Skip to content

Commit 66d9543

Browse files
sbrunksbrunk
committed
Make PyTorch cross-compilation more consistent with numpy and scipy
1 parent 4ca07ee commit 66d9543

File tree

2 files changed

+57
-74
lines changed

2 files changed

+57
-74
lines changed

‎pytorch/cppbuild.sh

+42-30
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,8 @@ git submodule foreach --recursive 'git reset --hard'
5656
# https://github.com/pytorch/pytorch/pull/66219
5757
#patch -Np1 < ../../../pytorch.patch
5858

59+
CPYTHON_HOST_PATH="$INSTALL_PATH/../../../cpython/cppbuild/$PLATFORM/host/"
5960
CPYTHON_PATH="$INSTALL_PATH/../../../cpython/cppbuild/$PLATFORM/"
60-
# local cross-compilation requires x86_64 cpython
61-
if [ "$ARCH" = "x86_64" ] && [ "$PLATFORM" = "macosx-arm64" ]; then
62-
CPYTHON_PATH="$INSTALL_PATH/../../../cpython/cppbuild/macosx-x86_64/"
63-
fi
6461
OPENBLAS_PATH="$INSTALL_PATH/../../../openblas/cppbuild/$PLATFORM/"
6562
NUMPY_PATH="$INSTALL_PATH/../../../numpy/cppbuild/$PLATFORM/"
6663

@@ -69,7 +66,12 @@ if [[ -n "${BUILD_PATH:-}" ]]; then
6966
IFS="$BUILD_PATH_SEPARATOR"
7067
for P in $BUILD_PATH; do
7168
if [[ $(find "$P" -name Python.h) ]]; then
72-
CPYTHON_PATH="$P"
69+
if [[ "$(basename $P)" == "$PLATFORM_HOST" ]]; then
70+
CPYTHON_HOST_PATH="$P"
71+
fi
72+
if [[ "$(basename $P)" == "$PLATFORM" ]]; then
73+
CPYTHON_PATH="$P"
74+
fi
7375
elif [[ -f "$P/include/openblas_config.h" ]]; then
7476
OPENBLAS_PATH="$P"
7577
elif [[ -f "$P/python/numpy/core/include/numpy/numpyconfig.h" ]]; then
@@ -79,36 +81,46 @@ if [[ -n "${BUILD_PATH:-}" ]]; then
7981
IFS="$PREVIFS"
8082
fi
8183

84+
CPYTHON_HOST_PATH="${CPYTHON_HOST_PATH//\\//}"
8285
CPYTHON_PATH="${CPYTHON_PATH//\\//}"
8386
OPENBLAS_PATH="${OPENBLAS_PATH//\\//}"
8487
NUMPY_PATH="${NUMPY_PATH//\\//}"
8588

86-
if [[ -f "$CPYTHON_PATH/include/python3.12/Python.h" ]]; then
87-
# setup.py won't pick up the right libgfortran.so without this
88-
export LD_LIBRARY_PATH="$OPENBLAS_PATH/lib/:$CPYTHON_PATH/lib/:$NUMPY_PATH/lib/"
89-
export PYTHON_BIN_PATH="$CPYTHON_PATH/bin/python3.12"
90-
export PYTHON_INCLUDE_PATH="$CPYTHON_PATH/include/python3.12/"
91-
export PYTHON_LIB_PATH="$CPYTHON_PATH/lib/python3.12/"
92-
export PYTHON_INSTALL_PATH="$INSTALL_PATH/lib/python3.12/site-packages/"
93-
export SSL_CERT_FILE="$CPYTHON_PATH/lib/python3.12/site-packages/pip/_vendor/certifi/cacert.pem"
94-
chmod +x "$PYTHON_BIN_PATH"
95-
elif [[ -f "$CPYTHON_PATH/include/Python.h" ]]; then
96-
CPYTHON_PATH=$(cygpath $CPYTHON_PATH)
97-
OPENBLAS_PATH=$(cygpath $OPENBLAS_PATH)
98-
NUMPY_PATH=$(cygpath $NUMPY_PATH)
99-
export PATH="$OPENBLAS_PATH:$CPYTHON_PATH:$NUMPY_PATH:$PATH"
100-
export PYTHON_BIN_PATH="$CPYTHON_PATH/bin/python.exe"
101-
export PYTHON_INCLUDE_PATH="$CPYTHON_PATH/include/"
102-
export PYTHON_LIB_PATH="$CPYTHON_PATH/lib/"
103-
export PYTHON_INSTALL_PATH="$INSTALL_PATH/lib/site-packages/"
104-
export SSL_CERT_FILE="$CPYTHON_PATH/lib/pip/_vendor/certifi/cacert.pem"
105-
fi
106-
export PYTHONPATH="$PYTHON_INSTALL_PATH:$NUMPY_PATH/python/"
107-
mkdir -p "$PYTHON_INSTALL_PATH"
108-
109-
export CFLAGS="-I$CPYTHON_PATH/include/ -I$PYTHON_LIB_PATH/include/python/ -L$CPYTHON_PATH/lib/ -L$CPYTHON_PATH/libs/"
11089
export PYTHONNOUSERSITE=1
111-
$PYTHON_BIN_PATH -m pip install --target=$PYTHON_LIB_PATH setuptools==67.6.1 pyyaml==6.0.1 typing_extensions==4.8.0
90+
91+
TOOLS="setuptools==67.6.1 pyyaml==6.0.1 typing_extensions==4.8.0"
92+
93+
if [[ $PLATFORM == $PLATFORM_HOST ]]; then
94+
if [[ -f "$CPYTHON_PATH/include/python3.12/Python.h" ]]; then
95+
# setup.py won't pick up the right libgfortran.so without this
96+
export LD_LIBRARY_PATH="$OPENBLAS_PATH/lib/:$CPYTHON_PATH/lib/:$NUMPY_PATH/lib/"
97+
export PYTHON_BIN_PATH="$CPYTHON_PATH/bin/python3.12"
98+
export PYTHON_INCLUDE_PATH="$CPYTHON_PATH/include/python3.12/"
99+
export PYTHON_LIB_PATH="$CPYTHON_PATH/lib/python3.12/"
100+
export PYTHON_INSTALL_PATH="$INSTALL_PATH/lib/python3.12/site-packages/"
101+
export SSL_CERT_FILE="$CPYTHON_PATH/lib/python3.12/site-packages/pip/_vendor/certifi/cacert.pem"
102+
chmod +x "$PYTHON_BIN_PATH"
103+
elif [[ -f "$CPYTHON_PATH/include/Python.h" ]]; then
104+
CPYTHON_PATH=$(cygpath $CPYTHON_PATH)
105+
OPENBLAS_PATH=$(cygpath $OPENBLAS_PATH)
106+
NUMPY_PATH=$(cygpath $NUMPY_PATH)
107+
export PATH="$OPENBLAS_PATH:$CPYTHON_PATH:$NUMPY_PATH:$PATH"
108+
export PYTHON_BIN_PATH="$CPYTHON_PATH/bin/python.exe"
109+
export PYTHON_INCLUDE_PATH="$CPYTHON_PATH/include/"
110+
export PYTHON_LIB_PATH="$CPYTHON_PATH/lib/"
111+
export PYTHON_INSTALL_PATH="$INSTALL_PATH/lib/site-packages/"
112+
export SSL_CERT_FILE="$CPYTHON_PATH/lib/pip/_vendor/certifi/cacert.pem"
113+
fi
114+
export PYTHONPATH="$PYTHON_INSTALL_PATH:$NUMPY_PATH/python/"
115+
mkdir -p "$PYTHON_INSTALL_PATH"
116+
117+
export CFLAGS="-I$CPYTHON_PATH/include/ -I$PYTHON_LIB_PATH/include/python/ -L$CPYTHON_PATH/lib/ -L$CPYTHON_PATH/libs/"
118+
$PYTHON_BIN_PATH -m pip install --target=$PYTHON_LIB_PATH $TOOLS
119+
else # cross-compile
120+
export PYTHON_BIN_PATH="$CPYTHON_HOST_PATH/bin/python3.12"
121+
chmod +x $PYTHON_BIN_PATH
122+
$PYTHON_BIN_PATH -m pip install --target="$CPYTHON_HOST_PATH/lib/python3.12/" $TOOLS
123+
fi
112124

113125
case $PLATFORM in
114126
linux-x86)

‎pytorch/pom.xml

+15-44
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,8 @@
9191
<buildResource>/${javacpp.platform.library.path}/</buildResource>
9292
<buildResource>/org/bytedeco/openblas/${javacpp.platform}/</buildResource>
9393
<buildResource>/org/bytedeco/cpython/${javacpp.platform}/</buildResource>
94+
<!-- add host cpython for running setup.py when cross-compiling -->
95+
<buildResource>/org/bytedeco/cpython/${os.name}-${os.arch}/</buildResource>
9496
<buildResource>/org/bytedeco/mkldnn/${javacpp.platform}/</buildResource>
9597
<buildResource>/org/bytedeco/numpy/${javacpp.platform}/</buildResource>
9698
</buildResources>
@@ -138,7 +140,7 @@
138140
<!-- cross-compilation profile for macosx-arm64 builds on macosx-x86_64 CI machines -->
139141
<profiles>
140142
<profile>
141-
<id>macosx-arm64</id>
143+
<id>pytorch-cross-compile</id>
142144
<activation>
143145
<property>
144146
<name>javacpp.platform</name>
@@ -153,54 +155,23 @@
153155
<dependencies>
154156
<dependency>
155157
<groupId>org.bytedeco</groupId>
156-
<artifactId>openblas-platform</artifactId>
157-
<version>0.3.25-${project.parent.version}</version>
158+
<artifactId>cpython</artifactId>
159+
<version>3.12.1-${project.parent.version}</version>
160+
<classifier>${os.name}-${os.arch}</classifier>
158161
</dependency>
162+
<!-- we need to exclude target cpython until we have a macosx-arm64 build -->
159163
<dependency>
160164
<groupId>org.bytedeco</groupId>
161-
<artifactId>cpython</artifactId>
162-
<version>3.12.1-${project.parent.version}</version>
163-
<classifier>macosx-x86_64</classifier>
165+
<artifactId>numpy-platform</artifactId>
166+
<version>1.26.3-${project.parent.version}</version>
167+
<exclusions>
168+
<exclusion>
169+
<groupId>org.bytedeco</groupId>
170+
<artifactId>cpython</artifactId>
171+
</exclusion>
172+
</exclusions>
164173
</dependency>
165174
</dependencies>
166-
<configuration>
167-
<classPaths>
168-
<classPath>${basedir}/../openblas/target/classes/</classPath>
169-
<classPath>${project.build.outputDirectory}</classPath>
170-
</classPaths>
171-
<includePaths>
172-
<includePath>${basedir}/../openblas/src/main/resources/org/bytedeco/openblas/include/</includePath>
173-
<includePath>${basedir}/../openblas/target/classes/org/bytedeco/openblas/include/</includePath>
174-
<includePath>${basedir}/../openblas/cppbuild/${javacpp.platform}/include/</includePath>
175-
<includePath>${basedir}/cppbuild/${javacpp.platform}${javacpp.platform.extension}/include/torch/csrc/api/include/</includePath>
176-
<includePath>${basedir}/cppbuild/${javacpp.platform}${javacpp.platform.extension}/include/</includePath>
177-
<includePath>${basedir}/target/classes/org/bytedeco/${javacpp.packageName}/include/</includePath>
178-
</includePaths>
179-
<linkPaths>
180-
<linkPath>${basedir}/../openblas/cppbuild/${javacpp.platform}/lib/</linkPath>
181-
<linkPath>${basedir}/cppbuild/${javacpp.platform}${javacpp.platform.extension}/lib/</linkPath>
182-
</linkPaths>
183-
<preloadPaths>
184-
<preloadPath>${basedir}/../openblas/cppbuild/${javacpp.platform}/bin/</preloadPath>
185-
</preloadPaths>
186-
<buildResources>
187-
<buildResource>/${javacpp.platform.library.path}/</buildResource>
188-
<buildResource>/org/bytedeco/openblas/${javacpp.platform}/</buildResource>
189-
<!-- add host cpython for running setup.py when cross-compiling -->
190-
<buildResource>/org/bytedeco/cpython/macosx-x86_64/</buildResource>
191-
</buildResources>
192-
<includeResources>
193-
<includeResource>/${javacpp.platform.library.path}/include/</includeResource>
194-
<includeResource>/org/bytedeco/openblas/include/</includeResource>
195-
<includeResource>/org/bytedeco/openblas/${javacpp.platform}/include/</includeResource>
196-
</includeResources>
197-
<linkResources>
198-
<linkResource>/${javacpp.platform.library.path}/</linkResource>
199-
<linkResource>/${javacpp.platform.library.path}/lib/</linkResource>
200-
<linkResource>/org/bytedeco/openblas/${javacpp.platform}/</linkResource>
201-
<linkResource>/org/bytedeco/openblas/${javacpp.platform}/lib/</linkResource>
202-
</linkResources>
203-
</configuration>
204175
</plugin>
205176
</plugins>
206177
</build>

0 commit comments

Comments
 (0)