diff --git a/docker/manylinux2014/all.sh b/docker/manylinux2014/all.sh index 5423e707a268766a7223e0d5e56d35818d6e91e3..8106072d4ae611821e438c361a90509e8d1bbc3e 100755 --- a/docker/manylinux2014/all.sh +++ b/docker/manylinux2014/all.sh @@ -19,7 +19,7 @@ test() { if [ "$pyver" = all ] then for pyver in 2.7 3.5 3.6 3.7 3.8 3.9 - do build $pyver + do build $pyver & done wait else diff --git a/docker/manylinux2014/scripts/build_wheels.sh b/docker/manylinux2014/scripts/build_wheels.sh index f7df991d9b7f1922ffd03d2226fc6e3510779075..24a88711677c1368c84ebf9f9e152925868f7ddd 100755 --- a/docker/manylinux2014/scripts/build_wheels.sh +++ b/docker/manylinux2014/scripts/build_wheels.sh @@ -3,52 +3,50 @@ TARGET=${1:-eigenpy} PYVER=${2:-3.9} -export LD_LIBRARY_PATH=/opt/openrobots/lib:$LD_LIBRARY_PATH - VERSION="$(grep version "/io/config/$TARGET/setup.py" | head -1 | cut -d'"' -f2)" TARVER="$TARGET-$VERSION" +WHEEL_DIR="${TARGET/-/_}-$VERSION" PACKAGE="$TARGET" [ "$TARGET" = "hpp-fcl" ] && PACKAGE=hppfcl PYBIN="$(find /opt/python -name "cp${PYVER/.}*")/bin" +export CMAKE_PREFIX_PATH="~/.local:/opt/openrobots" +INSTALLED_PREFIX="$PWD/_skbuild/linux-x86_64-$PYVER/cmake-install" +SITE_PACKAGES="lib/python$PYVER/site-packages" +USER_PACKAGES="$HOME/.local/$SITE_PACKAGES" +WHEEL_NAME="$WHEEL_DIR-cp${PYVER/.}-" +LIB_DIR="${TARGET/-/_}.libs" +PACKAGE_DIR="$WHEEL_DIR/$WHEEL_DIR.data/data/$SITE_PACKAGES/$PACKAGE" # Install dependencies "$PYBIN/pip" install --user --find-links=/io/dist/ \ $(grep install_requires "/io/config/$TARGET/setup.py" | sed "s/.*\['//;s/'\].*//;s/,//") -export CMAKE_PREFIX_PATH="~/.local:/opt/openrobots" - # Build wheels "$PYBIN/python" setup.py bdist_wheel -j"$(nproc)" -DBUILD_TESTING=OFF -DINSTALL_DOCUMENTATION=OFF \ -DCMAKE_INSTALL_LIBDIR=lib -DPYTHON_STANDARD_LAYOUT=ON -DENFORCE_MINIMAL_CXX_STANDARD=ON # Bundle external shared libraries into the wheels -INSTALLED_PREFIX="$PWD/_skbuild/linux-x86_64-$PYVER/cmake-install" -SITE_PACKAGES="lib/python$PYVER/site-packages" -USER_PACKAGES="$HOME/.local/$SITE_PACKAGES" OTHER_LIB_DIRS=$(find "$USER_PACKAGES" -name '*.libs' | tr '\n' ':') -WHEEL_NAME=$(ls -Art dist/ | tail -n 1) # don't bundle ones already in another wheel # that's against pypa recomendations, but should work WHITELISTED="$(/scripts/patch_whitelist.py "$USER_PACKAGES")" # Repair it -LD_LIBRARY_PATH="$INSTALLED_PREFIX/lib:$OTHER_LIB_DIRS$LD_LIBRARY_PATH" \ - auditwheel repair "dist/$WHEEL_NAME" --plat "$PLAT" -w /io/wheelhouse/ +LD_LIBRARY_PATH="$INSTALLED_PREFIX/lib:$OTHER_LIB_DIRS/opt/openrobots/lib:$LD_LIBRARY_PATH" \ + auditwheel repair "dist/$WHEEL_NAME"* --plat "$PLAT" -w /io/wheelhouse/ + +# Clean build rm -rf _skbuild dist # Extract it -WHEEL_DIR="${TARGET/-/_}-$VERSION" -WHEEL="$(find /io/wheelhouse/ -name "$WHEEL_DIR-cp${PYVER/.}*-manylinux2014*.whl")" -wheel unpack "$WHEEL" -PACKAGE_DIR="$WHEEL_DIR/$WHEEL_DIR.data/data/$SITE_PACKAGES/$PACKAGE" +wheel unpack /io/wheelhouse/"$WHEEL_NAME"*-manylinux2014*.whl # set the RPATH right for the installed wheel # ref https://github.com/pypa/auditwheel/issues/257 -LIB_DIR="${TARGET/-/_}.libs" patchelf --set-rpath "\$ORIGIN/../$LIB_DIR$WHITELISTED" "$(find "$PACKAGE_DIR" -name '*.so')" -# Remove duplicated libs +# Remove moved libs rm -f "$WHEEL_DIR/$WHEEL_DIR.data/data/lib/"lib*so* # Set the rpath of the other shared libraries in LIB_DIR @@ -56,7 +54,7 @@ for lib in "$WHEEL_DIR/${LIB_DIR}"/* do patchelf --set-rpath '$ORIGIN' "$lib" done -# .cmake files should not have references to _skbuild, this is only a temporary fix to be able to build hpp-fcl +# .cmake files should not have references to _skbuild MAIN_LIB=$(find "$WHEEL_DIR/$LIB_DIR" -name "lib$TARGET-*.so") # eigenpy-2.5.0/eigenpy.libs/libeigenpy-0c5e8890.so MAIN_LIB_NAME=$(basename "$MAIN_LIB") # libeigenpy-0c5e8890.so MAIN_LIB_NAME_NO_HASH=$(echo "$MAIN_LIB_NAME" | sed 's/-[[:xdigit:]]\{8\}//') # libeigenpy.so @@ -71,13 +69,11 @@ done # Repack and clean wheel pack "$WHEEL_DIR" -d /io/wheelhouse -rm -rf "${WHEEL_DIR:?}"/ +rm -rf "${WHEEL_DIR:?}"/ "$TARGET".egg-info/ dist/ # Install packages and test "$PYBIN/pip" install --user "$TARGET" --no-index --find-links=/io/wheelhouse/ (cd "$HOME"; "$PYBIN/python" "/io/config/$TARGET/test.py") || touch "/io/wheelhouse/$PYVER-$TARGET" -rm -rf "$TARGET".egg-info/ dist/ - mkdir -p /io/dist -mv /io/wheelhouse/* /io/dist/ +mv /io/wheelhouse/"$WHEEL_NAME"*-manylinux2014*.whl /io/dist diff --git a/docker/manylinux2014/scripts/patch_whitelist.py b/docker/manylinux2014/scripts/patch_whitelist.py index c1366361125385b156da517ed89cc34d4f938964..2ca35266edb5864cfe2ef00555899aa61c1e1062 100755 --- a/docker/manylinux2014/scripts/patch_whitelist.py +++ b/docker/manylinux2014/scripts/patch_whitelist.py @@ -1,6 +1,8 @@ #!/opt/_internal/tools/bin/python """ -Patch auditwheel whitelist. +Patch auditwheel whitelist: allow reuse of shared objects from our other wheels, +and generate a string for a RPATH update. + Folks at pypa won't allow that in their codebase, as it's ugly, but they don't forbid us to do so: https://github.com/pypa/auditwheel/issues/76 """ @@ -9,19 +11,21 @@ import sys import json from pathlib import Path +import auditwheel + USER_PACKAGES = Path(sys.argv[-1]) -POLICY = Path('/opt/_internal/tools/lib/python3.7/site-packages/auditwheel/policy/policy.json') +POLICY = Path(auditwheel.__file__).parent / 'policy/policy.json' with POLICY.open() as f: - POLICIES = json.load(f) + policies = json.load(f) deps = [] for dep in USER_PACKAGES.glob('*.libs'): deps.append(dep.name) for lib in dep.glob('*.so*'): - POLICIES[-1]["lib_whitelist"].append(lib.name) + policies[-1]["lib_whitelist"].append(lib.name) with POLICY.open('w') as f: - json.dump(POLICIES, f) + json.dump(policies, f) print(''.join(f':$ORIGIN/../{dep}' for dep in deps))