manylinux: // on pyver

docker/manylinux2014/all.sh

@@ -19,7 +19,7 @@ test() {
 if [ "$pyver" = all ]
 then
     for pyver in 2.7 3.5 3.6 3.7 3.8 3.9
-    do build $pyver
+    do build $pyver &
     done
     wait
 else

docker/manylinux2014/scripts/build_wheels.sh

@@ -3,52 +3,50 @@
 TARGET=${1:-eigenpy}
 PYVER=${2:-3.9}
 
-export LD_LIBRARY_PATH=/opt/openrobots/lib:$LD_LIBRARY_PATH
-
 VERSION="$(grep version "/io/config/$TARGET/setup.py" | head -1 | cut -d'"' -f2)"
 TARVER="$TARGET-$VERSION"
+WHEEL_DIR="${TARGET/-/_}-$VERSION"
 PACKAGE="$TARGET"
 [ "$TARGET" = "hpp-fcl" ] && PACKAGE=hppfcl
 PYBIN="$(find /opt/python -name "cp${PYVER/.}*")/bin"
+export CMAKE_PREFIX_PATH="~/.local:/opt/openrobots"
+INSTALLED_PREFIX="$PWD/_skbuild/linux-x86_64-$PYVER/cmake-install"
+SITE_PACKAGES="lib/python$PYVER/site-packages"
+USER_PACKAGES="$HOME/.local/$SITE_PACKAGES"
+WHEEL_NAME="$WHEEL_DIR-cp${PYVER/.}-"
+LIB_DIR="${TARGET/-/_}.libs"
+PACKAGE_DIR="$WHEEL_DIR/$WHEEL_DIR.data/data/$SITE_PACKAGES/$PACKAGE"
 
 # Install dependencies
 "$PYBIN/pip" install --user --find-links=/io/dist/ \
     $(grep install_requires "/io/config/$TARGET/setup.py" | sed "s/.*\['//;s/'\].*//;s/,//")
 
-export CMAKE_PREFIX_PATH="~/.local:/opt/openrobots"
-
 # Build wheels
 "$PYBIN/python" setup.py bdist_wheel -j"$(nproc)" -DBUILD_TESTING=OFF -DINSTALL_DOCUMENTATION=OFF \
     -DCMAKE_INSTALL_LIBDIR=lib -DPYTHON_STANDARD_LAYOUT=ON -DENFORCE_MINIMAL_CXX_STANDARD=ON
 
 # Bundle external shared libraries into the wheels
-INSTALLED_PREFIX="$PWD/_skbuild/linux-x86_64-$PYVER/cmake-install"
-SITE_PACKAGES="lib/python$PYVER/site-packages"
-USER_PACKAGES="$HOME/.local/$SITE_PACKAGES"
 OTHER_LIB_DIRS=$(find "$USER_PACKAGES" -name '*.libs' | tr '\n' ':')
-WHEEL_NAME=$(ls -Art dist/ | tail -n 1)
 
 # don't bundle ones already in another wheel
 # that's against pypa recomendations, but should work
 WHITELISTED="$(/scripts/patch_whitelist.py "$USER_PACKAGES")"
 
 # Repair it
-LD_LIBRARY_PATH="$INSTALLED_PREFIX/lib:$OTHER_LIB_DIRS$LD_LIBRARY_PATH" \
-    auditwheel repair "dist/$WHEEL_NAME" --plat "$PLAT" -w /io/wheelhouse/
+LD_LIBRARY_PATH="$INSTALLED_PREFIX/lib:$OTHER_LIB_DIRS/opt/openrobots/lib:$LD_LIBRARY_PATH" \
+    auditwheel repair "dist/$WHEEL_NAME"* --plat "$PLAT" -w /io/wheelhouse/
+
+# Clean build
 rm -rf _skbuild dist
 
 # Extract it
-WHEEL_DIR="${TARGET/-/_}-$VERSION"
-WHEEL="$(find /io/wheelhouse/ -name "$WHEEL_DIR-cp${PYVER/.}*-manylinux2014*.whl")"
-wheel unpack "$WHEEL"
-PACKAGE_DIR="$WHEEL_DIR/$WHEEL_DIR.data/data/$SITE_PACKAGES/$PACKAGE"
+wheel unpack /io/wheelhouse/"$WHEEL_NAME"*-manylinux2014*.whl
 
 # set the RPATH right for the installed wheel
 # ref https://github.com/pypa/auditwheel/issues/257
-LIB_DIR="${TARGET/-/_}.libs"
 patchelf --set-rpath "\$ORIGIN/../$LIB_DIR$WHITELISTED" "$(find "$PACKAGE_DIR" -name '*.so')"
 
-# Remove duplicated libs
+# Remove moved libs
 rm -f "$WHEEL_DIR/$WHEEL_DIR.data/data/lib/"lib*so*
 
 # Set the rpath of the other shared libraries in LIB_DIR
@@ -56,7 +54,7 @@ for lib in "$WHEEL_DIR/${LIB_DIR}"/*
 do patchelf --set-rpath '$ORIGIN' "$lib"
 done
 
-# .cmake files should not have references to _skbuild, this is only a temporary fix to be able to build hpp-fcl
+# .cmake files should not have references to _skbuild
 MAIN_LIB=$(find "$WHEEL_DIR/$LIB_DIR" -name "lib$TARGET-*.so") # eigenpy-2.5.0/eigenpy.libs/libeigenpy-0c5e8890.so
 MAIN_LIB_NAME=$(basename "$MAIN_LIB") # libeigenpy-0c5e8890.so
 MAIN_LIB_NAME_NO_HASH=$(echo "$MAIN_LIB_NAME" | sed 's/-[[:xdigit:]]\{8\}//') # libeigenpy.so
@@ -71,13 +69,11 @@ done
 
 # Repack and clean
 wheel pack "$WHEEL_DIR" -d /io/wheelhouse
-rm -rf "${WHEEL_DIR:?}"/
+rm -rf "${WHEEL_DIR:?}"/ "$TARGET".egg-info/ dist/
 
 # Install packages and test
 "$PYBIN/pip" install --user "$TARGET" --no-index --find-links=/io/wheelhouse/
 (cd "$HOME"; "$PYBIN/python" "/io/config/$TARGET/test.py") || touch "/io/wheelhouse/$PYVER-$TARGET"
 
-rm -rf "$TARGET".egg-info/ dist/
-
 mkdir -p /io/dist
-mv /io/wheelhouse/* /io/dist/
+mv /io/wheelhouse/"$WHEEL_NAME"*-manylinux2014*.whl /io/dist

docker/manylinux2014/scripts/patch_whitelist.py

 #!/opt/_internal/tools/bin/python
 """
-Patch auditwheel whitelist.
+Patch auditwheel whitelist: allow reuse of shared objects from our other wheels, 
+and generate a string for a RPATH update.
+
 Folks at pypa won't allow that in their codebase, as it's ugly, but they don't forbid us to do so:
 https://github.com/pypa/auditwheel/issues/76
 """
@@ -9,19 +11,21 @@ import sys
 import json
 from pathlib import Path
 
+import auditwheel
+
 USER_PACKAGES = Path(sys.argv[-1])
-POLICY = Path('/opt/_internal/tools/lib/python3.7/site-packages/auditwheel/policy/policy.json')
+POLICY = Path(auditwheel.__file__).parent / 'policy/policy.json'
 
 with POLICY.open() as f:
-    POLICIES = json.load(f)
+    policies = json.load(f)
 
 deps = []
 for dep in USER_PACKAGES.glob('*.libs'):
     deps.append(dep.name)
     for lib in dep.glob('*.so*'):
-        POLICIES[-1]["lib_whitelist"].append(lib.name)
+        policies[-1]["lib_whitelist"].append(lib.name)
 
 with POLICY.open('w') as f:
-    json.dump(POLICIES, f)
+    json.dump(policies, f)
 
 print(''.join(f':$ORIGIN/../{dep}' for dep in deps))