manylinux: patch auditwheel whitelist

docker/manylinux2014/all.sh

 #!/bin/bash -eux
 
+pyver=${1:-all}
+
 docker build -t manylinux .
 docker run -v "$(pwd -P):/io" -v /local/users/ccache/:/root/.ccache --rm -t manylinux rm -rf /io/dist /io/wheelhouse
-docker run -v "$(pwd -P):/io" -v /local/users/ccache/:/root/.ccache --rm -t manylinux /scripts/setup.sh eigenpy
-docker run -v "$(pwd -P):/io" -v /local/users/ccache/:/root/.ccache --rm -t manylinux /scripts/setup.sh hpp-fcl
-docker run -v "$(pwd -P):/io" -v /local/users/ccache/:/root/.ccache --rm -t manylinux /scripts/setup.sh pinocchio
+
+build() {
+    for target in eigenpy hpp-fcl pinocchio
+    do docker run -v "$(pwd -P):/io" -v /local/users/ccache/:/root/.ccache --rm -t manylinux /scripts/setup.sh $target $1
+    done
+}
+
+test() {
+    docker build -f test.Dockerfile --build-arg PYVER=$pyver -t manylinux-test:$pyver .
+    docker run --rm -t manylinux-test:$pyver
+}
+
+if [ "$pyver" = all ]
+then
+    for pyver in 2.7 3.5 3.6 3.7 3.8 3.9
+    do build $pyver
+    done
+    wait
+else
+    build $pyver
+fi
+
 clear
-for PYVER in 2.7 3.5 3.6 3.7 3.8 3.9
-do
-    docker build -f test.Dockerfile --build-arg PYVER=$PYVER -t manylinux-test:$PYVER .
-    docker run --rm -t manylinux-test:$PYVER
-done
+
+if [ "$pyver" = all ]
+then
+    for pyver in 2.7 3.5 3.6 3.7 3.8 3.9
+    do test $pyver
+    done
+else
+    test $pyver
+fi

docker/manylinux2014/build/setup_python.sh

@@ -13,12 +13,9 @@ PY27_BIN=/opt/python/cp27-cp27mu/bin
 mkdir -p $PY27_BIN
 ln -s /usr/bin/python $PY27_BIN/
 ln -s /usr/bin/pip $PY27_BIN/
-touch $PY27_BIN/python2.7
-pip2 install --upgrade pip
-pip2 install scikit-build --user
 
 # Upgrade pip and install scikit-build
 for PYBIN in /opt/python/*/bin; do
     "$PYBIN"/pip install --upgrade pip
-    "$PYBIN"/pip install scikit-build numpy
+    "$PYBIN"/pip install --user scikit-build numpy
 done

docker/manylinux2014/config/hpp-fcl/pre_build.sh

-#!/bin/bash -eux
-
-for PYBIN in /opt/python/*/bin; do
-  "$PYBIN/pip" install --find-links=/io/dist/ eigenpy
-done

docker/manylinux2014/config/pinocchio/pre_build.sh

-#!/bin/bash -eux
-
-for PYBIN in /opt/python/*/bin; do
-  "$PYBIN/pip" install --find-links=/io/dist/ eigenpy hpp-fcl
-done

docker/manylinux2014/scripts/build_wheels.sh

 #!/bin/bash -eux
 
+TARGET=${1:-eigenpy}
+PYVER=${2:-3.9}
+
 export LD_LIBRARY_PATH=/opt/openrobots/lib:$LD_LIBRARY_PATH
 
-TARGET=${1:-eigenpy}
 VERSION="$(grep version "/io/config/$TARGET/setup.py" | head -1 | cut -d'"' -f2)"
 TARVER="$TARGET-$VERSION"
 PACKAGE="$TARGET"
 [ "$TARGET" = "hpp-fcl" ] && PACKAGE=hppfcl
+PYBIN="$(find /opt/python -name "cp${PYVER/.}*")/bin"
+
+# Install dependencies
+"$PYBIN/pip" install --user --find-links=/io/dist/ \
+    $(grep install_requires "/io/config/$TARGET/setup.py" | sed "s/.*\['//;s/'\].*//;s/,//")
 
-for PYBIN in /opt/python/*/bin; do
-  PYVERSION=$(find "$PYBIN" -type f -name 'python*.*' | head -1 | grep -Eo "[0-9]\.[0-9]")
-  export CMAKE_PREFIX_PATH="$(find / -path "*$PYVERSION*" -type d -name cmake | tr '\n' ':')/opt/openrobots"
+export CMAKE_PREFIX_PATH="~/.local:/opt/openrobots"
 
-  # Build wheels
-  "$PYBIN/python" setup.py bdist_wheel -j"$(nproc)" -DBUILD_TESTING=OFF -DINSTALL_DOCUMENTATION=OFF \
-      -DCMAKE_INSTALL_LIBDIR=lib -DPYTHON_STANDARD_LAYOUT=ON -DENFORCE_MINIMAL_CXX_STANDARD=ON
+# Build wheels
+"$PYBIN/python" setup.py bdist_wheel -j"$(nproc)" -DBUILD_TESTING=OFF -DINSTALL_DOCUMENTATION=OFF \
+    -DCMAKE_INSTALL_LIBDIR=lib -DPYTHON_STANDARD_LAYOUT=ON -DENFORCE_MINIMAL_CXX_STANDARD=ON
 
-  # Bundle external shared libraries into the wheels
-  INSTALLED_PREFIX="$PWD/_skbuild/linux-x86_64-$PYVERSION/cmake-install"
-  OTHER_LIB_DIRS=$(find / -path "*$PYVERSION*.libs/*" -name '*.so' -0 | xargs dirname | tr '\n' ':')
-  WHEEL_NAME=$(ls -Art dist/ | tail -n 1)
+# Bundle external shared libraries into the wheels
+INSTALLED_PREFIX="$PWD/_skbuild/linux-x86_64-$PYVER/cmake-install"
+SITE_PACKAGES="lib/python$PYVER/site-packages"
+USER_PACKAGES="$HOME/.local/$SITE_PACKAGES"
+OTHER_LIB_DIRS=$(find "$USER_PACKAGES" -name '*.libs' | tr '\n' ':')
+WHEEL_NAME=$(ls -Art dist/ | tail -n 1)
 
-  # Repair it
-  LD_LIBRARY_PATH="$INSTALLED_PREFIX/lib:$OTHER_LIB_DIRS$LD_LIBRARY_PATH" \
+# don't bundle ones already in another wheel
+# that's against pypa recomendations, but should work
+WHITELISTED="$(/scripts/patch_whitelist.py "$USER_PACKAGES")"
+
+# Repair it
+LD_LIBRARY_PATH="$INSTALLED_PREFIX/lib:$OTHER_LIB_DIRS$LD_LIBRARY_PATH" \
     auditwheel repair "dist/$WHEEL_NAME" --plat "$PLAT" -w /io/wheelhouse/
-  rm -rf _skbuild dist
-
-  # Extract it
-  WHEEL_DIR="${TARGET/-/_}-$VERSION"
-  WHEEL="$(find /io/wheelhouse/ -name "$WHEEL_DIR-cp${PYVERSION/.}*-manylinux2014*.whl")"
-  wheel unpack "$WHEEL"
-  PACKAGE_DIR="$WHEEL_DIR/$WHEEL_DIR.data/data/lib/python$PYVERSION/site-packages/$PACKAGE"
-  MAIN_LIB="$(find "$PACKAGE_DIR" -name '*.so')"
-
-  # set the RPATH right for the installed wheel
-  LIB_DIR="${TARGET/-/_}.libs"
-  patchelf --set-rpath "\$ORIGIN/../${LIB_DIR}" "$MAIN_LIB"
-
-  # Set the rpath of the other shared libraries in LIB_DIR
-  for lib in "$WHEEL_DIR/${LIB_DIR}"/*
-  do patchelf --set-rpath '$ORIGIN' "$lib"
-  done
-
-  # Remove libs in double
-  rm -f "$WHEEL_DIR/$WHEEL_DIR.data/data/lib/"lib*so*
-
-  MAIN_LIB=$(find "$WHEEL_DIR/$LIB_DIR" -name "lib$PACKAGE-*.so") # eigenpy-2.5.0/eigenpy.libs/libeigenpy-0c5e8890.so
-  MAIN_LIB_NAME=$(basename "$MAIN_LIB_NAME") # libeigenpy-0c5e8890.so
-  MAIN_LIB_NAME_NO_HASH=$(echo "$MAIN_LIB_NAME" | sed 's/-[[:xdigit:]]\{8\}//') # libeigenpy.so
-
-  # .cmake files should not have references to _skbuild, this is only a temporary fix to be able to build hpp-fcl
-  for file in $WHEEL_DIR/$WHEEL_DIR.data/data/lib/cmake/$TARGET/*.cmake
-  do
-    sed -i "s:/src/_skbuild/linux-x86_64-.*/cmake-install/lib/\(.*\)\.so:\${_IMPORT_PREFIX}/${LIB_DIR}/\1.so:" "$file" || true
+rm -rf _skbuild dist
+
+# Extract it
+WHEEL_DIR="${TARGET/-/_}-$VERSION"
+WHEEL="$(find /io/wheelhouse/ -name "$WHEEL_DIR-cp${PYVER/.}*-manylinux2014*.whl")"
+wheel unpack "$WHEEL"
+PACKAGE_DIR="$WHEEL_DIR/$WHEEL_DIR.data/data/$SITE_PACKAGES/$PACKAGE"
+
+# set the RPATH right for the installed wheel
+# ref https://github.com/pypa/auditwheel/issues/257
+LIB_DIR="${TARGET/-/_}.libs"
+patchelf --set-rpath "\$ORIGIN/../$LIB_DIR$WHITELISTED" "$(find "$PACKAGE_DIR" -name '*.so')"
+
+# Remove duplicated libs
+rm -f "$WHEEL_DIR/$WHEEL_DIR.data/data/lib/"lib*so*
+
+# Set the rpath of the other shared libraries in LIB_DIR
+for lib in "$WHEEL_DIR/${LIB_DIR}"/*
+do patchelf --set-rpath '$ORIGIN' "$lib"
+done
+
+# .cmake files should not have references to _skbuild, this is only a temporary fix to be able to build hpp-fcl
+MAIN_LIB=$(find "$WHEEL_DIR/$LIB_DIR" -name "lib$TARGET-*.so") # eigenpy-2.5.0/eigenpy.libs/libeigenpy-0c5e8890.so
+MAIN_LIB_NAME=$(basename "$MAIN_LIB") # libeigenpy-0c5e8890.so
+MAIN_LIB_NAME_NO_HASH=$(echo "$MAIN_LIB_NAME" | sed 's/-[[:xdigit:]]\{8\}//') # libeigenpy.so
+
+for file in $WHEEL_DIR/$WHEEL_DIR.data/data/lib/cmake/$TARGET/*.cmake
+do
+    sed -i "s:/src/_skbuild/linux-x86_64-.*/cmake-install/lib/\(.*\)\.so:\${_IMPORT_PREFIX}/$SITE_PACKAGES/$LIB_DIR/\1.so:" "$file" || true
     sed -i "s:$MAIN_LIB_NAME_NO_HASH:$MAIN_LIB_NAME:" "$file" || true
     sed -i 's:/src/_skbuild/linux-x86_64-.*/cmake-install/include:${_IMPORT_PREFIX}/include:' "$file" || true
     sed -i 's:/src/_skbuild/linux-x86_64-.*/cmake-install/lib:${_IMPORT_PREFIX}/lib:' "$file" || true
-  done
+done
 
-  # Repack and clean
-  wheel pack "$WHEEL_DIR" -d /io/wheelhouse
-  rm -rf "${WHEEL_DIR:?}"/
+# Repack and clean
+wheel pack "$WHEEL_DIR" -d /io/wheelhouse
+rm -rf "${WHEEL_DIR:?}"/
 
-  # Install packages and test
-  "$PYBIN/pip" install "$TARGET" --no-index --find-links=/io/wheelhouse/
-  (cd "$HOME"; "$PYBIN/python" "/io/config/$TARGET/test.py") || touch "/io/wheelhouse/$PYVERSION-$TARGET"
-done
+# Install packages and test
+"$PYBIN/pip" install --user "$TARGET" --no-index --find-links=/io/wheelhouse/
+(cd "$HOME"; "$PYBIN/python" "/io/config/$TARGET/test.py") || touch "/io/wheelhouse/$PYVER-$TARGET"
 
 rm -rf "$TARGET".egg-info/ dist/
 

docker/manylinux2014/scripts/patch_whitelist.py

+#!/opt/_internal/tools/bin/python
+"""
+Patch auditwheel whitelist.
+Folks at pypa won't allow that in their codebase, as it's ugly, but they don't forbid us to do so:
+https://github.com/pypa/auditwheel/issues/76
+"""
+
+import sys
+import json
+from pathlib import Path
+
+USER_PACKAGES = Path(sys.argv[-1])
+POLICY = Path('/opt/_internal/tools/lib/python3.7/site-packages/auditwheel/policy/policy.json')
+
+with POLICY.open() as f:
+    POLICIES = json.load(f)
+
+deps = []
+for dep in USER_PACKAGES.glob('*.libs'):
+    deps.append(dep.name)
+    for lib in dep.glob('*.so*'):
+        POLICIES[-1]["lib_whitelist"].append(lib.name)
+
+with POLICY.open('w') as f:
+    json.dump(POLICIES, f)
+
+print(''.join(f':$ORIGIN/../{dep}' for dep in deps))

docker/manylinux2014/scripts/setup.sh

 #!/bin/bash -eux
 
 TARGET=${1:-eigenpy}
+PYVER=${2:-3.9}
+
 URL="$(grep url "/io/config/$TARGET/setup.py" | cut -d'"' -f2)"
 VERSION="$(grep version "/io/config/$TARGET/setup.py" | head -1 | cut -d'"' -f2)"
 
@@ -17,8 +19,4 @@ cp /scripts/pyproject.toml .
 find . -name CMakeLists.txt | xargs sed -i 's/PYTHON_INCLUDE_DIRS/PYTHON_INCLUDE_DIR/'
 sed -i 's/REQUIRED COMPONENTS Interpreter Development/REQUIRED COMPONENTS Interpreter/' cmake/python.cmake
 
-[ -x "/io/config/$TARGET/pre_build.sh" ] && "/io/config/$TARGET/pre_build.sh"
-
-echo -e "\n=========================================================================================================\n"
-
-/scripts/build_wheels.sh "$TARGET"
+/scripts/build_wheels.sh "$TARGET" "$PYVER"

docker/manylinux2014/test.Dockerfile

 ARG PYVER=3.7
 FROM python:$PYVER
 
-RUN apt-get update -qqy && apt-get install -qqy libgl1 && rm -rf /var/lib/apt/lists/* && pip install --no-cache-dir numpy
+RUN apt-get update -qqy && apt-get install -qqy libgl1 && rm -rf /var/lib/apt/lists/* && pip install --user --no-cache-dir numpy
 
 ADD dist config test.sh /
 CMD /test.sh

docker/manylinux2014/test.sh

@@ -2,6 +2,6 @@
 
 while read tgt
 do  echo -e "\n================================ $tgt ===================================\n"
-    pip install --find-links=/ "$tgt"
+    pip install --user --find-links=/ "$tgt"
     python "/$tgt/test.py"
 done < targets