From cd34d867b7ffcf0403bfbf1052864176d1408324 Mon Sep 17 00:00:00 2001 From: Guilhem Saurel <guilhem.saurel@laas.fr> Date: Sun, 11 Oct 2020 14:36:51 +0200 Subject: [PATCH] manylinux: example-robot-data --- docker/manylinux2014/all.sh | 32 ++++---- .../config/example-robot-data/setup.py | 23 ++++++ .../config/example-robot-data/test.py | 12 +++ docker/manylinux2014/config/targets | 1 + docker/manylinux2014/scripts/build_wheels.sh | 75 +++++++++++-------- 5 files changed, 96 insertions(+), 47 deletions(-) create mode 100644 docker/manylinux2014/config/example-robot-data/setup.py create mode 100644 docker/manylinux2014/config/example-robot-data/test.py diff --git a/docker/manylinux2014/all.sh b/docker/manylinux2014/all.sh index 8106072..b5ba165 100755 --- a/docker/manylinux2014/all.sh +++ b/docker/manylinux2014/all.sh @@ -1,38 +1,38 @@ #!/bin/bash -eux -pyver=${1:-all} +pyver="${1:-all}" +pwd="$(pwd -P)" docker build -t manylinux . -docker run -v "$(pwd -P):/io" -v /local/users/ccache/:/root/.ccache --rm -t manylinux rm -rf /io/dist /io/wheelhouse +docker run -v "$pwd:/io" -v /local/users/ccache/:/root/.ccache --rm -t manylinux rm -rf /io/dist /io/wheelhouse -build() { - for target in eigenpy hpp-fcl pinocchio - do docker run -v "$(pwd -P):/io" -v /local/users/ccache/:/root/.ccache --rm -t manylinux /scripts/setup.sh $target $1 - done +build_wheel() { + while read target + do docker run -v "$pwd:/io" -v /local/users/ccache/:/root/.ccache --rm -t manylinux /scripts/setup.sh $target $1 + done < config/targets } -test() { - docker build -f test.Dockerfile --build-arg PYVER=$pyver -t manylinux-test:$pyver . - docker run --rm -t manylinux-test:$pyver +test_wheel() { + docker build -f test.Dockerfile --build-arg PYVER=$1 -t manylinux-test:$1 . + docker run --rm -t manylinux-test:$1 } if [ "$pyver" = all ] then - for pyver in 2.7 3.5 3.6 3.7 3.8 3.9 - do build $pyver & + for pv in 2.7 3.5 3.6 3.7 3.8 3.9 + do build_wheel $pv done - wait else - build $pyver + build_wheel $pyver fi clear if [ "$pyver" = all ] then - for pyver in 2.7 3.5 3.6 3.7 3.8 3.9 - do test $pyver + for pv in 2.7 3.5 3.6 3.7 3.8 3.9 + do test_wheel $pv done else - test $pyver + test_wheel $pyver fi diff --git a/docker/manylinux2014/config/example-robot-data/setup.py b/docker/manylinux2014/config/example-robot-data/setup.py new file mode 100644 index 0000000..0d342ad --- /dev/null +++ b/docker/manylinux2014/config/example-robot-data/setup.py @@ -0,0 +1,23 @@ +from skbuild import setup + +try: + with open("README.md", "r") as f: + long_description = f.read() +except FileNotFoundError: + long_description = "" + +setup( + name="example-robot-data", + version="VERSION", + description="Set of robot URDFs for benchmarking and developed examples.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/gepetto/example-robot-data", + install_requires=['pinocchio'], + cmake_minimum_required_version='3.1', + classifiers=[ + "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", + "License :: OSI Approved :: BSD License", "Operating System :: POSIX :: Linux" + ], + python_requires='>=2.7', +) diff --git a/docker/manylinux2014/config/example-robot-data/test.py b/docker/manylinux2014/config/example-robot-data/test.py new file mode 100644 index 0000000..325870f --- /dev/null +++ b/docker/manylinux2014/config/example-robot-data/test.py @@ -0,0 +1,12 @@ +import unittest + +import example_robot_data + + +class TestPinocchio(unittest.TestCase): + def test_trivial(self): + self.assertEqual(example_robot_data.load('talos').model.nq, 39) + + +if __name__ == '__main__': + unittest.main() diff --git a/docker/manylinux2014/config/targets b/docker/manylinux2014/config/targets index ed5626a..659eb56 100644 --- a/docker/manylinux2014/config/targets +++ b/docker/manylinux2014/config/targets @@ -1,3 +1,4 @@ eigenpy hpp-fcl pinocchio +example-robot-data diff --git a/docker/manylinux2014/scripts/build_wheels.sh b/docker/manylinux2014/scripts/build_wheels.sh index d4df0b7..04ef01b 100755 --- a/docker/manylinux2014/scripts/build_wheels.sh +++ b/docker/manylinux2014/scripts/build_wheels.sh @@ -4,17 +4,17 @@ TARGET=${1:-eigenpy} PYVER=${2:-3.9} export CMAKE_PREFIX_PATH="~/.local:/opt/openrobots" +PACKAGE="${TARGET//-/_}" VERSION="$(grep version setup.py | head -1 | cut -d'"' -f2)" TARVER="$TARGET-$VERSION" -WHEEL_DIR="${TARGET/-/_}-$VERSION" -PACKAGE="$TARGET" -[ "$TARGET" = "hpp-fcl" ] && PACKAGE=hppfcl +WHEEL_DIR="$PACKAGE-$VERSION" PYBIN="$(find /opt/python -name "cp${PYVER/.}*")/bin" INSTALLED_PREFIX="$PWD/_skbuild/linux-x86_64-$PYVER/cmake-install" SITE_PACKAGES="lib/python$PYVER/site-packages" USER_PACKAGES="$HOME/.local/$SITE_PACKAGES" WHEEL_NAME="$WHEEL_DIR-cp${PYVER/.}-" -LIB_DIR="${TARGET/-/_}.libs" +LIB_DIR="$PACKAGE.libs" +[ "$PACKAGE" = "hpp_fcl" ] && PACKAGE=hppfcl PACKAGE_DIR="$WHEEL_DIR/$WHEEL_DIR.data/data/$SITE_PACKAGES/$PACKAGE" # Install dependencies @@ -25,55 +25,68 @@ PACKAGE_DIR="$WHEEL_DIR/$WHEEL_DIR.data/data/$SITE_PACKAGES/$PACKAGE" "$PYBIN/python" setup.py bdist_wheel -j"$(nproc)" -DBUILD_TESTING=OFF -DINSTALL_DOCUMENTATION=OFF \ -DCMAKE_INSTALL_LIBDIR=lib -DPYTHON_STANDARD_LAYOUT=ON -DENFORCE_MINIMAL_CXX_STANDARD=ON -# Bundle external shared libraries into the wheels -OTHER_LIB_DIRS=$(find "$USER_PACKAGES" -name '*.libs' | tr '\n' ':') +BINARY_EXTENSIONS="$(find "$INSTALLED_PREFIX/$SITE_PACKAGES/$PACKAGE" -name '*.so*')" || true +if [ -n "$BINARY_EXTENSIONS" ] +then + # Bundle external shared libraries into the wheels + OTHER_LIB_DIRS=$(find "$USER_PACKAGES" -name '*.libs' | tr '\n' ':') -# don't bundle ones already in another wheel -# that's against pypa recomendations, but should work -WHITELISTED="$(/scripts/patch_whitelist.py "$USER_PACKAGES")" + # don't bundle ones already in another wheel + # that's against pypa recomendations, but should work + WHITELISTED="$(/scripts/patch_whitelist.py "$USER_PACKAGES")" -# Repair it -LD_LIBRARY_PATH="$INSTALLED_PREFIX/lib:$OTHER_LIB_DIRS/opt/openrobots/lib:$LD_LIBRARY_PATH" \ - auditwheel repair "dist/$WHEEL_NAME"* --plat "$PLAT" -w /io/wheelhouse/ + # Repair it + LD_LIBRARY_PATH="$INSTALLED_PREFIX/lib:$OTHER_LIB_DIRS/opt/openrobots/lib:$LD_LIBRARY_PATH" \ + auditwheel repair "dist/$WHEEL_NAME"* --plat "$PLAT" -w /io/wheelhouse/ +else + mv dist/"$WHEEL_NAME"*-linux*.whl /io/wheelhouse +fi # Clean build rm -rf _skbuild dist # Extract it -wheel unpack /io/wheelhouse/"$WHEEL_NAME"*-manylinux2014*.whl +wheel unpack /io/wheelhouse/"$WHEEL_NAME"*linux*.whl -# set the RPATH right for the installed wheel -# ref https://github.com/pypa/auditwheel/issues/257 -patchelf --set-rpath "\$ORIGIN/../$LIB_DIR$WHITELISTED" "$(find "$PACKAGE_DIR" -name '*.so')" +if [ -n "$BINARY_EXTENSIONS" ] +then + # set the RPATH right for the installed wheel + # ref https://github.com/pypa/auditwheel/issues/257 + patchelf --set-rpath "\$ORIGIN/../$LIB_DIR$WHITELISTED" "$(find "$PACKAGE_DIR" -name '*.so')" -# Remove moved libs -rm -f "$WHEEL_DIR/$WHEEL_DIR.data/data/lib/"lib*so* + # Remove moved libs + rm -f "$WHEEL_DIR/$WHEEL_DIR.data/data/lib/"lib*so* -# Set the rpath of the other shared libraries in LIB_DIR -for lib in "$WHEEL_DIR/${LIB_DIR}"/* -do patchelf --set-rpath '$ORIGIN' "$lib" -done + # Set the rpath of the other shared libraries in LIB_DIR + for lib in "$WHEEL_DIR/${LIB_DIR}"/* + do patchelf --set-rpath '$ORIGIN' "$lib" + done + + # fix .cmake files: remove _skbuild references & update lib name with their hash from auditwheel + MAIN_LIB=$(find "$WHEEL_DIR/$LIB_DIR" -name "lib$TARGET-*.so*") # eigenpy-2.5.0/eigenpy.libs/libeigenpy-0c5e8890.so + MAIN_LIB_NAME=$(basename "$MAIN_LIB") # libeigenpy-0c5e8890.so + MAIN_LIB_NAME_NO_HASH=$(echo "$MAIN_LIB_NAME" | sed 's/-[[:xdigit:]]\{8\}//') # libeigenpy.so -# fix .cmake files: remove _skbuild references & update lib name with their hash from auditwheel -MAIN_LIB=$(find "$WHEEL_DIR/$LIB_DIR" -name "lib$TARGET-*.so") # eigenpy-2.5.0/eigenpy.libs/libeigenpy-0c5e8890.so -MAIN_LIB_NAME=$(basename "$MAIN_LIB") # libeigenpy-0c5e8890.so -MAIN_LIB_NAME_NO_HASH=$(echo "$MAIN_LIB_NAME" | sed 's/-[[:xdigit:]]\{8\}//') # libeigenpy.so + for file in $WHEEL_DIR/$WHEEL_DIR.data/data/lib/cmake/$TARGET/*.cmake + do + sed -i "s:$MAIN_LIB_NAME_NO_HASH:$MAIN_LIB_NAME:" "$file" || true + done +fi for file in $WHEEL_DIR/$WHEEL_DIR.data/data/lib/cmake/$TARGET/*.cmake do - sed -i "s:/src/_skbuild/linux-x86_64-.*/cmake-install/lib/\(.*\)\.so:\${_IMPORT_PREFIX}/$SITE_PACKAGES/$LIB_DIR/\1.so:" "$file" || true - sed -i "s:$MAIN_LIB_NAME_NO_HASH:$MAIN_LIB_NAME:" "$file" || true + sed -i "s:/src/_skbuild/linux-x86_64-.*/cmake-install/lib/\([^/]*\)\.so:\${_IMPORT_PREFIX}/$SITE_PACKAGES/$LIB_DIR/\1.so:" "$file" || true sed -i 's:/src/_skbuild/linux-x86_64-.*/cmake-install/include:${_IMPORT_PREFIX}/include:' "$file" || true sed -i 's:/src/_skbuild/linux-x86_64-.*/cmake-install/lib:${_IMPORT_PREFIX}/lib:' "$file" || true done # Repack and clean wheel pack "$WHEEL_DIR" -d /io/wheelhouse -rm -rf "${WHEEL_DIR:?}"/ "$TARGET".egg-info/ dist/ +rm -rf "${WHEEL_DIR:?}"/ "$TARGET".egg-info # Install packages and test "$PYBIN/pip" install --user "$TARGET" --no-index --find-links=/io/wheelhouse/ -(cd "$HOME"; "$PYBIN/python" "/io/config/$TARGET/test.py") || touch "/io/wheelhouse/$PYVER-$TARGET" +(cd "$HOME"; "$PYBIN/python" "/io/config/$TARGET/test.py") mkdir -p /io/dist -mv /io/wheelhouse/"$WHEEL_NAME"*-manylinux2014*.whl /io/dist +mv /io/wheelhouse/"$WHEEL_NAME"*linux*.whl /io/dist -- GitLab