Commit f6017ec4 authored by Olivier Stasse's avatar Olivier Stasse
Browse files

Update doc + improve dockerfile generation.

parent 72eba0ca
......@@ -67,11 +67,8 @@ If you are not satisfied, you have to use the [web interface](http://axis-ptz1).
The second records the current view. It is meant to be run in `eur0c:/local/axis`
## `robotpkg-test3-wipuser.sh`
## `robotpkg_helpers`
See documentation in [robotpkg_helpers](robotpkg_helpers)
Creates a directory ${HOME}/devel-src/robotpkg-test3 in which robotpkg and ${USER} wip from gepgitlab
are cloned.
Then the script bootstrap robotpkg and add the file ./etc/addrobotpkg.conf to the default
${HOME}/devel-src/robotpkg-test3/install/etc/robotpkg.conf.
From this the script tries to compile talos-simulation and talos-dev.
PKG_FORMAT= deb
ACCEPTABLE_LICENSES+=openhrp-grx-license
ACCEPTABLE_LICENSES+=cnrs-hpp-closed-source
ACCEPTABLE_LICENSES+=gnu-gpl
ACCEPTABLE_LICENSES+=motion-analysis-license
PREFER.gnupg=system
PREFER.urdfdom=system
PREFER.urdfdom-headers=system
PREFER.ros-catkin = system
PREFER.ros-comm = system
PREFER.ros-genlisp = system
PREFER.ros-message-generation = system
PREFER.ros-std-msgs = system
PREFER.ros-rospack = system
PREFER.ros-message-runtime = system
PREFER.ros-roscpp-core = system
PREFER.ros-xacro = system
PREFER.ros-common-msgs = system
PREFER.ros-lint = system
PREFER.ros-com-msgs = system
PREFER.ros-com-msgs = system
PREFER.bullet = system
PREFER.ros-ros = system
PREFER.ros-cmake-modules = system
PREFER.ros-dynamic-reconfigure = system
PREFER.ros-realtime-tools = system
PREFER.ros-control-toolbox = system
PREFER.ros-bond-core = system
PREFER.ros-class-loader = system
PREFER.ros-pluginlib = system
PREFER.ros-rqt = system
PREFER.ros-humanoid-msgs = system
PREFER.ros-genmsg = system
PREFER.ros-actionlib = system
PREFER.ros-geometry = system
PREFER.collada-dom = system
PREFER.orocos-kdl = system
PREFER.ros-angles = system
PREFER.ros-console-bridge = system
PREFER.ros-eigen-stl-containers = system
PREFER.ros-random-numbers = system
PREFER.ros-resource-retriever = system
PREFER.ros-shape-tools = system
PREFER.ros-geometric-shapes = system
PREFER.ros-srdfdom = system
PREFER.ros-robot-model = system
PREFER.ros-orocos-kdl = system
PREFER.assimp=system
ACCEPTABLE_LICENSES+=pal-license
ROS_PACKAGE_PATH=${ROBOTPKG_ROOT}/install/share:$ROS_PACKAGE_PATH
# Robotpkg Helpers
This is a python package providing tools to handle robotpkg.
## RobotpkgPackage
This class is analyzing the directory of a package and extracts information
reading the files Makefile and depend.mk
## RobotpkgSrcIntrospection
This class is analyzing a clone of the robotpkg repository and is trying
to make simple inferences on the packages dependencies
## RobotpkgTests
This class is building instance of robotpkg to make deployment tests.
It is useful for instance to test if various releases of packages are coherent
together.
## Tests
Set of python scripts using the previous classes to perform:
- Deployment tests for a given set of packages and specific branches
- Deployment tests using a destfiles directory
- Generate a dockerfile based on parsing a Makefile using jrl-cmakemodules
......@@ -15,7 +15,7 @@ def valid_name(name):
class RobotpkgGenerateDockerFile:
def __init__(self,CMakeLists_txt_path,ROBOTPKG_ROOT_SRC, dst_dir="/tmp/robotpkg_docker"):
def __init__(self,CMakeLists_txt_path,ROBOTPKG_ROOT_SRC, dst_dir="/tmp/robotpkg_docker",debug=0):
""" Creates a docker file from a CMakeLists.txt file
Arguments:
......@@ -23,6 +23,7 @@ class RobotpkgGenerateDockerFile:
ROBOTPKG_ROOT_SRC: The path to robotpkg sources.
dst_dir: The path where the files should be generated
"""
self.debug=debug
self.CMakeLists_txt_path = CMakeLists_txt_path
self.dst_dir = dst_dir
self.check_dst_dir()
......@@ -35,6 +36,7 @@ class RobotpkgGenerateDockerFile:
self.generate_docker_file()
def init_colors(self):
""" Initialize colors for beautification
......@@ -58,6 +60,13 @@ class RobotpkgGenerateDockerFile:
list_py = glob.glob(src_dir+ "/*py")
for apy_file in list_py:
copy(apy_file,self.dst_dir+'/robotpkg_helpers')
def jrl_cmake_module_analysis(self,cmake_content):
# Search for ADD_[REQUIRED|OPTIONAL]_DEPENDENCY and put it in the list of files
for project_name in re.findall(r'SET\s*\(\s*PROJECT\_NAME\s*([^\s>="\')]+)\s*\)',
cmake_content, re.I):
self.project_name=project_name
print("Project name:" +self.project_name)
def cmake(self):
# Read CMake file
......@@ -66,17 +75,30 @@ class RobotpkgGenerateDockerFile:
self.dict_os_pkgs=[]
with open(filename,"r") as f:
content = f.read()
self.jrl_cmake_module_analysis(content)
# Search for ADD_[REQUIRED|OPTIONAL]_DEPENDENCY and put it in the list of files
for dependency,version in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^\s>="\')]+)\s*[>=]*\s*([\d|.]*)["\']?\)', content, re.I):
self.dict_robot_pkgs[dependency]=version
print("CMakeLists.txt needs the following packages:")
print(self.dict_robot_pkgs)
for dependency,version in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^\s>="\')]+)\s*[>=]*\s*([\d|.]*)["\']?\)', content, re.I):
self.dict_robot_pkgs[dependency]=version
if self.debug>1:
print("CMakeLists.txt needs the following packages:")
print(self.dict_robot_pkgs)
f.close()
def filter_robotpkg_packages(self):
""" This methods remove from the dictionnary dict_robot_pkgs the packages not handled by robotpkg.
The removed packages are put inside the dict_os_pkgs dictionary.
"""
# Need to use the keys() method to iterate
# as we are removing from the dictionnary using key.
print("Filter packages")
if self.debug>1:
print("Filter packages")
self.tree_of_includes_os = set()
new_dict_robot_pkgs = dict(self.dict_robot_pkgs)
for pkg_name in self.dict_robot_pkgs.keys():
r = self.robotpkg_src_intro.is_pkg_present(pkg_name)
......@@ -84,21 +106,33 @@ class RobotpkgGenerateDockerFile:
del new_dict_robot_pkgs[pkg_name]
self.dict_os_pkgs.append(pkg_name)
else:
print("\n**********************\nFound pkg " + pkg_name + " in robotpkg")
self.robotpkg_src_intro.package_dict[pkg_name].read_package_info()
if len(self.robotpkg_src_intro.package_dict[pkg_name].master_repository)==0 :
print("The package does not have a master repository")
if not len(self.robotpkg_src_intro.package_dict[pkg_name].version)==0 :
print("The package can only provide version :" + self.robotpkg_src_intro.package_dict[pkg_name].version[0])
if self.debug>3:
print("\n**********************\nFound pkg " + pkg_name + " in robotpkg")
found_pkg = self.robotpkg_src_intro.package_dict[pkg_name]
found_pkg.read_package_info()
self.robotpkg_src_intro.build_tree_of_dependencies(found_pkg)
self.tree_of_includes_os = self.tree_of_includes_os.union(found_pkg.tree_of_includes_os)
self.dict_robot_pkgs = new_dict_robot_pkgs
def display_packages(self):
""" Display the packages detected by analyzing only the repository
"""
for pkg_name,pkg_version in self.dict_robot_pkgs.items():
print("Package name:" + pkg_name + " " + pkg_version)
for pkg_name in self.tree_of_includes_os:
print("OS package name:" + pkg_name)
def display_tree_of_dependencies(self):
""" Display all the packages known in robotpkg needed to compile the analyzed repository
"""
print("Display tree of dependencies:")
for an_include_mk in self.tree_of_includes_mk:
print(an_include_mk)
def generate_test_rc(self):
""" This method generates a python file to compile the packages using robotpkg inside the dockerfile.
"""
# Generate python file
filename = self.dst_dir + "/robotpkg_for_docker_file.py"
with open(filename,"w") as f:
......@@ -106,7 +140,7 @@ class RobotpkgGenerateDockerFile:
f.write("from robotpkg_helpers import RobotpkgTests \n")
f.write("dict_robot_pkgs= [ \n")
for key in self.dict_robot_pkgs:
print(key)
#print(key)
if len(self.robotpkg_src_intro.package_dict[key].master_repository)==0 :
f.write(" (\'"+ key +"\', None), \n")
else:
......@@ -118,14 +152,19 @@ class RobotpkgGenerateDockerFile:
os.chmod(filename,stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP |stat.S_IROTH | stat.S_IXOTH )
def generate_docker_file(self):
""" This methods generates the dockerfile to compile robotpkg and the repository being analyzed.
"""
filename = self.dst_dir + "/Dockerfile"
with open(filename,"w") as f:
f.write("FROM ubuntu:16.04\n")
f.write("FROM ubuntu:16.04 as " + self.project_name+ "_img\n")
f.write("RUN apt-get update\n")
f.write("RUN apt-get install -y python3 git g++ make bzip2 libbz2-dev ccache zlib1g zlib1g-dev libssl-dev libncurses-dev pax tar\n")
f.write("RUN apt-get install -y assimp-utils libassimp-dev libboost-dev libboost-date-time-dev libboost-filesystem-dev\n")
f.write("RUN apt-get install -y libboost-system-dev libboost-thread-dev cmake\n")
f.write("RUN apt-get install -y libeigen3-dev pkg-config apt-utils\n")
f.write("RUN apt-get install -yyq python3 python3-dev git g++ make ccache libssl-dev libncurses-dev pax tar\n")
f.write("RUN apt-get install -yyq libboost-dev libboost-date-time-dev libboost-filesystem-dev\n")
f.write("RUN apt-get install -yyq libboost-system-dev libboost-thread-dev cmake\n")
f.write("RUN apt-get install -yyq libeigen3-dev pkg-config apt-utils doxygen \n")
# Add packages found as an OS dependency in the packages
for an_os_pkg in self.tree_of_includes_os:
f.write('RUN apt-get install -y '+an_os_pkg+'\n')
for an_os_pkg in self.dict_os_pkgs:
f.write('RUN apt-get install -y '+an_os_pkg+'\n')
f.write("RUN mkdir -p /workdir\n")
......
......@@ -3,43 +3,99 @@ import re
class RobotpkgPackage:
def __init__(self,name,path,group):
def __init__(self,name,path,group,subgroup=None,debug=0):
self.name=name
self.path=path
self.group=group
self.debug=debug
self.subgroup=subgroup
def display(self):
""" Display the information extracted from robotpkg
"""
print("Name: "+self.name)
print("Path: "+self.path)
print("Group: "+self.group)
if not self.subgroup==None:
print("Subgroup: "+self.subgroup)
if hasattr(self,'rpkg_name'):
if not len(self.rpkg_name)==0:
print("Robotpkg name: ")
print(self.rpkg_name)
if hasattr(self,'org_name'):
if not len(self.org_name)==0:
print("Organization: ")
print(self.org_name)
if hasattr(self,'master_repository'):
if not len(self.master_repository)==0:
print("Master repository: ")
print(self.master_repository)
if hasattr(self,'version'):
if not len(self.version)==0:
print("Version: ")
print(self.version)
if hasattr(self,'includes_depend'):
print("Dependencies in Makefile with depend.mk:")
print(self.includes_depend)
if hasattr(self,'includes_mk'):
print("Dependencies in Makefile in mk group:")
print(self.includes_mk)
if hasattr(self,'depend_mk_system_pkg'):
print("Depend SYSTEM in depend.mk:")
print(self.depend_mk_system_pkg)
def analyze_makefile(self,make_content):
""" Parse the content of a CMakeLists.txt """
""" This methods analyzes the contents of the string make_content.
The string is the contents of the Makefile file inside the package directory.
Right now it populates the attributes:
rpkg_name: The name of the package as PKGNAME (if provided)
org_name: The name of the organization in charge of the package (if provided)
license: The name of the package's license (if provided)
master_repository: Link to the repository of the package (if provided)
version: Version of the package (necessary)
includes_depend_mk: list of packages provided by robotpkg
includes_mk. list of packages needed for the current package but provided by the system.
"""
# Search for NAME
self.rpkg_name = re.findall("PKGNAME\s*=\s*([0-9azA-Z-]+)",make_content)
print("analyze_makefile for " + self.name)
if not len(self.rpkg_name)==0 :
print("analyze_makefile: " + self.rpkg_name[0])
if self.debug>3:
print("analyze_makefile for " + self.name)
if not len(self.rpkg_name)==0 :
print("analyze_makefile: " + self.rpkg_name[0])
# Search for ORG
self.org_name = re.findall("ORG\s*=\s*([azA-Z-]+)",make_content)
if not len(self.org_name)==0:
print("analyze_makefile: " + self.org_name[0])
if self.debug>3:
if not len(self.org_name)==0:
print("analyze_makefile: " + self.org_name[0])
# Search for LICENSE
self.license = re.findall("LICENSE\s*=\s*([a-zA-Z0-9\-]+)",make_content)
if not len(self.license)==0:
print("analyze_makefile license: " + self.license[0])
if self.debug>3:
if not len(self.license)==0:
print("analyze_makefile license: " + self.license[0])
# Search for MASTER_REPOSITORY
self.master_repository = re.findall("MASTER_REPOSITORY\s*=\s*([a-zA-Z0-9\-\${}_]+)",make_content)
if not len(self.master_repository)==0:
print("analyze_makefile master_repository: " + self.master_repository[0])
if self.debug>3:
if not len(self.master_repository)==0:
print("analyze_makefile master_repository: " + self.master_repository[0])
# Search for VERSION
self.version = re.findall("VERSION\s*=\s*([a-zA-Z0-9\-.]+)",make_content)
if not len(self.version)==0:
print("analyze_makefile version: " + self.version[0])
if self.debug>3:
if not len(self.version)==0:
print("analyze_makefile version: " + self.version[0])
# Search for include
self.includes_depend = re.findall("include\s*../../([0-9a-zA-Z-]+)/([0-9a-zA-Z-]+)/depend.mk",make_content)
......@@ -53,29 +109,50 @@ class RobotpkgPackage:
# Reading Makefile
os.chdir(self.path)
with open("Makefile","r") as f_cmakelists:
make_content = f_cmakelists.read()
self.analyze_makefile(make_content)
if os.path.isfile("Makefile"):
with open("Makefile",mode='r',encoding='utf-8') as f_cmakelists:
make_content = f_cmakelists.read()
self.analyze_makefile(make_content)
# Going back to where we were
os.chdir(current_path)
def analyze_depend_mk(self,depend_mk_content):
""" This methods analyzes the contents of the string depend_mk_content.
The string is the contents of the depend.mk file inside the package directory.
Right now it populates one list called depend_mk_system_pkg. It gives
the system packages needed for the current package.
"""
# Search for SYSTEM_PKG
self.depend_mk_system_pkg = re.findall("SYSTEM_PKG\.Ubuntu\.([0-9a-zA-Z\-]+)\s*=([0-9a-zA-Z\-]+)",depend_mk_content)
self.depend_mk_system_pkg = re.findall("SYSTEM_PKG\.Ubuntu\.([0-9a-zA-Z\-]+)\s*=\s*([0-9a-zA-Z\-]+)",depend_mk_content)
if self.debug>3:
print("depend_mk_system_pkg: " + self.name)
print(self.depend_mk_system_pkg)
if len(self.depend_mk_system_pkg)==0:
self.depend_mk_system_pkg = re.findall("SYSTEM_PKG\.Debian\.([0-9a-zA-Z\-]+)\s*=\s*([0-9a-zA-Z\-]+)",depend_mk_content)
print(self.depend_mk_system_pkg)
if self.debug>3:
print(self.depend_mk_system_pkg)
def read_depend_mk(self):
""" This methods open the file depend.mk inside a package directory.
Right it populates the packages with one variable depend_mk_system_pkg which are
the system packages needed for this package.
"""
# Keep current directory.
current_path=os.getcwd()
# Reading Makefile
os.chdir(self.path)
with open("depend.mk","r") as f_depend_mk:
depend_mk_content = f_depend_mk.read()
self.analyze_depend_mk(depend_mk_content)
if os.path.isfile("depend.mk"):
with open("depend.mk",mode='r',encoding='utf-8') as f_depend_mk:
depend_mk_content = f_depend_mk.read()
self.analyze_depend_mk(depend_mk_content)
# Going back to where we were
os.chdir(current_path)
......
......@@ -5,7 +5,7 @@ from .package import RobotpkgPackage
# This class handles the analysis of a robotpkg directory
class RobotpkgSrcIntrospection:
def __init__(self, ROBOTPKG_ROOT_SRC=None):
def __init__(self, ROBOTPKG_ROOT_SRC=None,debug=0):
""" Class to perform robotpkg introspection
Arguments:
......@@ -13,7 +13,23 @@ class RobotpkgSrcIntrospection:
"""
self.ROBOTPKG_ROOT_SRC= ROBOTPKG_ROOT_SRC
self.build_list_of_packages()
self.debug=debug
def create_new_pkg_description(self,group,pkg_name,dirname,subgroup=None):
""" Create a package object.
group: Given by the directory in which is the package
subgroup: Some groups have another level of hierarchy (mk for instance)
pkg_name: Name of the package here the directory in which the package is described
"""
self.package_dict[pkg_name]=RobotpkgPackage(pkg_name,
dirname + '/' + pkg_name,
group,subgroup)
# Analyzes the file Makefile
self.package_dict[pkg_name].read_makefile()
# Analyzes the file depend.mk
self.package_dict[pkg_name].read_depend_mk()
#print(dirname+'/'+pkg_name)
def build_list_of_packages(self):
""" Class to perform robotpkg introspection
......@@ -28,16 +44,28 @@ class RobotpkgSrcIntrospection:
for adir in dirs:
dirname = self.ROBOTPKG_ROOT_SRC+'/'+adir
if os.path.isdir(dirname):
os.chdir(dirname)
# In each subgroup search for package
subdirs= os.listdir()
for asubdir in subdirs:
if os.path.isdir(asubdir):
self.package_dict[asubdir]=RobotpkgPackage(asubdir,
dirname + '/' + asubdir,
adir)
os.chdir(self.ROBOTPKG_ROOT_SRC)
os.chdir(dirname)
# In each subgroup search for package
subdirs= os.listdir()
for asubdir in subdirs:
if os.path.isdir(asubdir):
os.chdir(asubdir)
# Special treatment is asubdir is mk
# There is another level of hierarchy
if adir=="mk":
subtwodirs = os.listdir()
for asubtwodir in subtwodirs:
if os.path.isdir(asubtwodir):
os.chdir(asubtwodir)
self.create_new_pkg_description(adir,asubdir,dirname,subgroup=asubtwodir)
os.chdir(dirname+'/'+asubdir)
else:
self.create_new_pkg_description(adir,asubdir,dirname)
os.chdir(dirname)
os.chdir(self.ROBOTPKG_ROOT_SRC)
# Going back to where we were
os.chdir(current_path)
......@@ -49,5 +77,50 @@ class RobotpkgSrcIntrospection:
return False
def display(self):
for aRbpkg in self.package_dict:
aRbpkg.display()
for pkg_name,a_rpkg in self.package_dict.items():
a_rpkg.display()
def build_tree_of_dependencies(self,a_rpkg):
""" This methods is looking for the whole set of dependencies inside robotpkg
It creates a python set called tree_of_includes_dep which is giving the set
of packages needed by the package a_rpkg.
For Ubuntu and Debian if available it provides a system equivalent.
"""
if self.debug>3:
print("Build tree of dependencies " + a_rpkg.name)
a_rpkg.tree_of_includes_os=set()
a_rpkg.tree_of_includes_dep=set()
# Check if the package is having an OS equivalent.
if self.debug>3:
print("Checking depend_mk_system_pkg")
print(a_rpkg.depend_mk_system_pkg)
if not len(a_rpkg.depend_mk_system_pkg)==0:
a_rpkg.tree_of_includes_os.add(a_rpkg.depend_mk_system_pkg[0][1])
return
# Deal with direct robotpkg dependencies
for lincludes_dep_group,lincludes_dep_name in a_rpkg.includes_depend:
# If the package was found
if lincludes_dep_name in self.package_dict.keys():
# Includes robotpkg dependencies
self.build_tree_of_dependencies(self.package_dict[lincludes_dep_name])
if len(self.package_dict[lincludes_dep_name].tree_of_includes_os)==0:
a_rpkg.tree_of_includes_dep.add(lincludes_dep_name)
a_rpkg.tree_of_includes_dep = a_rpkg.tree_of_includes_dep.union(self.package_dict[lincludes_dep_name].tree_of_includes_dep)
a_rpkg.tree_of_includes_os = a_rpkg.tree_of_includes_os.union(self.package_dict[lincludes_dep_name].tree_of_includes_os)
else:
print("Did not find "+lincludes_dep_name)
if self.debug>3:
print("build_tree_of_dependencies: " +a_rpkg.name)
print("robotpkg dep:")
print(a_rpkg.tree_of_includes_dep)
print("OS dep:")
print(a_rpkg.tree_of_includes_os)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment