Skip to content

GitLab

Commit 16011bcc authored by Olivier Stasse's avatar Olivier Stasse
Browse files

Split src_introspection.py and add package.py 

Use robotpkg information to decide if checkout is needed or not.
Make possible to go in other place than wip.
parent 14ea35ff
Hide whitespace changes
Inline Side-by-side
scripts/robotpkg_helpers/dockerfile.py
View file @ 16011bcc
#!/usr/bin/python3
import subprocess
import glob
import inspect
import os
import re
from shutil import copy
import stat
import subprocess
import sys
from .utils import execute, execute_call
from .src_introspection import RobotpkgSrcIntrospection,RobotpkgPackage
......@@ -11,20 +15,25 @@ def valid_name(name):
class RobotpkgGenerateDockerFile:
def __init__(self,CMakeLists_txt_path,ROBOTPKG_ROOT_SRC):
def __init__(self,CMakeLists_txt_path,ROBOTPKG_ROOT_SRC, dst_dir="/tmp/robotpkg_docker"):
""" Creates a docker file from a CMakeLists.txt file
Arguments:
CMakeLists_txt_path: The path to the CMakeLists.txt to analyze.
ROBOTPKG_ROOT_SRC: The path to robotpkg sources.
dst_dir: The path where the files should be generated
"""
self.CMakeLists_txt_path = CMakeLists_txt_path
self.dst_dir = dst_dir
self.check_dst_dir()
self.prepare_dst_dir()
self.robotpkg_src_intro = RobotpkgSrcIntrospection(ROBOTPKG_ROOT_SRC)
self.cmake()
self.filter_packages()
self.filter_robotpkg_packages()
self.display_packages()
self.generate_test_rc()
self.generate_docker_file()
#self.build_docker()
def init_colors(self):
""" Initialize colors for beautification
......@@ -36,58 +45,86 @@ class RobotpkgGenerateDockerFile:
self.GREEN= '\033[0;32m'
self.PURPLE='\033[0;35m'
self.NC = '\033[0m'
def check_dst_dir(self):
if not os.path.isdir(self.dst_dir):
os.mkdir(self.dst_dir)
os.mkdir(self.dst_dir+'/robotpkg_helpers')
def prepare_dst_dir(self):
current_py_file= (inspect.getfile(inspect.currentframe()) ) # script filename (usually with path)
# Find path where this current python script is located
src_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script director
list_py = glob.glob(src_dir+ "/*py")
for apy_file in list_py:
copy(apy_file,self.dst_dir+'/robotpkg_helpers')
def cmake(self):
# Read CMake file
filename = self.CMakeLists_txt_path
self.dict_packages={}
self.dict_robot_pkgs={}
self.dict_os_pkgs=[]
with open(filename,"r") as f:
content = f.read()
# Search for ADD_[REQUIRED|OPTIONAL]_DEPENDENCY and put it in the list of files
for dependency,version in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^\s>="\')]+)\s*[>=]*\s*([\d|.]*)["\']?\)', content, re.I):
self.dict_packages[dependency]=version
self.dict_robot_pkgs[dependency]=version
print("CMakeLists.txt needs the following packages:")
print(self.dict_packages)
print(self.dict_robot_pkgs)
f.close()
def filter_packages(self):
def filter_robotpkg_packages(self):
# Need to use the keys() method to iterate
# as we are removing from the dictionnary using key.
print("Filter packages")
new_dict_packages = dict(self.dict_packages)
for pkg_name in self.dict_packages.keys():
new_dict_robot_pkgs = dict(self.dict_robot_pkgs)
for pkg_name in self.dict_robot_pkgs.keys():
r = self.robotpkg_src_intro.is_pkg_present(pkg_name)
if not r:
del new_dict_packages[pkg_name]
del new_dict_robot_pkgs[pkg_name]
self.dict_os_pkgs.append(pkg_name)
else:
print("Found pkg " + pkg_name + " in robotpkg")
self.dict_packages = new_dict_packages
print("\n**********************\nFound pkg " + pkg_name + " in robotpkg")
self.robotpkg_src_intro.package_dict[pkg_name].read_makefile()
if len(self.robotpkg_src_intro.package_dict[pkg_name].master_repository)==0 :
print("\n The package does not have a master repository")
if not len(self.robotpkg_src_intro.package_dict[pkg_name].version)==0 :
print("\n The package can only provide version :" + self.robotpkg_src_intro.package_dict[pkg_name].version[0])
self.dict_robot_pkgs = new_dict_robot_pkgs
def display_packages(self):
for pkg_name,pkg_version in self.dict_packages.items():
for pkg_name,pkg_version in self.dict_robot_pkgs.items():
print("Package name:" + pkg_name + " " + pkg_version)
def generate_test_rc(self):
# Generate python file
filename = "robotpkg_for_docker_file.py"
filename = self.dst_dir + "/robotpkg_for_docker_file.py"
with open(filename,"w") as f:
f.write("#!/usr/bin/python3\n")
f.write("from robotpkg_tests import RobotpkgTests \n")
f.write("dict_packages= [ \n")
for key in self.dict_packages:
f.write(" (\'"+ key +"\',\'" + self.dict_packages[key] + "\'), \n")
f.write("from robotpkg_helpers import RobotpkgTests \n")
f.write("dict_robot_pkgs= [ \n")
for key in self.dict_robot_pkgs:
print(key)
if len(self.robotpkg_src_intro.package_dict[key].master_repository)==0 :
f.write(" (\'"+ key +"\', None), \n")
else:
f.write(" (\'"+ key +"\',\'" + self.dict_robot_pkgs[key] + "\'), \n")
f.write("]\n")
f.write("arpgtestrc =RobotpkgTests()\n")
f.write("arpgtestrc.perform_test_rc(dict_packages)\n")
f.write("arpgtestrc.perform_test_rc(dict_robot_pkgs)\n")
f.close()
os.chmod(filename,stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP |stat.S_IROTH | stat.S_IXOTH )
def generate_docker_file(self):
filename = "Dockerfile"
filename = self.dst_dir + "/Dockerfile"
with open(filename,"w") as f:
f.write("FROM ubuntu:16.04\n")
f.write("RUN apt-get update\n")
f.write("RUN apt-get install -y python3 git g++ make\n")
for an_os_pkg in self.dict_os_pkgs:
f.write('RUN apt-get install -y '+an_os_pkg+'\n')
f.write("RUN mkdir -p /workdir\n")
f.write("WORKDIR /workdir\n")
f.write("COPY . /workdir\n")
......@@ -100,4 +137,5 @@ class RobotpkgGenerateDockerFile:
def build_docker(self):
print("Build docker file")
self.env = os.environ.copy()
execute("docker build -f Dockerfile .", self.env,3)
bash_cmd = "docker build -f " + self.dst_dir + "/Dockerfile "+self.dst_dir
execute(bash_cmd, self.env,3)
scripts/robotpkg_helpers/package.py 0 → 100644
View file @ 16011bcc
import os
import re
class RobotpkgPackage:
def __init__(self,name,path,group):
self.name=name
self.path=path
self.group=group
def display(self):
print("Name: "+self.name)
print("Path: "+self.path)
print("Group: "+self.group)
def analyze_makefile(self,make_content):
""" Parse the content of a CMakeLists.txt """
# Search for NAME
self.rpkg_name = re.findall("PKGNAME\s*=\s*([azA-Z-]+)",make_content)
print("analyze_makefile for " + self.name)
if not len(self.rpkg_name)==0 :
print("analyze_makefile: " + self.rpkg_name[0])
# Search for ORG
self.org_name = re.findall("ORG\s*=\s*([azA-Z-]+)",make_content)
if not len(self.org_name)==0:
print("analyze_makefile: " + self.org_name[0])
# Search for LICENSE
self.license = re.findall("LICENSE\s*=\s*([a-zA-Z0-9\-]+)",make_content)
if not len(self.license)==0:
print("analyze_makefile license: " + self.license[0])
# Search for MASTER_REPOSITORY
self.master_repository = re.findall("MASTER_REPOSITORY\s*=\s*([a-zA-Z0-9\-\${}_]+)",make_content)
if not len(self.master_repository)==0:
print("analyze_makefile master_repository: " + self.master_repository[0])
# Search for VERSION
self.version = re.findall("VERSION\s*=\s*([a-zA-Z0-9\-.]+)",make_content)
if not len(self.version)==0:
print("analyze_makefile version: " + self.version[0])
def read_makefile(self):
# Keep current directory.
current_path=os.getcwd()
# Reading Makefile
os.chdir(self.path)
with open("Makefile","r") as f_cmakelists:
make_content = f_cmakelists.read()
self.analyze_makefile(make_content)
# Going back to where we were
os.chdir(current_path)
scripts/robotpkg_helpers/robotpkg_dockerfile.py deleted 100644 → 0
View file @ 14ea35ff
#!/usr/bin/python3
import subprocess
import os
import re
import stat
from robotpkg_tests import execute, execute_call
def valid_name(name):
return name.replace('_', ' ').replace('-', ' ').lower()
class RobotpkgGenerateDockerFile:
def __init__(self,CMakeLists_txt_path=None):
""" Creates a docker file from a CMakeLists.txt file
Arguments:
CMakeLists_txt_path: The path to the CMakeLists.txt to analyze.
"""
self.CMakeLists_txt_path = CMakeLists_txt_path
self.cmake()
self.generate_test_rc()
self.generate_docker_file()
self.build_docker()
def init_colors(self):
""" Initialize colors for beautification
The following variables are available:
REG, GREEN, PURPLE, NC (no color)
"""
self.RED = '\033[0;31m'
self.GREEN= '\033[0;32m'
self.PURPLE='\033[0;35m'
self.NC = '\033[0m'
def cmake(self):
# Read CMake file
filename = self.CMakeLists_txt_path
self.arch_release_candidates= {}
with open(filename,"r") as f:
content = f.read()
for dependency,version in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^>="\')]+)\s*[>=]*\s*([\d|.]*)["\']?\)', content, re.I):
self.arch_release_candidates[dependency]=version
print(self.arch_release_candidates)
f.close()
def generate_test_rc(self):
# Generate python file
filename = "robotpkg_for_docker_file.py"
with open(filename,"w") as f:
f.write("#!/usr/bin/python3\n")
f.write("from robotpkg_tests import RobotpkgTests \n")
f.write("arch_release_candidates= [ \n")
for key in self.arch_release_candidates:
f.write(" (\'"+ key +"\',\'" + self.arch_release_candidates[key] + "\'), \n")
f.write("]\n")
f.write("arpgtestrc =RobotpkgTests()\n")
f.write("arpgtestrc.perform_test_rc(arch_release_candidates)\n")
f.close()
os.chmod(filename,stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP |stat.S_IROTH | stat.S_IXOTH )
def generate_docker_file(self):
filename = "Dockerfile"
with open(filename,"w") as f:
f.write("FROM ubuntu:16.04\n")
f.write("RUN apt-get update\n")
f.write("RUN apt-get install -y python3 git g++ make\n")
f.write("RUN mkdir -p /workdir\n")
f.write("WORKDIR /workdir\n")
f.write("COPY . /workdir\n")
f.write("RUN ls /workdir\n")
f.write("RUN ls . \n")
f.write("RUN /usr/bin/python3 ./robotpkg_for_docker_file.py\n")
f.write("\n")
f.close()
def build_docker(self):
print("Build docker file")
self.env = os.environ.copy()
execute("docker build -f Dockerfile .", self.env,3)
scripts/robotpkg_helpers/src_introspection.py
View file @ 16011bcc
import os
import re
from .package import RobotpkgPackage
class RobotpkgPackage:
def __init__(self,name,path,group):
self.name=name
self.path=path
self.group=group
def display(self):
print("Name: "+self.name)
print("Path: "+self.path)
print("Group: "+self.group)
# This class handles the analysis of a robotpkg directory
class RobotpkgSrcIntrospection:
def __init__(self, ROBOTPKG_ROOT_SRC=None):
......@@ -28,7 +20,8 @@ class RobotpkgSrcIntrospection:
Arguments:
ROBOTPKG_ROOT_SRC: The directory where the whole robotpkg source is located.
"""
# Keep current directory.
current_path=os.getcwd()
self.package_dict={}
# Explore robotpkg src direction
dirs=os.listdir(self.ROBOTPKG_ROOT_SRC)
......@@ -44,9 +37,11 @@ class RobotpkgSrcIntrospection:
self.package_dict[asubdir]=RobotpkgPackage(asubdir,
dirname + '/' + asubdir,
adir)
os.chdir(self.ROBOTPKG_ROOT_SRC)
# Going back to where we were
os.chdir(current_path)
def is_pkg_present(self,package_name):
if package_name in self.package_dict.keys():
return True
......
scripts/robotpkg_helpers/test_deployment.py
View file @ 16011bcc
#!/usr/bin/python3
import subprocess
import os
import socket
from pathlib import Path
import socket
from .utils import execute, execute_call
from .src_introspection import RobotpkgPackage,RobotpkgSrcIntrospection
class RobotpkgTests:
def __init__(self,ROBOTPKG_ROOT=None):
""" Init colors and environment variables
""" Install and compile a robotpkg infrastructure
Arguments:
ROBOTPKG_ROOT: The directory where the whole robotpkg install takes place.
"""
"""
# Build internal variables for dir
if ROBOTPKG_ROOT is None:
self.ROBOTPKG_ROOT=str(Path.home())+'/devel-src/robotpkg-test-rc'
else:
self.ROBOTPKG_ROOT=ROBOTPKG_ROOT
self.ROBOTPKG_ROOT_SRC=self.ROBOTPKG_ROOT + '/robotpkg'
self.init_colors()
self.init_environment_variables(self.ROBOTPKG_ROOT)
self.init_robotpkg_conf_add()
self.debug = 0
self.debug = 5
self.ssh_git_openrobots = False
def init_colors(self):
""" Initialize colors for beautification
......@@ -33,6 +40,12 @@ class RobotpkgTests:
self.PURPLE='\033[0;35m'
self.NC = '\033[0m'
def execute(self,bashCommand):
execute(bashCommand,self.env,self.debug)
def execute_call(self,bashCommand):
execute_call(bashCommand,self.debug)
def init_environment_variables(self, ROBOTPKG_ROOT):
"""Prepare the environment variables.
......@@ -146,39 +159,6 @@ class RobotpkgTests:
'PREFER.assimp=system',
'ACCEPTABLE_LICENSES+=pal-license',
'ROS_PACKAGE_PATH='+self.ROBOTPKG_ROOT+'/install/share:$ROS_PACKAGE_PATH']
def execute_call(self,bashCommand):
""" Execute baschCommand
Keyword arguments:
bashCommand -- the bashCommand to be run in a bash script
It returns a list of binary string that can be iterate
and decode.
"""
subprocess.call(bashCommand,shell=True)
def execute(self,bashCommand):
""" Execute baschCommand
Keyword arguments:
bashCommand -- the bashCommand to be run in a bash script
It returns a list of binary string that can be iterate
and decode.
"""
# TODO: Handle error
if self.debug>3:
print("execute bash command: "+bashCommand)
process = subprocess.Popen(bashCommand.split(),
stdout=subprocess.PIPE,
env=self.env)
outputdata, error = process.communicate()
for stdout_line in outputdata.splitlines():
print(stdout_line.decode('utf-8'))
return outputdata
def prepare_robotpkg(self,wip_repository):
"""
......@@ -257,10 +237,11 @@ class RobotpkgTests:
file_robotpkgconf.close()
def build_rpkg_checkout_package(self,packagename):
''' Execute bashcmd in the working directory of packagename'''
""" Execute bashcmd in the working directory of packagename"""
# Going into the repository directory
hostname = socket.gethostname()
pathname = self.ROBOTPKG_ROOT+'/robotpkg/wip/'+packagename+'/work.'+hostname
group = self.robotpkg_src_intro.package_dict[packagename].group
pathname = self.ROBOTPKG_ROOT+'/robotpkg/'+group+'/'+packagename+'/work.'+hostname
return pathname
def apply_rpkg_checkout_package(self,packagename,branchname):
......@@ -271,7 +252,8 @@ class RobotpkgTests:
The location of the repository is specified in the robotpkg Makefile.
"""
print(self.GREEN+'Checkout '+ packagename +' in robotpkg/wip'+self.NC+'\n')
group = self.robotpkg_src_intro.package_dict[packagename].group
print(self.GREEN+'Checkout '+ packagename +' in robotpkg/'+group+self.NC+'\n')
# Checking if we need to clean or not the package
# First check if the working directory exists
......@@ -309,7 +291,7 @@ class RobotpkgTests:
print('Directory to clean: '+str(directory_to_clean))
if directory_to_clean:
# Going into the directory of the package
os.chdir(self.ROBOTPKG_ROOT+'/robotpkg/wip/'+packagename)
os.chdir(self.ROBOTPKG_ROOT+'/robotpkg/'+group+'/'+packagename)
self.execute("make clean confirm")
self.execute("make checkout")
else:
......@@ -343,8 +325,9 @@ class RobotpkgTests:
""" Performs make replace confirm in package working directory
"""
# Going into the directory of the package
os.chdir(self.ROBOTPKG_ROOT+'/robotpkg/wip/'+packagename)
print(self.GREEN+'Compile '+ packagename +' in robotpkg/wip'+self.NC+'\n')
group = self.robotpkg_src_intro.package_dict[packagename].group
os.chdir(self.ROBOTPKG_ROOT+'/robotpkg/'+group+'/'+packagename)
print(self.GREEN+'Compile '+ packagename +' in robotpkg/'+group+self.NC+'\n')
# Compiling the repository
self.execute("make replace confirm")
......@@ -355,22 +338,34 @@ class RobotpkgTests:
Then compile the package with make replace.
Do not use make update confirm, this install the release version (the tar file).
"""
self.apply_rpkg_checkout_package(packagename,branchname)
self.apply_git_checkout_branch(packagename,branchname)
if not branchname==None:
self.apply_rpkg_checkout_package(packagename,branchname)
self.apply_git_checkout_branch(packagename,branchname)
self.compile_package(packagename)
def perform_test_rc(self,arch_release_candidates):
"""Install packages specified by release_candidates
"""Install packages specified by arch_release_candidates
arch_release_candidates: tuple of pair [ ('package_name','branch_name'), ... ]
arch_release_candidates: tuple of list [ ('package_name','branch_name','group'), ... ]
"""
wip_repository="ssh://git@git.openrobots.org/robots/robotpkg/robotpkg-wip wip"
if self.ssh_git_openrobots==True:
wip_repository="ssh://git@git.openrobots.org/robots/robotpkg/robotpkg-wip wip"
else:
wip_repository="https://git.openrobots.org/robots/robotpkg/robotpkg-wip.git wip"
# Create the robotpkg structure.
self.prepare_robotpkg(wip_repository)
# Analysis the robotpkg src structure
self.robotpkg_src_intro= RobotpkgSrcIntrospection(self.ROBOTPKG_ROOT_SRC)
# Download and install each package
for package_name,branch_name in arch_release_candidates:
self.handle_package(package_name,branch_name)
def perform_test_dist_files(self,wip_repository,dist_files_path):
"""Install packages specified by wip_repository with distfiles located in dist_files_path
"""Install packages specified by wip_repository with distfiles
located in dist_files_path
wip_repository [ ('wip_repository'), ... ]
"""
......
scripts/robotpkg_helpers/tests/test_distfiles.py
View file @ 16011bcc
#!/usr/bin/python3
from robotpkg_tests import RobotpkgTests
import os
from robotpkg_helpers import RobotpkgTests, build_test_rc_robot_vars
ROBOTPKG_ROOT='/home/ostasse/devel-src/robotpkg-test-rc'
arpg_test_dist_files =RobotpkgTests(ROBOTPKG_ROOT)
wip_repository='git@gepgitlab.laas.fr:ostasse/robotpkg-wip.git wip'
dist_files_path='/home/ostasse/distfiles'
# This script assumes that a directory in your folder:
# $HOME/devel-src/distfiles
# contains all the distribution files needed to recompile robotpkg.
#
# Robotpkg is extracted here from the repository located at:
# git@gepgitlab.laas.fr:user/robotpkg-wip.git wip
#
# It then rebuild completly robotpkg by extracting the sources in the folder:
# $HOME/devel-src/robotpkg-test-rc/robotpkg
# Then everything is compiled and installed in:
# $HOME/devel-src/robotpkg-test-rc/install
#
robotpkg_vars = build_test_rc_robotpkg_vars()
arpg_test_dist_files =RobotpkgTests(robotpkg_vars['ROOT'])
wip_repository='git@gepgitlab.laas.fr:'+user+'/robotpkg-wip.git wip'
dist_files_path=robotpkg_vars['DISTFILES']
arpg_test_dist_files.perform_test_dist_files(wip_repository,dist_files_path)
arpg_test_dist_files.compile_package('talos-dev')
......
scripts/robotpkg_helpers/tests/test_gen_dockerfile.py
View file @ 16011bcc
......@@ -6,7 +6,7 @@ robotpkg_vars = build_test_rc_robotpkg_vars()
CMakeLists_txt_path="/home/ostasse/devel-src/pinocchio/CMakeLists.txt"
arpg_gen_docker=RobotpkgGenerateDockerFile(CMakeLists_txt_path,robotpkg_vars['SRC'])
#arpg_gen_docker.build_docker()
......
scripts/robotpkg_helpers/tests/test_rc.py
View file @ 16011bcc
#!/usr/bin/python3
from robotpkg_tests import RobotpkgTests
from robotpkg_helpers import RobotpkgTests
arch_release_candidates= [ ('dynamic-graph-v3','rc-v3.2.2'),
('sot-core-v3','rc-v3.3.1'),
......
scripts/robotpkg_helpers/tests/test_rc_other_path.py 0 → 100755
View file @ 16011bcc
#!/usr/bin/python3
from robotpkg_helpers import RobotpkgTests
arch_release_candidates= [ ('dynamic-graph-v3','rc-v3.2.2'),
('sot-core-v3','rc-v3.3.1'),
('py-dynamic-graph-bridge-v3','rc-v3.2.3'),
]
arpgtestrc =RobotpkgTests("/tmp/robotpkg-test-rc")
arpgtestrc.perform_test_rc(arch_release_candidates)
arpgtestrc.compile_package('talos-dev')
scripts/robotpkg_helpers/utils.py 0 → 100644
View file @ 16011bcc
import subprocess
def execute(bashCommand, lenv, debug=0):
""" Execute baschCommand
Keyword arguments:
bashCommand -- the bashCommand to be run in a bash script
It returns a list of binary string that can be iterate
and decode.
"""
# TODO: Handle error
if debug>3:
print("execute bash command: "+bashCommand)
process = subprocess.Popen(bashCommand.split(),
stdout=subprocess.PIPE,
env=lenv)