[robotpkg_helpers] Creates a real python module is proper format

scripts/robotpkg-test3-wipuser.sh

-#!/bin/bash
-set -x
-RED='\033[0;31m'
-NC='\033[0m' # No Color
-printf "User: ${USER}\n"
-
-#Phase 1
-printf "${RED}Creating the repositories${NC}\n"
-export ROBOTPKG_ROOT=${HOME}/devel-src/robotpkg-test3
-mkdir -p ${ROBOTPKG_ROOT}/
-mkdir -p ${ROBOTPKG_ROOT}/install
-
-printf "${RED}Cloning robotpkg${NC}\n"
-cd ${ROBOTPKG_ROOT}
-git clone https://git.openrobots.org/robots/robotpkg.git
-cd robotpkg
-printf "${RED}Cloning robotpkg/wip ${NC}\n"
-git clone git@gepgitlab.laas.fr:${USER}/robotpkg-wip.git wip
-printf "${RED}Boostrapping ${NC}\n"
-cd bootstrap
-./bootstrap --prefix=${ROBOTPKG_ROOT}/install
-
-# Phase 2
-export ROBOTPKG_BASE=${ROBOTPKG_ROOT}/install
-
-
-# Build talos simulation
-printf "${RED}Preparing robotpkg.conf${NC}\n"
-cat ${PWD}/etc/addrobotpkg.conf >> ${ROBOTPKG_BASE}/etc/robotpkg.conf
-cd ${ROBOTPKG_ROOT}/robotpkg/wip/talos-simulation
-env &> /tmp/check-env-rpkg
-printf "${RED}Compiling (quite long...)${NC}\n"
-make install -j 8
-cd ${ROBOTPKG_ROOT}/robotpkg/wip/talos-dev
-env &> /tmp/check-env-rpkg
-printf "${RED}Compiling (quite long...)${NC}\n"
-make install -j 8
-

scripts/robotpkg_helpers/__init__.py

+from .test_deployment import RobotpkgTests
+from .dockerfile import RobotpkgGenerateDockerFile
+from .src_introspection import RobotpkgSrcIntrospection,RobotpkgPackage
+from .utils import execute,execute_call,build_test_rc_robotpkg_vars

scripts/robotpkg_helpers/dockerfile.py

+#!/usr/bin/python3
+import subprocess
+import os
+import re
+import stat
+from .utils import execute, execute_call
+from .src_introspection import RobotpkgSrcIntrospection,RobotpkgPackage
+
+def valid_name(name):
+    return name.replace('_', ' ').replace('-', ' ').lower()
+
+class RobotpkgGenerateDockerFile:
+
+    def __init__(self,CMakeLists_txt_path,ROBOTPKG_ROOT_SRC):
+        """ Creates a docker file from a CMakeLists.txt file
+
+        Arguments:
+        CMakeLists_txt_path: The path to the CMakeLists.txt to analyze.
+        """
+        self.CMakeLists_txt_path = CMakeLists_txt_path
+        self.robotpkg_src_intro = RobotpkgSrcIntrospection(ROBOTPKG_ROOT_SRC)
+        self.cmake()
+        self.filter_packages()
+        self.display_packages()
+        self.generate_test_rc()
+        self.generate_docker_file()
+        #self.build_docker()
+        
+    def init_colors(self):
+        """ Initialize colors for beautification
+        
+        The following variables are available: 
+        REG, GREEN, PURPLE, NC (no color)
+        """
+        self.RED =  '\033[0;31m'
+        self.GREEN= '\033[0;32m'
+        self.PURPLE='\033[0;35m'
+        self.NC =   '\033[0m'
+    
+    def cmake(self):
+        # Read CMake file
+        filename = self.CMakeLists_txt_path
+        self.dict_packages={}
+        with open(filename,"r") as f:
+          content = f.read()
+          # Search for ADD_[REQUIRED|OPTIONAL]_DEPENDENCY and put it in the list of files
+          for dependency,version in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^\s>="\')]+)\s*[>=]*\s*([\d|.]*)["\']?\)', content, re.I):
+              self.dict_packages[dependency]=version
+          print("CMakeLists.txt needs the following packages:")
+          print(self.dict_packages)
+        f.close()
+
+    def filter_packages(self):
+        # Need to use the keys() method to iterate
+        # as we are removing from the dictionnary using key.
+        print("Filter packages")
+        new_dict_packages = dict(self.dict_packages)
+        for pkg_name in self.dict_packages.keys():
+            r = self.robotpkg_src_intro.is_pkg_present(pkg_name)
+            if not r:
+                del new_dict_packages[pkg_name]
+            else:
+                print("Found pkg " + pkg_name + " in robotpkg")
+        self.dict_packages = new_dict_packages
+
+    def display_packages(self):
+        for pkg_name,pkg_version in self.dict_packages.items():
+            print("Package name:" + pkg_name + " " + pkg_version)
+            
+    def generate_test_rc(self):
+        # Generate python file
+        filename = "robotpkg_for_docker_file.py"
+        with open(filename,"w") as f:
+            f.write("#!/usr/bin/python3\n")
+            f.write("from robotpkg_tests import RobotpkgTests \n")
+            f.write("dict_packages= [ \n")
+            for key in  self.dict_packages:
+              f.write("  (\'"+ key +"\',\'" + self.dict_packages[key] + "\'), \n")
+            f.write("]\n")
+            f.write("arpgtestrc =RobotpkgTests()\n")
+            f.write("arpgtestrc.perform_test_rc(dict_packages)\n")
+        f.close()
+        os.chmod(filename,stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP |stat.S_IROTH | stat.S_IXOTH )
+
+    def generate_docker_file(self):
+        filename = "Dockerfile"
+        with open(filename,"w") as f:
+            f.write("FROM ubuntu:16.04\n")
+            f.write("RUN apt-get update\n")
+            f.write("RUN apt-get install -y python3 git g++ make\n")
+            f.write("RUN mkdir -p /workdir\n")
+            f.write("WORKDIR /workdir\n")
+            f.write("COPY . /workdir\n")
+            f.write("RUN ls /workdir\n")
+            f.write("RUN ls . \n")            
+            f.write("RUN /usr/bin/python3 ./robotpkg_for_docker_file.py\n")
+            f.write("\n")
+        f.close()
+            
+    def build_docker(self):
+        print("Build docker file")
+        self.env = os.environ.copy()
+        execute("docker build -f Dockerfile .", self.env,3)

scripts/robotpkg_helpers/robotpkg_dockerfile.py

+#!/usr/bin/python3
+import subprocess
+import os
+import re
+import stat
+from robotpkg_tests import execute, execute_call
+
+def valid_name(name):
+    return name.replace('_', ' ').replace('-', ' ').lower()
+
+class RobotpkgGenerateDockerFile:
+
+    def __init__(self,CMakeLists_txt_path=None):
+        """ Creates a docker file from a CMakeLists.txt file
+
+        Arguments:
+        CMakeLists_txt_path: The path to the CMakeLists.txt to analyze.
+        """
+        self.CMakeLists_txt_path = CMakeLists_txt_path
+        self.cmake()
+        self.generate_test_rc()
+        self.generate_docker_file()
+        self.build_docker()
+        
+    def init_colors(self):
+        """ Initialize colors for beautification
+        
+        The following variables are available: 
+        REG, GREEN, PURPLE, NC (no color)
+        """
+        self.RED =  '\033[0;31m'
+        self.GREEN= '\033[0;32m'
+        self.PURPLE='\033[0;35m'
+        self.NC =   '\033[0m'
+    
+    def cmake(self):
+        # Read CMake file
+        filename = self.CMakeLists_txt_path
+        self.arch_release_candidates= {}
+        with open(filename,"r") as f:
+          content = f.read()
+          for dependency,version in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^>="\')]+)\s*[>=]*\s*([\d|.]*)["\']?\)', content, re.I):
+              self.arch_release_candidates[dependency]=version
+          print(self.arch_release_candidates)
+        f.close()
+
+        
+    def generate_test_rc(self):
+        # Generate python file
+        filename = "robotpkg_for_docker_file.py"
+        with open(filename,"w") as f:
+            f.write("#!/usr/bin/python3\n")
+            f.write("from robotpkg_tests import RobotpkgTests \n")
+            f.write("arch_release_candidates= [ \n")
+            for key in  self.arch_release_candidates:
+              f.write("  (\'"+ key +"\',\'" + self.arch_release_candidates[key] + "\'), \n")
+            f.write("]\n")
+            f.write("arpgtestrc =RobotpkgTests()\n")
+            f.write("arpgtestrc.perform_test_rc(arch_release_candidates)\n")
+        f.close()
+        os.chmod(filename,stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP |stat.S_IROTH | stat.S_IXOTH )
+
+    def generate_docker_file(self):
+        filename = "Dockerfile"
+        with open(filename,"w") as f:
+            f.write("FROM ubuntu:16.04\n")
+            f.write("RUN apt-get update\n")
+            f.write("RUN apt-get install -y python3 git g++ make\n")
+            f.write("RUN mkdir -p /workdir\n")
+            f.write("WORKDIR /workdir\n")
+            f.write("COPY . /workdir\n")
+            f.write("RUN ls /workdir\n")
+            f.write("RUN ls . \n")            
+            f.write("RUN /usr/bin/python3 ./robotpkg_for_docker_file.py\n")
+            f.write("\n")
+        f.close()
+            
+    def build_docker(self):
+        print("Build docker file")
+        self.env = os.environ.copy()
+        execute("docker build -f Dockerfile .", self.env,3)

scripts/robotpkg_helpers/src_introspection.py

+import os
+
+class RobotpkgPackage:
+    def __init__(self,name,path,group):
+        self.name=name
+        self.path=path
+        self.group=group
+
+    def display(self):
+        print("Name: "+self.name)
+        print("Path: "+self.path)
+        print("Group: "+self.group)
+    
+class RobotpkgSrcIntrospection:
+
+    def __init__(self, ROBOTPKG_ROOT_SRC=None):
+        """ Class to perform robotpkg introspection
+
+        Arguments:
+        ROBOTPKG_ROOT_SRC: The directory where the whole robotpkg source is located.
+        """ 
+        self.ROBOTPKG_ROOT_SRC= ROBOTPKG_ROOT_SRC
+        self.build_list_of_packages()
+        
+    def build_list_of_packages(self):
+        """ Class to perform robotpkg introspection
+
+        Arguments:
+        ROBOTPKG_ROOT_SRC: The directory where the whole robotpkg source is located.
+        """
+        
+        self.package_dict={}
+        # Explore robotpkg src direction
+        dirs=os.listdir(self.ROBOTPKG_ROOT_SRC)
+        for adir in dirs:
+            dirname = self.ROBOTPKG_ROOT_SRC+'/'+adir
+            if os.path.isdir(dirname):
+              os.chdir(dirname)
+
+              # In each subgroup search for package
+              subdirs= os.listdir()
+              for asubdir in subdirs:
+                  if os.path.isdir(asubdir):
+                      self.package_dict[asubdir]=RobotpkgPackage(asubdir,
+                                                                 dirname + '/' + asubdir,
+                                                                 adir)
+                      
+              os.chdir(self.ROBOTPKG_ROOT_SRC)
+                
+    def is_pkg_present(self,package_name):
+        if package_name in self.package_dict.keys():
+            return True
+        else:
+            return False
+
+    def display(self):
+        for aRbpkg in self.package_dict:
+            aRbpkg.display()

scripts/robotpkg_tests.py → scripts/robotpkg_helpers/test_deployment.py

@@ -147,6 +147,18 @@ class RobotpkgTests:
             'ACCEPTABLE_LICENSES+=pal-license',
             'ROS_PACKAGE_PATH='+self.ROBOTPKG_ROOT+'/install/share:$ROS_PACKAGE_PATH']
 
+    def execute_call(self,bashCommand):
+        """ Execute baschCommand 
+        
+        Keyword arguments:
+        bashCommand -- the bashCommand to be run in a bash script
+        
+        It returns a list of binary string that can be iterate 
+        and decode.
+        
+        """
+        subprocess.call(bashCommand,shell=True)
+        
     def execute(self,bashCommand):
         """ Execute baschCommand 
         
@@ -165,7 +177,7 @@ class RobotpkgTests:
                                    env=self.env)
         outputdata, error = process.communicate()
         for stdout_line in outputdata.splitlines():
-            print(stdout_line.decode('ascii'))
+            print(stdout_line.decode('utf-8'))
         return outputdata
         
     def prepare_robotpkg(self,wip_repository):
@@ -286,7 +298,7 @@ class RobotpkgTests:
                     # Check the branch
                     outputdata =self.execute("git symbolic-ref --short -q HEAD")
                     for stdout_line in outputdata.splitlines():
-                        lstr = str(stdout_line.decode('ascii'))
+                        lstr = str(stdout_line.decode('utf-8'))
                         if lstr != branchname:
                           print(self.RED+' Wrong branch name: '+lstr+' instead of '+branchname+self.NC)
                         else:
@@ -367,5 +379,5 @@ class RobotpkgTests:
         os.makedirs(dest_dist_files_path,0o777,True)
         bashcmd="cp -r "+dist_files_path+"/* "+dest_dist_files_path
         print("bashcmd: "+bashcmd)
-        self.execute(bashcmd)
+        self.execute_call(bashcmd)
         self.compile_package('talos-dev')

scripts/robotpkg_helpers/tests/robotpkg_helpers

+..
\ No newline at end of file

scripts/robotpkg-test-distfiles.py → scripts/robotpkg_helpers/tests/test_distfiles.py

 #!/usr/bin/python3
 from robotpkg_tests import RobotpkgTests
 
-ROBOTPKG_ROOT='/home/ostasse/devel-src/robotpkg-test3'
+ROBOTPKG_ROOT='/home/ostasse/devel-src/robotpkg-test-rc'
 arpg_test_dist_files =RobotpkgTests(ROBOTPKG_ROOT)
 wip_repository='git@gepgitlab.laas.fr:ostasse/robotpkg-wip.git wip'
 dist_files_path='/home/ostasse/distfiles'
 
 arpg_test_dist_files.perform_test_dist_files(wip_repository,dist_files_path)
+arpg_test_dist_files.compile_package('talos-dev')
+

scripts/robotpkg_helpers/tests/test_gen_dockerfile.py

+#!/usr/bin/python3
+
+from robotpkg_helpers import RobotpkgGenerateDockerFile,build_test_rc_robotpkg_vars
+
+robotpkg_vars = build_test_rc_robotpkg_vars()
+
+CMakeLists_txt_path="/home/ostasse/devel-src/pinocchio/CMakeLists.txt"
+arpg_gen_docker=RobotpkgGenerateDockerFile(CMakeLists_txt_path,robotpkg_vars['SRC'])
+
+
+
+
+
+
+

scripts/robotpkg_helpers/tests/test_src_introspection.py

+#!/usr/bin/python3
+from unittest import TestCase
+
+from robotpkg_helpers import RobotpkgSrcIntrospection, build_test_rc_robotpkg_vars
+
+robotpkg_vars = build_test_rc_robotpkg_vars()
+
+arpg_src_intros = RobotpkgSrcIntrospection(robotpkg_vars['SRC'])
+arpg_src_intros.display()
+