dashboard: clean, improve models

dashboard/Dockerfile

@@ -3,7 +3,6 @@ ENV PYTHONUNBUFFERED 1
 
 RUN mkdir /app /static
 WORKDIR /app
-COPY requirements.txt .
 
 # Get Dependencies
 RUN apk update && apk add --virtual deps \
@@ -13,7 +12,8 @@ RUN apk update && apk add --virtual deps \
    musl-dev \
    postgresql-dev \
    && apk add --no-cache libpq
-RUN pip install -r requirements.txt
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
 RUN apk del deps
 
 COPY . .
@@ -23,4 +23,4 @@ ENV REDMINE_TOKEN=placeholder \
    GITHUB_TOKEN=placeholder \
    DJANGO_SECRET_KEY=placeholder \
    POSTGRES_PASSWORD=placeholder
-RUN python manage.py collectstatic
+RUN python manage.py collectstatic --noinput

dashboard/gepetto_packages/admin.py

-from django.contrib import admin
+from django.contrib.admin import site
 
-# Register your models here.
+from .models import License, Package, Project, Repo
+
+for model in [License, Package, Project, Repo]:
+    site.register(model)

dashboard/gepetto_packages/migrations/0001_initial.py

-# -*- coding: utf-8 -*-
-# Generated by Django 1.11.7 on 2017-11-15 17:31
-from __future__ import unicode_literals
+# Generated by Django 2.0 on 2017-12-19 18:30
 
-import autoslug.fields
-from django.db import migrations, models
 import django.db.models.deletion
+from django.db import migrations, models
+
+import autoslug.fields
 
 
 class Migration(migrations.Migration):
@@ -67,6 +66,9 @@ class Migration(migrations.Migration):
                 ('open_issues', models.PositiveSmallIntegerField(blank=True, null=True)),
                 ('open_pr', models.PositiveSmallIntegerField(blank=True, null=True)),
                 ('repo_id', models.PositiveIntegerField()),
+                ('source_type', models.PositiveSmallIntegerField(choices=[(1, 'github'), (2, 'gitlab'), (3, 'redmine'), (4, 'robotpkg')])),
+                ('api_url', models.CharField(max_length=100)),
+                ('token', models.CharField(blank=True, max_length=50, null=True)),
                 ('license', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='gepetto_packages.License')),
                 ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gepetto_packages.Package')),
             ],

dashboard/gepetto_packages/migrations/0002_github.py

@@ -2,17 +2,17 @@
 # Generated by Django 1.11.7 on 2017-11-09 15:55
 from __future__ import unicode_literals
 
-from django.conf import settings
+import os
+
 from django.db import migrations
 
 import requests
 
-GITHUB_API = 'https://api.github.com'
-HEADERS = {
-    'Authorization': f'token {settings.GITHUB_TOKEN}',
-    'Accept': 'application/vnd.github.drax-preview+json',
-}
+from gepetto_packages.utils import SOURCES, api_headers, api_data
 
+GITHUB_API = 'https://api.github.com'
+GITHUB_TOKEN = os.getenv('GITHUB_TOKEN', '')
+HEADERS = api_headers(source=SOURCES.github, token=GITHUB_TOKEN)
 PROJECTS = (
     'Humanoid Path Planner',
     'Stack of Tasks'
@@ -28,22 +28,21 @@ def github_projects(apps, schema_editor):
     Project, License, Package, Repo = (apps.get_model('gepetto_packages', model)
                                        for model in ['Project', 'License', 'Package', 'Repo'])
     for project_name in PROJECTS:
-        project = Project(name=project_name)
-        project.save()
+        project = Project.objects.create(name=project_name)
         for data in requests.get(f'{GITHUB_API}/orgs/{project.slug}/repos', headers=HEADERS).json():
-            package = Package(name=data['name'], project=project, homepage=data['homepage'])
-            package.save()
-            repo = Repo(package=package, url=data['html_url'], homepage=data['homepage'], repo_id=data['id'],
-                        default_branch=data['default_branch'], open_issues=data['open_issues'])
-            repo_api_url = f'{GITHUB_API}/repos/{project.slug}/{package.slug}'
-            repo_data = requests.get(repo_api_url, headers=HEADERS).json()
+            package = Package.objects.create(name=data['name'], project=project, homepage=data['homepage'])
+            repo = Repo.objects.create(package=package, url=data['html_url'], homepage=data['homepage'],
+                                       repo_id=data['id'], default_branch=data['default_branch'],
+                                       open_issues=data['open_issues'], source_type=SOURCES.github,
+                                       api_url=GITHUB_API, token=GITHUB_TOKEN)
+            repo_data = api_data(repo)
             if 'license' in repo_data and repo_data['license']:
                 license_data = repo_data['license']
                 license, _ = License.objects.get_or_create(github_key=license_data['key'], name=license_data['name'])
                 repo.license = license
                 package.license = license
                 package.save()
-            repo.open_pr = len(requests.get(f'{repo_api_url}/pulls', headers=HEADERS).json())
+            repo.open_pr = len(api_data(repo, '/pulls'))
             repo.save()
 
 class Migration(migrations.Migration):

dashboard/gepetto_packages/migrations/0003_gitlab.py

@@ -6,6 +6,8 @@ from django.db import migrations
 
 import requests
 
+from gepetto_packages.utils import SOURCES
+
 GITLAB_API = 'https://eur0c.laas.fr/api/v4'
 
 def gitlab(apps, schema_editor):
@@ -14,7 +16,8 @@ def gitlab(apps, schema_editor):
     for data in requests.get(f'{GITLAB_API}/projects', verify=False).json():
         package_qs = Package.objects.filter(name=data['name'])
         if package_qs.exists():
-            Repo.objects.create(package=package_qs.first(), url=data['web_url'], repo_id=data['id'])
+            Repo.objects.create(package=package_qs.first(), url=data['web_url'], repo_id=data['id'],
+                                source_type=SOURCES.gitlab, api_url=GITLAB_API)
 
 
 class Migration(migrations.Migration):

dashboard/gepetto_packages/migrations/0004_redmine.py

@@ -2,18 +2,19 @@
 # Generated by Django 1.11.7 on 2017-11-14 09:29
 from __future__ import unicode_literals
 
+import os
+
 from django.conf import settings
 from django.db import migrations
 
 import requests
 
-REDMINE_APIS = [
-    'https://redmine.laas.fr',
-    # 'https://git.openrobots.org',
+from gepetto_packages.utils import SOURCES, api_headers, api_data
+
+REDMINES = [
+    ('https://redmine.laas.fr', os.getenv('REDMINE_TOKEN', '')),
+    # ('https://git.openrobots.org', os.getenv('OPENROB_TOKEN', ''),
 ]
-HEADERS = {
-    'X-Redmine-API-Key': settings.REDMINE_TOKEN,
-}
 PACKAGES = [
     'openhrp3-hrp2',
     'Pyrene Talos',
@@ -23,17 +24,19 @@ def redmine(apps, schema_editor):
     Project, License, Package, Repo = (apps.get_model('gepetto_packages', model)
                                        for model in ['Project', 'License', 'Package', 'Repo'])
     dummy_project, _ = Project.objects.get_or_create(name='dummy')
-    for api in REDMINE_APIS:
-        for data in requests.get(f'{api}/projects.json?limit=100', headers=HEADERS).json()['projects']:
+    for api, token in REDMINES:
+        headers = api_headers(source=SOURCES.redmine, token=token)
+        for data in requests.get(f'{api}/projects.json?limit=100', headers=headers).json()['projects']:
             if data['name'] in PACKAGES:
                 Package.objects.get_or_create(name=data['identifier'], project=dummy_project)
             package_qs = Package.objects.filter(name=data['identifier'])
             if package_qs.exists():
-                r = Repo(package=package_qs.first(), repo_id=data['id'], open_pr=0)
-                repo_data = requests.get(f'{api}/projects/{r.repo_id}.json', headers=HEADERS).json()['project']
+                r = Repo.objects.create(package=package_qs.first(), repo_id=data['id'], open_pr=0,
+                                        source_type=SOURCES.redmine, api_url=api, token=token)
+                repo_data = api_data(r)['project']
                 r.homepage = repo_data['homepage']
                 r.url = f'{api}/projects/{r.package.name}'
-                issues_data = requests.get(f'{api}/issues.json?project_id={r.repo_id}&status_id=open', headers=HEADERS)
+                issues_data = requests.get(f'{api}/issues.json?project_id={r.repo_id}&status_id=open', headers=headers)
                 r.open_issues = issues_data.json()['total_count']
                 r.save()
                 if r.homepage and not r.package.homepage:

dashboard/gepetto_packages/models.py

 from django.db import models
 
-from ndh.models import TimeStampedModel, NamedModel
-from ndh.utils import query_sum
+from ndh.models import NamedModel, TimeStampedModel
+from ndh.utils import enum_to_choices, query_sum
+
+from .utils import SOURCES, api_headers, api_url
 
 
 class Project(NamedModel):
@@ -41,6 +43,9 @@ class Repo(TimeStampedModel):
     open_issues = models.PositiveSmallIntegerField(blank=True, null=True)
     open_pr = models.PositiveSmallIntegerField(blank=True, null=True)
     repo_id = models.PositiveIntegerField()
+    source_type = models.PositiveSmallIntegerField(choices=enum_to_choices(SOURCES))
+    api_url = models.CharField(max_length=100)
+    token = models.CharField(max_length=50, blank=True, null=True)
 
     class Meta:
         ordering = ('package', 'url')

dashboard/gepetto_packages/templatetags/gepetto_packages.py

@@ -3,6 +3,7 @@ from django.utils.safestring import mark_safe
 
 register = template.Library()
 
+
 @register.filter
 def domain(url):
     if url:

dashboard/gepetto_packages/tests.py

-from django.test import TestCase
-
-# Create your tests here.

dashboard/gepetto_packages/utils.py

+from enum import IntEnum
+
+import requests
+
+SOURCES = IntEnum('Source', 'github gitlab redmine robotpkg')
+
+
+def api_url(repo):
+    if repo.source_type == SOURCES.github:
+        return f'{repo.api_url}/repos/{repo.package.project.slug}/{repo.package.slug}'
+    if repo.source_type == SOURCES.redmine:
+        return f'{repo.api_url}/projects/{repo.repo_id}.json'
+
+
+def api_headers(repo=None, source=None, token=None):
+    if repo is not None:
+        source = repo.source_type
+        token = repo.token
+    if source == SOURCES.github:
+        return {
+            'Authorization': f'token {token}',
+            'Accept': 'application/vnd.github.drax-preview+json',
+        }
+    if source == SOURCES.redmine:
+        return {'X-Redmine-API-Key': token}
+
+
+def api_data(repo, url=''):
+    return requests.get(api_url(repo) + url, headers=api_headers(repo)).json()

dashboard/gepetto_packages/views.py

-from django.shortcuts import render
-
-# Create your views here.

dashboard/gepkg/settings.py

@@ -50,11 +50,11 @@ WSGI_APPLICATION = f'{PROJECT}.wsgi.application'
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.postgresql',
+        'ENGINE': 'django.db.backends.' + ('postgresql' if 'POSTGRES_PASSWORD' in os.environ else 'sqlite3'),
         'HOST': 'db',
         'USER': 'postgres',
-        'NAME': 'postgres',
-        'PASSWORD': os.environ['POSTGRES_PASSWORD'],
+        'NAME': 'postgres' if 'POSTGRES_PASSWORD' in os.environ else os.path.join(BASE_DIR, 'db.sqlite3'),
+        'PASSWORD': os.getenv('POSTGRES_PASSWORD', '')
     }
 }
 
@@ -84,6 +84,4 @@ STATIC_ROOT = '/static/'
 
 SITE_ID = 1
 
-REDMINE_TOKEN = os.environ['REDMINE_TOKEN']
-GITHUB_TOKEN = os.environ['GITHUB_TOKEN']
-SECRET_KEY = os.environ['DJANGO_SECRET_KEY']
+SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'placeholder')

dashboard/manage.py

@@ -11,7 +11,7 @@ if __name__ == "__main__":
         # issue is really that Django is missing to avoid masking other
         # exceptions on Python 2.
         try:
-            import django
+            import django  # noqa
         except ImportError:
             raise ImportError(
                 "Couldn't import Django. Are you sure it's installed and "