Commit 7931c27f authored by Guilhem Saurel's avatar Guilhem Saurel
Browse files

dashboard: clean, improve models

parent c98e40b8
......@@ -3,7 +3,6 @@ ENV PYTHONUNBUFFERED 1
RUN mkdir /app /static
WORKDIR /app
COPY requirements.txt .
# Get Dependencies
RUN apk update && apk add --virtual deps \
......@@ -13,7 +12,8 @@ RUN apk update && apk add --virtual deps \
musl-dev \
postgresql-dev \
&& apk add --no-cache libpq
RUN pip install -r requirements.txt
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
RUN apk del deps
COPY . .
......@@ -23,4 +23,4 @@ ENV REDMINE_TOKEN=placeholder \
GITHUB_TOKEN=placeholder \
DJANGO_SECRET_KEY=placeholder \
POSTGRES_PASSWORD=placeholder
RUN python manage.py collectstatic
RUN python manage.py collectstatic --noinput
from django.contrib import admin
from django.contrib.admin import site
# Register your models here.
from .models import License, Package, Project, Repo
for model in [License, Package, Project, Repo]:
site.register(model)
# -*- coding: utf-8 -*-
# Generated by Django 1.11.7 on 2017-11-15 17:31
from __future__ import unicode_literals
# Generated by Django 2.0 on 2017-12-19 18:30
import autoslug.fields
from django.db import migrations, models
import django.db.models.deletion
from django.db import migrations, models
import autoslug.fields
class Migration(migrations.Migration):
......@@ -67,6 +66,9 @@ class Migration(migrations.Migration):
('open_issues', models.PositiveSmallIntegerField(blank=True, null=True)),
('open_pr', models.PositiveSmallIntegerField(blank=True, null=True)),
('repo_id', models.PositiveIntegerField()),
('source_type', models.PositiveSmallIntegerField(choices=[(1, 'github'), (2, 'gitlab'), (3, 'redmine'), (4, 'robotpkg')])),
('api_url', models.CharField(max_length=100)),
('token', models.CharField(blank=True, max_length=50, null=True)),
('license', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='gepetto_packages.License')),
('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gepetto_packages.Package')),
],
......
......@@ -2,17 +2,17 @@
# Generated by Django 1.11.7 on 2017-11-09 15:55
from __future__ import unicode_literals
from django.conf import settings
import os
from django.db import migrations
import requests
GITHUB_API = 'https://api.github.com'
HEADERS = {
'Authorization': f'token {settings.GITHUB_TOKEN}',
'Accept': 'application/vnd.github.drax-preview+json',
}
from gepetto_packages.utils import SOURCES, api_headers, api_data
GITHUB_API = 'https://api.github.com'
GITHUB_TOKEN = os.getenv('GITHUB_TOKEN', '')
HEADERS = api_headers(source=SOURCES.github, token=GITHUB_TOKEN)
PROJECTS = (
'Humanoid Path Planner',
'Stack of Tasks'
......@@ -28,22 +28,21 @@ def github_projects(apps, schema_editor):
Project, License, Package, Repo = (apps.get_model('gepetto_packages', model)
for model in ['Project', 'License', 'Package', 'Repo'])
for project_name in PROJECTS:
project = Project(name=project_name)
project.save()
project = Project.objects.create(name=project_name)
for data in requests.get(f'{GITHUB_API}/orgs/{project.slug}/repos', headers=HEADERS).json():
package = Package(name=data['name'], project=project, homepage=data['homepage'])
package.save()
repo = Repo(package=package, url=data['html_url'], homepage=data['homepage'], repo_id=data['id'],
default_branch=data['default_branch'], open_issues=data['open_issues'])
repo_api_url = f'{GITHUB_API}/repos/{project.slug}/{package.slug}'
repo_data = requests.get(repo_api_url, headers=HEADERS).json()
package = Package.objects.create(name=data['name'], project=project, homepage=data['homepage'])
repo = Repo.objects.create(package=package, url=data['html_url'], homepage=data['homepage'],
repo_id=data['id'], default_branch=data['default_branch'],
open_issues=data['open_issues'], source_type=SOURCES.github,
api_url=GITHUB_API, token=GITHUB_TOKEN)
repo_data = api_data(repo)
if 'license' in repo_data and repo_data['license']:
license_data = repo_data['license']
license, _ = License.objects.get_or_create(github_key=license_data['key'], name=license_data['name'])
repo.license = license
package.license = license
package.save()
repo.open_pr = len(requests.get(f'{repo_api_url}/pulls', headers=HEADERS).json())
repo.open_pr = len(api_data(repo, '/pulls'))
repo.save()
class Migration(migrations.Migration):
......
......@@ -6,6 +6,8 @@ from django.db import migrations
import requests
from gepetto_packages.utils import SOURCES
GITLAB_API = 'https://eur0c.laas.fr/api/v4'
def gitlab(apps, schema_editor):
......@@ -14,7 +16,8 @@ def gitlab(apps, schema_editor):
for data in requests.get(f'{GITLAB_API}/projects', verify=False).json():
package_qs = Package.objects.filter(name=data['name'])
if package_qs.exists():
Repo.objects.create(package=package_qs.first(), url=data['web_url'], repo_id=data['id'])
Repo.objects.create(package=package_qs.first(), url=data['web_url'], repo_id=data['id'],
source_type=SOURCES.gitlab, api_url=GITLAB_API)
class Migration(migrations.Migration):
......
......@@ -2,18 +2,19 @@
# Generated by Django 1.11.7 on 2017-11-14 09:29
from __future__ import unicode_literals
import os
from django.conf import settings
from django.db import migrations
import requests
REDMINE_APIS = [
'https://redmine.laas.fr',
# 'https://git.openrobots.org',
from gepetto_packages.utils import SOURCES, api_headers, api_data
REDMINES = [
('https://redmine.laas.fr', os.getenv('REDMINE_TOKEN', '')),
# ('https://git.openrobots.org', os.getenv('OPENROB_TOKEN', ''),
]
HEADERS = {
'X-Redmine-API-Key': settings.REDMINE_TOKEN,
}
PACKAGES = [
'openhrp3-hrp2',
'Pyrene Talos',
......@@ -23,17 +24,19 @@ def redmine(apps, schema_editor):
Project, License, Package, Repo = (apps.get_model('gepetto_packages', model)
for model in ['Project', 'License', 'Package', 'Repo'])
dummy_project, _ = Project.objects.get_or_create(name='dummy')
for api in REDMINE_APIS:
for data in requests.get(f'{api}/projects.json?limit=100', headers=HEADERS).json()['projects']:
for api, token in REDMINES:
headers = api_headers(source=SOURCES.redmine, token=token)
for data in requests.get(f'{api}/projects.json?limit=100', headers=headers).json()['projects']:
if data['name'] in PACKAGES:
Package.objects.get_or_create(name=data['identifier'], project=dummy_project)
package_qs = Package.objects.filter(name=data['identifier'])
if package_qs.exists():
r = Repo(package=package_qs.first(), repo_id=data['id'], open_pr=0)
repo_data = requests.get(f'{api}/projects/{r.repo_id}.json', headers=HEADERS).json()['project']
r = Repo.objects.create(package=package_qs.first(), repo_id=data['id'], open_pr=0,
source_type=SOURCES.redmine, api_url=api, token=token)
repo_data = api_data(r)['project']
r.homepage = repo_data['homepage']
r.url = f'{api}/projects/{r.package.name}'
issues_data = requests.get(f'{api}/issues.json?project_id={r.repo_id}&status_id=open', headers=HEADERS)
issues_data = requests.get(f'{api}/issues.json?project_id={r.repo_id}&status_id=open', headers=headers)
r.open_issues = issues_data.json()['total_count']
r.save()
if r.homepage and not r.package.homepage:
......
from django.db import models
from ndh.models import TimeStampedModel, NamedModel
from ndh.utils import query_sum
from ndh.models import NamedModel, TimeStampedModel
from ndh.utils import enum_to_choices, query_sum
from .utils import SOURCES, api_headers, api_url
class Project(NamedModel):
......@@ -41,6 +43,9 @@ class Repo(TimeStampedModel):
open_issues = models.PositiveSmallIntegerField(blank=True, null=True)
open_pr = models.PositiveSmallIntegerField(blank=True, null=True)
repo_id = models.PositiveIntegerField()
source_type = models.PositiveSmallIntegerField(choices=enum_to_choices(SOURCES))
api_url = models.CharField(max_length=100)
token = models.CharField(max_length=50, blank=True, null=True)
class Meta:
ordering = ('package', 'url')
......
......@@ -3,6 +3,7 @@ from django.utils.safestring import mark_safe
register = template.Library()
@register.filter
def domain(url):
if url:
......
from django.test import TestCase
# Create your tests here.
from enum import IntEnum
import requests
SOURCES = IntEnum('Source', 'github gitlab redmine robotpkg')
def api_url(repo):
if repo.source_type == SOURCES.github:
return f'{repo.api_url}/repos/{repo.package.project.slug}/{repo.package.slug}'
if repo.source_type == SOURCES.redmine:
return f'{repo.api_url}/projects/{repo.repo_id}.json'
def api_headers(repo=None, source=None, token=None):
if repo is not None:
source = repo.source_type
token = repo.token
if source == SOURCES.github:
return {
'Authorization': f'token {token}',
'Accept': 'application/vnd.github.drax-preview+json',
}
if source == SOURCES.redmine:
return {'X-Redmine-API-Key': token}
def api_data(repo, url=''):
return requests.get(api_url(repo) + url, headers=api_headers(repo)).json()
from django.shortcuts import render
# Create your views here.
......@@ -50,11 +50,11 @@ WSGI_APPLICATION = f'{PROJECT}.wsgi.application'
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'ENGINE': 'django.db.backends.' + ('postgresql' if 'POSTGRES_PASSWORD' in os.environ else 'sqlite3'),
'HOST': 'db',
'USER': 'postgres',
'NAME': 'postgres',
'PASSWORD': os.environ['POSTGRES_PASSWORD'],
'NAME': 'postgres' if 'POSTGRES_PASSWORD' in os.environ else os.path.join(BASE_DIR, 'db.sqlite3'),
'PASSWORD': os.getenv('POSTGRES_PASSWORD', '')
}
}
......@@ -84,6 +84,4 @@ STATIC_ROOT = '/static/'
SITE_ID = 1
REDMINE_TOKEN = os.environ['REDMINE_TOKEN']
GITHUB_TOKEN = os.environ['GITHUB_TOKEN']
SECRET_KEY = os.environ['DJANGO_SECRET_KEY']
SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'placeholder')
......@@ -11,7 +11,7 @@ if __name__ == "__main__":
# issue is really that Django is missing to avoid masking other
# exceptions on Python 2.
try:
import django
import django # noqa
except ImportError:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment