Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Olivier Stasse
Gepetto Utils
Commits
7931c27f
Commit
7931c27f
authored
Dec 19, 2017
by
Guilhem Saurel
Browse files
dashboard: clean, improve models
parent
c98e40b8
Changes
13
Hide whitespace changes
Inline
Side-by-side
dashboard/Dockerfile
View file @
7931c27f
...
...
@@ -3,7 +3,6 @@ ENV PYTHONUNBUFFERED 1
RUN
mkdir
/app /static
WORKDIR
/app
COPY
requirements.txt .
# Get Dependencies
RUN
apk update
&&
apk add
--virtual
deps
\
...
...
@@ -13,7 +12,8 @@ RUN apk update && apk add --virtual deps \
musl-dev
\
postgresql-dev
\
&&
apk add
--no-cache
libpq
RUN
pip
install
-r
requirements.txt
COPY
requirements.txt .
RUN
pip
install
--no-cache-dir
-r
requirements.txt
RUN
apk del deps
COPY
. .
...
...
@@ -23,4 +23,4 @@ ENV REDMINE_TOKEN=placeholder \
GITHUB_TOKEN=placeholder \
DJANGO_SECRET_KEY=placeholder \
POSTGRES_PASSWORD=placeholder
RUN
python manage.py collectstatic
RUN
python manage.py collectstatic
--noinput
dashboard/gepetto_packages/admin.py
View file @
7931c27f
from
django.contrib
import
admin
from
django.contrib
.admin
import
site
# Register your models here.
from
.models
import
License
,
Package
,
Project
,
Repo
for
model
in
[
License
,
Package
,
Project
,
Repo
]:
site
.
register
(
model
)
dashboard/gepetto_packages/migrations/0001_initial.py
View file @
7931c27f
# -*- coding: utf-8 -*-
# Generated by Django 1.11.7 on 2017-11-15 17:31
from
__future__
import
unicode_literals
# Generated by Django 2.0 on 2017-12-19 18:30
import
autoslug.fields
from
django.db
import
migrations
,
models
import
django.db.models.deletion
from
django.db
import
migrations
,
models
import
autoslug.fields
class
Migration
(
migrations
.
Migration
):
...
...
@@ -67,6 +66,9 @@ class Migration(migrations.Migration):
(
'open_issues'
,
models
.
PositiveSmallIntegerField
(
blank
=
True
,
null
=
True
)),
(
'open_pr'
,
models
.
PositiveSmallIntegerField
(
blank
=
True
,
null
=
True
)),
(
'repo_id'
,
models
.
PositiveIntegerField
()),
(
'source_type'
,
models
.
PositiveSmallIntegerField
(
choices
=
[(
1
,
'github'
),
(
2
,
'gitlab'
),
(
3
,
'redmine'
),
(
4
,
'robotpkg'
)])),
(
'api_url'
,
models
.
CharField
(
max_length
=
100
)),
(
'token'
,
models
.
CharField
(
blank
=
True
,
max_length
=
50
,
null
=
True
)),
(
'license'
,
models
.
ForeignKey
(
blank
=
True
,
null
=
True
,
on_delete
=
django
.
db
.
models
.
deletion
.
CASCADE
,
to
=
'gepetto_packages.License'
)),
(
'package'
,
models
.
ForeignKey
(
on_delete
=
django
.
db
.
models
.
deletion
.
CASCADE
,
to
=
'gepetto_packages.Package'
)),
],
...
...
dashboard/gepetto_packages/migrations/0002_github.py
View file @
7931c27f
...
...
@@ -2,17 +2,17 @@
# Generated by Django 1.11.7 on 2017-11-09 15:55
from
__future__
import
unicode_literals
from
django.conf
import
settings
import
os
from
django.db
import
migrations
import
requests
GITHUB_API
=
'https://api.github.com'
HEADERS
=
{
'Authorization'
:
f
'token
{
settings
.
GITHUB_TOKEN
}
'
,
'Accept'
:
'application/vnd.github.drax-preview+json'
,
}
from
gepetto_packages.utils
import
SOURCES
,
api_headers
,
api_data
GITHUB_API
=
'https://api.github.com'
GITHUB_TOKEN
=
os
.
getenv
(
'GITHUB_TOKEN'
,
''
)
HEADERS
=
api_headers
(
source
=
SOURCES
.
github
,
token
=
GITHUB_TOKEN
)
PROJECTS
=
(
'Humanoid Path Planner'
,
'Stack of Tasks'
...
...
@@ -28,22 +28,21 @@ def github_projects(apps, schema_editor):
Project
,
License
,
Package
,
Repo
=
(
apps
.
get_model
(
'gepetto_packages'
,
model
)
for
model
in
[
'Project'
,
'License'
,
'Package'
,
'Repo'
])
for
project_name
in
PROJECTS
:
project
=
Project
(
name
=
project_name
)
project
.
save
()
project
=
Project
.
objects
.
create
(
name
=
project_name
)
for
data
in
requests
.
get
(
f
'
{
GITHUB_API
}
/orgs/
{
project
.
slug
}
/repos'
,
headers
=
HEADERS
).
json
():
package
=
Package
(
name
=
data
[
'name'
],
project
=
project
,
homepage
=
data
[
'homepage'
])
package
.
save
()
repo
=
Repo
(
package
=
package
,
url
=
data
[
'html_url'
],
homepage
=
data
[
'homepage'
],
repo_id
=
data
[
'id
'
],
default_branch
=
data
[
'default_branch'
],
open_issues
=
data
[
'open_issues'
]
)
repo_
api_url
=
f
'
{
GITHUB_API
}
/repos/
{
project
.
slug
}
/
{
package
.
slug
}
'
repo_data
=
requests
.
get
(
repo_api_url
,
headers
=
HEADERS
).
json
(
)
package
=
Package
.
objects
.
create
(
name
=
data
[
'name'
],
project
=
project
,
homepage
=
data
[
'homepage'
])
repo
=
Repo
.
objects
.
create
(
package
=
package
,
url
=
data
[
'html_url'
],
homepage
=
data
[
'homepage'
],
repo_id
=
data
[
'id'
],
default_branch
=
data
[
'default_branch
'
],
open_issues
=
data
[
'open_issues'
]
,
source_type
=
SOURCES
.
github
,
api_url
=
GITHUB_API
,
token
=
GITHUB_TOKEN
)
repo_data
=
api_data
(
repo
)
if
'license'
in
repo_data
and
repo_data
[
'license'
]:
license_data
=
repo_data
[
'license'
]
license
,
_
=
License
.
objects
.
get_or_create
(
github_key
=
license_data
[
'key'
],
name
=
license_data
[
'name'
])
repo
.
license
=
license
package
.
license
=
license
package
.
save
()
repo
.
open_pr
=
len
(
requests
.
get
(
f
'
{
repo_api_url
}
/pulls'
,
headers
=
HEADERS
).
json
(
))
repo
.
open_pr
=
len
(
api_data
(
repo
,
'/pulls'
))
repo
.
save
()
class
Migration
(
migrations
.
Migration
):
...
...
dashboard/gepetto_packages/migrations/0003_gitlab.py
View file @
7931c27f
...
...
@@ -6,6 +6,8 @@ from django.db import migrations
import
requests
from
gepetto_packages.utils
import
SOURCES
GITLAB_API
=
'https://eur0c.laas.fr/api/v4'
def
gitlab
(
apps
,
schema_editor
):
...
...
@@ -14,7 +16,8 @@ def gitlab(apps, schema_editor):
for
data
in
requests
.
get
(
f
'
{
GITLAB_API
}
/projects'
,
verify
=
False
).
json
():
package_qs
=
Package
.
objects
.
filter
(
name
=
data
[
'name'
])
if
package_qs
.
exists
():
Repo
.
objects
.
create
(
package
=
package_qs
.
first
(),
url
=
data
[
'web_url'
],
repo_id
=
data
[
'id'
])
Repo
.
objects
.
create
(
package
=
package_qs
.
first
(),
url
=
data
[
'web_url'
],
repo_id
=
data
[
'id'
],
source_type
=
SOURCES
.
gitlab
,
api_url
=
GITLAB_API
)
class
Migration
(
migrations
.
Migration
):
...
...
dashboard/gepetto_packages/migrations/0004_redmine.py
View file @
7931c27f
...
...
@@ -2,18 +2,19 @@
# Generated by Django 1.11.7 on 2017-11-14 09:29
from
__future__
import
unicode_literals
import
os
from
django.conf
import
settings
from
django.db
import
migrations
import
requests
REDMINE_APIS
=
[
'https://redmine.laas.fr'
,
# 'https://git.openrobots.org',
from
gepetto_packages.utils
import
SOURCES
,
api_headers
,
api_data
REDMINES
=
[
(
'https://redmine.laas.fr'
,
os
.
getenv
(
'REDMINE_TOKEN'
,
''
)),
# ('https://git.openrobots.org', os.getenv('OPENROB_TOKEN', ''),
]
HEADERS
=
{
'X-Redmine-API-Key'
:
settings
.
REDMINE_TOKEN
,
}
PACKAGES
=
[
'openhrp3-hrp2'
,
'Pyrene Talos'
,
...
...
@@ -23,17 +24,19 @@ def redmine(apps, schema_editor):
Project
,
License
,
Package
,
Repo
=
(
apps
.
get_model
(
'gepetto_packages'
,
model
)
for
model
in
[
'Project'
,
'License'
,
'Package'
,
'Repo'
])
dummy_project
,
_
=
Project
.
objects
.
get_or_create
(
name
=
'dummy'
)
for
api
in
REDMINE_APIS
:
for
data
in
requests
.
get
(
f
'
{
api
}
/projects.json?limit=100'
,
headers
=
HEADERS
).
json
()[
'projects'
]:
for
api
,
token
in
REDMINES
:
headers
=
api_headers
(
source
=
SOURCES
.
redmine
,
token
=
token
)
for
data
in
requests
.
get
(
f
'
{
api
}
/projects.json?limit=100'
,
headers
=
headers
).
json
()[
'projects'
]:
if
data
[
'name'
]
in
PACKAGES
:
Package
.
objects
.
get_or_create
(
name
=
data
[
'identifier'
],
project
=
dummy_project
)
package_qs
=
Package
.
objects
.
filter
(
name
=
data
[
'identifier'
])
if
package_qs
.
exists
():
r
=
Repo
(
package
=
package_qs
.
first
(),
repo_id
=
data
[
'id'
],
open_pr
=
0
)
repo_data
=
requests
.
get
(
f
'
{
api
}
/projects/
{
r
.
repo_id
}
.json'
,
headers
=
HEADERS
).
json
()[
'project'
]
r
=
Repo
.
objects
.
create
(
package
=
package_qs
.
first
(),
repo_id
=
data
[
'id'
],
open_pr
=
0
,
source_type
=
SOURCES
.
redmine
,
api_url
=
api
,
token
=
token
)
repo_data
=
api_data
(
r
)[
'project'
]
r
.
homepage
=
repo_data
[
'homepage'
]
r
.
url
=
f
'
{
api
}
/projects/
{
r
.
package
.
name
}
'
issues_data
=
requests
.
get
(
f
'
{
api
}
/issues.json?project_id=
{
r
.
repo_id
}
&status_id=open'
,
headers
=
HEADERS
)
issues_data
=
requests
.
get
(
f
'
{
api
}
/issues.json?project_id=
{
r
.
repo_id
}
&status_id=open'
,
headers
=
headers
)
r
.
open_issues
=
issues_data
.
json
()[
'total_count'
]
r
.
save
()
if
r
.
homepage
and
not
r
.
package
.
homepage
:
...
...
dashboard/gepetto_packages/models.py
View file @
7931c27f
from
django.db
import
models
from
ndh.models
import
TimeStampedModel
,
NamedModel
from
ndh.utils
import
query_sum
from
ndh.models
import
NamedModel
,
TimeStampedModel
from
ndh.utils
import
enum_to_choices
,
query_sum
from
.utils
import
SOURCES
,
api_headers
,
api_url
class
Project
(
NamedModel
):
...
...
@@ -41,6 +43,9 @@ class Repo(TimeStampedModel):
open_issues
=
models
.
PositiveSmallIntegerField
(
blank
=
True
,
null
=
True
)
open_pr
=
models
.
PositiveSmallIntegerField
(
blank
=
True
,
null
=
True
)
repo_id
=
models
.
PositiveIntegerField
()
source_type
=
models
.
PositiveSmallIntegerField
(
choices
=
enum_to_choices
(
SOURCES
))
api_url
=
models
.
CharField
(
max_length
=
100
)
token
=
models
.
CharField
(
max_length
=
50
,
blank
=
True
,
null
=
True
)
class
Meta
:
ordering
=
(
'package'
,
'url'
)
...
...
dashboard/gepetto_packages/templatetags/gepetto_packages.py
View file @
7931c27f
...
...
@@ -3,6 +3,7 @@ from django.utils.safestring import mark_safe
register
=
template
.
Library
()
@
register
.
filter
def
domain
(
url
):
if
url
:
...
...
dashboard/gepetto_packages/tests.py
deleted
100644 → 0
View file @
c98e40b8
from
django.test
import
TestCase
# Create your tests here.
dashboard/gepetto_packages/utils.py
0 → 100644
View file @
7931c27f
from
enum
import
IntEnum
import
requests
SOURCES
=
IntEnum
(
'Source'
,
'github gitlab redmine robotpkg'
)
def
api_url
(
repo
):
if
repo
.
source_type
==
SOURCES
.
github
:
return
f
'
{
repo
.
api_url
}
/repos/
{
repo
.
package
.
project
.
slug
}
/
{
repo
.
package
.
slug
}
'
if
repo
.
source_type
==
SOURCES
.
redmine
:
return
f
'
{
repo
.
api_url
}
/projects/
{
repo
.
repo_id
}
.json'
def
api_headers
(
repo
=
None
,
source
=
None
,
token
=
None
):
if
repo
is
not
None
:
source
=
repo
.
source_type
token
=
repo
.
token
if
source
==
SOURCES
.
github
:
return
{
'Authorization'
:
f
'token
{
token
}
'
,
'Accept'
:
'application/vnd.github.drax-preview+json'
,
}
if
source
==
SOURCES
.
redmine
:
return
{
'X-Redmine-API-Key'
:
token
}
def
api_data
(
repo
,
url
=
''
):
return
requests
.
get
(
api_url
(
repo
)
+
url
,
headers
=
api_headers
(
repo
)).
json
()
dashboard/gepetto_packages/views.py
deleted
100644 → 0
View file @
c98e40b8
from
django.shortcuts
import
render
# Create your views here.
dashboard/gepkg/settings.py
View file @
7931c27f
...
...
@@ -50,11 +50,11 @@ WSGI_APPLICATION = f'{PROJECT}.wsgi.application'
DATABASES
=
{
'default'
:
{
'ENGINE'
:
'django.db.backends.postgresql'
,
'ENGINE'
:
'django.db.backends.
'
+
(
'
postgresql'
if
'POSTGRES_PASSWORD'
in
os
.
environ
else
'sqlite3'
)
,
'HOST'
:
'db'
,
'USER'
:
'postgres'
,
'NAME'
:
'postgres'
,
'PASSWORD'
:
os
.
environ
[
'POSTGRES_PASSWORD'
]
,
'NAME'
:
'postgres'
if
'POSTGRES_PASSWORD'
in
os
.
environ
else
os
.
path
.
join
(
BASE_DIR
,
'db.sqlite3'
)
,
'PASSWORD'
:
os
.
getenv
(
'POSTGRES_PASSWORD'
,
''
)
}
}
...
...
@@ -84,6 +84,4 @@ STATIC_ROOT = '/static/'
SITE_ID
=
1
REDMINE_TOKEN
=
os
.
environ
[
'REDMINE_TOKEN'
]
GITHUB_TOKEN
=
os
.
environ
[
'GITHUB_TOKEN'
]
SECRET_KEY
=
os
.
environ
[
'DJANGO_SECRET_KEY'
]
SECRET_KEY
=
os
.
getenv
(
'DJANGO_SECRET_KEY'
,
'placeholder'
)
dashboard/manage.py
View file @
7931c27f
...
...
@@ -11,7 +11,7 @@ if __name__ == "__main__":
# issue is really that Django is missing to avoid masking other
# exceptions on Python 2.
try
:
import
django
import
django
# noqa
except
ImportError
:
raise
ImportError
(
"Couldn't import Django. Are you sure it's installed and "
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment