Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Metagenomics
metagenedb
Commits
9ea066fd
Commit
9ea066fd
authored
Aug 05, 2019
by
Kenzo-Hugo Hillion
♻
Browse files
cleaning up backend and rm useless code
parent
0ee67a41
Pipeline
#13531
passed with stages
in 1 minute and 51 seconds
Changes
19
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
backend/metagenedb/api/accounts/tests.py
View file @
9ea066fd
...
...
@@ -12,7 +12,7 @@ class TestAccounts(APITestCase):
def
test_obtain_jwt
(
self
):
# create an inactive user
url
=
reverse
(
'auth:api-jwt-auth'
)
url
=
reverse
(
'
api:
auth:api-jwt-auth'
)
u
=
User
.
objects
.
create_user
(
username
=
'user'
,
email
=
'user@foo.com'
,
password
=
'pass'
)
u
.
is_active
=
False
u
.
save
()
...
...
@@ -39,7 +39,7 @@ class TestAccounts(APITestCase):
payload
=
jwt_payload_handler
(
user
)
token
=
jwt_encode_handler
(
payload
)
verify_url
=
reverse
(
'auth:api-jwt-verify'
)
verify_url
=
reverse
(
'
api:
auth:api-jwt-verify'
)
credentials
=
{
'token'
:
token
}
...
...
backend/metagenedb/api/catalog/urls.py
View file @
9ea066fd
from
django.urls
import
re_path
from
django.conf.urls
import
url
,
include
from
rest_framework.routers
import
DefaultRouter
from
metagenedb.api.catalog.views
import
GeneViewSet
,
gene_length
from
metagenedb.api.catalog.views
import
GeneViewSet
api_router
=
DefaultRouter
()
...
...
@@ -11,6 +9,5 @@ api_router.register(r'genes', GeneViewSet, base_name='genes')
urlpatterns
=
[
re_path
(
r
'^gene_length$'
,
gene_length
,
name
=
'gene_length'
),
url
(
r
'v1/'
,
include
((
api_router
.
urls
,
'v1'
),
namespace
=
'v1'
))
url
(
r
'v1/'
,
include
((
api_router
.
urls
,
'v1'
)))
]
backend/metagenedb/api/catalog/views/__init__.py
View file @
9ea066fd
from
.gene
import
GeneViewSet
# noqa
from
.statistics
import
gene_length
# noqa
__all__
=
[
'GeneViewSet'
]
backend/metagenedb/api/catalog/views/gene.py
View file @
9ea066fd
import
pandas
as
pd
from
rest_framework.viewsets
import
GenericViewSet
from
rest_framework
import
mixins
from
rest_framework.decorators
import
action
from
rest_framework.response
import
Response
from
metagenedb.common.utils.df_operations
import
get_mask
from
metagenedb.apps.catalog.models
import
Gene
from
metagenedb.apps.catalog.serializers
import
GeneSerializer
...
...
@@ -8,9 +12,34 @@ from metagenedb.apps.catalog.serializers import GeneSerializer
class
GeneViewSet
(
mixins
.
ListModelMixin
,
mixins
.
RetrieveModelMixin
,
GenericViewSet
):
"""
The following endpoints are fully provided by mixins:
* List gene
"""
queryset
=
Gene
.
objects
.
all
()
serializer_class
=
GeneSerializer
GENE_LENGTH_COL
=
'gene_length'
def
_count_windows
(
self
,
df
,
window_size
=
10000
,
window_col
=
GENE_LENGTH_COL
):
"""
Count how many line of the df belong to each windows defined by the window_size for the window_col
:param df:
:param window_col: column concerned by the window
:param window_size: size of the window
:return: {'data': COUNTS_BY_WINDOW, 'labels': START-END}
"""
all_ranges
=
[(
i
,
i
+
window_size
)
for
i
in
range
(
0
,
df
[
window_col
].
max
(),
window_size
)]
data
=
[]
labels
=
[]
for
rg
in
all_ranges
:
labels
.
append
(
f
"
{
rg
[
0
]
}
-
{
rg
[
1
]
-
1
}
"
)
data
.
append
(
df
[
get_mask
(
df
,
rg
,
window_col
)].
count
()[
'gene_length'
])
return
{
'counts'
:
data
,
'labels'
:
labels
}
@
action
(
methods
=
[
'get'
],
detail
=
False
)
def
gene_length
(
self
,
request
,
window_size
=
10000
):
if
'window_size'
in
request
.
query_params
:
window_size
=
int
(
request
.
query_params
.
get
(
'window_size'
))
df
=
pd
.
DataFrame
(
list
(
self
.
queryset
.
values
(
self
.
GENE_LENGTH_COL
)))
return
Response
(
{
'results'
:
self
.
_count_windows
(
df
,
window_size
)}
)
backend/metagenedb/api/catalog/views/statistics.py
deleted
100644 → 0
View file @
0ee67a41
import
pandas
as
pd
from
rest_framework.decorators
import
api_view
from
rest_framework.response
import
Response
from
metagenedb.apps.catalog.models
import
Gene
GENE_LENGTH_COL
=
'gene_length'
def
_get_mask
(
df
,
rg
,
col_name
):
"""
rg is a range, e.g. (10-20)
"""
return
(
df
[
col_name
]
>=
rg
[
0
])
&
(
df
[
col_name
]
<
rg
[
1
])
def
_count_windows
(
df
,
window_col
,
window_size
=
10000
):
"""
Count how many line of the df belong to each windows defined by the window_size for the window_col
:param df:
:param window_col: column concerned by the window
:param window_size: size of the window
:return: {'data': COUNTS_BY_WINDOW, 'labels': START-END}
"""
all_ranges
=
[(
i
,
i
+
window_size
)
for
i
in
range
(
0
,
df
[
window_col
].
max
(),
window_size
)]
data
=
[]
labels
=
[]
for
rg
in
all_ranges
:
labels
.
append
(
f
"
{
rg
[
0
]
}
-
{
rg
[
1
]
-
1
}
"
)
data
.
append
(
df
[
_get_mask
(
df
,
rg
,
window_col
)].
count
()[
'gene_length'
])
return
{
'counts'
:
data
,
'labels'
:
labels
}
@
api_view
([
'GET'
])
def
gene_length
(
request
,
window_size
=
10000
):
if
'window_size'
in
request
.
query_params
:
window_size
=
int
(
request
.
query_params
.
get
(
'window_size'
))
df
=
pd
.
DataFrame
(
list
(
Gene
.
objects
.
all
().
values
(
GENE_LENGTH_COL
)))
return
Response
({
'data'
:
_count_windows
(
df
,
GENE_LENGTH_COL
,
window_size
)
})
backend/metagenedb/api/catalog/views/test_gene.py
View file @
9ea066fd
...
...
@@ -2,19 +2,21 @@ from django.contrib.auth.models import User
from
django.test
import
TestCase
from
django.urls
import
reverse
import
pandas
as
pd
from
rest_framework
import
status
from
rest_framework_jwt.settings
import
api_settings
from
metagenedb.api.catalog.views.gene
import
GeneViewSet
class
TestGenes
(
TestCase
):
"""Post Tests"""
def
test_get_genes_no_auth
(
self
):
"""
Unauthenticated users should be able to access genes
@TODO make unaccessible
"""
url
=
reverse
(
'catalog:v1:genes-list'
)
url
=
reverse
(
'
api:
catalog:v1:genes-list'
)
resp
=
self
.
client
.
get
(
url
)
self
.
assertEqual
(
resp
.
status_code
,
status
.
HTTP_200_OK
)
...
...
@@ -32,6 +34,25 @@ class TestGenes(TestCase):
payload
=
jwt_payload_handler
(
user
)
token
=
jwt_encode_handler
(
payload
)
url
=
reverse
(
'catalog:v1:genes-list'
)
url
=
reverse
(
'
api:
catalog:v1:genes-list'
)
resp
=
self
.
client
.
get
(
url
,
format
=
'json'
,
HTTP_AUTHORIZATION
=
f
"JWT
{
token
}
"
)
self
.
assertEqual
(
resp
.
status_code
,
status
.
HTTP_200_OK
)
class
TestCountWindows
(
TestCase
):
def
setUp
(
self
):
self
.
window_col
=
"gene_length"
self
.
df
=
pd
.
DataFrame
(
[
22
,
29
,
35
],
columns
=
[
self
.
window_col
]
)
def
test_simple_count_window10
(
self
):
expected_dict
=
{
'labels'
:
[
'0-9'
,
'10-19'
,
'20-29'
,
'30-39'
],
'counts'
:
[
0
,
0
,
2
,
1
]
}
geneviewset
=
GeneViewSet
()
test_dict
=
geneviewset
.
_count_windows
(
self
.
df
,
10
,
window_col
=
self
.
window_col
)
self
.
assertDictEqual
(
test_dict
,
expected_dict
)
backend/metagenedb/api/catalog/views/test_statistics.py
deleted
100644 → 0
View file @
0ee67a41
import
pandas
as
pd
from
unittest
import
TestCase
from
.statistics
import
_count_windows
class
TestCountWindows
(
TestCase
):
def
setUp
(
self
):
self
.
window_col
=
"gene_length"
self
.
df
=
pd
.
DataFrame
(
[
22
,
29
,
35
],
columns
=
[
self
.
window_col
]
)
def
test_simple_count_window10
(
self
):
expected_dict
=
{
'labels'
:
[
'0-9'
,
'10-19'
,
'20-29'
,
'30-39'
],
'counts'
:
[
0
,
0
,
2
,
1
]
}
test_dict
=
_count_windows
(
self
.
df
,
self
.
window_col
,
10
)
self
.
assertDictEqual
(
test_dict
,
expected_dict
)
backend/metagenedb/api/urls.py
View file @
9ea066fd
...
...
@@ -2,6 +2,6 @@ from django.urls import include, path
urlpatterns
=
[
path
(
'auth/'
,
include
((
'metagenedb.api.accounts.urls'
,
'auth'
)
,
namespace
=
'auth'
)),
path
(
'catalog/'
,
include
((
'metagenedb.api.catalog.urls'
,
'catalog'
)
,
namespace
=
'catalog'
))
path
(
'auth/'
,
include
((
'metagenedb.api.accounts.urls'
,
'auth'
))),
path
(
'catalog/'
,
include
((
'metagenedb.api.catalog.urls'
,
'catalog'
)))
]
backend/metagenedb/utils/__init__.py
→
backend/metagenedb/
common/
utils/__init__.py
View file @
9ea066fd
File moved
backend/metagenedb/common/utils/df_operations.py
0 → 100644
View file @
9ea066fd
def
get_mask
(
df
,
range
,
col_name
):
"""
Get mask for a dataframe for a given range on a column defined by col_name
Lower is included but not upper:
e.g (4,8): 4, 5, 6 and 7 are selected
"""
return
(
df
[
col_name
]
>=
range
[
0
])
&
(
df
[
col_name
]
<
range
[
1
])
backend/metagenedb/utils/parsers.py
→
backend/metagenedb/
common/
utils/parsers.py
View file @
9ea066fd
File moved
backend/metagenedb/common/utils/test_df_operations.py
0 → 100644
View file @
9ea066fd
from
unittest
import
TestCase
import
pandas
as
pd
from
pandas.util.testing
import
assert_series_equal
from
metagenedb.common.utils.df_operations
import
get_mask
class
TestGetMask
(
TestCase
):
def
setUp
(
self
):
self
.
df
=
pd
.
DataFrame
(
[
[
'breizh'
,
6
],
[
'loquemo'
,
7
],
[
'sea'
,
3
],
],
columns
=
[
'name'
,
'length'
]
)
def
test_get_mask
(
self
):
range
=
(
5
,
10
)
expected_series
=
pd
.
Series
(
[
True
,
True
,
False
],
name
=
'length'
)
test_mask
=
get_mask
(
self
.
df
,
range
,
'length'
)
assert_series_equal
(
test_mask
,
expected_series
)
def
test_get_mask_lower_equal
(
self
):
range
=
(
3
,
5
)
expected_series
=
pd
.
Series
(
[
False
,
False
,
True
],
name
=
'length'
)
test_mask
=
get_mask
(
self
.
df
,
range
,
'length'
)
assert_series_equal
(
test_mask
,
expected_series
)
def
test_get_mask_upper_equal
(
self
):
range
=
(
4
,
6
)
expected_series
=
pd
.
Series
(
[
False
,
False
,
False
],
name
=
'length'
)
test_mask
=
get_mask
(
self
.
df
,
range
,
'length'
)
assert_series_equal
(
test_mask
,
expected_series
)
backend/metagenedb/utils/test_parsers.py
→
backend/metagenedb/
common/
utils/test_parsers.py
View file @
9ea066fd
from
unittest
import
TestCase
from
metagenedb.utils.parsers
import
KEGGLineParser
,
NCBITaxonomyLineParser
from
metagenedb.
common.
utils.parsers
import
KEGGLineParser
,
NCBITaxonomyLineParser
class
TestKEGGLineParser
(
TestCase
):
...
...
backend/metagenedb/urls.py
View file @
9ea066fd
...
...
@@ -18,6 +18,6 @@ from django.urls import include, path
urlpatterns
=
[
path
(
'api/'
,
include
(
'metagenedb.api.urls'
)),
path
(
'api/'
,
include
(
(
'metagenedb.api.urls'
,
'api'
)
)),
path
(
'admin/'
,
admin
.
site
.
urls
),
]
backend/metagenedb/utils/dict_operations.py
deleted
100644 → 0
View file @
0ee67a41
import
logging
logging
.
basicConfig
(
level
=
logging
.
INFO
)
_LOGGER
=
logging
.
getLogger
(
__name__
)
def
extract_dict
(
source_dict
,
keys
,
keep_original_source
=
False
):
"""
Extract a dict from a given dict based on a given set of keys
"""
extracted_dict
=
{}
for
key
in
keys
:
try
:
extracted_dict
[
key
]
=
source_dict
[
key
]
if
not
keep_original_source
:
del
source_dict
[
key
]
except
KeyError
:
_LOGGER
.
warning
(
f
"[
{
key
}
] is not found in the source dict, extraction skipped for this key."
)
return
extracted_dict
backend/metagenedb/utils/test_dict_operations.py
deleted
100644 → 0
View file @
0ee67a41
from
unittest
import
TestCase
from
metagenedb.utils.dict_operations
import
extract_dict
class
TestExtractDict
(
TestCase
):
def
test_extract_dict
(
self
):
source_dict
=
{
'a'
:
1
,
'b'
:
2
}
extract_keys
=
[
'b'
]
expected_source_dict
=
{
'a'
:
1
}
expected_extract_dict
=
{
'b'
:
2
}
test_extract_dict
=
extract_dict
(
source_dict
,
extract_keys
)
self
.
assertDictEqual
(
source_dict
,
expected_source_dict
)
self
.
assertDictEqual
(
test_extract_dict
,
expected_extract_dict
)
def
test_extract_dict_wrong_key
(
self
):
source_dict
=
{
'a'
:
1
,
'b'
:
2
}
extract_keys
=
[
'c'
]
expected_source_dict
=
{
'a'
:
1
,
'b'
:
2
}
expected_extract_dict
=
{}
test_extract_dict
=
extract_dict
(
source_dict
,
extract_keys
)
self
.
assertDictEqual
(
source_dict
,
expected_source_dict
)
self
.
assertDictEqual
(
test_extract_dict
,
expected_extract_dict
)
def
test_extract_dict_original_source
(
self
):
source_dict
=
{
'a'
:
1
,
'b'
:
2
}
extract_keys
=
[
'b'
]
expected_source_dict
=
{
'a'
:
1
,
'b'
:
2
}
expected_extract_dict
=
{
'b'
:
2
}
test_extract_dict
=
extract_dict
(
source_dict
,
extract_keys
,
keep_original_source
=
True
)
self
.
assertDictEqual
(
source_dict
,
expected_source_dict
)
self
.
assertDictEqual
(
test_extract_dict
,
expected_extract_dict
)
backend/scripts/populate_db/import_ncbi_taxonomy.py
View file @
9ea066fd
...
...
@@ -6,7 +6,7 @@ import sys
import
django
from
metagenedb.utils.parsers
import
NCBITaxonomyLineParser
from
metagenedb.
common.
utils.parsers
import
NCBITaxonomyLineParser
# Before model import, we need to called django.setup() to Load apps
os
.
environ
.
setdefault
(
"DJANGO_SETTINGS_MODULE"
,
"metagenedb.settings"
)
...
...
backend/scripts/populate_db/load_kegg_ko.py
View file @
9ea066fd
...
...
@@ -8,7 +8,7 @@ import sys
import
django
from
django.core.exceptions
import
ValidationError
from
metagenedb.utils.parsers
import
KEGGLineParser
from
metagenedb.
common.
utils.parsers
import
KEGGLineParser
# Before model import, we need to called django.setup() to Load apps
os
.
environ
.
setdefault
(
"DJANGO_SETTINGS_MODULE"
,
"metagenedb.settings"
)
...
...
frontend/src/views/Stats.vue
View file @
9ea066fd
...
...
@@ -42,7 +42,7 @@ export default {
},
methods
:
{
fetchGeneLength
(
geneLengthWindowSize
)
{
fetch
(
'
/api/catalog/gene_length?window_size=
'
.
concat
(
geneLengthWindowSize
),
{
fetch
(
'
/api/catalog/
v1/genes/
gene_length?window_size=
'
.
concat
(
geneLengthWindowSize
),
{
method
:
'
GET
'
,
headers
:
{
Accept
:
'
application/json
'
,
...
...
@@ -51,7 +51,7 @@ export default {
.
then
((
response
)
=>
{
if
(
response
.
ok
)
{
response
.
json
().
then
((
json
)
=>
{
this
.
geneLengthData
=
json
.
data
;
this
.
geneLengthData
=
json
.
results
;
});
}
});
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment