create_update_eggnog.py 4.47 KB
Newer Older
1
2
3
import logging

from django.core.management.base import BaseCommand
Kenzo-Hugo Hillion's avatar
Kenzo-Hugo Hillion committed
4
from django.core.exceptions import ValidationError
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

from metagenedb.apps.catalog.models import EggNog, EggNogFunctionalCategory
from metagenedb.common.utils.chunks import file_len
from metagenedb.common.utils.parsers import EggNogAnnotationLineParser

logging.basicConfig(format='[%(asctime)s] %(levelname)s:%(name)s:%(message)s')
logger = logging.getLogger(__name__)


class ImportEggNog(object):

    def __init__(self, file_path):
        self.annotation_file = file_path
        self.eggnog_parser = EggNogAnnotationLineParser()
        self.processed_count = 0
        self.created_count = 0
        self.updated_count = 0
        self.skipped_count = 0
        self.skipped_ids = []
Kenzo-Hugo Hillion's avatar
Kenzo-Hugo Hillion committed
24
        self.skipped_errors = []
25
26
27

    def _build_functional_category_dict(self):
        all_categories = EggNogFunctionalCategory.objects.all()
28
29
        if not all_categories:
            raise Exception("You need to create Functional categories first.")
30
        self.functional_cat_instances = {cat.category_id: cat for cat in all_categories}
31

32
33
34
    def add_functional_categories(self, eggnog, functional_category_keys):
        for key in functional_category_keys:
            eggnog.functional_categories.add(self.functional_cat_instances[key])
35
36
37
38
39
40
41

    def load_all(self, test=False):
        self._build_functional_category_dict()
        self.total_eggnog_nb = file_len(self.annotation_file)
        with open(self.annotation_file, "r") as file:
            for line in file:
                eggnog_dict = self.eggnog_parser.get_dict(line)
42
                functional_category_keys = eggnog_dict.pop('functional_categories')  # link later
43
                payload = {k: v for k, v in eggnog_dict.items() if v != ""}
44
                eggnog = None
45
46
                try:
                    eggnog = EggNog(**payload)
Kenzo-Hugo Hillion's avatar
Kenzo-Hugo Hillion committed
47
                    eggnog.full_clean()
48
49
                    eggnog.save()
                    self.created_count += 1
Kenzo-Hugo Hillion's avatar
Kenzo-Hugo Hillion committed
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
                except ValidationError as validation_error:
                    if 'function_id' in validation_error.error_dict.keys():
                        try:
                            eggnog = EggNog.objects.get(function_id=payload.get('function_id'))
                            for k, v in payload.items():
                                setattr(eggnog, k, v)
                            eggnog.full_clean()
                            eggnog.save()
                            self.updated_count += 1
                        except ValidationError as validation_error:
                            logger.error(validation_error)
                            self.skipped_errors.append(validation_error)
                            self.skipped_ids.append(payload.get('function_id'))
                            self.skipped_count += 1
                    else:
                        logger.error(validation_error)
                        self.skipped_errors.append(validation_error)
67
68
                        self.skipped_ids.append(payload.get('function_id'))
                        self.skipped_count += 1
69
70
71
                if eggnog is not None:
                    self.add_functional_categories(eggnog, functional_category_keys)
                    eggnog.save()
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
                self.processed_count += 1
                if self.processed_count % 1000 == 0:
                    logger.info("%s/%s EggNog processed so far...", self.processed_count, self.total_eggnog_nb)
                    if test:
                        break
        logger.info("[DONE] %s/%s EggNog created.", self.created_count, self.total_eggnog_nb)
        logger.info("[DONE] %s/%s EggNog updated.", self.updated_count, self.total_eggnog_nb)
        logger.info("[DONE] %s/%s EggNog skipped. List: %s", self.skipped_count, self.total_eggnog_nb,
                    self.skipped_ids)


class Command(BaseCommand):
    help = 'Create or update all Eggnog entries from annotations.tsv file.'

    def add_arguments(self, parser):
        parser.add_argument('annotation', help='annotations.tsv file from EggNog')
        parser.add_argument('--test', action='store_true', help='Run only on first 1000 entries.')

    def set_logger_level(self, verbosity):
        if verbosity > 2:
            logger.setLevel(logging.DEBUG)
        elif verbosity > 1:
            logger.setLevel(logging.INFO)

    def handle(self, *args, **options):
        self.set_logger_level(int(options['verbosity']))
        import_eggnog = ImportEggNog(options['annotation'])
        import_eggnog.load_all(test=options['test'])