biomaj-migrate.py 7.97 KB
Newer Older
root's avatar
root committed
1
2
#!/usr/bin/python

Manu's avatar
Manu committed
3
4
import os
import sys
root's avatar
root committed
5
import argparse
Manu's avatar
Manu committed
6
7
import datetime
import time
root's avatar
root committed
8
9
10
11
12
13
14
15
16
17
18
import logging
import re
import fnmatch

import MySQLdb

from biomaj.bank import Bank
from biomaj.config import BiomajConfig
from biomaj.workflow import UpdateWorkflow, RemoveWorkflow, Workflow

def migrate_bank(cur, bank):
Manu's avatar
Manu committed
19
    cur.execute("SELECT path, session, creation, remove, size from productionDirectory WHERE ref_idbank="+str(bank['dbid']))
root's avatar
root committed
20
21
22
    bank['productionDirectory'] = []
    for row in cur.fetchall():
      if not row[3]:
Manu's avatar
Manu committed
23
24
25
26
27
28
          bank['productionDirectory'].append({
            'path': row[0],
            'session': row[1],
            'creation': row[2],
            'size': row[4],
          })
root's avatar
root committed
29
30
31
32
33
34
35
36
37
38
39
40
41
    for prod in bank['productionDirectory']:
      if prod['session']:
        cur.execute("SELECT ref_idupdateBank from session WHERE idsession="+str(prod['session']))
        for row in cur.fetchall():
          prod['ref_idupdateBank'] = row[0]
    for prod in bank['productionDirectory']:
      if 'ref_idupdateBank' in prod:
        cur.execute("SELECT updateRelease from updateBank where idupdateBank="+str(prod['ref_idupdateBank']))
        for row in cur.fetchall():
          prod['release'] = row[0]
          prod['remoterelease'] = row[0]
          pathelts = prod['path'].split('/')
          release_dir = pathelts[len(pathelts)-1]
42
          prod['prod_dir'] = release_dir
root's avatar
root committed
43
44
45
46
          pattern = re.compile(".*__(\d+)$")
          relmatch = pattern.match(release_dir)
          if relmatch:
            prod['release'] = prod['release']+'__'+relmatch.group(1)
Manu's avatar
Manu committed
47
48
49
50

        # We create the session id from the productionDirectory.creation field
        session_id = time.mktime(datetime.datetime.strptime(str(prod['creation']), "%Y-%m-%d %H:%M:%S").timetuple())
        session_exists = False
51
        b = Bank(bank['name'], no_log=True)
Manu's avatar
Manu committed
52
53
54
55
56
57
58
59
60

        # We check we did not already imported this session into the database
        for s in b.bank['sessions']:
          if s['id'] == session_id:
            logging.warn('Session already imported: ' + b.name + ':' + str(prod['creation']))
            session_exists = True
        if session_exists:
          continue

root's avatar
root committed
61
62
63
64
65
        for p in b.bank['production']:
          if p['release'] == prod['release']:
            logging.warn('Prod release already imported: '+b.name+":"+p['release'])
            continue
        b.load_session(UpdateWorkflow.FLOW)
66
        b.session.set('prod_dir', prod['prod_dir'])
root's avatar
root committed
67
68
69
        b.session.set('action','update')
        b.session.set('release', prod['release'])
        b.session.set('remoterelease', prod['remoterelease'])
Manu's avatar
Manu committed
70
71
        # Set production size from productionDirectory.size field
        b.session.set('fullsize', prod['size'])
root's avatar
root committed
72
73
        b.session._session['status'][Workflow.FLOW_OVER] = True
        b.session._session['update'] = True
Manu's avatar
Manu committed
74
75
        # We set the session.id (timestamp) with creation field fron productionDirectory table
        b.session.set('id', session_id)
root's avatar
root committed
76
        b.save_session()
Manu's avatar
Manu committed
77

78
79
80
81
82
83
84
85
86
87
88
89
90
91
        # Listing files ?
        root_files = []
        if os.path.exists(prod['path']):
          root_files = os.listdir(prod['path'])
        for root_file in root_files:
          if root_file.startswith('listing.'):
            fileName, fileExtension = os.path.splitext(root_file)
            f = open(os.path.join(prod['path'], root_file), 'r')
            listing = f.read()
            f.close()
            f = open(os.path.join(prod['path'], 'listingv1'+fileExtension), 'w')
            listing = "{\"files\": [], \"name\": \""+fileExtension.replace('.','')+"\"," +listing+"}"
            f.write(listing)
            f.close()
root's avatar
root committed
92
        # Current link?
93
        pathelts = prod['path'].split('/')
root's avatar
root committed
94
95
96
97
98
        del pathelts[-1]
        current_link = os.path.join('/'.join(pathelts),'current')
        if os.path.lexists(current_link):
          b.bank['current'] = b.session._session['id']
          b.banks.update({'name': b.name},
Manu's avatar
Manu committed
99
100
101
                         {
                           '$set': {'current': b.session._session['id']}
                         })
root's avatar
root committed
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

def main():

  parser = argparse.ArgumentParser()
  parser.add_argument('-c', '--config', dest="config",help="Biomaj3 Configuration file")
  parser.add_argument('-o', '--oldconfig', dest="oldconfig",help="Old configuration file")
  parser.add_argument('-u', '--user', dest="user", help="MySQL user to override global properties")
  parser.add_argument('-p', '--password', dest="password", help="MySQL password to override global properties")
  parser.add_argument('-l', '--host', dest="host", help="MySQL host to override global properties")
  parser.add_argument('-d', '--database', dest="database", help="MySQL database to override global properties")


  args = parser.parse_args()

  biomajconfig = {}
  banks = []
  with open(args.oldconfig,'r') as old:
    for line in old:
      vals = line.split('=')
      key = None
      if len(vals) > 1:
        biomajconfig[vals[0].strip()] = vals[1].strip()

  BiomajConfig.load_config(args.config, allow_user_config=False)
  db_properties_dir = os.path.dirname(args.oldconfig)
  if db_properties_dir == os.path.dirname(args.config):
    logging.error("Bank properties use the same directory, please use a different conf.dir")
    sys.exit(1)

  data_dir = biomajconfig['data.dir']
132
133
134
  if data_dir.endswith('/'):
    data_dir = data_dir[:-1]

root's avatar
root committed
135
136
137
138
139
140
141
142
143
144
145
146
147
  if not os.path.dirname(data_dir) == os.path.dirname(BiomajConfig.global_config.get('GENERAL','data.dir')):
    logging.error('Data dirs are different, please use the same data dirs')
    sys.exit(1)

  prop_files = []
  for root, dirnames, filenames in os.walk(db_properties_dir):
    for filename in fnmatch.filter(filenames, '*.properties'):
      if filename != 'global.properties':
        prop_files.append(os.path.join(root, filename))

  if not os.path.exists(BiomajConfig.global_config.get('GENERAL','conf.dir')):
    os.makedirs(BiomajConfig.global_config.get('GENERAL','conf.dir'))
  for prop_file in prop_files:
148
149
150
151
152
153
154
155

    propbankconfig = {}
    with open(prop_file,'r') as old:
        for line in old:
            vals = line.split('=')
            if len(vals) > 1:
                propbankconfig[vals[0].strip()] = vals[1].strip()

root's avatar
root committed
156
157
158
159
160
161
162
    newpropfile = os.path.join(BiomajConfig.global_config.get('GENERAL','conf.dir'),os.path.basename(prop_file))
    newprop = open(newpropfile,'w')
    #logging.warn("manage "+prop_file+" => "+newpropfile)
    newprop.write("[GENERAL]\n")
    with open(prop_file,'r') as props:
      for line in props:
        if not (line.startswith('*') or  line.startswith('/*')):
163
164
165
166
167
168
169
170
            # Replace config variables with new syntax ${xx} => %(xx)s, not other env variables
            pattern = re.compile("\$\{([a-zA-Z0-9-_.]+)\}")
            varmatch = pattern.findall(line)
            if varmatch:
                for match in varmatch:
                    if match in biomajconfig or match in propbankconfig:
                        line = line.replace('${'+match+'}','%('+match+')s')
            newprop.write(line.replace('\\\\','\\').replace('db.source','depends'))
root's avatar
root committed
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
    newprop.close()
    b = Bank(os.path.basename(prop_file).replace('.properties',''),no_log=True)
    banks.append(b.name)

    #database.url=jdbc\:mysql\://genobdd.genouest.org/biomaj_log
  vals = biomajconfig['database.url'].split('/')
  urllen = len(vals)
  db_name = vals[urllen-1]
  if args.database:
    db_name = args.database
  db_host = vals[urllen -2]
  if args.host:
    db_host = args.host
  db_user = biomajconfig['database.login']
  if args.user:
    db_user = args.user
  db_password = biomajconfig['database.password']
  if args.password:
    db_password = args.password
  db = MySQLdb.connect(host=db_host, # your host, usually localhost
                   user=db_user, # your username
                    passwd=db_password, # your password
                    db=db_name) # name of the data base
  cur = db.cursor()
  oldbanks = {}
  cur.execute("SELECT name,idbank from bank")
  for row in cur.fetchall():
    oldbanks[row[0]] = { "dbid": row[1], "name": row[0] }
  for bank,value in oldbanks.iteritems():
    migrate_bank(cur, value)

if __name__ == '__main__':
    main()