biomaj-migrate.py 5.78 KB
Newer Older
root's avatar
root committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
#!/usr/bin/python

import os,sys
import argparse
import pkg_resources
import logging
import re
import fnmatch

import MySQLdb

from biomaj.bank import Bank
from biomaj.config import BiomajConfig
from biomaj.workflow import UpdateWorkflow, RemoveWorkflow, Workflow

def migrate_bank(cur, bank):
    cur.execute("SELECT path, session, creation, remove from productionDirectory WHERE ref_idbank="+str(bank['dbid']))
    bank['productionDirectory'] = []
    for row in cur.fetchall():
      if not row[3]:
        bank['productionDirectory'].append({
          'session': row[1], 'path': row[0],
          'creation': row[2]
      })
    for prod in bank['productionDirectory']:
      if prod['session']:
        cur.execute("SELECT ref_idupdateBank from session WHERE idsession="+str(prod['session']))
        for row in cur.fetchall():
          prod['ref_idupdateBank'] = row[0]
    for prod in bank['productionDirectory']:
      if 'ref_idupdateBank' in prod:
        cur.execute("SELECT updateRelease from updateBank where idupdateBank="+str(prod['ref_idupdateBank']))
        for row in cur.fetchall():
          prod['release'] = row[0]
          prod['remoterelease'] = row[0]
          pathelts = prod['path'].split('/')
          release_dir = pathelts[len(pathelts)-1]
          pattern = re.compile(".*__(\d+)$")
          relmatch = pattern.match(release_dir)
          if relmatch:
            prod['release'] = prod['release']+'__'+relmatch.group(1)
        b = Bank(bank['name'])
        prod_present = False
        for p in b.bank['production']:
          if p['release'] == prod['release']:
            logging.warn('Prod release already imported: '+b.name+":"+p['release'])
            continue
        b.load_session(UpdateWorkflow.FLOW)
        b.session.set('action','update')
        b.session.set('release', prod['release'])
        b.session.set('remoterelease', prod['remoterelease'])
        b.session._session['status'][Workflow.FLOW_OVER] = True
        b.session._session['update'] = True
        b.save_session()
        # Current link?
        pathelts = prod['path'].split('/') 
        del pathelts[-1]
        current_link = os.path.join('/'.join(pathelts),'current')
        if os.path.lexists(current_link):
          b.bank['current'] = b.session._session['id']
          b.banks.update({'name': b.name},
                      {
                      '$set': {'current': b.session._session['id']}
                      })

def main():

  parser = argparse.ArgumentParser()
  parser.add_argument('-c', '--config', dest="config",help="Biomaj3 Configuration file")
  parser.add_argument('-o', '--oldconfig', dest="oldconfig",help="Old configuration file")
  parser.add_argument('-u', '--user', dest="user", help="MySQL user to override global properties")
  parser.add_argument('-p', '--password', dest="password", help="MySQL password to override global properties")
  parser.add_argument('-l', '--host', dest="host", help="MySQL host to override global properties")
  parser.add_argument('-d', '--database', dest="database", help="MySQL database to override global properties")


  args = parser.parse_args()

  biomajconfig = {}
  banks = []
  with open(args.oldconfig,'r') as old:
    for line in old:
      vals = line.split('=')
      key = None
      if len(vals) > 1:
        biomajconfig[vals[0].strip()] = vals[1].strip()

  BiomajConfig.load_config(args.config, allow_user_config=False)
  db_properties_dir = os.path.dirname(args.oldconfig)
  if db_properties_dir == os.path.dirname(args.config):
    logging.error("Bank properties use the same directory, please use a different conf.dir")
    sys.exit(1)

  data_dir = biomajconfig['data.dir']
95
96
97
  if data_dir.endswith('/'):
    data_dir = data_dir[:-1]

root's avatar
root committed
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
  if not os.path.dirname(data_dir) == os.path.dirname(BiomajConfig.global_config.get('GENERAL','data.dir')):
    logging.error('Data dirs are different, please use the same data dirs')
    sys.exit(1)

  prop_files = []
  for root, dirnames, filenames in os.walk(db_properties_dir):
    for filename in fnmatch.filter(filenames, '*.properties'):
      if filename != 'global.properties':
        prop_files.append(os.path.join(root, filename))

  if not os.path.exists(BiomajConfig.global_config.get('GENERAL','conf.dir')):
    os.makedirs(BiomajConfig.global_config.get('GENERAL','conf.dir'))
  for prop_file in prop_files:
    newpropfile = os.path.join(BiomajConfig.global_config.get('GENERAL','conf.dir'),os.path.basename(prop_file))
    newprop = open(newpropfile,'w')
    #logging.warn("manage "+prop_file+" => "+newpropfile)
    newprop.write("[GENERAL]\n")
    with open(prop_file,'r') as props:
      for line in props:
        if not (line.startswith('*') or  line.startswith('/*')):
          newprop.write(line.replace('\\\\','\\').replace('db.source','depends'))
    newprop.close()
    b = Bank(os.path.basename(prop_file).replace('.properties',''),no_log=True)
    banks.append(b.name)

    #database.url=jdbc\:mysql\://genobdd.genouest.org/biomaj_log
  vals = biomajconfig['database.url'].split('/')
  urllen = len(vals)
  db_name = vals[urllen-1]
  if args.database:
    db_name = args.database
  db_host = vals[urllen -2]
  if args.host:
    db_host = args.host
  db_user = biomajconfig['database.login']
  if args.user:
    db_user = args.user
  db_password = biomajconfig['database.password']
  if args.password:
    db_password = args.password
  db = MySQLdb.connect(host=db_host, # your host, usually localhost
                   user=db_user, # your username
                    passwd=db_password, # your password
                    db=db_name) # name of the data base
  cur = db.cursor()
  oldbanks = {}
  cur.execute("SELECT name,idbank from bank")
  for row in cur.fetchall():
    oldbanks[row[0]] = { "dbid": row[1], "name": row[0] }
  for bank,value in oldbanks.iteritems():
    migrate_bank(cur, value)

if __name__ == '__main__':
    main()