Commit 4722801f authored by root's avatar root

ignore global.properties

parent 26fc1454
global.properties
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
......
# Migration
## Setup
Make sure Biomaj is installed and running
A "biomaj-cli.py --status" should work.
Make sure that data.dir is the same for old and new install.
New version is fully compatible with the old release directory management.
Running twice the migration script will not overwrite existing values.
## One same server
Use a different properties location, migration script will create new property files based on old ones.
## On different server
Copy old properties to the new server (/etc/biomaj/db_properties)
data.dir should be mounted on the new server.
If a new data.dir is expected, then do not run any migration and start from a scratch new biomaj install.
## Process scripts
Process script are compatible, and can be kept on previous location, simply do not forget to set its location in new configuration file.
## Running migration
Migration script will copy properties files to the new conf.dir, modifying old scripts on-the-fly to the new syntax.
It will import all existing productio directory info in the new database.
Production directories are not modified nor copied/moved.
python bin/biomaj-migrate.py --config global.properties --oldconfig ../oldbiomaj/biomaj/db_properties/global.properties
OR specifying db parameters
python bin/biomaj-migrate.py --config global.properties --oldconfig ../oldbiomaj/biomaj/db_properties/global.properties --user biomaj --password 'XXX' --host 'YYY' --database biomaj_log
## Checks
"biomaj-cli --status" should show the old databases.
# Requirements
mysql-devel
#!/usr/bin/python
import os,sys
import argparse
import pkg_resources
import logging
import re
import fnmatch
import MySQLdb
from biomaj.bank import Bank
from biomaj.config import BiomajConfig
from biomaj.workflow import UpdateWorkflow, RemoveWorkflow, Workflow
def migrate_bank(cur, bank):
cur.execute("SELECT path, session, creation, remove from productionDirectory WHERE ref_idbank="+str(bank['dbid']))
bank['productionDirectory'] = []
for row in cur.fetchall():
if not row[3]:
bank['productionDirectory'].append({
'session': row[1], 'path': row[0],
'creation': row[2]
})
for prod in bank['productionDirectory']:
if prod['session']:
cur.execute("SELECT ref_idupdateBank from session WHERE idsession="+str(prod['session']))
for row in cur.fetchall():
prod['ref_idupdateBank'] = row[0]
for prod in bank['productionDirectory']:
if 'ref_idupdateBank' in prod:
cur.execute("SELECT updateRelease from updateBank where idupdateBank="+str(prod['ref_idupdateBank']))
for row in cur.fetchall():
prod['release'] = row[0]
prod['remoterelease'] = row[0]
pathelts = prod['path'].split('/')
release_dir = pathelts[len(pathelts)-1]
pattern = re.compile(".*__(\d+)$")
relmatch = pattern.match(release_dir)
if relmatch:
prod['release'] = prod['release']+'__'+relmatch.group(1)
b = Bank(bank['name'])
prod_present = False
for p in b.bank['production']:
if p['release'] == prod['release']:
logging.warn('Prod release already imported: '+b.name+":"+p['release'])
continue
b.load_session(UpdateWorkflow.FLOW)
b.session.set('action','update')
b.session.set('release', prod['release'])
b.session.set('remoterelease', prod['remoterelease'])
b.session._session['status'][Workflow.FLOW_OVER] = True
b.session._session['update'] = True
b.save_session()
# Current link?
pathelts = prod['path'].split('/')
del pathelts[-1]
current_link = os.path.join('/'.join(pathelts),'current')
if os.path.lexists(current_link):
b.bank['current'] = b.session._session['id']
b.banks.update({'name': b.name},
{
'$set': {'current': b.session._session['id']}
})
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-c', '--config', dest="config",help="Biomaj3 Configuration file")
parser.add_argument('-o', '--oldconfig', dest="oldconfig",help="Old configuration file")
parser.add_argument('-u', '--user', dest="user", help="MySQL user to override global properties")
parser.add_argument('-p', '--password', dest="password", help="MySQL password to override global properties")
parser.add_argument('-l', '--host', dest="host", help="MySQL host to override global properties")
parser.add_argument('-d', '--database', dest="database", help="MySQL database to override global properties")
args = parser.parse_args()
biomajconfig = {}
banks = []
with open(args.oldconfig,'r') as old:
for line in old:
vals = line.split('=')
key = None
if len(vals) > 1:
biomajconfig[vals[0].strip()] = vals[1].strip()
BiomajConfig.load_config(args.config, allow_user_config=False)
db_properties_dir = os.path.dirname(args.oldconfig)
if db_properties_dir == os.path.dirname(args.config):
logging.error("Bank properties use the same directory, please use a different conf.dir")
sys.exit(1)
data_dir = biomajconfig['data.dir']
if not os.path.dirname(data_dir) == os.path.dirname(BiomajConfig.global_config.get('GENERAL','data.dir')):
logging.error('Data dirs are different, please use the same data dirs')
sys.exit(1)
prop_files = []
for root, dirnames, filenames in os.walk(db_properties_dir):
for filename in fnmatch.filter(filenames, '*.properties'):
if filename != 'global.properties':
prop_files.append(os.path.join(root, filename))
if not os.path.exists(BiomajConfig.global_config.get('GENERAL','conf.dir')):
os.makedirs(BiomajConfig.global_config.get('GENERAL','conf.dir'))
for prop_file in prop_files:
newpropfile = os.path.join(BiomajConfig.global_config.get('GENERAL','conf.dir'),os.path.basename(prop_file))
newprop = open(newpropfile,'w')
#logging.warn("manage "+prop_file+" => "+newpropfile)
newprop.write("[GENERAL]\n")
with open(prop_file,'r') as props:
for line in props:
if not (line.startswith('*') or line.startswith('/*')):
newprop.write(line.replace('\\\\','\\').replace('db.source','depends'))
newprop.close()
b = Bank(os.path.basename(prop_file).replace('.properties',''),no_log=True)
banks.append(b.name)
#database.url=jdbc\:mysql\://genobdd.genouest.org/biomaj_log
vals = biomajconfig['database.url'].split('/')
urllen = len(vals)
db_name = vals[urllen-1]
if args.database:
db_name = args.database
db_host = vals[urllen -2]
if args.host:
db_host = args.host
db_user = biomajconfig['database.login']
if args.user:
db_user = args.user
db_password = biomajconfig['database.password']
if args.password:
db_password = args.password
db = MySQLdb.connect(host=db_host, # your host, usually localhost
user=db_user, # your username
passwd=db_password, # your password
db=db_name) # name of the data base
cur = db.cursor()
oldbanks = {}
cur.execute("SELECT name,idbank from bank")
for row in cur.fetchall():
oldbanks[row[0]] = { "dbid": row[1], "name": row[0] }
for bank,value in oldbanks.iteritems():
migrate_bank(cur, value)
if __name__ == '__main__':
main()
try:
from setuptools import setup, find_packages
except ImportError:
from distutils.core import setup
config = {
'description': 'BioMAJ Migration tool',
'author': 'Olivier Sallou',
'url': 'http://biomaj.genouest.org',
'download_url': 'http://biomaj.genouest.org',
'author_email': 'olivier.sallou@irisa.fr',
'version': '3.0.0',
'install_requires': ['nose',
'MySQL-python',
'pymongo'],
'packages': find_packages(),
'include_package_data': True,
'scripts': ['bin/biomaj-migrate.py'],
'name': 'biomajmigrate'
}
setup(**config)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment