Skip to content

GitLab

Commit eff57768 authored by Fabrice ALLAIN's avatar Fabrice ALLAIN
Browse files

fix: solve several format issues in the new logging system

parent 223c58d6
Expand all Hide whitespace changes
Inline Side-by-side
aria/aria2.py
View file @ eff57768
......@@ -29,10 +29,11 @@
"""
from __future__ import absolute_import, division, print_function
from future.builtins import input
from aria.core.ariabase import get_aria_root, ARIA_ENV
from aria.core.ariabase import get_aria_root, ARIA_ENV, LOG_CONF
import os
import sys
import logging
# ARIA_ENV = 'ARIA2'
......@@ -349,6 +350,8 @@ Conversion:
actual conversion.
'''
LOG = logging.getLogger(__name__)
def print_sequence(data):
"""
......@@ -517,9 +520,8 @@ def welcome():
"""
import aria.core.ariabase as ariabase
message = \
"""
ARIA Version %s. Authors: Benjamin Bardiaux, Michael Habeck, Jens Linge,
message = """\
ARIA Version {0}. Authors: Benjamin Bardiaux, Michael Habeck, Jens Linge,
Therese Malliavin, Sean O'Donoghue, Wolfgang Rieping, and Michael Nilges.
If you use this software, please quote the following reference(s):
......@@ -527,8 +529,8 @@ If you use this software, please quote the following reference(s):
Rieping W., Habeck M., Bardiaux B., Bernard A., Malliavin T.E.,
Nilges M.(2007) ARIA2: automated NOE assignment and data integration in NMR
structure calculation. Bioinformatics 23:381-382
"""
print(message % ariabase.AriaBaseClass().get_version_string())
""".format(ariabase.AriaBaseClass().get_version_string())
return message
def check_python():
......@@ -1011,13 +1013,16 @@ def run_aria(project_file, log_file=None, debug=0, test_commands=1,
try:
log_file = os.path.expanduser(log_file)
f = open(log_file, 'w')
# f = open(log_file, 'w')
except Exception as msg:
print(msg)
print('Could not create log-file: %s', log_file)
f = None
AriaBaseClass.log_file = f
LOG.exception(msg)
# print(msg)
# print('Could not create log-file: %s', log_file)
# f = None
# AriaBaseClass.log_file = f
# Update logging out files
LOG_CONF.up_outfiles(log_file)
if debug:
from aria.core.TypeChecking import check_type
......
aria/conbox/analysis.py
View file @ eff57768
......@@ -191,6 +191,7 @@ class EnsembleAnalysis(object):
dref = None
if dists_ref:
# TODO: prendre en compte un gap dans les matrices
try:
dref = [dists_ref(*sp.getAtoms()) for sp in contrib]
dref = np.power(np.sum(np.power(dref, -6.), axis=0),
......
aria/conbox/commands.py
View file @ eff57768
......@@ -156,14 +156,7 @@ class CLI(object):
"""
parser = argp.ArgumentParser(
formatter_class=argp.ArgumentDefaultsHelpFormatter)
parser.add_argument("-o", "--output", dest="output_directory", type=str,
help="Output directory", required=True,
action=ReadableDir)
parser.add_argument("--nolog", action="store_true",
default=False, help="Don't generate log files")
parser.add_argument("-d", "--debug", dest="verbose", default=False,
action='store_true',
help="Increase output verbosity")
return parser
# TODO: should be called in child classes .....
......@@ -252,9 +245,17 @@ class AriaEcCommands(CLI):
Update default CLI in order to add the confi
"""
parser = super(AriaEcCommands, self)._create_argparser()
parser.add_argument("-c", "--conf", action=ReadableFile,
dest="conf_file",
default=None, help="configuration file")
# parser.add_argument("-c", "--conf", action=ReadableFile,
# dest="conf_file",
# default=None, help="configuration file")
# parser.add_argument("-o", "--output", dest="output_directory", type=str,
# help="Output directory", required=True,
# action=ReadableDir)
# parser.add_argument("--nolog", action="store_true",
# default=False, help="Don't generate log files")
# parser.add_argument("-d", "--debug", dest="verbose", default=False,
# action='store_true',
# help="Increase output verbosity")
# TODO: not really practical, we HAVE to call line below since every
# subcommand argparser have to be defined in the same scope
self.add_subparsers(parser.add_subparsers(dest="command"))
......@@ -280,13 +281,24 @@ class AriaEcCommands(CLI):
# Args
group = parser.add_argument_group('required arguments')
group.add_argument("seq", action=ReadableFile,
help="sequence file [FASTA]")
help="Sequence file [FASTA]")
# group.add_argument("sspred", action=ReadableFile,
# help="secondary structure prediction file")
group.add_argument("infiles", nargs="*", metavar="infile",
action=ReadableFile,
help="contact or pdb file(s) used to build aria "
help="Contact or PDB file(s) used to build ARIA "
"distance restraints")
parser.add_argument("-c", "--conf", action=ReadableFile,
dest="conf_file",
default=None, help="Configuration file")
parser.add_argument("-o", "--output", dest="output_directory", type=str,
help="Output directory", required=True,
action=ReadableDir)
parser.add_argument("--nolog", action="store_true",
default=False, help="Don't generate log files")
parser.add_argument("-d", "--debug", dest="verbose", default=False,
action='store_true',
help="Increase output verbosity")
group.add_argument("-d", "--distfile", dest="distfile",
help="Pdb or distance matrix iif distance_type "
"set to distfile in conf file, "
......@@ -299,7 +311,7 @@ class AriaEcCommands(CLI):
action=ReadableFile,
help="ARIA project file used to initialize a new "
"project with contact map data.")
group.add_argument("-t", "--type",
group.add_argument("-t", "--type", required=True,
nargs="*", dest="contact_types",
choices=self.contact_types, help="Infile(s) contact "
"type(s)")
......@@ -349,6 +361,17 @@ class AriaEcCommands(CLI):
parser.add_argument("-t", "--type", required=True, dest="contact_type",
choices=self.contact_types, help="Infile contact "
"type")
parser.add_argument("-c", "--conf", action=ReadableFile,
dest="conf_file",
default=None, help="configuration file")
parser.add_argument("-o", "--output", dest="output_directory", type=str,
help="Output directory", required=True,
action=ReadableDir)
parser.add_argument("--nolog", action="store_true",
default=False, help="Don't generate log files")
parser.add_argument("-d", "--debug", dest="verbose", default=False,
action='store_true',
help="Increase output verbosity")
return parser
def _iniconv_argparser(self, desc=None):
......@@ -370,6 +393,17 @@ class AriaEcCommands(CLI):
# args
parser.add_argument("confiles", nargs='+',
type=str, help="config files")
parser.add_argument("-c", "--conf", action=ReadableFile,
dest="conf_file",
default=None, help="configuration file")
parser.add_argument("-o", "--output", dest="output_directory", type=str,
help="Output directory", required=True,
action=ReadableDir)
parser.add_argument("--nolog", action="store_true",
default=False, help="Don't generate log files")
parser.add_argument("-d", "--debug", dest="verbose", default=False,
action='store_true',
help="Increase output verbosity")
return parser
def _maplot_argparser(self, desc=None):
......@@ -421,6 +455,17 @@ class AriaEcCommands(CLI):
parser.add_argument("--prefixname", dest="prefixname",
default="",
help="Prefix name for file names")
parser.add_argument("-c", "--conf", action=ReadableFile,
dest="conf_file",
default=None, help="configuration file")
parser.add_argument("-o", "--output", dest="output_directory", type=str,
help="Output directory", required=True,
action=ReadableDir)
parser.add_argument("--nolog", action="store_true",
default=False, help="Don't generate log files")
parser.add_argument("-d", "--debug", dest="verbose", default=False,
action='store_true',
help="Increase output verbosity")
return parser
def _pdbqual_argparser(self, desc=None):
......@@ -430,6 +475,17 @@ class AriaEcCommands(CLI):
action=ReadableFile,
help="PDB file(s) used to run quality tools with "
"aria API")
parser.add_argument("-c", "--conf", action=ReadableFile,
dest="conf_file",
default=None, help="configuration file")
parser.add_argument("-o", "--output", dest="output_directory", type=str,
help="Output directory", required=True,
action=ReadableDir)
parser.add_argument("--nolog", action="store_true",
default=False, help="Don't generate log files")
parser.add_argument("-d", "--debug", dest="verbose", default=False,
action='store_true',
help="Increase output verbosity")
return parser
def _analysis_argparser(self, desc=None):
......@@ -452,6 +508,17 @@ class AriaEcCommands(CLI):
parser.add_argument("-p", "--prefixname", dest="prefixname",
default=None,
help="Prefix name for file names")
parser.add_argument("-c", "--conf", action=ReadableFile,
dest="conf_file",
default=None, help="configuration file")
parser.add_argument("-o", "--output", dest="output_directory", type=str,
help="Output directory", required=True,
action=ReadableDir)
parser.add_argument("--nolog", action="store_true",
default=False, help="Don't generate log files")
parser.add_argument("-d", "--debug", dest="verbose", default=False,
action='store_true',
help="Increase output verbosity")
return parser
def _tbl2xml_argparser(self, desc=None):
......@@ -478,6 +545,17 @@ class AriaEcCommands(CLI):
parser.add_argument("infiles", nargs="+", metavar="infile.tbl",
action=ReadableFile,
help="TBL distance restraint file(s)")
parser.add_argument("-c", "--conf", action=ReadableFile,
dest="conf_file",
default=None, help="configuration file")
parser.add_argument("-o", "--output", dest="output_directory", type=str,
help="Output directory", required=True,
action=ReadableDir)
parser.add_argument("--nolog", action="store_true",
default=False, help="Don't generate log files")
parser.add_argument("-d", "--debug", dest="verbose", default=False,
action='store_true',
help="Increase output verbosity")
return parser
def _pdbdist_argparser(self, desc=None):
......@@ -511,6 +589,17 @@ class AriaEcCommands(CLI):
"--pdbdir", dest="pdbdir", default='',
metavar="PDB_FOLDER",
help="Folder containing pdb file entries")
parser.add_argument("-c", "--conf", action=ReadableFile,
dest="conf_file",
default=None, help="configuration file")
parser.add_argument("-o", "--output", dest="output_directory", type=str,
help="Output directory", required=True,
action=ReadableDir)
parser.add_argument("--nolog", action="store_true",
default=False, help="Don't generate log files")
parser.add_argument("-d", "--debug", dest="verbose", default=False,
action='store_true',
help="Increase output verbosity")
return parser
def _pdbstat_argparser(self, desc=None):
......@@ -546,6 +635,17 @@ class AriaEcCommands(CLI):
"--min", dest="minflag", action="store_true", default=False,
help="Compute stats only on a minimized list of atom (CA, CB and 1 "
"SC)")
parser.add_argument("-c", "--conf", action=ReadableFile,
dest="conf_file",
default=None, help="configuration file")
parser.add_argument("-o", "--output", dest="output_directory", type=str,
help="Output directory", required=True,
action=ReadableDir)
parser.add_argument("--nolog", action="store_true",
default=False, help="Don't generate log files")
parser.add_argument("-d", "--debug", dest="verbose", default=False,
action='store_true',
help="Increase output verbosity")
return parser
def create_settings(self):
......
aria/conbox/common.py
View file @ eff57768
......@@ -452,7 +452,8 @@ class CustomLogging(object):
# "conf/logging.json")
default_file = "conf/logging.json"
def __init__(self, level=logging.INFO, desc=None, welcome=True):
def __init__(self, level=logging.INFO, desc=None, welcome=True,
decorator=True):
"""
Parameters
......@@ -468,7 +469,7 @@ class CustomLogging(object):
self.config = self.load_config()
self.update_log()
if welcome:
self.welcome()
self.welcome(decorator=decorator)
def update_msg(self, desc):
"""
......@@ -507,16 +508,20 @@ class CustomLogging(object):
conf = pkgr.resource_stream(__name__, self.default_file).read().decode()
config = json.loads(conf)
# if abc:
# reg = re.compile(r"(?P<prefix>[A-Za-z0-9]*(?=_))?_?(?P<name>\w+)")
# for ft_name, formatter in config.get("formatters", {}).iteritems():
# prefix, name = [reg.match(ft_name).groupdict()[_]
# for _ in ('prefix', 'name')]
# if prefix == "abc" and name in config["formatters"]:
# config["formatters"][name] = config["formatters"][ft_name]
return config
def set_outdir(self, outdir):
def up_outfiles(self, outfile):
base = os.path.splitext(outfile)[0]
for hand in self.config["handlers"]:
if "filename" in self.config["handlers"][hand]:
oldpath = self.config["handlers"][hand]["filename"]
old_ext = os.path.splitext(oldpath)[1]
newpath = os.path.abspath(base + old_ext)
self.config["handlers"][hand]["filename"] = newpath
shutil.copy2(oldpath, newpath)
self.update_log()
def set_outdir(self, outdir, logdir=True):
"""
Create log directory and change log files location
......@@ -530,7 +535,7 @@ class CustomLogging(object):
"""
outdir = os.path.join(outdir,
"log") if "log" not in outdir else outdir
"log") if "log" not in outdir and logdir else outdir
if os.path.exists(os.path.abspath(outdir)):
# Avoid overwriting files with w mode after copy2 call
shutil.rmtree(os.path.abspath(outdir))
......@@ -546,7 +551,7 @@ class CustomLogging(object):
shutil.copy2(oldpath, newpath)
self.update_log()
def welcome(self):
def welcome(self, decorator=True):
"""
Returns
......@@ -559,7 +564,7 @@ class CustomLogging(object):
{:^80}
================================================================================
'''.format(self.msg)
'''.format(self.msg) if decorator else self.msg
for hand in self.config.get("handlers"):
if "filename" in self.config["handlers"][hand]:
with open(self.config["handlers"][hand]["filename"],
......
aria/conbox/protein.py
View file @ eff57768
......@@ -47,7 +47,7 @@ class SsList(object):
r'\s+(?P<ss_conf>\d?)'),
'ss2': re.compile(r'^\s*(?P<up_index>\d+)'
r'\s+(?P<up_residue>[AC-IK-NP-TVWYZ])'
r'\s+(?P<ss_pred>[HEC])'
r'\s+(?P<ss_pred>[HECBTG])'
r'\s+(?P<h_conf>\d?\.?\d*)'
r'\s+(?P<e_conf>\d?\.?\d*)'
r'\s+(?P<c_conf>\d?\.?\d*)'),
......
aria/conbox/templates/aria_project_v2.3.7.xml
View file @ eff57768
......@@ -76,7 +76,7 @@
<procheck executable="${procheck_executable}" enabled="${procheck_enabled}"/>
<prosa executable="${prosa_executable}" enabled="${prosa_enabled}"/>
<whatif executable="${whatif_executable}" enabled="${whatif_enabled}"/>
<clashlist executable="${clashlist_executable}" enabled="${clahlist_enabled}"/>
<clashlist executable="${clashlist_executable}" enabled="${clashlist_enabled}"/>
</analysis>
<report>
<ccpn export_assignments="no" export_noe_restraint_list="no" export_structures="no"/>
......
aria/core/JobManager.py
View file @ eff57768
......@@ -214,7 +214,8 @@ class Job(Thread, AriaBaseClass):
# 'sge_job_hostname' : qsubhosts} ##### BARDIAUX
f = open(settings['script'])
s = f.read() % d
s = f.read()
s = string.Formatter().vformat(s, (), d)
f.close()
# write new csh script
......@@ -246,7 +247,7 @@ class Job(Thread, AriaBaseClass):
try:
[os.unlink(foo) for foo in (filename, checkfile)]
except Exception as e:
LOG.exception(e)
self.logger.exception(e)
# start job
......@@ -266,10 +267,14 @@ class Job(Thread, AriaBaseClass):
# self.message(msg % job_desc)
self.debug(command)
process = subprocess.Popen(command, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = process.communicate()
try:
process = subprocess.Popen(command, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = process.communicate()
except Exception as e:
self.logger.exception(e)
self.message(out.replace("\n", "") + " (%s)" % job_desc)
# Parse output to get jobid and launch another job which depend on it to
......
aria/core/TypeChecking.py
View file @ eff57768
......@@ -31,6 +31,7 @@ from __future__ import absolute_import, division, print_function
import numpy
import logging
from numpy import zeros as _zeros
from collections import OrderedDict
FLOAT = 'FLOAT'
......@@ -39,6 +40,7 @@ INT = 'INT'
STRING = 'STRING'
UNICODE = 'UNICODE'
DICT = 'DICT'
ORDEREDICT = 'ORDEREDICT'
LIST = 'LIST'
TUPLE = 'TUPLE'
ARRAY = 'ARRAY'
......@@ -51,6 +53,7 @@ TYPES = {type(0.): FLOAT,
type(''): STRING,
type(u''): UNICODE,
type({}): DICT,
type(OrderedDict()): ORDEREDICT,
type([]): LIST,
type(()): TUPLE,
type(_zeros(0)): ARRAY,
......@@ -164,7 +167,7 @@ check_float = lambda x: check_type(x, FLOAT, FLOAT64)
check_string = lambda x: check_type(x, STRING, UNICODE)
check_list = lambda x: check_type(x, LIST)
check_tuple = lambda x: check_type(x, TUPLE)
check_dict = lambda x: check_type(x, DICT)
check_dict = lambda x: check_type(x, DICT, ORDEREDICT)
check_array = lambda x: check_type(x, ARRAY)
check_file = lambda x: check_type(x, 'file')
check_bool = lambda x: check_type(x, INT, BOOL)
aria/core/ViolationAnalyser.py
View file @ eff57768
......@@ -259,7 +259,7 @@ class ViolationAnalyser(AriaBaseClass):
"""
check_type(ensemble, 'StructureEnsemble')
check_type(peak, 'AriaPeak')
check_type(peak, 'AriaPeak', 'DistanceRestraint')
check_int(store_analysis)
check_type(lower_correction, FLOAT, NONE)
check_type(upper_correction, FLOAT, NONE)
......
aria/core/ariabase.py
View file @ eff57768
......@@ -21,9 +21,6 @@
.. Distribution of substantively modified versions of this module is ..
.. prohibited without the explicit permission of the copyright holders. ..
.. ..
.. $Author: bardiaux $ ..
.. $Revision: 1.1.1.1 $ ..
.. $Date: 2010/03/23 15:27:24 $ ..
.. .......................................................................... ..
"""
from __future__ import absolute_import, division, print_function
......@@ -60,7 +57,8 @@ VL_LOW = 1
PRINT_LOCK = Lock()
# Logging Config
LOG_CONF = CustomLogging(welcome=False)
# TODO: generate doc string at the package level with up to date version number
LOG_CONF = CustomLogging(desc=__doc__, decorator=False)
LOG = logging.getLogger(__name__)
......@@ -228,8 +226,25 @@ class AriaBaseClass:
if name is not None:
self._set_name(name)
def get_version_string(self):
return self.VERSION_FORMAT % (self.VERSION, self.VERSION_RELEASE)
if self.__class__.display_debug:
logging.getLogger().setLevel(logging.DEBUG)
def __getstate__(self):
"""
Remove logger attribute in order to pickle every ariabase object instance
"""
d = dict(self.__dict__)
del d['logger']
return d
def __setstate__(self, d):
d['logger'] = logging.getLogger(self.__module__)
self.__dict__.update(d)
@staticmethod
def get_version_string():
return AriaBaseClass.VERSION_FORMAT % (
AriaBaseClass.VERSION, AriaBaseClass.VERSION_RELEASE)
def _set_name(self, name):
check_string(name)
......@@ -314,12 +329,12 @@ class AriaBaseClass:
filename = code.co_filename
lineno = frame.f_lineno
descr = '\nFile "%s", line %d in %s\n%s'
descr = 'File "%s", line %d in %s\n%s'
msg += descr % (filename, lineno, func_name, str(error))
msg = 'USER ERROR <%s> ' % str(self.__class__) + msg
self.__log(msg)
# msg = 'USER ERROR <%s> \n' % str(self.__class__) + msg
self.message(msg, 'ERROR')
# self.__log(msg)
raise exception(msg)
......@@ -331,11 +346,11 @@ class AriaBaseClass:
from . import tools as tools
if self.wrap_lines:
lines = tools.make_block(msg, self.line_length - len(tag))
lines = tools.make_block(msg, self.line_length - len(tag) - 8)
else:
lines = [msg]
lines = tools.indent(lines, tag)
lines = tools.indent(lines, tag, extra=9)
return lines
......@@ -350,9 +365,8 @@ class AriaBaseClass:
def __print(self, prefix, msg, verbose_level, func=None):
if verbose_level <= self.verbose_level or self.display_debug:
tag = prefix
prefix = self.__compile_name("")
lines = self.__format(prefix, msg)
lvl = logging.getLevelName(prefix)
lines = self.__format(self.__compile_name(""), msg)
if self.log_gui:
self.log_gui.write(lines + '\n')
if self.log_stdout:
......@@ -361,18 +375,15 @@ class AriaBaseClass:
"abc_name": func.co_filename,
"abc_funcName": func.co_name,
"abc_lineno": func.co_firstlineno}}
self.logger.warning(lines, **kw) if "WARNING" in tag else \
self.logger.error(lines, **kw) if "ERROR" in tag else \
self.logger.info(lines, **kw)
# print(lines)
self.__log(lines)
self.logger.log(lvl, lines, **kw)
# self.__log(lines)
def __log(self, s):
if self.log_file is not None:
self.log_file.write(s + '\n')
self.log_file.flush()
def message(self, msg, prefix='MESSAGE', verbose_level=VL_STANDARD):
def message(self, msg, prefix="INFO", verbose_level=VL_STANDARD):
"""
Message log
......@@ -381,7 +392,7 @@ class AriaBaseClass:
msg :
prefix :
(Default value = 'MESSAGE')
(Default value = 'INFO')
verbose_level :
(Default value = VL_STANDARD)
......@@ -392,9 +403,7 @@ class AriaBaseClass:
"""
func = inspect.currentframe().f_back.f_code
if self.display_messages:
prefix = self.__compile_name(prefix)
msg = str(msg)
self.__print(prefix, msg, verbose_level, func=func)
self.__print(prefix, str(msg), verbose_level, func=func)
def warning(self, msg, verbose_level=VL_STANDARD):
"""
......@@ -412,7 +421,7 @@ class AriaBaseClass:
"""
func = inspect.currentframe().f_back.f_code
if self.display_warnings:
msg = str(msg)
......@@ -420,8 +429,7 @@ class AriaBaseClass:
if self.warnings_as_errors:
self.error(msg=msg)
else:
prefix = self.__compile_name('WARNING')
self.__print(prefix, msg, verbose_level)
self.__print('WARNING', msg, verbose_level, func=func)
def halt(self):
"""aborts ARIA"""
......
aria/core/cns.py
View file @ eff57768
......@@ -92,8 +92,8 @@ CSH_SCRIPT_REFINE = '''\
#BSUB -L {sge_job_shell}
#BSUB -oo refine.lsf.out
#BSUB -eo refine.lsf.err
#SBATCH -o refine.{%j}.slurm.out
#SBATCH -e refine.{%j}.slurm.err