Commit 78f1f2ce authored by Hervé  MENAGER's avatar Hervé MENAGER
Browse files

more compatibility work

parent 004a93ec
Pipeline #11373 failed with stages
in 9 minutes and 57 seconds
......@@ -11,7 +11,6 @@ RUN apt-get update && apt-get install -y \
wget \
curl \
libpq-dev
#RUN git clone --depth 1 --progress https://github.com/hmenager/galaxy.git /app/galaxy
RUN git clone --depth 1 --progress https://github.com/common-workflow-language/galaxy.git /app/galaxy
RUN git clone --depth 1 --single-branch --branch master --progress https://github.com/hmenager/workflow-is-cwl /app/workflow-is-cwl
RUN git clone --depth 1 --single-branch --branch assembly --progress https://github.com/hmenager/workflow-is-cwl /app/workflow-is-cwl_assembly
......@@ -20,18 +19,25 @@ RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources
RUN apt-get update && apt-get install -y yarn
#quickfix due to record not being in the right folder following rebase (04/03/2019)
RUN mv /app/galaxy/lib/galaxy/dataset_collections/types/record.py /app/galaxy/lib/galaxy/model/dataset_collections/types/record.py
# init Galaxy (download deps and init DB...)
RUN cd /app/galaxy && ./scripts/common_startup.sh
RUN cd /app/galaxy && ls database/*
#RUN cd /app/galaxy && . .venv/bin/activate && cwltool --pack /app/workflow-is-cwl/tools/Diamond/Diamon.blastx-v0.9.21.cwl > /app/workflow-is-cwl/tools/Diamond/Diamon.blastx-v0.9.21.packed.cwl
RUN cd /app/galaxy && ./create_db.sh
RUN cd /app/galaxy && ./manage_db.sh upgrade
# DEBUG complex types
COPY representation.py /app/galaxy/lib/galaxy/tools/cwl/representation.py
COPY parser.py /app/galaxy/lib/galaxy/tools/cwl/parser.py
#COPY __init__.py /app/galaxy/lib/galaxy/tools/__init__.py
COPY process.py /app/galaxy/.venv/local/lib/python2.7/site-packages/cwltool/process.py
COPY wrappers.py /app/galaxy/lib/galaxy/tools/wrappers.py
COPY TransDecoder.LongOrfs-v5.cwl /app/workflow-is-cwl/tools/TransDecoder/TransDecoder.LongOrfs-v5.cwl
# local config
COPY job_conf.xml /app/galaxy/config/job_conf.xml
COPY dependency_resolvers_conf.xml /app/galaxy/config/dependency_resolvers_conf.xml
COPY galaxy.yml /app/galaxy/config/galaxy.yml
#COPY tool_conf_test.xml /app/galaxy/config/tool_conf.xml
#COPY concatenate.cwl /app/galaxy/tools/
#COPY concatenate-docker.cwl /app/galaxy/tools/
COPY tool_conf.xml /app/galaxy/config/tool_conf.xml
COPY welcome.html /app/galaxy/static/welcome.html
# create entry point
COPY start.sh /app/start.sh
CMD ["sh", "/app/start.sh"]
EXPOSE 8080
......@@ -9,7 +9,7 @@ docker build . -t galaxycwlis
run server locally:
```bash
docker run -i -t -p 8080:8080 -v /var/run/docker.sock:/var/run/docker.sock galaxycwlis:latest
docker run --privileged -i -t -p 8080:8080 galaxycwlis:latest
```
open terminal in server:
......
"""
Classes encapsulating galaxy tools and tool configuration.
"""
import json
import logging
import os
import re
import uuid
import tarfile
import tempfile
import threading
from collections import OrderedDict
from datetime import datetime
from xml.etree import ElementTree
import packaging.version
import webob.exc
from mako.template import Template
from six import itervalues, string_types
from six.moves.urllib.parse import unquote_plus
from webob.compat import cgi_FieldStorage
import tool_shed.util.repository_util as repository_util
import tool_shed.util.shed_util_common
from galaxy import (
exceptions,
model
)
from galaxy.managers.jobs import JobSearch
from galaxy.metadata import get_metadata_compute_strategy
from galaxy.model.tags import GalaxyTagHandler
from galaxy.queue_worker import send_control_task
from galaxy.tools import expressions
from galaxy.tools.actions import DefaultToolAction
from galaxy.tools.actions.data_manager import DataManagerToolAction
from galaxy.tools.actions.data_source import DataSourceToolAction
from galaxy.tools.actions.model_operations import ModelOperationToolAction
from galaxy.tools.deps import (
CachedDependencyManager,
)
from galaxy.tools.fetcher import ToolLocationFetcher
from galaxy.tools.parameters import (
check_param,
params_from_strings,
params_to_incoming,
params_to_strings,
populate_state,
visit_input_values
)
from galaxy.tools.parameters import output_collect
from galaxy.tools.parameters.basic import (
BaseURLToolParameter,
DataCollectionToolParameter,
DataToolParameter,
HiddenToolParameter,
SelectToolParameter,
ToolParameter,
workflow_building_modes,
)
from galaxy.tools.parameters.dataset_matcher import (
set_dataset_matcher_factory,
unset_dataset_matcher_factory,
)
from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repeat, Section, UploadDataset
from galaxy.tools.parameters.input_translation import ToolInputTranslator
from galaxy.tools.parameters.meta import expand_meta_parameters
from galaxy.tools.parameters.wrapped_json import json_wrap
from galaxy.tools.parser import (
get_tool_source,
get_tool_source_from_representation,
ToolOutputCollectionPart
)
from galaxy.tools.cwl import needs_shell_quoting, shellescape, to_galaxy_parameters
from galaxy.tools.parser.xml import XmlPageSource
from galaxy.tools.test import parse_tests
from galaxy.tools.toolbox import BaseGalaxyToolBox
from galaxy.util import (
ExecutionTimer,
in_directory,
listify,
Params,
rst_to_html,
safe_makedirs,
string_as_bool,
unicodify
)
from galaxy.util.bunch import Bunch
from galaxy.util.dictifiable import Dictifiable
from galaxy.util.expressions import ExpressionContext
from galaxy.util.form_builder import SelectField
from galaxy.util.json import safe_loads
from galaxy.util.odict import odict
from galaxy.util.rules_dsl import RuleSet
from galaxy.util.template import fill_template
from galaxy.version import VERSION_MAJOR
from galaxy.work.context import WorkRequestContext
from tool_shed.util import common_util
from .execute import (
execute as execute_job,
MappingParameters,
)
from .loader import (
imported_macro_paths,
raw_tool_xml_tree,
template_macro_params
)
from .provided_metadata import parse_tool_provided_metadata
log = logging.getLogger(__name__)
REQUIRES_JS_RUNTIME_MESSAGE = ("The tool [%s] requires a nodejs runtime to execute "
"but node or nodejs could not be found. Please contact the Galaxy adminstrator")
HELP_UNINITIALIZED = threading.Lock()
MODEL_TOOLS_PATH = os.path.abspath(os.path.dirname(__file__))
# Tools that require Galaxy's Python environment to be preserved.
GALAXY_LIB_TOOLS_UNVERSIONED = [
"upload1",
"send_to_cloud",
"__DATA_FETCH__",
# Legacy tools bundled with Galaxy.
"vcf_to_maf_customtrack1",
"laj_1",
"meme_fimo",
"secure_hash_message_digest",
"join1",
"gff2bed1",
"gff_filter_by_feature_count",
"aggregate_scores_in_intervals2",
"Interval_Maf_Merged_Fasta2",
"GeneBed_Maf_Fasta2",
"maf_stats1",
"Interval2Maf1",
"Interval2Maf_pairwise1",
"MAF_To_Interval1",
"MAF_filter",
"MAF_To_Fasta1",
"MAF_Reverse_Complement_1",
"MAF_split_blocks_by_species1",
"MAF_Limit_To_Species1",
"maf_by_block_number1",
"wiggle2simple1",
# Converters
"CONVERTER_bed_to_fli_0",
"CONVERTER_fastq_to_fqtoc0",
"CONVERTER_gff_to_fli_0",
"CONVERTER_gff_to_interval_index_0",
"CONVERTER_maf_to_fasta_0",
"CONVERTER_maf_to_interval_0",
"CONVERTER_wiggle_to_interval_0",
"CONVERTER_tar_to_directory",
# Tools improperly migrated to the tool shed (devteam)
"qualityFilter",
"winSplitter",
"pileup_interval",
"count_gff_features",
"Convert characters1",
"lastz_paired_reads_wrapper",
"subRate1",
"substitutions1",
"sam_pileup",
"find_diag_hits",
"cufflinks",
# Tools improperly migrated to the tool shed (iuc)
"tabular_to_dbnsfp",
# Tools improperly migrated using Galaxy (from shed other)
"column_join",
"gd_coverage_distributions", # Genome Diversity tools from miller-lab
"gd_dpmix",
"gd_pca",
"gd_phylogenetic_tree",
"gd_population_structure",
"gd_prepare_population_structure",
# Datasources
"genomespace_importer"
]
# Tools that needed galaxy on the PATH in the past but no longer do along
# with the version at which they were fixed.
GALAXY_LIB_TOOLS_VERSIONED = {
"sam_to_bam": packaging.version.parse("1.1.3"),
"PEsortedSAM2readprofile": packaging.version.parse("1.1.1"),
"fetchflank": packaging.version.parse("1.0.1"),
"Extract genomic DNA 1": packaging.version.parse("3.0.0"),
"lastz_wrapper_2": packaging.version.parse("1.3"),
}
class ToolErrorLog(object):
def __init__(self):
self.error_stack = []
self.max_errors = 100
def add_error(self, file, phase, exception):
self.error_stack.insert(0, {
"file": file,
"time": str(datetime.now()),
"phase": phase,
"error": str(exception)
})
if len(self.error_stack) > self.max_errors:
self.error_stack.pop()
global_tool_errors = ToolErrorLog()
class ToolInputsNotReadyException(Exception):
pass
class ToolNotFoundException(Exception):
pass
def create_tool_from_source(app, tool_source, config_file=None, **kwds):
# Allow specifying a different tool subclass to instantiate
tool_module = tool_source.parse_tool_module()
if tool_module is not None:
module, cls = tool_module
mod = __import__(module, globals(), locals(), [cls])
ToolClass = getattr(mod, cls)
elif tool_source.parse_tool_type():
tool_type = tool_source.parse_tool_type()
ToolClass = tool_types.get(tool_type)
else:
# Normal tool
root = getattr(tool_source, 'root', None)
ToolClass = Tool
tool = ToolClass(config_file, tool_source, app, **kwds)
return tool
class ToolBox(BaseGalaxyToolBox):
""" A derivative of AbstractToolBox with knowledge about Tool internals -
how to construct them, action types, dependency management, etc....
"""
def __init__(self, config_filenames, tool_root_dir, app):
self._reload_count = 0
self.tool_location_fetcher = ToolLocationFetcher()
super(ToolBox, self).__init__(
config_filenames=config_filenames,
tool_root_dir=tool_root_dir,
app=app,
)
def handle_panel_update(self, section_dict):
"""
Sends a panel update to all threads/processes.
"""
send_control_task(self.app, 'create_panel_section', kwargs=section_dict)
# The following local call to self.create_section should be unnecessary
# but occasionally the local ToolPanelElements instance appears to not
# get updated.
self.create_section(section_dict)
def has_reloaded(self, other_toolbox):
return self._reload_count != other_toolbox._reload_count
@property
def all_requirements(self):
reqs = set([req for _, tool in self.tools() for req in tool.tool_requirements])
return [r.to_dict() for r in reqs]
@property
def tools_by_id(self):
# Deprecated method, TODO - eliminate calls to this in test/.
return self._tools_by_id
def create_tool(self, config_file, **kwds):
tool_source = get_tool_source(
config_file,
enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False),
tool_location_fetcher=self.tool_location_fetcher,
strict_cwl_validation=getattr(self.app.config, "strict_cwl_validation", True),
)
return self._create_tool_from_source(tool_source, config_file=config_file, **kwds)
def _create_tool_from_source(self, tool_source, **kwds):
return create_tool_from_source(self.app, tool_source, **kwds)
def create_dynamic_tool(self, dynamic_tool, **kwds):
tool_format = dynamic_tool.tool_format
tool_representation = dynamic_tool.value
get_source_kwds = dict(
tool_format=tool_format,
tool_representation=tool_representation,
uuid=dynamic_tool.uuid,
)
if dynamic_tool.tool_directory:
get_source_kwds["tool_directory"] = dynamic_tool.tool_directory
if dynamic_tool.tool_path:
config_file = dynamic_tool.tool_path
# TODO: uuid probably needed here...
tool_source = get_tool_source(
config_file,
enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", True),
tool_location_fetcher=self.tool_location_fetcher,
strict_cwl_validation=getattr(self.app.config, "strict_cwl_validation", True),
)
else:
tool_source = get_tool_source_from_representation(**get_source_kwds)
kwds["dynamic"] = True
tool = self._create_tool_from_source(tool_source, **kwds)
tool.dynamic_tool = dynamic_tool
tool.uuid = dynamic_tool.uuid
if not tool.id:
tool.id = dynamic_tool.tool_id
if not tool.name:
tool.name = tool.id
return tool
def get_tool_components(self, tool_id, tool_version=None, get_loaded_tools_by_lineage=False, set_selected=False):
"""
Retrieve all loaded versions of a tool from the toolbox and return a select list enabling
selection of a different version, the list of the tool's loaded versions, and the specified tool.
"""
toolbox = self
tool_version_select_field = None
tools = []
tool = None
# Backwards compatibility for datasource tools that have default tool_id configured, but which
# are now using only GALAXY_URL.
tool_ids = listify(tool_id)
for tool_id in tool_ids:
if get_loaded_tools_by_lineage:
tools = toolbox.get_loaded_tools_by_lineage(tool_id)
else:
tools = toolbox.get_tool(tool_id, tool_version=tool_version, get_all_versions=True)
if tools:
tool = toolbox.get_tool(tool_id, tool_version=tool_version, get_all_versions=False)
if len(tools) > 1:
tool_version_select_field = self.__build_tool_version_select_field(tools, tool.id, set_selected)
break
return tool_version_select_field, tools, tool
def _path_template_kwds(self):
return {
"model_tools_path": MODEL_TOOLS_PATH,
}
def _get_tool_shed_repository(self, tool_shed, name, owner, installed_changeset_revision):
# Abstract toolbox doesn't have a dependency on the the database, so
# override _get_tool_shed_repository here to provide this information.
return repository_util.get_installed_repository(
self.app,
tool_shed=tool_shed,
name=name,
owner=owner,
installed_changeset_revision=installed_changeset_revision
)
def __build_tool_version_select_field(self, tools, tool_id, set_selected):
"""Build a SelectField whose options are the ids for the received list of tools."""
options = []
for tool in tools:
options.insert(0, (tool.version, tool.id))
select_field = SelectField(name='tool_id')
for option_tup in options:
selected = set_selected and option_tup[1] == tool_id
if selected:
select_field.add_option('version %s' % option_tup[0], option_tup[1], selected=True)
else:
select_field.add_option('version %s' % option_tup[0], option_tup[1])
return select_field
class DefaultToolState(object):
"""
Keeps track of the state of a users interaction with a tool between
requests.
"""
def __init__(self):
self.page = 0
self.rerun_remap_job_id = None
self.inputs = {}
def initialize(self, trans, tool):
"""
Create a new `DefaultToolState` for this tool. It will be initialized
with default values for inputs. Grouping elements are filled in recursively.
"""
self.inputs = {}
context = ExpressionContext(self.inputs)
for input in tool.inputs.values():
self.inputs[input.name] = input.get_initial_value(trans, context)
def encode(self, tool, app, nested=False):
"""
Convert the data to a string
"""
value = params_to_strings(tool.inputs, self.inputs, app, nested=nested)
value["__page__"] = self.page
value["__rerun_remap_job_id__"] = self.rerun_remap_job_id
return value
def decode(self, values, tool, app):
"""
Restore the state from a string
"""
values = safe_loads(values) or {}
self.page = values.pop("__page__") if "__page__" in values else None
self.rerun_remap_job_id = values.pop("__rerun_remap_job_id__") if "__rerun_remap_job_id__" in values else None
self.inputs = params_from_strings(tool.inputs, values, app, ignore_errors=True)
def copy(self):
"""
Shallow copy of the state
"""
new_state = DefaultToolState()
new_state.page = self.page
new_state.rerun_remap_job_id = self.rerun_remap_job_id
new_state.inputs = self.inputs
return new_state
class Tool(Dictifiable):
"""
Represents a computational tool that can be executed through Galaxy.
"""
tool_type = 'default'
requires_setting_metadata = True
default_tool_action = DefaultToolAction
dict_collection_visible_keys = ['id', 'name', 'version', 'description', 'labels']
may_use_container_entry_point = False
def __init__(self, config_file, tool_source, app, guid=None, repository_id=None, tool_shed_repository=None, allow_code_files=True, dynamic=False):
"""Load a tool from the config named by `config_file`"""
# Determine the full path of the directory where the tool config is
if config_file is not None:
self.config_file = config_file
self.tool_dir = os.path.dirname(config_file)
else:
self.config_file = None
self.tool_dir = None
self.app = app
self.repository_id = repository_id
self._allow_code_files = allow_code_files
# setup initial attribute values
self.inputs = odict()
self.stdio_exit_codes = list()
self.stdio_regexes = list()
self.inputs_by_page = list()
self.display_by_page = list()
self.action = '/tool_runner/index'
self.target = 'galaxy_main'
self.method = 'post'
self.labels = []
self.check_values = True
self.nginx_upload = False
self.input_required = False
self.display_interface = True
self.require_login = False
self.rerun = False
# This will be non-None for tools loaded from the database (DynamicTool objects).
self.dynamic_tool = None
# Define a place to keep track of all input These
# differ from the inputs dictionary in that inputs can be page
# elements like conditionals, but input_params are basic form
# parameters like SelectField objects. This enables us to more
# easily ensure that parameter dependencies like index files or
# tool_data_table_conf.xml entries exist.
self.input_params = []
# Attributes of tools installed from Galaxy tool sheds.
self.tool_shed = None
self.repository_name = None
self.repository_owner = None
self.changeset_revision = None
self.installed_changeset_revision = None
self.sharable_url = None
# The tool.id value will be the value of guid, but we'll keep the
# guid attribute since it is useful to have.
self.guid = guid
self.old_id = None
self.version = None
self._lineage = None
self.dependencies = []
# populate toolshed repository info, if available
self.populate_tool_shed_info(tool_shed_repository)
# add tool resource parameters
self.populate_resource_parameters(tool_source)
self.tool_errors = None
self.parse(tool_source, guid=guid, dynamic=dynamic)
# Parse XML element containing configuration
try:
self.parse(tool_source, guid=guid, dynamic=dynamic)
except Exception as e:
global_tool_errors.add_error(config_file, "Tool Loading", e)
raise e
# The job search is only relevant in a galaxy context, and breaks
# loading tools into the toolshed for validation.
if self.app.name == 'galaxy':
self.job_search = JobSearch(app=self.app)
@property
def history_manager(self):
return self.app.history_manager
@property
def _view(self):
return self.app.dependency_resolvers_view
@property
def version_object(self):
return packaging.version.parse(self.version)
@property
def sa_session(self):
"""Returns a SQLAlchemy session"""
return self.app.model.context
@property
def lineage(self):
"""Return ToolLineage for this tool."""
return self._lineage
@property
def tool_versions(self):
# If we have versions, return them.
if self.lineage:
return self.lineage.tool_versions
else:
return []
@property
def tool_shed_repository(self):
# If this tool is included in an installed tool shed repository, return it.
if self.tool_shed:
return repository_util.get_installed_repository(self.app,
tool_shed=self.tool_shed,
name=self.repository_name,
owner=self.repository_owner,
installed_changeset_revision=self.installed_changeset_revision,
repository_id=self.repository_id)
@property
def produces_collections_with_unknown_structure(self):
def output_is_dynamic(output):
if not output.collection:
return False
return output.dynamic_structure
return any(map(output_is_dynamic, self.outputs.values()))
@property
def valid_input_states(self):
return model.Dataset.valid_input_states
@property
def requires_galaxy_python_environment(self):
"""Indicates this tool's runtime requires Galaxy's Python environment."""
# All special tool types (data source, history import/export, etc...)
# seem to require Galaxy's Python.
if self.tool_type not in ["default", "manage_data"]:
return True
if self.tool_type == "manage_data" and self.profile < 18.09:
return True
config = self.app.config
preserve_python_environment = config.preserve_python_environment
if preserve_python_environment == "always":
return True
elif preserve_python_environment == "legacy_and_local" and self.tool_shed is None:
return True
else:
unversioned_legacy_tool = self.old_id in GALAXY_LIB_TOOLS_UNVERSIONED
versioned_legacy_tool = self.old_id in GALAXY_LIB_TOOLS_VERSIONED
legacy_tool = unversioned_legacy_tool or \
(versioned_legacy_tool and self.version_object < GALAXY_LIB_TOOLS_VERSIONED[self.old_id])
return legacy_tool