Commit 66cd8a4f authored by Keith Jolley's avatar Keith Jolley

Initial work to submit XMFA file creation to the job queue database.

parent 310866ee
......@@ -92,6 +92,7 @@ use BIGSdb::FieldHelpPage;
use BIGSdb::TableQueryPage;
use BIGSdb::DownloadSeqbinPage;
use BIGSdb::AlleleSequencePage;
use BIGSdb::OfflineJobManager;
my $r = shift; #Apache request object (used for mod_perl)
Log::Log4perl->init_once(CONFIG_DIR . '/logging.conf');
......
......@@ -75,6 +75,7 @@ use BIGSdb::FieldHelpPage;
use BIGSdb::ExtractedSequencePage;
use BIGSdb::AlleleQueryPage;
use BIGSdb::LocusInfoPage;
use BIGSdb::OfflineJobManager;
my $r = shift; #Apache request object (used for mod_perl)
Log::Log4perl->init_once( CONFIG_DIR . '/logging.conf' );
......
......@@ -13,10 +13,9 @@ log4perl.logger.BIGSdb.Application_Authentication=WARN, A1
log4perl.logger.BIGSdb.Prefs=WARN, A1
log4perl.logger.BIGSdb.Locus=WARN, A1
log4perl.logger.BIGSdb.Scheme=WARN, A1
log4perl.logger.BIGSdb.ClientDB=WARN, A1
log4perl.logger.BIGSdb.Page=WARN, A1
log4perl.logger.BIGSdb.Plugins=WARN, A1
log4perl.logger.BIGSdb.Charts=WARN, A1
log4perl.logger.BIGSdb.Job=WARN, A1
log4perl.appender.A1=Log::Dispatch::File
log4perl.appender.A1.filename=/var/log/bigsdb_jobs.log
......
......@@ -44,7 +44,7 @@ sub new {
$self->{'curate'} = $curate;
bless( $self, $class );
$self->_initiate( $config_dir, $dbase_config_dir );
$self->{'dataConnector'}->initiate($self->{'system'});
$self->{'dataConnector'}->initiate( $self->{'system'} );
my $logger_benchmark = get_logger('BIGSdb.Application_Benchmark');
if ( !$self->{'error'} ) {
......@@ -52,8 +52,9 @@ sub new {
if ( $self->{'db'} ) {
$self->_setup_datastore();
$self->_setup_prefstore();
$self->_initiate_authdb if $self->{'system'}->{'authentication'} eq 'builtin';
$self->_initiate_plugins($plugin_dir);
$self->_initiate_authdb if $self->{'system'}->{'authentication'} eq 'builtin';
$self->_initiate_jobmanager( $config_dir, $plugin_dir, $dbase_config_dir );
$self->_initiate_plugins($plugin_dir);
}
}
( my $elapsed = gettimeofday() - $self->{'start_time'} ) =~ s/(^\d{1,}\.\d{4}).*$/$1/;
......@@ -116,20 +117,19 @@ sub _initiate {
$self->{'system'}->{'user'} = 'apache' if !$self->{'system'}->{'user'};
$self->{'system'}->{'password'} = 'remote'
if !$self->{'system'}->{'password'};
$self->{'system'}->{'privacy'} = $self->{'system'}->{'privacy'} eq 'no' ? 0 : 1;
if ($self->{'system'}->{'dbtype'} eq 'isolates'){
$self->{'system'}->{'view'} = 'isolates' if !$self->{'system'}->{'view'};
$self->{'system'}->{'labelfield'} = 'isolate' if !$self->{'system'}->{'labelfield'};
if (!$self->{'xmlHandler'}->is_field($self->{'system'}->{'labelfield'})){
$logger->error("The defined labelfield '$self->{'system'}->{'labelfield'}' does not exist in the database. Please set the labelfield attribute in the system tag of the database XML file.");
if ( $self->{'system'}->{'dbtype'} eq 'isolates' ) {
$self->{'system'}->{'view'} = 'isolates' if !$self->{'system'}->{'view'};
$self->{'system'}->{'labelfield'} = 'isolate' if !$self->{'system'}->{'labelfield'};
if ( !$self->{'xmlHandler'}->is_field( $self->{'system'}->{'labelfield'} ) ) {
$logger->error(
"The defined labelfield '$self->{'system'}->{'labelfield'}' does not exist in the database. Please set the labelfield attribute in the system tag of the database XML file."
);
}
}
}
sub _initiate_authdb {
my ($self) = @_;
my $logger = get_logger('BIGSdb.Application_Initiate');
......@@ -184,8 +184,6 @@ SQL
}
}
sub _initiate_plugins {
my ( $self, $plugin_dir ) = @_;
$self->{'pluginManager'} = BIGSdb::PluginManager->new(
......@@ -199,10 +197,22 @@ sub _initiate_plugins {
'xmlHandler' => $self->{'xmlHandler'},
'dataConnector' => $self->{'dataConnector'},
'mod_perl_request' => $self->{'mod_perl_request'},
'jobManager' => $self->{'jobManager'},
'pluginDir' => $plugin_dir
);
}
sub _initiate_jobmanager {
my ( $self, $config_dir, $plugin_dir, $dbase_config_dir, ) = @_;
$self->{'jobManager'} = BIGSdb::OfflineJobManager->new(
$config_dir, $plugin_dir, $dbase_config_dir,
$self->{'system'}->{'host'},
$self->{'system'}->{'port'},
$self->{'system'}->{'user'},
$self->{'system'}->{'password'},
);
}
sub _read_config_file {
my ( $self, $config_dir ) = @_;
my $logger = get_logger('BIGSdb.Application_Initiate');
......@@ -322,8 +332,8 @@ sub print_page {
'alleleInfo' => 'AlleleInfoPage',
'fieldValues' => 'FieldHelpPage',
'extractedSequence' => 'ExtractedSequencePage',
'alleleQuery' => 'AlleleQueryPage',
'locusInfo' => 'LocusInfoPage'
'alleleQuery' => 'AlleleQueryPage',
'locusInfo' => 'LocusInfoPage'
);
my $page;
my %page_attributes = (
......@@ -360,13 +370,12 @@ sub print_page {
if ( $self->{'page'} eq 'options'
&& ( $self->{'cgi'}->param('set') || $self->{'cgi'}->param('reset') ) )
{
$page = BIGSdb::OptionsPage->new(%page_attributes);
$page = BIGSdb::OptionsPage->new(%page_attributes);
$page->initiate_prefs;
$page->set_options;
$self->{'page'} = 'index';
$self->{'cgi'}->param('page','index'); #stop prefs initiating twice
$set_options = 1;
$self->{'cgi'}->param( 'page', 'index' ); #stop prefs initiating twice
$set_options = 1;
}
if ( !$self->{'db'} ) {
$page_attributes{'error'} = 'noConnect';
......
......@@ -17,39 +17,39 @@
#You should have received a copy of the GNU General Public License
#along with BIGSdb. If not, see <http://www.gnu.org/licenses/>.
package BIGSdb::OfflineJobManager;
use base qw(BIGSdb::Application);
use Error qw(:try);
use Log::Log4perl qw(get_logger);
sub new {
#The job manager uses its own Dataconnector since it may be called by a stand-alone script.
my ( $class, $config_dir, $plugin_dir, $dbase_config_dir, $host, $port, $user, $password ) = @_;
my $self = {};
$self->{'system'} = {};
$self->{'host'} = $host;
$self->{'port'} = $port;
$self->{'user'} = $user;
$self->{'password'} = $password;
$self->{'xmlHandler'} = undef;
$self->{'dataConnector'} = new BIGSdb::Dataconnector;
$self->{'system'} = {};
$self->{'host'} = $host;
$self->{'port'} = $port;
$self->{'user'} = $user;
$self->{'password'} = $password;
$self->{'xmlHandler'} = undef;
$self->{'dataConnector'} = new BIGSdb::Dataconnector;
bless( $self, $class );
$self->_initiate( $config_dir, $dbase_config_dir );
# $self->{'dataConnector'}->initiate($self->{'system'});
$self->_initiate( $config_dir, $dbase_config_dir );
$self->_db_connect;
return $self;
}
sub _initiate {
my ( $self, $config_dir, $dbase_config_dir ) = @_;
$self->_read_config_file($config_dir);
my $logger = get_logger('BIGSdb.Application_Initiate');
# $logger->error("Test logging");
# $logger->error("Test logging");
}
sub _db_connect {
my ($self) = @_;
my $logger = get_logger('BIGSdb.Application_Initiate');
if (!$self->{'config'}->{'jobs_db'}){
if ( !$self->{'config'}->{'jobs_db'} ) {
$logger->fatal("jobs_db not set in config file.");
return;
}
......@@ -71,4 +71,51 @@ sub _db_connect {
};
}
1;
\ No newline at end of file
sub add_job {
#Required params:
#dbase_config: name of db configuration direction in /etc/dbases
#ip_address: connecting address
#module: Plugin module name
#function: Function in module to call
#
#Optional params:
#username and email of user
#priority: (highest 1 - lowest 9)
#query: SQL of query used to generate dataset
#parameters: any additional parameters needed by plugin (| separated list)
my ( $self, $params ) = @_;
my $logger = get_logger('BIGSdb.Job');
foreach (qw (dbase_config ip_address module function)) {
if ( !$params->{$_} ) {
$logger->error("Parameter $_ not passed");
throw BIGSdb::DataException("Parameter $_ not passed");
}
}
$params->{'priority'} = 5 if !$params->{'priority'};
my $id = BIGSdb::Utils::get_random();
eval {
$self->{'db'}->do(
"INSERT INTO jobs (id,dbase_config,username,email,ip_address,submit_time,module,function,query,parameters,status,priority) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
undef,
$id,
$params->{'dbase_config'},
$params->{'username'},
$params->{'email'},
$params->{'ip_address'},
'now',
$params->{'module'},
$params->{'function'},
$params->{'query'},
$params->{'parameters'},
'submitted',
$params->{'priority'}
);
};
if ($@){
$logger->error($@);
$self->{'db'}->rollback;
} else {
$self->{'db'}->commit;
}
}
1;
......@@ -58,6 +58,7 @@ sub initiate {
'db' => $self->{'db'},
'xmlHandler' => $self->{'xmlHandler'},
'dataConnector' => $self->{'dataConnector'},
'jobManager' => $self->{'jobManager'},
'mod_perl_request' => $self->{'mod_perl_request'}
);
$self->{'plugins'}->{$plugin_name} = $plugin;
......
......@@ -127,26 +127,34 @@ sub run {
$list = $self->{'datastore'}->run_list_query($qry);
}
}
print "<div class=\"box\" id=\"resultstable\">";
print "<p>Please wait for processing to finish (do not refresh page).</p>\n";
print "<p>Output file being generated ...";
print <<"HTML";
<div class="box" id="resultstable">
<p>The output file has been submitted to the job queue.</p>
<p>Please be aware that this job may take a long time depending on the number of sequences to align.</p>
HTML
my $filename = (BIGSdb::Utils::get_random()) . '.txt';
my $full_path = "$self->{'config'}->{'tmp_dir'}/$filename";
$| = 1;
my ( $problem_ids, $no_output ) = $self->_write_xmfa( $list, \@fields_selected, $full_path, $pk );
print " done</p>";
if ($no_output) {
print "<p>No output generated. Please ensure that your sequences have been defined for these isolates.</p>\n";
} else {
print "<p><a href=\"/tmp/$filename\">Output file</a> (right-click to save)</p>\n";
}
print "</div>\n";
if (@$problem_ids) {
$" = '; ';
print
"<div class=\"box\" id=\"statusbad\"><p>The following ids could not be processed (they do not exist): @$problem_ids.</p></div>\n";
}
$self->{'jobManager'}->add_job({
'dbase_config' => $self->{'instance'},
'ip_address' => $q->remote_host,
'module' => 'XmfaExport',
'function' => 'write_xmfa'
});
# $| = 1;
# my ( $problem_ids, $no_output ) = $self->_write_xmfa( $list, \@fields_selected, $full_path, $pk );
# print " done</p>";
#
# if ($no_output) {
# print "<p>No output generated. Please ensure that your sequences have been defined for these isolates.</p>\n";
# } else {
# print "<p><a href=\"/tmp/$filename\">Output file</a> (right-click to save)</p>\n";
# }
# print "</div>\n";
# if (@$problem_ids) {
# $" = '; ';
# print
#"<div class=\"box\" id=\"statusbad\"><p>The following ids could not be processed (they do not exist): @$problem_ids.</p></div>\n";
# }
return;
}
}
......
......@@ -5,8 +5,8 @@ username text,
email text,
ip_address text NOT NULL,
submit_time timestamp NOT NULL,
start_time timestamp NOT NULL,
stop_time timestamp NOT NULL,
start_time timestamp,
stop_time timestamp,
module text NOT NULL,
function text NOT NULL,
query text,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment