Skip to content

GitLab

BIGSdb
BIGSdb
Commit 5d64fb0b authored by Keith Jolley's avatar Keith Jolley
Browse files
Options

Merge branch 'classification_groups' into develop

parents 2b761674 a804fdba
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 3448 additions and 1141 deletions
Upgrading_README.txt
View file @ 5d64fb0b
Important notes about upgrading
-------------------------------
===============================
Version 1.1: Offline job manager - set up new user account and cron job.
Version 1.2: Change of isolate database structure. New package 'exonerate'
required.
......@@ -19,6 +18,7 @@ Version 1.11: Change of authentication database structure.
required.
Version 1.12: Change of authentication and seqdef database structures.
Version 1.13: Change of seqdef and isolate database structures.
Version 1.14: Change of seqdef and isolate database structures.
Details can be found below.
......@@ -286,3 +286,38 @@ the sql/upgrade directory, against your seqdef database.
The isolate database has also been modified to support future functionality.
Please run isolatedb_v1.13.sql against your isolate databases.
Version 1.14
------------
There are large-scale changes to both the sequence definition and isolate
database structures to support cgMLST schemes with primary key indexing. The
previous method of caching scheme profiles did not support schemes with a ST
value and more than ~1600 loci due to PostgreSQL column limits. This did not
apply to schemes that were simply a collection of loci. Even before this limit
was reached, such large schemes would suffer performance penalties. The new
structure has no such limits.
Classification groups for cgMLST schemes are also introduced with this version.
This has necessitated a number of new tables in the database schemas.
Please run the seqdefdb_v1.14.sql against sequence definition databases and
isolatedb_v1.14.sql against isolate databases. These can be found in the
sql/upgrade directory.
Any scheme views set up in the sequence definition database named 'scheme_1'
etc. can be dropped as these are no longer used. To do this log in to the
database using psql, and type 'DROP VIEW scheme_1;'. Do this for each of the
scheme views.
If you are planning to use cgMLST schemes, you should ensure that your database
configuration has a temp_buffers setting of at least 64MB. This can be done by
editing bigsdb.conf and setting:
temp_buffers=64
Alternatively, this can be set globally in the postgresql.conf file (probably
/etc/postgresql/9.x/main/postgresql.conf) with the following line:
SET temp_buffers=64MB
Without this, the database engine is likely to run out of memory during cache
renewal.
conf/bigsdb.conf
View file @ 5d64fb0b
......@@ -62,6 +62,10 @@ doclink=http://bigsdb.readthedocs.io/en/latest
#Maximum file upload size (MB)
max_upload_size=32
#Set PostgreSQL temporary buffers (MB) - likely to be needed if using cgMLST
#schemes. Alternatively this can be set globally in postgresql.conf.
temp_buffers=64
###REST INTERFACE###########
#Name of database describing resources for REST interface
......
lib/BIGSdb/Application.pm
View file @ 5d64fb0b
......@@ -27,6 +27,7 @@ use BIGSdb::AuthorizeClientPage;
use BIGSdb::BatchProfileQueryPage;
use BIGSdb::BIGSException;
use BIGSdb::ChangePasswordPage;
use BIGSdb::ClassificationScheme;
use BIGSdb::CombinationQueryPage;
use BIGSdb::CurateSubmissionExcelPage;
use BIGSdb::CustomizePage;
......@@ -303,7 +304,7 @@ sub read_config_file {
blast+_path blast_threads muscle_path max_muscle_mb mafft_path mafft_threads mogrify_path ipcress_path
splitstree_path reference refdb ref_db chartdirector disable_updates disable_update_message intranet
debug results_deleted_days cache_days doclink rest_behind_proxy bcrypt_cost curate_script query_script
submissions_deleted_days smtp_server stylesheet domain max_upload_size)
submissions_deleted_days smtp_server stylesheet domain max_upload_size temp_buffers)
)
{
$self->{'config'}->{$param} = $config->{_}->{$param};
......
lib/BIGSdb/BatchProfileQueryPage.pm
View file @ 5d64fb0b
......@@ -83,11 +83,12 @@ sub print_content {
print qq(<th>$cleaned</th>);
}
local $" = ',';
my $scheme_view =
$self->{'datastore'}->materialized_view_exists($scheme_id) ? "mv_scheme_$scheme_id" : "scheme_$scheme_id";
my $qry = "SELECT @$scheme_fields FROM $scheme_view WHERE ";
my @cleaned_loci_db = @$loci;
$_ =~ s/'/_PRIME_/gx foreach @cleaned_loci_db;
my $scheme_warehouse = "mv_scheme_$scheme_id";
my $qry = "SELECT @$scheme_fields FROM $scheme_warehouse WHERE ";
my @cleaned_loci_db;
foreach my $locus (@$loci) {
push @cleaned_loci_db, $self->{'datastore'}->get_scheme_warehouse_locus_name( $scheme_id, $locus );
}
my $set_id = $self->get_set_id;
my $scheme_info = $self->{'datastore'}->get_scheme_info( $scheme_id, { set_id => $set_id } );
local $" = $scheme_info->{'allow_missing_loci'} ? q[ IN (?, 'N')) AND (] : q[=?) AND (];
......@@ -164,10 +165,9 @@ sub print_content {
return;
}
#Generate example data file for batch profile query
#Get up to 15 random profiles from the database
sub _print_examples {
#Generate example data file for batch profile query
#Get up to 15 random profiles from the database
my ($self) = @_;
my $q = $self->{'cgi'};
my $scheme_id = $q->param('scheme_id');
......@@ -183,22 +183,23 @@ sub _print_examples {
print "Invalid scheme selected.\n";
return;
}
my $loci = $self->{'datastore'}->get_scheme_loci($scheme_id);
my @cleaned_loci = @$loci;
$_ =~ s/'/_PRIME_/gx foreach @cleaned_loci;
local $" = ',';
my $scheme_view =
$self->{'datastore'}->materialized_view_exists($scheme_id) ? "mv_scheme_$scheme_id" : "scheme_$scheme_id";
my $data = $self->{'datastore'}->run_query( "SELECT @cleaned_loci FROM $scheme_view ORDER BY random() LIMIT 15",
undef, { fetch => 'all_arrayref' } );
my $loci = $self->{'datastore'}->get_scheme_loci($scheme_id);
my $scheme_warehouse = "mv_scheme_$scheme_id";
my $data = $self->{'datastore'}->run_query( "SELECT profile FROM $scheme_warehouse ORDER BY random() LIMIT 15",
undef, { fetch => 'col_arrayref' } );
if ( !@$data ) {
say q(No profiles have yet been defined for this scheme.);
return;
}
my $i = 1;
my $indices = $self->{'datastore'}->get_scheme_locus_indices($scheme_id);
local $" = "\t";
my $i = 1;
foreach my $profile (@$data) {
say qq(isolate_$i\t@$profile);
my @alleles;
foreach my $locus (@$loci) {
push @alleles, $profile->[ $indices->{$locus} ];
}
say qq(isolate_$i\t@alleles);
$i++;
}
return;
......
lib/BIGSdb/ClassificationScheme.pm 0 → 100644
View file @ 5d64fb0b
#Written by Keith Jolley
#Copyright (c) 2016, University of Oxford
#E-mail: keith.jolley@zoo.ox.ac.uk
#
#This file is part of Bacterial Isolate Genome Sequence Database (BIGSdb).
#
#BIGSdb is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation, either version 3 of the License, or
#(at your option) any later version.
#
#BIGSdb is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#GNU General Public License for more details.
#
#You should have received a copy of the GNU General Public License
#along with BIGSdb. If not, see <http://www.gnu.org/licenses/>.
package BIGSdb::ClassificationScheme;
use strict;
use warnings;
use 5.010;
use Log::Log4perl qw(get_logger);
my $logger = get_logger('BIGSdb.Scheme');
sub new { ## no critic (RequireArgUnpacking)
my $class = shift;
my $self = {@_};
$self->{'sql'} = {};
bless( $self, $class );
$logger->info("Classification scheme#$self->{'id'} ($self->{'name'}) set up.");
return $self;
}
sub get_db {
my ($self) = @_;
return $self->{'db'} if $self->{'db'};
return;
}
1;
\ No newline at end of file
lib/BIGSdb/CombinationQueryPage.pm
View file @ 5d64fb0b
#Written by Keith Jolley
#Copyright (c) 2010-2015, University of Oxford
#Copyright (c) 2010-2016, University of Oxford
#E-mail: keith.jolley@zoo.ox.ac.uk
#
#This file is part of Bacterial Isolate Genome Sequence Database (BIGSdb).
......@@ -60,7 +60,7 @@ sub print_content {
if $scheme_id
&& $self->is_scheme_invalid( $scheme_id, { with_pk => $with_pk, all_loci => $all_loci } );
$self->print_scheme_section( { with_pk => $with_pk, all_loci => $all_loci } );
$scheme_id = $q->param('scheme_id'); #Will be set by scheme section method
$scheme_id = $q->param('scheme_id'); #Will be set by scheme section method
$scheme_id = 0 if !BIGSdb::Utils::is_int($scheme_id);
$self->_print_query_interface($scheme_id);
}
......@@ -97,20 +97,12 @@ sub _autofill {
push @errors, 'Error retrieving information from remote database - check configuration.';
};
} else {
my @cleaned_loci = @$loci;
foreach my $locus (@cleaned_loci) {
$locus =~ s/'/_PRIME_/gx;
}
local $" = ',';
my $scheme_view =
$self->{'datastore'}->materialized_view_exists($scheme_id)
? "mv_scheme_$scheme_id"
: "scheme_$scheme_id";
my $qry = "SELECT @cleaned_loci FROM $scheme_view WHERE $primary_key=?";
my $loci_values = $self->{'datastore'}->run_query( $qry, $pk_value, { fetch => 'row_hashref' } );
my $scheme_warehouse = "mv_scheme_$scheme_id";
my $qry = "SELECT profile FROM $scheme_warehouse WHERE $primary_key=?";
my $profile = $self->{'datastore'}->run_query( $qry, $pk_value );
my $locus_indices = $self->{'datastore'}->get_scheme_locus_indices($scheme_id);
foreach my $locus (@$loci) {
( my $cleaned = $locus ) =~ s/'/_PRIME_/gx;
$q->param( "l_$locus", $loci_values->{ lc($cleaned) } );
$q->param( "l_$locus", $profile->[ $locus_indices->{$locus} ] );
}
}
}
......@@ -195,7 +187,12 @@ sub _print_query_interface {
say qq(<tr>$header_row</tr>);
say qq(<tr>$form_row</tr>);
say q(</table></fieldset>);
if ($primary_key) {
if (
$primary_key
&& ( ( $self->{'system'}->{'dbtype'} eq 'isolates' && $scheme_info->{'dbase_table'} )
|| $self->{'system'}->{'dbtype'} eq 'sequences' )
)
{
my $remote = $self->{'system'}->{'dbtype'} eq 'isolates' ? ' by searching remote database' : '';
say qq(<fieldset id="autofill_fieldset" style="float:left"><legend>Autofill profile$remote</legend><ul>);
my $first = 1;
......@@ -264,22 +261,12 @@ sub _print_query_interface {
return;
}
sub _get_scheme_view {
my ( $self, $scheme_id ) = @_;
if ( $self->{'system'}->{'dbtype'} eq 'sequences' ) {
return $self->{'datastore'}->materialized_view_exists($scheme_id)
? "mv_scheme_$scheme_id"
: "scheme_$scheme_id";
}
return;
}
sub _generate_query {
my ( $self, $scheme_id ) = @_;
my $scheme_info = $self->{'datastore'}->get_scheme_info( $scheme_id, { get_pk => 1 } );
my $scheme_view = $self->_get_scheme_view($scheme_id);
my $q = $self->{'cgi'};
my @params = $q->param;
my $scheme_info = $self->{'datastore'}->get_scheme_info( $scheme_id, { get_pk => 1 } );
my $scheme_warehouse = "mv_scheme_$scheme_id";
my $q = $self->{'cgi'};
my @params = $q->param;
my @errors;
my $msg;
my $qry;
......@@ -343,7 +330,7 @@ sub _generate_query {
if (@lqry) {
local $" = ' OR ';
my $required_matches = $q->param('matches_list');
$required_matches = scalar @lqry if $required_matches == @loci;
$required_matches = @lqry if $required_matches == @loci;
my $lqry;
if ( $self->{'system'}->{'dbtype'} eq 'isolates' ) {
$lqry = "(SELECT DISTINCT($view.id) FROM $view LEFT JOIN allele_designations ON "
......@@ -353,7 +340,7 @@ sub _generate_query {
'(SELECT DISTINCT(profiles.profile_id) FROM profiles LEFT JOIN profile_members ON '
. 'profiles.profile_id=profile_members.profile_id AND profiles.scheme_id=profile_members.scheme_id '
. "AND profile_members.scheme_id=$scheme_id WHERE "
. "$scheme_view.$scheme_info->{'primary_key'}=profiles.profile_id AND (@lqry)";
. "$scheme_warehouse.$scheme_info->{'primary_key'}=profiles.profile_id AND (@lqry)";
}
if ( $required_matches == 0 ) { #Find out the greatest number of matches
my $match = $self->_find_best_match_count( $scheme_id, \@lqry );
......@@ -369,7 +356,7 @@ sub _generate_query {
$qry =
$self->{'system'}->{'dbtype'} eq 'isolates'
? "SELECT * FROM $view WHERE $view.id IN $lqry"
: "SELECT * FROM $scheme_view WHERE EXISTS $lqry";
: "SELECT * FROM $scheme_warehouse WHERE EXISTS $lqry";
}
$self->_modify_qry_by_filters( \$qry );
$self->_add_query_ordering( \$qry, $scheme_id );
......
lib/BIGSdb/ConfigCheckPage.pm
View file @ 5d64fb0b
......@@ -38,6 +38,7 @@ sub print_content {
if ( $self->{'system'}->{'dbtype'} eq 'isolates' ) {
$self->_check_locus_databases;
$self->_check_scheme_databases;
$self->_check_classification_scheme_databases;
} else {
$self->_check_client_databases;
}
......@@ -222,6 +223,61 @@ sub _check_scheme_databases {
return;
}
sub _check_classification_scheme_databases {
my ($self) = @_;
say q(<div class="box resultstable">);
say q(<h2>Classification scheme databases</h2>);
my $cschemes =
$self->{'datastore'}
->run_query( 'SELECT id FROM classification_schemes ORDER BY id', undef, { fetch => 'col_arrayref' } );
my $td = 1;
if (@$cschemes) {
say q(<div class="scrollable"><table class="resultstable"><tr><th>Classification scheme</th>)
. q(<th>Scheme</th><th>Database</th><th>Host</th><th>Port</th><th>Table</th><th>Database accessible</th>)
. q(<th>Seqdef classification scheme id</th><th>Classification data</th></tr>);
foreach my $cscheme_id (@$cschemes) {
my $cscheme_info = $self->{'datastore'}->get_classification_scheme_info($cscheme_id);
my $scheme_info = $self->{'datastore'}->get_scheme_info( $cscheme_info->{'scheme_id'} );
$cscheme_info->{'name'} =~ s/&/&amp;/gx;
$scheme_info->{'description'} =~ s/&/&amp;/gx;
print qq(<tr class="td$td"><td>$cscheme_info->{'id'}: $cscheme_info->{'name'}</td><td>)
. ("$scheme_info->{'id'}: $scheme_info->{'description'}")
. q(</td><td>)
. ( $scheme_info->{'dbase_name'} // q() )
. q(</td><td>)
. ( $scheme_info->{'dbase_host'} // $self->{'system'}->{'host'} )
. q(</td><td>)
. ( $scheme_info->{'dbase_port'} // $self->{'system'}->{'port'} )
. q(</td><td>)
. ( $scheme_info->{'dbase_table'} // q() )
. q(</td><td>);
if ( $self->{'datastore'}->get_classification_scheme($cscheme_id)->get_db ) {
print q(<span class="statusgood fa fa-check"></span>);
} else {
print q(<span class="statusbad fa fa-times"></span>);
}
print qq(</td><td>$cscheme_info->{'seqdef_cscheme_id'}</td><td>);
my $seqdef_db = $self->{'datastore'}->get_scheme( $cscheme_info->{'scheme_id'} )->get_db;
my $exists = $self->{'datastore'}->run_query(
'SELECT EXISTS(SELECT * FROM classification_group_profiles WHERE cg_scheme_id=?)',
$cscheme_info->{'seqdef_cscheme_id'},
{ db => $seqdef_db }
);
if ($exists) {
print q(<span class="statusgood fa fa-check"></span>);
} else {
print q(<span class="statusbad fa fa-times"></span>);
}
my $classification_data = say q(</td></tr>);
$td = $td == 1 ? 2 : 1;
}
say q(</table></div></div>);
} else {
say q(<p>No schemes with databases defined.</p>);
}
return;
}
sub _check_client_databases {
my ($self) = @_;
my $client_dbs =
......
lib/BIGSdb/ConfigRepairPage.pm
View file @ 5d64fb0b
#Written by Keith Jolley
#Copyright (c) 2011-2015, University of Oxford
#Copyright (c) 2011-2016, University of Oxford
#E-mail: keith.jolley@zoo.ox.ac.uk
#
#This file is part of Bacterial Isolate Genome Sequence Database (BIGSdb).
......@@ -41,11 +41,10 @@ sub print_content {
return;
}
say q(<div class="box" id="queryform">);
say q(<h2>Rebuild scheme views</h2>);
say q(<p>Scheme views can become damaged if the database is modifed outside of the web interface. This )
say q(<h2>Rebuild scheme warehouse tables</h2>);
say q(<p>Schemes can become damaged if the database is modifed outside of the web interface. This )
. q(is especially likely if loci that belong to schemes are renamed.</p>);
say q(<p>As materialized views are enabled for this database, these will also be (re)created.</p>)
if $self->{'system'}->{'materialized_views'} && $self->{'system'}->{'materialized_views'} eq 'yes';
say q(<p>Warehouse tables will be (re)created.</p>);
my $schemes = $self->{'datastore'}->run_query(
'SELECT id,description FROM schemes WHERE id IN (SELECT scheme_id FROM scheme_fields '
. 'WHERE primary_key) AND id IN (SELECT scheme_id FROM scheme_members) ORDER BY id',
......@@ -78,8 +77,7 @@ sub print_content {
sub _rebuild {
my ( $self, $scheme_id ) = @_;
eval {
$self->drop_scheme_view($scheme_id);
$self->create_scheme_view($scheme_id);
$self->{'datastore'}->run_query("SELECT initiate_scheme_warehouse($scheme_id)");
};
if ($@) {
$logger->error($@);
......
lib/BIGSdb/Constants.pm
View file @ 5d64fb0b
......@@ -24,11 +24,10 @@ our @EXPORT_OK;
our %EXPORT_TAGS;
#Limits
use constant MAX_POSTGRES_COLS => 1664;
use constant MAX_SPLITS_TAXA => 200;
use constant MAX_MUSCLE_MB => 4 * 1024; #4GB
push @EXPORT_OK, qw(MAX_POSTGRES_COLS MAX_SPLITS_TAXA MAX_MUSCLE_MB);
$EXPORT_TAGS{'limits'} = [qw(MAX_POSTGRES_COLS MAX_SPLITS_TAXA MAX_MUSCLE_MB)];
push @EXPORT_OK, qw(MAX_SPLITS_TAXA MAX_MUSCLE_MB);
$EXPORT_TAGS{'limits'} = [qw(MAX_SPLITS_TAXA MAX_MUSCLE_MB)];
#Methods
use constant SEQ_METHODS =>
......
lib/BIGSdb/CurateAddPage.pm
View file @ 5d64fb0b
......@@ -25,7 +25,7 @@ use BIGSdb::Utils;
use Log::Log4perl qw(get_logger);
my $logger = get_logger('BIGSdb.Page');
use List::MoreUtils qw(any none uniq);
use BIGSdb::Constants qw(ALLELE_FLAGS LOCUS_PATTERN DIPLOID HAPLOID MAX_POSTGRES_COLS DATABANKS);
use BIGSdb::Constants qw(ALLELE_FLAGS LOCUS_PATTERN DIPLOID HAPLOID DATABANKS);
use constant SUCCESS => 1;
sub initiate {
......@@ -191,19 +191,6 @@ sub _check_locus_descriptions {
return;
}
sub _too_many_cols {
my ( $self, $has_pk, $field_count ) = @_;
if ( $has_pk && $field_count > MAX_POSTGRES_COLS ) {
say q(<div class="box" id="statusbad"><p>Indexed scheme tables are limited to a maximum of )
. MAX_POSTGRES_COLS
. qq( columns - yours would have $field_count. This is a limitation of PostgreSQL, but it's )
. q(not really sensible to have indexed schemes (those with a primary key field) to have so )
. q(many fields. Update failed.</p></div);
return 1;
}
return;
}
sub _insert {
my ( $self, $table, $newdata ) = @_;
my $q = $self->{'cgi'};
......@@ -213,7 +200,7 @@ sub _insert {
@problems = $self->check_record( $table, $newdata );
my $extra_inserts = [];
my %check_tables = map { $_ => 1 } qw(accession loci locus_aliases locus_descriptions profile_refs scheme_fields
scheme_group_group_members sequences sequence_bin sequence_refs retired_profiles);
scheme_group_group_members sequences sequence_bin sequence_refs retired_profiles classification_group_fields);
if (
defined $newdata->{'isolate_id'}
......@@ -247,20 +234,8 @@ sub _insert {
}
if ( $self->{'system'}->{'dbtype'} eq 'sequences' ) {
my %modifies_scheme = map { $_ => 1 } qw(scheme_members scheme_fields);
if ( $table eq 'schemes' ) {
$self->create_scheme_view( $newdata->{'id'} );
} elsif ( $modifies_scheme{$table} ) {
my $scheme_fields = $self->{'datastore'}->get_scheme_fields( $newdata->{'scheme_id'} );
my $scheme_loci = $self->{'datastore'}->get_scheme_loci( $newdata->{'scheme_id'} );
my $scheme_info = $self->{'datastore'}->get_scheme_info( $newdata->{'scheme_id'}, { get_pk => 1 } );
my $field_count = @$scheme_fields + @$scheme_loci;
if ( $self->_too_many_cols( $scheme_info->{'primary_key'}, $field_count ) ) {
$continue = 0;
} else {
$self->remove_profile_data( $newdata->{'scheme_id'} );
$self->drop_scheme_view( $newdata->{'scheme_id'} );
$self->create_scheme_view( $newdata->{'scheme_id'} );
}
if ( $modifies_scheme{$table} ) {
$self->remove_profile_data( $newdata->{'scheme_id'} );
} elsif ( $table eq 'sequences' ) {
$self->{'datastore'}->mark_cache_stale;
}
......@@ -467,7 +442,7 @@ sub _check_sequences { ## no critic (ProhibitUnusedPrivateSu
sub _check_sequence_retired {
my ( $self, $newdata, $problems ) = @_;
my $retired = $self->{'datastore'}->is_sequence_retired($newdata->{'locus'}, $newdata->{'allele_id'});
my $retired = $self->{'datastore'}->is_sequence_retired( $newdata->{'locus'}, $newdata->{'allele_id'} );
if ($retired) {
push @$problems, "Allele $newdata->{'allele_id'} has been retired.";
}
......@@ -626,7 +601,7 @@ sub _check_sequence_extended_attributes {
return;
}
sub _check_scheme_fields { ## no critic (ProhibitUnusedPrivateSubroutines) #Called by dispatch table
sub _check_scheme_fields {## no critic (ProhibitUnusedPrivateSubroutines) #Called by dispatch table
my ( $self, $newdata, $problems ) = @_;
#special case to check that only one primary key field is set for a scheme field
......@@ -639,11 +614,21 @@ sub _check_scheme_fields { ## no critic (ProhibitUnusedPrivateSubroutines) #C
#special case to check that scheme field is not called 'id' (this causes problems when joining tables)
if ( $newdata->{'field'} eq 'id' ) {
push @$problems, q(Scheme fields can not be called 'id'.);
push @$problems, q(Scheme fields cannot be called 'id'.);
}
return;
}
sub _check_classification_group_fields { ## no critic (ProhibitUnusedPrivateSubroutines) #Called by dispatch table
my ( $self, $newdata, $problems ) = @_;
#special case to check that scheme field is not called 'id' (this causes problems when joining tables)
if ( $newdata->{'field'} eq 'id' ) {
push @$problems, q(Scheme fields cannot be called 'id'.);
}
return
}
sub _check_locus_aliases { ## no critic (ProhibitUnusedPrivateSubroutines) #Called by dispatch table
my ( $self, $newdata, $problems ) = @_;
if ( $newdata->{'locus'} eq $newdata->{'alias'} ) {
......
lib/BIGSdb/CurateBatchAddPage.pm
View file @ 5d64fb0b
......@@ -24,7 +24,7 @@ use Digest::MD5 qw(md5);
use List::MoreUtils qw(any none uniq);
use parent qw(BIGSdb::CurateAddPage);
use Log::Log4perl qw(get_logger);
use BIGSdb::Constants qw(SEQ_STATUS ALLELE_FLAGS DIPLOID HAPLOID MAX_POSTGRES_COLS :submissions);
use BIGSdb::Constants qw(SEQ_STATUS ALLELE_FLAGS DIPLOID HAPLOID :submissions);
use BIGSdb::Utils;
use Error qw(:try);
my $logger = get_logger('BIGSdb.Page');
......@@ -741,6 +741,9 @@ sub _run_table_specific_field_checks {
},
retired_profiles => sub {
$self->_check_retired_profile_id($new_args);
},
classification_group_fields => sub {
$self->_check_data_scheme_fields($new_args);
}
);
$further_checks{$table}->() if $further_checks{$table};
......@@ -760,7 +763,7 @@ sub _report_check {
my ( $self, $data ) = @_;
my ( $table, $buffer, $problems, $advisories, $checked_buffer, $sender_message ) =
@{$data}{qw (table buffer problems advisories checked_buffer sender_message)};
if (!@$checked_buffer){
if ( !@$checked_buffer ) {
say q(<div class="box" id="statusbad"><h2>Import status</h2>);
say q(<p>No valid records to upload after filtering.</p></div>);
return;
......@@ -1567,10 +1570,9 @@ sub _check_retired_profile_id {
sub _upload_data {
my ( $self, $arg_ref ) = @_;
my $table = $arg_ref->{'table'};
my $locus = $arg_ref->{'locus'};
my $q = $self->{'cgi'};
my %schemes;
my $table = $arg_ref->{'table'};
my $locus = $arg_ref->{'locus'};
my $q = $self->{'cgi'};
my $records = $self->_extract_checked_records;
return if !@$records;
my $field_order = $self->_get_field_order($records);
......@@ -1601,9 +1603,6 @@ sub _upload_data {
user_status => ( $user_info->{'status'} // undef )
}
) // undef;
if ( $field eq 'scheme_id' ) {
$schemes{ $data[ $field_order->{'scheme_id'} ] } = 1;
}
}
if ( $table eq 'loci' || $table eq 'isolates' ) {
@extras = split /;/x, $data[ $field_order->{'aliases'} ]
......@@ -1689,7 +1688,6 @@ sub _upload_data {
}
}
}
$self->_regenerate_scheme_view_if_needed( $table, \%schemes );
$self->{'db'}->commit && say q(<div class="box" id="resultsheader"><p>Database updated ok</p>);
foreach (@history) {
my ( $isolate_id, $action ) = split /\|/x, $_;
......@@ -1768,32 +1766,6 @@ sub _display_update_footer_links {
return;
}
sub _regenerate_scheme_view_if_needed {
my ( $self, $table, $schemes ) = @_;
if ( ( $table eq 'scheme_members' || $table eq 'scheme_fields' )
&& $self->{'system'}->{'dbtype'} eq 'sequences' )
{
foreach my $scheme_id ( keys %$schemes ) {
my $scheme_fields = $self->{'datastore'}->get_scheme_fields($scheme_id);
my $scheme_loci = $self->{'datastore'}->get_scheme_loci($scheme_id);
my $scheme_info = $self->{'datastore'}->get_scheme_info( $scheme_id, { get_pk => 1 } );
my $field_count = @$scheme_fields + @$scheme_loci;
if ( $scheme_info->{'primary_key'} && $field_count > MAX_POSTGRES_COLS ) {
say q(<div class="box" id="statusbad"><p>Indexed scheme tables are limited to a maximum of )
. MAX_POSTGRES_COLS
. qq( columns - yours would have $field_count. This is a limitation of PostgreSQL, but it's not really sensible )
. q(to have indexed schemes (those with a primary key field) to have so many fields. Update failed.</p></div);
$self->{'db'}->rollback;
return;
}
$self->remove_profile_data($scheme_id);
$self->drop_scheme_view($scheme_id);
$self->create_scheme_view($scheme_id);
}
}
return;
}
sub _get_locus_list {
my ($self) = @_;
if ( !$self->{'cache'}->{'loci'} ) {
......@@ -2082,7 +2054,8 @@ sub _update_scheme_caches {
config_dir => $self->{'config_dir'},
lib_dir => $self->{'lib_dir'},
dbase_config_dir => $self->{'dbase_config_dir'},
instance => $self->{'system'}->{'curate_config'} // $self->{'instance'}
instance => $self->{'system'}->{'curate_config'} // $self->{'instance'},
options => { method => 'daily' }
}
);
CORE::exit(0);
......
lib/BIGSdb/CurateBatchProfileUpdatePage.pm
View file @ 5d64fb0b
......@@ -131,6 +131,7 @@ sub _check {
shift @rows; #Remove header
my $td = 1;
my $buffer;
my $indices = $self->{'datastore'}->get_scheme_locus_indices( $scheme_info->{'id'} );