Commit 8ce612eb authored by Keith Jolley's avatar Keith Jolley

Create view of scheme fields against isolate id for use in query.

This can also be generated offline as a permanent table within the
database with indexes for very quick response times.  This will need to
be regenerated frequently (probably once a day).  This is necessary for
larger schemes and/or databases with many isolates.
parent f9e49891
......@@ -67,7 +67,7 @@ sub print_content {
if ( $query =~ /temp_scheme_$_\s/ ) {
try {
$self->{'datastore'}->create_temp_scheme_table($_);
$self->{'datastore'}->create_temp_isolate_scheme_table($_);
$self->{'datastore'}->create_temp_isolate_scheme_loci_view($_);
}
catch BIGSdb::DatabaseConnectionException with {
say "<div class=\"box\" id=\"statusbad\"><p>Can't copy data into temporary table - please check scheme configuration "
......
......@@ -38,7 +38,7 @@ sub print_content {
if ( $query =~ /temp_scheme_$_\s/ ) {
try {
$self->{'datastore'}->create_temp_scheme_table($_);
$self->{'datastore'}->create_temp_isolate_scheme_table($_);
$self->{'datastore'}->create_temp_isolate_scheme_loci_view($_);
}
catch BIGSdb::DatabaseConnectionException with {
say "<div class=\"box\" id=\"statusbad\"><p>Can't copy data into temporary table - please check scheme configuration "
......
......@@ -787,7 +787,7 @@ sub _create_temp_tables {
foreach (@$schemes) {
if ( $qry =~ /temp_scheme_$_\s/ || $qry =~ /ORDER BY s_$_\_/ ) {
$self->{'datastore'}->create_temp_scheme_table($_);
$self->{'datastore'}->create_temp_isolate_scheme_table($_);
$self->{'datastore'}->create_temp_isolate_scheme_loci_view($_);
}
}
}
......
......@@ -900,10 +900,10 @@ sub is_scheme_field {
return any { $_ eq $field } @$fields;
}
sub create_temp_isolate_scheme_table {
sub create_temp_isolate_scheme_loci_view {
my ( $self, $scheme_id ) = @_;
my $view = $self->{'system'}->{'view'};
my $table = "temp_$view\_scheme_$scheme_id";
my $table = "temp_$view\_scheme_loci_$scheme_id";
#Test if view already exists
my $exists = $self->run_simple_query( "SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_name=?)", $table );
......@@ -929,6 +929,64 @@ sub create_temp_isolate_scheme_table {
return $table;
}
sub create_temp_isolate_scheme_fields_view {
my ( $self, $scheme_id, $options ) = @_;
#Create view containing isolate_id and scheme_fields.
#This view can instead be created as a persistent indexed table using the update_scheme_cache.pl script.
#This should be done once the scheme size/number of isolates results in a slowdown of queries.
$options = {} if ref $options ne 'HASH';
my $view = $self->{'system'}->{'view'};
my $table = "temp_$view\_scheme_fields_$scheme_id";
if ( !$options->{'cache'} ) {
my $exists = $self->run_simple_query( "SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_name=?)", $table )->[0];
return $table if $exists;
}
my $scheme_table = $self->create_temp_scheme_table($scheme_id);
my $loci_table = $self->create_temp_isolate_scheme_loci_view($scheme_id);
my $scheme_loci = $self->get_scheme_loci($scheme_id);
my $scheme_fields = $self->get_scheme_fields($scheme_id);
my $scheme_info = $self->get_scheme_info($scheme_id);
my @temp;
foreach my $locus (@$scheme_loci) {
( my $cleaned = $locus ) =~ s/'/\\'/g;
( my $named = $locus ) =~ s/'/_PRIME_/g;
#Use correct cast to ensure that database indexes are used.
my $locus_info = $self->get_locus_info($locus);
if ( $scheme_info->{'allow_missing_loci'} ) {
push @temp, "$scheme_table.$named=ANY($loci_table.$named || 'N'::text)";
} else {
if ( $locus_info->{'allele_id_format'} eq 'integer' ) {
push @temp, "$scheme_table.$named=ANY(CAST($loci_table.$named AS int[]))";
} else {
push @temp, "$scheme_table.$named=ANY($loci_table.$named)";
}
}
}
local $" = ",$scheme_table.";
s/'/\\'/g foreach @$scheme_fields;
my $table_type = 'TEMP VIEW';
if ( $options->{'cache'} ) {
$table_type = 'TABLE';
eval { $self->{'db'}->do("DROP TABLE IF EXISTS $table") };
$logger->error($@) if $@;
}
my $qry =
"CREATE $table_type $table AS SELECT $loci_table.id,$scheme_table.@$scheme_fields FROM $loci_table LEFT JOIN $scheme_table ON ";
local $" = ' AND ';
$qry .= "@temp";
eval { $self->{'db'}->do($qry) };
$logger->error($@) if $@;
if ( $options->{'cache'} ) {
foreach my $field (@$scheme_fields) {
$self->{'db'}->do("CREATE INDEX i_$table\_$field ON $table ($field)");
}
$self->{'db'}->commit;
}
return $table;
}
sub create_temp_scheme_table {
my ( $self, $id ) = @_;
my $scheme_info = $self->get_scheme_info($id);
......@@ -937,18 +995,17 @@ sub create_temp_scheme_table {
$logger->error("No scheme database for scheme $id");
throw BIGSdb::DatabaseConnectionException("Database does not exist");
}
my $temp_table = "temp_scheme_$id";
#Test if table already exists
my ($exists) =
$self->run_simple_query( "SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_name=?)", "temp_scheme_$id" );
my ($exists) = $self->run_simple_query( "SELECT EXISTS(SELECT * FROM information_schema.tables WHERE table_name=?)", $temp_table );
if ( $exists->[0] ) {
$logger->debug("Table already exists");
return;
return $temp_table;
}
my $fields = $self->get_scheme_fields($id);
my $loci = $self->get_scheme_loci($id);
my $temp_table = "temp_scheme_$id";
my $create = "CREATE TEMP TABLE $temp_table (";
my $fields = $self->get_scheme_fields($id);
my $loci = $self->get_scheme_loci($id);
my $create = "CREATE TEMP TABLE $temp_table (";
my @table_fields;
foreach (@$fields) {
my $type = $self->get_scheme_field_info( $id, $_ )->{'type'};
......
......@@ -965,7 +965,7 @@ sub _modify_query_for_filters {
my $scheme_fields = $self->{'datastore'}->get_scheme_fields($scheme_id);
foreach my $field (@$scheme_fields) {
if ( defined $q->param("scheme_$scheme_id\_$field\_list") && $q->param("scheme_$scheme_id\_$field\_list") ne '' ) {
my $temp_table = $self->{'datastore'}->create_temp_isolate_scheme_table($scheme_id);
my $temp_table = $self->{'datastore'}->create_temp_isolate_scheme_loci_view($scheme_id);
my $value = $q->param("scheme_$scheme_id\_$field\_list");
$value =~ s/'/\\'/g;
my $clause;
......@@ -1137,7 +1137,8 @@ sub _modify_query_for_designations {
push @$errors_ref, "$operator is not a valid operator.";
next;
}
$field = "scheme_$scheme_id\.$field";
my $isolate_scheme_field_view = $self->{'datastore'}->create_temp_isolate_scheme_fields_view($scheme_id);
$field = "$isolate_scheme_field_view.$field";
my $scheme_loci = $self->{'datastore'}->get_scheme_loci($scheme_id);
my ( %cleaned, %named, %scheme_named );
foreach my $locus (@$scheme_loci) {
......@@ -1145,52 +1146,44 @@ sub _modify_query_for_designations {
( $named{$locus} = $locus ) =~ s/'/_PRIME_/g;
( $scheme_named{$locus} = $locus ) =~ s/'/_PRIME_/g;
}
my $temp_table = $self->{'datastore'}->create_temp_isolate_scheme_table($scheme_id);
my @temp;
foreach my $locus (@$scheme_loci) {
push @temp,
$self->get_scheme_locus_query_clause( $scheme_id, $temp_table, $locus, $scheme_named{$locus}, $named{$locus} );
}
local $" = ' AND ';
my $joined_query = "SELECT $temp_table.id FROM $temp_table LEFT JOIN temp_scheme_$scheme_id AS scheme_$scheme_id ON @temp";
my $temp_qry = "SELECT $isolate_scheme_field_view.id FROM $isolate_scheme_field_view";
$text =~ s/'/\\'/g;
if ( $operator eq 'NOT' ) {
if ( $text eq 'null' ) {
push @sqry, "($view.id NOT IN ($joined_query WHERE $field is null) AND $view.id IN ($joined_query))";
push @sqry, "($view.id NOT IN ($temp_qry WHERE $field IS NULL) AND $view.id IN ($temp_qry))";
} else {
push @sqry,
$scheme_field_info->{'type'} eq 'integer'
? "($view.id NOT IN ($joined_query WHERE CAST($field AS text)= E'$text' AND $view.id IN ($joined_query)))"
: "($view.id NOT IN ($joined_query WHERE upper($field)=upper(E'$text') AND $view.id IN ($joined_query)))";
? "($view.id NOT IN ($temp_qry WHERE CAST($field AS text)= E'$text' AND $view.id IN ($temp_qry)))"
: "($view.id NOT IN ($temp_qry WHERE upper($field)=upper(E'$text') AND $view.id IN ($temp_qry)))";
}
} elsif ( $operator eq "contains" ) {
push @sqry,
$scheme_field_info->{'type'} eq 'integer'
? "($view.id IN ($joined_query WHERE CAST($field AS text) ~* E'$text'))"
: "($view.id IN ($joined_query WHERE $field ~* E'$text'))";
? "($view.id IN ($temp_qry WHERE CAST($field AS text) ~* E'$text'))"
: "($view.id IN ($temp_qry WHERE $field ~* E'$text'))";
} elsif ( $operator eq "starts with" ) {
push @sqry,
$scheme_field_info->{'type'} eq 'integer'
? "($view.id IN ($joined_query WHERE CAST($field AS text) LIKE E'$text\%'))"
: "($view.id IN ($joined_query WHERE $field ILIKE E'$text\%'))";
? "($view.id IN ($temp_qry WHERE CAST($field AS text) LIKE E'$text\%'))"
: "($view.id IN ($temp_qry WHERE $field ILIKE E'$text\%'))";
} elsif ( $operator eq "ends with" ) {
push @sqry,
$scheme_field_info->{'type'} eq 'integer'
? "($view.id IN ($joined_query WHERE CAST($field AS text) LIKE E'\%$text'))"
: "($view.id IN ($joined_query WHERE $field ILIKE E'\%$text'))";
? "($view.id IN ($temp_qry WHERE CAST($field AS text) LIKE E'\%$text'))"
: "($view.id IN ($temp_qry WHERE $field ILIKE E'\%$text'))";
} elsif ( $operator eq "NOT contain" ) {
push @sqry,
$scheme_field_info->{'type'} eq 'integer'
? "($view.id IN ($joined_query WHERE CAST($field AS text) !~* E'$text'))"
: "($view.id IN ($joined_query WHERE $field !~* E'$text'))";
? "($view.id IN ($temp_qry WHERE CAST($field AS text) !~* E'$text'))"
: "($view.id IN ($temp_qry WHERE $field !~* E'$text'))";
} elsif ( $operator eq '=' ) {
if ( $text eq 'null' ) {
push @lqry_blank, "($view.id IN ($joined_query WHERE $field is null) OR $view.id NOT IN ($joined_query))";
push @lqry_blank, "($view.id IN ($temp_qry WHERE $field IS NULL) OR $view.id NOT IN ($temp_qry))";
} else {
push @sqry,
$scheme_field_info->{'type'} eq 'text'
? "($view.id IN ($joined_query WHERE upper($field)=upper(E'$text')))"
: "($view.id IN ($joined_query WHERE $field=E'$text'))";
push @sqry, $scheme_field_info->{'type'} eq 'text'
? "($view.id IN ($temp_qry WHERE upper($field)=upper(E'$text')))"
: "($view.id IN ($temp_qry WHERE $field=E'$text'))";
}
} else {
if ( $text eq 'null' ) {
......@@ -1198,9 +1191,9 @@ sub _modify_query_for_designations {
next;
}
if ( $scheme_field_info->{'type'} eq 'integer' ) {
push @sqry, "($view.id IN ($joined_query WHERE CAST($field AS int) $operator E'$text'))";
push @sqry, "($view.id IN ($temp_qry WHERE CAST($field AS int) $operator E'$text'))";
} else {
push @sqry, "($view.id IN ($joined_query WHERE $field $operator E'$text'))";
push @sqry, "($view.id IN ($temp_qry WHERE $field $operator E'$text'))";
}
}
}
......
......@@ -234,7 +234,7 @@ sub _run_isolate_query {
$fieldtype = 'scheme_field';
my $scheme_loci = $self->{'datastore'}->get_scheme_loci($scheme_id);
my $scheme_info = $self->{'datastore'}->get_scheme_info($scheme_id);
my $temp_table = $self->{'datastore'}->create_temp_isolate_scheme_table($scheme_id);
my $temp_table = $self->{'datastore'}->create_temp_isolate_scheme_loci_view($scheme_id);
my ( %cleaned, %named, %scheme_named );
foreach my $locus (@$scheme_loci) {
......
#!/usr/bin/perl
#Create scheme profile caches in an isolate database
#
#Written by Keith Jolley
#Copyright (c) 2014, University of Oxford
#E-mail: keith.jolley@zoo.ox.ac.uk
#
#This file is part of Bacterial Isolate Genome Sequence Database (BIGSdb).
#
#BIGSdb is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation, either version 3 of the License, or
#(at your option) any later version.
#
#BIGSdb is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#GNU General Public License for more details.
#
#You should have received a copy of the GNU General Public License
#along with BIGSdb. If not, see <http://www.gnu.org/licenses/>.
package BIGSdb::Offline::UpdateSchemeCaches;
use strict;
use warnings;
use 5.010;
use parent qw(BIGSdb::Offline::Script);
sub run_script {
my ($self) = @_;
die "No connection to database (check logs).\n" if !defined $self->{'db'};
die "This script can only be run against an isolate database.\n" if ( $self->{'system'}->{'dbtype'} // '' ) ne 'isolates';
my $schemes = $self->{'datastore'}->run_list_query("SELECT id FROM schemes WHERE dbase_name IS NOT NULL AND dbase_table IS NOT NULL");
foreach my $scheme_id (@$schemes) {
if ( !$self->{'options'}->{'q'} ) {
my $scheme_info = $self->{'datastore'}->get_scheme_info($scheme_id);
say "Updating scheme $scheme_id cache ($scheme_info->{'description'})";
}
$self->{'datastore'}->create_temp_isolate_scheme_fields_view( $scheme_id, { cache => 1 } );
}
return;
}
1;
......@@ -1349,7 +1349,7 @@ sub _create_join_sql_for_scheme {
my $scheme_field = $2;
my $loci = $self->{'datastore'}->get_scheme_loci($scheme_id);
my $scheme_info = $self->{'datastore'}->get_scheme_info($scheme_id);
my $isolate_scheme_table = $self->{'datastore'}->create_temp_isolate_scheme_table($scheme_id);
my $isolate_scheme_table = $self->{'datastore'}->create_temp_isolate_scheme_loci_view($scheme_id);
my %named;
foreach my $locus (@$loci) {
( $named{$locus} = $locus ) =~ s/'/_PRIME_/g;
......
......@@ -185,7 +185,7 @@ sub create_temp_tables {
foreach (@$schemes) {
if ( $qry =~ /temp_scheme_$_\s/ || $qry =~ /ORDER BY s_$_\_/ ) {
$self->{'datastore'}->create_temp_scheme_table($_);
$self->{'datastore'}->create_temp_isolate_scheme_table($_);
$self->{'datastore'}->create_temp_isolate_scheme_loci_view($_);
}
}
}
......
......@@ -132,7 +132,7 @@ sub _do_analysis {
try {
$scheme_fields_qry = $self->_get_scheme_fields_sql($scheme_id);
$self->{'datastore'}->create_temp_scheme_table($scheme_id);
$self->{'datastore'}->create_temp_isolate_scheme_table($scheme_id);
$self->{'datastore'}->create_temp_isolate_scheme_loci_view($scheme_id);
}
catch BIGSdb::DatabaseConnectionException with {
say "<div class=\"box\" id=\"statusbad\"><p>The database for scheme $scheme_id is not accessible. This may be a "
......@@ -354,7 +354,7 @@ sub _print_scheme_table {
try {
$scheme_fields_qry = $self->_get_scheme_fields_sql($scheme_id);
$self->{'datastore'}->create_temp_scheme_table($scheme_id);
$self->{'datastore'}->create_temp_isolate_scheme_table($scheme_id);
$self->{'datastore'}->create_temp_isolate_scheme_loci_view($scheme_id);
}
catch BIGSdb::DatabaseConnectionException with {
say "</table>\n</div>";
......
......@@ -96,6 +96,7 @@ sub get_grouped_fields {
sub get_scheme_locus_query_clause {
my ( $self, $scheme_id, $table, $locus, $scheme_named, $named ) = @_;
$logger->logcarp("QueryPage::get_scheme_locus_query_clause is deprecated"); #TODO Remove
my $scheme_info = $self->{'datastore'}->get_scheme_info($scheme_id);
#Use correct cast to ensure that database indexes are used.
......
......@@ -51,7 +51,7 @@ sub paged_display {
foreach (@$schemes) {
if ( $qry =~ /temp_scheme_$_\s/ || $qry =~ /ORDER BY s_$_\_/ ) {
$self->{'datastore'}->create_temp_scheme_table($_);
$self->{'datastore'}->create_temp_isolate_scheme_table($_);
$self->{'datastore'}->create_temp_isolate_scheme_loci_view($_);
}
}
}
......
#!/usr/bin/perl -T
#Update tables of scheme field values linked to isolate
#Written by Keith Jolley
#Copyright (c) 2014, University of Oxford
#E-mail: keith.jolley@zoo.ox.ac.uk
#
#This file is part of Bacterial Isolate Genome Sequence Database (BIGSdb).
#
#BIGSdb is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation, either version 3 of the License, or
#(at your option) any later version.
#
#BIGSdb is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#GNU General Public License for more details.
#
#You should have received a copy of the GNU General Public License
#along with BIGSdb. If not, see <http://www.gnu.org/licenses/>.
use strict;
use warnings;
use 5.010;
###########Local configuration################################
use constant {
CONFIG_DIR => '/etc/bigsdb',
LIB_DIR => '/usr/local/lib',
DBASE_CONFIG_DIR => '/etc/bigsdb/dbases',
};
#######End Local configuration################################
use lib (LIB_DIR);
use Getopt::Std;
use BIGSdb::Offline::UpdateSchemeCaches;
my %opts;
getopts( 'd:q', \%opts );
if (!$opts{'d'}){
say "Usage: update_scheme_caches.pl -d <database configuration>";
exit;
}
BIGSdb::Offline::UpdateSchemeCaches->new(
{
config_dir => CONFIG_DIR,
lib_dir => LIB_DIR,
dbase_config_dir => DBASE_CONFIG_DIR,
options => \%opts,
instance => $opts{'d'},
}
);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment