2 # BioPerl module for Bio::DB::Taxonomy::flatfile
4 # Please direct questions and support issues to <bioperl-l@bioperl.org>
6 # Cared for by Chris Fields <cjfields-at-cpan-dot-org>
8 # Copyright Chris Fields
10 # You may distribute this module under the same terms as perl itself
12 # POD documentation - main docs before the code
16 Bio::DB::Taxonomy::sqlite - SQLite-based implementation of Bio::DB::Taxonomy::flatfile
20 use Bio::DB::Taxonomy;
22 my $db = Bio::DB::Taxonomy->new(-source => 'sqlite' ,
23 -nodesfile => 'nodes.dmp',
24 -namesfile => 'names.dmp');
28 This is an implementation of Bio::DB::Taxonomy which stores and accesses the
29 NCBI taxonomy using a simple SQLite3 database stored locally on disk.
31 The required database files, nodes.dmp and names.dmp can be obtained from
32 ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz
38 =item Optimize codebase
40 =item Clean up SQL (kinda a mess right now)
42 =item Check compat. with other NCBI-specific L<Bio::DB::Taxonomy> implementations
44 =item Plan out feasibility of allowing other backends (Neo4J, other DBI, etc)
48 Beyond completing the implementation and optimization, this will
49 likely be rolled into a more flexible backend at some future point.
55 User feedback is an integral part of the evolution of this and other
56 Bioperl modules. Send your comments and suggestions preferably to
57 the Bioperl mailing list. Your participation is much appreciated.
59 bioperl-l@bioperl.org - General discussion
60 http://bioperl.org/wiki/Mailing_lists - About the mailing lists
64 Please direct usage questions or support issues to the mailing list:
66 I<bioperl-l@bioperl.org>
68 rather than to the module maintainer directly. Many experienced and
69 reponsive experts will be able look at the problem and quickly
70 address it. Please include a thorough description of the problem
71 with code and data examples if at all possible.
75 Report bugs to the Bioperl bug tracking system to help us keep track
76 of the bugs and their resolution. Bug reports can be submitted via
79 https://github.com/bioperl/bioperl-live/issues
81 =head1 AUTHOR - Chris Fields
83 Email cjfields-at-cpan-dot-org
87 The rest of the documentation details each of the object methods.
88 Internal methods are usually preceded with a _
92 # Let the code begin...
94 package Bio
::DB
::Taxonomy
::sqlite
;
100 use File
::Spec
::Functions
;
104 use constant SEPARATOR
=> ':';
106 our $DEFAULT_INDEX_DIR = $Bio::Root
::IO
::TEMPDIR
; # /tmp
107 our $DEFAULT_CACHE_SIZE = 0; # /tmp
108 our $DEFAULT_DB_NAME = 'taxonomy.sqlite';
112 [qw(INV Invertebrates)],
115 [qw(PLN Plants)], # (and fungi)
119 [qw(UNA Unassigned)],
121 [qw(VRT Vertebrates)],
122 [qw(ENV 'Environmental samples')]
125 use base
qw(Bio::DB::Taxonomy);
130 Usage : my $obj = Bio::DB::Taxonomy::flatfile->new();
131 Function: Builds a new Bio::DB::Taxonomy::flatfile object
132 Returns : an instance of Bio::DB::Taxonomy::flatfile
133 Args : -directory => name of directory where index files should be created
134 -nodesfile => name of file containing nodes (nodes.dmp from NCBI)
135 -namesfile => name of the file containing names(names.dmp from NCBI)
136 -force => 1 to replace current indexes even if they exist
140 # TODO: get rid of globals!
142 my ( $class, @args ) = @_;
144 my $self = $class->SUPER::new
(@args);
146 my ( $dir, $nodesfile, $namesfile, $db, $force, $cs ) =
147 $self->_rearrange( [qw(DIRECTORY NODESFILE NAMESFILE DB FORCE CACHE_SIZE)], @args );
149 $self->index_directory( $dir || $DEFAULT_INDEX_DIR );
151 $self->db_name( $db || $DEFAULT_DB_NAME );
153 $self->cache_size($cs // $DEFAULT_CACHE_SIZE);
156 $self->_build_index( $nodesfile, $namesfile, $force );
163 =head2 Bio::DB::Taxonomy interface implementation
168 Usage : my $num = $db->get_num_taxa();
169 Function: Get the number of taxa stored in the database.
178 my $ct = $self->_dbh_fetch(<<SQL);
179 SELECT COUNT(*) FROM taxon
188 Usage : my $taxon = $db->get_taxon(-taxonid => $taxonid)
189 Function: Get a Bio::Taxon object from the database.
190 Returns : Bio::Taxon object
191 Args : just a single value which is the database id, OR named args:
192 -taxonid => taxonomy id (to query by taxonid)
194 -name => string (to query by a taxonomy name: common name,
195 scientific name, etc)
201 my ( $taxonid, $name );
204 ( $taxonid, $name ) = $self->_rearrange( [qw(TAXONID NAME)], @_ );
206 ( $taxonid, my @others ) = $self->get_taxonids($name);
208 "There were multiple ids ($taxonid @others) matching '$name', using '$taxonid'"
216 return unless $taxonid;
218 $taxonid =~ /^\d+$/ || $self->throw("TaxID must be integer, got [$taxonid]");
220 my ( $parent_id, $rank, $code, $divid, $gen_code, $mito, $nm, $uniq, $class );
221 # single join or two calls?
222 my $sth = $self->_prepare_cached(<<SQL);
223 SELECT tax.parent_id, tax.rank, tax.code, tax.division_id, tax.gencode_id, tax.mito_id, names.name, names.uniq_name, names.class
224 FROM taxon as tax, names
228 names.taxon_id = tax.taxon_id
231 $sth->bind_columns(\
$parent_id, \
$rank, \
$code, \
$divid, \
$gen_code, \
$mito, \
$nm, \
$uniq, \
$class);
233 $sth->execute($taxonid) or $self->throw($sth->errstr);
235 my ($sci_name, @common_names);
237 while ($sth->fetch) {
238 if ($class eq 'scientific name') {
241 push @common_names, $nm;
245 my $taxon = Bio
::Taxon
->new(
247 -common_names
=> [@common_names],
248 -ncbi_taxid
=> $taxonid,
251 # Okay, this is a pretty goofy thing; we have the parent_id in hand
252 # but can't assign it b/c of semantics (one apparently must call
253 # ancestor() to get this, which seems roundabout if the information is
256 -parent_id
=> $parent_id,
258 -division
=> $DIVISIONS[$divid]->[1],
259 -genetic_code
=> $gen_code,
260 -mito_genetic_code
=> $mito
263 # we can't use -dbh or the db_handle() method ourselves or we'll go
264 # infinite on the merge attempt
265 $taxon->{'db_handle'} = $self;
267 $self->_handle_internal_id($taxon);
272 *get_Taxonomy_Node
= \
&get_taxon
;
277 Usage : my @taxonids = $db->get_taxonids('Homo sapiens');
278 Function: Searches for a taxonid (typically ncbi_taxon_id) based on a query
279 string. Note that multiple taxonids can match to the same supplied
281 Returns : array of integer ids in list context, one of these in scalar context
282 Args : string representing taxon's name
287 my ( $self, $query ) = @_;
289 # TODO: note we're not cleaning the query here, so you could technically
290 # have a fuzzy match (or Bobby Tables someone)
292 # TODO: OR'd match seems poor optimally
293 my $taxids = $self->{dbh
}->selectcol_arrayref(<<SQL);
294 SELECT DISTINCT taxon_id FROM names
298 uniq_name LIKE "$query"
301 return wantarray() ? @
{$taxids} : @
{$taxids}[0];
304 *get_taxonid
= \
&get_taxonids
;
306 =head2 get_Children_Taxids
308 Title : get_Children_Taxids
309 Usage : my @childrenids = $db->get_Children_Taxids
310 Function: Get the ids of the children of a node in the taxonomy
311 Returns : Array of Ids
312 Args : Bio::Taxon or a taxon_id
313 Status : deprecated (use each_Descendent())
317 sub get_Children_Taxids
{
318 my ( $self, $node ) = @_;
319 $self->deprecated(); # ?
321 # "get_Children_Taxids is deprecated, use each_Descendent instead");
324 # if ( $node->can('object_id') ) {
325 # $id = $node->object_id;
327 # elsif ( $node->can('ncbi_taxid') ) {
328 # $id = $node->ncbi_taxid;
332 # "Don't know how to extract a taxon id from the object of type "
338 #else { $id = $node }
339 #my @vals = $self->{'_parentbtree'}->get_dup($id);
346 Usage : my $ancestor_taxon = $db->ancestor($taxon)
347 Function: Retrieve the full ancestor taxon of a supplied Taxon from the
350 Args : Bio::Taxon (that was retrieved from this database)
355 my ( $self, $taxon ) = @_;
356 $self->throw("Must supply a Bio::Taxon")
357 unless ref($taxon) && $taxon->isa('Bio::Taxon');
358 $self->throw("The supplied Taxon must belong to this database")
359 unless $taxon->db_handle && $taxon->db_handle eq $self;
361 $taxon->id || $self->throw("The supplied Taxon is missing its id!");
364 # Note here we explicitly set the parent ID, but use a separate method to
365 # check whether it is defined. Mixing back-end databases, even if from the
366 # same source, should still work (since a different backend wouldn't
367 # explicitly set the parent_id)
369 if ($taxon->trusted_parent_id) {
370 # this is the failsafe when we hit the root node
371 if ($taxon->parent_id eq $id) {
374 my $anc = $self->get_taxon(-taxonid
=> $taxon->parent_id);
375 $taxon->ancestor($anc);
379 # TODO: would there be any other option?
387 Usage : my @ancestor_taxa = $db->ancestors($taxon)
388 Function: Retrieve the full ancestor taxon of a supplied Taxon from the
390 Returns : List of Bio::Taxon
391 Args : Bio::Taxon (that was retrieved from this database)
396 # my ( $self, $taxon ) = @_;
397 # $self->throw("Must supply a Bio::Taxon")
398 # unless ref($taxon) && $taxon->isa('Bio::Taxon');
399 # $self->throw("The supplied Taxon must belong to this database")
400 # unless $taxon->db_handle && $taxon->db_handle eq $self;
402 # $taxon->id || $self->throw("The supplied Taxon is missing its id!");
404 # my $sth = $self->{dbh}->prepare_cached(<<SQL);
407 # SELECT taxon_id FROM taxon WHERE id=@BASELINE
409 # SELECT derivedfrom.xfrom, checkin.mtime
410 # FROM ancestor, derivedfrom, checkin
411 # WHERE ancestor.id=derivedfrom.xto
412 # AND checkin.id=derivedfrom.xfrom
413 # ORDER BY checkin.mtime DESC
416 # SELECT * FROM checkin JOIN ancestor USING(id);
419 # if ($taxon->trusted_parent_id) {
420 # # this is the failsafe when we hit the root node
421 # if ($taxon->parent_id eq $id) {
424 # my $anc = $self->get_taxon(-taxonid => $taxon->parent_id);
427 # # TODO: would there be any other option?
432 =head2 each_Descendent
434 Title : each_Descendent
435 Usage : my @taxa = $db->each_Descendent($taxon);
436 Function: Get all the descendents of the supplied Taxon (but not their
437 descendents, ie. not a recursive fetchall).
438 Returns : Array of Bio::Taxon objects
439 Args : Bio::Taxon (that was retrieved from this database)
443 sub each_Descendent
{
444 my ( $self, $taxon ) = @_;
445 $self->throw("Must supply a Bio::Taxon")
446 unless ref($taxon) && $taxon->isa('Bio::Taxon');
447 $self->throw("The supplied Taxon must belong to this database")
448 unless $taxon->db_handle && $taxon->db_handle eq $self; # yikes
451 $taxon->id || $self->throw("The supplied Taxon is missing its id!");
453 #my ( $parent_id, $rank, $code, $divid, $gen_code, $mito, $nm, $uniq, $class );
454 # single join or two calls?
456 # probably not optimal, maybe set up as a cached statement with bindings?
457 my $desc_ids = $self->{dbh
}->selectcol_arrayref(<<SQL) or $self->throw($self->{dbh}->errstr);
464 return unless ref $desc_ids eq 'ARRAY';
467 foreach my $desc_id (@
$desc_ids) {
468 push( @descs, $self->get_taxon($desc_id) || next );
473 =head2 Helper methods
477 =head2 index_directory
479 Title : index_directory
480 Funtion : Get/set the location that index files are stored. (this module
481 will index the supplied database)
482 Usage : $obj->index_directory($newval)
483 Returns : value of index_directory (a scalar)
484 Args : on set, new value (a scalar or undef, optional)
485 Note : kept for backwards compatibility with older DB_File implementation
489 sub index_directory
{
491 return $self->{'index_directory'} = shift if @_;
492 return $self->{'index_directory'};
498 Funtion : Get/set the name of the SQLite3 database where data is stored
499 Usage : $obj->db_name($newval)
500 Returns : value of db_name (a scalar)
501 Args : on set, new value (a scalar or undef, optional)
505 # TODO: this may need some disambiguation w/ index_directory above; for now we
506 # assume this doesn't have a full path name (though I see no reason why this
507 # shouldn't allow that)
511 return $self->{'db_name'} = shift if @_;
512 return $self->{'db_name'};
518 Funtion : Get/set the cachesize used for loading the SQLite3 database
519 Usage : $obj->cache_size($newval)
520 Returns : value of cache_size (a scalar)
521 Args : on set, new value (a scalar or undef, optional)
522 Note : we do no checking on whether this value is an integer (SQLite does this for use)
528 return $self->{'cache_size'} = shift if defined($_[0]);
529 return $self->{'cache_size'};
532 # internal method which does the indexing
534 my ( $self, $nodesfile, $namesfile, $force ) = @_;
536 # TODO: need to disambiguate using index_directory here since we only have
537 # one file. Mayeb ignore it in favor of having full path for db_name?
538 my ($dir, $db_name) = ($self->index_directory, $self->db_name);
539 if (! -e
$db_name || $force) {
541 # TODO: we're ignoring index_directory for now, may add support for this
543 my $dbh = DBI
->connect("dbi:SQLite:dbname=$db_name","","") or die $!;
545 $self->debug("Running SQLite version:".$dbh->{sqlite_version
}."\n");
547 #$dbh->do('PRAGMA synchronous = 0'); # Non transaction safe!!!
549 if ($self->cache_size) {
550 my $cs = $self->cache_size;
551 $self->debug("Setting cache size $cs\n");
552 $dbh->do("PRAGMA cache_size = $cs")
555 $self->debug("Loading taxon table data\n");
556 $self->_init_db($dbh);
557 open my $NODES, '<', $nodesfile
558 or $self->throw("Could not read node file '$nodesfile': $!");
560 # TODO: this has the really unnecessary 'OR IGNORE' option added,
561 # apparently b.c the test data expects to handle cases where the TaxID
562 # is repeated in this table (which should never happen in this table). I
563 # will likely change this to throw under those circumstances
565 my $sth = $dbh->prepare_cached(<<SQL);
566 INSERT OR IGNORE INTO taxon (taxon_id, parent_id, rank, code, division_id, gencode_id, mito_id) VALUES (?,?,?,?,?,?,?)
572 my ($taxid,$parent,$rank,$code,$divid,undef,$gen_code,undef,$mito) = split(/\t\|\t/,$_);
578 $sth->execute($taxid, $parent, $rank, $code, $divid, $gen_code, $mito) or die $sth->errstr.": TaxID $taxid";
580 $dbh->do("COMMIT") or $self->throw($dbh->errstr);
584 $self->debug("Loading name table data\n");
585 open my $NAMES, '<', $namesfile
586 or $self->throw("Could not read names file '$namesfile': $!");
588 my $sth = $dbh->prepare_cached(<<SQL) or $self->throw($dbh->errstr);
589 INSERT INTO names (taxon_id, name, uniq_name, class) VALUES (?,?,?,?)
595 my ($taxid, $name, $unique_name, $class) = split(/\t\|\t/,$_);
597 # don't include the fake root node 'root' or 'all' with id 1
600 $class =~ s/\s+\|\s*$//;
602 #if ($name =~ /\(class\)$/) { # it seems that only rank of class is ever used in this situation
603 # $name =~ s/\s+\(class\)$//;
606 $sth->execute($taxid, $name, $unique_name, $class) or $self->throw($sth->errstr);
610 $self->debug("Creating taxon/parent index\n");
611 $dbh->do("CREATE INDEX parent_idx ON taxon (parent_id)") or $self->throw($dbh->errstr);
612 $self->debug("Creating name/taxon index\n");
613 $dbh->do("CREATE INDEX name_taxon_idx ON names (taxon_id,name)") or $self->throw($dbh->errstr);
614 $dbh->do("PRAGMA foreign_keys = ON");
618 #$dbh->do('PRAGMA synchronous = 1');
620 $self->{'_initialized'} = 1;
625 # connect the internal db handle
628 return if $self->{'_initialized'};
630 my ($dir, $db_name) = ($self->index_directory, $self->db_name);
632 # TODO: we're ignoring index_directory for now, may add support for this
634 my $dbh = DBI
->connect("dbi:SQLite:dbname=$db_name","","") or die $!;
635 $dbh->do("PRAGMA foreign_keys = ON");
636 if ($self->cache_size) {
637 my $cs = $self->cache_size;
638 $self->debug("Setting cache size $cs\n");
639 $dbh->do("PRAGMA cache_size = $cs")
643 $self->{'_initialized'} = 1;
647 my ($self, $dbh) = @_;
648 my $schema = $self->taxon_schema();
649 # TODO: set up handler parameters here
650 for my $table (sort keys %$schema) {
651 $dbh->do("DROP TABLE IF EXISTS $table") or $self->throw($dbh->errstr);
652 $dbh->do("CREATE TABLE $table ".$schema->{$table}) or $self->throw($dbh->errstr);
658 my ($self, $sql) = @_;
659 # TODO: more sanity checks
660 my $rows = $self->{dbh
}->selectrow_arrayref($sql) or $self->throw( $self->{dbh
}->errstr );
664 sub _prepare_cached
{
665 my ($self, $sql) = @_;
666 # TODO: more sanity checks
667 my $sth = $self->{dbh
}->prepare_cached($sql) or $self->throw( $self->{dbh
}->errstr );
672 # TODO: check data size, this is a ballpark estimate (could be reduced)
678 taxon_id INTEGER UNIQUE PRIMARY KEY NOT NULL,
687 FOREIGN KEY(parent_id) REFERENCES taxon(taxon_id)
693 name_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
696 uniq_name VARCHAR(50),
698 FOREIGN KEY(taxon_id) REFERENCES taxon(taxon_id)