1 /** @file glass_database.h
2 * @brief C++ class definition for glass database
4 /* Copyright 1999,2000,2001 BrightStation PLC
5 * Copyright 2002 Ananova Ltd
6 * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016 Olly Betts
7 * Copyright 2008 Lemur Consulting Ltd
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License as
11 * published by the Free Software Foundation; either version 2 of the
12 * License, or (at your option) any later version.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
25 #ifndef OM_HGUARD_GLASS_DATABASE_H
26 #define OM_HGUARD_GLASS_DATABASE_H
28 #include "backends/backends.h"
29 #include "backends/database.h"
30 #include "glass_changes.h"
31 #include "glass_docdata.h"
32 #include "glass_inverter.h"
33 #include "glass_positionlist.h"
34 #include "glass_postlist.h"
35 #include "glass_spelling.h"
36 #include "glass_synonym.h"
37 #include "glass_termlisttable.h"
38 #include "glass_values.h"
39 #include "glass_version.h"
40 #include "../flint_lock.h"
41 #include "glass_defs.h"
42 #include "backends/valuestats.h"
46 #include "xapian/compactor.h"
47 #include "xapian/constants.h"
52 class GlassAllDocsPostList
;
53 class RemoteConnection
;
55 /** A backend designed for efficient indexing and retrieval, using
56 * compressed posting lists and a btree storage scheme.
58 class GlassDatabase
: public Xapian::Database::Internal
{
59 friend class GlassWritableDatabase
;
60 friend class GlassTermList
;
61 friend class GlassPostList
;
62 friend class GlassAllTermsList
;
63 friend class GlassAllDocsPostList
;
65 /** Directory to store databases in.
69 /** Whether the database is readonly.
73 /** The file describing the Glass database.
74 * This file has information about the format of the database
75 * which can't easily be stored in any of the individual tables.
77 GlassVersion version_file
;
79 /** Table storing posting lists.
81 * Whenever an update is performed, this table is the first to be
82 * updated: therefore, its most recent revision number is the most
83 * recent anywhere in the database.
85 mutable GlassPostListTable postlist_table
;
87 /** Table storing position lists.
89 mutable GlassPositionListTable position_table
;
91 /** Table storing term lists.
93 GlassTermListTable termlist_table
;
96 mutable GlassValueManager value_manager
;
98 /** Table storing synonym data.
100 mutable GlassSynonymTable synonym_table
;
102 /** Table storing spelling correction data.
104 mutable GlassSpellingTable spelling_table
;
106 /** Table storing document data.
108 GlassDocDataTable docdata_table
;
113 /// Replication changesets.
114 GlassChanges changes
;
116 /** Return true if a database exists at the path specified for this
119 bool database_exists();
121 /** Create new tables, and open them.
122 * Any existing tables will be removed first.
124 void create_and_open_tables(int flags
, unsigned int blocksize
);
126 /** Open all tables at most recent revision.
128 * @exception Xapian::DatabaseCorruptError is thrown if a problem is
129 * found with the database's format.
131 * @return false if the tables were already open at the most recent
134 bool open_tables(int flags
);
136 /** Get a write lock on the database, or throw an
137 * Xapian::DatabaseLockError if failure.
139 * @param flags Bit-wise or of zero or more Xapian::DB_* constants
141 * @param creating true if the database is in the process of being
142 * created - if false, will throw a DatabaseOpening error if the lock
143 * can't be acquired and the database doesn't exist.
145 void get_database_write_lock(int flags
, bool creating
);
147 /** Get an object holding the next revision number which should be
148 * used in the tables.
150 * @return the next revision number.
152 glass_revision_number_t
get_next_revision_number() const;
154 /** Set the revision number in the tables.
156 * This updates the disk tables so that the currently open revision
157 * becomes the specified revision number.
159 * @param new_revision The new revision number to store. This must
160 * be greater than the current revision number. FIXME: If
161 * we support rewinding to a previous revision, maybe this
162 * needs to be greater than any previously used revision.
164 void set_revision_number(int flags
, glass_revision_number_t new_revision
);
166 /** Re-open tables to recover from an overwritten condition,
167 * or just get most up-to-date version.
171 /** Close all the tables permanently.
175 /** Called if a modifications fail.
177 * @param msg is a string description of the exception that was
178 * raised when the modifications failed.
180 void modifications_failed(glass_revision_number_t new_revision
,
181 const std::string
& msg
);
183 /** Apply any outstanding changes to the tables.
185 * If an error occurs during this operation, this will be signalled
186 * by an exception being thrown. In this case the contents of the
187 * tables on disk will be left in an unmodified state (though possibly
188 * with increased revision numbers), and the outstanding changes will
193 /** Cancel any outstanding changes to the tables.
197 /** Send a set of messages which transfer the whole database.
199 void send_whole_database(RemoteConnection
& conn
, double end_time
);
201 /** Get the revision stored in a changeset.
203 void get_changeset_revisions(const string
& path
,
204 glass_revision_number_t
* startrev
,
205 glass_revision_number_t
* endrev
) const;
208 /** Create and open a glass database.
210 * @exception Xapian::DatabaseCorruptError is thrown if a problem is
211 * found with the database's format.
213 * @exception Xapian::DatabaseOpeningError thrown if database can't
216 * @exception Xapian::DatabaseVersionError thrown if database is in an
217 * unsupported format. This implies that the database was
218 * created by an older or newer version of Xapian.
220 * @param dbdir directory holding glass tables
222 * @param block_size Block size, in bytes, to use when creating
223 * tables. This is only important, and has the
224 * correct value, when the database is being
227 explicit GlassDatabase(const string
&db_dir_
, int flags
= Xapian::DB_READONLY_
,
228 unsigned int block_size
= 0u);
230 explicit GlassDatabase(int fd
);
234 /// Get a postlist table cursor (used by GlassValueList).
235 GlassCursor
* get_postlist_cursor() const {
236 return postlist_table
.cursor_get();
239 /** Get an object holding the revision number which the tables are
242 * @return the current revision number.
244 glass_revision_number_t
get_revision_number() const;
246 /** Virtual methods of Database::Internal. */
248 Xapian::doccount
get_doccount() const;
249 Xapian::docid
get_lastdocid() const;
250 totlen_t
get_total_length() const;
251 Xapian::termcount
get_doclength(Xapian::docid did
) const;
252 Xapian::termcount
get_unique_terms(Xapian::docid did
) const;
253 void get_freqs(const string
& term
,
254 Xapian::doccount
* termfreq_ptr
,
255 Xapian::termcount
* collfreq_ptr
) const;
256 Xapian::doccount
get_value_freq(Xapian::valueno slot
) const;
257 std::string
get_value_lower_bound(Xapian::valueno slot
) const;
258 std::string
get_value_upper_bound(Xapian::valueno slot
) const;
259 Xapian::termcount
get_doclength_lower_bound() const;
260 Xapian::termcount
get_doclength_upper_bound() const;
261 Xapian::termcount
get_wdf_upper_bound(const string
& term
) const;
262 bool term_exists(const string
& tname
) const;
263 bool has_positions() const;
265 LeafPostList
* open_post_list(const string
& tname
) const;
266 ValueList
* open_value_list(Xapian::valueno slot
) const;
267 Xapian::Document::Internal
* open_document(Xapian::docid did
, bool lazy
) const;
269 PositionList
* open_position_list(Xapian::docid did
, const string
& term
) const;
270 TermList
* open_term_list(Xapian::docid did
) const;
271 TermList
* open_allterms(const string
& prefix
) const;
273 TermList
* open_spelling_termlist(const string
& word
) const;
274 TermList
* open_spelling_wordlist() const;
275 Xapian::doccount
get_spelling_frequency(const string
& word
) const;
277 TermList
* open_synonym_termlist(const string
& term
) const;
278 TermList
* open_synonym_keylist(const string
& prefix
) const;
280 string
get_metadata(const string
& key
) const;
281 TermList
* open_metadata_keylist(const std::string
&prefix
) const;
282 void write_changesets_to_fd(int fd
,
283 const string
& start_revision
,
285 Xapian::ReplicationInfo
* info
);
286 string
get_revision_info() const;
287 string
get_uuid() const;
289 void request_document(Xapian::docid
/*did*/) const;
290 void readahead_for_query(const Xapian::Query
&query
);
293 XAPIAN_NORETURN(void throw_termlist_table_close_exception() const);
295 int get_backend_info(string
* path
) const {
296 if (path
) *path
= db_dir
;
297 return BACKEND_GLASS
;
300 bool single_file() const { return version_file
.single_file(); }
302 void get_used_docid_range(Xapian::docid
& first
,
303 Xapian::docid
& last
) const;
305 /** Return true if there are uncommitted changes. */
306 virtual bool has_uncommitted_changes() const;
308 static void compact(Xapian::Compactor
* compactor
,
309 const char * destdir
,
311 const std::vector
<Xapian::Database::Internal
*> & sources
,
312 const std::vector
<Xapian::docid
> & offset
,
314 Xapian::Compactor::compaction_level compaction
,
316 Xapian::docid last_docid
);
319 /** A writable glass database.
321 class GlassWritableDatabase
: public GlassDatabase
{
322 mutable Inverter inverter
;
324 mutable map
<Xapian::valueno
, ValueStats
> value_stats
;
326 /** The number of documents added, deleted, or replaced since the last
329 mutable Xapian::doccount change_count
;
331 /// If change_count reaches this threshold we automatically flush.
332 Xapian::doccount flush_threshold
;
334 /** A pointer to the last document which was returned by
335 * open_document(), or NULL if there is no such valid document. This
336 * is used purely for comparing with a supplied document to help with
337 * optimising replace_document. When the document internals are
338 * deleted, this pointer gets set to NULL.
340 mutable Xapian::Document::Internal
* modify_shortcut_document
;
342 /** The document ID for the last document returned by open_document().
344 mutable Xapian::docid modify_shortcut_docid
;
346 /** Check if we should autoflush.
348 * Called at the end of each document changing operation.
350 void check_flush_threshold();
352 /// Flush any unflushed postlist changes, but don't commit them.
353 void flush_postlist_changes() const;
355 /// Close all the tables permanently.
362 /** Implementation of virtual methods: see Database::Internal for
367 /** Cancel pending modifications to the database. */
370 Xapian::docid
add_document(const Xapian::Document
& document
);
371 Xapian::docid
add_document_(Xapian::docid did
, const Xapian::Document
& document
);
372 // Stop the default implementation of delete_document(term) and
373 // replace_document(term) from being hidden. This isn't really
374 // a problem as we only try to call them through the base class
375 // (where they aren't hidden) but some compilers generate a warning
378 using Xapian::Database::Internal::delete_document
;
379 using Xapian::Database::Internal::replace_document
;
381 void delete_document(Xapian::docid did
);
382 void replace_document(Xapian::docid did
, const Xapian::Document
& document
);
384 Xapian::Document::Internal
* open_document(Xapian::docid did
,
390 /** Create and open a writable glass database.
392 * @exception Xapian::DatabaseOpeningError thrown if database can't
395 * @exception Xapian::DatabaseVersionError thrown if database is in an
396 * unsupported format. This implies that the database was
397 * created by an older or newer version of Xapian.
399 * @param dir directory holding glass tables
401 GlassWritableDatabase(const string
&dir
, int flags
, int block_size
);
403 ~GlassWritableDatabase();
405 /** Virtual methods of Database::Internal. */
407 Xapian::termcount
get_doclength(Xapian::docid did
) const;
408 void get_freqs(const string
& term
,
409 Xapian::doccount
* termfreq_ptr
,
410 Xapian::termcount
* collfreq_ptr
) const;
411 Xapian::doccount
get_value_freq(Xapian::valueno slot
) const;
412 std::string
get_value_lower_bound(Xapian::valueno slot
) const;
413 std::string
get_value_upper_bound(Xapian::valueno slot
) const;
414 bool term_exists(const string
& tname
) const;
415 bool has_positions() const;
417 LeafPostList
* open_post_list(const string
& tname
) const;
418 ValueList
* open_value_list(Xapian::valueno slot
) const;
419 PositionList
* open_position_list(Xapian::docid did
, const string
& term
) const;
420 TermList
* open_term_list(Xapian::docid did
) const;
421 TermList
* open_allterms(const string
& prefix
) const;
423 void add_spelling(const string
& word
, Xapian::termcount freqinc
) const;
424 void remove_spelling(const string
& word
, Xapian::termcount freqdec
) const;
425 TermList
* open_spelling_wordlist() const;
427 TermList
* open_synonym_keylist(const string
& prefix
) const;
428 void add_synonym(const string
& word
, const string
& synonym
) const;
429 void remove_synonym(const string
& word
, const string
& synonym
) const;
430 void clear_synonyms(const string
& word
) const;
432 void set_metadata(const string
& key
, const string
& value
);
433 void invalidate_doc_object(Xapian::Document::Internal
* obj
) const;
436 /** Return true if there are uncommitted changes. */
437 bool has_uncommitted_changes() const;
440 #endif /* OM_HGUARD_GLASS_DATABASE_H */