2 * @brief glass database
4 /* Copyright 1999,2000,2001 BrightStation PLC
5 * Copyright 2001 Hein Ragas
6 * Copyright 2002 Ananova Ltd
7 * Copyright 2002-2024 Olly Betts
8 * Copyright 2006,2008 Lemur Consulting Ltd
9 * Copyright 2009 Richard Boulton
10 * Copyright 2009 Kan-Ru Chen
11 * Copyright 2011 Dan Colish
13 * This program is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License as
15 * published by the Free Software Foundation; either version 2 of the
16 * License, or (at your option) any later version.
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
31 #include "glass_database.h"
33 #include "xapian/constants.h"
34 #include "xapian/error.h"
35 #include "xapian/valueiterator.h"
37 #include "backends/contiguousalldocspostlist.h"
38 #include "glass_alldocspostlist.h"
39 #include "glass_alltermslist.h"
40 #include "glass_defs.h"
41 #include "glass_docdata.h"
42 #include "glass_document.h"
43 #include "../flint_lock.h"
44 #include "glass_metadata.h"
45 #include "glass_positionlist.h"
46 #include "glass_postlist.h"
47 #include "glass_replicate_internal.h"
48 #include "glass_spellingwordslist.h"
49 #include "glass_termlist.h"
50 #include "glass_valuelist.h"
51 #include "glass_values.h"
54 #include "filetests.h"
58 #include "net/remoteconnection.h"
59 #include "api/replication.h"
60 #include "replicationprotocol.h"
61 #include "posixy_wrapper.h"
63 #include "stringutils.h"
64 #include "backends/valuestats.h"
66 #include "safesysstat.h"
67 #include <sys/types.h>
76 using namespace Xapian
;
77 using Xapian::Internal::intrusive_ptr
;
79 // The maximum safe term length is determined by the postlist. There we
80 // store the term using pack_string_preserving_sort() which takes the
81 // length of the string plus an extra byte (assuming the string doesn't
82 // contain any zero bytes), followed by the docid with encoded with
83 // pack_uint_preserving_sort() which takes up to 5 bytes (for a 32-bit
86 // The Btree manager's key length limit is 255 bytes so the maximum safe term
87 // length is 255 - 1 - 5 = 249 bytes. We actually set the limit at 245 for
88 // consistency with flint and chert, and also because this allows for 64-bit
91 // If the term contains zero bytes, the limit is lower (by one for each zero
93 #define MAX_SAFE_TERM_LENGTH 245
95 /* This opens the tables, determining the current and next revision numbers,
96 * and stores handles to the tables.
98 GlassDatabase::GlassDatabase(string_view glass_dir
, int flags
,
99 unsigned int block_size
)
100 : Xapian::Database::Internal(flags
== Xapian::DB_READONLY_
?
101 TRANSACTION_READONLY
:
104 readonly(flags
== Xapian::DB_READONLY_
),
105 version_file(db_dir
),
106 postlist_table(db_dir
, readonly
),
107 position_table(db_dir
, readonly
),
108 // Note: (Xapian::DB_READONLY_ & Xapian::DB_NO_TERMLIST) is true,
109 // so opening to read we always permit the termlist to be missing.
110 termlist_table(db_dir
, readonly
, (flags
& Xapian::DB_NO_TERMLIST
)),
111 value_manager(&postlist_table
, &termlist_table
),
112 synonym_table(db_dir
, readonly
),
113 spelling_table(db_dir
, readonly
),
114 docdata_table(db_dir
, readonly
),
118 LOGCALL_CTOR(DB
, "GlassDatabase", glass_dir
| flags
| block_size
);
125 // Block size must in the range GLASS_MIN_BLOCKSIZE..GLASS_MAX_BLOCKSIZE
126 // and a power of two.
127 if (block_size
< GLASS_MIN_BLOCKSIZE
||
128 block_size
> GLASS_MAX_BLOCKSIZE
||
129 (block_size
& (block_size
- 1)) != 0) {
130 block_size
= GLASS_DEFAULT_BLOCKSIZE
;
133 int action
= flags
& Xapian::DB_ACTION_MASK_
;
134 if (action
!= Xapian::DB_OPEN
&& !database_exists()) {
135 // Create the directory for the database, if it doesn't exist
137 if (mkdir(db_dir
.c_str(), 0755) < 0) {
138 int mkdir_errno
= errno
;
139 if (mkdir_errno
!= EEXIST
|| !dir_exists(db_dir
)) {
140 throw Xapian::DatabaseCreateError(db_dir
+ ": mkdir failed",
145 get_database_write_lock(flags
, true);
147 create_and_open_tables(flags
, block_size
);
151 if (action
== Xapian::DB_CREATE
) {
152 throw Xapian::DatabaseCreateError("Can't create new database at '" +
153 db_dir
+ "': a database already exists and I was told "
154 "not to overwrite it");
157 get_database_write_lock(flags
, false);
158 // if we're overwriting, pretend the db doesn't exist
159 if (action
== Xapian::DB_CREATE_OR_OVERWRITE
) {
160 create_and_open_tables(flags
, block_size
);
164 // Open the latest version of each table.
168 GlassDatabase::GlassDatabase(int fd
)
169 : Xapian::Database::Internal(TRANSACTION_READONLY
),
173 postlist_table(fd
, version_file
.get_offset(), readonly
),
174 position_table(fd
, version_file
.get_offset(), readonly
),
175 termlist_table(fd
, version_file
.get_offset(), readonly
, true),
176 value_manager(&postlist_table
, &termlist_table
),
177 synonym_table(fd
, version_file
.get_offset(), readonly
),
178 spelling_table(fd
, version_file
.get_offset(), readonly
),
179 docdata_table(fd
, version_file
.get_offset(), readonly
),
183 LOGCALL_CTOR(DB
, "GlassDatabase", fd
);
184 open_tables(Xapian::DB_READONLY_
);
187 GlassDatabase::~GlassDatabase()
189 LOGCALL_DTOR(DB
, "GlassDatabase");
193 GlassDatabase::database_exists() {
194 LOGCALL(DB
, bool, "GlassDatabase::database_exists", NO_ARGS
);
195 // The postlist table is the only non-optional one.
196 RETURN(postlist_table
.exists());
200 GlassDatabase::create_and_open_tables(int flags
, unsigned int block_size
)
202 LOGCALL_VOID(DB
, "GlassDatabase::create_and_open_tables", flags
|block_size
);
203 // The caller is expected to create the database directory if it doesn't
206 GlassVersion
&v
= version_file
;
207 v
.create(block_size
);
209 glass_revision_number_t rev
= v
.get_revision();
210 const string
& tmpfile
= v
.write(rev
, flags
);
212 position_table
.create_and_open(flags
, v
.get_root(Glass::POSITION
));
213 synonym_table
.create_and_open(flags
, v
.get_root(Glass::SYNONYM
));
214 spelling_table
.create_and_open(flags
, v
.get_root(Glass::SPELLING
));
215 docdata_table
.create_and_open(flags
, v
.get_root(Glass::DOCDATA
));
216 termlist_table
.create_and_open(flags
, v
.get_root(Glass::TERMLIST
));
217 postlist_table
.create_and_open(flags
, v
.get_root(Glass::POSTLIST
));
219 if (!v
.sync(tmpfile
, rev
, flags
)) {
220 throw Xapian::DatabaseCreateError("Failed to create iamglass file");
223 Assert(database_exists());
227 GlassDatabase::open_tables(int flags
)
229 LOGCALL(DB
, bool, "GlassDatabase::open_tables", flags
);
231 glass_revision_number_t cur_rev
= version_file
.get_revision();
234 // We're reopening, so ensure that we throw DatabaseError if close()
235 // was called. It could be argued that reopen() can be a no-op in this
236 // case, but that's confusing if someone expects that reopen() can undo
237 // the effects of close() - in fact reopen() is closer to
238 // update_reader_to_latest_revision().
239 if (!postlist_table
.is_open())
240 GlassTable::throw_database_closed();
244 glass_revision_number_t rev
= version_file
.get_revision();
245 if (cur_rev
&& cur_rev
== rev
) {
246 // We're reopening a database and the revision hasn't changed so we
247 // don't need to do anything.
251 docdata_table
.open(flags
, version_file
.get_root(Glass::DOCDATA
), rev
);
252 spelling_table
.open(flags
, version_file
.get_root(Glass::SPELLING
), rev
);
253 synonym_table
.open(flags
, version_file
.get_root(Glass::SYNONYM
), rev
);
254 termlist_table
.open(flags
, version_file
.get_root(Glass::TERMLIST
), rev
);
255 position_table
.open(flags
, version_file
.get_root(Glass::POSITION
), rev
);
256 postlist_table
.open(flags
, version_file
.get_root(Glass::POSTLIST
), rev
);
258 Xapian::termcount swfub
= version_file
.get_spelling_wordfreq_upper_bound();
259 spelling_table
.set_wordfreq_upper_bound(swfub
);
261 value_manager
.reset();
264 changes
.set_oldest_changeset(version_file
.get_oldest_changeset());
265 glass_revision_number_t revision
= version_file
.get_revision();
266 GlassChanges
* p
= changes
.start(revision
, revision
+ 1, flags
);
267 version_file
.set_changes(p
);
268 postlist_table
.set_changes(p
);
269 position_table
.set_changes(p
);
270 termlist_table
.set_changes(p
);
271 synonym_table
.set_changes(p
);
272 spelling_table
.set_changes(p
);
273 docdata_table
.set_changes(p
);
278 glass_revision_number_t
279 GlassDatabase::get_next_revision_number() const
281 LOGCALL(DB
, glass_revision_number_t
, "GlassDatabase::get_next_revision_number", NO_ARGS
);
282 // FIXME: If we permit a revision before the latest and then updating it
283 // (to roll back more recent changes) then we (probably) need this to be
284 // one more than the *highest* revision previously committed.
285 RETURN(version_file
.get_revision() + 1);
289 GlassDatabase::get_changeset_revisions(const string
& path
,
290 glass_revision_number_t
* startrev
,
291 glass_revision_number_t
* endrev
) const
293 FD
fd(posixy_open(path
.c_str(), O_RDONLY
| O_CLOEXEC
));
295 string message
= string("Couldn't open changeset ") +
297 throw Xapian::DatabaseError(message
, errno
);
300 char buf
[REASONABLE_CHANGESET_SIZE
];
301 const char *start
= buf
;
302 const char *end
= buf
+ io_read(fd
, buf
, REASONABLE_CHANGESET_SIZE
);
303 if (size_t(end
- start
) < CONST_STRLEN(CHANGES_MAGIC_STRING
))
304 throw Xapian::DatabaseError("Changeset too short at " + path
);
305 if (memcmp(start
, CHANGES_MAGIC_STRING
,
306 CONST_STRLEN(CHANGES_MAGIC_STRING
)) != 0) {
307 string message
= string("Changeset at ") +
308 path
+ " does not contain valid magic string";
309 throw Xapian::DatabaseError(message
);
311 start
+= CONST_STRLEN(CHANGES_MAGIC_STRING
);
313 unsigned int changes_version
;
314 if (!unpack_uint(&start
, end
, &changes_version
))
315 throw Xapian::DatabaseError("Couldn't read a valid version number for "
316 "changeset at " + path
);
317 if (changes_version
!= CHANGES_VERSION
)
318 throw Xapian::DatabaseError("Don't support version of changeset at " +
321 if (!unpack_uint(&start
, end
, startrev
))
322 throw Xapian::DatabaseError("Couldn't read a valid start revision from "
323 "changeset at " + path
);
325 if (!unpack_uint(&start
, end
, endrev
))
326 throw Xapian::DatabaseError("Couldn't read a valid end revision for "
327 "changeset at " + path
);
331 GlassDatabase::set_revision_number(int flags
, glass_revision_number_t new_revision
)
333 LOGCALL_VOID(DB
, "GlassDatabase::set_revision_number", flags
|new_revision
);
335 glass_revision_number_t rev
= version_file
.get_revision();
336 if (new_revision
<= rev
&& rev
!= 0) {
337 string m
= "New revision ";
338 m
+= str(new_revision
);
339 m
+= " <= old revision ";
341 throw Xapian::DatabaseError(m
);
344 value_manager
.merge_changes();
346 postlist_table
.flush_db();
347 position_table
.flush_db();
348 termlist_table
.flush_db();
349 synonym_table
.flush_db();
350 version_file
.set_spelling_wordfreq_upper_bound(spelling_table
.flush_db());
351 docdata_table
.flush_db();
353 postlist_table
.commit(new_revision
, version_file
.root_to_set(Glass::POSTLIST
));
354 position_table
.commit(new_revision
, version_file
.root_to_set(Glass::POSITION
));
355 termlist_table
.commit(new_revision
, version_file
.root_to_set(Glass::TERMLIST
));
356 synonym_table
.commit(new_revision
, version_file
.root_to_set(Glass::SYNONYM
));
357 spelling_table
.commit(new_revision
, version_file
.root_to_set(Glass::SPELLING
));
358 docdata_table
.commit(new_revision
, version_file
.root_to_set(Glass::DOCDATA
));
360 const string
& tmpfile
= version_file
.write(new_revision
, flags
);
361 if (!postlist_table
.sync() ||
362 !position_table
.sync() ||
363 !termlist_table
.sync() ||
364 !synonym_table
.sync() ||
365 !spelling_table
.sync() ||
366 !docdata_table
.sync() ||
367 !version_file
.sync(tmpfile
, new_revision
, flags
)) {
368 int saved_errno
= errno
;
369 (void)unlink(tmpfile
.c_str());
370 throw Xapian::DatabaseError("Commit failed", saved_errno
);
373 changes
.commit(new_revision
, flags
);
377 GlassDatabase::request_document(Xapian::docid did
) const
379 docdata_table
.readahead_for_document(did
);
383 GlassDatabase::readahead_for_query(const Xapian::Query
&query
) const
385 Xapian::TermIterator t
;
386 for (t
= query
.get_unique_terms_begin(); t
!= Xapian::TermIterator(); ++t
) {
387 const string
& term
= *t
;
388 if (!postlist_table
.readahead_key(GlassPostListTable::make_key(term
)))
394 GlassDatabase::reopen()
396 LOGCALL(DB
, bool, "GlassDatabase::reopen", NO_ARGS
);
397 if (!readonly
) RETURN(false);
398 RETURN(open_tables(postlist_table
.get_flags()));
402 GlassDatabase::close()
404 LOGCALL_VOID(DB
, "GlassDatabase::close", NO_ARGS
);
405 postlist_table
.close(true);
406 position_table
.close(true);
407 termlist_table
.close(true);
408 synonym_table
.close(true);
409 spelling_table
.close(true);
410 docdata_table
.close(true);
415 GlassDatabase::get_database_write_lock(int flags
, bool creating
)
417 LOGCALL_VOID(DB
, "GlassDatabase::get_database_write_lock", flags
|creating
);
418 // FIXME: Handle Xapian::DB_DANGEROUS here, perhaps by having readers
419 // get a lock on the revision they're reading, and then requiring the
420 // writer get an exclusive lock in this case.
422 bool retry
= flags
& Xapian::DB_RETRY_LOCK
;
423 FlintLock::reason why
= lock
.lock(true, retry
, explanation
);
424 if (why
!= FlintLock::SUCCESS
) {
425 if (why
== FlintLock::UNKNOWN
&& !creating
&& !database_exists()) {
426 string
msg("No glass database found at path '");
429 throw Xapian::DatabaseNotFoundError(msg
);
431 lock
.throw_databaselockerror(why
, db_dir
, explanation
);
436 GlassDatabase::send_whole_database(RemoteConnection
& conn
, double end_time
)
438 LOGCALL_VOID(DB
, "GlassDatabase::send_whole_database", conn
| end_time
);
439 #ifdef XAPIAN_HAS_REMOTE_BACKEND
440 // Send the current revision number in the header.
442 pack_string(buf
, get_uuid());
443 pack_uint(buf
, get_revision());
444 conn
.send_message(REPL_REPLY_DB_HEADER
, buf
, end_time
);
446 // Send all the tables. The tables which we want to be cached best after
447 // the copy finishes are sent last.
448 static const char filenames
[] =
449 "termlist." GLASS_TABLE_EXTENSION
"\0"
450 "synonym." GLASS_TABLE_EXTENSION
"\0"
451 "spelling." GLASS_TABLE_EXTENSION
"\0"
452 "docdata." GLASS_TABLE_EXTENSION
"\0"
453 "position." GLASS_TABLE_EXTENSION
"\0"
454 "postlist." GLASS_TABLE_EXTENSION
"\0"
456 string filepath
= db_dir
;
458 const char * p
= filenames
;
460 size_t len
= strlen(p
);
461 filepath
.replace(db_dir
.size() + 1, string::npos
, p
, len
);
462 FD
fd(posixy_open(filepath
.c_str(), O_RDONLY
| O_CLOEXEC
));
464 conn
.send_message(REPL_REPLY_DB_FILENAME
, {p
, len
}, end_time
);
465 conn
.send_file(REPL_REPLY_DB_FILEDATA
, fd
, end_time
);
476 GlassDatabase::write_changesets_to_fd(int fd
,
477 string_view revision
,
479 ReplicationInfo
* info
)
481 LOGCALL_VOID(DB
, "GlassDatabase::write_changesets_to_fd", fd
| revision
| need_whole_db
| info
);
482 #ifdef XAPIAN_HAS_REMOTE_BACKEND
483 int whole_db_copies_left
= MAX_DB_COPIES_PER_CONVERSATION
;
484 glass_revision_number_t start_rev_num
= 0;
485 string start_uuid
= get_uuid();
487 glass_revision_number_t needed_rev_num
= 0;
489 const char * rev_ptr
= revision
.data();
490 const char * rev_end
= rev_ptr
+ revision
.size();
491 if (!unpack_uint(&rev_ptr
, rev_end
, &start_rev_num
)) {
492 need_whole_db
= true;
495 RemoteConnection
conn(-1, fd
, string());
497 // While the starting revision number is less than the latest revision
498 // number, look for a changeset, and write it.
500 // FIXME - perhaps we should make hardlinks for all the changesets we're
501 // likely to need, first, and then start sending them, so that there's no
502 // risk of them disappearing while we're sending earlier ones.
505 // Decrease the counter of copies left to be sent, and fail
506 // if we've already copied the database enough. This ensures that
507 // synchronisation attempts always terminate eventually.
508 if (whole_db_copies_left
== 0) {
509 conn
.send_message(REPL_REPLY_FAIL
,
510 "Database changing too fast",
514 whole_db_copies_left
--;
516 // Send the whole database across.
517 start_rev_num
= get_revision();
518 start_uuid
= get_uuid();
520 send_whole_database(conn
, 0.0);
522 ++(info
->fullcopy_count
);
524 need_whole_db
= false;
527 if (start_uuid
== get_uuid()) {
528 // Send the latest revision number after sending the tables.
529 // The update must proceed to that revision number before the
530 // copy is safe to make live.
533 needed_rev_num
= get_revision();
534 pack_uint(buf
, needed_rev_num
);
535 conn
.send_message(REPL_REPLY_DB_FOOTER
, buf
, 0.0);
536 if (info
!= NULL
&& start_rev_num
== needed_rev_num
)
537 info
->changed
= true;
539 // Database has been replaced since we did the copy. Send a
540 // higher revision number than the revision we've just copied,
541 // so that the client doesn't make the copy we've just done
542 // live, and then mark that we need to do a copy again.
543 // The client will never actually get the required revision,
544 // because the next message is going to be the start of a new
545 // database transfer.
548 pack_uint(buf
, start_rev_num
+ 1);
549 conn
.send_message(REPL_REPLY_DB_FOOTER
, buf
, 0.0);
550 need_whole_db
= true;
553 // Check if we've sent all the updates.
554 if (start_rev_num
>= get_revision()) {
556 if (start_uuid
!= get_uuid()) {
557 need_whole_db
= true;
560 if (start_rev_num
>= get_revision()) {
565 // Look for the changeset for revision start_rev_num.
566 string changes_name
= db_dir
+ "/changes" + str(start_rev_num
);
567 FD
fd_changes(posixy_open(changes_name
.c_str(), O_RDONLY
| O_CLOEXEC
));
568 if (fd_changes
>= 0) {
569 // Send it, and also update start_rev_num to the new value
570 // specified in the changeset.
571 glass_revision_number_t changeset_start_rev_num
;
572 glass_revision_number_t changeset_end_rev_num
;
573 get_changeset_revisions(changes_name
,
574 &changeset_start_rev_num
,
575 &changeset_end_rev_num
);
576 if (changeset_start_rev_num
!= start_rev_num
) {
577 throw Xapian::DatabaseError("Changeset start revision does not match changeset filename");
579 if (changeset_start_rev_num
>= changeset_end_rev_num
) {
580 throw Xapian::DatabaseError("Changeset start revision is not less than end revision");
583 conn
.send_file(REPL_REPLY_CHANGESET
, fd_changes
, 0.0);
584 start_rev_num
= changeset_end_rev_num
;
586 ++(info
->changeset_count
);
587 if (start_rev_num
>= needed_rev_num
)
588 info
->changed
= true;
591 // The changeset doesn't exist: leave the revision number as it
592 // is, and mark for doing a full database copy.
593 need_whole_db
= true;
597 conn
.send_message(REPL_REPLY_END_OF_CHANGES
, {}, 0.0);
607 GlassDatabase::modifications_failed(glass_revision_number_t new_revision
,
608 const std::string
& msg
)
610 // Modifications failed. Wipe all the modifications from memory.
611 int flags
= postlist_table
.get_flags();
612 glass_revision_number_t old_revision
= version_file
.get_revision();
614 // Discard any buffered changes and reinitialised cached values
618 // Reopen tables with old revision number.
620 docdata_table
.open(flags
, version_file
.get_root(Glass::DOCDATA
), old_revision
);
621 spelling_table
.open(flags
, version_file
.get_root(Glass::SPELLING
), old_revision
);
622 synonym_table
.open(flags
, version_file
.get_root(Glass::SYNONYM
), old_revision
);
623 termlist_table
.open(flags
, version_file
.get_root(Glass::TERMLIST
), old_revision
);
624 position_table
.open(flags
, version_file
.get_root(Glass::POSITION
), old_revision
);
625 postlist_table
.open(flags
, version_file
.get_root(Glass::POSTLIST
), old_revision
);
627 value_manager
.reset();
628 } catch (const Xapian::Error
&e
) {
629 // We failed to roll-back so close the database to avoid the risk of
630 // database corruption.
631 GlassDatabase::close();
632 throw Xapian::DatabaseError("Modifications failed (" + msg
+ "), "
633 "and couldn't open at the old revision: " +
638 p
= changes
.start(old_revision
, new_revision
, flags
);
639 version_file
.set_changes(p
);
640 postlist_table
.set_changes(p
);
641 position_table
.set_changes(p
);
642 termlist_table
.set_changes(p
);
643 synonym_table
.set_changes(p
);
644 spelling_table
.set_changes(p
);
645 docdata_table
.set_changes(p
);
649 GlassDatabase::apply()
651 LOGCALL_VOID(DB
, "GlassDatabase::apply", NO_ARGS
);
652 if (!postlist_table
.is_modified() &&
653 !position_table
.is_modified() &&
654 !termlist_table
.is_modified() &&
655 !value_manager
.is_modified() &&
656 !synonym_table
.is_modified() &&
657 !spelling_table
.is_modified() &&
658 !docdata_table
.is_modified()) {
662 glass_revision_number_t new_revision
= get_next_revision_number();
664 int flags
= postlist_table
.get_flags();
666 set_revision_number(flags
, new_revision
);
667 } catch (const Xapian::Error
&e
) {
668 modifications_failed(new_revision
, e
.get_description());
671 modifications_failed(new_revision
, "Unknown error");
676 p
= changes
.start(new_revision
, new_revision
+ 1, flags
);
677 version_file
.set_changes(p
);
678 postlist_table
.set_changes(p
);
679 position_table
.set_changes(p
);
680 termlist_table
.set_changes(p
);
681 synonym_table
.set_changes(p
);
682 spelling_table
.set_changes(p
);
683 docdata_table
.set_changes(p
);
687 GlassDatabase::cancel()
689 LOGCALL_VOID(DB
, "GlassDatabase::cancel", NO_ARGS
);
690 version_file
.cancel();
691 glass_revision_number_t rev
= version_file
.get_revision();
692 postlist_table
.cancel(version_file
.get_root(Glass::POSTLIST
), rev
);
693 position_table
.cancel(version_file
.get_root(Glass::POSITION
), rev
);
694 termlist_table
.cancel(version_file
.get_root(Glass::TERMLIST
), rev
);
695 value_manager
.cancel();
696 synonym_table
.cancel(version_file
.get_root(Glass::SYNONYM
), rev
);
697 spelling_table
.cancel(version_file
.get_root(Glass::SPELLING
), rev
);
698 docdata_table
.cancel(version_file
.get_root(Glass::DOCDATA
), rev
);
700 Xapian::termcount ub
= version_file
.get_spelling_wordfreq_upper_bound();
701 spelling_table
.set_wordfreq_upper_bound(ub
);
705 GlassDatabase::get_doccount() const
707 LOGCALL(DB
, Xapian::doccount
, "GlassDatabase::get_doccount", NO_ARGS
);
708 RETURN(version_file
.get_doccount());
712 GlassDatabase::get_lastdocid() const
714 LOGCALL(DB
, Xapian::docid
, "GlassDatabase::get_lastdocid", NO_ARGS
);
715 RETURN(version_file
.get_last_docid());
719 GlassDatabase::get_total_length() const
721 LOGCALL(DB
, Xapian::totallength
, "GlassDatabase::get_total_length", NO_ARGS
);
722 RETURN(version_file
.get_total_doclen());
726 GlassDatabase::get_doclength(Xapian::docid did
) const
728 LOGCALL(DB
, Xapian::termcount
, "GlassDatabase::get_doclength", did
);
730 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
731 RETURN(postlist_table
.get_doclength(did
, ptrtothis
));
735 GlassDatabase::get_unique_terms(Xapian::docid did
) const
737 LOGCALL(DB
, Xapian::termcount
, "GlassDatabase::get_unique_terms", did
);
739 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
740 RETURN(GlassTermList(ptrtothis
, did
).get_unique_terms());
744 GlassDatabase::get_wdfdocmax(Xapian::docid did
) const
746 LOGCALL(DB
, Xapian::termcount
, "GlassDatabase::get_wdfdocmax", did
);
748 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
749 GlassTermList
termlist(ptrtothis
, did
);
750 Xapian::termcount max_wdf
= 0;
751 while (termlist
.next() == NULL
) {
752 Xapian::termcount current_wdf
= termlist
.get_wdf();
753 if (current_wdf
> max_wdf
) max_wdf
= current_wdf
;
759 GlassDatabase::get_freqs(string_view term
,
760 Xapian::doccount
* termfreq_ptr
,
761 Xapian::termcount
* collfreq_ptr
) const
763 LOGCALL_VOID(DB
, "GlassDatabase::get_freqs", term
| termfreq_ptr
| collfreq_ptr
);
764 Assert(!term
.empty());
765 postlist_table
.get_freqs(term
, termfreq_ptr
, collfreq_ptr
);
769 GlassDatabase::get_value_freq(Xapian::valueno slot
) const
771 LOGCALL(DB
, Xapian::doccount
, "GlassDatabase::get_value_freq", slot
);
772 RETURN(value_manager
.get_value_freq(slot
));
776 GlassDatabase::get_value_lower_bound(Xapian::valueno slot
) const
778 LOGCALL(DB
, std::string
, "GlassDatabase::get_value_lower_bound", slot
);
779 RETURN(value_manager
.get_value_lower_bound(slot
));
783 GlassDatabase::get_value_upper_bound(Xapian::valueno slot
) const
785 LOGCALL(DB
, std::string
, "GlassDatabase::get_value_upper_bound", slot
);
786 RETURN(value_manager
.get_value_upper_bound(slot
));
790 GlassDatabase::get_doclength_lower_bound() const
792 return version_file
.get_doclength_lower_bound();
796 GlassDatabase::get_doclength_upper_bound() const
798 return version_file
.get_doclength_upper_bound();
802 GlassDatabase::get_wdf_upper_bound(string_view term
) const
804 Assert(!term
.empty());
805 Xapian::termcount wdfub
;
806 postlist_table
.get_freqs(term
, NULL
, NULL
, &wdfub
);
807 return min(wdfub
, version_file
.get_wdf_upper_bound());
811 GlassDatabase::get_unique_terms_lower_bound() const
813 return version_file
.get_unique_terms_lower_bound();
817 GlassDatabase::term_exists(string_view term
) const
819 LOGCALL(DB
, bool, "GlassDatabase::term_exists", term
);
821 RETURN(get_doccount() != 0);
823 RETURN(postlist_table
.term_exists(term
));
827 GlassDatabase::has_positions() const
829 return !position_table
.empty();
833 GlassDatabase::open_post_list(string_view term
) const
835 LOGCALL(DB
, PostList
*, "GlassDatabase::open_post_list", term
);
836 RETURN(GlassDatabase::open_leaf_post_list(term
, false));
840 GlassDatabase::open_leaf_post_list(string_view term
, bool need_read_pos
) const
842 LOGCALL(DB
, LeafPostList
*, "GlassDatabase::open_leaf_post_list", term
| need_read_pos
);
844 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
847 Assert(!need_read_pos
);
848 Xapian::doccount doccount
= get_doccount();
849 if (rare(doccount
== 0)) {
852 if (version_file
.get_last_docid() == doccount
) {
853 RETURN(new ContiguousAllDocsPostList(doccount
));
855 RETURN(new GlassAllDocsPostList(ptrtothis
, doccount
));
858 auto pl
= new GlassPostList(ptrtothis
, term
, true);
859 if (pl
->get_termfreq() == 0) {
867 GlassDatabase::open_value_list(Xapian::valueno slot
) const
869 LOGCALL(DB
, ValueList
*, "GlassDatabase::open_value_list", slot
);
870 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
871 RETURN(new GlassValueList(slot
, ptrtothis
));
875 GlassDatabase::open_term_list(Xapian::docid did
) const
877 LOGCALL(DB
, TermList
*, "GlassDatabase::open_term_list", did
);
879 if (!termlist_table
.is_open())
880 throw_termlist_table_close_exception();
881 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
882 RETURN(new GlassTermList(ptrtothis
, did
));
886 GlassDatabase::open_term_list_direct(Xapian::docid did
) const
888 return GlassDatabase::open_term_list(did
);
891 Xapian::Document::Internal
*
892 GlassDatabase::open_document(Xapian::docid did
, bool lazy
) const
894 LOGCALL(DB
, Xapian::Document::Internal
*, "GlassDatabase::open_document", did
| lazy
);
897 // This will throw DocNotFoundError if the document doesn't exist.
898 (void)get_doclength(did
);
901 intrusive_ptr
<const Database::Internal
> ptrtothis(this);
902 RETURN(new GlassDocument(ptrtothis
, did
, &value_manager
, &docdata_table
));
906 GlassDatabase::read_position_list(GlassRePositionList
* pos_list
,
908 string_view term
) const
911 pos_list
->read_data(did
, term
);
915 GlassDatabase::positionlist_count(Xapian::docid did
, string_view term
) const
917 return position_table
.positionlist_count(did
, term
);
921 GlassDatabase::open_position_list(Xapian::docid did
, string_view term
) const
924 return position_table
.open_position_list(did
, term
);
928 GlassDatabase::open_allterms(string_view prefix
) const
930 LOGCALL(DB
, TermList
*, "GlassDatabase::open_allterms", prefix
);
931 RETURN(new GlassAllTermsList(intrusive_ptr
<const GlassDatabase
>(this),
936 GlassDatabase::open_spelling_termlist(string_view word
) const
938 return spelling_table
.open_termlist(word
);
942 GlassDatabase::open_spelling_wordlist() const
944 GlassCursor
* cursor
= spelling_table
.cursor_get();
945 if (!cursor
) return NULL
;
946 return new GlassSpellingWordsList(intrusive_ptr
<const GlassDatabase
>(this),
951 GlassDatabase::get_spelling_frequency(string_view word
) const
953 return spelling_table
.get_word_frequency(word
);
957 GlassDatabase::open_synonym_termlist(string_view term
) const
959 return synonym_table
.open_termlist(term
);
963 GlassDatabase::open_synonym_keylist(string_view prefix
) const
965 GlassCursor
* cursor
= synonym_table
.cursor_get();
966 if (!cursor
) return NULL
;
967 return new GlassSynonymTermList(intrusive_ptr
<const GlassDatabase
>(this),
972 GlassDatabase::get_metadata(string_view key
) const
974 LOGCALL(DB
, string
, "GlassDatabase::get_metadata", key
);
975 string
btree_key("\x00\xc0", 2);
978 (void)postlist_table
.get_exact_entry(btree_key
, tag
);
983 GlassDatabase::open_metadata_keylist(std::string_view prefix
) const
985 LOGCALL(DB
, TermList
*, "GlassDatabase::open_metadata_keylist", prefix
);
986 GlassCursor
* cursor
= postlist_table
.cursor_get();
987 if (!cursor
) RETURN(NULL
);
988 RETURN(new GlassMetadataTermList(intrusive_ptr
<const GlassDatabase
>(this),
993 GlassDatabase::get_revision() const
995 LOGCALL(DB
, Xapian::rev
, "GlassDatabase::get_revision", NO_ARGS
);
996 RETURN(version_file
.get_revision());
1000 GlassDatabase::get_uuid() const
1002 LOGCALL(DB
, string
, "GlassDatabase::get_uuid", NO_ARGS
);
1003 RETURN(version_file
.get_uuid_string());
1007 GlassDatabase::throw_termlist_table_close_exception() const
1009 // Either the database has been closed, or else there's no termlist table.
1010 // Check if the postlist table is open to determine which is the case.
1011 if (!postlist_table
.is_open())
1012 GlassTable::throw_database_closed();
1013 throw Xapian::FeatureUnavailableError("Database has no termlist");
1017 GlassDatabase::get_used_docid_range(Xapian::docid
& first
,
1018 Xapian::docid
& last
) const
1020 last
= version_file
.get_last_docid();
1021 if (last
== version_file
.get_doccount()) {
1022 // Contiguous range starting at 1.
1026 postlist_table
.get_used_docid_range(first
, last
);
1030 GlassDatabase::has_uncommitted_changes() const
1036 GlassDatabase::locked() const
1042 GlassDatabase::update_lock(int flags
)
1044 if (!postlist_table
.is_open())
1045 GlassTable::throw_database_closed();
1047 if (flags
== Xapian::DB_READONLY_
) return this;
1048 // Mask out backend type bits as these don't make sense here, and set the
1049 // action to DB_OPEN since we don't want to create or overwrite here.
1050 flags
&= ~(DB_ACTION_MASK_
| DB_BACKEND_MASK_
);
1051 flags
|= Xapian::DB_OPEN
;
1052 return new GlassWritableDatabase(db_dir
, Xapian::DB_OPEN
, flags
);
1056 GlassDatabase::get_description() const
1058 string desc
= "Glass(";
1060 desc
+= "writable, ";
1067 ///////////////////////////////////////////////////////////////////////////
1069 GlassWritableDatabase::GlassWritableDatabase(string_view dir
, int flags
,
1071 : GlassDatabase(dir
, flags
, block_size
),
1074 modify_shortcut_document(NULL
),
1075 modify_shortcut_docid(0)
1077 LOGCALL_CTOR(DB
, "GlassWritableDatabase", dir
| flags
| block_size
);
1079 const char *p
= getenv("XAPIAN_FLUSH_THRESHOLD");
1081 if (!parse_unsigned(p
, flush_threshold
)) {
1082 throw Xapian::InvalidArgumentError("XAPIAN_FLUSH_THRESHOLD must "
1083 "be a non-negative integer");
1086 if (flush_threshold
== 0)
1087 flush_threshold
= 10000;
1090 GlassWritableDatabase::~GlassWritableDatabase()
1092 LOGCALL_DTOR(DB
, "GlassWritableDatabase");
1097 GlassWritableDatabase::commit()
1099 if (transaction_active())
1100 throw Xapian::InvalidOperationError("Can't commit during a transaction");
1101 if (change_count
) flush_postlist_changes();
1106 GlassWritableDatabase::check_flush_threshold()
1108 // FIXME: this should be done by checking memory usage, not the number of
1109 // changes. We could also look at the amount of data the inverter object
1111 if (++change_count
>= flush_threshold
) {
1112 flush_postlist_changes();
1113 if (!transaction_active()) apply();
1118 GlassWritableDatabase::flush_postlist_changes()
1121 version_file
.set_oldest_changeset(changes
.get_oldest_changeset());
1122 inverter
.flush(postlist_table
);
1123 inverter
.flush_pos_lists(position_table
);
1128 GlassWritableDatabase::cancel();
1136 GlassWritableDatabase::close()
1138 LOGCALL_VOID(DB
, "GlassWritableDatabase::close", NO_ARGS
);
1139 if (!transaction_active()) {
1141 // FIXME: if commit() throws, should we still close?
1143 GlassDatabase::close();
1147 GlassWritableDatabase::apply()
1149 value_manager
.set_value_stats(value_stats
);
1150 GlassDatabase::apply();
1154 GlassWritableDatabase::add_document(const Xapian::Document
& document
)
1156 LOGCALL(DB
, Xapian::docid
, "GlassWritableDatabase::add_document", document
);
1157 // Make sure the docid counter doesn't overflow.
1158 if (version_file
.get_last_docid() == GLASS_MAX_DOCID
)
1159 throw Xapian::DatabaseError("Run out of docids - you'll have to use copydatabase to eliminate any gaps before you can add more documents");
1160 // Use the next unused document ID.
1161 RETURN(add_document_(version_file
.get_next_docid(), document
));
1165 GlassWritableDatabase::add_document_(Xapian::docid did
,
1166 const Xapian::Document
& document
)
1168 LOGCALL(DB
, Xapian::docid
, "GlassWritableDatabase::add_document_", did
| document
);
1171 // Set the document data.
1172 docdata_table
.replace_document_data(did
, document
.get_data());
1175 value_manager
.add_document(did
, document
, value_stats
);
1177 Xapian::termcount new_doclen
= 0;
1179 Xapian::TermIterator term
= document
.termlist_begin();
1180 for ( ; term
!= document
.termlist_end(); ++term
) {
1181 termcount wdf
= term
.get_wdf();
1182 // Calculate the new document length
1184 version_file
.check_wdf(wdf
);
1186 string tname
= *term
;
1187 if (tname
.size() > MAX_SAFE_TERM_LENGTH
)
1188 throw Xapian::InvalidArgumentError("Term too long (> " STRINGIZE(MAX_SAFE_TERM_LENGTH
) "): " + tname
);
1190 inverter
.add_posting(did
, tname
, wdf
);
1191 inverter
.set_positionlist(position_table
, did
, tname
, term
);
1194 LOGLINE(DB
, "Calculated doclen for new document " << did
<< " as " << new_doclen
);
1196 // Set the termlist.
1197 if (termlist_table
.is_open())
1198 termlist_table
.set_termlist(did
, document
, new_doclen
);
1200 // Set the new document length
1201 inverter
.set_doclength(did
, new_doclen
, true);
1202 version_file
.add_document(new_doclen
);
1204 // If an error occurs while adding a document, or doing any other
1205 // transaction, the modifications so far must be cleared before
1206 // returning control to the user - otherwise partial modifications will
1207 // persist in memory, and eventually get written to disk.
1212 check_flush_threshold();
1218 GlassWritableDatabase::delete_document(Xapian::docid did
)
1220 LOGCALL_VOID(DB
, "GlassWritableDatabase::delete_document", did
);
1223 if (!termlist_table
.is_open())
1224 throw_termlist_table_close_exception();
1226 // Remove the document data. If this fails, just propagate the exception since
1227 // the state should still be consistent.
1228 bool doc_really_existed
= docdata_table
.delete_document_data(did
);
1230 if (rare(modify_shortcut_docid
== did
)) {
1231 // The modify_shortcut document can't be used for a modification
1232 // shortcut now, because it's been deleted!
1233 modify_shortcut_document
= NULL
;
1234 modify_shortcut_docid
= 0;
1235 doc_really_existed
= true;
1238 if (!doc_really_existed
) {
1239 // Ensure we throw DocumentNotFound if the document doesn't exist.
1240 (void)get_doclength(did
);
1244 // Remove the values.
1245 value_manager
.delete_document(did
, value_stats
);
1247 // OK, now add entries to remove the postings in the underlying record.
1248 intrusive_ptr
<const GlassWritableDatabase
> ptrtothis(this);
1249 GlassTermList
termlist(ptrtothis
, did
);
1251 version_file
.delete_document(termlist
.get_doclength());
1253 while (termlist
.next() == NULL
) {
1254 string tname
= termlist
.get_termname();
1255 inverter
.delete_positionlist(did
, tname
);
1257 inverter
.remove_posting(did
, tname
, termlist
.get_wdf());
1260 // Remove the termlist.
1261 if (termlist_table
.is_open())
1262 termlist_table
.delete_termlist(did
);
1264 // Mark this document as removed.
1265 inverter
.delete_doclength(did
);
1267 // If an error occurs while deleting a document, or doing any other
1268 // transaction, the modifications so far must be cleared before
1269 // returning control to the user - otherwise partial modifications will
1270 // persist in memory, and eventually get written to disk.
1275 check_flush_threshold();
1279 GlassWritableDatabase::replace_document(Xapian::docid did
,
1280 const Xapian::Document
& document
)
1282 LOGCALL_VOID(DB
, "GlassWritableDatabase::replace_document", did
| document
);
1286 if (did
> version_file
.get_last_docid()) {
1287 version_file
.set_last_docid(did
);
1288 // If this docid is above the highwatermark, then we can't be
1289 // replacing an existing document.
1290 (void)add_document_(did
, document
);
1294 if (!termlist_table
.is_open()) {
1295 // We can replace an *unused* docid <= last_docid too.
1296 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
1297 if (!postlist_table
.document_exists(did
, ptrtothis
)) {
1298 (void)add_document_(did
, document
);
1301 throw_termlist_table_close_exception();
1304 // Check for a document read from this database being replaced - ie, a
1305 // modification operation.
1306 bool modifying
= false;
1307 if (modify_shortcut_docid
&&
1308 document
.internal
->get_docid() == modify_shortcut_docid
) {
1309 if (document
.internal
.get() == modify_shortcut_document
) {
1310 // We have a docid, it matches, and the pointer matches, so we
1311 // can skip modification of any data which hasn't been modified
1313 if (!document
.internal
->modified()) {
1314 // If the document is unchanged, we've nothing to do.
1318 LOGLINE(DB
, "Detected potential document modification shortcut.");
1320 // The modify_shortcut document can't be used for a
1321 // modification shortcut now, because it's about to be
1323 modify_shortcut_document
= NULL
;
1324 modify_shortcut_docid
= 0;
1328 if (!modifying
|| document
.internal
->terms_modified()) {
1329 bool pos_modified
= !modifying
||
1330 document
.internal
->positions_modified();
1332 // Avoid the cost of throwing and handling an exception if
1333 // there's not already a document with docid did.
1334 intrusive_ptr
<const GlassWritableDatabase
> ptrtothis(this);
1335 GlassTermList
termlist(ptrtothis
, did
, false);
1336 if (termlist
.not_present()) {
1337 (void)add_document_(did
, document
);
1341 Xapian::TermIterator term
= document
.termlist_begin();
1342 Xapian::termcount old_doclen
= termlist
.get_doclength();
1343 version_file
.delete_document(old_doclen
);
1344 Xapian::termcount new_doclen
= old_doclen
;
1346 string old_tname
, new_tname
;
1348 bool termlist_at_end
= (termlist
.next() != NULL
);
1349 while (!termlist_at_end
|| term
!= document
.termlist_end()) {
1351 if (termlist_at_end
) {
1355 old_tname
= termlist
.get_termname();
1356 if (term
!= document
.termlist_end()) {
1358 cmp
= old_tname
.compare(new_tname
);
1365 // Term old_tname has been deleted.
1366 termcount old_wdf
= termlist
.get_wdf();
1367 new_doclen
-= old_wdf
;
1368 inverter
.remove_posting(did
, old_tname
, old_wdf
);
1370 inverter
.delete_positionlist(did
, old_tname
);
1371 termlist_at_end
= (termlist
.next() != NULL
);
1372 } else if (cmp
> 0) {
1373 // Term new_tname as been added.
1374 termcount new_wdf
= term
.get_wdf();
1375 new_doclen
+= new_wdf
;
1376 version_file
.check_wdf(new_wdf
);
1377 if (new_tname
.size() > MAX_SAFE_TERM_LENGTH
)
1378 throw Xapian::InvalidArgumentError("Term too long (> " STRINGIZE(MAX_SAFE_TERM_LENGTH
) "): " + new_tname
);
1379 inverter
.add_posting(did
, new_tname
, new_wdf
);
1381 inverter
.set_positionlist(position_table
, did
, new_tname
, term
);
1384 } else if (cmp
== 0) {
1385 // Term already exists: look for wdf and positionlist changes.
1386 termcount old_wdf
= termlist
.get_wdf();
1387 termcount new_wdf
= term
.get_wdf();
1389 // Check the stats even if wdf hasn't changed, because if
1390 // this is the only document, the stats will have been
1392 version_file
.check_wdf(new_wdf
);
1394 if (old_wdf
!= new_wdf
) {
1395 new_doclen
= new_doclen
- old_wdf
+ new_wdf
;
1396 inverter
.update_posting(did
, new_tname
, old_wdf
, new_wdf
);
1400 inverter
.set_positionlist(position_table
, did
, new_tname
, term
, true);
1404 termlist_at_end
= (termlist
.next() != NULL
);
1407 LOGLINE(DB
, "Calculated doclen for replacement document " << did
<< " as " << new_doclen
);
1409 // Set the termlist.
1410 if (termlist_table
.is_open())
1411 termlist_table
.set_termlist(did
, document
, new_doclen
);
1413 // Set the new document length
1414 if (new_doclen
!= old_doclen
)
1415 inverter
.set_doclength(did
, new_doclen
, false);
1416 version_file
.add_document(new_doclen
);
1419 if (!modifying
|| document
.internal
->data_modified()) {
1420 // Update the document data.
1421 docdata_table
.replace_document_data(did
, document
.get_data());
1424 if (!modifying
|| document
.internal
->values_modified()) {
1425 // Replace the values.
1426 value_manager
.replace_document(did
, document
, value_stats
);
1429 // If an error occurs while replacing a document, or doing any other
1430 // transaction, the modifications so far must be cleared before
1431 // returning control to the user - otherwise partial modifications will
1432 // persist in memory, and eventually get written to disk.
1437 check_flush_threshold();
1440 Xapian::Document::Internal
*
1441 GlassWritableDatabase::open_document(Xapian::docid did
, bool lazy
) const
1443 LOGCALL(DB
, Xapian::Document::Internal
*, "GlassWritableDatabase::open_document", did
| lazy
);
1444 modify_shortcut_document
= GlassDatabase::open_document(did
, lazy
);
1445 // Store the docid only after open_document() successfully returns, so an
1446 // attempt to open a missing document doesn't overwrite this.
1447 modify_shortcut_docid
= did
;
1448 RETURN(modify_shortcut_document
);
1452 GlassWritableDatabase::get_doclength(Xapian::docid did
) const
1454 LOGCALL(DB
, Xapian::termcount
, "GlassWritableDatabase::get_doclength", did
);
1455 Xapian::termcount doclen
;
1456 if (inverter
.get_doclength(did
, doclen
))
1458 RETURN(GlassDatabase::get_doclength(did
));
1462 GlassWritableDatabase::get_unique_terms(Xapian::docid did
) const
1464 LOGCALL(DB
, Xapian::termcount
, "GlassWritableDatabase::get_unique_terms", did
);
1466 // Note that the "approximate" size should be exact in this case.
1468 // get_unique_terms() really ought to only count terms with wdf > 0, but
1469 // that's expensive to calculate on demand, so for now let's just ensure
1470 // unique_terms <= doclen.
1471 Xapian::termcount doclen
;
1472 if (inverter
.get_doclength(did
, doclen
)) {
1473 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
1474 GlassTermList
termlist(ptrtothis
, did
);
1475 RETURN(min(doclen
, termlist
.get_approx_size()));
1477 RETURN(GlassDatabase::get_unique_terms(did
));
1481 GlassWritableDatabase::get_freqs(string_view term
,
1482 Xapian::doccount
* termfreq_ptr
,
1483 Xapian::termcount
* collfreq_ptr
) const
1485 LOGCALL_VOID(DB
, "GlassWritableDatabase::get_freqs", term
| termfreq_ptr
| collfreq_ptr
);
1486 Assert(!term
.empty());
1487 GlassDatabase::get_freqs(term
, termfreq_ptr
, collfreq_ptr
);
1488 Xapian::termcount tf_delta
, cf_delta
;
1489 if (inverter
.get_deltas(term
, tf_delta
, cf_delta
)) {
1491 UNSIGNED_OVERFLOW_OK(*termfreq_ptr
+= tf_delta
);
1493 UNSIGNED_OVERFLOW_OK(*collfreq_ptr
+= cf_delta
);
1498 GlassWritableDatabase::get_value_freq(Xapian::valueno slot
) const
1500 LOGCALL(DB
, Xapian::doccount
, "GlassWritableDatabase::get_value_freq", slot
);
1501 map
<Xapian::valueno
, ValueStats
>::const_iterator i
;
1502 i
= value_stats
.find(slot
);
1503 if (i
!= value_stats
.end()) RETURN(i
->second
.freq
);
1504 RETURN(GlassDatabase::get_value_freq(slot
));
1508 GlassWritableDatabase::get_value_lower_bound(Xapian::valueno slot
) const
1510 LOGCALL(DB
, std::string
, "GlassWritableDatabase::get_value_lower_bound", slot
);
1511 map
<Xapian::valueno
, ValueStats
>::const_iterator i
;
1512 i
= value_stats
.find(slot
);
1513 if (i
!= value_stats
.end()) RETURN(i
->second
.lower_bound
);
1514 RETURN(GlassDatabase::get_value_lower_bound(slot
));
1518 GlassWritableDatabase::get_value_upper_bound(Xapian::valueno slot
) const
1520 LOGCALL(DB
, std::string
, "GlassWritableDatabase::get_value_upper_bound", slot
);
1521 map
<Xapian::valueno
, ValueStats
>::const_iterator i
;
1522 i
= value_stats
.find(slot
);
1523 if (i
!= value_stats
.end()) RETURN(i
->second
.upper_bound
);
1524 RETURN(GlassDatabase::get_value_upper_bound(slot
));
1528 GlassWritableDatabase::term_exists(string_view term
) const
1530 LOGCALL(DB
, bool, "GlassWritableDatabase::term_exists", term
);
1532 RETURN(get_doccount() != 0);
1534 Xapian::doccount tf
;
1535 get_freqs(term
, &tf
, NULL
);
1540 GlassWritableDatabase::has_positions() const
1542 return inverter
.has_positions(position_table
);
1546 GlassWritableDatabase::open_post_list(string_view term
) const
1548 LOGCALL(DB
, PostList
*, "GlassWritableDatabase::open_post_list", term
);
1549 RETURN(GlassWritableDatabase::open_leaf_post_list(term
, false));
1553 GlassWritableDatabase::open_leaf_post_list(string_view term
,
1554 bool need_read_pos
) const
1556 LOGCALL(DB
, LeafPostList
*, "GlassWritableDatabase::open_leaf_post_list", term
| need_read_pos
);
1557 (void)need_read_pos
;
1558 intrusive_ptr
<const GlassWritableDatabase
> ptrtothis(this);
1561 Assert(!need_read_pos
);
1562 Xapian::doccount doccount
= get_doccount();
1563 if (rare(doccount
== 0)) {
1566 if (version_file
.get_last_docid() == doccount
) {
1567 RETURN(new ContiguousAllDocsPostList(doccount
));
1569 inverter
.flush_doclengths(postlist_table
);
1570 RETURN(new GlassAllDocsPostList(ptrtothis
, doccount
));
1573 // Flush any buffered changes for this term's postlist so we can just
1574 // iterate from the flushed state.
1575 inverter
.flush_post_list(postlist_table
, term
);
1577 auto pl
= new GlassPostList(ptrtothis
, term
, true);
1578 if (pl
->get_termfreq() == 0) {
1586 GlassWritableDatabase::open_value_list(Xapian::valueno slot
) const
1588 LOGCALL(DB
, ValueList
*, "GlassWritableDatabase::open_value_list", slot
);
1589 // If there are changes, we don't have code to iterate the modified value
1590 // list so we need to flush (but don't commit - there may be a transaction
1592 if (change_count
) value_manager
.merge_changes();
1593 RETURN(GlassDatabase::open_value_list(slot
));
1597 GlassWritableDatabase::read_position_list(GlassRePositionList
* pos_list
,
1599 string_view term
) const
1603 if (inverter
.get_positionlist(did
, term
, data
)) {
1604 pos_list
->assign_data(std::move(data
));
1607 GlassDatabase::read_position_list(pos_list
, did
, term
);
1611 GlassWritableDatabase::positionlist_count(Xapian::docid did
,
1612 string_view term
) const
1616 if (inverter
.get_positionlist(did
, term
, data
)) {
1619 return position_table
.positionlist_count(data
);
1621 return GlassDatabase::positionlist_count(did
, term
);
1625 GlassWritableDatabase::open_position_list(Xapian::docid did
,
1626 string_view term
) const
1630 if (inverter
.get_positionlist(did
, term
, data
)) {
1631 if (data
.empty()) return nullptr;
1632 return new GlassPositionList(std::move(data
));
1634 return GlassDatabase::open_position_list(did
, term
);
1638 GlassWritableDatabase::open_allterms(string_view prefix
) const
1640 LOGCALL(DB
, TermList
*, "GlassWritableDatabase::open_allterms", prefix
);
1642 // There are changes, and terms may have been added or removed, and so
1643 // we need to flush changes for terms with the specified prefix (but
1644 // don't commit - there may be a transaction in progress).
1645 inverter
.flush_post_lists(postlist_table
, prefix
);
1646 if (prefix
.empty()) {
1647 // We've flushed all the posting list changes, but the positions,
1648 // document lengths and stats haven't been written, so set
1649 // change_count to 1.
1650 // FIXME: Can we handle this better?
1654 RETURN(GlassDatabase::open_allterms(prefix
));
1658 GlassWritableDatabase::cancel()
1660 GlassDatabase::cancel();
1662 value_stats
.clear();
1667 GlassWritableDatabase::add_spelling(string_view word
,
1668 Xapian::termcount freqinc
) const
1670 spelling_table
.add_word(word
, freqinc
);
1674 GlassWritableDatabase::remove_spelling(string_view word
,
1675 Xapian::termcount freqdec
) const
1677 return spelling_table
.remove_word(word
, freqdec
);
1681 GlassWritableDatabase::open_spelling_wordlist() const
1683 spelling_table
.merge_changes();
1684 return GlassDatabase::open_spelling_wordlist();
1688 GlassWritableDatabase::open_synonym_keylist(string_view prefix
) const
1690 synonym_table
.merge_changes();
1691 return GlassDatabase::open_synonym_keylist(prefix
);
1695 GlassWritableDatabase::add_synonym(string_view term
,
1696 string_view synonym
) const
1698 synonym_table
.add_synonym(term
, synonym
);
1702 GlassWritableDatabase::remove_synonym(string_view term
,
1703 string_view synonym
) const
1705 synonym_table
.remove_synonym(term
, synonym
);
1709 GlassWritableDatabase::clear_synonyms(string_view term
) const
1711 synonym_table
.clear_synonyms(term
);
1715 GlassWritableDatabase::set_metadata(string_view key
, string_view value
)
1717 LOGCALL_VOID(DB
, "GlassWritableDatabase::set_metadata", key
| value
);
1718 string
btree_key("\x00\xc0", 2);
1720 if (value
.empty()) {
1721 postlist_table
.del(btree_key
);
1723 postlist_table
.add(btree_key
, value
);
1728 GlassWritableDatabase::invalidate_doc_object(Xapian::Document::Internal
* obj
) const
1730 if (obj
== modify_shortcut_document
) {
1731 modify_shortcut_document
= NULL
;
1732 modify_shortcut_docid
= 0;
1737 GlassWritableDatabase::has_uncommitted_changes() const
1739 return change_count
> 0 ||
1740 postlist_table
.is_modified() ||
1741 position_table
.is_modified() ||
1742 termlist_table
.is_modified() ||
1743 value_manager
.is_modified() ||
1744 synonym_table
.is_modified() ||
1745 spelling_table
.is_modified() ||
1746 docdata_table
.is_modified();
1750 GlassWritableDatabase::update_lock(int flags
)
1752 if (!postlist_table
.is_open())
1753 GlassTable::throw_database_closed();
1755 if (flags
!= Xapian::DB_READONLY_
) {
1756 // Allow changing flags on an open DB.
1757 postlist_table
.set_flags(flags
);
1758 position_table
.set_flags(flags
);
1759 termlist_table
.set_flags(flags
);
1760 synonym_table
.set_flags(flags
);
1761 spelling_table
.set_flags(flags
);
1762 docdata_table
.set_flags(flags
);
1766 unique_ptr
<Database::Internal
> result(new GlassDatabase(db_dir
));
1768 return result
.release();
1771 #ifdef DISABLE_GPL_LIBXAPIAN
1772 # error GPL source we cannot relicense included in libxapian