1 /* glass_database.cc: glass database
3 * Copyright 1999,2000,2001 BrightStation PLC
4 * Copyright 2001 Hein Ragas
5 * Copyright 2002 Ananova Ltd
6 * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016 Olly Betts
7 * Copyright 2006,2008 Lemur Consulting Ltd
8 * Copyright 2009 Richard Boulton
9 * Copyright 2009 Kan-Ru Chen
10 * Copyright 2011 Dan Colish
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License as
14 * published by the Free Software Foundation; either version 2 of the
15 * License, or (at your option) any later version.
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
30 #include "glass_database.h"
32 #include "xapian/constants.h"
33 #include "xapian/error.h"
34 #include "xapian/valueiterator.h"
36 #include "backends/contiguousalldocspostlist.h"
37 #include "glass_alldocspostlist.h"
38 #include "glass_alltermslist.h"
39 #include "glass_defs.h"
40 #include "glass_docdata.h"
41 #include "glass_document.h"
42 #include "../flint_lock.h"
43 #include "glass_metadata.h"
44 #include "glass_positionlist.h"
45 #include "glass_postlist.h"
46 #include "glass_replicate_internal.h"
47 #include "glass_spellingwordslist.h"
48 #include "glass_termlist.h"
49 #include "glass_valuelist.h"
50 #include "glass_values.h"
55 #include "net/remoteconnection.h"
56 #include "api/replication.h"
57 #include "replicationprotocol.h"
58 #include "net/length.h"
59 #include "posixy_wrapper.h"
61 #include "stringutils.h"
62 #include "backends/valuestats.h"
64 #include "safeerrno.h"
65 #include "safesysstat.h"
66 #include <sys/types.h>
74 using namespace Xapian
;
75 using Xapian::Internal::intrusive_ptr
;
77 // The maximum safe term length is determined by the postlist. There we
78 // store the term using pack_string_preserving_sort() which takes the
79 // length of the string plus an extra byte (assuming the string doesn't
80 // contain any zero bytes), followed by the docid with encoded with
81 // pack_uint_preserving_sort() which takes up to 5 bytes (for a 32-bit
84 // The Btree manager's key length limit is 255 bytes so the maximum safe term
85 // length is 255 - 1 - 5 = 249 bytes. We actually set the limit at 245 for
86 // consistency with flint and chert, and also because this allows for 64-bit
89 // If the term contains zero bytes, the limit is lower (by one for each zero
91 #define MAX_SAFE_TERM_LENGTH 245
93 /* This opens the tables, determining the current and next revision numbers,
94 * and stores handles to the tables.
96 GlassDatabase::GlassDatabase(const string
&glass_dir
, int flags
,
97 unsigned int block_size
)
99 readonly(flags
== Xapian::DB_READONLY_
),
100 version_file(db_dir
),
101 postlist_table(db_dir
, readonly
),
102 position_table(db_dir
, readonly
),
103 // Note: (Xapian::DB_READONLY_ & Xapian::DB_NO_TERMLIST) is true,
104 // so opening to read we always permit the termlist to be missing.
105 termlist_table(db_dir
, readonly
, (flags
& Xapian::DB_NO_TERMLIST
)),
106 value_manager(&postlist_table
, &termlist_table
),
107 synonym_table(db_dir
, readonly
),
108 spelling_table(db_dir
, readonly
),
109 docdata_table(db_dir
, readonly
),
113 LOGCALL_CTOR(DB
, "GlassDatabase", glass_dir
| flags
| block_size
);
120 // Block size must in the range 2048..65536, and a power of two.
121 if (block_size
< 2048 || block_size
> 65536 ||
122 (block_size
& (block_size
- 1)) != 0) {
123 block_size
= GLASS_DEFAULT_BLOCKSIZE
;
126 int action
= flags
& Xapian::DB_ACTION_MASK_
;
127 if (action
!= Xapian::DB_OPEN
&& !database_exists()) {
129 // Create the directory for the database, if it doesn't exist
133 if (stat(db_dir
.c_str(), &statbuf
) == 0) {
134 if (!S_ISDIR(statbuf
.st_mode
)) fail
= true;
135 } else if (errno
!= ENOENT
|| mkdir(db_dir
.c_str(), 0755) == -1) {
139 throw Xapian::DatabaseCreateError("Cannot create directory '" +
140 db_dir
+ "'", errno
);
142 get_database_write_lock(flags
, true);
144 create_and_open_tables(flags
, block_size
);
148 if (action
== Xapian::DB_CREATE
) {
149 throw Xapian::DatabaseCreateError("Can't create new database at '" +
150 db_dir
+ "': a database already exists and I was told "
151 "not to overwrite it");
154 get_database_write_lock(flags
, false);
155 // if we're overwriting, pretend the db doesn't exist
156 if (action
== Xapian::DB_CREATE_OR_OVERWRITE
) {
157 create_and_open_tables(flags
, block_size
);
161 // Open the latest version of each table.
165 GlassDatabase::GlassDatabase(int fd
)
169 postlist_table(fd
, version_file
.get_offset(), readonly
),
170 position_table(fd
, version_file
.get_offset(), readonly
),
171 termlist_table(fd
, version_file
.get_offset(), readonly
, true),
172 value_manager(&postlist_table
, &termlist_table
),
173 synonym_table(fd
, version_file
.get_offset(), readonly
),
174 spelling_table(fd
, version_file
.get_offset(), readonly
),
175 docdata_table(fd
, version_file
.get_offset(), readonly
),
179 LOGCALL_CTOR(DB
, "GlassDatabase", fd
);
180 open_tables(Xapian::DB_READONLY_
);
183 GlassDatabase::~GlassDatabase()
185 LOGCALL_DTOR(DB
, "GlassDatabase");
189 GlassDatabase::database_exists() {
190 LOGCALL(DB
, bool, "GlassDatabase::database_exists", NO_ARGS
);
191 // The postlist table is the only non-optional one.
192 RETURN(postlist_table
.exists());
196 GlassDatabase::create_and_open_tables(int flags
, unsigned int block_size
)
198 LOGCALL_VOID(DB
, "GlassDatabase::create_and_open_tables", flags
|block_size
);
199 // The caller is expected to create the database directory if it doesn't
202 GlassVersion
&v
= version_file
;
203 v
.create(block_size
);
205 glass_revision_number_t rev
= v
.get_revision();
206 const string
& tmpfile
= v
.write(rev
, flags
);
208 position_table
.create_and_open(flags
, v
.get_root(Glass::POSITION
));
209 synonym_table
.create_and_open(flags
, v
.get_root(Glass::SYNONYM
));
210 spelling_table
.create_and_open(flags
, v
.get_root(Glass::SPELLING
));
211 docdata_table
.create_and_open(flags
, v
.get_root(Glass::DOCDATA
));
212 termlist_table
.create_and_open(flags
, v
.get_root(Glass::TERMLIST
));
213 postlist_table
.create_and_open(flags
, v
.get_root(Glass::POSTLIST
));
215 if (!v
.sync(tmpfile
, rev
, flags
)) {
216 throw Xapian::DatabaseCreateError("Failed to create iamglass file");
219 Assert(database_exists());
223 GlassDatabase::open_tables(int flags
)
225 LOGCALL(DB
, bool, "GlassDatabase::open_tables", flags
);
227 glass_revision_number_t cur_rev
= version_file
.get_revision();
230 // We're reopening, so ensure that we throw DatabaseError if close()
231 // was called. It could be argued that reopen() can be a no-op in this
232 // case, but that's confusing if someone expects that reopen() can undo
233 // the effects of close() - in fact reopen() is closer to
234 // update_reader_to_latest_revision().
235 if (!postlist_table
.is_open())
236 GlassTable::throw_database_closed();
240 glass_revision_number_t rev
= version_file
.get_revision();
241 if (cur_rev
&& cur_rev
== rev
) {
242 // We're reopening a database and the revision hasn't changed so we
243 // don't need to do anything.
247 docdata_table
.open(flags
, version_file
.get_root(Glass::DOCDATA
), rev
);
248 spelling_table
.open(flags
, version_file
.get_root(Glass::SPELLING
), rev
);
249 synonym_table
.open(flags
, version_file
.get_root(Glass::SYNONYM
), rev
);
250 termlist_table
.open(flags
, version_file
.get_root(Glass::TERMLIST
), rev
);
251 position_table
.open(flags
, version_file
.get_root(Glass::POSITION
), rev
);
252 postlist_table
.open(flags
, version_file
.get_root(Glass::POSTLIST
), rev
);
254 Xapian::termcount swfub
= version_file
.get_spelling_wordfreq_upper_bound();
255 spelling_table
.set_wordfreq_upper_bound(swfub
);
257 value_manager
.reset();
260 changes
.set_oldest_changeset(version_file
.get_oldest_changeset());
261 glass_revision_number_t revision
= version_file
.get_revision();
262 GlassChanges
* p
= changes
.start(revision
, revision
+ 1, flags
);
263 version_file
.set_changes(p
);
264 postlist_table
.set_changes(p
);
265 position_table
.set_changes(p
);
266 termlist_table
.set_changes(p
);
267 synonym_table
.set_changes(p
);
268 spelling_table
.set_changes(p
);
269 docdata_table
.set_changes(p
);
274 glass_revision_number_t
275 GlassDatabase::get_revision_number() const
277 LOGCALL(DB
, glass_revision_number_t
, "GlassDatabase::get_revision_number", NO_ARGS
);
278 RETURN(version_file
.get_revision());
281 glass_revision_number_t
282 GlassDatabase::get_next_revision_number() const
284 LOGCALL(DB
, glass_revision_number_t
, "GlassDatabase::get_next_revision_number", NO_ARGS
);
285 // FIXME: If we permit a revision before the latest and then updating it
286 // (to roll back more recent changes) then we (probably) need this to be
287 // one more than the *highest* revision previously committed.
288 RETURN(version_file
.get_revision() + 1);
292 GlassDatabase::get_changeset_revisions(const string
& path
,
293 glass_revision_number_t
* startrev
,
294 glass_revision_number_t
* endrev
) const
296 FD
fd(posixy_open(path
.c_str(), O_RDONLY
| O_CLOEXEC
));
298 string message
= string("Couldn't open changeset ")
300 throw Xapian::DatabaseError(message
, errno
);
303 char buf
[REASONABLE_CHANGESET_SIZE
];
304 const char *start
= buf
;
305 const char *end
= buf
+ io_read(fd
, buf
, REASONABLE_CHANGESET_SIZE
);
306 if (size_t(end
- start
) < CONST_STRLEN(CHANGES_MAGIC_STRING
))
307 throw Xapian::DatabaseError("Changeset too short at " + path
);
308 if (memcmp(start
, CHANGES_MAGIC_STRING
,
309 CONST_STRLEN(CHANGES_MAGIC_STRING
)) != 0) {
310 string message
= string("Changeset at ")
311 + path
+ " does not contain valid magic string";
312 throw Xapian::DatabaseError(message
);
314 start
+= CONST_STRLEN(CHANGES_MAGIC_STRING
);
316 unsigned int changes_version
;
317 if (!unpack_uint(&start
, end
, &changes_version
))
318 throw Xapian::DatabaseError("Couldn't read a valid version number for "
319 "changeset at " + path
);
320 if (changes_version
!= CHANGES_VERSION
)
321 throw Xapian::DatabaseError("Don't support version of changeset at "
324 if (!unpack_uint(&start
, end
, startrev
))
325 throw Xapian::DatabaseError("Couldn't read a valid start revision from "
326 "changeset at " + path
);
328 if (!unpack_uint(&start
, end
, endrev
))
329 throw Xapian::DatabaseError("Couldn't read a valid end revision for "
330 "changeset at " + path
);
334 GlassDatabase::set_revision_number(int flags
, glass_revision_number_t new_revision
)
336 LOGCALL_VOID(DB
, "GlassDatabase::set_revision_number", flags
|new_revision
);
338 glass_revision_number_t rev
= version_file
.get_revision();
339 if (new_revision
<= rev
&& rev
!= 0) {
340 string m
= "New revision ";
341 m
+= str(new_revision
);
342 m
+= " <= old revision ";
344 throw Xapian::DatabaseError(m
);
347 value_manager
.merge_changes();
349 postlist_table
.flush_db();
350 position_table
.flush_db();
351 termlist_table
.flush_db();
352 synonym_table
.flush_db();
353 version_file
.set_spelling_wordfreq_upper_bound(spelling_table
.flush_db());
354 docdata_table
.flush_db();
356 postlist_table
.commit(new_revision
, version_file
.root_to_set(Glass::POSTLIST
));
357 position_table
.commit(new_revision
, version_file
.root_to_set(Glass::POSITION
));
358 termlist_table
.commit(new_revision
, version_file
.root_to_set(Glass::TERMLIST
));
359 synonym_table
.commit(new_revision
, version_file
.root_to_set(Glass::SYNONYM
));
360 spelling_table
.commit(new_revision
, version_file
.root_to_set(Glass::SPELLING
));
361 docdata_table
.commit(new_revision
, version_file
.root_to_set(Glass::DOCDATA
));
363 const string
& tmpfile
= version_file
.write(new_revision
, flags
);
364 if (!postlist_table
.sync() ||
365 !position_table
.sync() ||
366 !termlist_table
.sync() ||
367 !synonym_table
.sync() ||
368 !spelling_table
.sync() ||
369 !docdata_table
.sync() ||
370 !version_file
.sync(tmpfile
, new_revision
, flags
)) {
371 (void)unlink(tmpfile
.c_str());
372 throw Xapian::DatabaseError("Commit failed", errno
);
375 changes
.commit(new_revision
, flags
);
379 GlassDatabase::request_document(Xapian::docid did
) const
381 docdata_table
.readahead_for_document(did
);
385 GlassDatabase::readahead_for_query(const Xapian::Query
&query
)
387 Xapian::TermIterator t
;
388 for (t
= query
.get_unique_terms_begin(); t
!= Xapian::TermIterator(); ++t
) {
389 const string
& term
= *t
;
390 if (!postlist_table
.readahead_key(GlassPostListTable::make_key(term
)))
396 GlassDatabase::reopen()
398 LOGCALL(DB
, bool, "GlassDatabase::reopen", NO_ARGS
);
399 if (!readonly
) RETURN(false);
400 RETURN(open_tables(postlist_table
.get_flags()));
404 GlassDatabase::close()
406 LOGCALL_VOID(DB
, "GlassDatabase::close", NO_ARGS
);
407 postlist_table
.close(true);
408 position_table
.close(true);
409 termlist_table
.close(true);
410 synonym_table
.close(true);
411 spelling_table
.close(true);
412 docdata_table
.close(true);
417 GlassDatabase::get_database_write_lock(int flags
, bool creating
)
419 LOGCALL_VOID(DB
, "GlassDatabase::get_database_write_lock", flags
|creating
);
420 // FIXME: Handle Xapian::DB_DANGEROUS here, perhaps by having readers
421 // get a lock on the revision they're reading, and then requiring the
422 // writer get an exclusive lock in this case.
424 bool retry
= flags
& Xapian::DB_RETRY_LOCK
;
425 FlintLock::reason why
= lock
.lock(true, retry
, explanation
);
426 if (why
!= FlintLock::SUCCESS
) {
427 if (why
== FlintLock::UNKNOWN
&& !creating
&& !database_exists()) {
428 string
msg("No glass database found at path '");
431 throw Xapian::DatabaseOpeningError(msg
);
433 lock
.throw_databaselockerror(why
, db_dir
, explanation
);
438 GlassDatabase::send_whole_database(RemoteConnection
& conn
, double end_time
)
440 LOGCALL_VOID(DB
, "GlassDatabase::send_whole_database", conn
| end_time
);
442 // Send the current revision number in the header.
444 string uuid
= get_uuid();
445 buf
+= encode_length(uuid
.size());
447 pack_uint(buf
, get_revision_number());
448 conn
.send_message(REPL_REPLY_DB_HEADER
, buf
, end_time
);
450 // Send all the tables. The tables which we want to be cached best after
451 // the copy finishes are sent last.
452 static const char filenames
[] =
453 "termlist." GLASS_TABLE_EXTENSION
"\0"
454 "synonym." GLASS_TABLE_EXTENSION
"\0"
455 "spelling." GLASS_TABLE_EXTENSION
"\0"
456 "docdata." GLASS_TABLE_EXTENSION
"\0"
457 "position." GLASS_TABLE_EXTENSION
"\0"
458 "postlist." GLASS_TABLE_EXTENSION
"\0"
460 string filepath
= db_dir
;
462 const char * p
= filenames
;
464 size_t len
= strlen(p
);
465 filepath
.replace(db_dir
.size() + 1, string::npos
, p
, len
);
466 FD
fd(posixy_open(filepath
.c_str(), O_RDONLY
| O_CLOEXEC
));
468 conn
.send_message(REPL_REPLY_DB_FILENAME
, string(p
, len
), end_time
);
469 conn
.send_file(REPL_REPLY_DB_FILEDATA
, fd
, end_time
);
476 GlassDatabase::write_changesets_to_fd(int fd
,
477 const string
& revision
,
479 ReplicationInfo
* info
)
481 LOGCALL_VOID(DB
, "GlassDatabase::write_changesets_to_fd", fd
| revision
| need_whole_db
| info
);
483 int whole_db_copies_left
= MAX_DB_COPIES_PER_CONVERSATION
;
484 glass_revision_number_t start_rev_num
= 0;
485 string start_uuid
= get_uuid();
487 glass_revision_number_t needed_rev_num
= 0;
489 const char * rev_ptr
= revision
.data();
490 const char * rev_end
= rev_ptr
+ revision
.size();
491 if (!unpack_uint(&rev_ptr
, rev_end
, &start_rev_num
)) {
492 need_whole_db
= true;
495 RemoteConnection
conn(-1, fd
, string());
497 // While the starting revision number is less than the latest revision
498 // number, look for a changeset, and write it.
500 // FIXME - perhaps we should make hardlinks for all the changesets we're
501 // likely to need, first, and then start sending them, so that there's no
502 // risk of them disappearing while we're sending earlier ones.
505 // Decrease the counter of copies left to be sent, and fail
506 // if we've already copied the database enough. This ensures that
507 // synchronisation attempts always terminate eventually.
508 if (whole_db_copies_left
== 0) {
509 conn
.send_message(REPL_REPLY_FAIL
,
510 "Database changing too fast",
514 whole_db_copies_left
--;
516 // Send the whole database across.
517 start_rev_num
= get_revision_number();
518 start_uuid
= get_uuid();
520 send_whole_database(conn
, 0.0);
522 ++(info
->fullcopy_count
);
524 need_whole_db
= false;
527 if (start_uuid
== get_uuid()) {
528 // Send the latest revision number after sending the tables.
529 // The update must proceed to that revision number before the
530 // copy is safe to make live.
533 needed_rev_num
= get_revision_number();
534 pack_uint(buf
, needed_rev_num
);
535 conn
.send_message(REPL_REPLY_DB_FOOTER
, buf
, 0.0);
536 if (info
!= NULL
&& start_rev_num
== needed_rev_num
)
537 info
->changed
= true;
539 // Database has been replaced since we did the copy. Send a
540 // higher revision number than the revision we've just copied,
541 // so that the client doesn't make the copy we've just done
542 // live, and then mark that we need to do a copy again.
543 // The client will never actually get the required revision,
544 // because the next message is going to be the start of a new
545 // database transfer.
548 pack_uint(buf
, start_rev_num
+ 1);
549 conn
.send_message(REPL_REPLY_DB_FOOTER
, buf
, 0.0);
550 need_whole_db
= true;
553 // Check if we've sent all the updates.
554 if (start_rev_num
>= get_revision_number()) {
556 if (start_uuid
!= get_uuid()) {
557 need_whole_db
= true;
560 if (start_rev_num
>= get_revision_number()) {
565 // Look for the changeset for revision start_rev_num.
566 string changes_name
= db_dir
+ "/changes" + str(start_rev_num
);
567 FD
fd_changes(posixy_open(changes_name
.c_str(), O_RDONLY
| O_CLOEXEC
));
568 if (fd_changes
>= 0) {
569 // Send it, and also update start_rev_num to the new value
570 // specified in the changeset.
571 glass_revision_number_t changeset_start_rev_num
;
572 glass_revision_number_t changeset_end_rev_num
;
573 get_changeset_revisions(changes_name
,
574 &changeset_start_rev_num
,
575 &changeset_end_rev_num
);
576 if (changeset_start_rev_num
!= start_rev_num
) {
577 throw Xapian::DatabaseError("Changeset start revision does not match changeset filename");
579 if (changeset_start_rev_num
>= changeset_end_rev_num
) {
580 throw Xapian::DatabaseError("Changeset start revision is not less than end revision");
583 conn
.send_file(REPL_REPLY_CHANGESET
, fd_changes
, 0.0);
584 start_rev_num
= changeset_end_rev_num
;
586 ++(info
->changeset_count
);
587 if (start_rev_num
>= needed_rev_num
)
588 info
->changed
= true;
591 // The changeset doesn't exist: leave the revision number as it
592 // is, and mark for doing a full database copy.
593 need_whole_db
= true;
597 conn
.send_message(REPL_REPLY_END_OF_CHANGES
, string(), 0.0);
601 GlassDatabase::modifications_failed(glass_revision_number_t new_revision
,
602 const std::string
& msg
)
604 // Modifications failed. Wipe all the modifications from memory.
605 int flags
= postlist_table
.get_flags();
606 glass_revision_number_t old_revision
= version_file
.get_revision();
608 // Discard any buffered changes and reinitialised cached values
612 // Reopen tables with old revision number.
613 version_file
.cancel();
614 docdata_table
.open(flags
, version_file
.get_root(Glass::DOCDATA
), old_revision
);
615 spelling_table
.open(flags
, version_file
.get_root(Glass::SPELLING
), old_revision
);
616 synonym_table
.open(flags
, version_file
.get_root(Glass::SYNONYM
), old_revision
);
617 termlist_table
.open(flags
, version_file
.get_root(Glass::TERMLIST
), old_revision
);
618 position_table
.open(flags
, version_file
.get_root(Glass::POSITION
), old_revision
);
619 postlist_table
.open(flags
, version_file
.get_root(Glass::POSTLIST
), old_revision
);
621 Xapian::termcount ub
= version_file
.get_spelling_wordfreq_upper_bound();
622 spelling_table
.set_wordfreq_upper_bound(ub
);
624 value_manager
.reset();
626 // Increase revision numbers to new revision number plus one,
627 // writing increased numbers to all tables.
629 set_revision_number(flags
, new_revision
);
630 } catch (const Xapian::Error
&e
) {
631 // We failed to roll-back so close the database to avoid the risk of
632 // database corruption.
633 GlassDatabase::close();
634 throw Xapian::DatabaseError("Modifications failed (" + msg
+ "), "
635 "and couldn't open at the old revision: " +
640 p
= changes
.start(old_revision
, new_revision
, flags
);
641 version_file
.set_changes(p
);
642 postlist_table
.set_changes(p
);
643 position_table
.set_changes(p
);
644 termlist_table
.set_changes(p
);
645 synonym_table
.set_changes(p
);
646 spelling_table
.set_changes(p
);
647 docdata_table
.set_changes(p
);
651 GlassDatabase::apply()
653 LOGCALL_VOID(DB
, "GlassDatabase::apply", NO_ARGS
);
654 if (!postlist_table
.is_modified() &&
655 !position_table
.is_modified() &&
656 !termlist_table
.is_modified() &&
657 !value_manager
.is_modified() &&
658 !synonym_table
.is_modified() &&
659 !spelling_table
.is_modified() &&
660 !docdata_table
.is_modified()) {
664 glass_revision_number_t new_revision
= get_next_revision_number();
666 int flags
= postlist_table
.get_flags();
668 set_revision_number(flags
, new_revision
);
669 } catch (const Xapian::Error
&e
) {
670 modifications_failed(new_revision
, e
.get_description());
673 modifications_failed(new_revision
, "Unknown error");
678 p
= changes
.start(new_revision
, new_revision
+ 1, flags
);
679 version_file
.set_changes(p
);
680 postlist_table
.set_changes(p
);
681 position_table
.set_changes(p
);
682 termlist_table
.set_changes(p
);
683 synonym_table
.set_changes(p
);
684 spelling_table
.set_changes(p
);
685 docdata_table
.set_changes(p
);
689 GlassDatabase::cancel()
691 LOGCALL_VOID(DB
, "GlassDatabase::cancel", NO_ARGS
);
692 version_file
.cancel();
693 glass_revision_number_t rev
= version_file
.get_revision();
694 postlist_table
.cancel(version_file
.get_root(Glass::POSTLIST
), rev
);
695 position_table
.cancel(version_file
.get_root(Glass::POSITION
), rev
);
696 termlist_table
.cancel(version_file
.get_root(Glass::TERMLIST
), rev
);
697 value_manager
.cancel();
698 synonym_table
.cancel(version_file
.get_root(Glass::SYNONYM
), rev
);
699 spelling_table
.cancel(version_file
.get_root(Glass::SPELLING
), rev
);
700 docdata_table
.cancel(version_file
.get_root(Glass::DOCDATA
), rev
);
702 Xapian::termcount ub
= version_file
.get_spelling_wordfreq_upper_bound();
703 spelling_table
.set_wordfreq_upper_bound(ub
);
707 GlassDatabase::get_doccount() const
709 LOGCALL(DB
, Xapian::doccount
, "GlassDatabase::get_doccount", NO_ARGS
);
710 RETURN(version_file
.get_doccount());
714 GlassDatabase::get_lastdocid() const
716 LOGCALL(DB
, Xapian::docid
, "GlassDatabase::get_lastdocid", NO_ARGS
);
717 RETURN(version_file
.get_last_docid());
721 GlassDatabase::get_total_length() const
723 LOGCALL(DB
, totlen_t
, "GlassDatabase::get_total_length", NO_ARGS
);
724 RETURN(version_file
.get_total_doclen());
728 GlassDatabase::get_doclength(Xapian::docid did
) const
730 LOGCALL(DB
, Xapian::termcount
, "GlassDatabase::get_doclength", did
);
732 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
733 RETURN(postlist_table
.get_doclength(did
, ptrtothis
));
737 GlassDatabase::get_unique_terms(Xapian::docid did
) const
739 LOGCALL(DB
, Xapian::termcount
, "GlassDatabase::get_unique_terms", did
);
741 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
742 GlassTermList
termlist(ptrtothis
, did
);
743 // Note that the "approximate" size should be exact in this case.
745 // get_unique_terms() really ought to only count terms with wdf > 0, but
746 // that's expensive to calculate on demand, so for now let's just ensure
747 // unique_terms <= doclen.
748 RETURN(min(termlist
.get_approx_size(), GlassDatabase::get_doclength(did
)));
752 GlassDatabase::get_freqs(const string
& term
,
753 Xapian::doccount
* termfreq_ptr
,
754 Xapian::termcount
* collfreq_ptr
) const
756 LOGCALL_VOID(DB
, "GlassDatabase::get_freqs", term
| termfreq_ptr
| collfreq_ptr
);
757 Assert(!term
.empty());
758 postlist_table
.get_freqs(term
, termfreq_ptr
, collfreq_ptr
);
762 GlassDatabase::get_value_freq(Xapian::valueno slot
) const
764 LOGCALL(DB
, Xapian::doccount
, "GlassDatabase::get_value_freq", slot
);
765 RETURN(value_manager
.get_value_freq(slot
));
769 GlassDatabase::get_value_lower_bound(Xapian::valueno slot
) const
771 LOGCALL(DB
, std::string
, "GlassDatabase::get_value_lower_bound", slot
);
772 RETURN(value_manager
.get_value_lower_bound(slot
));
776 GlassDatabase::get_value_upper_bound(Xapian::valueno slot
) const
778 LOGCALL(DB
, std::string
, "GlassDatabase::get_value_upper_bound", slot
);
779 RETURN(value_manager
.get_value_upper_bound(slot
));
783 GlassDatabase::get_doclength_lower_bound() const
785 return version_file
.get_doclength_lower_bound();
789 GlassDatabase::get_doclength_upper_bound() const
791 return version_file
.get_doclength_upper_bound();
795 GlassDatabase::get_wdf_upper_bound(const string
& term
) const
797 Xapian::termcount cf
;
798 get_freqs(term
, NULL
, &cf
);
799 return min(cf
, version_file
.get_wdf_upper_bound());
803 GlassDatabase::term_exists(const string
& term
) const
805 LOGCALL(DB
, bool, "GlassDatabase::term_exists", term
);
806 Assert(!term
.empty());
807 RETURN(postlist_table
.term_exists(term
));
811 GlassDatabase::has_positions() const
813 return !position_table
.empty();
817 GlassDatabase::open_post_list(const string
& term
) const
819 LOGCALL(DB
, LeafPostList
*, "GlassDatabase::open_post_list", term
);
820 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
823 Xapian::doccount doccount
= get_doccount();
824 if (version_file
.get_last_docid() == doccount
) {
825 RETURN(new ContiguousAllDocsPostList(ptrtothis
, doccount
));
827 RETURN(new GlassAllDocsPostList(ptrtothis
, doccount
));
830 RETURN(new GlassPostList(ptrtothis
, term
, true));
834 GlassDatabase::open_value_list(Xapian::valueno slot
) const
836 LOGCALL(DB
, ValueList
*, "GlassDatabase::open_value_list", slot
);
837 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
838 RETURN(new GlassValueList(slot
, ptrtothis
));
842 GlassDatabase::open_term_list(Xapian::docid did
) const
844 LOGCALL(DB
, TermList
*, "GlassDatabase::open_term_list", did
);
846 if (!termlist_table
.is_open())
847 throw_termlist_table_close_exception();
848 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
849 RETURN(new GlassTermList(ptrtothis
, did
));
852 Xapian::Document::Internal
*
853 GlassDatabase::open_document(Xapian::docid did
, bool lazy
) const
855 LOGCALL(DB
, Xapian::Document::Internal
*, "GlassDatabase::open_document", did
| lazy
);
858 // This will throw DocNotFoundError if the document doesn't exist.
859 (void)get_doclength(did
);
862 intrusive_ptr
<const Database::Internal
> ptrtothis(this);
863 RETURN(new GlassDocument(ptrtothis
, did
, &value_manager
, &docdata_table
));
867 GlassDatabase::open_position_list(Xapian::docid did
, const string
& term
) const
871 AutoPtr
<GlassPositionList
> poslist(new GlassPositionList
);
872 if (!poslist
->read_data(&position_table
, did
, term
)) {
873 // As of 1.1.0, we don't check if the did and term exist - we just
874 // return an empty positionlist. If the user really needs to know,
875 // they can check for themselves.
878 return poslist
.release();
882 GlassDatabase::open_allterms(const string
& prefix
) const
884 LOGCALL(DB
, TermList
*, "GlassDatabase::open_allterms", NO_ARGS
);
885 RETURN(new GlassAllTermsList(intrusive_ptr
<const GlassDatabase
>(this),
890 GlassDatabase::open_spelling_termlist(const string
& word
) const
892 return spelling_table
.open_termlist(word
);
896 GlassDatabase::open_spelling_wordlist() const
898 GlassCursor
* cursor
= spelling_table
.cursor_get();
899 if (!cursor
) return NULL
;
900 return new GlassSpellingWordsList(intrusive_ptr
<const GlassDatabase
>(this),
905 GlassDatabase::get_spelling_frequency(const string
& word
) const
907 return spelling_table
.get_word_frequency(word
);
911 GlassDatabase::open_synonym_termlist(const string
& term
) const
913 return synonym_table
.open_termlist(term
);
917 GlassDatabase::open_synonym_keylist(const string
& prefix
) const
919 GlassCursor
* cursor
= synonym_table
.cursor_get();
920 if (!cursor
) return NULL
;
921 return new GlassSynonymTermList(intrusive_ptr
<const GlassDatabase
>(this),
926 GlassDatabase::get_metadata(const string
& key
) const
928 LOGCALL(DB
, string
, "GlassDatabase::get_metadata", key
);
929 string
btree_key("\x00\xc0", 2);
932 (void)postlist_table
.get_exact_entry(btree_key
, tag
);
937 GlassDatabase::open_metadata_keylist(const std::string
&prefix
) const
939 LOGCALL(DB
, TermList
*, "GlassDatabase::open_metadata_keylist", NO_ARGS
);
940 GlassCursor
* cursor
= postlist_table
.cursor_get();
941 if (!cursor
) RETURN(NULL
);
942 RETURN(new GlassMetadataTermList(intrusive_ptr
<const GlassDatabase
>(this),
947 GlassDatabase::get_revision_info() const
949 LOGCALL(DB
, string
, "GlassDatabase::get_revision_info", NO_ARGS
);
951 pack_uint(buf
, get_revision_number());
956 GlassDatabase::get_uuid() const
958 LOGCALL(DB
, string
, "GlassDatabase::get_uuid", NO_ARGS
);
959 RETURN(version_file
.get_uuid_string());
963 GlassDatabase::throw_termlist_table_close_exception() const
965 // Either the database has been closed, or else there's no termlist table.
966 // Check if the postlist table is open to determine which is the case.
967 if (!postlist_table
.is_open())
968 GlassTable::throw_database_closed();
969 throw Xapian::FeatureUnavailableError("Database has no termlist");
973 GlassDatabase::get_used_docid_range(Xapian::docid
& first
,
974 Xapian::docid
& last
) const
976 last
= version_file
.get_last_docid();
977 if (last
== version_file
.get_doccount()) {
978 // Contiguous range starting at 1.
982 postlist_table
.get_used_docid_range(first
, last
);
986 GlassDatabase::has_uncommitted_changes() const
991 ///////////////////////////////////////////////////////////////////////////
993 GlassWritableDatabase::GlassWritableDatabase(const string
&dir
, int flags
,
995 : GlassDatabase(dir
, flags
, block_size
),
998 modify_shortcut_document(NULL
),
999 modify_shortcut_docid(0)
1001 LOGCALL_CTOR(DB
, "GlassWritableDatabase", dir
| flags
| block_size
);
1003 const char *p
= getenv("XAPIAN_FLUSH_THRESHOLD");
1005 flush_threshold
= atoi(p
);
1006 if (flush_threshold
== 0)
1007 flush_threshold
= 10000;
1010 GlassWritableDatabase::~GlassWritableDatabase()
1012 LOGCALL_DTOR(DB
, "GlassWritableDatabase");
1017 GlassWritableDatabase::commit()
1019 if (transaction_active())
1020 throw Xapian::InvalidOperationError("Can't commit during a transaction");
1021 if (change_count
) flush_postlist_changes();
1026 GlassWritableDatabase::check_flush_threshold()
1028 // FIXME: this should be done by checking memory usage, not the number of
1029 // changes. We could also look at the amount of data the inverter object
1031 if (++change_count
>= flush_threshold
) {
1032 flush_postlist_changes();
1033 if (!transaction_active()) apply();
1038 GlassWritableDatabase::flush_postlist_changes() const
1040 version_file
.set_oldest_changeset(changes
.get_oldest_changeset());
1041 inverter
.flush(postlist_table
);
1042 inverter
.flush_pos_lists(position_table
);
1048 GlassWritableDatabase::close()
1050 LOGCALL_VOID(DB
, "GlassWritableDatabase::close", NO_ARGS
);
1051 if (!transaction_active()) {
1053 // FIXME: if commit() throws, should we still close?
1055 GlassDatabase::close();
1059 GlassWritableDatabase::apply()
1061 value_manager
.set_value_stats(value_stats
);
1062 GlassDatabase::apply();
1066 GlassWritableDatabase::add_document(const Xapian::Document
& document
)
1068 LOGCALL(DB
, Xapian::docid
, "GlassWritableDatabase::add_document", document
);
1069 // Make sure the docid counter doesn't overflow.
1070 if (version_file
.get_last_docid() == GLASS_MAX_DOCID
)
1071 throw Xapian::DatabaseError("Run out of docids - you'll have to use copydatabase to eliminate any gaps before you can add more documents");
1072 // Use the next unused document ID.
1073 RETURN(add_document_(version_file
.get_next_docid(), document
));
1077 GlassWritableDatabase::add_document_(Xapian::docid did
,
1078 const Xapian::Document
& document
)
1080 LOGCALL(DB
, Xapian::docid
, "GlassWritableDatabase::add_document_", did
| document
);
1083 // Set the document data.
1084 docdata_table
.replace_document_data(did
, document
.get_data());
1087 value_manager
.add_document(did
, document
, value_stats
);
1089 Xapian::termcount new_doclen
= 0;
1091 Xapian::TermIterator term
= document
.termlist_begin();
1092 for ( ; term
!= document
.termlist_end(); ++term
) {
1093 termcount wdf
= term
.get_wdf();
1094 // Calculate the new document length
1096 version_file
.check_wdf(wdf
);
1098 string tname
= *term
;
1099 if (tname
.size() > MAX_SAFE_TERM_LENGTH
)
1100 throw Xapian::InvalidArgumentError("Term too long (> " STRINGIZE(MAX_SAFE_TERM_LENGTH
) "): " + tname
);
1102 inverter
.add_posting(did
, tname
, wdf
);
1103 inverter
.set_positionlist(position_table
, did
, tname
, term
);
1106 LOGLINE(DB
, "Calculated doclen for new document " << did
<< " as " << new_doclen
);
1108 // Set the termlist.
1109 if (termlist_table
.is_open())
1110 termlist_table
.set_termlist(did
, document
, new_doclen
);
1112 // Set the new document length
1113 inverter
.set_doclength(did
, new_doclen
, true);
1114 version_file
.add_document(new_doclen
);
1116 // If an error occurs while adding a document, or doing any other
1117 // transaction, the modifications so far must be cleared before
1118 // returning control to the user - otherwise partial modifications will
1119 // persist in memory, and eventually get written to disk.
1124 check_flush_threshold();
1130 GlassWritableDatabase::delete_document(Xapian::docid did
)
1132 LOGCALL_VOID(DB
, "GlassWritableDatabase::delete_document", did
);
1135 if (!termlist_table
.is_open())
1136 throw_termlist_table_close_exception();
1138 // Remove the document data. If this fails, just propagate the exception since
1139 // the state should still be consistent.
1140 bool doc_really_existed
= docdata_table
.delete_document_data(did
);
1142 if (rare(modify_shortcut_docid
== did
)) {
1143 // The modify_shortcut document can't be used for a modification
1144 // shortcut now, because it's been deleted!
1145 modify_shortcut_document
= NULL
;
1146 modify_shortcut_docid
= 0;
1147 doc_really_existed
= true;
1150 if (!doc_really_existed
) {
1151 // Ensure we throw DocumentNotFound if the document doesn't exist.
1152 (void)get_doclength(did
);
1156 // Remove the values.
1157 value_manager
.delete_document(did
, value_stats
);
1159 // OK, now add entries to remove the postings in the underlying record.
1160 intrusive_ptr
<const GlassWritableDatabase
> ptrtothis(this);
1161 GlassTermList
termlist(ptrtothis
, did
);
1163 version_file
.delete_document(termlist
.get_doclength());
1166 while (!termlist
.at_end()) {
1167 string tname
= termlist
.get_termname();
1168 inverter
.delete_positionlist(did
, tname
);
1170 inverter
.remove_posting(did
, tname
, termlist
.get_wdf());
1175 // Remove the termlist.
1176 if (termlist_table
.is_open())
1177 termlist_table
.delete_termlist(did
);
1179 // Mark this document as removed.
1180 inverter
.delete_doclength(did
);
1182 // If an error occurs while deleting a document, or doing any other
1183 // transaction, the modifications so far must be cleared before
1184 // returning control to the user - otherwise partial modifications will
1185 // persist in memory, and eventually get written to disk.
1190 check_flush_threshold();
1194 GlassWritableDatabase::replace_document(Xapian::docid did
,
1195 const Xapian::Document
& document
)
1197 LOGCALL_VOID(DB
, "GlassWritableDatabase::replace_document", did
| document
);
1201 if (did
> version_file
.get_last_docid()) {
1202 version_file
.set_last_docid(did
);
1203 // If this docid is above the highwatermark, then we can't be
1204 // replacing an existing document.
1205 (void)add_document_(did
, document
);
1209 if (!termlist_table
.is_open()) {
1210 // We can replace an *unused* docid <= last_docid too.
1211 intrusive_ptr
<const GlassDatabase
> ptrtothis(this);
1212 if (!postlist_table
.document_exists(did
, ptrtothis
)) {
1213 (void)add_document_(did
, document
);
1216 throw_termlist_table_close_exception();
1219 // Check for a document read from this database being replaced - ie, a
1220 // modification operation.
1221 bool modifying
= false;
1222 if (modify_shortcut_docid
&&
1223 document
.internal
->get_docid() == modify_shortcut_docid
) {
1224 if (document
.internal
.get() == modify_shortcut_document
) {
1225 // We have a docid, it matches, and the pointer matches, so we
1226 // can skip modification of any data which hasn't been modified
1228 if (!document
.internal
->modified()) {
1229 // If the document is unchanged, we've nothing to do.
1233 LOGLINE(DB
, "Detected potential document modification shortcut.");
1235 // The modify_shortcut document can't be used for a
1236 // modification shortcut now, because it's about to be
1238 modify_shortcut_document
= NULL
;
1239 modify_shortcut_docid
= 0;
1243 if (!modifying
|| document
.internal
->terms_modified()) {
1244 bool pos_modified
= !modifying
||
1245 document
.internal
->term_positions_modified();
1246 intrusive_ptr
<const GlassWritableDatabase
> ptrtothis(this);
1247 GlassTermList
termlist(ptrtothis
, did
);
1248 Xapian::TermIterator term
= document
.termlist_begin();
1249 Xapian::termcount old_doclen
= termlist
.get_doclength();
1250 version_file
.delete_document(old_doclen
);
1251 Xapian::termcount new_doclen
= old_doclen
;
1253 string old_tname
, new_tname
;
1256 while (!termlist
.at_end() || term
!= document
.termlist_end()) {
1258 if (termlist
.at_end()) {
1262 old_tname
= termlist
.get_termname();
1263 if (term
!= document
.termlist_end()) {
1265 cmp
= old_tname
.compare(new_tname
);
1272 // Term old_tname has been deleted.
1273 termcount old_wdf
= termlist
.get_wdf();
1274 new_doclen
-= old_wdf
;
1275 inverter
.remove_posting(did
, old_tname
, old_wdf
);
1277 inverter
.delete_positionlist(did
, old_tname
);
1279 } else if (cmp
> 0) {
1280 // Term new_tname as been added.
1281 termcount new_wdf
= term
.get_wdf();
1282 new_doclen
+= new_wdf
;
1283 version_file
.check_wdf(new_wdf
);
1284 if (new_tname
.size() > MAX_SAFE_TERM_LENGTH
)
1285 throw Xapian::InvalidArgumentError("Term too long (> " STRINGIZE(MAX_SAFE_TERM_LENGTH
) "): " + new_tname
);
1286 inverter
.add_posting(did
, new_tname
, new_wdf
);
1288 inverter
.set_positionlist(position_table
, did
, new_tname
, term
);
1291 } else if (cmp
== 0) {
1292 // Term already exists: look for wdf and positionlist changes.
1293 termcount old_wdf
= termlist
.get_wdf();
1294 termcount new_wdf
= term
.get_wdf();
1296 // Check the stats even if wdf hasn't changed, because if
1297 // this is the only document, the stats will have been
1299 version_file
.check_wdf(new_wdf
);
1301 if (old_wdf
!= new_wdf
) {
1302 new_doclen
+= new_wdf
- old_wdf
;
1303 inverter
.update_posting(did
, new_tname
, old_wdf
, new_wdf
);
1307 inverter
.set_positionlist(position_table
, did
, new_tname
, term
, true);
1314 LOGLINE(DB
, "Calculated doclen for replacement document " << did
<< " as " << new_doclen
);
1316 // Set the termlist.
1317 if (termlist_table
.is_open())
1318 termlist_table
.set_termlist(did
, document
, new_doclen
);
1320 // Set the new document length
1321 if (new_doclen
!= old_doclen
)
1322 inverter
.set_doclength(did
, new_doclen
, false);
1323 version_file
.add_document(new_doclen
);
1326 if (!modifying
|| document
.internal
->data_modified()) {
1327 // Update the document data.
1328 docdata_table
.replace_document_data(did
, document
.get_data());
1331 if (!modifying
|| document
.internal
->values_modified()) {
1332 // Replace the values.
1333 value_manager
.replace_document(did
, document
, value_stats
);
1335 } catch (const Xapian::DocNotFoundError
&) {
1336 (void)add_document_(did
, document
);
1339 // If an error occurs while replacing a document, or doing any other
1340 // transaction, the modifications so far must be cleared before
1341 // returning control to the user - otherwise partial modifications will
1342 // persist in memory, and eventually get written to disk.
1347 check_flush_threshold();
1350 Xapian::Document::Internal
*
1351 GlassWritableDatabase::open_document(Xapian::docid did
, bool lazy
) const
1353 LOGCALL(DB
, Xapian::Document::Internal
*, "GlassWritableDatabase::open_document", did
| lazy
);
1354 modify_shortcut_document
= GlassDatabase::open_document(did
, lazy
);
1355 // Store the docid only after open_document() successfully returns, so an
1356 // attempt to open a missing document doesn't overwrite this.
1357 modify_shortcut_docid
= did
;
1358 RETURN(modify_shortcut_document
);
1362 GlassWritableDatabase::get_doclength(Xapian::docid did
) const
1364 LOGCALL(DB
, Xapian::termcount
, "GlassWritableDatabase::get_doclength", did
);
1365 Xapian::termcount doclen
;
1366 if (inverter
.get_doclength(did
, doclen
))
1368 RETURN(GlassDatabase::get_doclength(did
));
1372 GlassWritableDatabase::get_freqs(const string
& term
,
1373 Xapian::doccount
* termfreq_ptr
,
1374 Xapian::termcount
* collfreq_ptr
) const
1376 LOGCALL_VOID(DB
, "GlassWritableDatabase::get_freqs", term
| termfreq_ptr
| collfreq_ptr
);
1377 Assert(!term
.empty());
1378 GlassDatabase::get_freqs(term
, termfreq_ptr
, collfreq_ptr
);
1379 Xapian::termcount_diff tf_delta
, cf_delta
;
1380 if (inverter
.get_deltas(term
, tf_delta
, cf_delta
)) {
1382 *termfreq_ptr
+= tf_delta
;
1384 *collfreq_ptr
+= cf_delta
;
1389 GlassWritableDatabase::get_value_freq(Xapian::valueno slot
) const
1391 LOGCALL(DB
, Xapian::doccount
, "GlassWritableDatabase::get_value_freq", slot
);
1392 map
<Xapian::valueno
, ValueStats
>::const_iterator i
;
1393 i
= value_stats
.find(slot
);
1394 if (i
!= value_stats
.end()) RETURN(i
->second
.freq
);
1395 RETURN(GlassDatabase::get_value_freq(slot
));
1399 GlassWritableDatabase::get_value_lower_bound(Xapian::valueno slot
) const
1401 LOGCALL(DB
, std::string
, "GlassWritableDatabase::get_value_lower_bound", slot
);
1402 map
<Xapian::valueno
, ValueStats
>::const_iterator i
;
1403 i
= value_stats
.find(slot
);
1404 if (i
!= value_stats
.end()) RETURN(i
->second
.lower_bound
);
1405 RETURN(GlassDatabase::get_value_lower_bound(slot
));
1409 GlassWritableDatabase::get_value_upper_bound(Xapian::valueno slot
) const
1411 LOGCALL(DB
, std::string
, "GlassWritableDatabase::get_value_upper_bound", slot
);
1412 map
<Xapian::valueno
, ValueStats
>::const_iterator i
;
1413 i
= value_stats
.find(slot
);
1414 if (i
!= value_stats
.end()) RETURN(i
->second
.upper_bound
);
1415 RETURN(GlassDatabase::get_value_upper_bound(slot
));
1419 GlassWritableDatabase::term_exists(const string
& tname
) const
1421 LOGCALL(DB
, bool, "GlassWritableDatabase::term_exists", tname
);
1422 Xapian::doccount tf
;
1423 get_freqs(tname
, &tf
, NULL
);
1428 GlassWritableDatabase::has_positions() const
1430 return inverter
.has_positions(position_table
);
1434 GlassWritableDatabase::open_post_list(const string
& tname
) const
1436 LOGCALL(DB
, LeafPostList
*, "GlassWritableDatabase::open_post_list", tname
);
1437 intrusive_ptr
<const GlassWritableDatabase
> ptrtothis(this);
1439 if (tname
.empty()) {
1440 Xapian::doccount doccount
= get_doccount();
1441 if (version_file
.get_last_docid() == doccount
) {
1442 RETURN(new ContiguousAllDocsPostList(ptrtothis
, doccount
));
1444 inverter
.flush_doclengths(postlist_table
);
1445 RETURN(new GlassAllDocsPostList(ptrtothis
, doccount
));
1448 // Flush any buffered changes for this term's postlist so we can just
1449 // iterate from the flushed state.
1450 inverter
.flush_post_list(postlist_table
, tname
);
1451 inverter
.flush_pos_lists(position_table
);
1452 RETURN(new GlassPostList(ptrtothis
, tname
, true));
1456 GlassWritableDatabase::open_value_list(Xapian::valueno slot
) const
1458 LOGCALL(DB
, ValueList
*, "GlassWritableDatabase::open_value_list", slot
);
1459 // If there are changes, we don't have code to iterate the modified value
1460 // list so we need to flush (but don't commit - there may be a transaction
1462 if (change_count
) value_manager
.merge_changes();
1463 RETURN(GlassDatabase::open_value_list(slot
));
1467 GlassWritableDatabase::open_term_list(Xapian::docid did
) const
1469 LOGCALL(DB
, TermList
*, "GlassWritableDatabase::open_term_list", did
);
1471 inverter
.flush_pos_lists(position_table
);
1472 RETURN(GlassDatabase::open_term_list(did
));
1476 GlassWritableDatabase::open_position_list(Xapian::docid did
, const string
& term
) const
1480 AutoPtr
<GlassPositionList
> poslist(new GlassPositionList
);
1483 if (inverter
.get_positionlist(did
, term
, data
)) {
1484 poslist
->read_data(data
);
1485 } else if (!poslist
->read_data(&position_table
, did
, term
)) {
1486 // As of 1.1.0, we don't check if the did and term exist - we just
1487 // return an empty positionlist. If the user really needs to know,
1488 // they can check for themselves.
1491 return poslist
.release();
1495 GlassWritableDatabase::open_allterms(const string
& prefix
) const
1497 LOGCALL(DB
, TermList
*, "GlassWritableDatabase::open_allterms", NO_ARGS
);
1499 // There are changes, and terms may have been added or removed, and so
1500 // we need to flush changes for terms with the specified prefix (but
1501 // don't commit - there may be a transaction in progress).
1502 inverter
.flush_post_lists(postlist_table
, prefix
);
1503 inverter
.flush_pos_lists(position_table
);
1504 if (prefix
.empty()) {
1505 // We've flushed all the posting list changes, but the document
1506 // length and stats haven't been written, so set change_count to 1.
1507 // FIXME: Can we handle this better?
1511 RETURN(GlassDatabase::open_allterms(prefix
));
1515 GlassWritableDatabase::cancel()
1517 GlassDatabase::cancel();
1519 value_stats
.clear();
1524 GlassWritableDatabase::add_spelling(const string
& word
,
1525 Xapian::termcount freqinc
) const
1527 spelling_table
.add_word(word
, freqinc
);
1531 GlassWritableDatabase::remove_spelling(const string
& word
,
1532 Xapian::termcount freqdec
) const
1534 spelling_table
.remove_word(word
, freqdec
);
1538 GlassWritableDatabase::open_spelling_wordlist() const
1540 spelling_table
.merge_changes();
1541 return GlassDatabase::open_spelling_wordlist();
1545 GlassWritableDatabase::open_synonym_keylist(const string
& prefix
) const
1547 synonym_table
.merge_changes();
1548 return GlassDatabase::open_synonym_keylist(prefix
);
1552 GlassWritableDatabase::add_synonym(const string
& term
,
1553 const string
& synonym
) const
1555 synonym_table
.add_synonym(term
, synonym
);
1559 GlassWritableDatabase::remove_synonym(const string
& term
,
1560 const string
& synonym
) const
1562 synonym_table
.remove_synonym(term
, synonym
);
1566 GlassWritableDatabase::clear_synonyms(const string
& term
) const
1568 synonym_table
.clear_synonyms(term
);
1572 GlassWritableDatabase::set_metadata(const string
& key
, const string
& value
)
1574 LOGCALL_VOID(DB
, "GlassWritableDatabase::set_metadata", key
| value
);
1575 string
btree_key("\x00\xc0", 2);
1577 if (value
.empty()) {
1578 postlist_table
.del(btree_key
);
1580 postlist_table
.add(btree_key
, value
);
1585 GlassWritableDatabase::invalidate_doc_object(Xapian::Document::Internal
* obj
) const
1587 if (obj
== modify_shortcut_document
) {
1588 modify_shortcut_document
= NULL
;
1589 modify_shortcut_docid
= 0;
1594 GlassWritableDatabase::has_uncommitted_changes() const
1596 return change_count
> 0 ||
1597 postlist_table
.is_modified() ||
1598 position_table
.is_modified() ||
1599 termlist_table
.is_modified() ||
1600 value_manager
.is_modified() ||
1601 synonym_table
.is_modified() ||
1602 spelling_table
.is_modified() ||
1603 docdata_table
.is_modified();