From 37b8ef371637c8f1b3db7903eebf80daf8c00404 Mon Sep 17 00:00:00 2001 From: Olly Betts Date: Wed, 20 Sep 2017 15:06:33 +1200 Subject: [PATCH] Implement TermIterator::positionlist_count() for remote backend This requires a minor remote protocol version bump (to 39.1). --- xapian-core/backends/remote/net_termlist.cc | 4 ++-- xapian-core/backends/remote/remote-database.cc | 23 ++++++++++++++++++++++- xapian-core/backends/remote/remote-database.h | 9 ++++++++- xapian-core/common/remoteprotocol.h | 7 +++++-- xapian-core/net/remote_protocol.rst | 8 ++++++++ xapian-core/net/remoteserver.cc | 22 ++++++++++++++++++++++ xapian-core/net/remoteserver.h | 5 ++++- xapian-core/tests/api_posdb.cc | 12 ++++-------- xapian-core/tests/dbcheck.cc | 6 +----- 9 files changed, 76 insertions(+), 20 deletions(-) diff --git a/xapian-core/backends/remote/net_termlist.cc b/xapian-core/backends/remote/net_termlist.cc index 25b570168..b540331b3 100644 --- a/xapian-core/backends/remote/net_termlist.cc +++ b/xapian-core/backends/remote/net_termlist.cc @@ -1,7 +1,7 @@ /* net_termlist.cc * * Copyright 1999,2000,2001 BrightStation PLC - * Copyright 2002,2003,2006,2007,2008,2009,2010 Olly Betts + * Copyright 2002,2003,2006,2007,2008,2009,2010,2017 Olly Betts * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -117,7 +117,7 @@ NetworkTermList::at_end() const Xapian::termcount NetworkTermList::positionlist_count() const { - throw Xapian::UnimplementedError("NetworkTermList::positionlist_count() not implemented"); + return this_db->positionlist_count(did, get_termname()); } Xapian::PositionIterator diff --git a/xapian-core/backends/remote/remote-database.cc b/xapian-core/backends/remote/remote-database.cc index d3c90a8dd..c66621f7e 100644 --- a/xapian-core/backends/remote/remote-database.cc +++ b/xapian-core/backends/remote/remote-database.cc @@ -1,7 +1,7 @@ /** @file remote-database.cc * @brief Remote backend database class */ -/* Copyright (C) 2006,2007,2008,2009,2010,2011,2012,2013,2014,2015 Olly Betts +/* Copyright (C) 2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2017 Olly Betts * Copyright (C) 2007,2009,2010 Lemur Consulting Ltd * * This program is free software; you can redistribute it and/or @@ -105,6 +105,27 @@ RemoteDatabase::RemoteDatabase(int fd, double timeout_, } } +Xapian::termcount +RemoteDatabase::positionlist_count(Xapian::docid did, + const std::string& term) const +{ + if (cached_stats_valid && !has_positional_info) + return 0; + + send_message(MSG_POSITIONLISTCOUNT, encode_length(did) + term); + + string message; + get_message(message, REPLY_POSITIONLISTCOUNT); + const char * p = message.data(); + const char * p_end = p + message.size(); + Xapian::termcount count; + decode_length(&p, p_end, count); + if (p != p_end) { + throw Xapian::NetworkError("Bad REPLY_POSITIONLISTCOUNT message received", context); + } + return count; +} + void RemoteDatabase::keep_alive() { diff --git a/xapian-core/backends/remote/remote-database.h b/xapian-core/backends/remote/remote-database.h index bbf27e97e..07246e407 100644 --- a/xapian-core/backends/remote/remote-database.h +++ b/xapian-core/backends/remote/remote-database.h @@ -1,7 +1,7 @@ /** @file remote-database.h * @brief RemoteDatabase is the baseclass for remote database implementations. */ -/* Copyright (C) 2006,2007,2009,2010,2011,2014,2015 Olly Betts +/* Copyright (C) 2006,2007,2009,2010,2011,2014,2015,2017 Olly Betts * Copyright (C) 2007,2009,2010 Lemur Consulting Ltd * * This program is free software; you can redistribute it and/or @@ -123,6 +123,13 @@ class RemoteDatabase : public Xapian::Database::Internal { double timeout; public: + /** Get the length of the position list. + * + * Extra method for RemoteDatabase. + */ + Xapian::termcount positionlist_count(Xapian::docid did, + const std::string& term) const; + /// Send a keep-alive message. void keep_alive(); diff --git a/xapian-core/common/remoteprotocol.h b/xapian-core/common/remoteprotocol.h index fcaa1ec6f..9d683f5f6 100644 --- a/xapian-core/common/remoteprotocol.h +++ b/xapian-core/common/remoteprotocol.h @@ -1,7 +1,7 @@ /** @file remoteprotocol.h * @brief Remote protocol version and message numbers */ -/* Copyright (C) 2006,2007,2008,2009,2010,2011,2013,2014,2015 Olly Betts +/* Copyright (C) 2006,2007,2008,2009,2010,2011,2013,2014,2015,2017 Olly Betts * Copyright (C) 2007,2010 Lemur Consulting Ltd * * This program is free software; you can redistribute it and/or modify @@ -49,8 +49,9 @@ // 37: 1.3.1 Prefix-compress termlists. // 38: 1.3.2 Stats serialisation now includes collection freq, and more... // 39: 1.3.3 New query operator OP_WILDCARD; sort keys in serialised MSet. +// 39.1: 1.5.0 MSG_POSITIONLISTCOUNT added. #define XAPIAN_REMOTE_PROTOCOL_MAJOR_VERSION 39 -#define XAPIAN_REMOTE_PROTOCOL_MINOR_VERSION 0 +#define XAPIAN_REMOTE_PROTOCOL_MINOR_VERSION 1 /** Message types (client -> server). * @@ -89,6 +90,7 @@ enum message_type { MSG_METADATAKEYLIST, // Iterator for metadata keys MSG_FREQS, // Get termfreq and collfreq MSG_UNIQUETERMS, // Get number of unique terms in doc + MSG_POSITIONLISTCOUNT, // Get PositionList length MSG_MAX }; @@ -117,6 +119,7 @@ enum reply_type { REPLY_METADATAKEYLIST, // Iterator for metadata keys REPLY_FREQS, // Get termfreq and collfreq REPLY_UNIQUETERMS, // Get number of unique terms in doc + REPLY_POSITIONLISTCOUNT, // Get PositionList length REPLY_MAX }; diff --git a/xapian-core/net/remote_protocol.rst b/xapian-core/net/remote_protocol.rst index 9111e5bf4..cf08b9954 100644 --- a/xapian-core/net/remote_protocol.rst +++ b/xapian-core/net/remote_protocol.rst @@ -183,6 +183,14 @@ Since positions must be strictly monotonically increasing, we encode values can still be encoded compactly. The first position is encoded as its true value. +Positionlist count +------------------ + +- ``MSG_POSITIONLISTCOUNT I `` +- ``REPLY_POSITIONLISTCOUNT I`` + +Get the length of the positionlist without fetching the list itself. + Postlist -------- diff --git a/xapian-core/net/remoteserver.cc b/xapian-core/net/remoteserver.cc index 6a7e45e30..b318ff11c 100644 --- a/xapian-core/net/remoteserver.cc +++ b/xapian-core/net/remoteserver.cc @@ -192,6 +192,7 @@ RemoteServer::run() &RemoteServer::msg_openmetadatakeylist, &RemoteServer::msg_freqs, &RemoteServer::msg_uniqueterms, + &RemoteServer::msg_positionlistcount, }; string message; @@ -308,6 +309,27 @@ RemoteServer::msg_positionlist(const string &message) } void +RemoteServer::msg_positionlistcount(const string &message) +{ + const char *p = message.data(); + const char *p_end = p + message.size(); + Xapian::docid did; + decode_length(&p, p_end, did); + + // This is kind of clumsy, but what the public API requires. + Xapian::termcount result = 0; + Xapian::TermIterator termit = db->termlist_begin(did); + if (termit != db->termlist_end(did)) { + string term(p, p_end - p); + termit.skip_to(term); + if (termit != db->termlist_end(did)) { + result = termit.positionlist_count(); + } + } + send_message(REPLY_POSITIONLISTCOUNT, encode_length(result)); +} + +void RemoteServer::msg_postlist(const string &message) { const string & term = message; diff --git a/xapian-core/net/remoteserver.h b/xapian-core/net/remoteserver.h index 857d7914c..6265570de 100644 --- a/xapian-core/net/remoteserver.h +++ b/xapian-core/net/remoteserver.h @@ -1,7 +1,7 @@ /** @file remoteserver.h * @brief Xapian remote backend server base class */ -/* Copyright (C) 2006,2007,2008,2009,2010,2014 Olly Betts +/* Copyright (C) 2006,2007,2008,2009,2010,2014,2017 Olly Betts * Copyright (C) 2007,2009,2010 Lemur Consulting Ltd * * This program is free software; you can redistribute it and/or modify @@ -122,6 +122,9 @@ class XAPIAN_VISIBILITY_DEFAULT RemoteServer : private RemoteConnection { // get positionlist void msg_positionlist(const std::string &message); + // get positionlist count + void msg_positionlistcount(const std::string &message); + // get write access void msg_writeaccess(const std::string & message); diff --git a/xapian-core/tests/api_posdb.cc b/xapian-core/tests/api_posdb.cc index cfc18cfe5..e8ab0a1c9 100644 --- a/xapian-core/tests/api_posdb.cc +++ b/xapian-core/tests/api_posdb.cc @@ -621,14 +621,10 @@ DEFINE_TESTCASE(positfromtermit1, positional) { Xapian::PositionIterator p = t.positionlist_begin(); TEST_NOT_EQUAL(p, t.positionlist_end()); - try { - TEST_EQUAL(t.positionlist_count(), 1); - t.skip_to("on"); - TEST_NOT_EQUAL(t, db.termlist_end(7)); - TEST_EQUAL(t.positionlist_count(), 2); - } catch (const Xapian::UnimplementedError &) { - SKIP_TEST("TermList::positionlist_count() not yet implemented for this backend"); - } + TEST_EQUAL(t.positionlist_count(), 1); + t.skip_to("on"); + TEST_NOT_EQUAL(t, db.termlist_end(7)); + TEST_EQUAL(t.positionlist_count(), 2); return true; } diff --git a/xapian-core/tests/dbcheck.cc b/xapian-core/tests/dbcheck.cc index 219e345f2..46c974a3b 100644 --- a/xapian-core/tests/dbcheck.cc +++ b/xapian-core/tests/dbcheck.cc @@ -183,11 +183,7 @@ dbcheck(const Xapian::Database & db, string posrepr2 = positions_to_string(it2, t2.positionlist_end(), &tc2); TEST_EQUAL(posrepr, posrepr2); TEST_EQUAL(tc1, tc2); - try { - TEST_EQUAL(tc1, t.positionlist_count()); - } catch (const Xapian::UnimplementedError &) { - // positionlist_count() isn't implemented for remote databases. - } + TEST_EQUAL(tc1, t.positionlist_count()); // Make a representation of the posting. if (!posrepr.empty()) { -- 2.11.4.GIT