xapian-letor/feature/tfidfdoclenfeature.cc

   1 /** @file tfidfdoclenfeature.cc
   2  * @brief TfIdfDoclenFeature class
   3  */
   4 /* Copyright (C) 2012 Parth Gupta
   5  * Copyright (C) 2016 Ayush Tomar
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "xapian-letor/feature.h"
  25
  26 #include "debuglog.h"
  27 #include "stringutils.h"
  28
  29 using namespace std;
  30
  31 namespace Xapian {
  32
  33 string
  34 TfIdfDoclenFeature::name() const
  35 {
  36     return "TfIdfDoclenFeature";
  37 }
  38
  39 /** A helper function for feature->get_value()
  40  *
  41  *  Checks if the term belongs to the title or is stemmed from the title.
  42  */
  43 inline bool
  44 is_title_term(const std::string& term)
  45 {
  46     return startswith(term, 'S') || startswith(term, "ZS");
  47 }
  48
  49 vector<double>
  50 TfIdfDoclenFeature::get_values() const
  51 {
  52     LOGCALL(API, vector<double>, "TfIdfDoclenFeature::get_values", NO_ARGS);
  53
  54     vector<double> values;
  55     double value = 0;
  56     double doc_len;
  57     auto doc_len_iterator = doc_length.find("title");
  58     if (doc_len_iterator != doc_length.end())
  59         doc_len = (double)doc_len_iterator->second;
  60     else
  61         doc_len = 0;
  62
  63     for (Xapian::TermIterator qt = feature_query.get_unique_terms_begin();
  64          qt != feature_query.get_terms_end(); ++qt) {
  65         if (is_title_term((*qt))) {
  66             double tf;
  67             double idf;
  68             auto tf_iterator = termfreq.find(*qt);
  69             auto idf_iterator = inverse_doc_freq.find(*qt);
  70             if (tf_iterator != termfreq.end())
  71                 tf = (double)tf_iterator->second;
  72             else
  73                 tf = 0;
  74             if (idf_iterator != inverse_doc_freq.end())
  75                 idf = idf_iterator->second;
  76             else
  77                 idf = 0;
  78             value += log10(1 + ((tf * idf) / (1 + doc_len)));
  79         }
  80     }
  81     values.push_back(value);
  82     value = 0;
  83     doc_len_iterator = doc_length.find("body");
  84     if (doc_len_iterator != doc_length.end())
  85         doc_len = (double)doc_len_iterator->second;
  86     else
  87         doc_len = 0;
  88
  89     for (Xapian::TermIterator qt = feature_query.get_unique_terms_begin();
  90          qt != feature_query.get_terms_end(); ++qt) {
  91         if (!is_title_term((*qt))) {
  92             double tf;
  93             double idf;
  94             auto tf_iterator = termfreq.find(*qt);
  95             auto idf_iterator = inverse_doc_freq.find(*qt);
  96             if (tf_iterator != termfreq.end())
  97                 tf = (double)tf_iterator->second;
  98             else
  99                 tf = 0;
 100             if (idf_iterator != inverse_doc_freq.end())
 101                 idf = idf_iterator->second;
 102             else
 103                 idf = 0;
 104             value += log10(1 + ((tf * idf) / (1 + doc_len)));
 105         }
 106     }
 107     values.push_back(value);
 108     value = 0;
 109     doc_len_iterator = doc_length.find("whole");
 110     if (doc_len_iterator != doc_length.end())
 111         doc_len = (double)doc_len_iterator->second;
 112     else
 113         doc_len = 0;
 114
 115     for (Xapian::TermIterator qt = feature_query.get_unique_terms_begin();
 116          qt != feature_query.get_terms_end(); ++qt) {
 117         double tf;
 118         double idf;
 119         auto tf_iterator = termfreq.find(*qt);
 120         auto idf_iterator = inverse_doc_freq.find(*qt);
 121         if (tf_iterator != termfreq.end())
 122             tf = (double)tf_iterator->second;
 123         else
 124             tf = 0;
 125         if (idf_iterator != inverse_doc_freq.end())
 126             idf = idf_iterator->second;
 127         else
 128             idf = 0;
 129         value += log10(1 + ((tf * idf) / (1 + doc_len)));
 130     }
 131     values.push_back(value);
 132
 133     return values;
 134 }
 135
 136 }