Add BM25 and TFIDF Scoring to the text index

src/global/Id.h

@@ -12,7 +12,7 @@
 #include "util/Exception.h"
 
 using Id = ValueId;
-typedef uint16_t Score;
+using Score = float;
 
 // TODO<joka921> Make the following ID and index types strong.
 using ColumnIndex = uint64_t;

src/global/IndexTypes.h

@@ -16,3 +16,4 @@ using LocalVocabIndex = const LocalVocabEntry*;
 using TextRecordIndex = ad_utility::TypedIndex<uint64_t, "TextRecordIndex">;
 using WordVocabIndex = ad_utility::TypedIndex<uint64_t, "WordVocabIndex">;
 using BlankNodeIndex = ad_utility::TypedIndex<uint64_t, "BlankNodeIndex">;
+using DocumentIndex = ad_utility::TypedIndex<uint64_t, "DocumentIndex">;

src/index/CMakeLists.txt

@@ -6,5 +6,5 @@ add_library(index
         DocsDB.cpp FTSAlgorithms.cpp
         PrefixHeuristic.cpp CompressedRelation.cpp
         PatternCreator.cpp ScanSpecification.cpp
-        DeltaTriples.cpp LocalVocabEntry.cpp)
+        DeltaTriples.cpp LocalVocabEntry.cpp TextScoring.cpp TextScoringEnum.cpp)
 qlever_target_link_libraries(index util parser vocabulary ${STXXL_LIBRARIES})

src/index/Index.cpp

@@ -25,9 +25,10 @@
 }
 
 // ____________________________________________________________________________
-void Index::addTextFromContextFile(const std::string& contextFile,
-                                   bool addWordsFromLiterals) {
-  pimpl_->addTextFromContextFile(contextFile, addWordsFromLiterals);
+void Index::buildTextIndexFile(
+    const std::pair<std::string, std::string>& wordsAndDocsFile,
+    bool addWordsFromLiterals) {
+  pimpl_->buildTextIndexFile(wordsAndDocsFile, addWordsFromLiterals);
 }
 
 // ____________________________________________________________________________
@@ -215,6 +216,16 @@
   return pimpl_->setNumTriplesPerBatch(numTriplesPerBatch);
 }
 
+// ____________________________________________________________________________
+void Index::setScoringMetricsUsedInSettings(TextScoringMetric scoringMetric) {
+  return pimpl_->setScoringMetricsUsedInSettings(scoringMetric);
+}
+
+// ____________________________________________________________________________
+void Index::setBM25ParmetersUsedInSettings(float b, float k) {
+  return pimpl_->setBM25ParametersInSettings(b, k);
+}
+
 // ____________________________________________________________________________
 const std::string& Index::getTextName() const { return pimpl_->getTextName(); }
 

src/index/Index.h

@@ -13,6 +13,7 @@
 #include "index/InputFileSpecification.h"
 #include "index/Permutation.h"
 #include "index/StringSortComparator.h"
+#include "index/TextScoringEnum.h"
 #include "index/Vocabulary.h"
 #include "parser/TripleComponent.h"
 #include "util/CancellationHandle.h"
@@ -61,7 +62,8 @@ class Index {
     // Stores the index of the entity of each result.
     vector<Id> eids_;
     // Stores for each result how often an entity
-    // appears in its associated TextRecord.
+    // appears in its associated TextRecord. [[OLD DEFINITION]]
+    // Now scores BM25 scores for all words that are in the voacabulary
     vector<Score> scores_;
   };
 
@@ -94,8 +96,9 @@ class Index {
 
   // Add a text index to a complete KB index. First read the given context
   // file (if file name not empty), then add words from literals (if true).
-  void addTextFromContextFile(const std::string& contextFile,
-                              bool addWordsFromLiterals);
+  void buildTextIndexFile(
+      const std::pair<std::string, std::string>& wordsAndDocsFile,
+      bool addWordsFromLiterals);
 
   // Build docsDB file from given file (one text record per line).
   void buildDocsDB(const std::string& docsFile);
@@ -206,6 +209,10 @@ class Index {
 
   void setNumTriplesPerBatch(uint64_t numTriplesPerBatch);
 
+  void setScoringMetricsUsedInSettings(TextScoringMetric scoringMetric);
+
+  void setBM25ParmetersUsedInSettings(float b, float k);
+
   const std::string& getTextName() const;
 
   const std::string& getKbName() const;

src/index/IndexBuilderMain.cpp

@@ -155,6 +155,7 @@ int main(int argc, char** argv) {
   string textIndexName;
   string kbIndexName;
   string settingsFile;
+  string scoringMetric = "count";
   std::vector<string> filetype;
   std::vector<string> inputFile;
   std::vector<string> defaultGraphs;
@@ -164,6 +165,8 @@ int main(int argc, char** argv) {
   bool keepTemporaryFiles = false;
   bool onlyPsoAndPos = false;
   bool addWordsFromLiterals = false;
+  float bScoringParam = 0.75;
+  float kScoringParam = 1.75;
   std::optional<ad_utility::MemorySize> stxxlMemory;
   std::optional<ad_utility::MemorySize> parserBufferSize;
   optind = 1;
@@ -214,6 +217,16 @@ int main(int argc, char** argv) {
   add("add-text-index,A", po::bool_switch(&onlyAddTextIndex),
       "Only build the text index. Assumes that a knowledge graph index with "
       "the same `index-basename` already exists.");
+  add("set-bm25-b-param", po::value(&bScoringParam),
+      "Sets the b param in the BM25 scoring metric. This has to be between "
+      "(including) 0 and 1. The default is 0.75.");
+  add("set-bm25-k-param", po::value(&kScoringParam),
+      "Sets the k param in the BM25 scoring metric. This has to be greater "
+      "than or equal to 0. The default is 1.75.");
+  add("set-scoring-metric,S", po::value(&scoringMetric),
+      "Sets the scoring metric used. Options are \"count\" for count, "
+      "\"tf-idf\" for tf idf "
+      "and \"bm25\" for bm25. The default is count.");
 
   // Options for the knowledge graph index.
   add("settings-file,s", po::value(&settingsFile),
@@ -245,6 +258,14 @@ int main(int argc, char** argv) {
       return EXIT_SUCCESS;
     }
     po::notify(optionsMap);
+    if (kScoringParam < 0) {
+      throw std::invalid_argument("The value of bm25-k must be >= 0");
+    }
+    if (bScoringParam < 0 || bScoringParam > 1) {
+      throw std::invalid_argument(
+          "The value of bm25-b must be between and "
+          "including 0 and 1");
+    }
   } catch (const std::exception& e) {
     std::cerr << "Error in command-line argument: " << e.what() << '\n';
     std::cerr << boostOptions << '\n';
@@ -330,8 +351,13 @@ int main(int argc, char** argv) {
       index.createFromFiles(fileSpecifications);
     }
 
-    if (!wordsfile.empty() || addWordsFromLiterals) {
-      index.addTextFromContextFile(wordsfile, addWordsFromLiterals);
+    if ((!wordsfile.empty() && !docsfile.empty()) || addWordsFromLiterals) {
+      index.setScoringMetricsUsedInSettings(
+          getTextScoringMetricFromString(scoringMetric));
+      index.setBM25ParmetersUsedInSettings(bScoringParam, kScoringParam);
+      index.buildTextIndexFile(
+          std::pair<std::string, std::string>{wordsfile, docsfile},
+          addWordsFromLiterals);
     }
 
     if (!docsfile.empty()) {