feat: wildcard pattern added for fields to find all non-null values (#4941)

fixed: #4937

Author: vyavdoshenko

src/core/search/ast_expr.h

@@ -22,6 +22,9 @@ struct AstNode;
 // Matches all documents
 struct AstStarNode {};
 
+// Matches all documents where this field has a non-null value
+struct AstStarFieldNode {};
+
 // Matches terms in text fields
 struct AstTermNode {
   explicit AstTermNode(std::string term);
@@ -108,9 +111,9 @@ struct AstKnnNode {
   std::optional<float> ef_runtime;
 };
 
-using NodeVariants =
-    std::variant<std::monostate, AstStarNode, AstTermNode, AstPrefixNode, AstRangeNode,
-                 AstNegateNode, AstLogicalNode, AstFieldNode, AstTagsNode, AstKnnNode>;
+using NodeVariants = std::variant<std::monostate, AstStarNode, AstStarFieldNode, AstTermNode,
+                                  AstPrefixNode, AstRangeNode, AstNegateNode, AstLogicalNode,
+                                  AstFieldNode, AstTagsNode, AstKnnNode>;
 
 struct AstNode : public NodeVariants {
   using variant::variant;

src/core/search/base.h

@@ -90,6 +90,11 @@ struct BaseIndex {
   // Returns true if the document was added / indexed
   virtual bool Add(DocId id, const DocumentAccessor& doc, std::string_view field) = 0;
   virtual void Remove(DocId id, const DocumentAccessor& doc, std::string_view field) = 0;
+
+  // Returns documents that have non-null values for this field (used for @field:* queries)
+  virtual std::optional<std::vector<DocId>> GetAllResults() const {
+    return std::nullopt;
+  }
 };
 
 // Base class for type-specific sorting indices.

src/core/search/indices.h

@@ -35,6 +35,10 @@ struct NumericIndex : public BaseIndex {
 
   std::vector<DocId> Range(double l, double r) const;
 
+  std::optional<std::vector<DocId>> GetAllResults() const override {
+    return Range(-std::numeric_limits<double>::infinity(), std::numeric_limits<double>::infinity());
+  }
+
  private:
   using Entry = std::pair<double, DocId>;
   absl::btree_set<Entry, std::less<Entry>, PMR_NS::polymorphic_allocator<Entry>> entries_;
@@ -58,6 +62,20 @@ template <typename C> struct BaseStringIndex : public BaseIndex {
   // Returns all the terms that appear as keys in the reverse index.
   std::vector<std::string> GetTerms() const;
 
+  std::optional<std::vector<DocId>> GetAllResults() const override {
+    absl::flat_hash_set<DocId> unique_docs;
+
+    for (const auto& [term, container] : entries_) {
+      for (const DocId& id : container) {
+        unique_docs.insert(id);
+      }
+    }
+
+    auto result = std::vector<DocId>(unique_docs.begin(), unique_docs.end());
+    std::sort(result.begin(), result.end());
+    return result;
+  }
+
  protected:
   using StringList = DocumentAccessor::StringList;
 
@@ -133,6 +151,33 @@ struct FlatVectorIndex : public BaseVectorIndex {
 
   const float* Get(DocId doc) const;
 
+  // Return all documents that have vectors in this index
+  std::optional<std::vector<DocId>> GetAllResults() const override {
+    std::vector<DocId> result;
+    size_t num_vectors = entries_.size() / dim_;
+    result.reserve(num_vectors);
+
+    for (DocId id = 0; id < num_vectors; ++id) {
+      // Check if the vector is not zero (all elements are 0)
+      // TODO: Valid vector can contain 0s, we should use a better approach
+      const float* vec = Get(id);
+      bool is_zero_vector = true;
+
+      for (size_t i = 0; i < dim_; ++i) {
+        if (vec[i] != 0.0f) {
+          is_zero_vector = false;
+          break;
+        }
+      }
+
+      if (!is_zero_vector) {
+        result.push_back(id);
+      }
+    }
+
+    return result;
+  }
+
  protected:
   void AddVector(DocId id, const VectorPtr& vector) override;
 
@@ -142,6 +187,10 @@ struct FlatVectorIndex : public BaseVectorIndex {
 
 struct HnswlibAdapter;
 
+// This index does't have GetAllResults method
+// because it's not possible to get all vectors from the index
+// It depends on the Hnswlib implementation
+// TODO: Consider adding GetAllResults method in the future
 struct HnswVectorIndex : public BaseVectorIndex {
   HnswVectorIndex(const SchemaField::VectorParams& params, PMR_NS::memory_resource* mr);
   ~HnswVectorIndex();

src/core/search/parser.y

@@ -10,6 +10,7 @@
 %define api.value.type variant
 %define api.parser.class {Parser}
 %define parse.assert
+%define api.value.automove true
 
 // Added to header file before parser declaration.
 %code requires {
@@ -136,6 +137,7 @@ search_unary_expr:
 field_cond:
   TERM                                                  { $$ = AstTermNode(std::move($1)); }
   | UINT32                                              { $$ = AstTermNode(std::move($1)); }
+  | STAR                                                { $$ = AstStarFieldNode(); }
   | NOT_OP field_cond                                   { $$ = AstNegateNode(std::move($2)); }
   | LPAREN field_cond_expr RPAREN                       { $$ = std::move($2); }
   | LBRACKET numeric_filter_expr RBRACKET               { $$ = std::move($2); }
@@ -168,7 +170,7 @@ field_or_expr:
 
 field_unary_expr:
   LPAREN field_cond_expr RPAREN                  { $$ = std::move($2); }
-  | NOT_OP field_unary_expr                      { $$ = AstNegateNode(std::move($2)); };
+  | NOT_OP field_unary_expr                      { $$ = AstNegateNode(std::move($2)); }
   | TERM                                         { $$ = AstTermNode(std::move($1)); }
   | UINT32                                       { $$ = AstTermNode(std::move($1)); }
 

src/core/search/search.cc

@@ -143,6 +143,7 @@ struct ProfileBuilder {
         [](const AstKnnNode& n) { return absl::StrCat("KNN{l=", n.limit, "}"); },
         [](const AstNegateNode& n) { return absl::StrCat("Negate{}"); },
         [](const AstStarNode& n) { return absl::StrCat("Star{}"); },
+        [](const AstStarFieldNode& n) { return absl::StrCat("StarField{}"); },
     };
     return visit(node_info, node.Variant());
   }
@@ -302,6 +303,32 @@ struct BasicSearch {
     return UnifyResults(GetSubResults(selected_indices, mapping), LogicOp::OR);
   }
 
+  IndexResult Search(const AstStarFieldNode& node, string_view active_field) {
+    // Try to get a sort index first, as `@field:*` might imply wanting sortable behavior
+    BaseSortIndex* sort_index = indices_->GetSortIndex(active_field);
+    if (sort_index) {
+      if (auto result = sort_index->GetAllResults()) {
+        return std::move(*result);
+      }
+    }
+
+    // If sort index doesn't exist or doesn't support GetAllResults, try regular index
+    BaseIndex* base_index = indices_->GetIndex(active_field);
+    if (base_index) {
+      if (auto result = base_index->GetAllResults()) {
+        return std::move(*result);
+      }
+    }
+
+    // If we get here, neither index could handle the request
+    if (!base_index && !sort_index) {
+      error_ = absl::StrCat("Invalid field: ", active_field);
+    } else {
+      error_ = absl::StrCat("Wrong access type for field: ", active_field);
+    }
+    return IndexResult{};
+  }
+
   IndexResult Search(const AstPrefixNode& node, string_view active_field) {
     vector<TextIndex*> indices;
     if (!active_field.empty()) {

src/core/search/search_test.cc

@@ -805,6 +805,45 @@ static void BM_VectorSearch(benchmark::State& state) {
 
 BENCHMARK(BM_VectorSearch)->Args({120, 10'000});
 
+TEST_F(SearchTest, MatchNonNullField) {
+  PrepareSchema({{"text_field", SchemaField::TEXT},
+                 {"tag_field", SchemaField::TAG},
+                 {"num_field", SchemaField::NUMERIC}});
+
+  {
+    PrepareQuery("@text_field:*");
+
+    ExpectAll(Map{{"text_field", "any value"}}, Map{{"text_field", "another value"}},
+              Map{{"text_field", "third"}, {"tag_field", "tag1"}});
+
+    ExpectNone(Map{{"tag_field", "wrong field"}}, Map{{"num_field", "123"}}, Map{});
+
+    EXPECT_TRUE(Check()) << GetError();
+  }
+
+  {
+    PrepareQuery("@tag_field:*");
+
+    ExpectAll(Map{{"tag_field", "tag1"}}, Map{{"tag_field", "tag2"}},
+              Map{{"text_field", "value"}, {"tag_field", "tag3"}});
+
+    ExpectNone(Map{{"text_field", "wrong field"}}, Map{{"num_field", "456"}}, Map{});
+
+    EXPECT_TRUE(Check()) << GetError();
+  }
+
+  {
+    PrepareQuery("@num_field:*");
+
+    ExpectAll(Map{{"num_field", "123"}}, Map{{"num_field", "456"}},
+              Map{{"text_field", "value"}, {"num_field", "789"}});
+
+    ExpectNone(Map{{"text_field", "wrong field"}}, Map{{"tag_field", "tag1"}}, Map{});
+
+    EXPECT_TRUE(Check()) << GetError();
+  }
+}
+
 }  // namespace search
 
 }  // namespace dfly

src/core/search/sort_indices.h

@@ -41,6 +41,23 @@ template <typename T> struct SimpleValueSortIndex : public BaseSortIndex {
   bool Add(DocId id, const DocumentAccessor& doc, std::string_view field) override;
   void Remove(DocId id, const DocumentAccessor& doc, std::string_view field) override;
 
+  // Override GetAllResults to return all documents with non-null values
+  std::optional<std::vector<DocId>> GetAllResults() const override {
+    std::vector<DocId> result;
+
+    for (DocId id = 0; id < values_.size(); ++id) {
+      // Check if id is not present in null_values_
+      // Also need to handle deleted elements - in them T should be empty
+      // Different types of T have different "empty" values, but we can check
+      // if this value is the default for the given type
+      if (!null_values_.contains(id) && !(values_[id] == T{})) {
+        result.push_back(id);
+      }
+    }
+
+    return result;
+  }
+
  protected:
   virtual ParsedSortValue Get(const DocumentAccessor& doc, std::string_view field_value) = 0;