RedisAI · DvirDukhan · Apr 3, 2023 · Apr 3, 2023 · Apr 24, 2023 · Apr 24, 2023
diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt
@@ -29,6 +29,7 @@ add_library(VectorSimilarity ${VECSIM_LIBTYPE}
     utils/vec_utils.cpp
     memory/vecsim_malloc.cpp
     memory/vecsim_base.cpp
+    info/vec_sim_info.cpp
     ${HEADER_LIST}
 )
 

diff --git a/src/VecSim/algorithms/brute_force/brute_force.h b/src/VecSim/algorithms/brute_force/brute_force.h
@@ -15,6 +15,7 @@
 #include "VecSim/spaces/spaces.h"
 #include "VecSim/query_result_struct.h"
 #include "VecSim/utils/vec_utils.h"
+#include "brute_force_info.h"
 
 #include <cstring>
 #include <cmath>
@@ -48,7 +49,7 @@ class BruteForceIndex : public VecSimIndexAbstract<DistType> {
                                              VecSimQueryParams *queryParams) override;
     VecSimQueryResult_List rangeQuery(const void *queryBlob, double radius,
                                       VecSimQueryParams *queryParams) override;
-    virtual VecSimIndexInfo info() const override;
+    virtual VecSimIndexInfo *info() const override;
     virtual VecSimInfoIterator *infoIterator() const override;
     virtual VecSimBatchIterator *newBatchIterator(const void *queryBlob,
                                                   VecSimQueryParams *queryParams) const override;
@@ -353,71 +354,17 @@ BruteForceIndex<DataType, DistType>::rangeQuery(const void *queryBlob, double ra
 }
 
 template <typename DataType, typename DistType>
-VecSimIndexInfo BruteForceIndex<DataType, DistType>::info() const {
-
-    VecSimIndexInfo info;
-    info.algo = VecSimAlgo_BF;
-    info.bfInfo.dim = this->dim;
-    info.bfInfo.type = this->vecType;
-    info.bfInfo.metric = this->metric;
-    info.bfInfo.indexSize = this->count;
-    info.bfInfo.indexLabelCount = this->indexLabelCount();
-    info.bfInfo.blockSize = this->blockSize;
-    info.bfInfo.memory = this->getAllocationSize();
-    info.bfInfo.isMulti = this->isMulti;
-    info.bfInfo.last_mode = this->last_mode;
-    return info;
+VecSimIndexInfo *BruteForceIndex<DataType, DistType>::info() const {
+    BruteForceInfo *bfInfo = new BruteForceInfo();
+    this->fillIndexInfo(bfInfo);
+    return bfInfo;
 }
 
 template <typename DataType, typename DistType>
 VecSimInfoIterator *BruteForceIndex<DataType, DistType>::infoIterator() const {
-    VecSimIndexInfo info = this->info();
-    // For readability. Update this number when needed.
-    size_t numberOfInfoFields = 8;
-    VecSimInfoIterator *infoIterator = new VecSimInfoIterator(numberOfInfoFields);
-
-    infoIterator->addInfoField(VecSim_InfoField{
-        .fieldName = VecSimCommonStrings::ALGORITHM_STRING,
-        .fieldType = INFOFIELD_STRING,
-        .fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(info.algo)}}});
-    infoIterator->addInfoField(VecSim_InfoField{
-        .fieldName = VecSimCommonStrings::TYPE_STRING,
-        .fieldType = INFOFIELD_STRING,
-        .fieldValue = {FieldValue{.stringValue = VecSimType_ToString(info.bfInfo.type)}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::DIMENSION_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.dim}}});
-    infoIterator->addInfoField(VecSim_InfoField{
-        .fieldName = VecSimCommonStrings::METRIC_STRING,
-        .fieldType = INFOFIELD_STRING,
-        .fieldValue = {FieldValue{.stringValue = VecSimMetric_ToString(info.bfInfo.metric)}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.isMulti}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.indexSize}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.indexLabelCount}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::BLOCK_SIZE_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.blockSize}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.memory}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::SEARCH_MODE_STRING,
-                         .fieldType = INFOFIELD_STRING,
-                         .fieldValue = {FieldValue{
-                             .stringValue = VecSimSearchMode_ToString(info.bfInfo.last_mode)}}});
-
+    VecSimIndexInfo *info = this->info();
+    VecSimInfoIterator *infoIterator = info->getIterator();
+    delete info;
     return infoIterator;
 }
 

diff --git a/src/VecSim/algorithms/brute_force/brute_force_info.h b/src/VecSim/algorithms/brute_force/brute_force_info.h
@@ -0,0 +1,11 @@
+/*
+ *Copyright Redis Ltd. 2021 - present
+ *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or
+ *the Server Side Public License v1 (SSPLv1).
+ */
+
+#pragma once
+
+#include "VecSim/vec_sim_info.h"
+
+struct BruteForceInfo : public VecSimIndexInfo {};
diff --git a/src/VecSim/algorithms/brute_force/copilot.cpp b/src/VecSim/algorithms/brute_force/copilot.cpp
diff --git a/src/VecSim/algorithms/hnsw/hnsw.h b/src/VecSim/algorithms/hnsw/hnsw.h
@@ -2194,69 +2194,6 @@ VecSimInfoIterator *HNSWIndex<DataType, DistType>::infoIterator() const {
     size_t numberOfInfoFields = 12;
     VecSimInfoIterator *infoIterator = new VecSimInfoIterator(numberOfInfoFields);
 
-    infoIterator->addInfoField(VecSim_InfoField{
-        .fieldName = VecSimCommonStrings::ALGORITHM_STRING,
-        .fieldType = INFOFIELD_STRING,
-        .fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(info.algo)}}});
-    infoIterator->addInfoField(VecSim_InfoField{
-        .fieldName = VecSimCommonStrings::TYPE_STRING,
-        .fieldType = INFOFIELD_STRING,
-        .fieldValue = {FieldValue{.stringValue = VecSimType_ToString(info.hnswInfo.type)}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::DIMENSION_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.dim}}});
-    infoIterator->addInfoField(VecSim_InfoField{
-        .fieldName = VecSimCommonStrings::METRIC_STRING,
-        .fieldType = INFOFIELD_STRING,
-        .fieldValue = {FieldValue{.stringValue = VecSimMetric_ToString(info.hnswInfo.metric)}}});
-
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.isMulti}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.indexSize}}});
-    infoIterator->addInfoField(VecSim_InfoField{
-        .fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING,
-        .fieldType = INFOFIELD_UINT64,
-        .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.indexLabelCount}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_M_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.M}}});
-    infoIterator->addInfoField(VecSim_InfoField{
-        .fieldName = VecSimCommonStrings::HNSW_EF_CONSTRUCTION_STRING,
-        .fieldType = INFOFIELD_UINT64,
-        .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efConstruction}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EF_RUNTIME_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efRuntime}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_MAX_LEVEL,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.max_level}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_ENTRYPOINT,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.entrypoint}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING,
-                         .fieldType = INFOFIELD_UINT64,
-                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.memory}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::SEARCH_MODE_STRING,
-                         .fieldType = INFOFIELD_STRING,
-                         .fieldValue = {FieldValue{
-                             .stringValue = VecSimSearchMode_ToString(info.hnswInfo.last_mode)}}});
-    infoIterator->addInfoField(
-        VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EPSILON_STRING,
-                         .fieldType = INFOFIELD_FLOAT64,
-                         .fieldValue = {FieldValue{.floatingPointValue = info.hnswInfo.epsilon}}});
-
     return infoIterator;
 }
 

diff --git a/src/VecSim/algorithms/hnsw/hnsw_info.cpp b/src/VecSim/algorithms/hnsw/hnsw_info.cpp
@@ -0,0 +1,39 @@
+/*
+ *Copyright Redis Ltd. 2021 - present
+ *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or
+ *the Server Side Public License v1 (SSPLv1).
+ */
+
+#include "hnsw_info.h"
+#include "VecSim/utils/vec_utils.h"
+
+VecSimInfoIterator *HNSWInfo::getIterator() {
+    VecSimInfoIterator *infoIterator = VecSimIndexInfo::getIterator();
+
+    infoIterator->addInfoField(
+        VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_M_STRING,
+                         .fieldType = INFOFIELD_UINT64,
+                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.M}}});
+    infoIterator->addInfoField(VecSim_InfoField{
+        .fieldName = VecSimCommonStrings::HNSW_EF_CONSTRUCTION_STRING,
+        .fieldType = INFOFIELD_UINT64,
+        .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efConstruction}}});
+    infoIterator->addInfoField(
+        VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EF_RUNTIME_STRING,
+                         .fieldType = INFOFIELD_UINT64,
+                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efRuntime}}});
+    infoIterator->addInfoField(
+        VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_MAX_LEVEL,
+                         .fieldType = INFOFIELD_UINT64,
+                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.max_level}}});
+    infoIterator->addInfoField(
+        VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_ENTRYPOINT,
+                         .fieldType = INFOFIELD_UINT64,
+                         .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.entrypoint}}});
+    infoIterator->addInfoField(
+        VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EPSILON_STRING,
+                         .fieldType = INFOFIELD_FLOAT64,
+                         .fieldValue = {FieldValue{.floatingPointValue = info.hnswInfo.epsilon}}});
+
+    return infoIterator;
+}
diff --git a/src/VecSim/algorithms/hnsw/hnsw_info.h b/src/VecSim/algorithms/hnsw/hnsw_info.h
@@ -0,0 +1,22 @@
+/*
+ *Copyright Redis Ltd. 2021 - present
+ *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or
+ *the Server Side Public License v1 (SSPLv1).
+ */
+
+#pragma once
+
+#include "VecSim/vec_sim_info.h"
+
+struct HNSWInfo : public VecSimIndexInfo {
+public:
+    size_t M;              // Number of allowed edges per node in graph.
+    size_t efConstruction; // EF parameter for HNSW graph accuracy/latency for indexing.
+    size_t efRuntime;      // EF parameter for HNSW graph accuracy/latency for search.
+    double epsilon;        // Epsilon parameter for HNSW graph accuracy/latency for range search.
+    size_t max_level;      // Number of graph levels.
+    size_t entrypoint;     // Entrypoint vector label.
+    size_t visitedNodesPoolSize; // The max number of parallel graph scans so far.
+
+    virtual VecSimInfoIterator *getIterator() override
+};
diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h
@@ -114,7 +114,8 @@ class TieredHNSWIndex : public VecSimTieredIndex<DataType, DistType> {
 public:
     TieredHNSWIndex(HNSWIndex<DataType, DistType> *hnsw_index,
                     BruteForceIndex<DataType, DistType> *bf_index,
-                    const TieredIndexParams &tieredParams);
+                    const TieredIndexParams &tieredParams,
+                    std::shared_ptr<VecSimAllocator> allocator);
     virtual ~TieredHNSWIndex();
 
     int addVector(const void *blob, labelType label, void *auxiliaryCtx = nullptr) override;
@@ -462,8 +463,9 @@ void TieredHNSWIndex<DataType, DistType>::executeRepairJob(HNSWRepairJob *job) {
 template <typename DataType, typename DistType>
 TieredHNSWIndex<DataType, DistType>::TieredHNSWIndex(HNSWIndex<DataType, DistType> *hnsw_index,
                                                      BruteForceIndex<DataType, DistType> *bf_index,
-                                                     const TieredIndexParams &tiered_index_params)
-    : VecSimTieredIndex<DataType, DistType>(hnsw_index, bf_index, tiered_index_params),
+                                                     const TieredIndexParams &tiered_index_params,
+                                                     std::shared_ptr<VecSimAllocator> allocator)
+    : VecSimTieredIndex<DataType, DistType>(hnsw_index, bf_index, tiered_index_params, allocator),
       labelToInsertJobs(this->allocator), idToRepairJobs(this->allocator),
       idToSwapJob(this->allocator) {
     // If the param for swapJobThreshold is 0 use the default value, if it exceeds the maximum

diff --git a/src/VecSim/index_factories/tiered_factory.cpp b/src/VecSim/index_factories/tiered_factory.cpp
@@ -27,7 +27,9 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) {
                           .multi = params->primaryIndexParams->hnswParams.multi,
                           .blockSize = params->primaryIndexParams->hnswParams.blockSize};
 
-    AbstractIndexInitParams abstractInitParams = {.allocator = hnsw_index->getAllocator(),
+    // Each part of the tiered index should have its own allocator.
+    std::shared_ptr<VecSimAllocator> flat_allocator = VecSimAllocator::newVecsimAllocator();
+    AbstractIndexInitParams abstractInitParams = {.allocator = flat_allocator,
                                                   .dim = bf_params.dim,
                                                   .vecType = bf_params.type,
                                                   .metric = bf_params.metric,
@@ -38,8 +40,9 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) {
         BruteForceFactory::NewIndex(&bf_params, abstractInitParams));
 
     // Create new tiered hnsw index
+    std::shared_ptr<VecSimAllocator> tiered_allocator = VecSimAllocator::newVecsimAllocator();
     return new (hnsw_index->getAllocator())
-        TieredHNSWIndex<DataType, DistType>(hnsw_index, frontendIndex, *params);
+        TieredHNSWIndex<DataType, DistType>(hnsw_index, frontendIndex, *params, tiered_allocator);
 }
 
 inline size_t EstimateInitialSize(const TieredIndexParams *params, BFParams &bf_params_output) {

diff --git a/src/VecSim/info_iterator_struct.h → src/VecSim/info/info_iterator_struct.h b/src/VecSim/info_iterator_struct.h → src/VecSim/info/info_iterator_struct.h
diff --git a/src/VecSim/info/vec_sim_info.cpp b/src/VecSim/info/vec_sim_info.cpp
@@ -0,0 +1,43 @@
+#include "vec_sim_info.h"
+#include "utils/vec_utils.h"
+
+VecSimInfoIterator *VecSimIndexInfo::getIterator() {
+    VecSimInfoIterator *infoIterator = new VecSimInfoIterator(7);
+
+    infoIterator->addInfoField(VecSim_InfoField{
+        .fieldName = VecSimCommonStrings::ALGORITHM_STRING,
+        .fieldType = INFOFIELD_STRING,
+        .fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(this->algo)}}});
+    infoIterator->addInfoField(VecSim_InfoField{
+        .fieldName = VecSimCommonStrings::TYPE_STRING,
+        .fieldType = INFOFIELD_STRING,
+        .fieldValue = {FieldValue{.stringValue = VecSimType_ToString(this->type)}}});
+    infoIterator->addInfoField(
+        VecSim_InfoField{.fieldName = VecSimCommonStrings::DIMENSION_STRING,
+                         .fieldType = INFOFIELD_UINT64,
+                         .fieldValue = {FieldValue{.uintegerValue = this->dim}}});
+    infoIterator->addInfoField(VecSim_InfoField{
+        .fieldName = VecSimCommonStrings::METRIC_STRING,
+        .fieldType = INFOFIELD_STRING,
+        .fieldValue = {FieldValue{.stringValue = VecSimMetric_ToString(this->metric)}}});
+    infoIterator->addInfoField(
+        VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING,
+                         .fieldType = INFOFIELD_UINT64,
+                         .fieldValue = {FieldValue{.uintegerValue = this->isMulti}}});
+    infoIterator->addInfoField(
+        VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING,
+                         .fieldType = INFOFIELD_UINT64,
+                         .fieldValue = {FieldValue{.uintegerValue = this->indexSize}}});
+    infoIterator->addInfoField(
+        VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING,
+                         .fieldType = INFOFIELD_UINT64,
+                         .fieldValue = {FieldValue{.uintegerValue = this->indexLabelCount}}});
+    infoIterator->addInfoField(
+        VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING,
+                         .fieldType = INFOFIELD_UINT64,
+                         .fieldValue = {FieldValue{.uintegerValue = this->memory}}});
+    infoIterator->addInfoField(VecSim_InfoField{
+        .fieldName = VecSimCommonStrings::SEARCH_MODE_STRING,
+        .fieldType = INFOFIELD_STRING,
+        .fieldValue = {FieldValue{.stringValue = VecSimSearchMode_ToString(this->last_mode)}}});
+}
diff --git a/src/VecSim/info/vec_sim_info.h b/src/VecSim/info/vec_sim_info.h
@@ -0,0 +1,26 @@
+#pragma once
+#include <stddef.h>
+#include <stdint.h>
+#include "vec_sim_common.h"
+#include "info_iterator_struct.h"
+
+/**
+ * @brief Struct that holds information about the index.
+ *
+ */
+struct VecSimIndexInfo {
+
+public:
+    VecSimAlgo algo;         // Index algorithm.
+    size_t indexSize;        // Current count of vectors.
+    size_t indexLabelCount;  // Current unique count of labels.
+    VecSimMetric metric;     // Index distance metric
+    uint64_t memory;         // Index memory consumption.
+    VecSimType type;         // Datatype the index holds.
+    bool isMulti;            // Determines if the index should multi-index or not.
+    size_t dim;              // Vector size (dimension).
+    VecSearchMode last_mode; // The mode in which the last query ran.
+
+    virtual VecSimInfoIterator *getIterator();
+    virtual ~VecSimIndexInfo() {}
+};
diff --git a/src/VecSim/info_iterator.cpp b/src/VecSim/info_iterator.cpp
@@ -4,7 +4,7 @@
  *the Server Side Public License v1 (SSPLv1).
  */
 
-#include "info_iterator_struct.h"
+#include "VecSim/info/info_iterator_struct.h"
 
 extern "C" size_t VecSimInfoIterator_NumberOfFields(VecSimInfoIterator *infoIterator) {
     return infoIterator->numberOfFields();

diff --git a/src/VecSim/info_iterator.h b/src/VecSim/info_iterator.h
@@ -21,14 +21,16 @@ typedef enum {
     INFOFIELD_STRING,
     INFOFIELD_INT64,
     INFOFIELD_UINT64,
-    INFOFIELD_FLOAT64
+    INFOFIELD_FLOAT64,
+    INFOFIELD_ITERATOR
 } VecSim_InfoFieldType;
 
 typedef union {
     double floatingPointValue; // Floating point value. 64 bits float.
     int64_t integerValue;      // Integer value. Signed 64 bits integer.
     u_int64_t uintegerValue;   // Unsigned value. Unsigned 64 buts integer.
     const char *stringValue;   // String value.
+    VecSimInfoIterator *iteratorValue; // Iterator value.
 } FieldValue;
 
 /**