diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index 54986b9ff..1631d72d9 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(VectorSimilarity ${VECSIM_LIBTYPE} utils/vec_utils.cpp memory/vecsim_malloc.cpp memory/vecsim_base.cpp + info/vec_sim_info.cpp ${HEADER_LIST} ) diff --git a/src/VecSim/algorithms/brute_force/brute_force.h b/src/VecSim/algorithms/brute_force/brute_force.h index 5758bbf05..38449d64a 100644 --- a/src/VecSim/algorithms/brute_force/brute_force.h +++ b/src/VecSim/algorithms/brute_force/brute_force.h @@ -15,6 +15,7 @@ #include "VecSim/spaces/spaces.h" #include "VecSim/query_result_struct.h" #include "VecSim/utils/vec_utils.h" +#include "brute_force_info.h" #include #include @@ -48,7 +49,7 @@ class BruteForceIndex : public VecSimIndexAbstract { VecSimQueryParams *queryParams) override; VecSimQueryResult_List rangeQuery(const void *queryBlob, double radius, VecSimQueryParams *queryParams) override; - virtual VecSimIndexInfo info() const override; + virtual VecSimIndexInfo *info() const override; virtual VecSimInfoIterator *infoIterator() const override; virtual VecSimBatchIterator *newBatchIterator(const void *queryBlob, VecSimQueryParams *queryParams) const override; @@ -353,71 +354,17 @@ BruteForceIndex::rangeQuery(const void *queryBlob, double ra } template -VecSimIndexInfo BruteForceIndex::info() const { - - VecSimIndexInfo info; - info.algo = VecSimAlgo_BF; - info.bfInfo.dim = this->dim; - info.bfInfo.type = this->vecType; - info.bfInfo.metric = this->metric; - info.bfInfo.indexSize = this->count; - info.bfInfo.indexLabelCount = this->indexLabelCount(); - info.bfInfo.blockSize = this->blockSize; - info.bfInfo.memory = this->getAllocationSize(); - info.bfInfo.isMulti = this->isMulti; - info.bfInfo.last_mode = this->last_mode; - return info; +VecSimIndexInfo *BruteForceIndex::info() const { + BruteForceInfo *bfInfo = new BruteForceInfo(); + this->fillIndexInfo(bfInfo); + return bfInfo; } template VecSimInfoIterator *BruteForceIndex::infoIterator() const { - VecSimIndexInfo info = this->info(); - // For readability. Update this number when needed. - size_t numberOfInfoFields = 8; - VecSimInfoIterator *infoIterator = new VecSimInfoIterator(numberOfInfoFields); - - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::ALGORITHM_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(info.algo)}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::TYPE_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimType_ToString(info.bfInfo.type)}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::DIMENSION_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.dim}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::METRIC_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimMetric_ToString(info.bfInfo.metric)}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.isMulti}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.indexSize}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.indexLabelCount}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::BLOCK_SIZE_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.blockSize}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.memory}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::SEARCH_MODE_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{ - .stringValue = VecSimSearchMode_ToString(info.bfInfo.last_mode)}}}); - + VecSimIndexInfo *info = this->info(); + VecSimInfoIterator *infoIterator = info->getIterator(); + delete info; return infoIterator; } diff --git a/src/VecSim/algorithms/brute_force/brute_force_info.h b/src/VecSim/algorithms/brute_force/brute_force_info.h new file mode 100644 index 000000000..d885d610c --- /dev/null +++ b/src/VecSim/algorithms/brute_force/brute_force_info.h @@ -0,0 +1,11 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + +#pragma once + +#include "VecSim/vec_sim_info.h" + +struct BruteForceInfo : public VecSimIndexInfo {}; diff --git a/src/VecSim/algorithms/brute_force/copilot.cpp b/src/VecSim/algorithms/brute_force/copilot.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/src/VecSim/algorithms/hnsw/hnsw.h b/src/VecSim/algorithms/hnsw/hnsw.h index 14c4a292c..5d623c178 100644 --- a/src/VecSim/algorithms/hnsw/hnsw.h +++ b/src/VecSim/algorithms/hnsw/hnsw.h @@ -2194,69 +2194,6 @@ VecSimInfoIterator *HNSWIndex::infoIterator() const { size_t numberOfInfoFields = 12; VecSimInfoIterator *infoIterator = new VecSimInfoIterator(numberOfInfoFields); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::ALGORITHM_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(info.algo)}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::TYPE_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimType_ToString(info.hnswInfo.type)}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::DIMENSION_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.dim}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::METRIC_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimMetric_ToString(info.hnswInfo.metric)}}}); - - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.isMulti}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.indexSize}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.indexLabelCount}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_M_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.M}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::HNSW_EF_CONSTRUCTION_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efConstruction}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EF_RUNTIME_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efRuntime}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_MAX_LEVEL, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.max_level}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_ENTRYPOINT, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.entrypoint}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.memory}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::SEARCH_MODE_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{ - .stringValue = VecSimSearchMode_ToString(info.hnswInfo.last_mode)}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EPSILON_STRING, - .fieldType = INFOFIELD_FLOAT64, - .fieldValue = {FieldValue{.floatingPointValue = info.hnswInfo.epsilon}}}); - return infoIterator; } diff --git a/src/VecSim/algorithms/hnsw/hnsw_info.cpp b/src/VecSim/algorithms/hnsw/hnsw_info.cpp new file mode 100644 index 000000000..273cf7d7e --- /dev/null +++ b/src/VecSim/algorithms/hnsw/hnsw_info.cpp @@ -0,0 +1,39 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + +#include "hnsw_info.h" +#include "VecSim/utils/vec_utils.h" + +VecSimInfoIterator *HNSWInfo::getIterator() { + VecSimInfoIterator *infoIterator = VecSimIndexInfo::getIterator(); + + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_M_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.M}}}); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::HNSW_EF_CONSTRUCTION_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efConstruction}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EF_RUNTIME_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efRuntime}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_MAX_LEVEL, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.max_level}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_ENTRYPOINT, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.entrypoint}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EPSILON_STRING, + .fieldType = INFOFIELD_FLOAT64, + .fieldValue = {FieldValue{.floatingPointValue = info.hnswInfo.epsilon}}}); + + return infoIterator; +} diff --git a/src/VecSim/algorithms/hnsw/hnsw_info.h b/src/VecSim/algorithms/hnsw/hnsw_info.h new file mode 100644 index 000000000..7f2685c6a --- /dev/null +++ b/src/VecSim/algorithms/hnsw/hnsw_info.h @@ -0,0 +1,22 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + +#pragma once + +#include "VecSim/vec_sim_info.h" + +struct HNSWInfo : public VecSimIndexInfo { +public: + size_t M; // Number of allowed edges per node in graph. + size_t efConstruction; // EF parameter for HNSW graph accuracy/latency for indexing. + size_t efRuntime; // EF parameter for HNSW graph accuracy/latency for search. + double epsilon; // Epsilon parameter for HNSW graph accuracy/latency for range search. + size_t max_level; // Number of graph levels. + size_t entrypoint; // Entrypoint vector label. + size_t visitedNodesPoolSize; // The max number of parallel graph scans so far. + + virtual VecSimInfoIterator *getIterator() override +}; diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index a9bf4014e..919260d4f 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -114,7 +114,8 @@ class TieredHNSWIndex : public VecSimTieredIndex { public: TieredHNSWIndex(HNSWIndex *hnsw_index, BruteForceIndex *bf_index, - const TieredIndexParams &tieredParams); + const TieredIndexParams &tieredParams, + std::shared_ptr allocator); virtual ~TieredHNSWIndex(); int addVector(const void *blob, labelType label, void *auxiliaryCtx = nullptr) override; @@ -462,8 +463,9 @@ void TieredHNSWIndex::executeRepairJob(HNSWRepairJob *job) { template TieredHNSWIndex::TieredHNSWIndex(HNSWIndex *hnsw_index, BruteForceIndex *bf_index, - const TieredIndexParams &tiered_index_params) - : VecSimTieredIndex(hnsw_index, bf_index, tiered_index_params), + const TieredIndexParams &tiered_index_params, + std::shared_ptr allocator) + : VecSimTieredIndex(hnsw_index, bf_index, tiered_index_params, allocator), labelToInsertJobs(this->allocator), idToRepairJobs(this->allocator), idToSwapJob(this->allocator) { // If the param for swapJobThreshold is 0 use the default value, if it exceeds the maximum diff --git a/src/VecSim/index_factories/tiered_factory.cpp b/src/VecSim/index_factories/tiered_factory.cpp index e2ede8843..c344f4efe 100644 --- a/src/VecSim/index_factories/tiered_factory.cpp +++ b/src/VecSim/index_factories/tiered_factory.cpp @@ -27,7 +27,9 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) { .multi = params->primaryIndexParams->hnswParams.multi, .blockSize = params->primaryIndexParams->hnswParams.blockSize}; - AbstractIndexInitParams abstractInitParams = {.allocator = hnsw_index->getAllocator(), + // Each part of the tiered index should have its own allocator. + std::shared_ptr flat_allocator = VecSimAllocator::newVecsimAllocator(); + AbstractIndexInitParams abstractInitParams = {.allocator = flat_allocator, .dim = bf_params.dim, .vecType = bf_params.type, .metric = bf_params.metric, @@ -38,8 +40,9 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) { BruteForceFactory::NewIndex(&bf_params, abstractInitParams)); // Create new tiered hnsw index + std::shared_ptr tiered_allocator = VecSimAllocator::newVecsimAllocator(); return new (hnsw_index->getAllocator()) - TieredHNSWIndex(hnsw_index, frontendIndex, *params); + TieredHNSWIndex(hnsw_index, frontendIndex, *params, tiered_allocator); } inline size_t EstimateInitialSize(const TieredIndexParams *params, BFParams &bf_params_output) { diff --git a/src/VecSim/info_iterator_struct.h b/src/VecSim/info/info_iterator_struct.h similarity index 100% rename from src/VecSim/info_iterator_struct.h rename to src/VecSim/info/info_iterator_struct.h diff --git a/src/VecSim/info/vec_sim_info.cpp b/src/VecSim/info/vec_sim_info.cpp new file mode 100644 index 000000000..18c063537 --- /dev/null +++ b/src/VecSim/info/vec_sim_info.cpp @@ -0,0 +1,43 @@ +#include "vec_sim_info.h" +#include "utils/vec_utils.h" + +VecSimInfoIterator *VecSimIndexInfo::getIterator() { + VecSimInfoIterator *infoIterator = new VecSimInfoIterator(7); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::ALGORITHM_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(this->algo)}}}); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::TYPE_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimType_ToString(this->type)}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::DIMENSION_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->dim}}}); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::METRIC_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimMetric_ToString(this->metric)}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->isMulti}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->indexSize}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->indexLabelCount}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->memory}}}); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::SEARCH_MODE_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimSearchMode_ToString(this->last_mode)}}}); +} diff --git a/src/VecSim/info/vec_sim_info.h b/src/VecSim/info/vec_sim_info.h new file mode 100644 index 000000000..f6633bdfd --- /dev/null +++ b/src/VecSim/info/vec_sim_info.h @@ -0,0 +1,26 @@ +#pragma once +#include +#include +#include "vec_sim_common.h" +#include "info_iterator_struct.h" + +/** + * @brief Struct that holds information about the index. + * + */ +struct VecSimIndexInfo { + +public: + VecSimAlgo algo; // Index algorithm. + size_t indexSize; // Current count of vectors. + size_t indexLabelCount; // Current unique count of labels. + VecSimMetric metric; // Index distance metric + uint64_t memory; // Index memory consumption. + VecSimType type; // Datatype the index holds. + bool isMulti; // Determines if the index should multi-index or not. + size_t dim; // Vector size (dimension). + VecSearchMode last_mode; // The mode in which the last query ran. + + virtual VecSimInfoIterator *getIterator(); + virtual ~VecSimIndexInfo() {} +}; diff --git a/src/VecSim/info_iterator.cpp b/src/VecSim/info_iterator.cpp index 7099db854..b68e95606 100644 --- a/src/VecSim/info_iterator.cpp +++ b/src/VecSim/info_iterator.cpp @@ -4,7 +4,7 @@ *the Server Side Public License v1 (SSPLv1). */ -#include "info_iterator_struct.h" +#include "VecSim/info/info_iterator_struct.h" extern "C" size_t VecSimInfoIterator_NumberOfFields(VecSimInfoIterator *infoIterator) { return infoIterator->numberOfFields(); diff --git a/src/VecSim/info_iterator.h b/src/VecSim/info_iterator.h index 2cc3d79ad..a7cf1a1e5 100644 --- a/src/VecSim/info_iterator.h +++ b/src/VecSim/info_iterator.h @@ -21,7 +21,8 @@ typedef enum { INFOFIELD_STRING, INFOFIELD_INT64, INFOFIELD_UINT64, - INFOFIELD_FLOAT64 + INFOFIELD_FLOAT64, + INFOFIELD_ITERATOR } VecSim_InfoFieldType; typedef union { @@ -29,6 +30,7 @@ typedef union { int64_t integerValue; // Integer value. Signed 64 bits integer. u_int64_t uintegerValue; // Unsigned value. Unsigned 64 buts integer. const char *stringValue; // String value. + VecSimInfoIterator *iteratorValue; // Iterator value. } FieldValue; /** diff --git a/src/VecSim/tiered_index_info.cpp b/src/VecSim/tiered_index_info.cpp new file mode 100644 index 000000000..8e905b043 --- /dev/null +++ b/src/VecSim/tiered_index_info.cpp @@ -0,0 +1,49 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + +#include "tiered_index_info.h" +#include "VecSim/utils/vec_utils.h" + +virtual VecSimInfoIterator *TieredIndexInfo::getIterator() { + VecSimInfoIterator *iterator = VecSimInfoIterator_New(9); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::ALGORITHM_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(this->algo)}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->isMulti}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->indexSize}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->indexLabelCount}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->memory}}}); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::SEARCH_MODE_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimSearchMode_ToString(this->last_mode)}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::FRONTEND_INDEX_STRING, + .fieldType = INFOFIELD_ITERATOR, + .fieldValue = {FieldValue{.iteratorValue = this->frontendIndexInfo->getIterator()}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::BACKEND_INDEX_STRING, + .fieldType = INFOFIELD_ITERATOR, + .fieldValue = {FieldValue{.iteratorValue = this->backendIndexInfo->getIterator()}}}); + + return infoIterator; + +} \ No newline at end of file diff --git a/src/VecSim/tiered_index_info.h b/src/VecSim/tiered_index_info.h new file mode 100644 index 000000000..664ef21c6 --- /dev/null +++ b/src/VecSim/tiered_index_info.h @@ -0,0 +1,19 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + +#pragma once + +#include "VecSim/info/vec_sim_info.h" + +struct TieredIndexInfo : public VecSimIndexInfo { +public: + VecSimIndexInfo *backendIndexInfo; + VecSimIndexInfo *frontendIndexInfo; + + size_t management_layer_memory; + + virtual VecSimInfoIterator *getIterator(); +}; diff --git a/src/VecSim/tombstone_interface.h b/src/VecSim/tombstone_interface.h index e4e114e66..1df3eea71 100644 --- a/src/VecSim/tombstone_interface.h +++ b/src/VecSim/tombstone_interface.h @@ -1,3 +1,9 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + #pragma once #include #include "vec_sim_common.h" diff --git a/src/VecSim/utils/vec_utils.cpp b/src/VecSim/utils/vec_utils.cpp index 6ab10cc7d..b2cba43f4 100644 --- a/src/VecSim/utils/vec_utils.cpp +++ b/src/VecSim/utils/vec_utils.cpp @@ -50,6 +50,9 @@ const char *VecSimCommonStrings::SEARCH_MODE_STRING = "LAST_SEARCH_MODE"; const char *VecSimCommonStrings::HYBRID_POLICY_STRING = "HYBRID_POLICY"; const char *VecSimCommonStrings::BATCH_SIZE_STRING = "BATCH_SIZE"; +const char *VecSimCommonStrings::FRONTEND_INDEX_STRING = "FRONTEND_INDEX"; +const char *VecSimCommonStrings::BACKEND_INDEX_STRING = "BACKEND_INDEX"; + void sort_results_by_id(VecSimQueryResult_List rl) { qsort(rl.results, VecSimQueryResult_Len(rl), sizeof(VecSimQueryResult), (__compar_fn_t)cmpVecSimQueryResultById); diff --git a/src/VecSim/utils/vec_utils.h b/src/VecSim/utils/vec_utils.h index 49aa2b6d8..1002716da 100644 --- a/src/VecSim/utils/vec_utils.h +++ b/src/VecSim/utils/vec_utils.h @@ -56,6 +56,9 @@ struct VecSimCommonStrings { static const char *SEARCH_MODE_STRING; static const char *HYBRID_POLICY_STRING; static const char *BATCH_SIZE_STRING; + + static const char *FRONTEND_INDEX_STRING; + static const char *BACKEND_INDEX_STRING; }; inline int cmpVecSimQueryResultById(const VecSimQueryResult *res1, const VecSimQueryResult *res2) { diff --git a/src/VecSim/vec_sim.cpp b/src/VecSim/vec_sim.cpp index ff61cceb4..82f8f21b1 100644 --- a/src/VecSim/vec_sim.cpp +++ b/src/VecSim/vec_sim.cpp @@ -235,8 +235,6 @@ extern "C" void VecSimIndex_Free(VecSimIndex *index) { delete index; } -extern "C" VecSimIndexInfo VecSimIndex_Info(VecSimIndex *index) { return index->info(); } - extern "C" VecSimInfoIterator *VecSimIndex_InfoIterator(VecSimIndex *index) { return index->infoIterator(); } diff --git a/src/VecSim/vec_sim.h b/src/VecSim/vec_sim.h index d1b293d27..b5aafe203 100644 --- a/src/VecSim/vec_sim.h +++ b/src/VecSim/vec_sim.h @@ -141,12 +141,6 @@ VecSimQueryResult_List VecSimIndex_TopKQuery(VecSimIndex *index, const void *que VecSimQueryResult_List VecSimIndex_RangeQuery(VecSimIndex *index, const void *queryBlob, double radius, VecSimQueryParams *queryParams, VecSimQueryResult_Order); -/** - * @brief Return index information. - * @param index the index to return its info. - * @return Index general and specific meta-data. - */ -VecSimIndexInfo VecSimIndex_Info(VecSimIndex *index); /** * @brief Returns an info iterator for generic reply purposes. diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 551e0d5f2..e61ff469d 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -189,45 +189,6 @@ typedef struct { // to get it from the parameters resolve function. } VecSimQueryParams; -/** - * @brief Index information. Mainly used for debug/testing. - * - */ -typedef struct { - union { - struct { - size_t indexSize; // Current count of vectors. - size_t indexLabelCount; // Current unique count of labels. - size_t blockSize; // Sets the amount to grow when resizing - size_t M; // Number of allowed edges per node in graph. - size_t efConstruction; // EF parameter for HNSW graph accuracy/latency for indexing. - size_t efRuntime; // EF parameter for HNSW graph accuracy/latency for search. - double epsilon; // Epsilon parameter for HNSW graph accuracy/latency for range search. - size_t max_level; // Number of graph levels. - size_t entrypoint; // Entrypoint vector label. - VecSimMetric metric; // Index distance metric - uint64_t memory; // Index memory consumption. - VecSimType type; // Datatype the index holds. - bool isMulti; // Determines if the index should multi-index or not. - size_t dim; // Vector size (dimension). - VecSearchMode last_mode; // The mode in which the last query ran. - size_t visitedNodesPoolSize; // The max number of parallel graph scans so far. - } hnswInfo; - struct { - size_t indexSize; // Current count of vectors. - size_t indexLabelCount; // Current unique count of labels. - size_t blockSize; // Brute force algorithm vector block (mini matrix) size - VecSimMetric metric; // Index distance metric - uint64_t memory; // Index memory consumption. - VecSimType type; // Datatype the index holds. - bool isMulti; // Determines if the index should multi-index or not. - size_t dim; // Vector size (dimension). - VecSearchMode last_mode; // The mode in which the last query ran. - } bfInfo; - }; - VecSimAlgo algo; // Algorithm being used. -} VecSimIndexInfo; - // Memory function declarations. typedef void *(*allocFn)(size_t n); typedef void *(*callocFn)(size_t nelem, size_t elemsz); diff --git a/src/VecSim/vec_sim_index.h b/src/VecSim/vec_sim_index.h index 607482c45..bea7d6ebd 100644 --- a/src/VecSim/vec_sim_index.h +++ b/src/VecSim/vec_sim_index.h @@ -13,6 +13,7 @@ #include "VecSim/utils/vec_utils.h" #include "VecSim/spaces/spaces.h" #include "info_iterator_struct.h" +#include "VecSim/vec_sim_info.h" #include using spaces::dist_func_t; @@ -56,6 +57,19 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { bool isMulti; // Determines if the index should multi-index or not. void *logCallbackCtx; // Context for the log callback. + void fillIndexInfo getInfo(VecSimIndexInfo *info) { + info->algo = this->getAlgo(); + info->indexSize = this->size(); + info->indexLabelCount = this->labelCount(); + info->metric = this->getMetric(); + info->memory = this->getMemoryUsage(); + info->type = this->getType(); + info->isMulti = this->isMultiValue(); + info->dim = this->getDim(); + info->last_mode = this->last_mode; + info->blockSize = this->blockSize; + } + public: /** * @brief Construct a new Vec Sim Index object diff --git a/src/VecSim/vec_sim_interface.h b/src/VecSim/vec_sim_interface.h index 6b4d28a97..4e070c21e 100644 --- a/src/VecSim/vec_sim_interface.h +++ b/src/VecSim/vec_sim_interface.h @@ -129,7 +129,7 @@ struct VecSimIndexInterface : public VecsimBaseObject { * * @return Index general and specific meta-data. */ - virtual VecSimIndexInfo info() const = 0; + virtual VecSimIndexInfo *info() const = 0; /** * @brief Returns an index information in an iterable structure. diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index e549b9cb1..3225f92cd 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -1,8 +1,14 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + #pragma once #include "vec_sim_index.h" #include "algorithms/brute_force/brute_force.h" - +#include "tiered_index_info.h" // TODO: Consider moving tiered index files from root to their own folder. #include /** @@ -44,8 +50,8 @@ class VecSimTieredIndex : public VecSimIndexInterface { public: VecSimTieredIndex(VecSimIndexAbstract *backendIndex_, BruteForceIndex *frontendIndex_, - TieredIndexParams tieredParams) - : VecSimIndexInterface(backendIndex_->getAllocator()), backendIndex(backendIndex_), + TieredIndexParams tieredParams, std::shared_ptr allocator) + : VecSimIndexInterface(allocator), backendIndex(backendIndex_), frontendIndex(frontendIndex_), jobQueue(tieredParams.jobQueue), jobQueueCtx(tieredParams.jobQueueCtx), SubmitJobsToQueue(tieredParams.submitCb), memoryCtx(tieredParams.memoryCtx), UpdateIndexMemory(tieredParams.UpdateMemCb) {} @@ -57,6 +63,8 @@ class VecSimTieredIndex : public VecSimIndexInterface { VecSimQueryResult_List topKQuery(const void *queryBlob, size_t k, VecSimQueryParams *queryParams) override; + + virtual VecSimIndexInfo *info() override; }; template @@ -109,3 +117,19 @@ VecSimTieredIndex::topKQuery(const void *queryBlob, size_t k } } } + +virtual VecSimIndexInfo *VecSimTieredIndex::info() { + TieredIndexInfo *info = new TieredIndexInfo(); + info->backendIndexInfo = this->backendIndex->info(); + info->frontendIndexInfo = this->frontendIndex->info(); + info->management_layer_memory = this->getAllocator()->getAllocatedMemory(); + info->memory = info->management_layer_memory + info->backendIndexInfo->memory + + info->frontendIndexInfo->memory; + info->indexType = VecSimAlgo_TIERED; + info->indexSize = this->indexSize(); + info->indexLabelCount = this->indexLabelCount(); + info->last_mode = this->last_mode; + info->isMultiValue = this->isMultiValue(); + + return info; +}