From ed9fc4faaf956c7e2d617cd87954ef88ede500f8 Mon Sep 17 00:00:00 2001 From: DvirDukhan Date: Mon, 3 Apr 2023 09:17:54 +0300 Subject: [PATCH 1/4] wip --- src/VecSim/vec_sim.cpp | 2 -- src/VecSim/vec_sim.h | 6 ----- src/VecSim/vec_sim_index.h | 19 ++++++++++++++++ src/VecSim/vec_sim_info.cpp | 44 +++++++++++++++++++++++++++++++++++++ src/VecSim/vec_sim_info.h | 27 +++++++++++++++++++++++ 5 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 src/VecSim/vec_sim_info.cpp create mode 100644 src/VecSim/vec_sim_info.h diff --git a/src/VecSim/vec_sim.cpp b/src/VecSim/vec_sim.cpp index 4dff71ec9..03dfc60f8 100644 --- a/src/VecSim/vec_sim.cpp +++ b/src/VecSim/vec_sim.cpp @@ -259,8 +259,6 @@ extern "C" void VecSimIndex_Free(VecSimIndex *index) { delete index; } -extern "C" VecSimIndexInfo VecSimIndex_Info(VecSimIndex *index) { return index->info(); } - extern "C" VecSimInfoIterator *VecSimIndex_InfoIterator(VecSimIndex *index) { return index->infoIterator(); } diff --git a/src/VecSim/vec_sim.h b/src/VecSim/vec_sim.h index 936b06cec..6655d7043 100644 --- a/src/VecSim/vec_sim.h +++ b/src/VecSim/vec_sim.h @@ -141,12 +141,6 @@ VecSimQueryResult_List VecSimIndex_TopKQuery(VecSimIndex *index, const void *que VecSimQueryResult_List VecSimIndex_RangeQuery(VecSimIndex *index, const void *queryBlob, double radius, VecSimQueryParams *queryParams, VecSimQueryResult_Order); -/** - * @brief Return index information. - * @param index the index to return its info. - * @return Index general and specific meta-data. - */ -VecSimIndexInfo VecSimIndex_Info(VecSimIndex *index); /** * @brief Returns an info iterator for generic reply purposes. diff --git a/src/VecSim/vec_sim_index.h b/src/VecSim/vec_sim_index.h index 7e33b69ad..12d213aa5 100644 --- a/src/VecSim/vec_sim_index.h +++ b/src/VecSim/vec_sim_index.h @@ -60,4 +60,23 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { inline bool isMultiValue() const { return isMulti; } inline VecSimType getType() const { return vecType; } inline VecSimMetric getMetric() const { return metric; } + + VecSimInfo* info() override const { + VecSimInfo* info = VecSimInfo_New(); + info->indexType = this->getType(); + info->metric = this->getMetric(); + info->dimension = this->getDim(); + info->size = this->size(); + info->lastSearchMode = this->last_mode; + info->isMultiValue = this->isMultiValue(); + info->labelCount = this->getLabelCount(); + return info; + } + + VecSimInfoIterator *infoIterator() override const { + VecSimInfo *info = this->info(); + VecSimInfoIterator *iter = info->infoIterator(); + delete info; + return iter; + } }; diff --git a/src/VecSim/vec_sim_info.cpp b/src/VecSim/vec_sim_info.cpp new file mode 100644 index 000000000..fa6b7f69c --- /dev/null +++ b/src/VecSim/vec_sim_info.cpp @@ -0,0 +1,44 @@ +#include "vec_sim_info.h" +#include "utils/vec_utils.h" + +VecSimInfoIterator* VecSimInfo::getIterator() { + VecSimInfoIterator *infoIterator = new VecSimInfoIterator(7); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::ALGORITHM_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(this->algo)}}}); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::TYPE_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimType_ToString(this->type)}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::DIMENSION_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->dim}}}); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::METRIC_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimMetric_ToString(this->metric)}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->isMulti}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->indexSize}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->indexLabelCount}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->memory}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::SEARCH_MODE_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{ + .stringValue = VecSimSearchMode_ToString(this->last_mode)}}}); +} \ No newline at end of file diff --git a/src/VecSim/vec_sim_info.h b/src/VecSim/vec_sim_info.h new file mode 100644 index 000000000..221ed76fb --- /dev/null +++ b/src/VecSim/vec_sim_info.h @@ -0,0 +1,27 @@ +#pragma once +#include +#include +#include "vec_sim_common.h" +#include "info_iterator_struct.h" + +/** + * @brief Struct that holds information about the index. + * + */ +struct VecSimInfo { + + public: + VecSimAlgo algo; // Index algorithm. + size_t indexSize; // Current count of vectors. + size_t indexLabelCount; // Current unique count of labels. + VecSimMetric metric; // Index distance metric + uint64_t memory; // Index memory consumption. + VecSimType type; // Datatype the index holds. + bool isMulti; // Determines if the index should multi-index or not. + size_t dim; // Vector size (dimension). + VecSearchMode last_mode; // The mode in which the last query ran. + + virtual VecSimInfoIterator* getIterator(); + virtual ~VecSimInfo() {} + +}; From ace16abbffe43575da5565c1c14d03e29b0f02d6 Mon Sep 17 00:00:00 2001 From: DvirDukhan Date: Mon, 3 Apr 2023 17:50:38 +0300 Subject: [PATCH 2/4] wip --- src/VecSim/vec_sim_index.h | 4 ++-- src/VecSim/vec_sim_info.cpp | 13 ++++++------- src/VecSim/vec_sim_info.h | 23 +++++++++++------------ 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/VecSim/vec_sim_index.h b/src/VecSim/vec_sim_index.h index 12d213aa5..c848bb38f 100644 --- a/src/VecSim/vec_sim_index.h +++ b/src/VecSim/vec_sim_index.h @@ -61,8 +61,8 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { inline VecSimType getType() const { return vecType; } inline VecSimMetric getMetric() const { return metric; } - VecSimInfo* info() override const { - VecSimInfo* info = VecSimInfo_New(); + VecSimInfo *info() override const { + VecSimInfo *info = VecSimInfo_New(); info->indexType = this->getType(); info->metric = this->getMetric(); info->dimension = this->getDim(); diff --git a/src/VecSim/vec_sim_info.cpp b/src/VecSim/vec_sim_info.cpp index fa6b7f69c..6d73f6bba 100644 --- a/src/VecSim/vec_sim_info.cpp +++ b/src/VecSim/vec_sim_info.cpp @@ -1,8 +1,8 @@ #include "vec_sim_info.h" #include "utils/vec_utils.h" -VecSimInfoIterator* VecSimInfo::getIterator() { - VecSimInfoIterator *infoIterator = new VecSimInfoIterator(7); +VecSimInfoIterator *VecSimInfo::getIterator() { + VecSimInfoIterator *infoIterator = new VecSimInfoIterator(7); infoIterator->addInfoField(VecSim_InfoField{ .fieldName = VecSimCommonStrings::ALGORITHM_STRING, @@ -36,9 +36,8 @@ VecSimInfoIterator* VecSimInfo::getIterator() { VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING, .fieldType = INFOFIELD_UINT64, .fieldValue = {FieldValue{.uintegerValue = this->memory}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::SEARCH_MODE_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{ - .stringValue = VecSimSearchMode_ToString(this->last_mode)}}}); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::SEARCH_MODE_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimSearchMode_ToString(this->last_mode)}}}); } \ No newline at end of file diff --git a/src/VecSim/vec_sim_info.h b/src/VecSim/vec_sim_info.h index 221ed76fb..2d1928584 100644 --- a/src/VecSim/vec_sim_info.h +++ b/src/VecSim/vec_sim_info.h @@ -10,18 +10,17 @@ */ struct VecSimInfo { - public: - VecSimAlgo algo; // Index algorithm. - size_t indexSize; // Current count of vectors. - size_t indexLabelCount; // Current unique count of labels. - VecSimMetric metric; // Index distance metric - uint64_t memory; // Index memory consumption. - VecSimType type; // Datatype the index holds. - bool isMulti; // Determines if the index should multi-index or not. - size_t dim; // Vector size (dimension). - VecSearchMode last_mode; // The mode in which the last query ran. +public: + VecSimAlgo algo; // Index algorithm. + size_t indexSize; // Current count of vectors. + size_t indexLabelCount; // Current unique count of labels. + VecSimMetric metric; // Index distance metric + uint64_t memory; // Index memory consumption. + VecSimType type; // Datatype the index holds. + bool isMulti; // Determines if the index should multi-index or not. + size_t dim; // Vector size (dimension). + VecSearchMode last_mode; // The mode in which the last query ran. - virtual VecSimInfoIterator* getIterator(); + virtual VecSimInfoIterator *getIterator(); virtual ~VecSimInfo() {} - }; From 48f5fac30b87a5b56f235d87d12c713e78e6e934 Mon Sep 17 00:00:00 2001 From: DvirDukhan Date: Mon, 24 Apr 2023 13:54:21 +0300 Subject: [PATCH 3/4] wip FLAT --- .../algorithms/brute_force/brute_force.h | 23 +++++++------------ .../brute_force/brute_force_info.cpp | 17 ++++++++++++++ .../algorithms/brute_force/brute_force_info.h | 16 +++++++++++++ src/VecSim/vec_sim_index.h | 15 ++++++++++++ src/VecSim/vec_sim_info.cpp | 2 +- src/VecSim/vec_sim_interface.h | 2 +- 6 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 src/VecSim/algorithms/brute_force/brute_force_info.cpp create mode 100644 src/VecSim/algorithms/brute_force/brute_force_info.h diff --git a/src/VecSim/algorithms/brute_force/brute_force.h b/src/VecSim/algorithms/brute_force/brute_force.h index 5758bbf05..f8494aab9 100644 --- a/src/VecSim/algorithms/brute_force/brute_force.h +++ b/src/VecSim/algorithms/brute_force/brute_force.h @@ -15,6 +15,7 @@ #include "VecSim/spaces/spaces.h" #include "VecSim/query_result_struct.h" #include "VecSim/utils/vec_utils.h" +#include "brute_force_info.h" #include #include @@ -48,7 +49,7 @@ class BruteForceIndex : public VecSimIndexAbstract { VecSimQueryParams *queryParams) override; VecSimQueryResult_List rangeQuery(const void *queryBlob, double radius, VecSimQueryParams *queryParams) override; - virtual VecSimIndexInfo info() const override; + virtual VecSimInfo *info() const override; virtual VecSimInfoIterator *infoIterator() const override; virtual VecSimBatchIterator *newBatchIterator(const void *queryBlob, VecSimQueryParams *queryParams) const override; @@ -353,20 +354,12 @@ BruteForceIndex::rangeQuery(const void *queryBlob, double ra } template -VecSimIndexInfo BruteForceIndex::info() const { - - VecSimIndexInfo info; - info.algo = VecSimAlgo_BF; - info.bfInfo.dim = this->dim; - info.bfInfo.type = this->vecType; - info.bfInfo.metric = this->metric; - info.bfInfo.indexSize = this->count; - info.bfInfo.indexLabelCount = this->indexLabelCount(); - info.bfInfo.blockSize = this->blockSize; - info.bfInfo.memory = this->getAllocationSize(); - info.bfInfo.isMulti = this->isMulti; - info.bfInfo.last_mode = this->last_mode; - return info; +VecSimInfo *BruteForceIndex::info() const { + VecSimInfo *info = VecSimIndexAbstract::info(); + BruteForceInfo *bfInfo = new BruteForceInfo(info); + bfInfo->blockSize = this->blockSize; + delete info; + return bfInfo; } template diff --git a/src/VecSim/algorithms/brute_force/brute_force_info.cpp b/src/VecSim/algorithms/brute_force/brute_force_info.cpp new file mode 100644 index 000000000..f1d60e62b --- /dev/null +++ b/src/VecSim/algorithms/brute_force/brute_force_info.cpp @@ -0,0 +1,17 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + +#include "brute_force_info.h" +#include "VecSim/utils/vec_utils.h" +BruteForceInfo::BruteForceInfo(VecSimInfo *info) : VecSimInfo(*info) {} + +VecSimInfoIterator *BruteForceInfo::getIterator() { + VecSimInfoIterator *infoIterator = VecSimInfo::getIterator(); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::BLOCK_SIZE_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->blockSize}}}); +} diff --git a/src/VecSim/algorithms/brute_force/brute_force_info.h b/src/VecSim/algorithms/brute_force/brute_force_info.h new file mode 100644 index 000000000..1cd66b9df --- /dev/null +++ b/src/VecSim/algorithms/brute_force/brute_force_info.h @@ -0,0 +1,16 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + +#pragma once + +#include "VecSim/vec_sim_info.h" + +struct BruteForceInfo : public VecSimInfo { +public: + BruteForceInfo(VecSimInfo *info); + size_t blockSize; // Brute force algorithm vector block (mini matrix) size + virtual VecSimInfoIterator *getIterator() override; +}; diff --git a/src/VecSim/vec_sim_index.h b/src/VecSim/vec_sim_index.h index 607482c45..c09627162 100644 --- a/src/VecSim/vec_sim_index.h +++ b/src/VecSim/vec_sim_index.h @@ -13,6 +13,7 @@ #include "VecSim/utils/vec_utils.h" #include "VecSim/spaces/spaces.h" #include "info_iterator_struct.h" +#include "VecSim/vec_sim_info.h" #include using spaces::dist_func_t; @@ -83,6 +84,20 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { inline VecSimType getType() const { return vecType; } inline VecSimMetric getMetric() const { return metric; } + virtual VecSimInfo *getInfo() { + VecSimInfo *info = new VecSimInfo(); + info->algo = this->getAlgo(); + info->indexSize = this->size(); + info->indexLabelCount = this->labelCount(); + info->metric = this->getMetric(); + info->memory = this->getMemoryUsage(); + info->type = this->getType(); + info->isMulti = this->isMultiValue(); + info->dim = this->getDim(); + info->last_mode = this->last_mode; + return info; + } + void log(const char *fmt, ...) const { if (VecSimIndexInterface::logCallback) { // Format the message and call the callback diff --git a/src/VecSim/vec_sim_info.cpp b/src/VecSim/vec_sim_info.cpp index 6d73f6bba..15a65e968 100644 --- a/src/VecSim/vec_sim_info.cpp +++ b/src/VecSim/vec_sim_info.cpp @@ -40,4 +40,4 @@ VecSimInfoIterator *VecSimInfo::getIterator() { .fieldName = VecSimCommonStrings::SEARCH_MODE_STRING, .fieldType = INFOFIELD_STRING, .fieldValue = {FieldValue{.stringValue = VecSimSearchMode_ToString(this->last_mode)}}}); -} \ No newline at end of file +} diff --git a/src/VecSim/vec_sim_interface.h b/src/VecSim/vec_sim_interface.h index 6b4d28a97..4e070c21e 100644 --- a/src/VecSim/vec_sim_interface.h +++ b/src/VecSim/vec_sim_interface.h @@ -129,7 +129,7 @@ struct VecSimIndexInterface : public VecsimBaseObject { * * @return Index general and specific meta-data. */ - virtual VecSimIndexInfo info() const = 0; + virtual VecSimIndexInfo *info() const = 0; /** * @brief Returns an index information in an iterable structure. From 170f56675433c4b3ff67e7997b843076b312a862 Mon Sep 17 00:00:00 2001 From: DvirDukhan Date: Mon, 24 Apr 2023 19:56:16 +0300 Subject: [PATCH 4/4] wip --- src/VecSim/CMakeLists.txt | 1 + .../algorithms/brute_force/brute_force.h | 60 +++--------------- .../brute_force/brute_force_info.cpp | 17 ----- .../algorithms/brute_force/brute_force_info.h | 7 +-- src/VecSim/algorithms/brute_force/copilot.cpp | 0 src/VecSim/algorithms/hnsw/hnsw.h | 63 ------------------- src/VecSim/algorithms/hnsw/hnsw_info.cpp | 39 ++++++++++++ src/VecSim/algorithms/hnsw/hnsw_info.h | 22 +++++++ src/VecSim/algorithms/hnsw/hnsw_tiered.h | 8 ++- src/VecSim/index_factories/tiered_factory.cpp | 7 ++- src/VecSim/{ => info}/info_iterator_struct.h | 0 src/VecSim/{ => info}/vec_sim_info.cpp | 2 +- src/VecSim/{ => info}/vec_sim_info.h | 4 +- src/VecSim/info_iterator.cpp | 2 +- src/VecSim/info_iterator.h | 4 +- src/VecSim/tiered_index_info.cpp | 49 +++++++++++++++ src/VecSim/tiered_index_info.h | 19 ++++++ src/VecSim/tombstone_interface.h | 6 ++ src/VecSim/utils/vec_utils.cpp | 3 + src/VecSim/utils/vec_utils.h | 3 + src/VecSim/vec_sim_common.h | 39 ------------ src/VecSim/vec_sim_index.h | 27 ++++---- src/VecSim/vec_sim_tiered_index.h | 30 ++++++++- 23 files changed, 207 insertions(+), 205 deletions(-) delete mode 100644 src/VecSim/algorithms/brute_force/brute_force_info.cpp create mode 100644 src/VecSim/algorithms/brute_force/copilot.cpp create mode 100644 src/VecSim/algorithms/hnsw/hnsw_info.cpp create mode 100644 src/VecSim/algorithms/hnsw/hnsw_info.h rename src/VecSim/{ => info}/info_iterator_struct.h (100%) rename src/VecSim/{ => info}/vec_sim_info.cpp (97%) rename src/VecSim/{ => info}/vec_sim_info.h (93%) create mode 100644 src/VecSim/tiered_index_info.cpp create mode 100644 src/VecSim/tiered_index_info.h diff --git a/src/VecSim/CMakeLists.txt b/src/VecSim/CMakeLists.txt index 54986b9ff..1631d72d9 100644 --- a/src/VecSim/CMakeLists.txt +++ b/src/VecSim/CMakeLists.txt @@ -29,6 +29,7 @@ add_library(VectorSimilarity ${VECSIM_LIBTYPE} utils/vec_utils.cpp memory/vecsim_malloc.cpp memory/vecsim_base.cpp + info/vec_sim_info.cpp ${HEADER_LIST} ) diff --git a/src/VecSim/algorithms/brute_force/brute_force.h b/src/VecSim/algorithms/brute_force/brute_force.h index f8494aab9..38449d64a 100644 --- a/src/VecSim/algorithms/brute_force/brute_force.h +++ b/src/VecSim/algorithms/brute_force/brute_force.h @@ -49,7 +49,7 @@ class BruteForceIndex : public VecSimIndexAbstract { VecSimQueryParams *queryParams) override; VecSimQueryResult_List rangeQuery(const void *queryBlob, double radius, VecSimQueryParams *queryParams) override; - virtual VecSimInfo *info() const override; + virtual VecSimIndexInfo *info() const override; virtual VecSimInfoIterator *infoIterator() const override; virtual VecSimBatchIterator *newBatchIterator(const void *queryBlob, VecSimQueryParams *queryParams) const override; @@ -354,63 +354,17 @@ BruteForceIndex::rangeQuery(const void *queryBlob, double ra } template -VecSimInfo *BruteForceIndex::info() const { - VecSimInfo *info = VecSimIndexAbstract::info(); - BruteForceInfo *bfInfo = new BruteForceInfo(info); - bfInfo->blockSize = this->blockSize; - delete info; +VecSimIndexInfo *BruteForceIndex::info() const { + BruteForceInfo *bfInfo = new BruteForceInfo(); + this->fillIndexInfo(bfInfo); return bfInfo; } template VecSimInfoIterator *BruteForceIndex::infoIterator() const { - VecSimIndexInfo info = this->info(); - // For readability. Update this number when needed. - size_t numberOfInfoFields = 8; - VecSimInfoIterator *infoIterator = new VecSimInfoIterator(numberOfInfoFields); - - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::ALGORITHM_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(info.algo)}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::TYPE_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimType_ToString(info.bfInfo.type)}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::DIMENSION_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.dim}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::METRIC_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimMetric_ToString(info.bfInfo.metric)}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.isMulti}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.indexSize}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.indexLabelCount}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::BLOCK_SIZE_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.blockSize}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.bfInfo.memory}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::SEARCH_MODE_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{ - .stringValue = VecSimSearchMode_ToString(info.bfInfo.last_mode)}}}); - + VecSimIndexInfo *info = this->info(); + VecSimInfoIterator *infoIterator = info->getIterator(); + delete info; return infoIterator; } diff --git a/src/VecSim/algorithms/brute_force/brute_force_info.cpp b/src/VecSim/algorithms/brute_force/brute_force_info.cpp deleted file mode 100644 index f1d60e62b..000000000 --- a/src/VecSim/algorithms/brute_force/brute_force_info.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - *Copyright Redis Ltd. 2021 - present - *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or - *the Server Side Public License v1 (SSPLv1). - */ - -#include "brute_force_info.h" -#include "VecSim/utils/vec_utils.h" -BruteForceInfo::BruteForceInfo(VecSimInfo *info) : VecSimInfo(*info) {} - -VecSimInfoIterator *BruteForceInfo::getIterator() { - VecSimInfoIterator *infoIterator = VecSimInfo::getIterator(); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::BLOCK_SIZE_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = this->blockSize}}}); -} diff --git a/src/VecSim/algorithms/brute_force/brute_force_info.h b/src/VecSim/algorithms/brute_force/brute_force_info.h index 1cd66b9df..d885d610c 100644 --- a/src/VecSim/algorithms/brute_force/brute_force_info.h +++ b/src/VecSim/algorithms/brute_force/brute_force_info.h @@ -8,9 +8,4 @@ #include "VecSim/vec_sim_info.h" -struct BruteForceInfo : public VecSimInfo { -public: - BruteForceInfo(VecSimInfo *info); - size_t blockSize; // Brute force algorithm vector block (mini matrix) size - virtual VecSimInfoIterator *getIterator() override; -}; +struct BruteForceInfo : public VecSimIndexInfo {}; diff --git a/src/VecSim/algorithms/brute_force/copilot.cpp b/src/VecSim/algorithms/brute_force/copilot.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/src/VecSim/algorithms/hnsw/hnsw.h b/src/VecSim/algorithms/hnsw/hnsw.h index 14c4a292c..5d623c178 100644 --- a/src/VecSim/algorithms/hnsw/hnsw.h +++ b/src/VecSim/algorithms/hnsw/hnsw.h @@ -2194,69 +2194,6 @@ VecSimInfoIterator *HNSWIndex::infoIterator() const { size_t numberOfInfoFields = 12; VecSimInfoIterator *infoIterator = new VecSimInfoIterator(numberOfInfoFields); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::ALGORITHM_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(info.algo)}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::TYPE_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimType_ToString(info.hnswInfo.type)}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::DIMENSION_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.dim}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::METRIC_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{.stringValue = VecSimMetric_ToString(info.hnswInfo.metric)}}}); - - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.isMulti}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.indexSize}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.indexLabelCount}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_M_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.M}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::HNSW_EF_CONSTRUCTION_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efConstruction}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EF_RUNTIME_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efRuntime}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_MAX_LEVEL, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.max_level}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_ENTRYPOINT, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.entrypoint}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.memory}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::SEARCH_MODE_STRING, - .fieldType = INFOFIELD_STRING, - .fieldValue = {FieldValue{ - .stringValue = VecSimSearchMode_ToString(info.hnswInfo.last_mode)}}}); - infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EPSILON_STRING, - .fieldType = INFOFIELD_FLOAT64, - .fieldValue = {FieldValue{.floatingPointValue = info.hnswInfo.epsilon}}}); - return infoIterator; } diff --git a/src/VecSim/algorithms/hnsw/hnsw_info.cpp b/src/VecSim/algorithms/hnsw/hnsw_info.cpp new file mode 100644 index 000000000..273cf7d7e --- /dev/null +++ b/src/VecSim/algorithms/hnsw/hnsw_info.cpp @@ -0,0 +1,39 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + +#include "hnsw_info.h" +#include "VecSim/utils/vec_utils.h" + +VecSimInfoIterator *HNSWInfo::getIterator() { + VecSimInfoIterator *infoIterator = VecSimIndexInfo::getIterator(); + + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_M_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.M}}}); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::HNSW_EF_CONSTRUCTION_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efConstruction}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EF_RUNTIME_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efRuntime}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_MAX_LEVEL, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.max_level}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_ENTRYPOINT, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.entrypoint}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EPSILON_STRING, + .fieldType = INFOFIELD_FLOAT64, + .fieldValue = {FieldValue{.floatingPointValue = info.hnswInfo.epsilon}}}); + + return infoIterator; +} diff --git a/src/VecSim/algorithms/hnsw/hnsw_info.h b/src/VecSim/algorithms/hnsw/hnsw_info.h new file mode 100644 index 000000000..7f2685c6a --- /dev/null +++ b/src/VecSim/algorithms/hnsw/hnsw_info.h @@ -0,0 +1,22 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + +#pragma once + +#include "VecSim/vec_sim_info.h" + +struct HNSWInfo : public VecSimIndexInfo { +public: + size_t M; // Number of allowed edges per node in graph. + size_t efConstruction; // EF parameter for HNSW graph accuracy/latency for indexing. + size_t efRuntime; // EF parameter for HNSW graph accuracy/latency for search. + double epsilon; // Epsilon parameter for HNSW graph accuracy/latency for range search. + size_t max_level; // Number of graph levels. + size_t entrypoint; // Entrypoint vector label. + size_t visitedNodesPoolSize; // The max number of parallel graph scans so far. + + virtual VecSimInfoIterator *getIterator() override +}; diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index a9bf4014e..919260d4f 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -114,7 +114,8 @@ class TieredHNSWIndex : public VecSimTieredIndex { public: TieredHNSWIndex(HNSWIndex *hnsw_index, BruteForceIndex *bf_index, - const TieredIndexParams &tieredParams); + const TieredIndexParams &tieredParams, + std::shared_ptr allocator); virtual ~TieredHNSWIndex(); int addVector(const void *blob, labelType label, void *auxiliaryCtx = nullptr) override; @@ -462,8 +463,9 @@ void TieredHNSWIndex::executeRepairJob(HNSWRepairJob *job) { template TieredHNSWIndex::TieredHNSWIndex(HNSWIndex *hnsw_index, BruteForceIndex *bf_index, - const TieredIndexParams &tiered_index_params) - : VecSimTieredIndex(hnsw_index, bf_index, tiered_index_params), + const TieredIndexParams &tiered_index_params, + std::shared_ptr allocator) + : VecSimTieredIndex(hnsw_index, bf_index, tiered_index_params, allocator), labelToInsertJobs(this->allocator), idToRepairJobs(this->allocator), idToSwapJob(this->allocator) { // If the param for swapJobThreshold is 0 use the default value, if it exceeds the maximum diff --git a/src/VecSim/index_factories/tiered_factory.cpp b/src/VecSim/index_factories/tiered_factory.cpp index e2ede8843..c344f4efe 100644 --- a/src/VecSim/index_factories/tiered_factory.cpp +++ b/src/VecSim/index_factories/tiered_factory.cpp @@ -27,7 +27,9 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) { .multi = params->primaryIndexParams->hnswParams.multi, .blockSize = params->primaryIndexParams->hnswParams.blockSize}; - AbstractIndexInitParams abstractInitParams = {.allocator = hnsw_index->getAllocator(), + // Each part of the tiered index should have its own allocator. + std::shared_ptr flat_allocator = VecSimAllocator::newVecsimAllocator(); + AbstractIndexInitParams abstractInitParams = {.allocator = flat_allocator, .dim = bf_params.dim, .vecType = bf_params.type, .metric = bf_params.metric, @@ -38,8 +40,9 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) { BruteForceFactory::NewIndex(&bf_params, abstractInitParams)); // Create new tiered hnsw index + std::shared_ptr tiered_allocator = VecSimAllocator::newVecsimAllocator(); return new (hnsw_index->getAllocator()) - TieredHNSWIndex(hnsw_index, frontendIndex, *params); + TieredHNSWIndex(hnsw_index, frontendIndex, *params, tiered_allocator); } inline size_t EstimateInitialSize(const TieredIndexParams *params, BFParams &bf_params_output) { diff --git a/src/VecSim/info_iterator_struct.h b/src/VecSim/info/info_iterator_struct.h similarity index 100% rename from src/VecSim/info_iterator_struct.h rename to src/VecSim/info/info_iterator_struct.h diff --git a/src/VecSim/vec_sim_info.cpp b/src/VecSim/info/vec_sim_info.cpp similarity index 97% rename from src/VecSim/vec_sim_info.cpp rename to src/VecSim/info/vec_sim_info.cpp index 15a65e968..18c063537 100644 --- a/src/VecSim/vec_sim_info.cpp +++ b/src/VecSim/info/vec_sim_info.cpp @@ -1,7 +1,7 @@ #include "vec_sim_info.h" #include "utils/vec_utils.h" -VecSimInfoIterator *VecSimInfo::getIterator() { +VecSimInfoIterator *VecSimIndexInfo::getIterator() { VecSimInfoIterator *infoIterator = new VecSimInfoIterator(7); infoIterator->addInfoField(VecSim_InfoField{ diff --git a/src/VecSim/vec_sim_info.h b/src/VecSim/info/vec_sim_info.h similarity index 93% rename from src/VecSim/vec_sim_info.h rename to src/VecSim/info/vec_sim_info.h index 2d1928584..f6633bdfd 100644 --- a/src/VecSim/vec_sim_info.h +++ b/src/VecSim/info/vec_sim_info.h @@ -8,7 +8,7 @@ * @brief Struct that holds information about the index. * */ -struct VecSimInfo { +struct VecSimIndexInfo { public: VecSimAlgo algo; // Index algorithm. @@ -22,5 +22,5 @@ struct VecSimInfo { VecSearchMode last_mode; // The mode in which the last query ran. virtual VecSimInfoIterator *getIterator(); - virtual ~VecSimInfo() {} + virtual ~VecSimIndexInfo() {} }; diff --git a/src/VecSim/info_iterator.cpp b/src/VecSim/info_iterator.cpp index 7099db854..b68e95606 100644 --- a/src/VecSim/info_iterator.cpp +++ b/src/VecSim/info_iterator.cpp @@ -4,7 +4,7 @@ *the Server Side Public License v1 (SSPLv1). */ -#include "info_iterator_struct.h" +#include "VecSim/info/info_iterator_struct.h" extern "C" size_t VecSimInfoIterator_NumberOfFields(VecSimInfoIterator *infoIterator) { return infoIterator->numberOfFields(); diff --git a/src/VecSim/info_iterator.h b/src/VecSim/info_iterator.h index 2cc3d79ad..a7cf1a1e5 100644 --- a/src/VecSim/info_iterator.h +++ b/src/VecSim/info_iterator.h @@ -21,7 +21,8 @@ typedef enum { INFOFIELD_STRING, INFOFIELD_INT64, INFOFIELD_UINT64, - INFOFIELD_FLOAT64 + INFOFIELD_FLOAT64, + INFOFIELD_ITERATOR } VecSim_InfoFieldType; typedef union { @@ -29,6 +30,7 @@ typedef union { int64_t integerValue; // Integer value. Signed 64 bits integer. u_int64_t uintegerValue; // Unsigned value. Unsigned 64 buts integer. const char *stringValue; // String value. + VecSimInfoIterator *iteratorValue; // Iterator value. } FieldValue; /** diff --git a/src/VecSim/tiered_index_info.cpp b/src/VecSim/tiered_index_info.cpp new file mode 100644 index 000000000..8e905b043 --- /dev/null +++ b/src/VecSim/tiered_index_info.cpp @@ -0,0 +1,49 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + +#include "tiered_index_info.h" +#include "VecSim/utils/vec_utils.h" + +virtual VecSimInfoIterator *TieredIndexInfo::getIterator() { + VecSimInfoIterator *iterator = VecSimInfoIterator_New(9); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::ALGORITHM_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(this->algo)}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->isMulti}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->indexSize}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->indexLabelCount}}}); + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = this->memory}}}); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::SEARCH_MODE_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = {FieldValue{.stringValue = VecSimSearchMode_ToString(this->last_mode)}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::FRONTEND_INDEX_STRING, + .fieldType = INFOFIELD_ITERATOR, + .fieldValue = {FieldValue{.iteratorValue = this->frontendIndexInfo->getIterator()}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::BACKEND_INDEX_STRING, + .fieldType = INFOFIELD_ITERATOR, + .fieldValue = {FieldValue{.iteratorValue = this->backendIndexInfo->getIterator()}}}); + + return infoIterator; + +} \ No newline at end of file diff --git a/src/VecSim/tiered_index_info.h b/src/VecSim/tiered_index_info.h new file mode 100644 index 000000000..664ef21c6 --- /dev/null +++ b/src/VecSim/tiered_index_info.h @@ -0,0 +1,19 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + +#pragma once + +#include "VecSim/info/vec_sim_info.h" + +struct TieredIndexInfo : public VecSimIndexInfo { +public: + VecSimIndexInfo *backendIndexInfo; + VecSimIndexInfo *frontendIndexInfo; + + size_t management_layer_memory; + + virtual VecSimInfoIterator *getIterator(); +}; diff --git a/src/VecSim/tombstone_interface.h b/src/VecSim/tombstone_interface.h index e4e114e66..1df3eea71 100644 --- a/src/VecSim/tombstone_interface.h +++ b/src/VecSim/tombstone_interface.h @@ -1,3 +1,9 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + #pragma once #include #include "vec_sim_common.h" diff --git a/src/VecSim/utils/vec_utils.cpp b/src/VecSim/utils/vec_utils.cpp index 6ab10cc7d..b2cba43f4 100644 --- a/src/VecSim/utils/vec_utils.cpp +++ b/src/VecSim/utils/vec_utils.cpp @@ -50,6 +50,9 @@ const char *VecSimCommonStrings::SEARCH_MODE_STRING = "LAST_SEARCH_MODE"; const char *VecSimCommonStrings::HYBRID_POLICY_STRING = "HYBRID_POLICY"; const char *VecSimCommonStrings::BATCH_SIZE_STRING = "BATCH_SIZE"; +const char *VecSimCommonStrings::FRONTEND_INDEX_STRING = "FRONTEND_INDEX"; +const char *VecSimCommonStrings::BACKEND_INDEX_STRING = "BACKEND_INDEX"; + void sort_results_by_id(VecSimQueryResult_List rl) { qsort(rl.results, VecSimQueryResult_Len(rl), sizeof(VecSimQueryResult), (__compar_fn_t)cmpVecSimQueryResultById); diff --git a/src/VecSim/utils/vec_utils.h b/src/VecSim/utils/vec_utils.h index 49aa2b6d8..1002716da 100644 --- a/src/VecSim/utils/vec_utils.h +++ b/src/VecSim/utils/vec_utils.h @@ -56,6 +56,9 @@ struct VecSimCommonStrings { static const char *SEARCH_MODE_STRING; static const char *HYBRID_POLICY_STRING; static const char *BATCH_SIZE_STRING; + + static const char *FRONTEND_INDEX_STRING; + static const char *BACKEND_INDEX_STRING; }; inline int cmpVecSimQueryResultById(const VecSimQueryResult *res1, const VecSimQueryResult *res2) { diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 551e0d5f2..e61ff469d 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -189,45 +189,6 @@ typedef struct { // to get it from the parameters resolve function. } VecSimQueryParams; -/** - * @brief Index information. Mainly used for debug/testing. - * - */ -typedef struct { - union { - struct { - size_t indexSize; // Current count of vectors. - size_t indexLabelCount; // Current unique count of labels. - size_t blockSize; // Sets the amount to grow when resizing - size_t M; // Number of allowed edges per node in graph. - size_t efConstruction; // EF parameter for HNSW graph accuracy/latency for indexing. - size_t efRuntime; // EF parameter for HNSW graph accuracy/latency for search. - double epsilon; // Epsilon parameter for HNSW graph accuracy/latency for range search. - size_t max_level; // Number of graph levels. - size_t entrypoint; // Entrypoint vector label. - VecSimMetric metric; // Index distance metric - uint64_t memory; // Index memory consumption. - VecSimType type; // Datatype the index holds. - bool isMulti; // Determines if the index should multi-index or not. - size_t dim; // Vector size (dimension). - VecSearchMode last_mode; // The mode in which the last query ran. - size_t visitedNodesPoolSize; // The max number of parallel graph scans so far. - } hnswInfo; - struct { - size_t indexSize; // Current count of vectors. - size_t indexLabelCount; // Current unique count of labels. - size_t blockSize; // Brute force algorithm vector block (mini matrix) size - VecSimMetric metric; // Index distance metric - uint64_t memory; // Index memory consumption. - VecSimType type; // Datatype the index holds. - bool isMulti; // Determines if the index should multi-index or not. - size_t dim; // Vector size (dimension). - VecSearchMode last_mode; // The mode in which the last query ran. - } bfInfo; - }; - VecSimAlgo algo; // Algorithm being used. -} VecSimIndexInfo; - // Memory function declarations. typedef void *(*allocFn)(size_t n); typedef void *(*callocFn)(size_t nelem, size_t elemsz); diff --git a/src/VecSim/vec_sim_index.h b/src/VecSim/vec_sim_index.h index c09627162..bea7d6ebd 100644 --- a/src/VecSim/vec_sim_index.h +++ b/src/VecSim/vec_sim_index.h @@ -57,6 +57,19 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { bool isMulti; // Determines if the index should multi-index or not. void *logCallbackCtx; // Context for the log callback. + void fillIndexInfo getInfo(VecSimIndexInfo *info) { + info->algo = this->getAlgo(); + info->indexSize = this->size(); + info->indexLabelCount = this->labelCount(); + info->metric = this->getMetric(); + info->memory = this->getMemoryUsage(); + info->type = this->getType(); + info->isMulti = this->isMultiValue(); + info->dim = this->getDim(); + info->last_mode = this->last_mode; + info->blockSize = this->blockSize; + } + public: /** * @brief Construct a new Vec Sim Index object @@ -84,20 +97,6 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { inline VecSimType getType() const { return vecType; } inline VecSimMetric getMetric() const { return metric; } - virtual VecSimInfo *getInfo() { - VecSimInfo *info = new VecSimInfo(); - info->algo = this->getAlgo(); - info->indexSize = this->size(); - info->indexLabelCount = this->labelCount(); - info->metric = this->getMetric(); - info->memory = this->getMemoryUsage(); - info->type = this->getType(); - info->isMulti = this->isMultiValue(); - info->dim = this->getDim(); - info->last_mode = this->last_mode; - return info; - } - void log(const char *fmt, ...) const { if (VecSimIndexInterface::logCallback) { // Format the message and call the callback diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index e549b9cb1..3225f92cd 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -1,8 +1,14 @@ +/* + *Copyright Redis Ltd. 2021 - present + *Licensed under your choice of the Redis Source Available License 2.0 (RSALv2) or + *the Server Side Public License v1 (SSPLv1). + */ + #pragma once #include "vec_sim_index.h" #include "algorithms/brute_force/brute_force.h" - +#include "tiered_index_info.h" // TODO: Consider moving tiered index files from root to their own folder. #include /** @@ -44,8 +50,8 @@ class VecSimTieredIndex : public VecSimIndexInterface { public: VecSimTieredIndex(VecSimIndexAbstract *backendIndex_, BruteForceIndex *frontendIndex_, - TieredIndexParams tieredParams) - : VecSimIndexInterface(backendIndex_->getAllocator()), backendIndex(backendIndex_), + TieredIndexParams tieredParams, std::shared_ptr allocator) + : VecSimIndexInterface(allocator), backendIndex(backendIndex_), frontendIndex(frontendIndex_), jobQueue(tieredParams.jobQueue), jobQueueCtx(tieredParams.jobQueueCtx), SubmitJobsToQueue(tieredParams.submitCb), memoryCtx(tieredParams.memoryCtx), UpdateIndexMemory(tieredParams.UpdateMemCb) {} @@ -57,6 +63,8 @@ class VecSimTieredIndex : public VecSimIndexInterface { VecSimQueryResult_List topKQuery(const void *queryBlob, size_t k, VecSimQueryParams *queryParams) override; + + virtual VecSimIndexInfo *info() override; }; template @@ -109,3 +117,19 @@ VecSimTieredIndex::topKQuery(const void *queryBlob, size_t k } } } + +virtual VecSimIndexInfo *VecSimTieredIndex::info() { + TieredIndexInfo *info = new TieredIndexInfo(); + info->backendIndexInfo = this->backendIndex->info(); + info->frontendIndexInfo = this->frontendIndex->info(); + info->management_layer_memory = this->getAllocator()->getAllocatedMemory(); + info->memory = info->management_layer_memory + info->backendIndexInfo->memory + + info->frontendIndexInfo->memory; + info->indexType = VecSimAlgo_TIERED; + info->indexSize = this->indexSize(); + info->indexLabelCount = this->indexLabelCount(); + info->last_mode = this->last_mode; + info->isMultiValue = this->isMultiValue(); + + return info; +}