From fd52bad8d0ce8ecfb8b8444ba7473275d268671e Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 26 Apr 2022 12:48:26 -0300 Subject: [PATCH 01/10] wip --- CMakeLists.txt | 2 +- clickhouse/CMakeLists.txt | 1 + clickhouse/columns/array.cpp | 29 ++++++++++--- clickhouse/columns/array.h | 11 ++++- clickhouse/columns/column.cpp | 38 +++++++++++++++++ clickhouse/columns/column.h | 17 +++++++- clickhouse/columns/date.cpp | 18 ++++----- clickhouse/columns/date.h | 12 +++--- clickhouse/columns/decimal.cpp | 6 +-- clickhouse/columns/decimal.h | 4 +- clickhouse/columns/enum.cpp | 4 +- clickhouse/columns/enum.h | 6 +-- clickhouse/columns/ip4.cpp | 6 +-- clickhouse/columns/ip4.h | 4 +- clickhouse/columns/ip6.cpp | 8 ++-- clickhouse/columns/ip6.h | 4 +- clickhouse/columns/lowcardinality.cpp | 47 ++++++++++++---------- clickhouse/columns/lowcardinality.h | 9 ++++- clickhouse/columns/lowcardinalityadaptor.h | 11 ++++- clickhouse/columns/nothing.h | 16 +++++++- clickhouse/columns/nullable.cpp | 24 ++++++++--- clickhouse/columns/nullable.h | 10 ++++- clickhouse/columns/numeric.cpp | 4 +- clickhouse/columns/numeric.h | 4 +- clickhouse/columns/string.cpp | 8 ++-- clickhouse/columns/string.h | 8 ++-- clickhouse/columns/tuple.cpp | 30 ++++++++++++-- clickhouse/columns/tuple.h | 10 ++++- clickhouse/columns/uuid.cpp | 6 +-- clickhouse/columns/uuid.h | 4 +- ut/CMakeLists.txt | 2 +- 31 files changed, 256 insertions(+), 107 deletions(-) create mode 100644 clickhouse/columns/column.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 31cd8a5a..2baba458 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ INCLUDE (cmake/subdirs.cmake) INCLUDE (cmake/openssl.cmake) OPTION (BUILD_BENCHMARK "Build benchmark" OFF) -OPTION (BUILD_TESTS "Build tests" OFF) +OPTION (BUILD_TESTS "Build tests" ON) OPTION (WITH_OPENSSL "Use OpenSSL for TLS connections" OFF) PROJECT (CLICKHOUSE-CLIENT) diff --git a/clickhouse/CMakeLists.txt b/clickhouse/CMakeLists.txt index e2afb3ae..6a851241 100644 --- a/clickhouse/CMakeLists.txt +++ b/clickhouse/CMakeLists.txt @@ -7,6 +7,7 @@ SET ( clickhouse-cpp-lib-src base/wire_format.cpp columns/array.cpp + columns/column.cpp columns/date.cpp columns/decimal.cpp columns/enum.cpp diff --git a/clickhouse/columns/array.cpp b/clickhouse/columns/array.cpp index c71684d1..049056ac 100644 --- a/clickhouse/columns/array.cpp +++ b/clickhouse/columns/array.cpp @@ -54,22 +54,38 @@ void ColumnArray::Append(ColumnRef column) { } } -bool ColumnArray::Load(InputStream* input, size_t rows) { +bool ColumnArray::LoadPrefix(InputStream* input, size_t rows) { if (!rows) { return true; } - if (!offsets_->Load(input, rows)) { + + return data_->LoadPrefix(input, rows); +} + +bool ColumnArray::LoadBody(InputStream* input, size_t rows) { + if (!rows) { + return true; + } + if (!offsets_->LoadBody(input, rows)) { return false; } - if (!data_->Load(input, (*offsets_)[rows - 1])) { + if (!data_->LoadBody(input, (*offsets_)[rows - 1])) { return false; } return true; } -void ColumnArray::Save(OutputStream* output) { - offsets_->Save(output); - data_->Save(output); +void ColumnArray::SavePrefix(OutputStream* output) { + data_->SavePrefix(output); +} + +void ColumnArray::SaveBody(OutputStream* output) { + offsets_->SaveBody(output); + data_->SaveBody(output); +} + +void ColumnArray::SaveSuffix(OutputStream* output) { + data_->SaveSuffix(output); } void ColumnArray::Clear() { @@ -92,6 +108,7 @@ void ColumnArray::OffsetsIncrease(size_t n) { } size_t ColumnArray::GetOffset(size_t n) const { + return (n == 0) ? 0 : (*offsets_)[n - 1]; } diff --git a/clickhouse/columns/array.h b/clickhouse/columns/array.h index e96e70c4..9c9ed502 100644 --- a/clickhouse/columns/array.h +++ b/clickhouse/columns/array.h @@ -24,11 +24,18 @@ class ColumnArray : public Column { /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; + bool LoadPrefix(InputStream* input, size_t rows) override; + /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; + + /// Saves column prefix to output stream. + void SavePrefix(OutputStream* output) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; + + void SaveSuffix(OutputStream* output) override; /// Clear column data . void Clear() override; diff --git a/clickhouse/columns/column.cpp b/clickhouse/columns/column.cpp new file mode 100644 index 00000000..b3d4d15c --- /dev/null +++ b/clickhouse/columns/column.cpp @@ -0,0 +1,38 @@ +#include "column.h" + +namespace clickhouse { + +bool Column::LoadPrefix([[maybe_unused]] InputStream* input, [[maybe_unused]] size_t rows) { + /// does nothing by default + return true; +} + +bool Column::LoadBody([[maybe_unused]] InputStream* input, [[maybe_unused]] size_t rows) { + /// does nothing by default + return true; +} + +bool Column::Load(InputStream* input, size_t rows) { + return LoadPrefix(input, rows) && LoadBody(input, rows); +} + +void Column::SavePrefix([[maybe_unused]] OutputStream* output) { + /// does nothing by default +} + +void Column::SaveBody([[maybe_unused]] OutputStream* output) { + /// does nothing by default +} + +void Column::SaveSuffix([[maybe_unused]] OutputStream* output) { + /// does nothing by default +} + +/// Saves column data to output stream. +void Column::Save(OutputStream* output) { + SavePrefix(output); + SaveBody(output); + SaveSuffix(output); +} + +} \ No newline at end of file diff --git a/clickhouse/columns/column.h b/clickhouse/columns/column.h index 6db5b39a..2a2a61ad 100644 --- a/clickhouse/columns/column.h +++ b/clickhouse/columns/column.h @@ -43,10 +43,23 @@ class Column : public std::enable_shared_from_this { virtual void Append(ColumnRef column) = 0; /// Loads column data from input stream. - virtual bool Load(InputStream* input, size_t rows) = 0; + bool Load(InputStream* input, size_t rows); + + virtual bool LoadPrefix(InputStream* input, size_t rows); + + virtual bool LoadBody(InputStream* input, size_t rows); /// Saves column data to output stream. - virtual void Save(OutputStream* output) = 0; + virtual void Save(OutputStream* output); + + /// Saves column prefix to output stream. Column types with prefixes must implement it + virtual void SavePrefix(OutputStream* output); + + /// Saves column body to output stream. + virtual void SaveBody(OutputStream* output); + + /// Saves column suffix to output stream. Column types with suffixes must implement it + virtual void SaveSuffix(OutputStream* output); /// Clear column data . virtual void Clear() = 0; diff --git a/clickhouse/columns/date.cpp b/clickhouse/columns/date.cpp index 78a86be1..6284cc5d 100644 --- a/clickhouse/columns/date.cpp +++ b/clickhouse/columns/date.cpp @@ -27,12 +27,12 @@ void ColumnDate::Append(ColumnRef column) { } } -bool ColumnDate::Load(InputStream* input, size_t rows) { +bool ColumnDate::LoadBody(InputStream* input, size_t rows) { return data_->Load(input, rows); } -void ColumnDate::Save(OutputStream* output) { - data_->Save(output); +void ColumnDate::SaveBody(OutputStream* output) { + data_->SaveBody(output); } size_t ColumnDate::Size() const { @@ -89,12 +89,12 @@ void ColumnDateTime::Append(ColumnRef column) { } } -bool ColumnDateTime::Load(InputStream* input, size_t rows) { +bool ColumnDateTime::LoadBody(InputStream* input, size_t rows) { return data_->Load(input, rows); } -void ColumnDateTime::Save(OutputStream* output) { - data_->Save(output); +void ColumnDateTime::SaveBody(OutputStream* output) { + data_->SaveBody(output); } size_t ColumnDateTime::Size() const { @@ -162,12 +162,12 @@ void ColumnDateTime64::Append(ColumnRef column) { } } -bool ColumnDateTime64::Load(InputStream* input, size_t rows) { +bool ColumnDateTime64::LoadBody(InputStream* input, size_t rows) { return data_->Load(input, rows); } -void ColumnDateTime64::Save(OutputStream* output) { - data_->Save(output); +void ColumnDateTime64::SaveBody(OutputStream* output) { + data_->SaveBody(output); } void ColumnDateTime64::Clear() { diff --git a/clickhouse/columns/date.h b/clickhouse/columns/date.h index 62ca4e05..d7841489 100644 --- a/clickhouse/columns/date.h +++ b/clickhouse/columns/date.h @@ -26,10 +26,10 @@ class ColumnDate : public Column { void Append(ColumnRef column) override; /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; /// Clear column data . void Clear() override; @@ -70,13 +70,13 @@ class ColumnDateTime : public Column { void Append(ColumnRef column) override; /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; /// Clear column data . void Clear() override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; /// Returns count of rows in the column. size_t Size() const override; @@ -118,13 +118,13 @@ class ColumnDateTime64 : public Column { void Append(ColumnRef column) override; /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; /// Clear column data . void Clear() override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; /// Returns count of rows in the column. size_t Size() const override; diff --git a/clickhouse/columns/decimal.cpp b/clickhouse/columns/decimal.cpp index 8a4186a0..7c3dcf68 100644 --- a/clickhouse/columns/decimal.cpp +++ b/clickhouse/columns/decimal.cpp @@ -197,12 +197,12 @@ void ColumnDecimal::Append(ColumnRef column) { } } -bool ColumnDecimal::Load(InputStream * input, size_t rows) { +bool ColumnDecimal::LoadBody(InputStream * input, size_t rows) { return data_->Load(input, rows); } -void ColumnDecimal::Save(OutputStream* output) { - data_->Save(output); +void ColumnDecimal::SaveBody(OutputStream* output) { + data_->SaveBody(output); } void ColumnDecimal::Clear() { diff --git a/clickhouse/columns/decimal.h b/clickhouse/columns/decimal.h index b28699ae..825cf5e6 100644 --- a/clickhouse/columns/decimal.h +++ b/clickhouse/columns/decimal.h @@ -21,8 +21,8 @@ class ColumnDecimal : public Column { public: void Append(ColumnRef column) override; - bool Load(InputStream* input, size_t rows) override; - void Save(OutputStream* output) override; + bool LoadBody(InputStream* input, size_t rows) override; + void SaveBody(OutputStream* output) override; void Clear() override; size_t Size() const override; ColumnRef Slice(size_t begin, size_t len) const override; diff --git a/clickhouse/columns/enum.cpp b/clickhouse/columns/enum.cpp index fc07e629..1192e479 100644 --- a/clickhouse/columns/enum.cpp +++ b/clickhouse/columns/enum.cpp @@ -74,13 +74,13 @@ void ColumnEnum::Append(ColumnRef column) { } template -bool ColumnEnum::Load(InputStream* input, size_t rows) { +bool ColumnEnum::LoadBody(InputStream* input, size_t rows) { data_.resize(rows); return WireFormat::ReadBytes(*input, data_.data(), data_.size() * sizeof(T)); } template -void ColumnEnum::Save(OutputStream* output) { +void ColumnEnum::SaveBody(OutputStream* output) { WireFormat::WriteBytes(*output, data_.data(), data_.size() * sizeof(T)); } diff --git a/clickhouse/columns/enum.h b/clickhouse/columns/enum.h index 34c672f6..892a818a 100644 --- a/clickhouse/columns/enum.h +++ b/clickhouse/columns/enum.h @@ -33,11 +33,11 @@ class ColumnEnum : public Column { void Append(ColumnRef column) override; /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; - + void SaveBody(OutputStream* output) override; + /// Clear column data . void Clear() override; diff --git a/clickhouse/columns/ip4.cpp b/clickhouse/columns/ip4.cpp index b2327e86..b9160bfb 100644 --- a/clickhouse/columns/ip4.cpp +++ b/clickhouse/columns/ip4.cpp @@ -71,12 +71,12 @@ void ColumnIPv4::Append(ColumnRef column) { } } -bool ColumnIPv4::Load(InputStream * input, size_t rows) { +bool ColumnIPv4::LoadBody(InputStream * input, size_t rows) { return data_->Load(input, rows); } -void ColumnIPv4::Save(OutputStream* output) { - data_->Save(output); +void ColumnIPv4::SaveBody(OutputStream* output) { + data_->SaveBody(output); } size_t ColumnIPv4::Size() const { diff --git a/clickhouse/columns/ip4.h b/clickhouse/columns/ip4.h index 0b1d41ab..713e9954 100644 --- a/clickhouse/columns/ip4.h +++ b/clickhouse/columns/ip4.h @@ -41,10 +41,10 @@ class ColumnIPv4 : public Column { void Append(ColumnRef column) override; /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; /// Clear column data . void Clear() override; diff --git a/clickhouse/columns/ip6.cpp b/clickhouse/columns/ip6.cpp index 1d9f14db..e92b23db 100644 --- a/clickhouse/columns/ip6.cpp +++ b/clickhouse/columns/ip6.cpp @@ -71,12 +71,12 @@ void ColumnIPv6::Append(ColumnRef column) { } } -bool ColumnIPv6::Load(InputStream* input, size_t rows) { - return data_->Load(input, rows); +bool ColumnIPv6::LoadBody(InputStream* input, size_t rows) { + return data_->LoadBody(input, rows); } -void ColumnIPv6::Save(OutputStream* output) { - data_->Save(output); +void ColumnIPv6::SaveBody(OutputStream* output) { + data_->SaveBody(output); } size_t ColumnIPv6::Size() const { diff --git a/clickhouse/columns/ip6.h b/clickhouse/columns/ip6.h index e523912a..e654dff3 100644 --- a/clickhouse/columns/ip6.h +++ b/clickhouse/columns/ip6.h @@ -39,10 +39,10 @@ class ColumnIPv6 : public Column { void Append(ColumnRef column) override; /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; /// Clear column data . void Clear() override; diff --git a/clickhouse/columns/lowcardinality.cpp b/clickhouse/columns/lowcardinality.cpp index 694e3eda..dfc52845 100644 --- a/clickhouse/columns/lowcardinality.cpp +++ b/clickhouse/columns/lowcardinality.cpp @@ -200,15 +200,6 @@ auto Load(ColumnRef new_dictionary_column, InputStream& input, size_t rows) { // As for now those features are not used in client-server protocol and minimal implementation suffices, // however some day they may. - // prefix - uint64_t key_version; - if (!WireFormat::ReadFixed(input, &key_version)) - throw ProtocolError("Failed to read key serialization version."); - - if (key_version != KeySerializationVersion::SharedDictionariesWithAdditionalKeys) - throw ProtocolError("Invalid key serialization version value."); - - // body uint64_t index_serialization_type; if (!WireFormat::ReadFixed(input, &index_serialization_type)) throw ProtocolError("Failed to read index serializaton type."); @@ -224,8 +215,8 @@ auto Load(ColumnRef new_dictionary_column, InputStream& input, size_t rows) { if (!WireFormat::ReadFixed(input, &number_of_keys)) throw ProtocolError("Failed to read number of rows in dictionary column."); - if (!new_dictionary_column->Load(&input, number_of_keys)) - throw ProtocolError("Failed to read values of dictionary column."); + if (!new_dictionary_column->LoadBody(&input, number_of_keys)) + throw std::runtime_error("Failed to read values of dictionary column."); uint64_t number_of_rows; if (!WireFormat::ReadFixed(input, &number_of_rows)) @@ -234,7 +225,7 @@ auto Load(ColumnRef new_dictionary_column, InputStream& input, size_t rows) { if (number_of_rows != rows) throw AssertionError("LowCardinality column must be read in full."); - new_index_column->Load(&input, number_of_rows); + new_index_column->LoadBody(&input, number_of_rows); ColumnLowCardinality::UniqueItems new_unique_items_map; for (size_t i = 0; i < new_dictionary_column->Size(); ++i) { @@ -250,7 +241,22 @@ auto Load(ColumnRef new_dictionary_column, InputStream& input, size_t rows) { } -bool ColumnLowCardinality::Load(InputStream* input, size_t rows) { +bool ColumnLowCardinality::LoadPrefix(InputStream* input, [[maybe_unused]] size_t rows) { + uint64_t key_version; + + if (!WireFormat::ReadFixed(*input, &key_version)) + throw std::runtime_error("Failed to read key serialization version."); + + if (key_version != KeySerializationVersion::SharedDictionariesWithAdditionalKeys) + throw std::runtime_error("Invalid key serialization version value."); + + + // does nothing with key_version, only check it? + + return true; +} + +bool ColumnLowCardinality::LoadBody(InputStream* input, size_t rows) { try { auto [new_dictionary, new_index, new_unique_items_map] = ::Load(dictionary_column_->Slice(0, 0), *input, rows); @@ -264,25 +270,22 @@ bool ColumnLowCardinality::Load(InputStream* input, size_t rows) { } } -void ColumnLowCardinality::Save(OutputStream* output) { - // prefix - const uint64_t version = static_cast(KeySerializationVersion::SharedDictionariesWithAdditionalKeys); +void ColumnLowCardinality::SavePrefix(OutputStream* output) { + const auto version = static_cast(KeySerializationVersion::SharedDictionariesWithAdditionalKeys); WireFormat::WriteFixed(*output, version); +} - // body +void ColumnLowCardinality::SaveBody(OutputStream* output) { const uint64_t index_serialization_type = indexTypeFromIndexColumn(*index_column_) | IndexFlag::HasAdditionalKeysBit; WireFormat::WriteFixed(*output, index_serialization_type); const uint64_t number_of_keys = dictionary_column_->Size(); WireFormat::WriteFixed(*output, number_of_keys); - dictionary_column_->Save(output); + dictionary_column_->SaveBody(output); const uint64_t number_of_rows = index_column_->Size(); WireFormat::WriteFixed(*output, number_of_rows); - index_column_->Save(output); - - // suffix - // NOP + index_column_->SaveBody(output); } void ColumnLowCardinality::Clear() { diff --git a/clickhouse/columns/lowcardinality.h b/clickhouse/columns/lowcardinality.h index 6d834428..7b0944af 100644 --- a/clickhouse/columns/lowcardinality.h +++ b/clickhouse/columns/lowcardinality.h @@ -54,11 +54,16 @@ class ColumnLowCardinality : public Column { /// Appends another LowCardinality column to the end of this one, updating dictionary. void Append(ColumnRef /*column*/) override; + bool LoadPrefix(InputStream* input, size_t rows) override; + /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; + + /// Saves column prefix to output stream. + void SavePrefix(OutputStream* output) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; /// Clear column data. void Clear() override; diff --git a/clickhouse/columns/lowcardinalityadaptor.h b/clickhouse/columns/lowcardinalityadaptor.h index 70261a51..6d76dd4b 100644 --- a/clickhouse/columns/lowcardinalityadaptor.h +++ b/clickhouse/columns/lowcardinalityadaptor.h @@ -27,12 +27,19 @@ class LowCardinalitySerializationAdaptor : public AdaptedColumnType public: using AdaptedColumnType::AdaptedColumnType; + bool LoadPrefix(InputStream* input, size_t rows) override { + auto new_data_column = this->Slice(0, 0)->template As(); + ColumnLowCardinalityT low_cardinality_col(new_data_column); + + return low_cardinality_col.LoadPrefix(input, rows); + } + /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override { + bool LoadBody(InputStream* input, size_t rows) override { auto new_data_column = this->Slice(0, 0)->template As(); ColumnLowCardinalityT low_cardinality_col(new_data_column); - if (!low_cardinality_col.Load(input, rows)) + if (!low_cardinality_col.LoadBody(input, rows)) return false; // It safe to reuse `flat_data_column` later since ColumnLowCardinalityT makes a deep copy, but still check just in case. diff --git a/clickhouse/columns/nothing.h b/clickhouse/columns/nothing.h index f3a1e5e2..c08b490d 100644 --- a/clickhouse/columns/nothing.h +++ b/clickhouse/columns/nothing.h @@ -51,16 +51,28 @@ class ColumnNothing : public Column { } /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override { + bool LoadBody(InputStream* input, size_t rows) override { input->Skip(rows); size_ += rows; return true; } + /// Saves column prefix to output stream. + void SavePrefix([[maybe_unused]] OutputStream* output) override { + throw std::runtime_error("method Save is not supported for Nothing column"); + } /// Saves column data to output stream. void Save(OutputStream*) override { throw UnimplementedError("method Save is not supported for Nothing column"); - } + } + + void SaveBody([[maybe_unused]] OutputStream*) override { + throw std::runtime_error("method SaveBody is not supported for Nothing column"); + } + + void SaveSuffix([[maybe_unused]] OutputStream* output) override { + throw std::runtime_error("method SaveSuffix is not supported for Nothing column"); + } /// Clear column data . void Clear() override { size_ = 0; } diff --git a/clickhouse/columns/nullable.cpp b/clickhouse/columns/nullable.cpp index 65336b31..357409e6 100644 --- a/clickhouse/columns/nullable.cpp +++ b/clickhouse/columns/nullable.cpp @@ -50,19 +50,31 @@ void ColumnNullable::Clear() { nulls_->Clear(); } -bool ColumnNullable::Load(InputStream* input, size_t rows) { - if (!nulls_->Load(input, rows)) { +bool ColumnNullable::LoadPrefix(InputStream* input, size_t rows) { + return nested_->LoadPrefix(input, rows); +} + +bool ColumnNullable::LoadBody(InputStream* input, size_t rows) { + if (!nulls_->LoadBody(input, rows)) { return false; } - if (!nested_->Load(input, rows)) { + if (!nested_->LoadBody(input, rows)) { return false; } return true; } -void ColumnNullable::Save(OutputStream* output) { - nulls_->Save(output); - nested_->Save(output); +void ColumnNullable::SavePrefix(OutputStream* output) { + nested_->SavePrefix(output); +} + +void ColumnNullable::SaveBody(OutputStream* output) { + nulls_->SaveBody(output); + nested_->SaveBody(output); +} + +void ColumnNullable::SaveSuffix(OutputStream* output) { + nested_->SaveSuffix(output); } size_t ColumnNullable::Size() const { diff --git a/clickhouse/columns/nullable.h b/clickhouse/columns/nullable.h index 8cde2781..64ddb3e9 100644 --- a/clickhouse/columns/nullable.h +++ b/clickhouse/columns/nullable.h @@ -28,11 +28,17 @@ class ColumnNullable : public Column { /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; + bool LoadPrefix(InputStream* input, size_t rows) override; + /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; + + void SavePrefix(OutputStream* output) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; + + void SaveSuffix(OutputStream* output) override; /// Clear column data . void Clear() override; diff --git a/clickhouse/columns/numeric.cpp b/clickhouse/columns/numeric.cpp index 479d1e79..d1e66fc9 100644 --- a/clickhouse/columns/numeric.cpp +++ b/clickhouse/columns/numeric.cpp @@ -61,14 +61,14 @@ void ColumnVector::Append(ColumnRef column) { } template -bool ColumnVector::Load(InputStream* input, size_t rows) { +bool ColumnVector::LoadBody(InputStream* input, size_t rows) { data_.resize(rows); return WireFormat::ReadBytes(*input, data_.data(), data_.size() * sizeof(T)); } template -void ColumnVector::Save(OutputStream* output) { +void ColumnVector::SaveBody(OutputStream* output) { WireFormat::WriteBytes(*output, data_.data(), data_.size() * sizeof(T)); } diff --git a/clickhouse/columns/numeric.h b/clickhouse/columns/numeric.h index 65b21300..e13e58f1 100644 --- a/clickhouse/columns/numeric.h +++ b/clickhouse/columns/numeric.h @@ -35,10 +35,10 @@ class ColumnVector : public Column { void Append(ColumnRef column) override; /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; /// Clear column data . void Clear() override; diff --git a/clickhouse/columns/string.cpp b/clickhouse/columns/string.cpp index 612f1a4c..f43d1c3d 100644 --- a/clickhouse/columns/string.cpp +++ b/clickhouse/columns/string.cpp @@ -77,7 +77,7 @@ void ColumnFixedString::Append(ColumnRef column) { } } -bool ColumnFixedString::Load(InputStream * input, size_t rows) { +bool ColumnFixedString::LoadBody(InputStream * input, size_t rows) { data_.resize(string_size_ * rows); if (!WireFormat::ReadBytes(*input, &data_[0], data_.size())) { return false; @@ -86,7 +86,7 @@ bool ColumnFixedString::Load(InputStream * input, size_t rows) { return true; } -void ColumnFixedString::Save(OutputStream* output) { +void ColumnFixedString::SaveBody(OutputStream* output) { WireFormat::WriteBytes(*output, data_.data(), data_.size()); } @@ -220,7 +220,7 @@ void ColumnString::Append(ColumnRef column) { } } -bool ColumnString::Load(InputStream* input, size_t rows) { +bool ColumnString::LoadBody(InputStream* input, size_t rows) { items_.clear(); blocks_.clear(); @@ -245,7 +245,7 @@ bool ColumnString::Load(InputStream* input, size_t rows) { return true; } -void ColumnString::Save(OutputStream* output) { +void ColumnString::SaveBody(OutputStream* output) { for (const auto & item : items_) { WireFormat::WriteString(*output, item); } diff --git a/clickhouse/columns/string.h b/clickhouse/columns/string.h index d1cec652..013511e1 100644 --- a/clickhouse/columns/string.h +++ b/clickhouse/columns/string.h @@ -43,10 +43,10 @@ class ColumnFixedString : public Column { void Append(ColumnRef column) override; /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; /// Clear column data . void Clear() override; @@ -95,10 +95,10 @@ class ColumnString : public Column { void Append(ColumnRef column) override; /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; /// Clear column data . void Clear() override; diff --git a/clickhouse/columns/tuple.cpp b/clickhouse/columns/tuple.cpp index 88201f36..d2fae362 100644 --- a/clickhouse/columns/tuple.cpp +++ b/clickhouse/columns/tuple.cpp @@ -44,9 +44,9 @@ ColumnRef ColumnTuple::Slice(size_t begin, size_t len) const { return std::make_shared(sliced_columns); } -bool ColumnTuple::Load(InputStream* input, size_t rows) { +bool ColumnTuple::LoadPrefix(InputStream* input, size_t rows) { for (auto ci = columns_.begin(); ci != columns_.end(); ++ci) { - if (!(*ci)->Load(input, rows)) { + if (!(*ci)->LoadPrefix(input, rows)) { return false; } } @@ -54,9 +54,31 @@ bool ColumnTuple::Load(InputStream* input, size_t rows) { return true; } -void ColumnTuple::Save(OutputStream* output) { +bool ColumnTuple::LoadBody(InputStream* input, size_t rows) { for (auto ci = columns_.begin(); ci != columns_.end(); ++ci) { - (*ci)->Save(output); + if (!(*ci)->LoadBody(input, rows)) { + return false; + } + } + + return true; +} + +void ColumnTuple::SavePrefix(OutputStream* output) { + for (auto & column : columns_) { + column->SavePrefix(output); + } +} + +void ColumnTuple::SaveBody(OutputStream* output) { + for (auto & column : columns_) { + column->SaveBody(output); + } +} + +void ColumnTuple::SaveSuffix(OutputStream* output) { + for (auto & column : columns_) { + column->SaveSuffix(output); } } diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index b1ac784a..e6ae3454 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -24,11 +24,17 @@ class ColumnTuple : public Column { /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; + bool LoadPrefix(InputStream* input, size_t rows) override; + /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; + + void SavePrefix(OutputStream* output) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; + + void SaveSuffix(OutputStream* output) override; /// Clear column data . void Clear() override; diff --git a/clickhouse/columns/uuid.cpp b/clickhouse/columns/uuid.cpp index b64bb6b5..36fd1390 100644 --- a/clickhouse/columns/uuid.cpp +++ b/clickhouse/columns/uuid.cpp @@ -44,12 +44,12 @@ void ColumnUUID::Append(ColumnRef column) { } } -bool ColumnUUID::Load(InputStream* input, size_t rows) { +bool ColumnUUID::LoadBody(InputStream* input, size_t rows) { return data_->Load(input, rows * 2); } -void ColumnUUID::Save(OutputStream* output) { - data_->Save(output); +void ColumnUUID::SaveBody(OutputStream* output) { + data_->SaveBody(output); } size_t ColumnUUID::Size() const { diff --git a/clickhouse/columns/uuid.h b/clickhouse/columns/uuid.h index 40319459..c1c6e78d 100644 --- a/clickhouse/columns/uuid.h +++ b/clickhouse/columns/uuid.h @@ -30,10 +30,10 @@ class ColumnUUID : public Column { void Append(ColumnRef column) override; /// Loads column data from input stream. - bool Load(InputStream* input, size_t rows) override; + bool LoadBody(InputStream* input, size_t rows) override; /// Saves column data to output stream. - void Save(OutputStream* output) override; + void SaveBody(OutputStream* output) override; /// Clear column data . void Clear() override; diff --git a/ut/CMakeLists.txt b/ut/CMakeLists.txt index baa72b91..26620934 100644 --- a/ut/CMakeLists.txt +++ b/ut/CMakeLists.txt @@ -15,7 +15,7 @@ SET ( clickhouse-cpp-ut-src utils.cpp readonly_client_test.cpp connection_failed_client_test.cpp -) + array_of_low_cardinality_tests.cpp) IF (WITH_OPENSSL) LIST (APPEND clickhouse-cpp-ut-src ssl_ut.cpp) From deaa0facfd6e4a13d6a10f677fc962d5859c3ba5 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 26 Apr 2022 13:20:50 -0300 Subject: [PATCH 02/10] wip --- ut/array_of_low_cardinality_tests.cpp | 132 ++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 ut/array_of_low_cardinality_tests.cpp diff --git a/ut/array_of_low_cardinality_tests.cpp b/ut/array_of_low_cardinality_tests.cpp new file mode 100644 index 00000000..fd9f69ab --- /dev/null +++ b/ut/array_of_low_cardinality_tests.cpp @@ -0,0 +1,132 @@ +#include +#include +#include +#include + +#include +#include +#include +#include "clickhouse/block.h" +#include "clickhouse/client.h" +#include "utils.h" +#include "clickhouse/base/buffer.h" +#include "clickhouse/base/output.h" + +namespace +{ +using namespace clickhouse; +} + +std::shared_ptr buildTestColumn(const std::vector>& rows) { + auto arrayColumn = std::make_shared(std::make_shared>()); + + for (const auto& row : rows) { + auto column = std::make_shared>(); + + for (const auto& string : row) { + column->Append(string); + } + + arrayColumn->AppendAsColumn(column); + } + + return arrayColumn; +} + +//std::string gen_random() { +// int len = 5; +// static const char alphanum[] = +// "0123456789" +// "ABCDEFGHIJKLMNOPQRSTUVWXYZ" +// "abcdefghijklmnopqrstuvwxyz"; +// std::string tmp_s; +// tmp_s.reserve(len); +// +// for (int i = 0; i < len; ++i) { +// tmp_s += alphanum[rand() % (sizeof(alphanum) - 1)]; +// } +// +// return tmp_s; +//} + +//std::vector generateRandomStringVector(int size) { +// std::random_device rnd_device; +// std::mt19937 mersenne_engine {rnd_device()}; +// std::uniform_int_distribution dist {1, 100}; +// +// auto gen = [&dist, &mersenne_engine](){ +// return dist(mersenne_engine); +// }; +// +// std::vector vec(size); +// +// std::generate(vec.begin(), vec.end(), gen); +// +// return vec; +//} + +const auto localHostEndpoint = ClientOptions() + .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) + .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); + + +TEST(Something, LowCardinalityString) { + const auto inputColumn = buildTestColumn({ + { "aa", "bb" }, + { "cc" } + }); + + // The serialization data was extracted from a successful insert. When compared to what Clickhouse/NativeWriter does for the same fields, + // the only differences are the index type and indexes. + // Since we are setting a different index type in clickhouse-cpp, it's expected to have different indexes. + const std::vector expectedSerialization { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x61, 0x61, + 0x02, 0x62, 0x62, 0x02, 0x63, 0x63, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00 + }; + + Buffer buf; + + BufferOutput output(&buf); + inputColumn->Save(&output); + output.Flush(); + + ASSERT_EQ(expectedSerialization, buf); +} + +TEST(Something, LowCardinalityString2) { + Client client(ClientOptions(localHostEndpoint) + .SetPingBeforeQuery(true)); + + const auto testData = std::vector> { + { "aa", "bb" }, + { "cc" } + }; + + auto column = buildTestColumn(testData); + + Block block; + block.AppendColumn("arr", column); + + client.Execute("DROP TABLE test.array_lc"); + client.Execute("CREATE TABLE IF NOT EXISTS test.array_lc (arr Array(LowCardinality(String))) ENGINE = Memory"); + client.Insert("test.array_lc", block); + + client.Select("SELECT * FROM test.array_lc", [&](const Block& bl) { + for (size_t c = 0; c < bl.GetRowCount(); ++c) { + auto col = bl[0]->As()->GetAsColumn(c); + for (size_t i = 0; i < col->Size(); ++i) { + auto stringColumn = col->As(); + const auto string = stringColumn->At(i); + + ASSERT_EQ(testData[c][i], string); + } + } + } + ); + +} \ No newline at end of file From bfc113e81f9245ec65430039eac7b56ffd6f471b Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 26 Apr 2022 13:28:24 -0300 Subject: [PATCH 03/10] wip --- clickhouse/columns/date.cpp | 6 +++--- clickhouse/columns/decimal.cpp | 2 +- clickhouse/columns/ip4.cpp | 2 +- clickhouse/columns/uuid.cpp | 2 +- ut/array_of_low_cardinality_tests.cpp | 5 ++--- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/clickhouse/columns/date.cpp b/clickhouse/columns/date.cpp index 6284cc5d..6e3c93d4 100644 --- a/clickhouse/columns/date.cpp +++ b/clickhouse/columns/date.cpp @@ -28,7 +28,7 @@ void ColumnDate::Append(ColumnRef column) { } bool ColumnDate::LoadBody(InputStream* input, size_t rows) { - return data_->Load(input, rows); + return data_->LoadBody(input, rows); } void ColumnDate::SaveBody(OutputStream* output) { @@ -90,7 +90,7 @@ void ColumnDateTime::Append(ColumnRef column) { } bool ColumnDateTime::LoadBody(InputStream* input, size_t rows) { - return data_->Load(input, rows); + return data_->LoadBody(input, rows); } void ColumnDateTime::SaveBody(OutputStream* output) { @@ -163,7 +163,7 @@ void ColumnDateTime64::Append(ColumnRef column) { } bool ColumnDateTime64::LoadBody(InputStream* input, size_t rows) { - return data_->Load(input, rows); + return data_->LoadBody(input, rows); } void ColumnDateTime64::SaveBody(OutputStream* output) { diff --git a/clickhouse/columns/decimal.cpp b/clickhouse/columns/decimal.cpp index 7c3dcf68..ad77a6c1 100644 --- a/clickhouse/columns/decimal.cpp +++ b/clickhouse/columns/decimal.cpp @@ -198,7 +198,7 @@ void ColumnDecimal::Append(ColumnRef column) { } bool ColumnDecimal::LoadBody(InputStream * input, size_t rows) { - return data_->Load(input, rows); + return data_->LoadBody(input, rows); } void ColumnDecimal::SaveBody(OutputStream* output) { diff --git a/clickhouse/columns/ip4.cpp b/clickhouse/columns/ip4.cpp index b9160bfb..82b22b04 100644 --- a/clickhouse/columns/ip4.cpp +++ b/clickhouse/columns/ip4.cpp @@ -72,7 +72,7 @@ void ColumnIPv4::Append(ColumnRef column) { } bool ColumnIPv4::LoadBody(InputStream * input, size_t rows) { - return data_->Load(input, rows); + return data_->LoadBody(input, rows); } void ColumnIPv4::SaveBody(OutputStream* output) { diff --git a/clickhouse/columns/uuid.cpp b/clickhouse/columns/uuid.cpp index 36fd1390..6bb6cd8c 100644 --- a/clickhouse/columns/uuid.cpp +++ b/clickhouse/columns/uuid.cpp @@ -45,7 +45,7 @@ void ColumnUUID::Append(ColumnRef column) { } bool ColumnUUID::LoadBody(InputStream* input, size_t rows) { - return data_->Load(input, rows * 2); + return data_->LoadBody(input, rows * 2); } void ColumnUUID::SaveBody(OutputStream* output) { diff --git a/ut/array_of_low_cardinality_tests.cpp b/ut/array_of_low_cardinality_tests.cpp index fd9f69ab..9364de52 100644 --- a/ut/array_of_low_cardinality_tests.cpp +++ b/ut/array_of_low_cardinality_tests.cpp @@ -73,7 +73,7 @@ const auto localHostEndpoint = ClientOptions() .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); -TEST(Something, LowCardinalityString) { +TEST(ArrayOfLowCardinality, Serialization) { const auto inputColumn = buildTestColumn({ { "aa", "bb" }, { "cc" } @@ -98,7 +98,7 @@ TEST(Something, LowCardinalityString) { ASSERT_EQ(expectedSerialization, buf); } -TEST(Something, LowCardinalityString2) { +TEST(ArrayOfLowCardinality, InsertAndQuery) { Client client(ClientOptions(localHostEndpoint) .SetPingBeforeQuery(true)); @@ -128,5 +128,4 @@ TEST(Something, LowCardinalityString2) { } } ); - } \ No newline at end of file From 56ee119688f209eee5feba318a0d2c6396f6ef76 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 26 Apr 2022 14:00:21 -0300 Subject: [PATCH 04/10] wip --- ut/CMakeLists.txt | 2 +- ut/array_of_low_cardinality_tests.cpp | 48 +++++---------------------- 2 files changed, 9 insertions(+), 41 deletions(-) diff --git a/ut/CMakeLists.txt b/ut/CMakeLists.txt index 26620934..50c1dd03 100644 --- a/ut/CMakeLists.txt +++ b/ut/CMakeLists.txt @@ -15,7 +15,7 @@ SET ( clickhouse-cpp-ut-src utils.cpp readonly_client_test.cpp connection_failed_client_test.cpp - array_of_low_cardinality_tests.cpp) + array_of_low_cardinality_tests.cpp) IF (WITH_OPENSSL) LIST (APPEND clickhouse-cpp-ut-src ssl_ut.cpp) diff --git a/ut/array_of_low_cardinality_tests.cpp b/ut/array_of_low_cardinality_tests.cpp index 9364de52..bc01117a 100644 --- a/ut/array_of_low_cardinality_tests.cpp +++ b/ut/array_of_low_cardinality_tests.cpp @@ -33,46 +33,6 @@ std::shared_ptr buildTestColumn(const std::vector generateRandomStringVector(int size) { -// std::random_device rnd_device; -// std::mt19937 mersenne_engine {rnd_device()}; -// std::uniform_int_distribution dist {1, 100}; -// -// auto gen = [&dist, &mersenne_engine](){ -// return dist(mersenne_engine); -// }; -// -// std::vector vec(size); -// -// std::generate(vec.begin(), vec.end(), gen); -// -// return vec; -//} - -const auto localHostEndpoint = ClientOptions() - .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) - .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) - .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) - .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) - .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); - - TEST(ArrayOfLowCardinality, Serialization) { const auto inputColumn = buildTestColumn({ { "aa", "bb" }, @@ -99,6 +59,14 @@ TEST(ArrayOfLowCardinality, Serialization) { } TEST(ArrayOfLowCardinality, InsertAndQuery) { + + const auto localHostEndpoint = ClientOptions() + .SetHost( getEnvOrDefault("CLICKHOUSE_HOST", "localhost")) + .SetPort( getEnvOrDefault("CLICKHOUSE_PORT", "9000")) + .SetUser( getEnvOrDefault("CLICKHOUSE_USER", "default")) + .SetPassword( getEnvOrDefault("CLICKHOUSE_PASSWORD", "")) + .SetDefaultDatabase(getEnvOrDefault("CLICKHOUSE_DB", "default")); + Client client(ClientOptions(localHostEndpoint) .SetPingBeforeQuery(true)); From 515011dae2a799d23d7f07c868fbf155ba730faf Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 26 Apr 2022 14:14:01 -0300 Subject: [PATCH 05/10] wip --- CMakeLists.txt | 2 +- clickhouse/columns/column.cpp | 2 +- clickhouse/columns/lowcardinality.cpp | 6 +++--- clickhouse/columns/nothing.h | 4 ++-- ut/array_of_low_cardinality_tests.cpp | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2baba458..31cd8a5a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,7 @@ INCLUDE (cmake/subdirs.cmake) INCLUDE (cmake/openssl.cmake) OPTION (BUILD_BENCHMARK "Build benchmark" OFF) -OPTION (BUILD_TESTS "Build tests" ON) +OPTION (BUILD_TESTS "Build tests" OFF) OPTION (WITH_OPENSSL "Use OpenSSL for TLS connections" OFF) PROJECT (CLICKHOUSE-CLIENT) diff --git a/clickhouse/columns/column.cpp b/clickhouse/columns/column.cpp index b3d4d15c..91e48261 100644 --- a/clickhouse/columns/column.cpp +++ b/clickhouse/columns/column.cpp @@ -35,4 +35,4 @@ void Column::Save(OutputStream* output) { SaveSuffix(output); } -} \ No newline at end of file +} diff --git a/clickhouse/columns/lowcardinality.cpp b/clickhouse/columns/lowcardinality.cpp index dfc52845..381279cb 100644 --- a/clickhouse/columns/lowcardinality.cpp +++ b/clickhouse/columns/lowcardinality.cpp @@ -216,7 +216,7 @@ auto Load(ColumnRef new_dictionary_column, InputStream& input, size_t rows) { throw ProtocolError("Failed to read number of rows in dictionary column."); if (!new_dictionary_column->LoadBody(&input, number_of_keys)) - throw std::runtime_error("Failed to read values of dictionary column."); + throw ProtocolError("Failed to read values of dictionary column."); uint64_t number_of_rows; if (!WireFormat::ReadFixed(input, &number_of_rows)) @@ -245,10 +245,10 @@ bool ColumnLowCardinality::LoadPrefix(InputStream* input, [[maybe_unused]] size_ uint64_t key_version; if (!WireFormat::ReadFixed(*input, &key_version)) - throw std::runtime_error("Failed to read key serialization version."); + throw ProtocolError("Failed to read key serialization version."); if (key_version != KeySerializationVersion::SharedDictionariesWithAdditionalKeys) - throw std::runtime_error("Invalid key serialization version value."); + throw ProtocolError("Invalid key serialization version value."); // does nothing with key_version, only check it? diff --git a/clickhouse/columns/nothing.h b/clickhouse/columns/nothing.h index c08b490d..29a1eb86 100644 --- a/clickhouse/columns/nothing.h +++ b/clickhouse/columns/nothing.h @@ -67,11 +67,11 @@ class ColumnNothing : public Column { } void SaveBody([[maybe_unused]] OutputStream*) override { - throw std::runtime_error("method SaveBody is not supported for Nothing column"); + throw UnimplementedError("method SaveBody is not supported for Nothing column"); } void SaveSuffix([[maybe_unused]] OutputStream* output) override { - throw std::runtime_error("method SaveSuffix is not supported for Nothing column"); + throw UnimplementedError("method SaveSuffix is not supported for Nothing column"); } /// Clear column data . diff --git a/ut/array_of_low_cardinality_tests.cpp b/ut/array_of_low_cardinality_tests.cpp index bc01117a..e2700068 100644 --- a/ut/array_of_low_cardinality_tests.cpp +++ b/ut/array_of_low_cardinality_tests.cpp @@ -96,4 +96,4 @@ TEST(ArrayOfLowCardinality, InsertAndQuery) { } } ); -} \ No newline at end of file +} From 3ecf492917e942de8188eafad52841500a984fad Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 26 Apr 2022 14:36:15 -0300 Subject: [PATCH 06/10] update comments --- clickhouse/columns/array.h | 1 + clickhouse/columns/column.h | 7 ++++--- clickhouse/columns/nothing.h | 2 ++ clickhouse/columns/nullable.h | 2 ++ clickhouse/columns/tuple.h | 2 ++ ut/array_of_low_cardinality_tests.cpp | 4 ++-- 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/clickhouse/columns/array.h b/clickhouse/columns/array.h index 9c9ed502..b4c591c0 100644 --- a/clickhouse/columns/array.h +++ b/clickhouse/columns/array.h @@ -35,6 +35,7 @@ class ColumnArray : public Column { /// Saves column data to output stream. void SaveBody(OutputStream* output) override; + /// Saves column suffix to output stream. void SaveSuffix(OutputStream* output) override; /// Clear column data . diff --git a/clickhouse/columns/column.h b/clickhouse/columns/column.h index 2a2a61ad..924e396e 100644 --- a/clickhouse/columns/column.h +++ b/clickhouse/columns/column.h @@ -49,9 +49,6 @@ class Column : public std::enable_shared_from_this { virtual bool LoadBody(InputStream* input, size_t rows); - /// Saves column data to output stream. - virtual void Save(OutputStream* output); - /// Saves column prefix to output stream. Column types with prefixes must implement it virtual void SavePrefix(OutputStream* output); @@ -61,6 +58,10 @@ class Column : public std::enable_shared_from_this { /// Saves column suffix to output stream. Column types with suffixes must implement it virtual void SaveSuffix(OutputStream* output); + /// Template method to save to output stream. It'll call SavePrefix, SaveBody and SaveSuffix respectively + /// Should be called only once from the client. Derived classes should not call it. + virtual void Save(OutputStream* output); + /// Clear column data . virtual void Clear() = 0; diff --git a/clickhouse/columns/nothing.h b/clickhouse/columns/nothing.h index 29a1eb86..bd479e4b 100644 --- a/clickhouse/columns/nothing.h +++ b/clickhouse/columns/nothing.h @@ -66,10 +66,12 @@ class ColumnNothing : public Column { throw UnimplementedError("method Save is not supported for Nothing column"); } + /// Saves column data to output stream. void SaveBody([[maybe_unused]] OutputStream*) override { throw UnimplementedError("method SaveBody is not supported for Nothing column"); } + /// Saves column suffix to output stream. void SaveSuffix([[maybe_unused]] OutputStream* output) override { throw UnimplementedError("method SaveSuffix is not supported for Nothing column"); } diff --git a/clickhouse/columns/nullable.h b/clickhouse/columns/nullable.h index 64ddb3e9..2bfd4cc5 100644 --- a/clickhouse/columns/nullable.h +++ b/clickhouse/columns/nullable.h @@ -33,11 +33,13 @@ class ColumnNullable : public Column { /// Loads column data from input stream. bool LoadBody(InputStream* input, size_t rows) override; + /// Saves column prefix to output stream. void SavePrefix(OutputStream* output) override; /// Saves column data to output stream. void SaveBody(OutputStream* output) override; + /// Saves column suffix to output stream. void SaveSuffix(OutputStream* output) override; /// Clear column data . diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index e6ae3454..e92c2b3d 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -29,11 +29,13 @@ class ColumnTuple : public Column { /// Loads column data from input stream. bool LoadBody(InputStream* input, size_t rows) override; + /// Saves column prefix to output stream. void SavePrefix(OutputStream* output) override; /// Saves column data to output stream. void SaveBody(OutputStream* output) override; + /// Saves column suffix to output stream. void SaveSuffix(OutputStream* output) override; /// Clear column data . diff --git a/ut/array_of_low_cardinality_tests.cpp b/ut/array_of_low_cardinality_tests.cpp index e2700068..4c99e897 100644 --- a/ut/array_of_low_cardinality_tests.cpp +++ b/ut/array_of_low_cardinality_tests.cpp @@ -39,8 +39,8 @@ TEST(ArrayOfLowCardinality, Serialization) { { "cc" } }); - // The serialization data was extracted from a successful insert. When compared to what Clickhouse/NativeWriter does for the same fields, - // the only differences are the index type and indexes. + // The serialization data was extracted from a successful insert. + // When compared to what Clickhouse/NativeWriter does for the same fields, the only differences are the index type and indexes. // Since we are setting a different index type in clickhouse-cpp, it's expected to have different indexes. const std::vector expectedSerialization { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, From 0a3e6089feabf65e51b55011847552eedf536c5a Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 26 Apr 2022 14:39:17 -0300 Subject: [PATCH 07/10] update comments --- clickhouse/columns/array.h | 1 + clickhouse/columns/column.h | 9 ++++++--- clickhouse/columns/nullable.h | 1 + clickhouse/columns/tuple.h | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/clickhouse/columns/array.h b/clickhouse/columns/array.h index b4c591c0..53186ae1 100644 --- a/clickhouse/columns/array.h +++ b/clickhouse/columns/array.h @@ -24,6 +24,7 @@ class ColumnArray : public Column { /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; + /// Loads column prefix from input stream. bool LoadPrefix(InputStream* input, size_t rows) override; /// Loads column data from input stream. diff --git a/clickhouse/columns/column.h b/clickhouse/columns/column.h index 924e396e..71310a6e 100644 --- a/clickhouse/columns/column.h +++ b/clickhouse/columns/column.h @@ -42,20 +42,23 @@ class Column : public std::enable_shared_from_this { /// Appends content of given column to the end of current one. virtual void Append(ColumnRef column) = 0; - /// Loads column data from input stream. + /// Template method to load column data from input stream. It'll call LoadPrefix and LoadBody. + /// Should be called only once from the client. Derived classes should not call it. bool Load(InputStream* input, size_t rows); + /// Loads column prefix from input stream. virtual bool LoadPrefix(InputStream* input, size_t rows); + /// Loads column data from input stream. virtual bool LoadBody(InputStream* input, size_t rows); - /// Saves column prefix to output stream. Column types with prefixes must implement it + /// Saves column prefix to output stream. Column types with prefixes must implement it. virtual void SavePrefix(OutputStream* output); /// Saves column body to output stream. virtual void SaveBody(OutputStream* output); - /// Saves column suffix to output stream. Column types with suffixes must implement it + /// Saves column suffix to output stream. Column types with suffixes must implement it. virtual void SaveSuffix(OutputStream* output); /// Template method to save to output stream. It'll call SavePrefix, SaveBody and SaveSuffix respectively diff --git a/clickhouse/columns/nullable.h b/clickhouse/columns/nullable.h index 2bfd4cc5..e3b26f88 100644 --- a/clickhouse/columns/nullable.h +++ b/clickhouse/columns/nullable.h @@ -28,6 +28,7 @@ class ColumnNullable : public Column { /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; + /// Loads column prefix from input stream. bool LoadPrefix(InputStream* input, size_t rows) override; /// Loads column data from input stream. diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index e92c2b3d..ace8784d 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -24,6 +24,7 @@ class ColumnTuple : public Column { /// Appends content of given column to the end of current one. void Append(ColumnRef column) override; + /// Loads column prefix from input stream. bool LoadPrefix(InputStream* input, size_t rows) override; /// Loads column data from input stream. From db75ab5a26bcdcce31b4587261112d4449c14706 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 28 Apr 2022 10:10:27 -0300 Subject: [PATCH 08/10] Remove SaveSuffix, maybe_unused and add comments --- clickhouse/columns/array.cpp | 4 ---- clickhouse/columns/array.h | 3 --- clickhouse/columns/column.cpp | 18 ++---------------- clickhouse/columns/column.h | 13 ++++++------- clickhouse/columns/lowcardinality.cpp | 2 +- clickhouse/columns/lowcardinalityadaptor.h | 4 ++-- clickhouse/columns/nothing.h | 14 ++------------ clickhouse/columns/nullable.cpp | 4 ---- clickhouse/columns/nullable.h | 3 --- clickhouse/columns/tuple.cpp | 6 ------ clickhouse/columns/tuple.h | 3 --- ut/array_of_low_cardinality_tests.cpp | 8 ++++---- 12 files changed, 17 insertions(+), 65 deletions(-) diff --git a/clickhouse/columns/array.cpp b/clickhouse/columns/array.cpp index 049056ac..983000dd 100644 --- a/clickhouse/columns/array.cpp +++ b/clickhouse/columns/array.cpp @@ -84,10 +84,6 @@ void ColumnArray::SaveBody(OutputStream* output) { data_->SaveBody(output); } -void ColumnArray::SaveSuffix(OutputStream* output) { - data_->SaveSuffix(output); -} - void ColumnArray::Clear() { offsets_->Clear(); data_->Clear(); diff --git a/clickhouse/columns/array.h b/clickhouse/columns/array.h index 53186ae1..cfe3eb4b 100644 --- a/clickhouse/columns/array.h +++ b/clickhouse/columns/array.h @@ -36,9 +36,6 @@ class ColumnArray : public Column { /// Saves column data to output stream. void SaveBody(OutputStream* output) override; - /// Saves column suffix to output stream. - void SaveSuffix(OutputStream* output) override; - /// Clear column data . void Clear() override; diff --git a/clickhouse/columns/column.cpp b/clickhouse/columns/column.cpp index 91e48261..7f881d7c 100644 --- a/clickhouse/columns/column.cpp +++ b/clickhouse/columns/column.cpp @@ -2,12 +2,7 @@ namespace clickhouse { -bool Column::LoadPrefix([[maybe_unused]] InputStream* input, [[maybe_unused]] size_t rows) { - /// does nothing by default - return true; -} - -bool Column::LoadBody([[maybe_unused]] InputStream* input, [[maybe_unused]] size_t rows) { +bool Column::LoadPrefix(InputStream*, size_t) { /// does nothing by default return true; } @@ -16,15 +11,7 @@ bool Column::Load(InputStream* input, size_t rows) { return LoadPrefix(input, rows) && LoadBody(input, rows); } -void Column::SavePrefix([[maybe_unused]] OutputStream* output) { - /// does nothing by default -} - -void Column::SaveBody([[maybe_unused]] OutputStream* output) { - /// does nothing by default -} - -void Column::SaveSuffix([[maybe_unused]] OutputStream* output) { +void Column::SavePrefix(OutputStream*) { /// does nothing by default } @@ -32,7 +19,6 @@ void Column::SaveSuffix([[maybe_unused]] OutputStream* output) { void Column::Save(OutputStream* output) { SavePrefix(output); SaveBody(output); - SaveSuffix(output); } } diff --git a/clickhouse/columns/column.h b/clickhouse/columns/column.h index 71310a6e..8a89134d 100644 --- a/clickhouse/columns/column.h +++ b/clickhouse/columns/column.h @@ -50,20 +50,19 @@ class Column : public std::enable_shared_from_this { virtual bool LoadPrefix(InputStream* input, size_t rows); /// Loads column data from input stream. - virtual bool LoadBody(InputStream* input, size_t rows); + virtual bool LoadBody(InputStream* input, size_t rows) = 0; /// Saves column prefix to output stream. Column types with prefixes must implement it. virtual void SavePrefix(OutputStream* output); /// Saves column body to output stream. - virtual void SaveBody(OutputStream* output); + virtual void SaveBody(OutputStream* output) = 0; - /// Saves column suffix to output stream. Column types with suffixes must implement it. - virtual void SaveSuffix(OutputStream* output); - - /// Template method to save to output stream. It'll call SavePrefix, SaveBody and SaveSuffix respectively + /// Template method to save to output stream. It'll call SavePrefix and SaveBody respectively /// Should be called only once from the client. Derived classes should not call it. - virtual void Save(OutputStream* output); + /// Save is split in Prefix and Body because some data types require prefixes and specific serialization order. + /// For instance, Array(LowCardinality(X)) requires LowCardinality.key_version bytes to come before Array.offsets + void Save(OutputStream* output); /// Clear column data . virtual void Clear() = 0; diff --git a/clickhouse/columns/lowcardinality.cpp b/clickhouse/columns/lowcardinality.cpp index 381279cb..a41baa33 100644 --- a/clickhouse/columns/lowcardinality.cpp +++ b/clickhouse/columns/lowcardinality.cpp @@ -241,7 +241,7 @@ auto Load(ColumnRef new_dictionary_column, InputStream& input, size_t rows) { } -bool ColumnLowCardinality::LoadPrefix(InputStream* input, [[maybe_unused]] size_t rows) { +bool ColumnLowCardinality::LoadPrefix(InputStream* input, size_t) { uint64_t key_version; if (!WireFormat::ReadFixed(*input, &key_version)) diff --git a/clickhouse/columns/lowcardinalityadaptor.h b/clickhouse/columns/lowcardinalityadaptor.h index 6d76dd4b..a8b1295f 100644 --- a/clickhouse/columns/lowcardinalityadaptor.h +++ b/clickhouse/columns/lowcardinalityadaptor.h @@ -53,8 +53,8 @@ class LowCardinalitySerializationAdaptor : public AdaptedColumnType } /// Saves column data to output stream. - void Save(OutputStream* output) override { - ColumnLowCardinalityT(this->template As()).Save(output); + void SaveBody(OutputStream* output) override { + ColumnLowCardinalityT(this->template As()).SaveBody(output); } }; diff --git a/clickhouse/columns/nothing.h b/clickhouse/columns/nothing.h index bd479e4b..822e41af 100644 --- a/clickhouse/columns/nothing.h +++ b/clickhouse/columns/nothing.h @@ -57,25 +57,15 @@ class ColumnNothing : public Column { return true; } /// Saves column prefix to output stream. - void SavePrefix([[maybe_unused]] OutputStream* output) override { + void SavePrefix(OutputStream*) override { throw std::runtime_error("method Save is not supported for Nothing column"); } /// Saves column data to output stream. - void Save(OutputStream*) override { - throw UnimplementedError("method Save is not supported for Nothing column"); - } - - /// Saves column data to output stream. - void SaveBody([[maybe_unused]] OutputStream*) override { + void SaveBody(OutputStream*) override { throw UnimplementedError("method SaveBody is not supported for Nothing column"); } - /// Saves column suffix to output stream. - void SaveSuffix([[maybe_unused]] OutputStream* output) override { - throw UnimplementedError("method SaveSuffix is not supported for Nothing column"); - } - /// Clear column data . void Clear() override { size_ = 0; } diff --git a/clickhouse/columns/nullable.cpp b/clickhouse/columns/nullable.cpp index 357409e6..9150e154 100644 --- a/clickhouse/columns/nullable.cpp +++ b/clickhouse/columns/nullable.cpp @@ -73,10 +73,6 @@ void ColumnNullable::SaveBody(OutputStream* output) { nested_->SaveBody(output); } -void ColumnNullable::SaveSuffix(OutputStream* output) { - nested_->SaveSuffix(output); -} - size_t ColumnNullable::Size() const { assert(nested_->Size() == nulls_->Size()); return nulls_->Size(); diff --git a/clickhouse/columns/nullable.h b/clickhouse/columns/nullable.h index e3b26f88..72d31f18 100644 --- a/clickhouse/columns/nullable.h +++ b/clickhouse/columns/nullable.h @@ -40,9 +40,6 @@ class ColumnNullable : public Column { /// Saves column data to output stream. void SaveBody(OutputStream* output) override; - /// Saves column suffix to output stream. - void SaveSuffix(OutputStream* output) override; - /// Clear column data . void Clear() override; diff --git a/clickhouse/columns/tuple.cpp b/clickhouse/columns/tuple.cpp index d2fae362..2b55bae9 100644 --- a/clickhouse/columns/tuple.cpp +++ b/clickhouse/columns/tuple.cpp @@ -76,12 +76,6 @@ void ColumnTuple::SaveBody(OutputStream* output) { } } -void ColumnTuple::SaveSuffix(OutputStream* output) { - for (auto & column : columns_) { - column->SaveSuffix(output); - } -} - void ColumnTuple::Clear() { columns_.clear(); } diff --git a/clickhouse/columns/tuple.h b/clickhouse/columns/tuple.h index ace8784d..c87943fd 100644 --- a/clickhouse/columns/tuple.h +++ b/clickhouse/columns/tuple.h @@ -36,9 +36,6 @@ class ColumnTuple : public Column { /// Saves column data to output stream. void SaveBody(OutputStream* output) override; - /// Saves column suffix to output stream. - void SaveSuffix(OutputStream* output) override; - /// Clear column data . void Clear() override; diff --git a/ut/array_of_low_cardinality_tests.cpp b/ut/array_of_low_cardinality_tests.cpp index 4c99e897..90711327 100644 --- a/ut/array_of_low_cardinality_tests.cpp +++ b/ut/array_of_low_cardinality_tests.cpp @@ -80,11 +80,11 @@ TEST(ArrayOfLowCardinality, InsertAndQuery) { Block block; block.AppendColumn("arr", column); - client.Execute("DROP TABLE test.array_lc"); - client.Execute("CREATE TABLE IF NOT EXISTS test.array_lc (arr Array(LowCardinality(String))) ENGINE = Memory"); - client.Insert("test.array_lc", block); + client.Execute("DROP TEMPORARY TABLE array_lc"); + client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS array_lc (arr Array(LowCardinality(String))) ENGINE = Memory"); + client.Insert("array_lc", block); - client.Select("SELECT * FROM test.array_lc", [&](const Block& bl) { + client.Select("SELECT * FROM array_lc", [&](const Block& bl) { for (size_t c = 0; c < bl.GetRowCount(); ++c) { auto col = bl[0]->As()->GetAsColumn(c); for (size_t i = 0; i < col->Size(); ++i) { From b0081a8f089092ed87a6459f5e85a566f1939f55 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Thu, 28 Apr 2022 10:18:15 -0300 Subject: [PATCH 09/10] add if exists clause to drop temporary table in array_lc_tests --- ut/array_of_low_cardinality_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ut/array_of_low_cardinality_tests.cpp b/ut/array_of_low_cardinality_tests.cpp index 90711327..e022c218 100644 --- a/ut/array_of_low_cardinality_tests.cpp +++ b/ut/array_of_low_cardinality_tests.cpp @@ -80,7 +80,7 @@ TEST(ArrayOfLowCardinality, InsertAndQuery) { Block block; block.AppendColumn("arr", column); - client.Execute("DROP TEMPORARY TABLE array_lc"); + client.Execute("DROP TEMPORARY TABLE IF EXISTS array_lc"); client.Execute("CREATE TEMPORARY TABLE IF NOT EXISTS array_lc (arr Array(LowCardinality(String))) ENGINE = Memory"); client.Insert("array_lc", block); From b68d7f3983e7e3769600f9bb6485f9a4f14f0494 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 3 May 2022 14:11:58 +0300 Subject: [PATCH 10/10] Removed superfluous comment --- clickhouse/columns/lowcardinality.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/clickhouse/columns/lowcardinality.cpp b/clickhouse/columns/lowcardinality.cpp index a41baa33..7e499131 100644 --- a/clickhouse/columns/lowcardinality.cpp +++ b/clickhouse/columns/lowcardinality.cpp @@ -250,9 +250,6 @@ bool ColumnLowCardinality::LoadPrefix(InputStream* input, size_t) { if (key_version != KeySerializationVersion::SharedDictionariesWithAdditionalKeys) throw ProtocolError("Invalid key serialization version value."); - - // does nothing with key_version, only check it? - return true; }