[Issue pixelsdb#802] implement cpp version of writer (pixelsdb#803)

Co-authored-by: whzruc <whzsxdt@126.com>
Co-authored-by: AntiO2 <antio2@qq.com>

Author: 3 people

.gitmodules

@@ -4,3 +4,9 @@
 [submodule "cpp/pixels-duckdb"]
 	path = cpp/pixels-duckdb
 	url = git@github.com:pixelsdb/duckdb.git
+[submodule "writer_cpp_test/third-party/protobuf"]
+	path = writer_cpp_test/third-party/protobuf
+	url = https://github.com/protocolbuffers/protobuf
+[submodule "cpp/third-party/googletest"]
+	path = cpp/third-party/googletest
+	url = git@github.com:google/googletest.git

cpp/CMakeLists.txt

@@ -20,6 +20,8 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 add_subdirectory(pixels-common)
 add_subdirectory(pixels-core)
+ add_subdirectory(pixels-cli)
+add_subdirectory(third-party/googletest)
 add_subdirectory(tests)
 
 include_directories(pixels-common/include)

cpp/Makefile

@@ -49,11 +49,9 @@ clean:
 
 # Main build
 debug: deps
-	mkdir -p  build/debug && \
 	cmake $(GENERATOR) $(FORCE_COLOR) $(EXTENSION_FLAGS) ${CLIENT_FLAGS} -DEXTENSION_STATIC_BUILD=1 -DCMAKE_BUILD_TYPE=Debug ${BUILD_FLAGS} -S pixels-duckdb/ -B build/debug && \
 	cmake --build build/debug --config Debug
 
 release: deps
-	mkdir -p build/release && \
 	cmake $(GENERATOR) $(FORCE_COLOR) $(EXTENSION_FLAGS) ${CLIENT_FLAGS} -DEXTENSION_STATIC_BUILD=1 -DCMAKE_BUILD_TYPE=Release ${BUILD_FLAGS} -S pixels-duckdb/ -B build/release && \
 	cmake --build build/release --config Release

cpp/README.md

@@ -246,3 +246,37 @@ def clean_page_cache():
 
 ### 5. The protobuf version issue
 I use protobuf [v3.21.6](https://github.com/protocolbuffers/protobuf/releases/tag/v3.21.6). The latest protobuf version doesn't work for pixels c++ reader. 
+
+### 6. Install Boost C++ Libraries
+We need the Boost C++ Libraries in pixels-cli, which can be installed with the following command.
+
+```
+sudo apt-get install libboost-all-dev
+```
+
+For every Boost release this information is added by the CMake maintainers and it gets part of the next CMake release. So you have to make sure, that your CMake version was released after the Boost version you try to find.
+
+```
+Boost 1.63 requires CMake 3.7 or newer.
+Boost 1.64 requires CMake 3.8 or newer.
+Boost 1.65 and 1.65.1 require CMake 3.9.3 or newer.
+Boost 1.66 requires CMake 3.11 or newer.
+Boost 1.67 requires CMake 3.12 or newer.
+Boost 1.68, 1.69 require CMake 3.13 or newer.
+Boost 1.70 requires CMake 3.14 or newer.
+Boost 1.71 requires CMake 3.15.3 or newer.
+Boost 1.72 requires CMake 3.16.2 or newer.
+Boost 1.73 requires CMake 3.17.2 or newer.
+Boost 1.74 requires CMake 3.19 or newer.
+Boost 1.75 requires CMake 3.19.5 or newer.
+Boost 1.76 requires CMake 3.20.3 or newer.
+Boost 1.77 requires CMake 3.21.3 or newer.
+Boost 1.78 requires CMake 3.22.2 or newer.
+Boost 1.79 requires CMake 3.23.2 or newer.
+Boost 1.80 requires CMake 3.24.2 or newer.
+Boost 1.81 requires CMake 3.25.2 or newer.
+Boost 1.82 requires CMake 3.27.0 or newer.
+Boost 1.83 requires CMake 3.27.4 or newer.
+Boost 1.84 requires CMake 3.28.2 or newer.
+Boost 1.85 requires CMake 3.29.3 or newer.
+```

cpp/pixels-cli/CMakeLists.txt

@@ -0,0 +1,32 @@
+cmake_minimum_required(VERSION 3.0)
+project(pixels-cli)
+
+set(CMAKE_CXX_STANDARD 17)
+set(Boost_USE_STATIC_LIBS ON)
+set(Boost_USE_MULTITHREADED ON)
+
+if(POLICY CMP0057)
+    cmake_policy(SET CMP0057 NEW)
+endif()
+
+if(POLICY CMP0167)
+  cmake_policy(SET CMP0167 NEW)
+endif()
+
+find_package(Boost REQUIRED COMPONENTS program_options regex)
+if(Boost_FOUND)
+    include_directories(${Boost_INCLUDE_DIRS})
+else()
+    message(FATAL_ERROR "Could not find Boost")
+endif()
+
+set(pixels_cli_cxx
+        main.cpp
+        lib/executor/LoadExecutor.cpp
+        lib/load/Parameters.cpp
+        lib/load/PixelsConsumer.cpp)
+add_executable(pixels-cli ${pixels_cli_cxx})
+include_directories(include)
+include_directories(../pixels-core/include)
+include_directories(../pixels-common/include)
+target_link_libraries(pixels-cli Boost::program_options Boost::regex pixels-core pixels-common duckdb)

cpp/pixels-cli/Makefile

@@ -0,0 +1,8 @@
+.PHONY: all clean
+
+all:
+	cmake -B build
+	cmake --build build
+
+clean:
+	rm -rf build

cpp/pixels-cli/include/executor/CommandExecutor.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2024 PixelsDB.
+ *
+ * This file is part of Pixels.
+ *
+ * Pixels is free software: you can redistribute it and/or modify
+ * it under the terms of the Affero GNU General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * Pixels is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * Affero GNU General Public License for more details.
+ *
+ * You should have received a copy of the Affero GNU General Public
+ * License along with Pixels.  If not, see
+ * <https://www.gnu.org/licenses/>.
+ */
+
+//
+// Created by gengdy on 24-11-17.
+//
+
+#ifndef PIXELS_COMMANDEXECUTOR_H
+#define PIXELS_COMMANDEXECUTOR_H
+
+#include <boost/program_options.hpp>
+#include <string>
+
+namespace bpo = boost::program_options;
+
+class CommandExecutor {
+public:
+    virtual ~CommandExecutor() = default;
+    virtual void execute(const bpo::variables_map& ns, const std::string& command) = 0;
+};
+#endif //PIXELS_COMMANDEXECUTOR_H

cpp/pixels-cli/include/executor/LoadExecutor.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2024 PixelsDB.
+ *
+ * This file is part of Pixels.
+ *
+ * Pixels is free software: you can redistribute it and/or modify
+ * it under the terms of the Affero GNU General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * Pixels is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * Affero GNU General Public License for more details.
+ *
+ * You should have received a copy of the Affero GNU General Public
+ * License along with Pixels.  If not, see
+ * <https://www.gnu.org/licenses/>.
+ */
+
+//
+// Created by gengdy on 24-11-17.
+//
+
+#ifndef PIXELS_LOADEXECUTOR_H
+#define PIXELS_LOADEXECUTOR_H
+
+#include <executor/CommandExecutor.h>
+#include <vector>
+#include <load/Parameters.h>
+
+class LoadExecutor : public CommandExecutor {
+public:
+    void execute(const bpo::variables_map& ns, const std::string& command) override;
+private:
+    bool startConsumers(const std::vector<std::string> &inputFiles, Parameters parameters,
+                        const std::vector<std::string> &loadedFiles);
+};
+#endif //PIXELS_LOADEXECUTOR_H

cpp/pixels-cli/include/load/Parameters.h

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2024 PixelsDB.
+ *
+ * This file is part of Pixels.
+ *
+ * Pixels is free software: you can redistribute it and/or modify
+ * it under the terms of the Affero GNU General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * Pixels is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * Affero GNU General Public License for more details.
+ *
+ * You should have received a copy of the Affero GNU General Public
+ * License along with Pixels.  If not, see
+ * <https://www.gnu.org/licenses/>.
+ */
+
+//
+// Created by gengdy on 24-11-19.
+//
+
+#ifndef PIXELS_PARAMETERS_H
+#define PIXELS_PARAMETERS_H
+
+#include <string>
+#include <encoding/EncodingLevel.h>
+
+class Parameters {
+public:
+    Parameters(const std::string &schema, int maxRowNum, const std::string &regex,
+               const std::string &loadingPath, EncodingLevel encodingLevel, bool nullsPadding);
+    std::string getLoadingPath() const;
+    std::string getSchema() const;
+    int getMaxRowNum() const;
+    std::string getRegex() const;
+    EncodingLevel getEncodingLevel() const;
+    bool isNullsPadding() const;
+
+private:
+    std::string schema;
+    int maxRowNum;
+    std::string regex;
+    std::string loadingPath;
+    EncodingLevel encodingLevel;
+    bool nullsPadding;
+};
+#endif //PIXELS_PARAMETERS_H

cpp/pixels-cli/include/load/PixelsConsumer.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2024 PixelsDB.
+ *
+ * This file is part of Pixels.
+ *
+ * Pixels is free software: you can redistribute it and/or modify
+ * it under the terms of the Affero GNU General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * Pixels is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * Affero GNU General Public License for more details.
+ *
+ * You should have received a copy of the Affero GNU General Public
+ * License along with Pixels.  If not, see
+ * <https://www.gnu.org/licenses/>.
+ */
+
+//
+// Created by gengdy on 24-11-22.
+//
+
+#ifndef PIXELS_PIXELSCONSUMER_H
+#define PIXELS_PIXELSCONSUMER_H
+
+#include <vector>
+#include <string>
+#include <load/Parameters.h>
+
+class PixelsConsumer {
+public:
+    PixelsConsumer(const std::vector<std::string> &queue, const Parameters &parameters, const std::vector<std::string> &loadedFiles);
+    void run();
+private:
+    static int GlobalTargetPathId;
+    std::vector<std::string> queue;
+    Parameters parameters;
+    std::vector<std::string> loadedFiles;
+};
+#endif //PIXELS_PIXELSCONSUMER_H