[Issue pixelsdb#806] move submodule pixels-duckdb to pixels-duckdb/duckdb (pixelsdb#807)

Also specify protobuf and googletest versions in 'make update', and move extension cpp files into pixels-duckdb.

Author: bianhq

.gitmodules

@@ -1,12 +1,9 @@
 [submodule "cpp/third_party/protobuf"]
 	path = cpp/third-party/protobuf
 	url = https://github.com/protocolbuffers/protobuf
-[submodule "cpp/pixels-duckdb"]
-	path = cpp/pixels-duckdb
+[submodule "cpp/pixels-duckdb/duckdb"]
+	path = cpp/pixels-duckdb/duckdb
 	url = git@github.com:pixelsdb/duckdb.git
-[submodule "writer_cpp_test/third-party/protobuf"]
-	path = writer_cpp_test/third-party/protobuf
-	url = https://github.com/protocolbuffers/protobuf
 [submodule "cpp/third-party/googletest"]
 	path = cpp/third-party/googletest
 	url = git@github.com:google/googletest.git

cpp/CMakeLists.txt

@@ -8,8 +8,8 @@ project(${TARGET_NAME})
 include_directories(include)
 
 set(EXTENSION_SOURCES
-        pixels_extension.cpp
-        PixelsScanFunction.cpp
+        pixels-duckdb/pixels_extension.cpp
+        pixels-duckdb/PixelsScanFunction.cpp
 )
 add_library(${EXTENSION_NAME} STATIC ${EXTENSION_SOURCES})
 

cpp/Makefile

@@ -35,7 +35,9 @@ pull:
 	git submodule update --recursive --init
 
 update:
-	git submodule update --remote --merge
+	git submodule update --remote --merge pixels-duckdb/duckdb
+	git -C third-party/googletest checkout v1.15.2
+	git -C third-party/protobuf checkout v3.21.6
 
 deps:
 	mkdir -p "${PROTOBUF_DIR}/cmake/build" && cd "third-party/protobuf/cmake/build" && \
@@ -45,13 +47,13 @@ deps:
 
 clean:
 	rm -rf build
-	cd pixels-duckdb && make clean
+	cd pixels-duckdb/duckdb && make clean
 
 # Main build
 debug: deps
-	cmake $(GENERATOR) $(FORCE_COLOR) $(EXTENSION_FLAGS) ${CLIENT_FLAGS} -DEXTENSION_STATIC_BUILD=1 -DCMAKE_BUILD_TYPE=Debug ${BUILD_FLAGS} -S pixels-duckdb/ -B build/debug && \
+	cmake $(GENERATOR) $(FORCE_COLOR) $(EXTENSION_FLAGS) ${CLIENT_FLAGS} -DEXTENSION_STATIC_BUILD=1 -DCMAKE_BUILD_TYPE=Debug ${BUILD_FLAGS} -S pixels-duckdb/duckdb -B build/debug && \
 	cmake --build build/debug --config Debug
 
 release: deps
-	cmake $(GENERATOR) $(FORCE_COLOR) $(EXTENSION_FLAGS) ${CLIENT_FLAGS} -DEXTENSION_STATIC_BUILD=1 -DCMAKE_BUILD_TYPE=Release ${BUILD_FLAGS} -S pixels-duckdb/ -B build/release && \
+	cmake $(GENERATOR) $(FORCE_COLOR) $(EXTENSION_FLAGS) ${CLIENT_FLAGS} -DEXTENSION_STATIC_BUILD=1 -DCMAKE_BUILD_TYPE=Release ${BUILD_FLAGS} -S pixels-duckdb/duckdb -B build/release && \
 	cmake --build build/release --config Release

cpp/README.md

@@ -1,14 +1,14 @@
-# Pixels Reader C++ Implementation
+# Pixels C++ Implementation
 
 ## Usage
 
 ### Compilation
 
-The repository relies on [duckdb](https://github.com/yuly16/duckdb), It is refered from [pixels reader](https://github.com/yuly16/pixels-reader-cxx). 
-It also relies on protobuf and liburing. We don't need to manually install these prerequisites, since our compilation code would
-automatically download them.
+Pixels C++ relies on protobuf and liburing. And it builds the pixels extension of duckdb by default,
+which relies on [duckdb](https://github.com/pixelsdb/duckdb).
+We don't need to manually install these prerequisites, since the Makefile would automatically download them.
 
-Pixels C++ reader uses `iouring` system call. You can use the following command to check if iouring is supported in your system:
+Pixels C++ reader uses `iouring` system calls. You can use the following command to check if iouring is supported in your system:
 
 ```shell
 grep io_uring_setup /proc/kallsyms
@@ -35,27 +35,34 @@ Then set the `PIXELS_SRC` and `PIXELS_HOME` environment variable (Ignore it if y
 
 Pull the dependency code:
 
-```
+```shell
 make pull
 ```
 
-Finally, compile the code:
+If it is not the first time to execute `make pull`, you can check out the latest submodules:
 
+```shell
+make update
 ```
+
+Finally, compile the code:
+
+```shell
 make -j
 ```
 
 ### Example
 
-Here is a pixels reader example in the directory `duckdb/examples/pixels-example`. This example validates the correctness of compilation and gives you an idea how to load the pixels data. 
+Here is a pixels reader example in the directory `duckdb/examples/pixels-example`.
+This example validates the correctness of compilation and gives you an idea how to load the Pixels data. 
 
 To run this binary:
 
 ```
 ./build/release/examples/pixels-example/pixels-example
 ```
 
-### Run pixels reader in CLion
+### Run pixels C++ in CLion
 In order to run the code in CLion, we should set the cmake configuration in CLion. 
 
 In `"setting"`->`"Build,Execution,Deployment"`->`"CMake"`, set `"Generator"` as `"Let CMake decide"`, and
@@ -74,55 +81,55 @@ We run one simple query by the following command:
 ```
 build/release/benchmark/benchmark_runner "benchmark/tpch/pixels/tpch_1/q01.benchmark"
 ```
-#### 2. TPCH benchmark
+#### 2. TPC-H benchmark
 The benchmark script is `run_benchmark.py` in `duckdb/scripts` directory.
 
 Check the usage:
 
 ```
-cd pixels-duckdb
+cd pixels-duckdb/duckdb
 python scripts/run_benchmark.py --help
 ```
 
 Run TPC-H 1 benchmarks:
 
 ```
-cd pixels-duckdb
+cd pixels-duckdb/duckdb
 python scripts/run_benchmark.py --pixels "benchmark/tpch/pixels/tpch_1/" --parquet "benchmark/tpch/parquet/tpch_1/" -v --repeat-time-disk 1
 ```
 
 or enabling the encoding of pixels:
 
 ```
-cd pixels-duckdb
+cd pixels-duckdb/duckdb
 python scripts/run_benchmark.py --pixels "benchmark/tpch/pixels/tpch_1_encoding/" --parquet "benchmark/tpch/parquet/tpch_1/" -v --repeat-time-disk 1
 ```
 
 Run TPCH 300 benchmark:
 
 ```
-cd pixels-duckdb
+cd pixels-duckdb/duckdb
 python scripts/run_benchmark.py --pixels "benchmark/tpch/pixels/tpch_300/" --parquet "benchmark/tpch/parquet/tpch_300/" -v --repeat-time-disk 1
 ```
 
 or enabling the encoding of pixels:
 
 ```
-cd pixels-duckdb
+cd pixels-duckdb/duckdb
 python scripts/run_benchmark.py --pixels "benchmark/tpch/pixels/tpch_300_encoding/" --parquet "benchmark/tpch/parquet/tpch_300/" -v --repeat-time-disk 1
 ```
 
 We also support to just run pixels or parquet:
 
 ```
-cd pixels-duckdb
+cd pixels-duckdb/duckdb
 python scripts/run_benchmark.py --pixels "benchmark/tpch/pixels/tpch_1/" -v --repeat-time-disk 1
 ```
 
 We also offer self-defined queries:
 
 ```
-cd pixels-duckdb
+cd pixels-duckdb/duckdb
 python scripts/run_benchmark.py --pixels "benchmark/tpch/pixels/micro-benchmark/tpch_1/" -v  --repeat-time-disk 1
 ```
 
@@ -138,7 +145,7 @@ SSD array needs pixels data to be in multiple directories. The following instruc
 We offer a convenient python script to distribute the pixels data in a single directory to multiple directories. For example:
 
 ```
-cd pixels-duckdb
+cd pixels-duckdb/duckdb
 python scripts/pixels-multidir-generator.py -i /data/s1725-1/liyu/pixels_data/pixels-tpch-1-small-endian \
 -o /data/s1725-1/liyu/pixels_data/pixels-tpch-1-small-endian-partition1 \
 /data/s1725-1/liyu/pixels_data/pixels-tpch-1-small-endian-partition2 \
@@ -152,7 +159,7 @@ The python script copies files from the input directory to all output paths. The
 (e.g. the file index of `20230809035030_4630.pxl` is 4630) and then copied to the output paths in the round-rubin fashion. Pixels C++ reader reads 
 the pixels data sorted by the file index, so all SSDs can be utilized simultaneously.
 
-#### 2. Modity the benchmark
+#### 2. Modify the benchmark
 
 The above python script outputs the following queries:
 
@@ -171,7 +178,7 @@ CREATE VIEW region AS SELECT * FROM pixels_scan(["/data/s1725-1/liyu/pixels_data
 We replace these `CREATE` queries in `pixels_tpch_template.benchmark.in`, and run the benchmark:
 
 ```
-cd pixels-duckdb
+cd pixels-duckdb/duckdb
 python scripts/run_benchmark.py --pixels "benchmark/tpch/pixels/tpch_1/" -v --repeat-time-disk 1
 ```
 
@@ -208,15 +215,9 @@ make update
 
 ### 2. The compilation fails in duckdb
 
-Please make sure you don't use the official `duckdb` repository. The official `duckdb` has some name conflicts with `iouring` (which is a linux async IO library), which would lead to the compilation failure. 
-
-
-### 3. I can't load the pixels data via pixels C++ reader
-
-Currently, the pixels Java writer and reader uses big endian to write/read pixels data. We find that small endian is more efficient for pixels c++ reader. Therefore, in order to generate the pixels data with small endian, please use Pixels in [little-endian branch](https://github.com/pixelsdb/pixels/tree/little-endian). We will merge the small endian to pixels java reader in the future. 
-
+Please make sure you don't use the official `duckdb` repository. The official `duckdb` has some name conflicts with `iouring` (which is a linux async IO library), which would lead to the compilation failure.
 
-### 4. I fail to run the pixels and parquet benchmark
+### 3. I fail to run the pixels and parquet benchmark
 
 This code is tested in diascld31 server. I hardcode the pixels data directory and parquet data directory in `parquet_tpch_template.benchmark.in`, `parquet_tpch_template_no_verification.benchmark.in`, `pixels_tpch_template.benchmark.in` and `pixels_tpch_template_no_verification.benchmark.in`. If you want to run this benchmark in another machine, make sure to modify the pixels and parquet directory to the correct location. `TODO`: In the future I will rewrite this to a more user-friendly benchmark.
 
@@ -244,10 +245,10 @@ def clean_page_cache():
     os.system(cmd)
 ```
 
-### 5. The protobuf version issue
-I use protobuf [v3.21.6](https://github.com/protocolbuffers/protobuf/releases/tag/v3.21.6). The latest protobuf version doesn't work for pixels c++ reader. 
+### 4. The protobuf version issue
+We use protobuf [v3.21.6](https://github.com/protocolbuffers/protobuf/releases/tag/v3.21.6). It is pulled as a submodule. The latest protobuf version doesn't work for pixels c++ reader. 
 
-### 6. Install Boost C++ Libraries
+### 5. Install Boost C++ Libraries
 We need the Boost C++ Libraries in pixels-cli, which can be installed with the following command.
 
 ```

…ixels-cache/include/memory_mapped_file.h …/pixels-cache/include/MemoryMappedFile.hcpp/pixels-cache/include/memory_mapped_file.h renamed to cpp/pixels-cache/include/MemoryMappedFile.h cpp/pixels-cache/include/memory_mapped_file.h renamed to cpp/pixels-cache/include/MemoryMappedFile.h

@@ -1,6 +1,6 @@
 #include <stdlib.h>
 #include <jni.h>
-#include "memory_mapped_file.h"
+#include "MemoryMappedFile.h"
 
 #ifndef _Included_utils
 #define _Included_utils

cpp/pixels-cache/include/utils.h cpp/pixels-cache/include/Utils.hcpp/pixels-cache/include/utils.h renamed to cpp/pixels-cache/include/Utils.h

@@ -2,7 +2,7 @@
 #include <stdint.h>
 #include <string.h>
 #include <byteswap.h>
-#include "../include/memory_mapped_file.h"
+#include "../include/MemoryMappedFile.h"
 #include "../include/io_pixelsdb_pixels_cache_NativeHashIndexReader.h"
 
 #define INDEX_HASH_HEAD_OFFSET 24 // 16 + 8

cpp/pixels-cache/lib/io_pixelsdb_pixels_cache_NativeHashIndexReader.c

@@ -2,7 +2,7 @@
 #include <stdint.h>
 #include <string.h>
 #include <byteswap.h>
-#include "../include/memory_mapped_file.h"
+#include "../include/MemoryMappedFile.h"
 #include "../include/io_pixelsdb_pixels_cache_NativeRadixIndexReader.h"
 
 #define INDEX_RADIX_OFFSET 16

cpp/pixels-cache/lib/io_pixelsdb_pixels_cache_NativeRadixIndexReader.c

@@ -1,8 +1,8 @@
 #include <stdio.h>
 #include <string.h>
 #include <byteswap.h>
-#include "../include/utils.h"
-#include "../include/memory_mapped_file.h"
+#include "../include/Utils.h"
+#include "../include/MemoryMappedFile.h"
 #include "../include/io_pixelsdb_pixels_cache_utils_RadixIndexEndianRewriter.h"
 
 #define INDEX_RADIX_OFFSET 16

cpp/pixels-cache/lib/io_pixelsdb_pixels_cache_utils_RadixIndexEndianRewriter.c

@@ -18,7 +18,7 @@ pixel.stride=2
 pixel.threads=-1
 # column size path. It is optional. If no column size path is designated, the
 # size of first pixels data is used. For example:
-# pixel.column.size.path=/scratch/liyu/opt/pixels/cpp/pixels-duckdb/benchmark/clickbench/clickbench-size.csv
+# pixel.column.size.path=/scratch/liyu/opt/pixels/cpp/pixels-duckdb/duckdb/benchmark/clickbench/clickbench-size.csv
 pixel.column.size.path=
 
 # the work thread to run parquet. -1 means using all CPU cores