Index Trace

Author: AntiO2

pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/retina/RetinaServerImpl.java

@@ -50,6 +50,7 @@
 public class RetinaServerImpl extends RetinaWorkerServiceGrpc.RetinaWorkerServiceImplBase
 {
     private static final Logger logger = LogManager.getLogger(RetinaServerImpl.class);
+
     private final MetadataService metadataService;
     private final IndexService indexService;
     private final RetinaResourceManager retinaResourceManager;

pixels-daemon/src/main/resources/log4j2.properties

@@ -19,3 +19,15 @@ appender.rolling.layout.pattern = %-d{yyyy-MM-dd HH:mm:ss,SSS} [%c]-[%p] %m%n
 rootLogger.level = info
 rootLogger.appenderRef.stdout.ref = STDOUT
 rootLogger.appenderRef.log.ref = log
+
+appender.index.type = File
+appender.index.name = INDEX_OUT
+appender.index.fileName = ${env:PIXELS_HOME}/logs/index.trace
+appender.index.append = true
+appender.index.layout.type = PatternLayout
+appender.index.layout.pattern = %m%n
+
+logger.index.name = index
+logger.index.level = info
+logger.index.additivity = false
+logger.index.appenderRef.index.ref = INDEX_OUT

pixels-index/pixels-index-rocksdb/pom.xml

@@ -66,6 +66,19 @@
             <artifactId>grpc-stub</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <!-- this is only for third-party libs that use slf4j -->
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <!-- this is only for third-party libs that use log4j-1 -->
+            <groupId>org.slf4j</groupId>
+            <artifactId>log4j-over-slf4j</artifactId>
+            <optional>true</optional>
+        </dependency>
     </dependencies>
 
     <build>

pixels-index/pixels-index-rocksdb/src/main/java/io/pixelsdb/pixels/index/rocksdb/RocksDBIndex.java

@@ -26,6 +26,8 @@
 import io.pixelsdb.pixels.index.IndexProto;
 import org.apache.commons.io.FileUtils;
 import org.rocksdb.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -40,6 +42,8 @@
  */
 public class RocksDBIndex implements SinglePointIndex
 {
+    private static final Logger indexLogger = LoggerFactory.getLogger("index");
+
     private final RocksDB rocksDB;
     private final String rocksDBPath;
     private final WriteOptions writeOptions;
@@ -166,6 +170,12 @@ public boolean putEntry(IndexProto.IndexKey key, long rowId) throws SinglePointI
                 ByteBuffer nonUniqueKeyBuffer = toNonUniqueKeyBuffer(key, rowId);
                 rocksDB.put(writeOptions, nonUniqueKeyBuffer, EMPTY_VALUE_BUFFER);
             }
+            indexLogger.info("P\t{}\t{}\t{}\t{}\t{}",
+                    tableId,
+                    indexId,
+                    key.getKey().asReadOnlyByteBuffer().getInt(),
+                    key.getTimestamp(),
+                    rowId);
             return true;
         }
         catch (RocksDBException e)
@@ -187,6 +197,13 @@ public boolean putPrimaryEntries(List<IndexProto.PrimaryIndexEntry> entries) thr
                 ByteBuffer valueBuffer = RocksDBThreadResources.getValueBuffer();
                 valueBuffer.putLong(rowId).position(0);
                 writeBatch.put(keyBuffer, valueBuffer);
+                indexLogger.info("P\t{}\t{}\t{}\t{}\t{}",
+                        tableId,
+                        indexId,
+                        entry.getIndexKey().getKey().asReadOnlyByteBuffer().getInt(),
+                        entry.getIndexKey().getTimestamp(),
+                        entry.getRowId()
+                );
             }
             rocksDB.write(writeOptions, writeBatch);
             return true;
@@ -238,6 +255,13 @@ public long updatePrimaryEntry(IndexProto.IndexKey key, long rowId) throws Singl
             ByteBuffer valueBuffer = RocksDBThreadResources.getValueBuffer();
             valueBuffer.putLong(rowId).position(0);
             rocksDB.put(writeOptions, keyBuffer, valueBuffer);
+            indexLogger.info("U\t{}\t{}\t{}\t{}\t{}",
+                    tableId,
+                    indexId,
+                    key.getKey().asReadOnlyByteBuffer().getInt(),
+                    key.getTimestamp(),
+                    rowId
+            );
             return prevRowId;
         }
         catch (RocksDBException e)
@@ -289,6 +313,13 @@ public List<Long> updatePrimaryEntries(List<IndexProto.PrimaryIndexEntry> entrie
                 ByteBuffer valueBuffer = RocksDBThreadResources.getValueBuffer();
                 valueBuffer.putLong(rowId).position(0);
                 writeBatch.put(keyBuffer, valueBuffer);
+                indexLogger.info("U\t{}\t{}\t{}\t{}\t{}",
+                        tableId,
+                        indexId,
+                        key.getKey().asReadOnlyByteBuffer().getInt(),
+                        key.getTimestamp(),
+                        rowId
+                );
             }
             rocksDB.write(writeOptions, writeBatch);
             return builder.build();
@@ -399,6 +430,11 @@ public List<Long> deleteEntries(List<IndexProto.IndexKey> keys) throws SinglePoi
             // Delete single point index
             for(IndexProto.IndexKey key : keys)
             {
+                indexLogger.info("D\t{}\t{}\t{}\t{}",
+                        tableId,
+                        indexId,
+                        key.getKey().asReadOnlyByteBuffer().getInt(),
+                        key.getTimestamp());
                 if(unique)
                 {
                     long rowId = getUniqueRowId(key);