diff --git a/beeline/pom.xml b/beeline/pom.xml index c3b65f283fa7..dbe6593c9d71 100644 --- a/beeline/pom.xml +++ b/beeline/pom.xml @@ -193,6 +193,11 @@ ${powermock.version} test + + org.apache.hadoop + hadoop-hdfs + test + org.postgresql postgresql diff --git a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out index 923d7f93679a..48f37290945c 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out @@ -255,7 +255,7 @@ POSTHOOK: query: select summary from default.ice_meta_3.snapshots POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_meta_3 POSTHOOK: Output: hdfs://### HDFS PATH ### -{"added-data-files":"7","added-records":"15","added-files-size":"1329","changed-partition-count":"7","total-records":"15","total-files-size":"1329","total-data-files":"7","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0"} +{"added-data-files":"7","added-records":"15","added-files-size":"1378","changed-partition-count":"7","total-records":"15","total-files-size":"1378","total-data-files":"7","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0"} PREHOOK: query: select summary['changed-partition-count'] from default.ice_meta_2.snapshots PREHOOK: type: QUERY PREHOOK: Input: default@ice_meta_2 @@ -551,7 +551,7 @@ POSTHOOK: query: select summary from default.ice_meta_3.snapshots POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_meta_3 POSTHOOK: Output: hdfs://### HDFS PATH ### -{"added-data-files":"7","added-records":"15","added-files-size":"1329","changed-partition-count":"7","total-records":"15","total-files-size":"1329","total-data-files":"7","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0"} +{"added-data-files":"7","added-records":"15","added-files-size":"1378","changed-partition-count":"7","total-records":"15","total-files-size":"1378","total-data-files":"7","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0"} PREHOOK: query: select summary['changed-partition-count'] from default.ice_meta_2.snapshots PREHOOK: type: QUERY PREHOOK: Input: default@ice_meta_2 diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java index f7cd620ec320..d0f3ebd19d69 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java @@ -390,7 +390,7 @@ public void testStatsAfterCompactionPartTbl() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "1373", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "1396", parameters.get("totalSize")); parameters = partitions .stream() @@ -400,7 +400,7 @@ public void testStatsAfterCompactionPartTbl() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "1442", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "1453", parameters.get("totalSize")); //Do a major compaction CompactionRequest rqst = new CompactionRequest(dbName, tblName, CompactionType.MAJOR); @@ -425,7 +425,7 @@ public void testStatsAfterCompactionPartTbl() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "801", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "808", parameters.get("totalSize")); parameters = partitions .stream() @@ -435,7 +435,7 @@ public void testStatsAfterCompactionPartTbl() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "1442", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "1453", parameters.get("totalSize")); } /** @@ -478,7 +478,7 @@ public void testStatsAfterCompactionTbl() throws Exception { Map parameters = Hive.get().getTable(tblName).getParameters(); Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "1434", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "1446", parameters.get("totalSize")); //Do a major compaction CompactionRequest rqst = new CompactionRequest(dbName, tblName, CompactionType.MAJOR); @@ -496,7 +496,7 @@ public void testStatsAfterCompactionTbl() throws Exception { parameters = Hive.get().getTable(tblName).getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "776", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "783", parameters.get("totalSize")); } @Test diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java index 62d4a0080273..838c4bd34952 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java @@ -177,18 +177,18 @@ private void testRebalanceCompactionWithParallelDeleteAsSecond(boolean optimisti "{\"writeid\":7,\"bucketid\":536870912,\"rowid\":4}\t13\t13", }, { - "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":6}\t3\t4", - "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":7}\t4\t4", - "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":8}\t2\t4", + "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":6}\t2\t4", + "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":7}\t3\t4", + "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":8}\t4\t4", "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":9}\t5\t4", "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":10}\t6\t4", - "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":11}\t4\t3", + "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":11}\t5\t3", }, { - "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":12}\t2\t3", - "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":13}\t3\t3", - "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":14}\t6\t3", - "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":15}\t5\t3", + "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":12}\t6\t3", + "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":13}\t4\t3", + "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":14}\t2\t3", + "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":15}\t3\t3", "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":16}\t6\t2", "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":17}\t5\t2", }, @@ -231,18 +231,18 @@ public void testRebalanceCompactionOfNotPartitionedImplicitlyBucketedTableWithOr "{\"writeid\":7,\"bucketid\":536870912,\"rowid\":5}\t12\t12", }, { - "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":6}\t3\t4", - "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":7}\t4\t4", - "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":8}\t2\t4", + "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":6}\t2\t4", + "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":7}\t3\t4", + "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":8}\t4\t4", "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":9}\t5\t4", "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":10}\t6\t4", - "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":11}\t4\t3", + "{\"writeid\":7,\"bucketid\":536936448,\"rowid\":11}\t5\t3", }, { - "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":12}\t2\t3", - "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":13}\t3\t3", - "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":14}\t6\t3", - "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":15}\t5\t3", + "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":12}\t6\t3", + "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":13}\t4\t3", + "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":14}\t2\t3", + "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":15}\t3\t3", "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":16}\t6\t2", "{\"writeid\":7,\"bucketid\":537001984,\"rowid\":17}\t5\t2", }, @@ -519,6 +519,7 @@ private TestDataProvider prepareRebalanceTestData(String tableName) throws Excep "{\"writeid\":1,\"bucketid\":536870912,\"rowid\":3}\t6\t4", "{\"writeid\":1,\"bucketid\":536870912,\"rowid\":4}\t5\t2", "{\"writeid\":1,\"bucketid\":536870912,\"rowid\":5}\t5\t3", + "{\"writeid\":1,\"bucketid\":536870912,\"rowid\":6}\t2\t4", "{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t12\t12", "{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t13\t13", "{\"writeid\":4,\"bucketid\":536870912,\"rowid\":0}\t14\t14", @@ -527,14 +528,13 @@ private TestDataProvider prepareRebalanceTestData(String tableName) throws Excep "{\"writeid\":7,\"bucketid\":536870912,\"rowid\":0}\t17\t17", }, { - "{\"writeid\":1,\"bucketid\":536936448,\"rowid\":0}\t2\t4", - "{\"writeid\":1,\"bucketid\":536936448,\"rowid\":1}\t3\t3", - "{\"writeid\":1,\"bucketid\":536936448,\"rowid\":2}\t4\t4", - "{\"writeid\":1,\"bucketid\":536936448,\"rowid\":3}\t4\t3", + "{\"writeid\":1,\"bucketid\":536936448,\"rowid\":0}\t3\t3", + "{\"writeid\":1,\"bucketid\":536936448,\"rowid\":1}\t4\t4", + "{\"writeid\":1,\"bucketid\":536936448,\"rowid\":2}\t4\t3", + "{\"writeid\":1,\"bucketid\":536936448,\"rowid\":3}\t2\t3", }, { - "{\"writeid\":1,\"bucketid\":537001984,\"rowid\":0}\t2\t3", - "{\"writeid\":1,\"bucketid\":537001984,\"rowid\":1}\t3\t4", + "{\"writeid\":1,\"bucketid\":537001984,\"rowid\":0}\t3\t4", }, }; AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf); @@ -810,7 +810,7 @@ public void testStatsAfterQueryCompactionOnTez() throws Exception { Map parameters = Hive.get().getTable(tblName).getParameters(); Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "1434", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "1446", parameters.get("totalSize")); //Do a major compaction CompactorTestUtil.runCompaction(conf, dbName, tblName, CompactionType.MAJOR, true); @@ -826,7 +826,7 @@ public void testStatsAfterQueryCompactionOnTez() throws Exception { parameters = Hive.get().getTable(tblName).getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "727", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "735", parameters.get("totalSize")); } @Test diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java index 9459a4ff64f6..c0a2daa68897 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java @@ -58,14 +58,13 @@ import org.apache.orc.impl.SchemaEvolution; import org.apache.orc.impl.TreeReaderFactory; import org.apache.orc.impl.TreeReaderFactory.StructTreeReader; -import org.apache.orc.impl.TreeReaderFactory.TreeReader; import org.apache.orc.impl.WriterImpl; import org.apache.orc.OrcProto; - +import org.apache.orc.impl.reader.tree.TypeReader; public class OrcEncodedDataConsumer extends EncodedDataConsumer { - private TreeReaderFactory.TreeReader[] columnReaders; + private TypeReader[] columnReaders; private int previousStripeIndex = -1; private ConsumerFileMetadata fileMetadata; // We assume one request is only for one file. private CompressionCodec codec; @@ -191,9 +190,9 @@ protected void decodeBatch(OrcEncodedColumnBatch batch, * it doesn't get confused. * */ - TreeReader reader = columnReaders[idx]; + TypeReader reader = columnReaders[idx]; ColumnVector cv = prepareColumnVector(cvb, idx, batchSize); - reader.nextVector(cv, null, batchSize); + reader.nextVector(cv, null, batchSize, cvb.filterContext, TypeReader.ReadPhase.ALL); } // we are done reading a batch, send it to consumer for processing @@ -299,18 +298,18 @@ private ColumnVector createColumn(TypeDescription type, int batchSize, final boo } } - private void positionInStreams(TreeReaderFactory.TreeReader[] columnReaders, + private void positionInStreams(TypeReader[] columnReaders, OrcBatchKey batchKey, ConsumerStripeMetadata stripeMetadata) throws IOException { PositionProvider[] pps = createPositionProviders(columnReaders, batchKey, stripeMetadata); if (pps == null) return; for (int i = 0; i < columnReaders.length; i++) { if (columnReaders[i] == null) continue; // TODO: we could/should trace seek destinations; pps needs a "peek" method - columnReaders[i].seek(pps); + columnReaders[i].seek(pps, TypeReader.ReadPhase.ALL); } } - private void repositionInStreams(TreeReaderFactory.TreeReader[] columnReaders, + private void repositionInStreams(TypeReader[] columnReaders, EncodedColumnBatch batch, boolean sameStripe, ConsumerStripeMetadata stripeMetadata) throws IOException { PositionProvider[] pps = createPositionProviders( @@ -320,7 +319,7 @@ private void repositionInStreams(TreeReaderFactory.TreeReader[] columnReaders, } if (pps == null) return; for (int i = 0; i < columnReaders.length; i++) { - TreeReader reader = columnReaders[i]; + TypeReader reader = columnReaders[i]; if (reader == null) continue; // Note: we assume this never happens for SerDe reader - the batch would never have vectors. // That is always true now; but it wasn't some day, the below would throw in getColumnData. @@ -331,7 +330,7 @@ private void repositionInStreams(TreeReaderFactory.TreeReader[] columnReaders, ((EncodedTreeReaderFactory.TimestampStreamReader) reader) .updateTimezone(stripeMetadata.getWriterTimezone()); } - reader.seek(pps); + reader.seek(pps, TypeReader.ReadPhase.ALL); } } @@ -352,7 +351,7 @@ public String toString() { } private PositionProvider[] createPositionProviders( - TreeReaderFactory.TreeReader[] columnReaders, OrcBatchKey batchKey, + TypeReader[] columnReaders, OrcBatchKey batchKey, ConsumerStripeMetadata stripeMetadata) throws IOException { if (columnReaders.length == 0) return null; PositionProvider[] pps = null; diff --git a/pom.xml b/pom.xml index 9f20d6649f9f..c0cf4c2ac1af 100644 --- a/pom.xml +++ b/pom.xml @@ -179,7 +179,7 @@ 42.5.1 21.3.0.0 2.3 - 1.6.9 + 1.8.3 3.4.4 2.0.2 2.0.0-M5 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java index 9ad75ddc92f6..46e5a3c3be89 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.io.orc.encoded; import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; +import org.apache.hadoop.hive.ql.io.filter.FilterContext; import org.apache.orc.OrcFile; import org.apache.orc.impl.RunLengthByteReader; @@ -35,11 +36,13 @@ import org.apache.orc.impl.PositionProvider; import org.apache.orc.impl.TreeReaderFactory; import org.apache.orc.OrcProto; +import org.apache.orc.impl.reader.tree.TypeReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class EncodedTreeReaderFactory extends TreeReaderFactory { private static final Logger LOG = LoggerFactory.getLogger(EncodedTreeReaderFactory.class); + /** * We choose to use a toy programming language, so we cannot use multiple inheritance. * If we could, we could have this inherit TreeReader to contain the common impl, and then @@ -50,8 +53,7 @@ public interface SettableTreeReader { void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException; } - public static class TimestampStreamReader extends TimestampTreeReader - implements SettableTreeReader { + public static class TimestampStreamReader extends TimestampTreeReader implements SettableTreeReader { private boolean isFileCompressed; private SettableUncompressedStream _presentStream; private SettableUncompressedStream _secondsStream; @@ -59,11 +61,9 @@ public static class TimestampStreamReader extends TimestampTreeReader private List vectors; private int vectorIndex = 0; - private TimestampStreamReader(int columnId, SettableUncompressedStream present, - SettableUncompressedStream data, SettableUncompressedStream nanos, - boolean isFileCompressed, OrcProto.ColumnEncoding encoding, - TreeReaderFactory.Context context, - List vectors, boolean isInstant) throws IOException { + private TimestampStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, + SettableUncompressedStream nanos, boolean isFileCompressed, OrcProto.ColumnEncoding encoding, + TreeReaderFactory.Context context, List vectors, boolean isInstant) throws IOException { super(columnId, present, data, nanos, encoding, context, isInstant); this.isFileCompressed = isFileCompressed; this._presentStream = present; @@ -73,10 +73,10 @@ private TimestampStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } // Note: we assume that batchSize will be consistent with vectors passed in. @@ -88,8 +88,9 @@ public void nextVector( } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase phase) throws IOException { + if (vectors != null) + return; if (present != null) { if (isFileCompressed) { index.getNext(); @@ -115,8 +116,7 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { // The situation here and in other readers is currently as such - setBuffers is never called // in SerDe reader case, and SerDe reader case is the only one that uses vector-s. // When the readers are created with vectors, streams are actually not created at all. @@ -191,21 +191,18 @@ public StreamReaderBuilder setIsInstant(boolean isInstant) { } public TimestampStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); - SettableUncompressedStream nanos = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.SECONDARY.name(), - nanosStream); + SettableUncompressedStream nanos = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.SECONDARY.name(), nanosStream); boolean isFileCompressed = compressionCodec != null; - return new TimestampStreamReader(columnIndex, present, data, nanos, - isFileCompressed, columnEncoding, context, vectors, isInstant); + return new TimestampStreamReader(columnIndex, present, data, nanos, isFileCompressed, columnEncoding, context, + vectors, isInstant); } public StreamReaderBuilder setVectors(List vectors) { @@ -220,12 +217,12 @@ public static StreamReaderBuilder builder() { } private static void skipCompressedIndex(boolean isCompressed, PositionProvider index) { - if (!isCompressed) return; + if (!isCompressed) + return; index.getNext(); } - protected static class StringStreamReader extends StringTreeReader - implements SettableTreeReader { + protected static class StringStreamReader extends StringTreeReader implements SettableTreeReader { private boolean _isFileCompressed; private boolean _isDictionaryEncoding; private SettableUncompressedStream _presentStream; @@ -233,13 +230,12 @@ protected static class StringStreamReader extends StringTreeReader private SettableUncompressedStream _lengthStream; private SettableUncompressedStream _dictionaryStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; - private StringStreamReader(int columnId, SettableUncompressedStream present, - SettableUncompressedStream data, SettableUncompressedStream length, - SettableUncompressedStream dictionary, - boolean isFileCompressed, OrcProto.ColumnEncoding encoding, - TreeReaderFactory.Context context, List vectors) throws IOException { + private StringStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, + SettableUncompressedStream length, SettableUncompressedStream dictionary, boolean isFileCompressed, + OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, List vectors) + throws IOException { super(columnId, present, data, length, dictionary, encoding, context); this._isDictionaryEncoding = dictionary != null; this._isFileCompressed = isFileCompressed; @@ -251,19 +247,20 @@ private StringStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { // This string reader should simply redirect to its own seek (what other types already do). - this.seek(index[columnId]); + this.seek(index[columnId], readPhase); } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); } - reader.getPresent().seek(index); + ((TreeReader) reader).getPresent().seek(index); } if (_isDictionaryEncoding) { @@ -299,10 +296,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, int batchSize, FilterContext filterContext, + TypeReader.ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -312,8 +309,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -334,7 +330,8 @@ public void setBuffers(EncodedColumnBatch batch, boolean sameStripe _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE])); } if (_dictionaryStream != null) { - _dictionaryStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DICTIONARY_DATA_VALUE])); + _dictionaryStream.setBuffers( + StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DICTIONARY_DATA_VALUE])); } } } @@ -391,24 +388,21 @@ public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { } public StringStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); - SettableUncompressedStream length = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.LENGTH.name(), - lengthStream); + SettableUncompressedStream length = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.LENGTH.name(), lengthStream); SettableUncompressedStream dictionary = StreamUtils.createSettableUncompressedStream( OrcProto.Stream.Kind.DICTIONARY_DATA.name(), dictionaryStream); boolean isFileCompressed = compressionCodec != null; - return new StringStreamReader(columnIndex, present, data, length, dictionary, - isFileCompressed, columnEncoding, context, vectors); + return new StringStreamReader(columnIndex, present, data, length, dictionary, isFileCompressed, columnEncoding, + context, vectors); } public StreamReaderBuilder setVectors(List vectors) { @@ -428,11 +422,10 @@ protected static class ShortStreamReader extends ShortTreeReader implements Sett private SettableUncompressedStream _presentStream; private SettableUncompressedStream _dataStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; - private ShortStreamReader(int columnId, SettableUncompressedStream present, - SettableUncompressedStream data, boolean isFileCompressed, - OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, + private ShortStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, + boolean isFileCompressed, OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, List vectors) throws IOException { super(columnId, present, data, encoding, context); this.isFileCompressed = isFileCompressed; @@ -442,8 +435,9 @@ private ShortStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase phase) throws IOException { + if (vectors != null) + return; if (present != null) { if (isFileCompressed) { index.getNext(); @@ -462,10 +456,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -475,8 +469,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -527,17 +520,14 @@ public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { } public ShortStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); boolean isFileCompressed = compressionCodec != null; - return new ShortStreamReader(columnIndex, present, data, isFileCompressed, - columnEncoding, context, vectors); + return new ShortStreamReader(columnIndex, present, data, isFileCompressed, columnEncoding, context, vectors); } public StreamReaderBuilder setVectors(List vectors) { @@ -559,9 +549,8 @@ protected static class LongStreamReader extends LongTreeReader implements Settab private List vectors; private int vectorIndex = 0; - private LongStreamReader(int columnId, SettableUncompressedStream present, - SettableUncompressedStream data, boolean isFileCompressed, - OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, + private LongStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, + boolean isFileCompressed, OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, List vectors) throws IOException { super(columnId, present, data, encoding, context); this._isFileCompressed = isFileCompressed; @@ -571,8 +560,9 @@ private LongStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); @@ -591,10 +581,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -604,8 +594,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -625,7 +614,6 @@ public static class StreamReaderBuilder { private TreeReaderFactory.Context context; private List vectors; - public StreamReaderBuilder setColumnIndex(int columnIndex) { this.columnIndex = columnIndex; return this; @@ -657,17 +645,14 @@ public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { } public LongStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); boolean isFileCompressed = compressionCodec != null; - return new LongStreamReader(columnIndex, present, data, isFileCompressed, - columnEncoding, context, vectors); + return new LongStreamReader(columnIndex, present, data, isFileCompressed, columnEncoding, context, vectors); } public StreamReaderBuilder setVectors(List vectors) { @@ -687,11 +672,10 @@ protected static class IntStreamReader extends IntTreeReader implements Settable private SettableUncompressedStream _presentStream; private SettableUncompressedStream _dataStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; - private IntStreamReader(int columnId, SettableUncompressedStream present, - SettableUncompressedStream data, boolean isFileCompressed, - OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, + private IntStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, + boolean isFileCompressed, OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, List vectors) throws IOException { super(columnId, present, data, encoding, context); this._isFileCompressed = isFileCompressed; @@ -701,8 +685,9 @@ private IntStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); @@ -721,10 +706,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -734,8 +719,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -786,17 +770,14 @@ public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { } public IntStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); boolean isFileCompressed = compressionCodec != null; - return new IntStreamReader(columnIndex, present, data, isFileCompressed, - columnEncoding, context, vectors); + return new IntStreamReader(columnIndex, present, data, isFileCompressed, columnEncoding, context, vectors); } public StreamReaderBuilder setVectors(List vectors) { @@ -817,12 +798,11 @@ protected static class FloatStreamReader extends FloatTreeReader implements Sett private SettableUncompressedStream _presentStream; private SettableUncompressedStream _dataStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; - private FloatStreamReader(int columnId, SettableUncompressedStream present, - SettableUncompressedStream data, boolean isFileCompressed, - List vectors) throws IOException { - super(columnId, present, data); + private FloatStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, + boolean isFileCompressed, List vectors, TreeReaderFactory.Context context) throws IOException { + super(columnId, present, data, context); this._isFileCompressed = isFileCompressed; this._presentStream = present; this._dataStream = data; @@ -830,8 +810,9 @@ private FloatStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); @@ -848,10 +829,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -861,8 +842,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -879,6 +859,7 @@ public static class StreamReaderBuilder { private ColumnStreamData dataStream; private CompressionCodec compressionCodec; private List vectors; + private Context context; public StreamReaderBuilder setColumnIndex(int columnIndex) { this.columnIndex = columnIndex; @@ -901,16 +882,19 @@ public StreamReaderBuilder setCompressionCodec(CompressionCodec compressionCodec } public FloatStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); boolean isFileCompressed = compressionCodec != null; - return new FloatStreamReader(columnIndex, present, data, isFileCompressed, vectors); + return new FloatStreamReader(columnIndex, present, data, isFileCompressed, vectors, context); + } + + public StreamReaderBuilder setContext(Context context) { + this.context = context; + return this; } public StreamReaderBuilder setVectors(List vectors) { @@ -930,12 +914,11 @@ protected static class DoubleStreamReader extends DoubleTreeReader implements Se private SettableUncompressedStream _presentStream; private SettableUncompressedStream _dataStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; - private DoubleStreamReader(int columnId, SettableUncompressedStream present, - SettableUncompressedStream data, boolean isFileCompressed, - List vectors) throws IOException { - super(columnId, present, data); + private DoubleStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, + boolean isFileCompressed, List vectors, TreeReaderFactory.Context context) throws IOException { + super(columnId, present, data, context); this._isFileCompressed = isFileCompressed; this._presentStream = present; this._dataStream = data; @@ -943,8 +926,9 @@ private DoubleStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); @@ -961,10 +945,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -974,8 +958,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -1020,17 +1003,15 @@ public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { } public DoubleStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); boolean isFileCompressed = compressionCodec != null; // TODO: why doesn't this use context? - return new DoubleStreamReader(columnIndex, present, data, isFileCompressed, vectors); + return new DoubleStreamReader(columnIndex, present, data, isFileCompressed, vectors, context); } public StreamReaderBuilder setVectors(List vectors) { @@ -1050,16 +1031,13 @@ protected static class DecimalStreamReader extends DecimalTreeReader implements private SettableUncompressedStream _valueStream; private SettableUncompressedStream _scaleStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; - private DecimalStreamReader(int columnId, int precision, int scale, - SettableUncompressedStream presentStream, - SettableUncompressedStream valueStream, SettableUncompressedStream scaleStream, - boolean isFileCompressed, - OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, - List vectors) throws IOException { - super(columnId, presentStream, valueStream, scaleStream, encoding, - precision, scale, context); + private DecimalStreamReader(int columnId, int precision, int scale, SettableUncompressedStream presentStream, + SettableUncompressedStream valueStream, SettableUncompressedStream scaleStream, boolean isFileCompressed, + OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, List vectors) + throws IOException { + super(columnId, presentStream, valueStream, scaleStream, encoding, precision, scale, context); this._isFileCompressed = isFileCompressed; this._presentStream = presentStream; this._valueStream = valueStream; @@ -1068,8 +1046,9 @@ private DecimalStreamReader(int columnId, int precision, int scale, } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); @@ -1095,10 +1074,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1108,8 +1087,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -1191,9 +1169,8 @@ public DecimalStreamReader build() throws IOException { OrcProto.Stream.Kind.SECONDARY.name(), scaleStream); boolean isFileCompressed = compressionCodec != null; - return new DecimalStreamReader(columnIndex, precision, scale, presentInStream, - valueInStream, - scaleInStream, isFileCompressed, columnEncoding, context, vectors); + return new DecimalStreamReader(columnIndex, precision, scale, presentInStream, valueInStream, scaleInStream, + isFileCompressed, columnEncoding, context, vectors); } public StreamReaderBuilder setVectors(List vectors) { @@ -1214,14 +1191,10 @@ protected static class Decimal64StreamReader extends Decimal64TreeReader impleme private List vectors; private int vectorIndex = 0; - private Decimal64StreamReader(int columnId, int precision, int scale, - SettableUncompressedStream presentStream, - SettableUncompressedStream valueStream, - boolean isFileCompressed, - OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, - List vectors) throws IOException { - super(columnId, presentStream, valueStream, encoding, - precision, scale, context); + private Decimal64StreamReader(int columnId, int precision, int scale, SettableUncompressedStream presentStream, + SettableUncompressedStream valueStream, boolean isFileCompressed, OrcProto.ColumnEncoding encoding, + TreeReaderFactory.Context context, List vectors) throws IOException { + super(columnId, presentStream, valueStream, encoding, precision, scale, context); this._isFileCompressed = isFileCompressed; this._presentStream = presentStream; this._valueStream = valueStream; @@ -1229,8 +1202,9 @@ private Decimal64StreamReader(int columnId, int precision, int scale, } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); @@ -1249,10 +1223,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1314,7 +1288,6 @@ public StreamReaderBuilder setValueStream(ColumnStreamData valueStream) { return this; } - public StreamReaderBuilder setCompressionCodec(CompressionCodec compressionCodec) { this.compressionCodec = compressionCodec; return this; @@ -1327,14 +1300,14 @@ public StreamReaderBuilder setColumnEncoding(OrcProto.ColumnEncoding encoding) { public Decimal64StreamReader build() throws IOException { SettableUncompressedStream presentInStream = StreamUtils.createSettableUncompressedStream( - OrcProto.Stream.Kind.PRESENT.name(), presentStream); + OrcProto.Stream.Kind.PRESENT.name(), presentStream); SettableUncompressedStream valueInStream = StreamUtils.createSettableUncompressedStream( - OrcProto.Stream.Kind.DATA.name(), valueStream); + OrcProto.Stream.Kind.DATA.name(), valueStream); boolean isFileCompressed = compressionCodec != null; - return new Decimal64StreamReader(columnIndex, precision, scale, presentInStream, - valueInStream, isFileCompressed, columnEncoding, context, vectors); + return new Decimal64StreamReader(columnIndex, precision, scale, presentInStream, valueInStream, + isFileCompressed, columnEncoding, context, vectors); } public StreamReaderBuilder setVectors(List vectors) { @@ -1353,12 +1326,11 @@ protected static class DateStreamReader extends DateTreeReader implements Settab private SettableUncompressedStream _presentStream; private SettableUncompressedStream _dataStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; - private DateStreamReader(int columnId, SettableUncompressedStream present, - SettableUncompressedStream data, boolean isFileCompressed, - OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, List vectors - ) throws IOException { + private DateStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, + boolean isFileCompressed, OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, + List vectors) throws IOException { super(columnId, present, data, encoding, context); this.isFileCompressed = isFileCompressed; this._presentStream = present; @@ -1367,8 +1339,9 @@ private DateStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (isFileCompressed) { index.getNext(); @@ -1387,10 +1360,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1400,8 +1373,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -1457,18 +1429,14 @@ public StreamReaderBuilder setVectors(List vectors) { } public DateStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); - + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); boolean isFileCompressed = compressionCodec != null; - return new DateStreamReader(columnIndex, present, data, isFileCompressed, - columnEncoding, context, vectors); + return new DateStreamReader(columnIndex, present, data, isFileCompressed, columnEncoding, context, vectors); } } @@ -1485,15 +1453,13 @@ protected static class CharStreamReader extends CharTreeReader implements Settab private SettableUncompressedStream _lengthStream; private SettableUncompressedStream _dictionaryStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; - private CharStreamReader(int columnId, int maxLength, - SettableUncompressedStream present, SettableUncompressedStream data, - SettableUncompressedStream length, SettableUncompressedStream dictionary, - boolean isFileCompressed, OrcProto.ColumnEncoding encoding, - List vectors) throws IOException { - super(columnId, maxLength, present, data, length, - dictionary, encoding); + private CharStreamReader(int columnId, int maxLength, SettableUncompressedStream present, + SettableUncompressedStream data, SettableUncompressedStream length, SettableUncompressedStream dictionary, + boolean isFileCompressed, OrcProto.ColumnEncoding encoding, List vectors, Context context) + throws IOException { + super(columnId, maxLength, present, data, length, dictionary, encoding, context); this._isDictionaryEncoding = dictionary != null; this._isFileCompressed = isFileCompressed; this._presentStream = present; @@ -1504,19 +1470,20 @@ private CharStreamReader(int columnId, int maxLength, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { // This string reader should simply redirect to its own seek (what other types already do). - this.seek(index[columnId]); + this.seek(index[columnId], readPhase); } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); } - reader.getPresent().seek(index); + ((TreeReader) reader).getPresent().seek(index); } if (_isDictionaryEncoding) { @@ -1551,10 +1518,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1564,8 +1531,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -1586,7 +1552,8 @@ public void setBuffers(EncodedColumnBatch batch, boolean sameStripe _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE])); } if (_dictionaryStream != null) { - _dictionaryStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DICTIONARY_DATA_VALUE])); + _dictionaryStream.setBuffers( + StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DICTIONARY_DATA_VALUE])); } } } @@ -1601,6 +1568,7 @@ public static class StreamReaderBuilder { private CompressionCodec compressionCodec; private OrcProto.ColumnEncoding columnEncoding; private List vectors; + private Context context; public StreamReaderBuilder setColumnIndex(int columnIndex) { this.columnIndex = columnIndex; @@ -1642,25 +1610,27 @@ public StreamReaderBuilder setColumnEncoding(OrcProto.ColumnEncoding encoding) { return this; } + public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { + this.context = context; + return this; + } + public CharStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); - SettableUncompressedStream length = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.LENGTH.name(), - lengthStream); + SettableUncompressedStream length = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.LENGTH.name(), lengthStream); SettableUncompressedStream dictionary = StreamUtils.createSettableUncompressedStream( OrcProto.Stream.Kind.DICTIONARY_DATA.name(), dictionaryStream); boolean isFileCompressed = compressionCodec != null; - return new CharStreamReader(columnIndex, maxLength, present, data, length, - dictionary, isFileCompressed, columnEncoding, vectors); + return new CharStreamReader(columnIndex, maxLength, present, data, length, dictionary, isFileCompressed, + columnEncoding, vectors, context); } public StreamReaderBuilder setVectors(List vectors) { @@ -1683,15 +1653,13 @@ protected static class VarcharStreamReader extends VarcharTreeReader implements private SettableUncompressedStream _lengthStream; private SettableUncompressedStream _dictionaryStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; - private VarcharStreamReader(int columnId, int maxLength, - SettableUncompressedStream present, SettableUncompressedStream data, - SettableUncompressedStream length, SettableUncompressedStream dictionary, - boolean isFileCompressed, OrcProto.ColumnEncoding encoding, - List vectors) throws IOException { - super(columnId, maxLength, present, data, length, - dictionary, encoding); + private VarcharStreamReader(int columnId, int maxLength, SettableUncompressedStream present, + SettableUncompressedStream data, SettableUncompressedStream length, SettableUncompressedStream dictionary, + boolean isFileCompressed, OrcProto.ColumnEncoding encoding, List vectors, Context context) + throws IOException { + super(columnId, maxLength, present, data, length, dictionary, encoding, context); this._isDictionaryEncoding = dictionary != null; this._isFileCompressed = isFileCompressed; this._presentStream = present; @@ -1702,19 +1670,20 @@ private VarcharStreamReader(int columnId, int maxLength, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { // This string reader should simply redirect to its own seek (what other types already do). - this.seek(index[columnId]); + this.seek(index[columnId], readPhase); } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); } - reader.getPresent().seek(index); + ((TreeReader) reader).getPresent().seek(index); } if (_isDictionaryEncoding) { @@ -1749,10 +1718,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1762,8 +1731,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -1784,7 +1752,8 @@ public void setBuffers(EncodedColumnBatch batch, boolean sameStripe _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE])); } if (_dictionaryStream != null) { - _dictionaryStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DICTIONARY_DATA_VALUE])); + _dictionaryStream.setBuffers( + StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DICTIONARY_DATA_VALUE])); } } } @@ -1799,6 +1768,7 @@ public static class StreamReaderBuilder { private CompressionCodec compressionCodec; private OrcProto.ColumnEncoding columnEncoding; private List vectors; + private Context context; public StreamReaderBuilder setColumnIndex(int columnIndex) { this.columnIndex = columnIndex; @@ -1840,25 +1810,27 @@ public StreamReaderBuilder setColumnEncoding(OrcProto.ColumnEncoding encoding) { return this; } + public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { + this.context = context; + return this; + } + public VarcharStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); - SettableUncompressedStream length = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.LENGTH.name(), - lengthStream); + SettableUncompressedStream length = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.LENGTH.name(), lengthStream); SettableUncompressedStream dictionary = StreamUtils.createSettableUncompressedStream( OrcProto.Stream.Kind.DICTIONARY_DATA.name(), dictionaryStream); boolean isFileCompressed = compressionCodec != null; - return new VarcharStreamReader(columnIndex, maxLength, present, data, length, - dictionary, isFileCompressed, columnEncoding, vectors); + return new VarcharStreamReader(columnIndex, maxLength, present, data, length, dictionary, isFileCompressed, + columnEncoding, vectors, context); } public StreamReaderBuilder setVectors(List vectors) { @@ -1878,12 +1850,12 @@ protected static class ByteStreamReader extends ByteTreeReader implements Settab private SettableUncompressedStream _presentStream; private SettableUncompressedStream _dataStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; - private ByteStreamReader(int columnId, SettableUncompressedStream present, - SettableUncompressedStream data, boolean isFileCompressed, - List vectors) throws IOException { - super(columnId, present, data); + private ByteStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, + boolean isFileCompressed, List vectors, Context context) throws IOException { + super(columnId, present, data, context); + ; this._isFileCompressed = isFileCompressed; this._presentStream = present; this._dataStream = data; @@ -1891,8 +1863,9 @@ private ByteStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); @@ -1911,10 +1884,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1924,8 +1897,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -1942,6 +1914,7 @@ public static class StreamReaderBuilder { private ColumnStreamData dataStream; private CompressionCodec compressionCodec; private List vectors; + private TreeReaderFactory.Context context; public StreamReaderBuilder setColumnIndex(int columnIndex) { this.columnIndex = columnIndex; @@ -1963,17 +1936,20 @@ public StreamReaderBuilder setCompressionCodec(CompressionCodec compressionCodec return this; } + public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { + this.context = context; + return this; + } + public ByteStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); boolean isFileCompressed = compressionCodec != null; - return new ByteStreamReader(columnIndex, present, data, isFileCompressed, vectors); + return new ByteStreamReader(columnIndex, present, data, isFileCompressed, vectors, context); } public StreamReaderBuilder setVectors(List vectors) { @@ -1993,12 +1969,11 @@ protected static class BinaryStreamReader extends BinaryTreeReader implements Se private SettableUncompressedStream _dataStream; private SettableUncompressedStream _lengthsStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; - private BinaryStreamReader(int columnId, SettableUncompressedStream present, - SettableUncompressedStream data, SettableUncompressedStream length, - boolean isFileCompressed, - OrcProto.ColumnEncoding encoding, TreeReaderFactory.Context context, List vectors) throws IOException { + private BinaryStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, + SettableUncompressedStream length, boolean isFileCompressed, OrcProto.ColumnEncoding encoding, + TreeReaderFactory.Context context, List vectors) throws IOException { super(columnId, present, data, length, encoding, context); this._isFileCompressed = isFileCompressed; this._presentStream = present; @@ -2008,10 +1983,10 @@ private BinaryStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -2021,8 +1996,9 @@ public void nextVector( } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); @@ -2048,8 +2024,7 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -2112,15 +2087,15 @@ public BinaryStreamReader build() throws IOException { SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream( - OrcProto.Stream.Kind.DATA.name(), dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); SettableUncompressedStream length = StreamUtils.createSettableUncompressedStream( OrcProto.Stream.Kind.LENGTH.name(), lengthStream); boolean isFileCompressed = compressionCodec != null; - return new BinaryStreamReader(columnIndex, present, data, length, isFileCompressed, - columnEncoding, context, vectors); + return new BinaryStreamReader(columnIndex, present, data, length, isFileCompressed, columnEncoding, context, + vectors); } public StreamReaderBuilder setVectors(List vectors) { @@ -2141,10 +2116,9 @@ protected static class BooleanStreamReader extends BooleanTreeReader implements private List vectors; private int vectorIndex = 0; - private BooleanStreamReader(int columnId, SettableUncompressedStream present, - SettableUncompressedStream data, boolean isFileCompressed, - List vectors) throws IOException { - super(columnId, present, data); + private BooleanStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, + boolean isFileCompressed, List vectors, Context context) throws IOException { + super(columnId, present, data, context); this._isFileCompressed = isFileCompressed; this._presentStream = present; this._dataStream = data; @@ -2152,8 +2126,9 @@ private BooleanStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { - if (vectors != null) return; + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + if (vectors != null) + return; if (present != null) { if (_isFileCompressed) { index.getNext(); @@ -2172,10 +2147,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -2185,8 +2160,7 @@ public void nextVector( } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { assert vectors == null; // See the comment in TimestampStreamReader.setBuffers. ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { @@ -2203,6 +2177,7 @@ public static class StreamReaderBuilder { private ColumnStreamData dataStream; private CompressionCodec compressionCodec; private List vectors; + private Context context; public StreamReaderBuilder setColumnIndex(int columnIndex) { this.columnIndex = columnIndex; @@ -2224,17 +2199,20 @@ public StreamReaderBuilder setCompressionCodec(CompressionCodec compressionCodec return this; } + public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { + this.context = context; + return this; + } + public BooleanStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), - dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); boolean isFileCompressed = compressionCodec != null; - return new BooleanStreamReader(columnIndex, present, data, isFileCompressed, vectors); + return new BooleanStreamReader(columnIndex, present, data, isFileCompressed, vectors, context); } public StreamReaderBuilder setVectors(List vectors) { @@ -2249,15 +2227,15 @@ public static StreamReaderBuilder builder() { } public static StructTreeReader createRootTreeReader(TypeDescription[] batchSchemas, - List encodings, OrcEncodedColumnBatch batch, - CompressionCodec codec, Context context, final boolean useDecimal64ColumnVectors) throws IOException { + List encodings, OrcEncodedColumnBatch batch, CompressionCodec codec, Context context, + final boolean useDecimal64ColumnVectors) throws IOException { // Note: we only look at the schema here to deal with complex types. Somebody has set up the // reader with whatever ideas they had to the schema and we just trust the reader to // produce the CVBs that was asked for. However, we only need to look at top level columns. int includedCount = batch.getColumnsWithDataCount(); if (batchSchemas.length > includedCount) { - throw new AssertionError("For " + Arrays.toString(batchSchemas) + ", only received " - + includedCount + " columns"); + throw new AssertionError( + "For " + Arrays.toString(batchSchemas) + ", only received " + includedCount + " columns"); } TreeReader[] childReaders = new TreeReader[batchSchemas.length]; for (int i = 0; i < batchSchemas.length; ++i) { @@ -2265,29 +2243,23 @@ public static StructTreeReader createRootTreeReader(TypeDescription[] batchSchem if (!batch.hasData(batchColIx) && !batch.hasVectors(batchColIx)) { throw new AssertionError("No data for column " + batchColIx + ": " + batchSchemas[i]); } - childReaders[i] = createEncodedTreeReader(batchSchemas[i], encodings, batch, codec, context, useDecimal64ColumnVectors); + childReaders[i] = createEncodedTreeReader(batchSchemas[i], encodings, batch, codec, context, + useDecimal64ColumnVectors); } // TODO: do we actually need this reader? the caller just extracts child readers. - return StructStreamReader.builder() - .setColumnIndex(0) - .setCompressionCodec(codec) - .setColumnEncoding(encodings.get(0)) - .setChildReaders(childReaders) - .setContext(context) - .build(); + return StructStreamReader.builder().setColumnIndex(0).setCompressionCodec(codec).setColumnEncoding(encodings.get(0)) + .setChildReaders(childReaders).setContext(context).build(); } - private static void skipSeek(PositionProvider index) { // Must be consistent with uncompressed stream seek in ORC. See call site comments. index.getNext(); } - - private static TreeReader createEncodedTreeReader(TypeDescription schema, - List encodings, OrcEncodedColumnBatch batch, - CompressionCodec codec, Context context, final boolean useDecimal64ColumnVectors) throws IOException { + private static TreeReader createEncodedTreeReader(TypeDescription schema, List encodings, + OrcEncodedColumnBatch batch, CompressionCodec codec, Context context, final boolean useDecimal64ColumnVectors) + throws IOException { int columnIndex = schema.getId(); ColumnStreamData[] streamBuffers = null; List vectors = null; @@ -2310,8 +2282,7 @@ private static TreeReader createEncodedTreeReader(TypeDescription schema, OrcProto.ColumnEncoding columnEncoding = encodings.get(columnIndex); // stream buffers are arranged in enum order of stream kind - ColumnStreamData present = null, data = null, dictionary = null, - lengths = null, secondary = null; + ColumnStreamData present = null, data = null, dictionary = null, lengths = null, secondary = null; if (streamBuffers != null) { present = streamBuffers[OrcProto.Stream.Kind.PRESENT_VALUE]; data = streamBuffers[OrcProto.Stream.Kind.DATA_VALUE]; @@ -2320,286 +2291,156 @@ private static TreeReader createEncodedTreeReader(TypeDescription schema, secondary = streamBuffers[OrcProto.Stream.Kind.SECONDARY_VALUE]; } - if (LOG.isDebugEnabled()) { - LOG.debug("columnIndex: {} columnType: {} streamBuffers.length: {} vectors: {} columnEncoding: {}" + - " present: {} data: {} dictionary: {} lengths: {} secondary: {} tz: {}", - columnIndex, schema, streamBuffers == null ? 0 : streamBuffers.length, - vectors == null ? 0 : vectors.size(), columnEncoding, present != null, - data, dictionary != null, lengths != null, secondary != null, + LOG.debug( + "columnIndex: {} columnType: {} streamBuffers.length: {} vectors: {} columnEncoding: {}" + " present: {} data: {} dictionary: {} lengths: {} secondary: {} tz: {}", + columnIndex, schema, streamBuffers == null ? 0 : streamBuffers.length, vectors == null ? 0 : vectors.size(), + columnEncoding, present != null, data, dictionary != null, lengths != null, secondary != null, context.getWriterTimezone()); } // TODO: get rid of the builders - they serve no purpose... just call ctors directly. switch (schema.getCategory()) { - case BINARY: - case BOOLEAN: - case BYTE: - case SHORT: - case INT: - case LONG: - case FLOAT: - case DOUBLE: - case CHAR: - case VARCHAR: - case STRING: - case DECIMAL: - case TIMESTAMP: - case DATE: - return getPrimitiveTreeReader(columnIndex, schema, codec, columnEncoding, - present, data, dictionary, lengths, secondary, context, vectors, useDecimal64ColumnVectors); - case LIST: - assert vectors == null; // Not currently supported. - TypeDescription elementType = schema.getChildren().get(0); - TreeReader elementReader = createEncodedTreeReader( - elementType, encodings, batch, codec, context, useDecimal64ColumnVectors); - return ListStreamReader.builder() - .setColumnIndex(columnIndex) - .setColumnEncoding(columnEncoding) - .setCompressionCodec(codec) - .setPresentStream(present) - .setLengthStream(lengths) - .setElementReader(elementReader) - .setContext(context) - .build(); - case MAP: - assert vectors == null; // Not currently supported. - TypeDescription keyType = schema.getChildren().get(0); - TypeDescription valueType = schema.getChildren().get(1); - TreeReader keyReader = createEncodedTreeReader( - keyType, encodings, batch, codec, context, useDecimal64ColumnVectors); - TreeReader valueReader = createEncodedTreeReader( - valueType, encodings, batch, codec, context, useDecimal64ColumnVectors); - return MapStreamReader.builder() - .setColumnIndex(columnIndex) - .setColumnEncoding(columnEncoding) - .setCompressionCodec(codec) - .setPresentStream(present) - .setLengthStream(lengths) - .setKeyReader(keyReader) - .setValueReader(valueReader) - .setContext(context) - .build(); - case STRUCT: { - assert vectors == null; // Not currently supported. - int childCount = schema.getChildren().size(); - TreeReader[] childReaders = new TreeReader[childCount]; - for (int i = 0; i < childCount; i++) { - TypeDescription childType = schema.getChildren().get(i); - childReaders[i] = createEncodedTreeReader( - childType, encodings, batch, codec, context, useDecimal64ColumnVectors); - } - return StructStreamReader.builder() - .setColumnIndex(columnIndex) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setPresentStream(present) - .setChildReaders(childReaders) - .setContext(context) - .build(); - } - case UNION: { - assert vectors == null; // Not currently supported. - int childCount = schema.getChildren().size(); - TreeReader[] childReaders = new TreeReader[childCount]; - for (int i = 0; i < childCount; i++) { - TypeDescription childType = schema.getChildren().get(i); - childReaders[i] = createEncodedTreeReader( - childType, encodings, batch, codec, context, useDecimal64ColumnVectors); - } - return UnionStreamReader.builder() - .setColumnIndex(columnIndex) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setPresentStream(present) - .setDataStream(data) - .setChildReaders(childReaders) - .setContext(context) - .build(); - } - default: - throw new UnsupportedOperationException("Data type not supported: " + schema); + case BINARY: + case BOOLEAN: + case BYTE: + case SHORT: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case CHAR: + case VARCHAR: + case STRING: + case DECIMAL: + case TIMESTAMP: + case DATE: + return getPrimitiveTreeReader(columnIndex, schema, codec, columnEncoding, present, data, dictionary, lengths, + secondary, context, vectors, useDecimal64ColumnVectors); + case LIST: + assert vectors == null; // Not currently supported. + TypeDescription elementType = schema.getChildren().get(0); + TreeReader elementReader = createEncodedTreeReader(elementType, encodings, batch, codec, context, + useDecimal64ColumnVectors); + return ListStreamReader.builder().setColumnIndex(columnIndex).setColumnEncoding(columnEncoding) + .setCompressionCodec(codec).setPresentStream(present).setLengthStream(lengths).setElementReader(elementReader) + .setContext(context).build(); + case MAP: + assert vectors == null; // Not currently supported. + TypeDescription keyType = schema.getChildren().get(0); + TypeDescription valueType = schema.getChildren().get(1); + TreeReader keyReader = createEncodedTreeReader(keyType, encodings, batch, codec, context, + useDecimal64ColumnVectors); + TreeReader valueReader = createEncodedTreeReader(valueType, encodings, batch, codec, context, + useDecimal64ColumnVectors); + return MapStreamReader.builder().setColumnIndex(columnIndex).setColumnEncoding(columnEncoding) + .setCompressionCodec(codec).setPresentStream(present).setLengthStream(lengths).setKeyReader(keyReader) + .setValueReader(valueReader).setContext(context).build(); + case STRUCT: { + assert vectors == null; // Not currently supported. + int childCount = schema.getChildren().size(); + TreeReader[] childReaders = new TreeReader[childCount]; + for (int i = 0; i < childCount; i++) { + TypeDescription childType = schema.getChildren().get(i); + childReaders[i] = createEncodedTreeReader(childType, encodings, batch, codec, context, + useDecimal64ColumnVectors); + } + return StructStreamReader.builder().setColumnIndex(columnIndex).setCompressionCodec(codec) + .setColumnEncoding(columnEncoding).setPresentStream(present).setChildReaders(childReaders).setContext(context) + .build(); + } + case UNION: { + assert vectors == null; // Not currently supported. + int childCount = schema.getChildren().size(); + TreeReader[] childReaders = new TreeReader[childCount]; + for (int i = 0; i < childCount; i++) { + TypeDescription childType = schema.getChildren().get(i); + childReaders[i] = createEncodedTreeReader(childType, encodings, batch, codec, context, + useDecimal64ColumnVectors); + } + return UnionStreamReader.builder().setColumnIndex(columnIndex).setCompressionCodec(codec) + .setColumnEncoding(columnEncoding).setPresentStream(present).setDataStream(data).setChildReaders(childReaders) + .setContext(context).build(); + } + default: + throw new UnsupportedOperationException("Data type not supported: " + schema); } } - private static TreeReader getPrimitiveTreeReader(final int columnIndex, - TypeDescription columnType, CompressionCodec codec, OrcProto.ColumnEncoding columnEncoding, - ColumnStreamData present, ColumnStreamData data, ColumnStreamData dictionary, - ColumnStreamData lengths, ColumnStreamData secondary, Context context, - List vectors, final boolean useDecimal64ColumnVectors) throws IOException { + private static TreeReader getPrimitiveTreeReader(final int columnIndex, TypeDescription columnType, + CompressionCodec codec, OrcProto.ColumnEncoding columnEncoding, ColumnStreamData present, ColumnStreamData data, + ColumnStreamData dictionary, ColumnStreamData lengths, ColumnStreamData secondary, Context context, + List vectors, final boolean useDecimal64ColumnVectors) throws IOException { switch (columnType.getCategory()) { - case BINARY: - return BinaryStreamReader.builder() - .setColumnIndex(columnIndex) - .setPresentStream(present) - .setDataStream(data) - .setLengthStream(lengths) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setVectors(vectors) - .setContext(context) - .build(); - case BOOLEAN: - return BooleanStreamReader.builder() - .setColumnIndex(columnIndex) - .setPresentStream(present) - .setDataStream(data) - .setCompressionCodec(codec) - .setVectors(vectors) - .build(); - case BYTE: - return ByteStreamReader.builder() - .setColumnIndex(columnIndex) - .setPresentStream(present) - .setDataStream(data) - .setCompressionCodec(codec) - .setVectors(vectors) - .build(); - case SHORT: - return ShortStreamReader.builder() - .setColumnIndex(columnIndex) - .setPresentStream(present) - .setDataStream(data) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setVectors(vectors) - .setContext(context) - .build(); - case INT: - return IntStreamReader.builder() - .setColumnIndex(columnIndex) - .setPresentStream(present) - .setDataStream(data) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setVectors(vectors) - .setContext(context) - .build(); - case LONG: - return LongStreamReader.builder() - .setColumnIndex(columnIndex) - .setPresentStream(present) - .setDataStream(data) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setVectors(vectors) - .setContext(context) - .build(); - case FLOAT: - return FloatStreamReader.builder() - .setColumnIndex(columnIndex) - .setPresentStream(present) - .setDataStream(data) - .setCompressionCodec(codec) - .setVectors(vectors) - .build(); - case DOUBLE: - return DoubleStreamReader.builder() - .setColumnIndex(columnIndex) - .setPresentStream(present) - .setDataStream(data) - .setCompressionCodec(codec) - .setVectors(vectors) - .build(); - case CHAR: - return CharStreamReader.builder() - .setColumnIndex(columnIndex) - .setMaxLength(columnType.getMaxLength()) - .setPresentStream(present) - .setDataStream(data) - .setLengthStream(lengths) - .setDictionaryStream(dictionary) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setVectors(vectors) - .build(); - case VARCHAR: - return VarcharStreamReader.builder() - .setColumnIndex(columnIndex) - .setMaxLength(columnType.getMaxLength()) - .setPresentStream(present) - .setDataStream(data) - .setLengthStream(lengths) - .setDictionaryStream(dictionary) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setVectors(vectors) - .build(); - case STRING: - return StringStreamReader.builder() - .setColumnIndex(columnIndex) - .setPresentStream(present) - .setDataStream(data) - .setLengthStream(lengths) - .setDictionaryStream(dictionary) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setVectors(vectors) - .build(); - case DECIMAL: - // special handling for serde reader (text) in llap IO. - // if file format version is null, then we are processing text IF in LLAP IO, in which case - // we get vectors instead of streams. If vectors contain instance of Decimal64ColumnVector we - // should use Decimal64StreamReader (which acts as a wrapper around vectors) - boolean useDecimal64Reader = context.getFileFormat() == null && vectors != null && useDecimal64ColumnVectors; - if (useDecimal64Reader) { - boolean containDecimal64CV = false; - for (ColumnVector vector : vectors) { - if (vector instanceof Decimal64ColumnVector) { - containDecimal64CV = true; - break; - } + case BINARY: + return BinaryStreamReader.builder().setColumnIndex(columnIndex).setPresentStream(present).setDataStream(data) + .setLengthStream(lengths).setCompressionCodec(codec).setColumnEncoding(columnEncoding).setVectors(vectors) + .setContext(context).build(); + case BOOLEAN: + return BooleanStreamReader.builder().setColumnIndex(columnIndex).setPresentStream(present).setDataStream(data) + .setCompressionCodec(codec).setVectors(vectors).setContext(context).build(); + case BYTE: + return ByteStreamReader.builder().setColumnIndex(columnIndex).setPresentStream(present).setDataStream(data) + .setCompressionCodec(codec).setVectors(vectors).setContext(context).build(); + case SHORT: + return ShortStreamReader.builder().setColumnIndex(columnIndex).setPresentStream(present).setDataStream(data) + .setCompressionCodec(codec).setColumnEncoding(columnEncoding).setVectors(vectors).setContext(context).build(); + case INT: + return IntStreamReader.builder().setColumnIndex(columnIndex).setPresentStream(present).setDataStream(data) + .setCompressionCodec(codec).setColumnEncoding(columnEncoding).setVectors(vectors).setContext(context).build(); + case LONG: + return LongStreamReader.builder().setColumnIndex(columnIndex).setPresentStream(present).setDataStream(data) + .setCompressionCodec(codec).setColumnEncoding(columnEncoding).setVectors(vectors).setContext(context).build(); + case FLOAT: + return FloatStreamReader.builder().setColumnIndex(columnIndex).setPresentStream(present).setDataStream(data) + .setCompressionCodec(codec).setVectors(vectors).setContext(context).build(); + case DOUBLE: + return DoubleStreamReader.builder().setColumnIndex(columnIndex).setPresentStream(present).setDataStream(data) + .setCompressionCodec(codec).setVectors(vectors).setContext(context).build(); + case CHAR: + return CharStreamReader.builder().setColumnIndex(columnIndex).setMaxLength(columnType.getMaxLength()) + .setPresentStream(present).setDataStream(data).setLengthStream(lengths).setDictionaryStream(dictionary) + .setCompressionCodec(codec).setColumnEncoding(columnEncoding).setVectors(vectors).setContext(context).build(); + case VARCHAR: + return VarcharStreamReader.builder().setColumnIndex(columnIndex).setMaxLength(columnType.getMaxLength()) + .setPresentStream(present).setDataStream(data).setLengthStream(lengths).setDictionaryStream(dictionary) + .setCompressionCodec(codec).setColumnEncoding(columnEncoding).setVectors(vectors).setContext(context).build(); + case STRING: + return StringStreamReader.builder().setColumnIndex(columnIndex).setPresentStream(present).setDataStream(data) + .setLengthStream(lengths).setDictionaryStream(dictionary).setCompressionCodec(codec) + .setColumnEncoding(columnEncoding).setVectors(vectors).setContext(context).build(); + case DECIMAL: + // special handling for serde reader (text) in llap IO. + // if file format version is null, then we are processing text IF in LLAP IO, in which case + // we get vectors instead of streams. If vectors contain instance of Decimal64ColumnVector we + // should use Decimal64StreamReader (which acts as a wrapper around vectors) + boolean useDecimal64Reader = context.getFileFormat() == null && vectors != null && useDecimal64ColumnVectors; + if (useDecimal64Reader) { + boolean containDecimal64CV = false; + for (ColumnVector vector : vectors) { + if (vector instanceof Decimal64ColumnVector) { + containDecimal64CV = true; + break; } - useDecimal64Reader &= containDecimal64CV; - } - if ((context.getFileFormat() == OrcFile.Version.UNSTABLE_PRE_2_0 || useDecimal64Reader) && - columnType.getPrecision() <= TypeDescription.MAX_DECIMAL64_PRECISION) { - return Decimal64StreamReader.builder() - .setColumnIndex(columnIndex) - .setPrecision(columnType.getPrecision()) - .setScale(columnType.getScale()) - .setPresentStream(present) - .setValueStream(data) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setVectors(vectors) - .setContext(context) - .build(); - } else { - return DecimalStreamReader.builder() - .setColumnIndex(columnIndex) - .setPrecision(columnType.getPrecision()) - .setScale(columnType.getScale()) - .setPresentStream(present) - .setValueStream(data) - .setScaleStream(secondary) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setVectors(vectors) - .setContext(context) - .build(); } - case TIMESTAMP: - return TimestampStreamReader.builder() - .setColumnIndex(columnIndex) - .setPresentStream(present) - .setSecondsStream(data) - .setNanosStream(secondary) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setVectors(vectors) - .setContext(context) - .setIsInstant(false) - .build(); - case DATE: - return DateStreamReader.builder() - .setColumnIndex(columnIndex) - .setPresentStream(present) - .setDataStream(data) - .setCompressionCodec(codec) - .setColumnEncoding(columnEncoding) - .setVectors(vectors) - .setContext(context) + useDecimal64Reader &= containDecimal64CV; + } + if ((context.getFileFormat() == OrcFile.Version.UNSTABLE_PRE_2_0 || useDecimal64Reader) && columnType.getPrecision() <= TypeDescription.MAX_DECIMAL64_PRECISION) { + return Decimal64StreamReader.builder().setColumnIndex(columnIndex).setPrecision(columnType.getPrecision()) + .setScale(columnType.getScale()).setPresentStream(present).setValueStream(data).setCompressionCodec(codec) + .setColumnEncoding(columnEncoding).setVectors(vectors).setContext(context).build(); + } else { + return DecimalStreamReader.builder().setColumnIndex(columnIndex).setPrecision(columnType.getPrecision()) + .setScale(columnType.getScale()).setPresentStream(present).setValueStream(data).setScaleStream(secondary) + .setCompressionCodec(codec).setColumnEncoding(columnEncoding).setVectors(vectors).setContext(context) .build(); + } + case TIMESTAMP: + return TimestampStreamReader.builder().setColumnIndex(columnIndex).setPresentStream(present) + .setSecondsStream(data).setNanosStream(secondary).setCompressionCodec(codec).setColumnEncoding(columnEncoding) + .setVectors(vectors).setContext(context).setIsInstant(false).build(); + case DATE: + return DateStreamReader.builder().setColumnIndex(columnIndex).setPresentStream(present).setDataStream(data) + .setCompressionCodec(codec).setColumnEncoding(columnEncoding).setVectors(vectors).setContext(context).build(); default: throw new AssertionError("Not a primitive category: " + columnType.getCategory()); } @@ -2610,11 +2451,9 @@ protected static class ListStreamReader extends ListTreeReader implements Settab private SettableUncompressedStream _presentStream; private SettableUncompressedStream _lengthStream; - public ListStreamReader(final int columnIndex, - final SettableUncompressedStream present, final SettableUncompressedStream lengthStream, - final OrcProto.ColumnEncoding columnEncoding, final boolean isFileCompressed, - final TreeReader elementReader, - TreeReaderFactory.Context context) throws IOException { + public ListStreamReader(final int columnIndex, final SettableUncompressedStream present, + final SettableUncompressedStream lengthStream, final OrcProto.ColumnEncoding columnEncoding, + final boolean isFileCompressed, final TreeReader elementReader, Context context) throws IOException { super(columnIndex, present, context, lengthStream, columnEncoding, elementReader); this._isFileCompressed = isFileCompressed; this._presentStream = present; @@ -2622,7 +2461,7 @@ public ListStreamReader(final int columnIndex, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { PositionProvider ownIndex = index[columnId]; if (present != null) { if (_isFileCompressed) { @@ -2638,27 +2477,24 @@ public void seek(PositionProvider[] index) throws IOException { ownIndex.getNext(); } lengths.seek(ownIndex); - elementReader.seek(index); + elementReader.seek(index, readPhase); } } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { // Only our parent class can call this. throw new IOException("Should never be called"); } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { - _presentStream.setBuffers( - StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE])); + _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE])); } if (_lengthStream != null) { - _lengthStream.setBuffers( - StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE])); + _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE])); } if (elementReader != null) { @@ -2705,23 +2541,21 @@ public ListStreamReader.StreamReaderBuilder setElementReader(TreeReader elementR return this; } - public ListStreamReader.StreamReaderBuilder setContext(TreeReaderFactory.Context context) { + public ListStreamReader.StreamReaderBuilder setContext(Context context) { this.context = context; return this; } public ListStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream length = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.LENGTH.name(), - lengthStream); + SettableUncompressedStream length = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.LENGTH.name(), lengthStream); boolean isFileCompressed = compressionCodec != null; - return new ListStreamReader(columnIndex, present, length, columnEncoding, isFileCompressed, - elementReader, context); + return new ListStreamReader(columnIndex, present, length, columnEncoding, isFileCompressed, elementReader, + context); } } @@ -2730,16 +2564,15 @@ public static ListStreamReader.StreamReaderBuilder builder() { } } - protected static class MapStreamReader extends MapTreeReader implements SettableTreeReader{ + protected static class MapStreamReader extends MapTreeReader implements SettableTreeReader { private boolean _isFileCompressed; private SettableUncompressedStream _presentStream; private SettableUncompressedStream _lengthStream; - public MapStreamReader(final int columnIndex, - final SettableUncompressedStream present, final SettableUncompressedStream lengthStream, - final OrcProto.ColumnEncoding columnEncoding, final boolean isFileCompressed, - final TreeReader keyReader, final TreeReader valueReader, - TreeReaderFactory.Context context) throws IOException { + public MapStreamReader(final int columnIndex, final SettableUncompressedStream present, + final SettableUncompressedStream lengthStream, final OrcProto.ColumnEncoding columnEncoding, + final boolean isFileCompressed, final TreeReader keyReader, final TreeReader valueReader, Context context) + throws IOException { super(columnIndex, present, context, lengthStream, columnEncoding, keyReader, valueReader); this._isFileCompressed = isFileCompressed; this._presentStream = present; @@ -2747,7 +2580,7 @@ public MapStreamReader(final int columnIndex, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { // We are not calling super.seek since we handle the present stream differently. PositionProvider ownIndex = index[columnId]; if (present != null) { @@ -2764,28 +2597,25 @@ public void seek(PositionProvider[] index) throws IOException { ownIndex.getNext(); } lengths.seek(ownIndex); - keyReader.seek(index); - valueReader.seek(index); + keyReader.seek(index, readPhase); + valueReader.seek(index, readPhase); } } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { // Only our parent class can call this. throw new IOException("Should never be called"); } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { - ColumnStreamData[] streamsData = batch.getColumnData(columnId); - if (_presentStream != null) { - _presentStream.setBuffers( - StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE])); + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { + ColumnStreamData[] streamsData = batch.getColumnData(columnId); + if (_presentStream != null) { + _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE])); } if (_lengthStream != null) { - _lengthStream.setBuffers( - StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE])); + _lengthStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.LENGTH_VALUE])); } if (keyReader != null) { @@ -2848,17 +2678,15 @@ public MapStreamReader.StreamReaderBuilder setContext(TreeReaderFactory.Context } public MapStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream length = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.LENGTH.name(), - lengthStream); + SettableUncompressedStream length = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.LENGTH.name(), lengthStream); boolean isFileCompressed = compressionCodec != null; - return new MapStreamReader(columnIndex, present, length, columnEncoding, isFileCompressed, - keyReader, valueReader, context); + return new MapStreamReader(columnIndex, present, length, columnEncoding, isFileCompressed, keyReader, + valueReader, context); } } @@ -2867,22 +2695,20 @@ public static MapStreamReader.StreamReaderBuilder builder() { } } - protected static class StructStreamReader extends StructTreeReader - implements SettableTreeReader { + protected static class StructStreamReader extends StructTreeReader implements SettableTreeReader { private boolean _isFileCompressed; private SettableUncompressedStream _presentStream; - public StructStreamReader(final int columnIndex, - final SettableUncompressedStream present, - final OrcProto.ColumnEncoding columnEncoding, final boolean isFileCompressed, - final TreeReader[] childReaders, TreeReaderFactory.Context context) throws IOException { + public StructStreamReader(final int columnIndex, final SettableUncompressedStream present, + final OrcProto.ColumnEncoding columnEncoding, final boolean isFileCompressed, final TreeReader[] childReaders, + TreeReaderFactory.Context context) throws IOException { super(columnIndex, present, context, columnEncoding, childReaders); this._isFileCompressed = isFileCompressed; this._presentStream = present; } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { PositionProvider ownIndex = index[columnId]; if (present != null) { if (_isFileCompressed) { @@ -2891,31 +2717,28 @@ public void seek(PositionProvider[] index) throws IOException { present.seek(ownIndex); } if (fields != null) { - for (TreeReader child : fields) { + for (TypeReader child : fields) { if (child != null) { - child.seek(index); + child.seek(index, readPhase); } } } } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { // Only our parent class can call this. throw new IOException("Should never be called"); } - @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { - _presentStream.setBuffers( - StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE])); + _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE])); } if (fields != null) { - for (TreeReader child : fields) { + for (TypeReader child : fields) { if (child != null) { ((SettableTreeReader) child).setBuffers(batch, sameStripe); } @@ -2962,13 +2785,11 @@ public StructStreamReader.StreamReaderBuilder setContext(TreeReaderFactory.Conte } public StructStreamReader build() throws IOException { - SettableUncompressedStream present = StreamUtils - .createSettableUncompressedStream(OrcProto.Stream.Kind.PRESENT.name(), - presentStream); + SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( + OrcProto.Stream.Kind.PRESENT.name(), presentStream); boolean isFileCompressed = compressionCodec != null; - return new StructStreamReader(columnIndex, present, columnEncoding, isFileCompressed, - childReaders, context); + return new StructStreamReader(columnIndex, present, columnEncoding, isFileCompressed, childReaders, context); } } @@ -2982,10 +2803,10 @@ protected static class UnionStreamReader extends UnionTreeReader implements Sett private SettableUncompressedStream _presentStream; private SettableUncompressedStream _dataStream; - public UnionStreamReader(final int columnIndex, - final SettableUncompressedStream present, final SettableUncompressedStream dataStream, - final OrcProto.ColumnEncoding columnEncoding, final boolean isFileCompressed, - final TreeReader[] childReaders, TreeReaderFactory.Context context) throws IOException { + public UnionStreamReader(final int columnIndex, final SettableUncompressedStream present, + final SettableUncompressedStream dataStream, final OrcProto.ColumnEncoding columnEncoding, + final boolean isFileCompressed, final TreeReader[] childReaders, TreeReaderFactory.Context context) + throws IOException { super(columnIndex, present, context, columnEncoding, childReaders); this._isFileCompressed = isFileCompressed; this._presentStream = present; @@ -2995,7 +2816,7 @@ public UnionStreamReader(final int columnIndex, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { PositionProvider ownIndex = index[columnId]; if (present != null) { if (_isFileCompressed) { @@ -3012,33 +2833,30 @@ public void seek(PositionProvider[] index) throws IOException { } tags.seek(ownIndex); if (fields != null) { - for (TreeReader child : fields) { - child.seek(index); + for (TypeReader child : fields) { + child.seek(index, readPhase); } } } } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { // Only our parent class can call this. throw new IOException("Should never be called"); } @Override - public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) - throws IOException { + public void setBuffers(EncodedColumnBatch batch, boolean sameStripe) throws IOException { ColumnStreamData[] streamsData = batch.getColumnData(columnId); if (_presentStream != null) { - _presentStream.setBuffers( - StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE])); + _presentStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE])); } if (_dataStream != null) { - _dataStream.setBuffers( - StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE])); + _dataStream.setBuffers(StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE])); } if (fields != null) { - for (TreeReader child : fields) { + for (TypeReader child : fields) { ((SettableTreeReader) child).setBuffers(batch, sameStripe); } } @@ -3092,12 +2910,12 @@ public UnionStreamReader build() throws IOException { SettableUncompressedStream present = StreamUtils.createSettableUncompressedStream( OrcProto.Stream.Kind.PRESENT.name(), presentStream); - SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream( - OrcProto.Stream.Kind.DATA.name(), dataStream); + SettableUncompressedStream data = StreamUtils.createSettableUncompressedStream(OrcProto.Stream.Kind.DATA.name(), + dataStream); boolean isFileCompressed = compressionCodec != null; - return new UnionStreamReader(columnIndex, present, data, - columnEncoding, isFileCompressed, childReaders, context); + return new UnionStreamReader(columnIndex, present, data, columnEncoding, isFileCompressed, childReaders, + context); } } @@ -3106,3 +2924,4 @@ public static UnionStreamReader.StreamReaderBuilder builder() { } } } + diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java index 26c08358f858..fa43d2a39f55 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java @@ -793,7 +793,7 @@ public void testCompactStatsGather() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "686", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "692", parameters.get("totalSize")); int[][] targetVals2 = {{5, 1, 1}, {5, 2, 2}, {5, 3, 1}, {5, 4, 2}}; runStatementOnDriver("insert into T partition(p=1,q) " + makeValuesClause(targetVals2)); @@ -847,7 +847,7 @@ public void testCompactStatsGather() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "4", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "698", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "704", parameters.get("totalSize")); } @Test diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index c98f7f408eb1..b81884d9790e 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -2603,14 +2603,14 @@ public void testCombinationInputFormatWithAcid() throws Exception { assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00000", split.getPath().toString()); assertEquals(0, split.getStart()); - assertEquals(702, split.getLength()); + assertEquals(709, split.getLength()); split = (HiveInputFormat.HiveInputSplit) splits[1]; assertEquals("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", split.inputFormatClassName()); assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00001", split.getPath().toString()); assertEquals(0, split.getStart()); - assertEquals(726, split.getLength()); + assertEquals(733, split.getLength()); CombineHiveInputFormat.CombineHiveInputSplit combineSplit = (CombineHiveInputFormat.CombineHiveInputSplit) splits[2]; assertEquals(BUCKETS, combineSplit.getNumPaths()); @@ -2618,7 +2618,7 @@ public void testCombinationInputFormatWithAcid() throws Exception { assertEquals("mock:/combinationAcid/p=1/00000" + bucket + "_0", combineSplit.getPath(bucket).toString()); assertEquals(0, combineSplit.getOffset(bucket)); - assertEquals(253, combineSplit.getLength(bucket)); + assertEquals(260, combineSplit.getLength(bucket)); } String[] hosts = combineSplit.getLocations(); assertEquals(2, hosts.length); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java index 923726eb2527..ca238aa276a9 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java @@ -1234,7 +1234,7 @@ public void emptyFile() throws Exception { assertEquals(false, reader.rows().hasNext()); assertEquals(CompressionKind.NONE, reader.getCompression()); assertEquals(0, reader.getNumberOfRows()); - assertEquals(0, reader.getCompressionSize()); + assertEquals(262144, reader.getCompressionSize()); assertEquals(false, reader.getMetadataKeys().iterator().hasNext()); assertEquals(3, reader.getContentLength()); assertEquals(false, reader.getStripes().iterator().hasNext()); diff --git a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out index f560d3a35b8a..74a840041b4b 100644 --- a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out +++ b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out @@ -63,7 +63,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 408 -totalSize 468 +totalSize 474 #### A masked pattern was here #### PREHOOK: query: create materialized view if not exists cmv_mat_view2_n4 as select a, c from cmv_basetable_n10 where a = 3 @@ -99,7 +99,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 232 -totalSize 334 +totalSize 340 #### A masked pattern was here #### PREHOOK: query: explain select a, c from cmv_basetable_n10 where a = 3 diff --git a/ql/src/test/results/clientpositive/llap/acid_bloom_filter_orc_file_dump.q.out b/ql/src/test/results/clientpositive/llap/acid_bloom_filter_orc_file_dump.q.out index 89d5d6506c33..dc181e7eb951 100644 --- a/ql/src/test/results/clientpositive/llap/acid_bloom_filter_orc_file_dump.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_bloom_filter_orc_file_dump.q.out @@ -76,7 +76,7 @@ PREHOOK: Input: default@bloomtest #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.3 Rows: 1 Compression: ZLIB Compression size: 32768 @@ -180,7 +180,7 @@ Stripes: Entry 0: numHashFunctions: 6 bitCount: 81472 popCount: 6 loadFactor: 0.0001 expectedFpp: 1.5953551E-25 Stripe level merge: numHashFunctions: 6 bitCount: 81472 popCount: 6 loadFactor: 0.0001 expectedFpp: 1.5953551E-25 -File length: 1212 bytes +File length: 1217 bytes Padding length: 0 bytes Padding ratio: 0% @@ -193,7 +193,7 @@ ________________________________________________________________________________ -- END ORC FILE DUMP -- -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.3 Rows: 1 Compression: ZLIB Compression size: 32768 @@ -297,7 +297,7 @@ Stripes: Entry 0: numHashFunctions: 6 bitCount: 81472 popCount: 6 loadFactor: 0.0001 expectedFpp: 1.5953551E-25 Stripe level merge: numHashFunctions: 6 bitCount: 81472 popCount: 6 loadFactor: 0.0001 expectedFpp: 1.5953551E-25 -File length: 1205 bytes +File length: 1211 bytes Padding length: 0 bytes Padding ratio: 0% diff --git a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out index 5e812d19d013..0677800ac2f6 100644 --- a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out @@ -334,7 +334,7 @@ Table Parameters: numPartitions 4 numRows 2003 rawDataSize 0 - totalSize 18043 + totalSize 18100 transactional true transactional_properties default #### A masked pattern was here #### @@ -444,7 +444,7 @@ Table Parameters: numPartitions 4 numRows 2003 rawDataSize 0 - totalSize 18043 + totalSize 18100 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/acid_table_stats.q.out b/ql/src/test/results/clientpositive/llap/acid_table_stats.q.out index 36c65fae4f86..d9b3801e8fb4 100644 --- a/ql/src/test/results/clientpositive/llap/acid_table_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_table_stats.q.out @@ -96,7 +96,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4361 + totalSize 4375 #### A masked pattern was here #### # Storage Information @@ -181,7 +181,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4361 + totalSize 4375 #### A masked pattern was here #### # Storage Information @@ -230,7 +230,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4361 + totalSize 4375 #### A masked pattern was here #### # Storage Information @@ -325,7 +325,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 8722 + totalSize 8750 #### A masked pattern was here #### # Storage Information @@ -372,7 +372,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 8722 + totalSize 8750 #### A masked pattern was here #### # Storage Information @@ -593,7 +593,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 176000 - totalSize 3325 + totalSize 3339 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/alter_merge_orc.q.out b/ql/src/test/results/clientpositive/llap/alter_merge_orc.q.out index 571651ccb647..c4fefac2d78b 100644 --- a/ql/src/test/results/clientpositive/llap/alter_merge_orc.q.out +++ b/ql/src/test/results/clientpositive/llap/alter_merge_orc.q.out @@ -48,9 +48,9 @@ columns:struct columns { i32 key, string value} partitioned:false partitionColumns: totalNumberFiles:3 -totalFileSize:7596 -maxFileSize:2532 -minFileSize:2532 +totalFileSize:7617 +maxFileSize:2539 +minFileSize:2539 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test @@ -91,9 +91,9 @@ columns:struct columns { i32 key, string value} partitioned:false partitionColumns: totalNumberFiles:1 -totalFileSize:7216 -maxFileSize:7216 -minFileSize:7216 +totalFileSize:7224 +maxFileSize:7224 +minFileSize:7224 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test @@ -171,9 +171,9 @@ columns:struct columns { i32 key, string value} partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:3 -totalFileSize:7596 -maxFileSize:2532 -minFileSize:2532 +totalFileSize:7617 +maxFileSize:2539 +minFileSize:2539 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test_part_n2 @@ -218,9 +218,9 @@ columns:struct columns { i32 key, string value} partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:1 -totalFileSize:7216 -maxFileSize:7216 -minFileSize:7216 +totalFileSize:7224 +maxFileSize:7224 +minFileSize:7224 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test_part_n2 diff --git a/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out b/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out index dab660d37021..1ba05bdff261 100644 --- a/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out +++ b/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out @@ -48,9 +48,9 @@ columns:struct columns { i32 key, string value} partitioned:false partitionColumns: totalNumberFiles:3 -totalFileSize:7596 -maxFileSize:2532 -minFileSize:2532 +totalFileSize:7617 +maxFileSize:2539 +minFileSize:2539 #### A masked pattern was here #### PREHOOK: query: desc extended src_orc_merge_test_stat @@ -93,7 +93,7 @@ Table Parameters: numFiles 3 numRows 1500 rawDataSize 147000 - totalSize 7596 + totalSize 7617 #### A masked pattern was here #### # Storage Information @@ -144,7 +144,7 @@ Table Parameters: numFiles 1 numRows 1500 rawDataSize 147000 - totalSize 7216 + totalSize 7224 #### A masked pattern was here #### # Storage Information @@ -214,9 +214,9 @@ columns:struct columns { i32 key, string value} partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:3 -totalFileSize:7596 -maxFileSize:2532 -minFileSize:2532 +totalFileSize:7617 +maxFileSize:2539 +minFileSize:2539 #### A masked pattern was here #### PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') @@ -243,7 +243,7 @@ Partition Parameters: numFiles 3 numRows 1500 rawDataSize 141000 - totalSize 7596 + totalSize 7617 #### A masked pattern was here #### # Storage Information @@ -290,7 +290,7 @@ Partition Parameters: numFiles 3 numRows 1500 rawDataSize 147000 - totalSize 7596 + totalSize 7617 #### A masked pattern was here #### # Storage Information @@ -345,7 +345,7 @@ Partition Parameters: numFiles 1 numRows 1500 rawDataSize 147000 - totalSize 7216 + totalSize 7224 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out b/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out index c30b58f525ea..1ebc46204dc4 100644 --- a/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out +++ b/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out @@ -214,7 +214,7 @@ Table Parameters: numFiles 2 numRows 10 rawDataSize 0 - totalSize 1884 + totalSize 1898 transactional true transactional_properties default #### A masked pattern was here #### @@ -257,7 +257,7 @@ Table Parameters: numFiles 3 numRows 8 rawDataSize 0 - totalSize 2596 + totalSize 2615 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/autoColumnStats_6.q.out b/ql/src/test/results/clientpositive/llap/autoColumnStats_6.q.out index e6ea95c8489e..af8016e2957a 100644 --- a/ql/src/test/results/clientpositive/llap/autoColumnStats_6.q.out +++ b/ql/src/test/results/clientpositive/llap/autoColumnStats_6.q.out @@ -213,7 +213,7 @@ POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=1).value SIMPLE [ POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 351 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 358 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge2a diff --git a/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_2.q.out b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_2.q.out index 915c73e0f63e..d34cb7553a60 100644 --- a/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_2.q.out +++ b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_2.q.out @@ -42,7 +42,7 @@ Table Parameters: numFiles 2 numRows 3 rawDataSize 24 - totalSize 571 + totalSize 585 #### A masked pattern was here #### # Storage Information @@ -87,7 +87,7 @@ Table Parameters: numFiles 2 numRows 3 rawDataSize 36 - totalSize 571 + totalSize 585 #### A masked pattern was here #### # Storage Information @@ -185,7 +185,7 @@ Table Parameters: numFiles 2 numRows 3 rawDataSize 36 - totalSize 571 + totalSize 585 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out b/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out index 5eb0ef9ad252..21676f9e2999 100644 --- a/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out +++ b/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out @@ -42,7 +42,7 @@ Table Parameters: numFiles 1 numRows 1 rawDataSize 170 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information @@ -221,7 +221,7 @@ Table Parameters: numFiles 1 numRows 1 rawDataSize 174 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information @@ -298,7 +298,7 @@ Table Parameters: numPartitions 2 numRows 2 rawDataSize 340 - totalSize 570 + totalSize 584 #### A masked pattern was here #### # Storage Information @@ -540,7 +540,7 @@ Table Parameters: numPartitions 2 numRows 2 rawDataSize 348 - totalSize 570 + totalSize 584 #### A masked pattern was here #### # Storage Information @@ -578,7 +578,7 @@ Partition Parameters: numFiles 1 numRows 1 rawDataSize 174 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information @@ -616,7 +616,7 @@ Partition Parameters: numFiles 1 numRows 1 rawDataSize 174 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information @@ -697,7 +697,7 @@ Table Parameters: numPartitions 2 numRows 2 rawDataSize 340 - totalSize 570 + totalSize 584 #### A masked pattern was here #### # Storage Information @@ -897,7 +897,7 @@ Table Parameters: numPartitions 2 numRows 2 rawDataSize 344 - totalSize 570 + totalSize 584 #### A masked pattern was here #### # Storage Information @@ -935,7 +935,7 @@ Partition Parameters: numFiles 1 numRows 1 rawDataSize 174 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information @@ -973,7 +973,7 @@ Partition Parameters: numFiles 1 numRows 1 rawDataSize 170 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/ctas.q.out b/ql/src/test/results/clientpositive/llap/ctas.q.out index b6f1d24faf8d..20d587cfa1b8 100644 --- a/ql/src/test/results/clientpositive/llap/ctas.q.out +++ b/ql/src/test/results/clientpositive/llap/ctas.q.out @@ -1183,4 +1183,4 @@ POSTHOOK: Input: default@acid_ctas_part@k=0 POSTHOOK: Input: default@acid_ctas_part@k=10 POSTHOOK: Input: default@acid_ctas_part@k=100 #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"acid_ctas_part\"\n ],\n \"table:alias\": \"acid_ctas_part\",\n \"inputs\": [],\n \"rowCount\": 270.0,\n \"avgRowSize\": 200.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"k\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"ROW__IS__DELETED\"\n }\n ],\n \"partitionColumns\": [\n \"k\"\n ],\n \"colStats\": [\n {\n \"name\": \"value\",\n \"ndv\": 54\n },\n {\n \"name\": \"k\",\n \"ndv\": 3\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"k\",\n \"value\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 270.0\n }\n ]\n}","optimizedSQL":"SELECT `k`, `value`\nFROM `default`.`acid_ctas_part`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"acid_ctas_part","columns:":["value"],"database:":"default","table:":"acid_ctas_part","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"k (type: string), value (type: string)","columnExprMap:":{"_col0":"k","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"acid_ctas_part\"\n ],\n \"table:alias\": \"acid_ctas_part\",\n \"inputs\": [],\n \"rowCount\": 273.0,\n \"avgRowSize\": 200.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"k\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"ROW__IS__DELETED\"\n }\n ],\n \"partitionColumns\": [\n \"k\"\n ],\n \"colStats\": [\n {\n \"name\": \"value\",\n \"ndv\": 54\n },\n {\n \"name\": \"k\",\n \"ndv\": 3\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"k\",\n \"value\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 273.0\n }\n ]\n}","optimizedSQL":"SELECT `k`, `value`\nFROM `default`.`acid_ctas_part`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"acid_ctas_part","columns:":["value"],"database:":"default","table:":"acid_ctas_part","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"k (type: string), value (type: string)","columnExprMap:":{"_col0":"k","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} diff --git a/ql/src/test/results/clientpositive/llap/default_constraint.q.out b/ql/src/test/results/clientpositive/llap/default_constraint.q.out index 48347175aa1c..6d739f69a177 100644 --- a/ql/src/test/results/clientpositive/llap/default_constraint.q.out +++ b/ql/src/test/results/clientpositive/llap/default_constraint.q.out @@ -1556,7 +1556,7 @@ Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 numFiles 1 - totalSize 1101 + totalSize 1107 transactional true transactional_properties default #### A masked pattern was here #### @@ -1730,7 +1730,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 2 - totalSize 2202 + totalSize 2213 transactional true transactional_properties default #### A masked pattern was here #### @@ -1808,7 +1808,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 2 - totalSize 2202 + totalSize 2213 transactional true transactional_properties default #### A masked pattern was here #### @@ -1993,7 +1993,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 3 - totalSize 3290 + totalSize 3306 transactional true transactional_properties default #### A masked pattern was here #### @@ -2074,7 +2074,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 3 - totalSize 3290 + totalSize 3306 transactional true transactional_properties default #### A masked pattern was here #### @@ -2152,7 +2152,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 3 - totalSize 3290 + totalSize 3306 transactional true transactional_properties default #### A masked pattern was here #### @@ -2756,7 +2756,7 @@ Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 numFiles 1 - totalSize 1101 + totalSize 1107 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out b/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out index 030b61d6dc45..41f27e5b0181 100644 --- a/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out +++ b/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out @@ -54,7 +54,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 634 - totalSize 598 + totalSize 604 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out index fa852eb143c8..c0a00d0b60b5 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out @@ -1362,7 +1362,7 @@ Partition Parameters: numFiles 2 numRows 32 rawDataSize 640 - totalSize 1432 + totalSize 1446 #### A masked pattern was here #### # Storage Information @@ -1402,7 +1402,7 @@ Partition Parameters: numFiles 2 numRows 6 rawDataSize 120 - totalSize 1136 + totalSize 1150 #### A masked pattern was here #### # Storage Information @@ -1442,7 +1442,7 @@ Partition Parameters: numFiles 2 numRows 14 rawDataSize 280 - totalSize 1250 + totalSize 1264 #### A masked pattern was here #### # Storage Information @@ -1482,7 +1482,7 @@ Partition Parameters: numFiles 2 numRows 6 rawDataSize 120 - totalSize 1136 + totalSize 1150 #### A masked pattern was here #### # Storage Information @@ -1521,7 +1521,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4656 + totalSize 4712 #### A masked pattern was here #### # Storage Information @@ -1560,7 +1560,7 @@ Partition Parameters: numFiles 4 numRows 6 rawDataSize 120 - totalSize 2082 + totalSize 2110 #### A masked pattern was here #### # Storage Information @@ -1599,7 +1599,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4674 + totalSize 4730 #### A masked pattern was here #### # Storage Information @@ -1638,7 +1638,7 @@ Partition Parameters: numFiles 4 numRows 6 rawDataSize 120 - totalSize 2082 + totalSize 2110 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out index 3f7cfa8ce9fd..86bf490d1971 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out @@ -1373,7 +1373,7 @@ Partition Parameters: numFiles 1 numRows 11 rawDataSize 88 - totalSize 466 + totalSize 473 #### A masked pattern was here #### # Storage Information @@ -1431,7 +1431,7 @@ Partition Parameters: numFiles 1 numRows 13 rawDataSize 104 - totalSize 489 + totalSize 496 #### A masked pattern was here #### # Storage Information @@ -1633,7 +1633,7 @@ Partition Parameters: numFiles 1 numRows 11 rawDataSize 88 - totalSize 466 + totalSize 473 #### A masked pattern was here #### # Storage Information @@ -1691,7 +1691,7 @@ Partition Parameters: numFiles 1 numRows 13 rawDataSize 104 - totalSize 489 + totalSize 496 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/explainanalyze_acid_with_direct_insert.q.out b/ql/src/test/results/clientpositive/llap/explainanalyze_acid_with_direct_insert.q.out index d06bd281d7d8..6b33ee77b2d1 100644 --- a/ql/src/test/results/clientpositive/llap/explainanalyze_acid_with_direct_insert.q.out +++ b/ql/src/test/results/clientpositive/llap/explainanalyze_acid_with_direct_insert.q.out @@ -679,24 +679,24 @@ STAGE PLANS: TableScan alias: analyze_part_table filterExpr: (b = 3) (type: boolean) - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) null sort order: z sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 11 (type: int), 3 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -711,10 +711,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 3 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -818,17 +818,17 @@ STAGE PLANS: TableScan alias: analyze_part_table filterExpr: (b = 1) (type: boolean) - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) null sort order: z sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: may be used (ACID table) Reducer 2 @@ -837,10 +837,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1639,24 +1639,24 @@ STAGE PLANS: TableScan alias: analyze_part_table filterExpr: (b = 3) (type: boolean) - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) null sort order: z sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 11 (type: int), 3 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1671,10 +1671,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 3 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1778,17 +1778,17 @@ STAGE PLANS: TableScan alias: analyze_part_table filterExpr: (b = 1) (type: boolean) - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) null sort order: z sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: may be used (ACID table) Reducer 2 @@ -1797,10 +1797,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/insert_only_to_acid_convert.q.out b/ql/src/test/results/clientpositive/llap/insert_only_to_acid_convert.q.out index 284e5d419a2c..079b22d84c28 100644 --- a/ql/src/test/results/clientpositive/llap/insert_only_to_acid_convert.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_only_to_acid_convert.q.out @@ -38,7 +38,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 182 - totalSize 310 + totalSize 317 transactional true transactional_properties insert_only #### A masked pattern was here #### @@ -84,7 +84,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 182 - totalSize 310 + totalSize 317 transactional true transactional_properties default #### A masked pattern was here #### @@ -159,7 +159,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 182 - totalSize 310 + totalSize 317 transactional true transactional_properties insert_only #### A masked pattern was here #### @@ -204,7 +204,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 182 - totalSize 310 + totalSize 317 transactional true transactional_properties insert_only #### A masked pattern was here #### @@ -296,7 +296,7 @@ Table Parameters: numPartitions 2 numRows 20 rawDataSize 2008 - totalSize 1920 + totalSize 1948 transactional true transactional_properties insert_only #### A masked pattern was here #### @@ -347,7 +347,7 @@ Table Parameters: numPartitions 2 numRows 20 rawDataSize 2008 - totalSize 1920 + totalSize 1948 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out index a2f11d24d4e7..7ddf658d5a73 100644 --- a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out @@ -171,7 +171,7 @@ Table Parameters: numFiles 1 numRows 12288 rawDataSize 0 - totalSize 309566 + totalSize 309572 transactional true transactional_properties default #### A masked pattern was here #### @@ -340,7 +340,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 0 - totalSize 1668 + totalSize 1673 transactional true transactional_properties default #### A masked pattern was here #### @@ -436,7 +436,7 @@ Table Parameters: numFiles 2 numRows 4 rawDataSize 0 - totalSize 3337 + totalSize 3346 transactional true transactional_properties default #### A masked pattern was here #### @@ -528,7 +528,7 @@ Table Parameters: numFiles 3 numRows 12292 rawDataSize 0 - totalSize 312904 + totalSize 312919 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/masking_mv.q.out b/ql/src/test/results/clientpositive/llap/masking_mv.q.out index 01f6b95cb17f..ba4f4b82c741 100644 --- a/ql/src/test/results/clientpositive/llap/masking_mv.q.out +++ b/ql/src/test/results/clientpositive/llap/masking_mv.q.out @@ -160,7 +160,7 @@ Table Parameters: numFiles 1 numRows 500 rawDataSize 2000 - totalSize 976 + totalSize 983 #### A masked pattern was here #### # Storage Information @@ -862,7 +862,7 @@ Table Parameters: numFiles 1 numRows 500 rawDataSize 2000 - totalSize 976 + totalSize 983 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/masking_mv_by_text.q.out b/ql/src/test/results/clientpositive/llap/masking_mv_by_text.q.out index 2f797c8d7706..bd7182b92640 100644 --- a/ql/src/test/results/clientpositive/llap/masking_mv_by_text.q.out +++ b/ql/src/test/results/clientpositive/llap/masking_mv_by_text.q.out @@ -160,7 +160,7 @@ Table Parameters: numFiles 1 numRows 500 rawDataSize 2000 - totalSize 976 + totalSize 983 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out index 132ad39b95e4..65d6ba544faf 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out @@ -53,7 +53,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 1025 - totalSize 511 + totalSize 516 #### A masked pattern was here #### # Storage Information @@ -121,7 +121,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 580 - totalSize 356 + totalSize 362 #### A masked pattern was here #### # Storage Information @@ -273,7 +273,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 5 rawDataSize 1605 -totalSize 727 +totalSize 732 #### A masked pattern was here #### PREHOOK: query: drop materialized view cmv_mat_view_n4 PREHOOK: type: DROP_MATERIALIZED_VIEW @@ -373,4 +373,4 @@ POSTHOOK: Input: default@acid_cmv_part@k=0 POSTHOOK: Input: default@acid_cmv_part@k=10 POSTHOOK: Input: default@acid_cmv_part@k=100 #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"acid_cmv_part\"\n ],\n \"table:alias\": \"acid_cmv_part\",\n \"inputs\": [],\n \"rowCount\": 270.0,\n \"avgRowSize\": 200.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"k\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"ROW__IS__DELETED\"\n }\n ],\n \"partitionColumns\": [\n \"k\"\n ],\n \"colStats\": [\n {\n \"name\": \"value\",\n \"ndv\": 54\n },\n {\n \"name\": \"k\",\n \"ndv\": 3\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"k\",\n \"value\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 270.0\n }\n ]\n}","optimizedSQL":"SELECT `k`, `value`\nFROM `default`.`acid_cmv_part`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"acid_cmv_part","columns:":["value"],"database:":"default","table:":"acid_cmv_part","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"k (type: string), value (type: string)","columnExprMap:":{"_col0":"k","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"acid_cmv_part\"\n ],\n \"table:alias\": \"acid_cmv_part\",\n \"inputs\": [],\n \"rowCount\": 273.0,\n \"avgRowSize\": 200.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"k\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"ROW__IS__DELETED\"\n }\n ],\n \"partitionColumns\": [\n \"k\"\n ],\n \"colStats\": [\n {\n \"name\": \"value\",\n \"ndv\": 54\n },\n {\n \"name\": \"k\",\n \"ndv\": 3\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"k\",\n \"value\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 273.0\n }\n ]\n}","optimizedSQL":"SELECT `k`, `value`\nFROM `default`.`acid_cmv_part`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"acid_cmv_part","columns:":["value"],"database:":"default","table:":"acid_cmv_part","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"k (type: string), value (type: string)","columnExprMap:":{"_col0":"k","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_acid.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_acid.q.out index f68b72535844..23aa956d04e8 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_acid.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_acid.q.out @@ -55,7 +55,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 0 - totalSize 926 + totalSize 931 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out index 9ccf526a515b..2af2b9bc0078 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out @@ -63,7 +63,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 408 -totalSize 468 +totalSize 474 #### A masked pattern was here #### PREHOOK: query: create materialized view if not exists cmv_mat_view2_n4 as select a, c from cmv_basetable_n10 where a = 3 @@ -99,7 +99,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 232 -totalSize 334 +totalSize 340 #### A masked pattern was here #### PREHOOK: query: explain select a, c from cmv_basetable_n10 where a = 3 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_10.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_10.q.out index 7fb9d79d31aa..86cce82f092c 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_10.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_10.q.out @@ -79,17 +79,17 @@ STAGE PLANS: TableScan alias: t1 filterExpr: (b = 1) (type: boolean) - Statistics: Num rows: 69 Data size: 13590 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 69 Data size: 13710 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (b = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), 1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 196 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out index 4b8cd42ef9d6..974dc15cfb02 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out @@ -296,7 +296,7 @@ Table Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1542 + totalSize 1554 transactional true transactional_properties default #### A masked pattern was here #### @@ -540,7 +540,7 @@ Table Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1542 + totalSize 1554 transactional true transactional_properties default #### A masked pattern was here #### @@ -1029,7 +1029,7 @@ Table Parameters: numFiles 3 numRows 3 rawDataSize 0 - totalSize 2301 + totalSize 2319 transactional true transactional_properties default #### A masked pattern was here #### @@ -1345,7 +1345,7 @@ Table Parameters: numFiles 2 numRows 3 rawDataSize 0 - totalSize 1047 + totalSize 1061 transactional true transactional_properties default #### A masked pattern was here #### @@ -1655,7 +1655,7 @@ Table Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1044 + totalSize 1059 transactional true transactional_properties default #### A masked pattern was here #### @@ -2071,7 +2071,7 @@ Table Parameters: numFiles 3 numRows 3 rawDataSize 0 - totalSize 1801 + totalSize 1822 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out index abeab03cf397..bed471893793 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out @@ -393,7 +393,7 @@ Table Parameters: numFiles 2 numRows 5 rawDataSize 0 - totalSize 1505 + totalSize 1516 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_by_text_multi_db.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_by_text_multi_db.q.out index 34d28e3b5edb..b4fd488bc8cf 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_by_text_multi_db.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_by_text_multi_db.q.out @@ -87,7 +87,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 408 -totalSize 468 +totalSize 474 #### A masked pattern was here #### PREHOOK: query: create materialized view if not exists cmv_mat_view2_n2 as select a, c from db1.cmv_basetable_n7 where a = 3 @@ -123,7 +123,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 232 -totalSize 334 +totalSize 340 #### A masked pattern was here #### PREHOOK: query: create database db3 PREHOOK: type: CREATEDATABASE diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out index 7f1db9ffd9ba..bf53cf8cbf47 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out @@ -63,7 +63,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 408 -totalSize 468 +totalSize 474 #### A masked pattern was here #### PREHOOK: query: create materialized view if not exists cmv_mat_view2 as select a, c from cmv_basetable_n0 where a = 3 @@ -99,7 +99,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 232 -totalSize 334 +totalSize 340 #### A masked pattern was here #### PREHOOK: query: explain select a, c from cmv_basetable_n0 where a = 3 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out index 34d28e3b5edb..b4fd488bc8cf 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out @@ -87,7 +87,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 408 -totalSize 468 +totalSize 474 #### A masked pattern was here #### PREHOOK: query: create materialized view if not exists cmv_mat_view2_n2 as select a, c from db1.cmv_basetable_n7 where a = 3 @@ -123,7 +123,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 232 -totalSize 334 +totalSize 340 #### A masked pattern was here #### PREHOOK: query: create database db3 PREHOOK: type: CREATEDATABASE diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out index 87933394f39b..784c7fd8d1e6 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out @@ -290,7 +290,7 @@ Table Parameters: numRows 2 rawDataSize 232 rewriting.time.window 5min - totalSize 610 + totalSize 624 #### A masked pattern was here #### # Storage Information @@ -529,7 +529,7 @@ Table Parameters: numRows 2 rawDataSize 232 rewriting.time.window 5min - totalSize 610 + totalSize 624 #### A masked pattern was here #### # Storage Information @@ -873,7 +873,7 @@ Table Parameters: numRows 3 rawDataSize 348 rewriting.time.window 5min - totalSize 640 + totalSize 654 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window_2.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window_2.q.out index e9ddcb0722b9..0faf8dc18fac 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window_2.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window_2.q.out @@ -96,7 +96,7 @@ Table Parameters: numFiles 2 numRows 2 rawDataSize 232 - totalSize 610 + totalSize 624 #### A masked pattern was here #### # Storage Information @@ -160,7 +160,7 @@ Table Parameters: numFiles 2 numRows 2 rawDataSize 232 - totalSize 610 + totalSize 624 #### A masked pattern was here #### # Storage Information @@ -224,7 +224,7 @@ Table Parameters: numFiles 2 numRows 3 rawDataSize 348 - totalSize 640 + totalSize 654 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out index 6c45f0b7c441..d35e1dc5ae52 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out @@ -75,7 +75,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 580 - totalSize 356 + totalSize 362 #### A masked pattern was here #### # Storage Information @@ -108,7 +108,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 5 rawDataSize 580 -totalSize 356 +totalSize 362 #### A masked pattern was here #### PREHOOK: query: select a, c from cmv_mat_view_n8 PREHOOK: type: QUERY @@ -259,7 +259,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 1025 - totalSize 511 + totalSize 516 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_drop.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_drop.q.out index ce73b68498c7..fb415df88867 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_drop.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_drop.q.out @@ -22,9 +22,9 @@ columns:struct columns { i32 cint, string cstring1} partitioned:false partitionColumns: totalNumberFiles:1 -totalFileSize:47139 -maxFileSize:47139 -minFileSize:47139 +totalFileSize:47146 +maxFileSize:47146 +minFileSize:47146 #### A masked pattern was here #### PREHOOK: query: drop materialized view dmv_mat_view diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out index 9c7eacb07dbd..65562130e795 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out @@ -301,7 +301,7 @@ Table Parameters: numPartitions 32 numRows 55 rawDataSize 9790 - totalSize 10870 + totalSize 11094 #### A masked pattern was here #### # Storage Information @@ -1828,7 +1828,7 @@ Table Parameters: numPartitions 32 numRows 57 rawDataSize 11062 - totalSize 17166 + totalSize 17390 #### A masked pattern was here #### # Storage Information @@ -2629,7 +2629,7 @@ Table Parameters: numPartitions 32 numRows 58 rawDataSize 11257 - totalSize 17169 + totalSize 17393 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out index 17ec0c351f40..9d989b20e379 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out @@ -263,7 +263,7 @@ Table Parameters: numPartitions 32 numRows 55 rawDataSize 5005 - totalSize 7682 + totalSize 7906 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out index f81ec3c9d23b..4e44988118e1 100644 --- a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out @@ -102,7 +102,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 53000 - totalSize 3231 + totalSize 3238 #### A masked pattern was here #### # Storage Information @@ -150,7 +150,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 53000 - totalSize 3231 + totalSize 3238 #### A masked pattern was here #### # Storage Information @@ -237,7 +237,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 52600 - totalSize 3231 + totalSize 3238 #### A masked pattern was here #### # Storage Information @@ -346,7 +346,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22150 - totalSize 2131 + totalSize 2138 #### A masked pattern was here #### # Storage Information @@ -387,7 +387,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22250 - totalSize 2143 + totalSize 2150 #### A masked pattern was here #### # Storage Information @@ -440,7 +440,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22150 - totalSize 2131 + totalSize 2138 #### A masked pattern was here #### # Storage Information @@ -481,7 +481,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22250 - totalSize 2143 + totalSize 2150 #### A masked pattern was here #### # Storage Information @@ -578,7 +578,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 21950 - totalSize 2131 + totalSize 2138 #### A masked pattern was here #### # Storage Information @@ -619,7 +619,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22050 - totalSize 2143 + totalSize 2150 #### A masked pattern was here #### # Storage Information @@ -734,7 +734,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22155 - totalSize 5396 + totalSize 5424 #### A masked pattern was here #### # Storage Information @@ -775,7 +775,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22243 - totalSize 5378 + totalSize 5403 #### A masked pattern was here #### # Storage Information @@ -828,7 +828,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22155 - totalSize 5396 + totalSize 5424 #### A masked pattern was here #### # Storage Information @@ -869,7 +869,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22243 - totalSize 5378 + totalSize 5403 #### A masked pattern was here #### # Storage Information @@ -972,7 +972,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 21955 - totalSize 5396 + totalSize 5424 #### A masked pattern was here #### # Storage Information @@ -1013,7 +1013,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22043 - totalSize 5378 + totalSize 5403 #### A masked pattern was here #### # Storage Information @@ -1122,7 +1122,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22150 - totalSize 2131 + totalSize 2138 #### A masked pattern was here #### # Storage Information @@ -1175,7 +1175,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22150 - totalSize 2131 + totalSize 2138 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/orc_file_dump.q.out b/ql/src/test/results/clientpositive/llap/orc_file_dump.q.out index 467f8e9d9055..f1122b11e713 100644 --- a/ql/src/test/results/clientpositive/llap/orc_file_dump.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_file_dump.q.out @@ -93,7 +93,7 @@ PREHOOK: Input: default@orc_ppd_n0 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.3 Rows: 1049 Compression: ZLIB Compression size: 262144 @@ -130,49 +130,49 @@ File Statistics: Column 11: count: 1049 hasNull: false bytesOnDisk: 2468 sum: 13278 Stripes: - Stripe: offset: 3 data: 22405 rows: 1049 tail: 243 index: 8973 + Stripe: offset: 3 data: 22405 rows: 1049 tail: 239 index: 13620 Stream: column 0 section ROW_INDEX start: 3 length 20 - Stream: column 0 section BLOOM_FILTER_UTF8 start: 23 length 34 - Stream: column 1 section ROW_INDEX start: 57 length 58 - Stream: column 1 section BLOOM_FILTER_UTF8 start: 115 length 696 - Stream: column 2 section ROW_INDEX start: 811 length 58 - Stream: column 2 section BLOOM_FILTER_UTF8 start: 869 length 867 - Stream: column 3 section ROW_INDEX start: 1736 length 61 - Stream: column 3 section BLOOM_FILTER_UTF8 start: 1797 length 861 - Stream: column 4 section ROW_INDEX start: 2658 length 69 - Stream: column 4 section BLOOM_FILTER_UTF8 start: 2727 length 850 - Stream: column 5 section ROW_INDEX start: 3577 length 78 - Stream: column 5 section BLOOM_FILTER_UTF8 start: 3655 length 1172 - Stream: column 6 section ROW_INDEX start: 4827 length 85 - Stream: column 6 section BLOOM_FILTER_UTF8 start: 4912 length 1167 - Stream: column 7 section ROW_INDEX start: 6079 length 41 - Stream: column 7 section BLOOM_FILTER_UTF8 start: 6120 length 34 - Stream: column 8 section ROW_INDEX start: 6154 length 86 - Stream: column 8 section BLOOM_FILTER_UTF8 start: 6240 length 1051 - Stream: column 9 section ROW_INDEX start: 7291 length 63 - Stream: column 9 section BLOOM_FILTER_UTF8 start: 7354 length 52 - Stream: column 10 section ROW_INDEX start: 7406 length 80 - Stream: column 10 section BLOOM_FILTER_UTF8 start: 7486 length 1189 - Stream: column 11 section ROW_INDEX start: 8675 length 47 - Stream: column 11 section BLOOM_FILTER_UTF8 start: 8722 length 254 - Stream: column 1 section PRESENT start: 8976 length 17 - Stream: column 1 section DATA start: 8993 length 962 - Stream: column 2 section PRESENT start: 9955 length 17 - Stream: column 2 section DATA start: 9972 length 1441 - Stream: column 3 section DATA start: 11413 length 1704 - Stream: column 4 section DATA start: 13117 length 1998 - Stream: column 5 section DATA start: 15115 length 2925 - Stream: column 6 section DATA start: 18040 length 3323 - Stream: column 7 section DATA start: 21363 length 137 - Stream: column 8 section DATA start: 21500 length 1572 - Stream: column 8 section LENGTH start: 23072 length 310 - Stream: column 8 section DICTIONARY_DATA start: 23382 length 1548 - Stream: column 9 section DATA start: 24930 length 19 - Stream: column 9 section SECONDARY start: 24949 length 1783 - Stream: column 10 section DATA start: 26732 length 2166 - Stream: column 10 section SECONDARY start: 28898 length 15 - Stream: column 11 section DATA start: 28913 length 1877 - Stream: column 11 section LENGTH start: 30790 length 591 + Stream: column 0 section BLOOM_FILTER_UTF8 start: 23 length 43 + Stream: column 1 section ROW_INDEX start: 66 length 58 + Stream: column 1 section BLOOM_FILTER_UTF8 start: 124 length 1091 + Stream: column 2 section ROW_INDEX start: 1215 length 58 + Stream: column 2 section BLOOM_FILTER_UTF8 start: 1273 length 1372 + Stream: column 3 section ROW_INDEX start: 2645 length 61 + Stream: column 3 section BLOOM_FILTER_UTF8 start: 2706 length 1374 + Stream: column 4 section ROW_INDEX start: 4080 length 69 + Stream: column 4 section BLOOM_FILTER_UTF8 start: 4149 length 1375 + Stream: column 5 section ROW_INDEX start: 5524 length 78 + Stream: column 5 section BLOOM_FILTER_UTF8 start: 5602 length 1824 + Stream: column 6 section ROW_INDEX start: 7426 length 85 + Stream: column 6 section BLOOM_FILTER_UTF8 start: 7511 length 1795 + Stream: column 7 section ROW_INDEX start: 9306 length 41 + Stream: column 7 section BLOOM_FILTER_UTF8 start: 9347 length 43 + Stream: column 8 section ROW_INDEX start: 9390 length 86 + Stream: column 8 section BLOOM_FILTER_UTF8 start: 9476 length 1653 + Stream: column 9 section ROW_INDEX start: 11129 length 63 + Stream: column 9 section BLOOM_FILTER_UTF8 start: 11192 length 80 + Stream: column 10 section ROW_INDEX start: 11272 length 80 + Stream: column 10 section BLOOM_FILTER_UTF8 start: 11352 length 1802 + Stream: column 11 section ROW_INDEX start: 13154 length 47 + Stream: column 11 section BLOOM_FILTER_UTF8 start: 13201 length 422 + Stream: column 1 section PRESENT start: 13623 length 17 + Stream: column 1 section DATA start: 13640 length 962 + Stream: column 2 section PRESENT start: 14602 length 17 + Stream: column 2 section DATA start: 14619 length 1441 + Stream: column 3 section DATA start: 16060 length 1704 + Stream: column 4 section DATA start: 17764 length 1998 + Stream: column 5 section DATA start: 19762 length 2925 + Stream: column 6 section DATA start: 22687 length 3323 + Stream: column 7 section DATA start: 26010 length 137 + Stream: column 8 section DATA start: 26147 length 1572 + Stream: column 8 section LENGTH start: 27719 length 310 + Stream: column 8 section DICTIONARY_DATA start: 28029 length 1548 + Stream: column 9 section DATA start: 29577 length 19 + Stream: column 9 section SECONDARY start: 29596 length 1783 + Stream: column 10 section DATA start: 31379 length 2166 + Stream: column 10 section SECONDARY start: 33545 length 15 + Stream: column 11 section DATA start: 33560 length 1877 + Stream: column 11 section LENGTH start: 35437 length 591 Encoding column 0: DIRECT Encoding column 1: DIRECT Encoding column 2: DIRECT_V2 @@ -189,88 +189,88 @@ Stripes: Entry 0: count: 1000 hasNull: false positions: Entry 1: count: 49 hasNull: false positions: Bloom filters for column 0: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 Row group indices for column 1: Entry 0: count: 997 hasNull: true min: -3 max: 124 sum: 59325 positions: 0,0,0,0,0,0,0 Entry 1: count: 49 hasNull: false min: 2 max: 123 sum: 3105 positions: 0,10,113,0,0,903,101 Bloom filters for column 1: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 492 loadFactor: 0.0784 expectedFpp: 3.7864847E-5 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 168 loadFactor: 0.0268 expectedFpp: 5.147697E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 492 loadFactor: 0.0784 expectedFpp: 3.7864847E-5 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 849 loadFactor: 0.0884 expectedFpp: 4.231118E-8 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 285 loadFactor: 0.0297 expectedFpp: 2.0324289E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 849 loadFactor: 0.0884 expectedFpp: 4.231118E-8 Row group indices for column 2: Entry 0: count: 997 hasNull: true min: 256 max: 511 sum: 381766 positions: 0,0,0,0,0,0,0 Entry 1: count: 49 hasNull: false min: 256 max: 507 sum: 17123 positions: 0,10,114,0,0,1026,485 Bloom filters for column 2: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 928 loadFactor: 0.148 expectedFpp: 4.792562E-4 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 175 loadFactor: 0.0279 expectedFpp: 6.0607726E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 928 loadFactor: 0.148 expectedFpp: 4.792562E-4 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 1617 loadFactor: 0.1684 expectedFpp: 3.8465505E-6 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 306 loadFactor: 0.0319 expectedFpp: 3.343115E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 1617 loadFactor: 0.1684 expectedFpp: 3.8465505E-6 Row group indices for column 3: Entry 0: count: 1000 hasNull: false min: 65536 max: 65791 sum: 65662251 positions: 0,0,0 Entry 1: count: 49 hasNull: false min: 65540 max: 65788 sum: 3218800 positions: 0,1538,488 Bloom filters for column 3: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 947 loadFactor: 0.151 expectedFpp: 5.1972753E-4 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 171 loadFactor: 0.0273 expectedFpp: 5.525356E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 951 loadFactor: 0.1516 expectedFpp: 5.285643E-4 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 1632 loadFactor: 0.17 expectedFpp: 4.1033873E-6 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 296 loadFactor: 0.0308 expectedFpp: 2.6493746E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 1638 loadFactor: 0.1706 expectedFpp: 4.2101606E-6 Row group indices for column 4: Entry 0: count: 1000 hasNull: false min: 4294967296 max: 4294967551 sum: 4294967421966 positions: 0,0,0 Entry 1: count: 49 hasNull: false min: 4294967301 max: 4294967550 sum: 210453403987 positions: 0,2885,422 Bloom filters for column 4: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 900 loadFactor: 0.1435 expectedFpp: 4.2398053E-4 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 177 loadFactor: 0.0282 expectedFpp: 6.342622E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 905 loadFactor: 0.1443 expectedFpp: 4.3348098E-4 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 1611 loadFactor: 0.1678 expectedFpp: 3.7477455E-6 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 318 loadFactor: 0.0331 expectedFpp: 4.3761383E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 1622 loadFactor: 0.169 expectedFpp: 3.9305864E-6 Row group indices for column 5: Entry 0: count: 1000 hasNull: false min: 0.07999999821186066 max: 99.91999816894531 sum: 49977.420025609434 positions: 0,0 Entry 1: count: 49 hasNull: false min: 2.7899999618530273 max: 97.6500015258789 sum: 2767.280002593994 positions: 0,4000 Bloom filters for column 5: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 2846 loadFactor: 0.4538 expectedFpp: 0.042395078 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 192 loadFactor: 0.0306 expectedFpp: 8.781743E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 2940 loadFactor: 0.4688 expectedFpp: 0.048279762 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 4847 loadFactor: 0.5049 expectedFpp: 0.008363968 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 341 loadFactor: 0.0355 expectedFpp: 7.134803E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 4989 loadFactor: 0.5197 expectedFpp: 0.010237543 Row group indices for column 6: Entry 0: count: 1000 hasNull: false min: 0.02 max: 49.85 sum: 25045.26999999998 positions: 0,0 Entry 1: count: 49 hasNull: false min: 1.71 max: 48.78 sum: 1241.08 positions: 0,8000 Bloom filters for column 6: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 2696 loadFactor: 0.4298 expectedFpp: 0.03413936 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 195 loadFactor: 0.0311 expectedFpp: 9.3436E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 2792 loadFactor: 0.4452 expectedFpp: 0.03926788 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 4562 loadFactor: 0.4752 expectedFpp: 0.005472533 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 337 loadFactor: 0.0351 expectedFpp: 6.569173E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 4710 loadFactor: 0.4906 expectedFpp: 0.006843018 Row group indices for column 7: Entry 0: count: 1000 hasNull: false true: 501 positions: 0,0,0,0 Entry 1: count: 49 hasNull: false true: 25 positions: 0,0,125,0 Bloom filters for column 7: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 Row group indices for column 8: Entry 0: count: 1000 hasNull: false min: max: zach zipper sum: 12798 positions: 0,0,0 Entry 1: count: 49 hasNull: false min: alice allen max: zach xylophone sum: 645 positions: 0,1026,488 Bloom filters for column 8: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 1735 loadFactor: 0.2766 expectedFpp: 0.0058556325 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 182 loadFactor: 0.029 expectedFpp: 7.090246E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 1772 loadFactor: 0.2825 expectedFpp: 0.0063713384 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 2961 loadFactor: 0.3084 expectedFpp: 2.655646E-4 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 317 loadFactor: 0.033 expectedFpp: 4.2807122E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 3015 loadFactor: 0.3141 expectedFpp: 3.0137875E-4 Row group indices for column 9: Entry 0: count: 1000 hasNull: false min: 2013-03-01 09:11:58.70307 max: 2013-03-01 09:11:58.703325 positions: 0,0,0,0,0,0 Entry 1: count: 49 hasNull: false min: 2013-03-01 09:11:58.703076 max: 2013-03-01 09:11:58.703325 positions: 0,7,488,0,1538,488 Bloom filters for column 9: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 4 loadFactor: 0.0006 expectedFpp: 1.6543056E-13 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 4 loadFactor: 0.0006 expectedFpp: 1.6543056E-13 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 4 loadFactor: 0.0006 expectedFpp: 1.6543056E-13 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 7 loadFactor: 0.0007 expectedFpp: 1.0959422E-22 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 7 loadFactor: 0.0007 expectedFpp: 1.0959422E-22 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 7 loadFactor: 0.0007 expectedFpp: 1.0959422E-22 Row group indices for column 10: Entry 0: count: 1000 hasNull: false min: 0.08 max: 99.94 sum: 51182.11 positions: 0,0,0,0,0 Entry 1: count: 49 hasNull: false min: 2.48 max: 94.9 sum: 2464.05 positions: 0,2194,0,4,488 Bloom filters for column 10: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 2848 loadFactor: 0.4541 expectedFpp: 0.042514365 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 194 loadFactor: 0.0309 expectedFpp: 9.153406E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 2952 loadFactor: 0.4707 expectedFpp: 0.049072847 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 4796 loadFactor: 0.4996 expectedFpp: 0.0077670407 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 339 loadFactor: 0.0353 expectedFpp: 6.846983E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 4947 loadFactor: 0.5153 expectedFpp: 0.009649276 Row group indices for column 11: Entry 0: count: 1000 hasNull: false sum: 12674 positions: 0,0,0,0,0 Entry 1: count: 49 hasNull: false sum: 604 positions: 0,12674,0,979,20 Bloom filters for column 11: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 102 loadFactor: 0.0163 expectedFpp: 6.9948186E-8 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 98 loadFactor: 0.0156 expectedFpp: 5.9604645E-8 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 102 loadFactor: 0.0163 expectedFpp: 6.9948186E-8 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 181 loadFactor: 0.0189 expectedFpp: 8.4693775E-13 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 174 loadFactor: 0.0181 expectedFpp: 6.426078E-13 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 181 loadFactor: 0.0189 expectedFpp: 8.4693775E-13 -File length: 32344 bytes +File length: 36995 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -291,7 +291,7 @@ PREHOOK: Input: default@orc_ppd_n0 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.3 Rows: 1049 Compression: ZLIB Compression size: 262144 @@ -468,7 +468,7 @@ Stripes: Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 174 loadFactor: 0.0181 expectedFpp: 6.426078E-13 Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 181 loadFactor: 0.0189 expectedFpp: 8.4693775E-13 -File length: 36988 bytes +File length: 36995 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -501,7 +501,7 @@ PREHOOK: Input: default@orc_ppd_part@ds=2015/hr=10 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.3 Rows: 1049 Compression: ZLIB Compression size: 262144 @@ -538,49 +538,49 @@ File Statistics: Column 11: count: 1049 hasNull: false bytesOnDisk: 2468 sum: 13278 Stripes: - Stripe: offset: 3 data: 22405 rows: 1049 tail: 243 index: 8973 + Stripe: offset: 3 data: 22405 rows: 1049 tail: 239 index: 13620 Stream: column 0 section ROW_INDEX start: 3 length 20 - Stream: column 0 section BLOOM_FILTER_UTF8 start: 23 length 34 - Stream: column 1 section ROW_INDEX start: 57 length 58 - Stream: column 1 section BLOOM_FILTER_UTF8 start: 115 length 696 - Stream: column 2 section ROW_INDEX start: 811 length 58 - Stream: column 2 section BLOOM_FILTER_UTF8 start: 869 length 867 - Stream: column 3 section ROW_INDEX start: 1736 length 61 - Stream: column 3 section BLOOM_FILTER_UTF8 start: 1797 length 861 - Stream: column 4 section ROW_INDEX start: 2658 length 69 - Stream: column 4 section BLOOM_FILTER_UTF8 start: 2727 length 850 - Stream: column 5 section ROW_INDEX start: 3577 length 78 - Stream: column 5 section BLOOM_FILTER_UTF8 start: 3655 length 1172 - Stream: column 6 section ROW_INDEX start: 4827 length 85 - Stream: column 6 section BLOOM_FILTER_UTF8 start: 4912 length 1167 - Stream: column 7 section ROW_INDEX start: 6079 length 41 - Stream: column 7 section BLOOM_FILTER_UTF8 start: 6120 length 34 - Stream: column 8 section ROW_INDEX start: 6154 length 86 - Stream: column 8 section BLOOM_FILTER_UTF8 start: 6240 length 1051 - Stream: column 9 section ROW_INDEX start: 7291 length 63 - Stream: column 9 section BLOOM_FILTER_UTF8 start: 7354 length 52 - Stream: column 10 section ROW_INDEX start: 7406 length 80 - Stream: column 10 section BLOOM_FILTER_UTF8 start: 7486 length 1189 - Stream: column 11 section ROW_INDEX start: 8675 length 47 - Stream: column 11 section BLOOM_FILTER_UTF8 start: 8722 length 254 - Stream: column 1 section PRESENT start: 8976 length 17 - Stream: column 1 section DATA start: 8993 length 962 - Stream: column 2 section PRESENT start: 9955 length 17 - Stream: column 2 section DATA start: 9972 length 1441 - Stream: column 3 section DATA start: 11413 length 1704 - Stream: column 4 section DATA start: 13117 length 1998 - Stream: column 5 section DATA start: 15115 length 2925 - Stream: column 6 section DATA start: 18040 length 3323 - Stream: column 7 section DATA start: 21363 length 137 - Stream: column 8 section DATA start: 21500 length 1572 - Stream: column 8 section LENGTH start: 23072 length 310 - Stream: column 8 section DICTIONARY_DATA start: 23382 length 1548 - Stream: column 9 section DATA start: 24930 length 19 - Stream: column 9 section SECONDARY start: 24949 length 1783 - Stream: column 10 section DATA start: 26732 length 2166 - Stream: column 10 section SECONDARY start: 28898 length 15 - Stream: column 11 section DATA start: 28913 length 1877 - Stream: column 11 section LENGTH start: 30790 length 591 + Stream: column 0 section BLOOM_FILTER_UTF8 start: 23 length 43 + Stream: column 1 section ROW_INDEX start: 66 length 58 + Stream: column 1 section BLOOM_FILTER_UTF8 start: 124 length 1091 + Stream: column 2 section ROW_INDEX start: 1215 length 58 + Stream: column 2 section BLOOM_FILTER_UTF8 start: 1273 length 1372 + Stream: column 3 section ROW_INDEX start: 2645 length 61 + Stream: column 3 section BLOOM_FILTER_UTF8 start: 2706 length 1374 + Stream: column 4 section ROW_INDEX start: 4080 length 69 + Stream: column 4 section BLOOM_FILTER_UTF8 start: 4149 length 1375 + Stream: column 5 section ROW_INDEX start: 5524 length 78 + Stream: column 5 section BLOOM_FILTER_UTF8 start: 5602 length 1824 + Stream: column 6 section ROW_INDEX start: 7426 length 85 + Stream: column 6 section BLOOM_FILTER_UTF8 start: 7511 length 1795 + Stream: column 7 section ROW_INDEX start: 9306 length 41 + Stream: column 7 section BLOOM_FILTER_UTF8 start: 9347 length 43 + Stream: column 8 section ROW_INDEX start: 9390 length 86 + Stream: column 8 section BLOOM_FILTER_UTF8 start: 9476 length 1653 + Stream: column 9 section ROW_INDEX start: 11129 length 63 + Stream: column 9 section BLOOM_FILTER_UTF8 start: 11192 length 80 + Stream: column 10 section ROW_INDEX start: 11272 length 80 + Stream: column 10 section BLOOM_FILTER_UTF8 start: 11352 length 1802 + Stream: column 11 section ROW_INDEX start: 13154 length 47 + Stream: column 11 section BLOOM_FILTER_UTF8 start: 13201 length 422 + Stream: column 1 section PRESENT start: 13623 length 17 + Stream: column 1 section DATA start: 13640 length 962 + Stream: column 2 section PRESENT start: 14602 length 17 + Stream: column 2 section DATA start: 14619 length 1441 + Stream: column 3 section DATA start: 16060 length 1704 + Stream: column 4 section DATA start: 17764 length 1998 + Stream: column 5 section DATA start: 19762 length 2925 + Stream: column 6 section DATA start: 22687 length 3323 + Stream: column 7 section DATA start: 26010 length 137 + Stream: column 8 section DATA start: 26147 length 1572 + Stream: column 8 section LENGTH start: 27719 length 310 + Stream: column 8 section DICTIONARY_DATA start: 28029 length 1548 + Stream: column 9 section DATA start: 29577 length 19 + Stream: column 9 section SECONDARY start: 29596 length 1783 + Stream: column 10 section DATA start: 31379 length 2166 + Stream: column 10 section SECONDARY start: 33545 length 15 + Stream: column 11 section DATA start: 33560 length 1877 + Stream: column 11 section LENGTH start: 35437 length 591 Encoding column 0: DIRECT Encoding column 1: DIRECT Encoding column 2: DIRECT_V2 @@ -597,88 +597,88 @@ Stripes: Entry 0: count: 1000 hasNull: false positions: Entry 1: count: 49 hasNull: false positions: Bloom filters for column 0: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 Row group indices for column 1: Entry 0: count: 997 hasNull: true min: -3 max: 124 sum: 59325 positions: 0,0,0,0,0,0,0 Entry 1: count: 49 hasNull: false min: 2 max: 123 sum: 3105 positions: 0,10,113,0,0,903,101 Bloom filters for column 1: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 492 loadFactor: 0.0784 expectedFpp: 3.7864847E-5 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 168 loadFactor: 0.0268 expectedFpp: 5.147697E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 492 loadFactor: 0.0784 expectedFpp: 3.7864847E-5 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 849 loadFactor: 0.0884 expectedFpp: 4.231118E-8 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 285 loadFactor: 0.0297 expectedFpp: 2.0324289E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 849 loadFactor: 0.0884 expectedFpp: 4.231118E-8 Row group indices for column 2: Entry 0: count: 997 hasNull: true min: 256 max: 511 sum: 381766 positions: 0,0,0,0,0,0,0 Entry 1: count: 49 hasNull: false min: 256 max: 507 sum: 17123 positions: 0,10,114,0,0,1026,485 Bloom filters for column 2: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 928 loadFactor: 0.148 expectedFpp: 4.792562E-4 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 175 loadFactor: 0.0279 expectedFpp: 6.0607726E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 928 loadFactor: 0.148 expectedFpp: 4.792562E-4 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 1617 loadFactor: 0.1684 expectedFpp: 3.8465505E-6 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 306 loadFactor: 0.0319 expectedFpp: 3.343115E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 1617 loadFactor: 0.1684 expectedFpp: 3.8465505E-6 Row group indices for column 3: Entry 0: count: 1000 hasNull: false min: 65536 max: 65791 sum: 65662251 positions: 0,0,0 Entry 1: count: 49 hasNull: false min: 65540 max: 65788 sum: 3218800 positions: 0,1538,488 Bloom filters for column 3: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 947 loadFactor: 0.151 expectedFpp: 5.1972753E-4 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 171 loadFactor: 0.0273 expectedFpp: 5.525356E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 951 loadFactor: 0.1516 expectedFpp: 5.285643E-4 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 1632 loadFactor: 0.17 expectedFpp: 4.1033873E-6 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 296 loadFactor: 0.0308 expectedFpp: 2.6493746E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 1638 loadFactor: 0.1706 expectedFpp: 4.2101606E-6 Row group indices for column 4: Entry 0: count: 1000 hasNull: false min: 4294967296 max: 4294967551 sum: 4294967421966 positions: 0,0,0 Entry 1: count: 49 hasNull: false min: 4294967301 max: 4294967550 sum: 210453403987 positions: 0,2885,422 Bloom filters for column 4: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 900 loadFactor: 0.1435 expectedFpp: 4.2398053E-4 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 177 loadFactor: 0.0282 expectedFpp: 6.342622E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 905 loadFactor: 0.1443 expectedFpp: 4.3348098E-4 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 1611 loadFactor: 0.1678 expectedFpp: 3.7477455E-6 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 318 loadFactor: 0.0331 expectedFpp: 4.3761383E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 1622 loadFactor: 0.169 expectedFpp: 3.9305864E-6 Row group indices for column 5: Entry 0: count: 1000 hasNull: false min: 0.07999999821186066 max: 99.91999816894531 sum: 49977.420025609434 positions: 0,0 Entry 1: count: 49 hasNull: false min: 2.7899999618530273 max: 97.6500015258789 sum: 2767.280002593994 positions: 0,4000 Bloom filters for column 5: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 2846 loadFactor: 0.4538 expectedFpp: 0.042395078 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 192 loadFactor: 0.0306 expectedFpp: 8.781743E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 2940 loadFactor: 0.4688 expectedFpp: 0.048279762 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 4847 loadFactor: 0.5049 expectedFpp: 0.008363968 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 341 loadFactor: 0.0355 expectedFpp: 7.134803E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 4989 loadFactor: 0.5197 expectedFpp: 0.010237543 Row group indices for column 6: Entry 0: count: 1000 hasNull: false min: 0.02 max: 49.85 sum: 25045.26999999998 positions: 0,0 Entry 1: count: 49 hasNull: false min: 1.71 max: 48.78 sum: 1241.08 positions: 0,8000 Bloom filters for column 6: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 2696 loadFactor: 0.4298 expectedFpp: 0.03413936 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 195 loadFactor: 0.0311 expectedFpp: 9.3436E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 2792 loadFactor: 0.4452 expectedFpp: 0.03926788 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 4562 loadFactor: 0.4752 expectedFpp: 0.005472533 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 337 loadFactor: 0.0351 expectedFpp: 6.569173E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 4710 loadFactor: 0.4906 expectedFpp: 0.006843018 Row group indices for column 7: Entry 0: count: 1000 hasNull: false true: 501 positions: 0,0,0,0 Entry 1: count: 49 hasNull: false true: 25 positions: 0,0,125,0 Bloom filters for column 7: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 0 loadFactor: 0 expectedFpp: 0.0 Row group indices for column 8: Entry 0: count: 1000 hasNull: false min: max: zach zipper sum: 12798 positions: 0,0,0 Entry 1: count: 49 hasNull: false min: alice allen max: zach xylophone sum: 645 positions: 0,1026,488 Bloom filters for column 8: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 1735 loadFactor: 0.2766 expectedFpp: 0.0058556325 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 182 loadFactor: 0.029 expectedFpp: 7.090246E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 1772 loadFactor: 0.2825 expectedFpp: 0.0063713384 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 2961 loadFactor: 0.3084 expectedFpp: 2.655646E-4 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 317 loadFactor: 0.033 expectedFpp: 4.2807122E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 3015 loadFactor: 0.3141 expectedFpp: 3.0137875E-4 Row group indices for column 9: Entry 0: count: 1000 hasNull: false min: 2013-03-01 09:11:58.70307 max: 2013-03-01 09:11:58.703325 positions: 0,0,0,0,0,0 Entry 1: count: 49 hasNull: false min: 2013-03-01 09:11:58.703076 max: 2013-03-01 09:11:58.703325 positions: 0,7,488,0,1538,488 Bloom filters for column 9: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 4 loadFactor: 0.0006 expectedFpp: 1.6543056E-13 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 4 loadFactor: 0.0006 expectedFpp: 1.6543056E-13 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 4 loadFactor: 0.0006 expectedFpp: 1.6543056E-13 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 7 loadFactor: 0.0007 expectedFpp: 1.0959422E-22 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 7 loadFactor: 0.0007 expectedFpp: 1.0959422E-22 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 7 loadFactor: 0.0007 expectedFpp: 1.0959422E-22 Row group indices for column 10: Entry 0: count: 1000 hasNull: false min: 0.08 max: 99.94 sum: 51182.11 positions: 0,0,0,0,0 Entry 1: count: 49 hasNull: false min: 2.48 max: 94.9 sum: 2464.05 positions: 0,2194,0,4,488 Bloom filters for column 10: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 2848 loadFactor: 0.4541 expectedFpp: 0.042514365 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 194 loadFactor: 0.0309 expectedFpp: 9.153406E-7 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 2952 loadFactor: 0.4707 expectedFpp: 0.049072847 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 4796 loadFactor: 0.4996 expectedFpp: 0.0077670407 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 339 loadFactor: 0.0353 expectedFpp: 6.846983E-11 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 4947 loadFactor: 0.5153 expectedFpp: 0.009649276 Row group indices for column 11: Entry 0: count: 1000 hasNull: false sum: 12674 positions: 0,0,0,0,0 Entry 1: count: 49 hasNull: false sum: 604 positions: 0,12674,0,979,20 Bloom filters for column 11: - Entry 0: numHashFunctions: 4 bitCount: 6272 popCount: 102 loadFactor: 0.0163 expectedFpp: 6.9948186E-8 - Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 98 loadFactor: 0.0156 expectedFpp: 5.9604645E-8 - Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 102 loadFactor: 0.0163 expectedFpp: 6.9948186E-8 + Entry 0: numHashFunctions: 7 bitCount: 9600 popCount: 181 loadFactor: 0.0189 expectedFpp: 8.4693775E-13 + Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 174 loadFactor: 0.0181 expectedFpp: 6.426078E-13 + Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 181 loadFactor: 0.0189 expectedFpp: 8.4693775E-13 -File length: 32344 bytes +File length: 36995 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index 3a270924a3b2..79029107e957 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -239,7 +239,7 @@ Table Parameters: orc.bloom.filter.columns * orc.row.index.stride 1000 rawDataSize 1139514 - totalSize 55696 + totalSize 64520 #### A masked pattern was here #### # Storage Information @@ -257,7 +257,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n1 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 16680 + HDFS_BYTES_READ: 16679 HDFS_BYTES_WRITTEN: 104 HDFS_READ_OPS: 5 HDFS_LARGE_READ_OPS: 0 @@ -319,7 +319,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n1 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1047 + HDFS_BYTES_READ: 1471 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -341,9 +341,9 @@ Stage-1 HIVE COUNTERS: TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1048576 - ALLOCATED_USED_BYTES: 2731 + ALLOCATED_USED_BYTES: 3979 CACHE_HIT_BYTES: 24 - CACHE_MISS_BYTES: 1047 + CACHE_MISS_BYTES: 1471 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 @@ -381,7 +381,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -420,7 +420,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -459,7 +459,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -498,7 +498,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -537,7 +537,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -576,7 +576,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -615,7 +615,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -654,7 +654,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -693,7 +693,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -732,7 +732,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -771,7 +771,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -810,7 +810,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -895,7 +895,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 823 + CACHE_HIT_BYTES: 1247 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -931,7 +931,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 823 + CACHE_HIT_BYTES: 1247 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -969,7 +969,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n1 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 5911 + HDFS_BYTES_READ: 6850 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -991,9 +991,9 @@ Stage-1 HIVE COUNTERS: TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1310720 - ALLOCATED_USED_BYTES: 13812 + ALLOCATED_USED_BYTES: 15060 CACHE_HIT_BYTES: 24 - CACHE_MISS_BYTES: 5911 + CACHE_MISS_BYTES: 6850 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 @@ -1031,7 +1031,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1070,7 +1070,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1132,7 +1132,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -1171,7 +1171,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1728 + CACHE_HIT_BYTES: 2667 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1207,7 +1207,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1246,7 +1246,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1285,7 +1285,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1324,7 +1324,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1363,7 +1363,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1402,7 +1402,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1441,7 +1441,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1480,7 +1480,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1519,7 +1519,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1558,7 +1558,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1728 + CACHE_HIT_BYTES: 2667 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1594,7 +1594,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1728 + CACHE_HIT_BYTES: 2667 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1630,7 +1630,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1728 + CACHE_HIT_BYTES: 2667 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1666,7 +1666,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1705,7 +1705,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1744,7 +1744,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 6982 + CACHE_HIT_BYTES: 8345 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1783,7 +1783,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 6982 + CACHE_HIT_BYTES: 8345 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out index 24b42becd299..59348799e8c3 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out @@ -239,7 +239,7 @@ Table Parameters: orc.bloom.filter.columns * orc.row.index.stride 1000 rawDataSize 1139514 - totalSize 55696 + totalSize 64520 #### A masked pattern was here #### # Storage Information @@ -257,7 +257,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 17727 + HDFS_BYTES_READ: 18150 HDFS_BYTES_WRITTEN: 104 HDFS_READ_OPS: 5 HDFS_LARGE_READ_OPS: 0 @@ -279,9 +279,9 @@ Stage-1 HIVE COUNTERS: TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1310720 - ALLOCATED_USED_BYTES: 2757 + ALLOCATED_USED_BYTES: 4005 CACHE_HIT_BYTES: 0 - CACHE_MISS_BYTES: 1071 + CACHE_MISS_BYTES: 1495 METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 3 NUM_VECTOR_BATCHES: 3 @@ -319,7 +319,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out index 9864e4838cf1..ee7ef8b210a0 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out @@ -1348,7 +1348,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: orc_llap_nonvector_2 - Statistics: Num rows: 12288 Data size: 4468180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: 4468250 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/orc_merge1.q.out b/ql/src/test/results/clientpositive/llap/orc_merge1.q.out index e28e2e647b35..c1e508eefd15 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge1.q.out @@ -167,12 +167,12 @@ POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).value SIMPLE [(src)s POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 6 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 557 ### HDFS DATE ### hdfs://### HDFS PATH ### -rw-rw-rw- 3 ### USER ### ### GROUP ### 564 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 571 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 570 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 505 ### HDFS DATE ### hdfs://### HDFS PATH ### -rw-rw-rw- 3 ### USER ### ### GROUP ### 563 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 498 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 556 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 480 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 487 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b_n1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -350,7 +350,7 @@ POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=0).value SIMPLE [(src) POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1362 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1369 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1c_n1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -520,7 +520,7 @@ POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=0).value SIMPLE [(src) POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2463 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 2470 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1_n1 WHERE ds='1' diff --git a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out index 7784e6e5aeee..864345f908a3 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out @@ -182,7 +182,7 @@ POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).value SIMPLE [(src)src. POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1756 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1763 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -375,7 +375,7 @@ POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=0).value SIMPLE [(src)src POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1756 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1763 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -560,7 +560,7 @@ POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=0).value SIMPLE [(src)src POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1756 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1763 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1 WHERE ds='1' @@ -672,7 +672,7 @@ POSTHOOK: type: ALTER_PARTITION_MERGE POSTHOOK: Input: default@orcfile_merge1 POSTHOOK: Output: default@orcfile_merge1@ds=1/part=0 Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1756 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1763 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1c WHERE ds='1' @@ -744,7 +744,7 @@ PREHOOK: Input: default@orcfile_merge1@ds=1/part=0 PREHOOK: Output: hdfs://### HDFS PATH ### -- BEGIN ORC FILE DUMP -- Structure for hdfs://### HDFS PATH ### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.3 Rows: 242 Compression: SNAPPY Compression size: 4096 @@ -781,7 +781,7 @@ Stripes: Row group indices for column 2: Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0 -File length: 1756 bytes +File length: 1763 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -795,7 +795,7 @@ PREHOOK: Input: default@orcfile_merge1c@ds=1/part=0 PREHOOK: Output: hdfs://### HDFS PATH ### -- BEGIN ORC FILE DUMP -- Structure for hdfs://### HDFS PATH ### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.3 Rows: 242 Compression: SNAPPY Compression size: 4096 @@ -832,7 +832,7 @@ Stripes: Row group indices for column 2: Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0 -File length: 1756 bytes +File length: 1763 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ diff --git a/ql/src/test/results/clientpositive/llap/orc_merge11.q.out b/ql/src/test/results/clientpositive/llap/orc_merge11.q.out index dbc6d828d5de..fe6335c90ade 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge11.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge11.q.out @@ -72,7 +72,7 @@ PREHOOK: Input: default@orcfile_merge1_n2 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.3 Rows: 50000 Compression: ZLIB Compression size: 4096 @@ -156,7 +156,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,506,294,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,666,54,0,312,64 -File length: 6685 bytes +File length: 6692 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -164,7 +164,7 @@ ________________________________________________________________________________ -- END ORC FILE DUMP -- -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.3 Rows: 50000 Compression: ZLIB Compression size: 4096 @@ -248,7 +248,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,506,294,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,666,54,0,312,64 -File length: 6685 bytes +File length: 6692 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -277,7 +277,7 @@ PREHOOK: Input: default@orcfile_merge1_n2 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.3 Rows: 100000 Compression: ZLIB Compression size: 4096 @@ -426,7 +426,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,506,294,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,666,54,0,312,64 -File length: 12994 bytes +File length: 13001 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ diff --git a/ql/src/test/results/clientpositive/llap/orc_merge2.q.out b/ql/src/test/results/clientpositive/llap/orc_merge2.q.out index a9708e303cfd..1964f1c03d60 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge2.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge2.q.out @@ -213,7 +213,7 @@ POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=1).value SIMPL POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 351 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 358 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge2a_n0 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge3.q.out b/ql/src/test/results/clientpositive/llap/orc_merge3.q.out index a4da817607a0..82a32433fd0c 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge3.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge3.q.out @@ -168,7 +168,7 @@ POSTHOOK: Output: default@orcfile_merge3b_n0 POSTHOOK: Lineage: orcfile_merge3b_n0.key SIMPLE [(orcfile_merge3a_n0)orcfile_merge3a_n0.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: orcfile_merge3b_n0.value SIMPLE [(orcfile_merge3a_n0)orcfile_merge3a_n0.FieldSchema(name:value, type:string, comment:null), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2574 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 2581 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) FROM orcfile_merge3a_n0 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge4.q.out b/ql/src/test/results/clientpositive/llap/orc_merge4.q.out index 7c28d26b5cfd..8d784f8fae6f 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge4.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge4.q.out @@ -37,7 +37,7 @@ POSTHOOK: Output: default@orcfile_merge3a@ds=1 POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2532 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 2539 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') SELECT * FROM src PREHOOK: type: QUERY @@ -63,9 +63,9 @@ POSTHOOK: Output: default@orcfile_merge3a@ds=2 POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2532 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 2539 ### HDFS DATE ### hdfs://### HDFS PATH ### Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2532 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 2539 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b SELECT key, value FROM orcfile_merge3a PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index 27346e0a737d..af114d90bf17 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -207,7 +207,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n2 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 16680 + HDFS_BYTES_READ: 16679 HDFS_BYTES_WRITTEN: 104 HDFS_READ_OPS: 5 HDFS_LARGE_READ_OPS: 0 @@ -269,7 +269,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n2 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1047 + HDFS_BYTES_READ: 1471 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -291,9 +291,9 @@ Stage-1 HIVE COUNTERS: TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1048576 - ALLOCATED_USED_BYTES: 2731 + ALLOCATED_USED_BYTES: 3979 CACHE_HIT_BYTES: 24 - CACHE_MISS_BYTES: 1047 + CACHE_MISS_BYTES: 1471 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 @@ -331,7 +331,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -370,7 +370,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -409,7 +409,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -448,7 +448,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -487,7 +487,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -526,7 +526,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -565,7 +565,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -604,7 +604,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -643,7 +643,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -682,7 +682,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -721,7 +721,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -760,7 +760,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -845,7 +845,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 823 + CACHE_HIT_BYTES: 1247 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -881,7 +881,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 823 + CACHE_HIT_BYTES: 1247 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -919,7 +919,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n2 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 5911 + HDFS_BYTES_READ: 6850 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -941,9 +941,9 @@ Stage-1 HIVE COUNTERS: TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1310720 - ALLOCATED_USED_BYTES: 13812 + ALLOCATED_USED_BYTES: 15060 CACHE_HIT_BYTES: 24 - CACHE_MISS_BYTES: 5911 + CACHE_MISS_BYTES: 6850 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 @@ -981,7 +981,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1020,7 +1020,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1082,7 +1082,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -1121,7 +1121,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1728 + CACHE_HIT_BYTES: 2667 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1157,7 +1157,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1196,7 +1196,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1235,7 +1235,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1274,7 +1274,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1313,7 +1313,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1352,7 +1352,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1391,7 +1391,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1430,7 +1430,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1469,7 +1469,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1508,7 +1508,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1728 + CACHE_HIT_BYTES: 2667 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1544,7 +1544,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1728 + CACHE_HIT_BYTES: 2667 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1580,7 +1580,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 0 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1728 + CACHE_HIT_BYTES: 2667 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -1616,7 +1616,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1655,7 +1655,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1694,7 +1694,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 6982 + CACHE_HIT_BYTES: 8345 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1733,7 +1733,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 6982 + CACHE_HIT_BYTES: 8345 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1792,7 +1792,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n2 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1759 + HDFS_BYTES_READ: 2763 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1814,9 +1814,9 @@ Stage-1 HIVE COUNTERS: TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 - ALLOCATED_USED_BYTES: 2376 + ALLOCATED_USED_BYTES: 3624 CACHE_HIT_BYTES: 4916 - CACHE_MISS_BYTES: 1759 + CACHE_MISS_BYTES: 2763 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 @@ -1893,7 +1893,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 6675 + CACHE_HIT_BYTES: 7679 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1912,8 +1912,8 @@ PREHOOK: Input: default@orc_ppd_staging_n1 PREHOOK: Output: database:default PREHOOK: Output: default@orc_ppd_1 Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 10583 - HDFS_BYTES_WRITTEN: 1468 + HDFS_BYTES_READ: 11581 + HDFS_BYTES_WRITTEN: 1475 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 3 @@ -1931,9 +1931,9 @@ Stage-1 HIVE COUNTERS: TOTAL_TABLE_ROWS_WRITTEN: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 2359296 - ALLOCATED_USED_BYTES: 44198 + ALLOCATED_USED_BYTES: 45446 CACHE_HIT_BYTES: 30633 - CACHE_MISS_BYTES: 10583 + CACHE_MISS_BYTES: 11581 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 @@ -1949,7 +1949,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_1 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1592 + HDFS_BYTES_READ: 1599 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 5 HDFS_LARGE_READ_OPS: 0 diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out index a93503c8998c..c6f3862a37f9 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_schema_evol_3a.q.out @@ -207,7 +207,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 17010 + HDFS_BYTES_READ: 17009 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 5 HDFS_LARGE_READ_OPS: 0 @@ -310,7 +310,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 717 + HDFS_BYTES_READ: 1141 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -332,9 +332,9 @@ Stage-1 HIVE COUNTERS: TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 - ALLOCATED_USED_BYTES: 2376 + ALLOCATED_USED_BYTES: 3624 CACHE_HIT_BYTES: 354 - CACHE_MISS_BYTES: 717 + CACHE_MISS_BYTES: 1141 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 @@ -411,7 +411,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -555,7 +555,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -633,7 +633,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -777,7 +777,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -855,7 +855,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -999,7 +999,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 1000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 @@ -1077,7 +1077,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2000 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 1071 + CACHE_HIT_BYTES: 1495 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 2 @@ -1099,7 +1099,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 17176 + HDFS_BYTES_READ: 17174 HDFS_BYTES_WRITTEN: 104 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1132,7 +1132,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 17999 + HDFS_BYTES_READ: 18421 HDFS_BYTES_WRITTEN: 104 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1165,7 +1165,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 17176 + HDFS_BYTES_READ: 17174 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1198,7 +1198,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 17999 + HDFS_BYTES_READ: 18421 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1231,7 +1231,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 17176 + HDFS_BYTES_READ: 17174 HDFS_BYTES_WRITTEN: 102 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1264,7 +1264,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 17999 + HDFS_BYTES_READ: 18421 HDFS_BYTES_WRITTEN: 102 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1338,7 +1338,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1759 + HDFS_BYTES_READ: 2763 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1360,9 +1360,9 @@ Stage-1 HIVE COUNTERS: TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 - ALLOCATED_USED_BYTES: 2376 + ALLOCATED_USED_BYTES: 3624 CACHE_HIT_BYTES: 4916 - CACHE_MISS_BYTES: 1759 + CACHE_MISS_BYTES: 2763 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 1 NUM_VECTOR_BATCHES: 1 @@ -1383,7 +1383,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 21717 + HDFS_BYTES_READ: 21715 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1416,7 +1416,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 23603 + HDFS_BYTES_READ: 24605 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1453,7 +1453,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 21717 + HDFS_BYTES_READ: 21715 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1486,7 +1486,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 23603 + HDFS_BYTES_READ: 24605 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1560,7 +1560,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1585 + HDFS_BYTES_READ: 2524 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1582,9 +1582,9 @@ Stage-1 HIVE COUNTERS: TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 - ALLOCATED_USED_BYTES: 2376 + ALLOCATED_USED_BYTES: 3624 CACHE_HIT_BYTES: 4350 - CACHE_MISS_BYTES: 1585 + CACHE_MISS_BYTES: 2524 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 NUM_VECTOR_BATCHES: 3 @@ -1605,7 +1605,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 21135 + HDFS_BYTES_READ: 21133 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1638,7 +1638,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 22863 + HDFS_BYTES_READ: 23800 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1675,7 +1675,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 21135 + HDFS_BYTES_READ: 21133 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1708,7 +1708,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 22863 + HDFS_BYTES_READ: 23800 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1745,7 +1745,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 21135 + HDFS_BYTES_READ: 21133 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1778,7 +1778,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 22863 + HDFS_BYTES_READ: 23800 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1875,7 +1875,7 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_OPERATOR_TS_0: 2100 TOTAL_TABLE_ROWS_WRITTEN: 0 Stage-1 LLAP IO COUNTERS: - CACHE_HIT_BYTES: 5935 + CACHE_HIT_BYTES: 6874 CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 NUM_DECODED_BATCHES: 3 @@ -1897,7 +1897,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 18902 + HDFS_BYTES_READ: 18900 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1930,7 +1930,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 18902 + HDFS_BYTES_READ: 18900 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1963,7 +1963,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 20221 + HDFS_BYTES_READ: 20986 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 @@ -1996,7 +1996,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_n3 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 20221 + HDFS_BYTES_READ: 20986 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 3 HDFS_LARGE_READ_OPS: 0 diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out index 8f3f96aebcce..c1ce4e06cc52 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out @@ -294,7 +294,7 @@ Table Parameters: numPartitions 2 numRows 8 rawDataSize 1148 - totalSize 852 + totalSize 866 #### A masked pattern was here #### # Storage Information @@ -331,7 +331,7 @@ Partition Parameters: numFiles 1 numRows 4 rawDataSize 400 - totalSize 325 + totalSize 332 #### A masked pattern was here #### # Storage Information @@ -370,7 +370,7 @@ Partition Parameters: numFiles 1 numRows 4 rawDataSize 748 - totalSize 527 + totalSize 534 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out b/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out index b757bbc9d773..96b9952adc63 100644 --- a/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out @@ -91,7 +91,7 @@ Table Parameters: numFiles 1 numRows 1 rawDataSize 0 - totalSize 659 + totalSize 666 transactional true transactional_properties default #### A masked pattern was here #### @@ -447,7 +447,7 @@ Table Parameters: numFiles 4 numRows 2 rawDataSize 0 - totalSize 2703 + totalSize 2730 transactional true transactional_properties default #### A masked pattern was here #### @@ -511,7 +511,7 @@ Table Parameters: numFiles 6 numRows 0 rawDataSize 0 - totalSize 4075 + totalSize 4116 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/stats_cbo_1.q.out b/ql/src/test/results/clientpositive/llap/stats_cbo_1.q.out index 65684ffac394..47e2405c15f9 100644 --- a/ql/src/test/results/clientpositive/llap/stats_cbo_1.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_cbo_1.q.out @@ -175,7 +175,7 @@ STAGE PLANS: TableScan alias: repro filterExpr: i is not null (type: boolean) - Statistics: Num rows: 1 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: i is not null (type: boolean) Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE @@ -202,7 +202,7 @@ STAGE PLANS: TableScan alias: repro filterExpr: i is not null (type: boolean) - Statistics: Num rows: 1 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: i is not null (type: boolean) Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/stats_histogram.q.out b/ql/src/test/results/clientpositive/llap/stats_histogram.q.out index ab433500e7b9..5be73b3d1ba5 100644 --- a/ql/src/test/results/clientpositive/llap/stats_histogram.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_histogram.q.out @@ -260,7 +260,7 @@ Table Parameters: numFiles 15 numRows 15 rawDataSize 4287 - totalSize 11029 + totalSize 11142 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/stats_nonpart.q.out b/ql/src/test/results/clientpositive/llap/stats_nonpart.q.out index 3136e352e5b5..15f2101b9335 100644 --- a/ql/src/test/results/clientpositive/llap/stats_nonpart.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_nonpart.q.out @@ -239,7 +239,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 0 - totalSize 720 + totalSize 727 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/stats_part.q.out b/ql/src/test/results/clientpositive/llap/stats_part.q.out index 5a208e0a37c1..55a7a2fc08be 100644 --- a/ql/src/test/results/clientpositive/llap/stats_part.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_part.q.out @@ -227,7 +227,7 @@ Table Parameters: numPartitions 3 numRows 6 rawDataSize 0 - totalSize 2274 + totalSize 2292 transactional true transactional_properties default #### A masked pattern was here #### @@ -288,7 +288,7 @@ Table Parameters: numPartitions 3 numRows 8 rawDataSize 0 - totalSize 3042 + totalSize 3066 transactional true transactional_properties default #### A masked pattern was here #### @@ -445,7 +445,7 @@ Table Parameters: numPartitions 3 numRows 8 rawDataSize 0 - totalSize 3042 + totalSize 3066 transactional true transactional_properties default #### A masked pattern was here #### @@ -530,7 +530,7 @@ Table Parameters: numPartitions 3 numRows 8 rawDataSize 0 - totalSize 3042 + totalSize 3066 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/stats_part2.q.out b/ql/src/test/results/clientpositive/llap/stats_part2.q.out index e90f93a03c50..80f8883c7733 100644 --- a/ql/src/test/results/clientpositive/llap/stats_part2.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_part2.q.out @@ -275,7 +275,7 @@ Table Parameters: numPartitions 3 numRows 6 rawDataSize 0 - totalSize 4387 + totalSize 4425 transactional true transactional_properties default #### A masked pattern was here #### @@ -372,7 +372,7 @@ Table Parameters: numPartitions 3 numRows 8 rawDataSize 0 - totalSize 5850 + totalSize 5899 transactional true transactional_properties default #### A masked pattern was here #### @@ -526,7 +526,7 @@ Partition Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1459 + totalSize 1471 #### A masked pattern was here #### # Storage Information @@ -561,7 +561,7 @@ Partition Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1464 + totalSize 1477 #### A masked pattern was here #### # Storage Information @@ -596,7 +596,7 @@ Partition Parameters: numFiles 4 numRows 4 rawDataSize 0 - totalSize 2927 + totalSize 2951 #### A masked pattern was here #### # Storage Information @@ -732,7 +732,7 @@ Partition Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1459 + totalSize 1471 #### A masked pattern was here #### # Storage Information @@ -767,7 +767,7 @@ Partition Parameters: numFiles 4 numRows 2 rawDataSize 0 - totalSize 2919 + totalSize 2945 #### A masked pattern was here #### # Storage Information @@ -802,7 +802,7 @@ Partition Parameters: numFiles 4 numRows 4 rawDataSize 0 - totalSize 2927 + totalSize 2951 #### A masked pattern was here #### # Storage Information @@ -868,7 +868,7 @@ Partition Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1459 + totalSize 1471 #### A masked pattern was here #### # Storage Information @@ -903,7 +903,7 @@ Partition Parameters: numFiles 4 numRows 2 rawDataSize 0 - totalSize 2919 + totalSize 2945 #### A masked pattern was here #### # Storage Information @@ -938,7 +938,7 @@ Partition Parameters: numFiles 4 numRows 4 rawDataSize 0 - totalSize 2927 + totalSize 2951 #### A masked pattern was here #### # Storage Information @@ -1006,7 +1006,7 @@ Partition Parameters: numFiles 3 numRows 1 rawDataSize 0 - totalSize 2163 + totalSize 2182 #### A masked pattern was here #### # Storage Information @@ -1041,7 +1041,7 @@ Partition Parameters: numFiles 5 numRows 1 rawDataSize 0 - totalSize 3617 + totalSize 3649 #### A masked pattern was here #### # Storage Information @@ -1076,7 +1076,7 @@ Partition Parameters: numFiles 4 numRows 4 rawDataSize 0 - totalSize 2927 + totalSize 2951 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/stats_part_multi_insert_acid.q.out b/ql/src/test/results/clientpositive/llap/stats_part_multi_insert_acid.q.out index c40f7d765208..1850b32d131d 100644 --- a/ql/src/test/results/clientpositive/llap/stats_part_multi_insert_acid.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_part_multi_insert_acid.q.out @@ -81,7 +81,7 @@ Table Parameters: numPartitions 1 numRows 2 rawDataSize 0 - totalSize 1470 + totalSize 1483 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/stats_sizebug.q.out b/ql/src/test/results/clientpositive/llap/stats_sizebug.q.out index 051ccca3c339..15f2c2d481b7 100644 --- a/ql/src/test/results/clientpositive/llap/stats_sizebug.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_sizebug.q.out @@ -177,7 +177,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 0 - totalSize 720 + totalSize 726 transactional true transactional_properties default #### A masked pattern was here #### @@ -222,7 +222,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 0 - totalSize 720 + totalSize 726 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out index fe2abd5916e2..d1462a2a7ae5 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out @@ -3908,7 +3908,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesnullorc - Statistics: Num rows: 12288 Data size: 9470 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: 9540 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true Select Operator @@ -3916,7 +3916,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [] - Statistics: Num rows: 12288 Data size: 9470 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: 9540 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() Group By Vectorization: diff --git a/ql/src/test/results/clientpositive/row__id.q.out b/ql/src/test/results/clientpositive/row__id.q.out index a1123ac7cfef..d67d56426122 100644 --- a/ql/src/test/results/clientpositive/row__id.q.out +++ b/ql/src/test/results/clientpositive/row__id.q.out @@ -72,25 +72,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hello_acid - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20144 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID.writeid (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20144 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: bigint) null sort order: z sort order: + - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20144 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20144 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20144 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -146,17 +146,17 @@ STAGE PLANS: TableScan alias: hello_acid filterExpr: (ROW__ID.writeid = 3L) (type: boolean) - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20144 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (ROW__ID.writeid = 3L) (type: boolean) - Statistics: Num rows: 1 Data size: 6648 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 6714 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID.writeid (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 6648 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 6714 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 6648 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 6714 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out index 57a21e9a8693..4fc991901e81 100644 --- a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out +++ b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out @@ -372,10 +372,10 @@ POSTHOOK: Lineage: over10k_orc_bucketed_n0.si SIMPLE [(over10k_n9)over10k_n9.Fie POSTHOOK: Lineage: over10k_orc_bucketed_n0.t SIMPLE [(over10k_n9)over10k_n9.FieldSchema(name:t, type:tinyint, comment:null), ] POSTHOOK: Lineage: over10k_orc_bucketed_n0.ts SIMPLE [(over10k_n9)over10k_n9.FieldSchema(name:ts, type:timestamp, comment:null), ] Found 4 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 8746 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7525 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7167 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7058 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 8753 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7531 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7174 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7066 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: insert into over10k_orc_bucketed_n0 select * from over10k_n9 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n9 @@ -396,14 +396,14 @@ POSTHOOK: Lineage: over10k_orc_bucketed_n0.si SIMPLE [(over10k_n9)over10k_n9.Fie POSTHOOK: Lineage: over10k_orc_bucketed_n0.t SIMPLE [(over10k_n9)over10k_n9.FieldSchema(name:t, type:tinyint, comment:null), ] POSTHOOK: Lineage: over10k_orc_bucketed_n0.ts SIMPLE [(over10k_n9)over10k_n9.FieldSchema(name:ts, type:timestamp, comment:null), ] Found 8 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 8746 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 8746 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7525 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7525 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7167 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7167 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7058 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7058 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 8753 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 8753 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7531 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7531 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7174 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7174 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7066 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7066 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: select distinct 7 as seven, INPUT__FILE__NAME from over10k_orc_bucketed_n0 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_orc_bucketed_n0 diff --git a/ql/src/test/results/clientpositive/tez/orc_merge12.q.out b/ql/src/test/results/clientpositive/tez/orc_merge12.q.out index c7615751fd65..d7f278b466ba 100644 --- a/ql/src/test/results/clientpositive/tez/orc_merge12.q.out +++ b/ql/src/test/results/clientpositive/tez/orc_merge12.q.out @@ -144,7 +144,7 @@ PREHOOK: Input: default@alltypesorc3xcols PREHOOK: Output: hdfs://### HDFS PATH ### -- BEGIN ORC FILE DUMP -- Structure for hdfs://### HDFS PATH ### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.3 Rows: 24576 Compression: ZLIB Compression size: 131072 @@ -814,7 +814,7 @@ Stripes: Entry 0: count: 6889 hasNull: true true: 3402 positions: 0,0,0,0,0,0,0,0 Entry 1: count: 2284 hasNull: true true: 581 positions: 0,168,8,0,0,520,97,1 -File length: 3004632 bytes +File length: 3004637 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 212c72d79fa5..25b7ff2b497d 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -88,7 +88,7 @@ 0.16.0 2.18.0 3.3.3 - 1.6.9 + 1.8.3 3.21.7 1.51.0 1.9.0