diff --git a/beeline/pom.xml b/beeline/pom.xml index c3b65f283fa7..42cb1d6ec2bc 100644 --- a/beeline/pom.xml +++ b/beeline/pom.xml @@ -84,6 +84,11 @@ jline jline + + org.apache.hadoop + hadoop-hdfs + true + org.apache.hadoop hadoop-common diff --git a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out index a5a04313c9cc..1cd23dafc416 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_partitioned_table.q.out @@ -255,7 +255,7 @@ POSTHOOK: query: select summary from default.ice_meta_3.snapshots POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_meta_3 POSTHOOK: Output: hdfs://### HDFS PATH ### -{"added-data-files":"7","added-records":"15","added-files-size":"1329","changed-partition-count":"7","total-records":"15","total-files-size":"1329","total-data-files":"7","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0"} +{"added-data-files":"7","added-records":"15","added-files-size":"1378","changed-partition-count":"7","total-records":"15","total-files-size":"1378","total-data-files":"7","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0"} PREHOOK: query: select summary['changed-partition-count'] from default.ice_meta_2.snapshots PREHOOK: type: QUERY PREHOOK: Input: default@ice_meta_2 @@ -424,7 +424,7 @@ POSTHOOK: query: select summary from default.ice_meta_3.snapshots POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_meta_3 POSTHOOK: Output: hdfs://### HDFS PATH ### -{"added-data-files":"7","added-records":"15","added-files-size":"1329","changed-partition-count":"7","total-records":"15","total-files-size":"1329","total-data-files":"7","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0"} +{"added-data-files":"7","added-records":"15","added-files-size":"1378","changed-partition-count":"7","total-records":"15","total-files-size":"1378","total-data-files":"7","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0"} PREHOOK: query: select summary['changed-partition-count'] from default.ice_meta_2.snapshots PREHOOK: type: QUERY PREHOOK: Input: default@ice_meta_2 diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java index f7cd620ec320..714372d046cf 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java @@ -390,7 +390,7 @@ public void testStatsAfterCompactionPartTbl() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "1373", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "1395", parameters.get("totalSize")); parameters = partitions .stream() @@ -400,7 +400,7 @@ public void testStatsAfterCompactionPartTbl() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "1442", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "1453", parameters.get("totalSize")); //Do a major compaction CompactionRequest rqst = new CompactionRequest(dbName, tblName, CompactionType.MAJOR); @@ -425,7 +425,7 @@ public void testStatsAfterCompactionPartTbl() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "801", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "808", parameters.get("totalSize")); parameters = partitions .stream() @@ -435,7 +435,7 @@ public void testStatsAfterCompactionPartTbl() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "1442", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "1453", parameters.get("totalSize")); } /** @@ -478,7 +478,7 @@ public void testStatsAfterCompactionTbl() throws Exception { Map parameters = Hive.get().getTable(tblName).getParameters(); Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "1434", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "1446", parameters.get("totalSize")); //Do a major compaction CompactionRequest rqst = new CompactionRequest(dbName, tblName, CompactionType.MAJOR); @@ -496,7 +496,7 @@ public void testStatsAfterCompactionTbl() throws Exception { parameters = Hive.get().getTable(tblName).getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "776", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "783", parameters.get("totalSize")); } @Test diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java index 07971abac7fb..60b32f6b6869 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java @@ -707,7 +707,7 @@ public void testStatsAfterQueryCompactionOnTez() throws Exception { Map parameters = Hive.get().getTable(tblName).getParameters(); Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "1434", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "1446", parameters.get("totalSize")); //Do a major compaction CompactorTestUtil.runCompaction(conf, dbName, tblName, CompactionType.MAJOR, true); @@ -723,7 +723,7 @@ public void testStatsAfterQueryCompactionOnTez() throws Exception { parameters = Hive.get().getTable(tblName).getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "727", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "735", parameters.get("totalSize")); } @Test diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java index 9459a4ff64f6..47a7c756d7a5 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch; import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl; import org.apache.orc.TypeDescription; +import org.apache.orc.impl.reader.tree.TypeReader; import org.apache.orc.impl.SchemaEvolution; import org.apache.orc.impl.TreeReaderFactory; import org.apache.orc.impl.TreeReaderFactory.StructTreeReader; @@ -62,7 +63,6 @@ import org.apache.orc.impl.WriterImpl; import org.apache.orc.OrcProto; - public class OrcEncodedDataConsumer extends EncodedDataConsumer { private TreeReaderFactory.TreeReader[] columnReaders; @@ -193,7 +193,7 @@ protected void decodeBatch(OrcEncodedColumnBatch batch, */ TreeReader reader = columnReaders[idx]; ColumnVector cv = prepareColumnVector(cvb, idx, batchSize); - reader.nextVector(cv, null, batchSize); + reader.nextVector(cv, null, batchSize, cvb.filterContext, TypeReader.ReadPhase.ALL); } // we are done reading a batch, send it to consumer for processing @@ -234,7 +234,7 @@ private void createColumnReaders(OrcEncodedColumnBatch batch, this.batchSchemas = includes.getBatchReaderTypes(fileSchema); StructTreeReader treeReader = EncodedTreeReaderFactory.createRootTreeReader( batchSchemas, stripeMetadata.getEncodings(), batch, codec, context, useDecimal64ColumnVectors); - this.columnReaders = treeReader.getChildReaders(); + this.columnReaders = (TreeReader[]) treeReader.getChildReaders(); if (LlapIoImpl.LOG.isDebugEnabled()) { for (int i = 0; i < columnReaders.length; ++i) { @@ -306,7 +306,7 @@ private void positionInStreams(TreeReaderFactory.TreeReader[] columnReaders, for (int i = 0; i < columnReaders.length; i++) { if (columnReaders[i] == null) continue; // TODO: we could/should trace seek destinations; pps needs a "peek" method - columnReaders[i].seek(pps); + columnReaders[i].seek(pps, TypeReader.ReadPhase.ALL); } } @@ -331,7 +331,7 @@ private void repositionInStreams(TreeReaderFactory.TreeReader[] columnReaders, ((EncodedTreeReaderFactory.TimestampStreamReader) reader) .updateTimezone(stripeMetadata.getWriterTimezone()); } - reader.seek(pps); + reader.seek(pps, TypeReader.ReadPhase.ALL); } } diff --git a/pom.xml b/pom.xml index 45de65a3704b..0bac6d9d7f07 100644 --- a/pom.xml +++ b/pom.xml @@ -180,7 +180,7 @@ 42.4.1 21.3.0.0 2.3 - 1.6.9 + 1.8.3 3.4.4 2.0.2 2.0.0-M5 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java index 9ad75ddc92f6..20ad5caa7df2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedTreeReaderFactory.java @@ -17,24 +17,34 @@ */ package org.apache.hadoop.hive.ql.io.orc.encoded; +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.Decimal64ColumnVector; -import org.apache.orc.OrcFile; -import org.apache.orc.impl.RunLengthByteReader; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch; -import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch; +import org.apache.hadoop.hive.ql.io.filter.FilterContext; + +import org.apache.orc.OrcFile; import org.apache.orc.CompressionCodec; import org.apache.orc.TypeDescription; +import org.apache.orc.OrcProto; +import org.apache.orc.impl.DynamicByteArray; import org.apache.orc.impl.InStream; +import org.apache.orc.impl.IntegerReader; import org.apache.orc.impl.PositionProvider; +import org.apache.orc.impl.RunLengthByteReader; +import org.apache.orc.impl.StreamName; import org.apache.orc.impl.TreeReaderFactory; -import org.apache.orc.OrcProto; +import org.apache.orc.impl.reader.StripePlanner; +import org.apache.orc.impl.reader.tree.TypeReader; + +import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch; +import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,10 +83,10 @@ private TimestampStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } // Note: we assume that batchSize will be consistent with vectors passed in. @@ -88,7 +98,7 @@ public void nextVector( } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase phase) throws IOException { if (vectors != null) return; if (present != null) { if (isFileCompressed) { @@ -224,7 +234,247 @@ private static void skipCompressedIndex(boolean isCompressed, PositionProvider i index.getNext(); } - protected static class StringStreamReader extends StringTreeReader + public static class StringDictionaryTreeReaderHive extends TreeReader { + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + private DynamicByteArray dictionaryBuffer; + private int[] dictionaryOffsets; + protected IntegerReader reader; + + private byte[] dictionaryBufferInBytesCache = null; + private final LongColumnVector scratchlcv; + + StringDictionaryTreeReaderHive(int columnId, Context context) throws IOException { + this(columnId, null, null, null, null, null, context); + } + + protected StringDictionaryTreeReaderHive(int columnId, InStream present, InStream data, InStream length, + InStream dictionary, OrcProto.ColumnEncoding encoding, Context context) throws IOException { + super(columnId, present, context); + scratchlcv = new LongColumnVector(); + if (data != null && encoding != null) { + this.reader = createIntegerReader(encoding.getKind(), data, false, context); + } + + if (dictionary != null && encoding != null) { + readDictionaryStream(dictionary); + } + + if (length != null && encoding != null) { + readDictionaryLengthStream(length, encoding); + } + } + + @Override + public void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException { + if (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DICTIONARY && + encoding.getKind() != OrcProto.ColumnEncoding.Kind.DICTIONARY_V2) { + throw new IOException("Unknown encoding " + encoding + " in column " + + columnId); + } + } + + @Override + public void startStripe(StripePlanner planner, ReadPhase readPhase) throws IOException { + super.startStripe(planner, readPhase); + + // read the dictionary blob + StreamName name = new StreamName(columnId, + OrcProto.Stream.Kind.DICTIONARY_DATA); + InStream in = planner.getStream(name); + readDictionaryStream(in); + + // read the lengths + name = new StreamName(columnId, OrcProto.Stream.Kind.LENGTH); + in = planner.getStream(name); + OrcProto.ColumnEncoding encoding = planner.getEncoding(columnId); + readDictionaryLengthStream(in, encoding); + + // set up the row reader + name = new StreamName(columnId, OrcProto.Stream.Kind.DATA); + reader = createIntegerReader(encoding.getKind(), + planner.getStream(name), false, context); + } + + private void readDictionaryLengthStream(InStream in, OrcProto.ColumnEncoding encoding) + throws IOException { + int dictionarySize = encoding.getDictionarySize(); + if (in != null) { // Guard against empty LENGTH stream. + IntegerReader lenReader = createIntegerReader(encoding.getKind(), in, false, context); + int offset = 0; + if (dictionaryOffsets == null || + dictionaryOffsets.length < dictionarySize + 1) { + dictionaryOffsets = new int[dictionarySize + 1]; + } + for (int i = 0; i < dictionarySize; ++i) { + dictionaryOffsets[i] = offset; + offset += (int) lenReader.next(); + } + dictionaryOffsets[dictionarySize] = offset; + in.close(); + } + + } + + private void readDictionaryStream(InStream in) throws IOException { + if (in != null) { // Guard against empty dictionary stream. + if (in.available() > 0) { + dictionaryBuffer = new DynamicByteArray(64, in.available()); + dictionaryBuffer.readAll(in); + // Since its start of strip invalidate the cache. + dictionaryBufferInBytesCache = null; + } + in.close(); + } else { + dictionaryBuffer = null; + } + } + + @Override + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { + seek(index[columnId], readPhase); + } + + @Override + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { + super.seek(index, readPhase); + reader.seek(index); + } + + @Override + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { + final BytesColumnVector result = (BytesColumnVector) previousVector; + + // Read present/isNull stream + super.nextVector(result, isNull, batchSize, filterContext, readPhase); + readDictionaryByteArray(result, filterContext, batchSize); + } + + private void readDictionaryByteArray(BytesColumnVector result, FilterContext filterContext, int batchSize) + throws IOException { + int offset; + int length; + + if (dictionaryBuffer != null) { + + // Load dictionaryBuffer into cache. + if (dictionaryBufferInBytesCache == null) { + dictionaryBufferInBytesCache = dictionaryBuffer.get(); + } + + // Read string offsets + scratchlcv.isRepeating = result.isRepeating; + scratchlcv.noNulls = result.noNulls; + scratchlcv.isNull = result.isNull; + scratchlcv.ensureSize(batchSize, false); + reader.nextVector(scratchlcv, scratchlcv.vector, batchSize); + if (!scratchlcv.isRepeating) { + // The vector has non-repeating strings. Iterate thru the batch + // and set strings one by one + if (filterContext.isSelectedInUse()) { + // Set all string values to null - offset and length is zero + for (int i = 0; i < batchSize; i++) { + result.setRef(i, dictionaryBufferInBytesCache, 0, 0); + } + // Read selected rows from stream + for (int i = 0; i != filterContext.getSelectedSize(); i++) { + int idx = filterContext.getSelected()[i]; + if (!scratchlcv.isNull[idx]) { + offset = dictionaryOffsets[(int) scratchlcv.vector[idx]]; + length = getDictionaryEntryLength((int) scratchlcv.vector[idx], offset); + result.setRef(idx, dictionaryBufferInBytesCache, offset, length); + } + } + } else { + for (int i = 0; i < batchSize; i++) { + if (!scratchlcv.isNull[i]) { + offset = dictionaryOffsets[(int) scratchlcv.vector[i]]; + length = getDictionaryEntryLength((int) scratchlcv.vector[i], offset); + result.setRef(i, dictionaryBufferInBytesCache, offset, length); + } else { + // If the value is null then set offset and length to zero (null string) + result.setRef(i, dictionaryBufferInBytesCache, 0, 0); + } + } + } + } else { + // If the value is repeating then just set the first value in the + // vector and set the isRepeating flag to true. No need to iterate thru and + // set all the elements to the same value + offset = dictionaryOffsets[(int) scratchlcv.vector[0]]; + length = getDictionaryEntryLength((int) scratchlcv.vector[0], offset); + result.setRef(0, dictionaryBufferInBytesCache, offset, length); + } + result.isRepeating = scratchlcv.isRepeating; + } else { + if (dictionaryOffsets == null) { + // Entire stripe contains null strings. + result.isRepeating = true; + result.noNulls = false; + result.isNull[0] = true; + result.setRef(0, EMPTY_BYTE_ARRAY, 0, 0); + } else { + // stripe contains nulls and empty strings + for (int i = 0; i < batchSize; i++) { + if (!result.isNull[i]) { + result.setRef(i, EMPTY_BYTE_ARRAY, 0, 0); + } + } + } + } + } + + int getDictionaryEntryLength(int entry, int offset) { + final int length; + // if it isn't the last entry, subtract the offsets otherwise use + // the buffer length. + if (entry < dictionaryOffsets.length - 1) { + length = dictionaryOffsets[entry + 1] - offset; + } else { + length = dictionaryBuffer.size() - offset; + } + return length; + } + + @Override + public void skipRows(long items, ReadPhase readPhase) throws IOException { + reader.skip(countNonNulls(items)); + } + + public IntegerReader getReader() { + return reader; + } + } + public static class StringTreeReaderHive extends StringTreeReader { + + protected StringTreeReaderHive(int columnId, InStream present, InStream data, InStream length, + InStream dictionary, OrcProto.ColumnEncoding encoding, Context context) throws IOException { + super(columnId, present, data, length, dictionary, encoding, context); + if (encoding != null) { + switch (encoding.getKind()) { + case DICTIONARY_V2: + case DICTIONARY: + this.reader = new StringDictionaryTreeReaderHive(columnId, present, data, length, dictionary, encoding, context); + break; + } + } + } + + @Override + public void startStripe(StripePlanner planner, TypeReader.ReadPhase readPhase) throws IOException { + switch (planner.getEncoding(this.columnId).getKind()) { + case DICTIONARY_V2: + case DICTIONARY: + this.reader = new StringDictionaryTreeReaderHive(this.columnId, this.context); + this.reader.startStripe(planner, readPhase); + break; + default: + super.startStripe(planner, readPhase); + } + } + } + + protected static class StringStreamReader extends StringTreeReaderHive implements SettableTreeReader { private boolean _isFileCompressed; private boolean _isDictionaryEncoding; @@ -251,19 +501,19 @@ private StringStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase phase) throws IOException { // This string reader should simply redirect to its own seek (what other types already do). - this.seek(index[columnId]); + this.seek(index[columnId], phase); } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase phase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { index.getNext(); } - reader.getPresent().seek(index); + ((TreeReader)reader).getPresent().seek(index); } if (_isDictionaryEncoding) { @@ -273,7 +523,7 @@ public void seek(PositionProvider index) throws IOException { // This can happen if all values in stream are nulls or last row group values are all null. skipCompressedIndex(_isFileCompressed, index); if (_dataStream != null && _dataStream.available() > 0) { - ((StringDictionaryTreeReader) reader).getReader().seek(index); + ((StringDictionaryTreeReaderHive) reader).getReader().seek(index); } // No need to skip seek here, index won't be used anymore. } else { // DIRECT encoding @@ -299,10 +549,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -442,7 +692,7 @@ private ShortStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase phase) throws IOException { if (vectors != null) return; if (present != null) { if (isFileCompressed) { @@ -462,10 +712,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -571,7 +821,7 @@ private LongStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { @@ -591,10 +841,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -701,7 +951,7 @@ private IntStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { @@ -721,10 +971,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -817,12 +1067,12 @@ protected static class FloatStreamReader extends FloatTreeReader implements Sett private SettableUncompressedStream _presentStream; private SettableUncompressedStream _dataStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; private FloatStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, boolean isFileCompressed, - List vectors) throws IOException { - super(columnId, present, data); + List vectors, TreeReaderFactory.Context context) throws IOException { + super(columnId, present, data, context); this._isFileCompressed = isFileCompressed; this._presentStream = present; this._dataStream = data; @@ -830,7 +1080,7 @@ private FloatStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { @@ -848,10 +1098,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -879,6 +1129,7 @@ public static class StreamReaderBuilder { private ColumnStreamData dataStream; private CompressionCodec compressionCodec; private List vectors; + private TreeReaderFactory.Context context; public StreamReaderBuilder setColumnIndex(int columnIndex) { this.columnIndex = columnIndex; @@ -910,7 +1161,12 @@ public FloatStreamReader build() throws IOException { dataStream); boolean isFileCompressed = compressionCodec != null; - return new FloatStreamReader(columnIndex, present, data, isFileCompressed, vectors); + return new FloatStreamReader(columnIndex, present, data, isFileCompressed, vectors, context); + } + + public StreamReaderBuilder setContext(Context context) { + this.context = context; + return this; } public StreamReaderBuilder setVectors(List vectors) { @@ -934,8 +1190,8 @@ protected static class DoubleStreamReader extends DoubleTreeReader implements Se private DoubleStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, boolean isFileCompressed, - List vectors) throws IOException { - super(columnId, present, data); + List vectors, TreeReaderFactory.Context context) throws IOException { + super(columnId, present, data, context); this._isFileCompressed = isFileCompressed; this._presentStream = present; this._dataStream = data; @@ -943,7 +1199,7 @@ private DoubleStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { @@ -961,10 +1217,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1030,7 +1286,7 @@ public DoubleStreamReader build() throws IOException { boolean isFileCompressed = compressionCodec != null; // TODO: why doesn't this use context? - return new DoubleStreamReader(columnIndex, present, data, isFileCompressed, vectors); + return new DoubleStreamReader(columnIndex, present, data, isFileCompressed, vectors, context); } public StreamReaderBuilder setVectors(List vectors) { @@ -1068,7 +1324,7 @@ private DecimalStreamReader(int columnId, int precision, int scale, } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { @@ -1095,10 +1351,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1229,7 +1485,7 @@ private Decimal64StreamReader(int columnId, int precision, int scale, } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { @@ -1249,10 +1505,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1367,7 +1623,7 @@ private DateStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (isFileCompressed) { @@ -1387,10 +1643,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1477,7 +1733,56 @@ public static StreamReaderBuilder builder() { } } - protected static class CharStreamReader extends CharTreeReader implements SettableTreeReader { + public static class CharTreeReaderHive extends StringTreeReaderHive { + int maxLength; + + CharTreeReaderHive(int columnId, int maxLength, Context context) throws IOException { + this(columnId, maxLength, (InStream)null, (InStream)null, (InStream)null, (InStream)null, (OrcProto.ColumnEncoding)null, context); + } + + protected CharTreeReaderHive(int columnId, int maxLength, InStream present, InStream data, InStream length, + InStream dictionary, OrcProto.ColumnEncoding encoding, Context context) throws IOException { + super(columnId, present, data, length, dictionary, encoding, context); + this.maxLength = maxLength; + } + + public void nextVector(ColumnVector previousVector, boolean[] isNull, int batchSize, FilterContext filterContext, + TypeReader.ReadPhase readPhase) throws IOException { + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); + BytesColumnVector result = (BytesColumnVector)previousVector; + int adjustedDownLen; + if (result.isRepeating) { + if (result.noNulls || !result.isNull[0]) { + adjustedDownLen = StringExpr.rightTrimAndTruncate(result.vector[0], result.start[0], result.length[0], this.maxLength); + if (adjustedDownLen < result.length[0]) { + result.setRef(0, result.vector[0], result.start[0], adjustedDownLen); + } + } + } else { + int i; + if (result.noNulls) { + for(i = 0; i < batchSize; ++i) { + adjustedDownLen = StringExpr.rightTrimAndTruncate(result.vector[i], result.start[i], result.length[i], this.maxLength); + if (adjustedDownLen < result.length[i]) { + result.setRef(i, result.vector[i], result.start[i], adjustedDownLen); + } + } + } else { + for(i = 0; i < batchSize; ++i) { + if (!result.isNull[i]) { + adjustedDownLen = StringExpr.rightTrimAndTruncate(result.vector[i], result.start[i], result.length[i], this.maxLength); + if (adjustedDownLen < result.length[i]) { + result.setRef(i, result.vector[i], result.start[i], adjustedDownLen); + } + } + } + } + } + + } + } + + protected static class CharStreamReader extends CharTreeReaderHive implements SettableTreeReader { private boolean _isFileCompressed; private boolean _isDictionaryEncoding; private SettableUncompressedStream _presentStream; @@ -1491,9 +1796,9 @@ private CharStreamReader(int columnId, int maxLength, SettableUncompressedStream present, SettableUncompressedStream data, SettableUncompressedStream length, SettableUncompressedStream dictionary, boolean isFileCompressed, OrcProto.ColumnEncoding encoding, - List vectors) throws IOException { + List vectors, TreeReaderFactory.Context context) throws IOException { super(columnId, maxLength, present, data, length, - dictionary, encoding); + dictionary, encoding, context); this._isDictionaryEncoding = dictionary != null; this._isFileCompressed = isFileCompressed; this._presentStream = present; @@ -1504,19 +1809,19 @@ private CharStreamReader(int columnId, int maxLength, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { // This string reader should simply redirect to its own seek (what other types already do). - this.seek(index[columnId]); + this.seek(index[columnId], readPhase); } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { index.getNext(); } - reader.getPresent().seek(index); + ((TreeReader)reader).getPresent().seek(index); } if (_isDictionaryEncoding) { @@ -1526,7 +1831,7 @@ public void seek(PositionProvider index) throws IOException { // This can happen if all values in stream are nulls or last row group values are all null. skipCompressedIndex(_isFileCompressed, index); if (_dataStream.available() > 0) { - ((StringDictionaryTreeReader) reader).getReader().seek(index); + ((StringDictionaryTreeReaderHive) reader).getReader().seek(index); } // No need to skip seek here, index won't be used anymore. } else { // DIRECT encoding @@ -1551,10 +1856,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1601,6 +1906,7 @@ public static class StreamReaderBuilder { private CompressionCodec compressionCodec; private OrcProto.ColumnEncoding columnEncoding; private List vectors; + private TreeReaderFactory.Context context; public StreamReaderBuilder setColumnIndex(int columnIndex) { this.columnIndex = columnIndex; @@ -1660,7 +1966,12 @@ public CharStreamReader build() throws IOException { boolean isFileCompressed = compressionCodec != null; return new CharStreamReader(columnIndex, maxLength, present, data, length, - dictionary, isFileCompressed, columnEncoding, vectors); + dictionary, isFileCompressed, columnEncoding, vectors, context); + } + + public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { + this.context = context; + return this; } public StreamReaderBuilder setVectors(List vectors) { @@ -1675,7 +1986,56 @@ public static StreamReaderBuilder builder() { } - protected static class VarcharStreamReader extends VarcharTreeReader implements SettableTreeReader { + public static class VarcharTreeReaderHive extends StringTreeReaderHive { + int maxLength; + + VarcharTreeReaderHive(int columnId, int maxLength, Context context) throws IOException { + this(columnId, maxLength, (InStream)null, (InStream)null, (InStream)null, (InStream)null, (OrcProto.ColumnEncoding)null, context); + } + + protected VarcharTreeReaderHive(int columnId, int maxLength, InStream present, InStream data, InStream length, + InStream dictionary, OrcProto.ColumnEncoding encoding, Context context) throws IOException { + super(columnId, present, data, length, dictionary, encoding, context); + this.maxLength = maxLength; + } + + public void nextVector(ColumnVector previousVector, boolean[] isNull, int batchSize, FilterContext filterContext, + TypeReader.ReadPhase readPhase) throws IOException { + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); + BytesColumnVector result = (BytesColumnVector)previousVector; + int adjustedDownLen; + if (result.isRepeating) { + if (result.noNulls || !result.isNull[0]) { + adjustedDownLen = StringExpr.truncate(result.vector[0], result.start[0], result.length[0], this.maxLength); + if (adjustedDownLen < result.length[0]) { + result.setRef(0, result.vector[0], result.start[0], adjustedDownLen); + } + } + } else { + int i; + if (result.noNulls) { + for(i = 0; i < batchSize; ++i) { + adjustedDownLen = StringExpr.truncate(result.vector[i], result.start[i], result.length[i], this.maxLength); + if (adjustedDownLen < result.length[i]) { + result.setRef(i, result.vector[i], result.start[i], adjustedDownLen); + } + } + } else { + for(i = 0; i < batchSize; ++i) { + if (!result.isNull[i]) { + adjustedDownLen = StringExpr.truncate(result.vector[i], result.start[i], result.length[i], this.maxLength); + if (adjustedDownLen < result.length[i]) { + result.setRef(i, result.vector[i], result.start[i], adjustedDownLen); + } + } + } + } + } + + } + } + + protected static class VarcharStreamReader extends VarcharTreeReaderHive implements SettableTreeReader { private boolean _isFileCompressed; private boolean _isDictionaryEncoding; private SettableUncompressedStream _presentStream; @@ -1683,15 +2043,15 @@ protected static class VarcharStreamReader extends VarcharTreeReader implements private SettableUncompressedStream _lengthStream; private SettableUncompressedStream _dictionaryStream; private List vectors; - private int vectorIndex = 0; + private int vectorIndex = 0; private VarcharStreamReader(int columnId, int maxLength, SettableUncompressedStream present, SettableUncompressedStream data, SettableUncompressedStream length, SettableUncompressedStream dictionary, boolean isFileCompressed, OrcProto.ColumnEncoding encoding, - List vectors) throws IOException { + List vectors, TreeReaderFactory.Context context) throws IOException { super(columnId, maxLength, present, data, length, - dictionary, encoding); + dictionary, encoding, context); this._isDictionaryEncoding = dictionary != null; this._isFileCompressed = isFileCompressed; this._presentStream = present; @@ -1702,19 +2062,19 @@ private VarcharStreamReader(int columnId, int maxLength, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { // This string reader should simply redirect to its own seek (what other types already do). - this.seek(index[columnId]); + this.seek(index[columnId], readPhase); } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { index.getNext(); } - reader.getPresent().seek(index); + ((TreeReader)reader).getPresent().seek(index); } if (_isDictionaryEncoding) { @@ -1724,7 +2084,7 @@ public void seek(PositionProvider index) throws IOException { // This can happen if all values in stream are nulls or last row group values are all null. skipCompressedIndex(_isFileCompressed, index); if (_dataStream.available() > 0) { - ((StringDictionaryTreeReader) reader).getReader().seek(index); + ((StringDictionaryTreeReaderHive) reader).getReader().seek(index); } // No need to skip seek here, index won't be used anymore. } else { // DIRECT encoding @@ -1749,10 +2109,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1799,6 +2159,7 @@ public static class StreamReaderBuilder { private CompressionCodec compressionCodec; private OrcProto.ColumnEncoding columnEncoding; private List vectors; + private TreeReaderFactory.Context context; public StreamReaderBuilder setColumnIndex(int columnIndex) { this.columnIndex = columnIndex; @@ -1858,7 +2219,12 @@ public VarcharStreamReader build() throws IOException { boolean isFileCompressed = compressionCodec != null; return new VarcharStreamReader(columnIndex, maxLength, present, data, length, - dictionary, isFileCompressed, columnEncoding, vectors); + dictionary, isFileCompressed, columnEncoding, vectors, context); + } + + public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { + this.context = context; + return this; } public StreamReaderBuilder setVectors(List vectors) { @@ -1882,8 +2248,8 @@ protected static class ByteStreamReader extends ByteTreeReader implements Settab private ByteStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, boolean isFileCompressed, - List vectors) throws IOException { - super(columnId, present, data); + List vectors, TreeReaderFactory.Context context) throws IOException { + super(columnId, present, data, context); this._isFileCompressed = isFileCompressed; this._presentStream = present; this._dataStream = data; @@ -1891,7 +2257,7 @@ private ByteStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { @@ -1911,10 +2277,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -1942,6 +2308,7 @@ public static class StreamReaderBuilder { private ColumnStreamData dataStream; private CompressionCodec compressionCodec; private List vectors; + private TreeReaderFactory.Context context; public StreamReaderBuilder setColumnIndex(int columnIndex) { this.columnIndex = columnIndex; @@ -1973,7 +2340,12 @@ public ByteStreamReader build() throws IOException { dataStream); boolean isFileCompressed = compressionCodec != null; - return new ByteStreamReader(columnIndex, present, data, isFileCompressed, vectors); + return new ByteStreamReader(columnIndex, present, data, isFileCompressed, vectors, context); + } + + public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { + this.context = context; + return this; } public StreamReaderBuilder setVectors(List vectors) { @@ -2008,10 +2380,10 @@ private BinaryStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -2021,7 +2393,7 @@ public void nextVector( } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { @@ -2143,8 +2515,8 @@ protected static class BooleanStreamReader extends BooleanTreeReader implements private BooleanStreamReader(int columnId, SettableUncompressedStream present, SettableUncompressedStream data, boolean isFileCompressed, - List vectors) throws IOException { - super(columnId, present, data); + List vectors, TreeReaderFactory.Context context) throws IOException { + super(columnId, present, data, context); this._isFileCompressed = isFileCompressed; this._presentStream = present; this._dataStream = data; @@ -2152,7 +2524,7 @@ private BooleanStreamReader(int columnId, SettableUncompressedStream present, } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { if (vectors != null) return; if (present != null) { if (_isFileCompressed) { @@ -2172,10 +2544,10 @@ public void seek(PositionProvider index) throws IOException { } @Override - public void nextVector( - ColumnVector previousVector, boolean[] isNull, int batchSize) throws IOException { + public void nextVector(ColumnVector previousVector, boolean[] isNull, final int batchSize, + FilterContext filterContext, ReadPhase readPhase) throws IOException { if (vectors == null) { - super.nextVector(previousVector, isNull, batchSize); + super.nextVector(previousVector, isNull, batchSize, filterContext, readPhase); return; } vectors.get(vectorIndex++).shallowCopyTo(previousVector); @@ -2203,6 +2575,7 @@ public static class StreamReaderBuilder { private ColumnStreamData dataStream; private CompressionCodec compressionCodec; private List vectors; + private TreeReaderFactory.Context context; public StreamReaderBuilder setColumnIndex(int columnIndex) { this.columnIndex = columnIndex; @@ -2234,7 +2607,12 @@ public BooleanStreamReader build() throws IOException { dataStream); boolean isFileCompressed = compressionCodec != null; - return new BooleanStreamReader(columnIndex, present, data, isFileCompressed, vectors); + return new BooleanStreamReader(columnIndex, present, data, isFileCompressed, vectors, context); + } + + public StreamReaderBuilder setContext(TreeReaderFactory.Context context) { + this.context = context; + return this; } public StreamReaderBuilder setVectors(List vectors) { @@ -2445,6 +2823,7 @@ private static TreeReader getPrimitiveTreeReader(final int columnIndex, .setDataStream(data) .setCompressionCodec(codec) .setVectors(vectors) + .setContext(context) .build(); case BYTE: return ByteStreamReader.builder() @@ -2453,6 +2832,7 @@ private static TreeReader getPrimitiveTreeReader(final int columnIndex, .setDataStream(data) .setCompressionCodec(codec) .setVectors(vectors) + .setContext(context) .build(); case SHORT: return ShortStreamReader.builder() @@ -2491,6 +2871,7 @@ private static TreeReader getPrimitiveTreeReader(final int columnIndex, .setDataStream(data) .setCompressionCodec(codec) .setVectors(vectors) + .setContext(context) .build(); case DOUBLE: return DoubleStreamReader.builder() @@ -2499,6 +2880,7 @@ private static TreeReader getPrimitiveTreeReader(final int columnIndex, .setDataStream(data) .setCompressionCodec(codec) .setVectors(vectors) + .setContext(context) .build(); case CHAR: return CharStreamReader.builder() @@ -2511,6 +2893,7 @@ private static TreeReader getPrimitiveTreeReader(final int columnIndex, .setCompressionCodec(codec) .setColumnEncoding(columnEncoding) .setVectors(vectors) + .setContext(context) .build(); case VARCHAR: return VarcharStreamReader.builder() @@ -2523,6 +2906,7 @@ private static TreeReader getPrimitiveTreeReader(final int columnIndex, .setCompressionCodec(codec) .setColumnEncoding(columnEncoding) .setVectors(vectors) + .setContext(context) .build(); case STRING: return StringStreamReader.builder() @@ -2534,6 +2918,7 @@ private static TreeReader getPrimitiveTreeReader(final int columnIndex, .setCompressionCodec(codec) .setColumnEncoding(columnEncoding) .setVectors(vectors) + .setContext(context) .build(); case DECIMAL: // special handling for serde reader (text) in llap IO. @@ -2622,7 +3007,7 @@ public ListStreamReader(final int columnIndex, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { PositionProvider ownIndex = index[columnId]; if (present != null) { if (_isFileCompressed) { @@ -2638,12 +3023,12 @@ public void seek(PositionProvider[] index) throws IOException { ownIndex.getNext(); } lengths.seek(ownIndex); - elementReader.seek(index); + elementReader.seek(index, readPhase); } } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { // Only our parent class can call this. throw new IOException("Should never be called"); } @@ -2747,7 +3132,7 @@ public MapStreamReader(final int columnIndex, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { // We are not calling super.seek since we handle the present stream differently. PositionProvider ownIndex = index[columnId]; if (present != null) { @@ -2764,13 +3149,13 @@ public void seek(PositionProvider[] index) throws IOException { ownIndex.getNext(); } lengths.seek(ownIndex); - keyReader.seek(index); - valueReader.seek(index); + keyReader.seek(index, readPhase); + valueReader.seek(index, readPhase); } } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { // Only our parent class can call this. throw new IOException("Should never be called"); } @@ -2882,7 +3267,7 @@ public StructStreamReader(final int columnIndex, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { PositionProvider ownIndex = index[columnId]; if (present != null) { if (_isFileCompressed) { @@ -2891,16 +3276,16 @@ public void seek(PositionProvider[] index) throws IOException { present.seek(ownIndex); } if (fields != null) { - for (TreeReader child : fields) { + for (TypeReader child : fields) { if (child != null) { - child.seek(index); + child.seek(index, readPhase); } } } } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { // Only our parent class can call this. throw new IOException("Should never be called"); } @@ -2915,7 +3300,7 @@ public void setBuffers(EncodedColumnBatch batch, boolean sameStripe StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.PRESENT_VALUE])); } if (fields != null) { - for (TreeReader child : fields) { + for (TypeReader child : fields) { if (child != null) { ((SettableTreeReader) child).setBuffers(batch, sameStripe); } @@ -2995,7 +3380,7 @@ public UnionStreamReader(final int columnIndex, } @Override - public void seek(PositionProvider[] index) throws IOException { + public void seek(PositionProvider[] index, ReadPhase readPhase) throws IOException { PositionProvider ownIndex = index[columnId]; if (present != null) { if (_isFileCompressed) { @@ -3012,15 +3397,15 @@ public void seek(PositionProvider[] index) throws IOException { } tags.seek(ownIndex); if (fields != null) { - for (TreeReader child : fields) { - child.seek(index); + for (TypeReader child : fields) { + child.seek(index, readPhase); } } } } @Override - public void seek(PositionProvider index) throws IOException { + public void seek(PositionProvider index, ReadPhase readPhase) throws IOException { // Only our parent class can call this. throw new IOException("Should never be called"); } @@ -3038,7 +3423,7 @@ public void setBuffers(EncodedColumnBatch batch, boolean sameStripe StreamUtils.createDiskRangeInfo(streamsData[OrcProto.Stream.Kind.DATA_VALUE])); } if (fields != null) { - for (TreeReader child : fields) { + for (TypeReader child : fields) { ((SettableTreeReader) child).setBuffers(batch, sameStripe); } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java index 26c08358f858..0b24c90fb287 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java @@ -793,7 +793,7 @@ public void testCompactStatsGather() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "686", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "691", parameters.get("totalSize")); int[][] targetVals2 = {{5, 1, 1}, {5, 2, 2}, {5, 3, 1}, {5, 4, 2}}; runStatementOnDriver("insert into T partition(p=1,q) " + makeValuesClause(targetVals2)); @@ -847,7 +847,7 @@ public void testCompactStatsGather() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "4", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "698", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "704", parameters.get("totalSize")); } @Test diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index c98f7f408eb1..b81884d9790e 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -2603,14 +2603,14 @@ public void testCombinationInputFormatWithAcid() throws Exception { assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00000", split.getPath().toString()); assertEquals(0, split.getStart()); - assertEquals(702, split.getLength()); + assertEquals(709, split.getLength()); split = (HiveInputFormat.HiveInputSplit) splits[1]; assertEquals("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat", split.inputFormatClassName()); assertEquals("mock:/combinationAcid/p=0/base_0000010/bucket_00001", split.getPath().toString()); assertEquals(0, split.getStart()); - assertEquals(726, split.getLength()); + assertEquals(733, split.getLength()); CombineHiveInputFormat.CombineHiveInputSplit combineSplit = (CombineHiveInputFormat.CombineHiveInputSplit) splits[2]; assertEquals(BUCKETS, combineSplit.getNumPaths()); @@ -2618,7 +2618,7 @@ public void testCombinationInputFormatWithAcid() throws Exception { assertEquals("mock:/combinationAcid/p=1/00000" + bucket + "_0", combineSplit.getPath(bucket).toString()); assertEquals(0, combineSplit.getOffset(bucket)); - assertEquals(253, combineSplit.getLength(bucket)); + assertEquals(260, combineSplit.getLength(bucket)); } String[] hosts = combineSplit.getLocations(); assertEquals(2, hosts.length); diff --git a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out index f560d3a35b8a..74a840041b4b 100644 --- a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out +++ b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out @@ -63,7 +63,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 408 -totalSize 468 +totalSize 474 #### A masked pattern was here #### PREHOOK: query: create materialized view if not exists cmv_mat_view2_n4 as select a, c from cmv_basetable_n10 where a = 3 @@ -99,7 +99,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 232 -totalSize 334 +totalSize 340 #### A masked pattern was here #### PREHOOK: query: explain select a, c from cmv_basetable_n10 where a = 3 diff --git a/ql/src/test/results/clientpositive/llap/acid_bloom_filter_orc_file_dump.q.out b/ql/src/test/results/clientpositive/llap/acid_bloom_filter_orc_file_dump.q.out index 89d5d6506c33..9d3d348d1738 100644 --- a/ql/src/test/results/clientpositive/llap/acid_bloom_filter_orc_file_dump.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_bloom_filter_orc_file_dump.q.out @@ -76,7 +76,7 @@ PREHOOK: Input: default@bloomtest #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.1 Rows: 1 Compression: ZLIB Compression size: 32768 @@ -180,7 +180,7 @@ Stripes: Entry 0: numHashFunctions: 6 bitCount: 81472 popCount: 6 loadFactor: 0.0001 expectedFpp: 1.5953551E-25 Stripe level merge: numHashFunctions: 6 bitCount: 81472 popCount: 6 loadFactor: 0.0001 expectedFpp: 1.5953551E-25 -File length: 1212 bytes +File length: 1217 bytes Padding length: 0 bytes Padding ratio: 0% @@ -193,7 +193,7 @@ ________________________________________________________________________________ -- END ORC FILE DUMP -- -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.1 Rows: 1 Compression: ZLIB Compression size: 32768 @@ -297,7 +297,7 @@ Stripes: Entry 0: numHashFunctions: 6 bitCount: 81472 popCount: 6 loadFactor: 0.0001 expectedFpp: 1.5953551E-25 Stripe level merge: numHashFunctions: 6 bitCount: 81472 popCount: 6 loadFactor: 0.0001 expectedFpp: 1.5953551E-25 -File length: 1205 bytes +File length: 1211 bytes Padding length: 0 bytes Padding ratio: 0% diff --git a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out index 5e812d19d013..e0a87b2bcb2b 100644 --- a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out @@ -334,7 +334,7 @@ Table Parameters: numPartitions 4 numRows 2003 rawDataSize 0 - totalSize 18043 + totalSize 18096 transactional true transactional_properties default #### A masked pattern was here #### @@ -444,7 +444,7 @@ Table Parameters: numPartitions 4 numRows 2003 rawDataSize 0 - totalSize 18043 + totalSize 18096 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/acid_table_stats.q.out b/ql/src/test/results/clientpositive/llap/acid_table_stats.q.out index 36c65fae4f86..d9b3801e8fb4 100644 --- a/ql/src/test/results/clientpositive/llap/acid_table_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_table_stats.q.out @@ -96,7 +96,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4361 + totalSize 4375 #### A masked pattern was here #### # Storage Information @@ -181,7 +181,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4361 + totalSize 4375 #### A masked pattern was here #### # Storage Information @@ -230,7 +230,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 0 - totalSize 4361 + totalSize 4375 #### A masked pattern was here #### # Storage Information @@ -325,7 +325,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 8722 + totalSize 8750 #### A masked pattern was here #### # Storage Information @@ -372,7 +372,7 @@ Partition Parameters: numFiles 4 numRows 2000 rawDataSize 0 - totalSize 8722 + totalSize 8750 #### A masked pattern was here #### # Storage Information @@ -593,7 +593,7 @@ Partition Parameters: numFiles 2 numRows 1000 rawDataSize 176000 - totalSize 3325 + totalSize 3339 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/alter_merge_orc.q.out b/ql/src/test/results/clientpositive/llap/alter_merge_orc.q.out index 571651ccb647..c4fefac2d78b 100644 --- a/ql/src/test/results/clientpositive/llap/alter_merge_orc.q.out +++ b/ql/src/test/results/clientpositive/llap/alter_merge_orc.q.out @@ -48,9 +48,9 @@ columns:struct columns { i32 key, string value} partitioned:false partitionColumns: totalNumberFiles:3 -totalFileSize:7596 -maxFileSize:2532 -minFileSize:2532 +totalFileSize:7617 +maxFileSize:2539 +minFileSize:2539 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test @@ -91,9 +91,9 @@ columns:struct columns { i32 key, string value} partitioned:false partitionColumns: totalNumberFiles:1 -totalFileSize:7216 -maxFileSize:7216 -minFileSize:7216 +totalFileSize:7224 +maxFileSize:7224 +minFileSize:7224 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test @@ -171,9 +171,9 @@ columns:struct columns { i32 key, string value} partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:3 -totalFileSize:7596 -maxFileSize:2532 -minFileSize:2532 +totalFileSize:7617 +maxFileSize:2539 +minFileSize:2539 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test_part_n2 @@ -218,9 +218,9 @@ columns:struct columns { i32 key, string value} partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:1 -totalFileSize:7216 -maxFileSize:7216 -minFileSize:7216 +totalFileSize:7224 +maxFileSize:7224 +minFileSize:7224 #### A masked pattern was here #### PREHOOK: query: select count(1) from src_orc_merge_test_part_n2 diff --git a/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out b/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out index dab660d37021..1ba05bdff261 100644 --- a/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out +++ b/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out @@ -48,9 +48,9 @@ columns:struct columns { i32 key, string value} partitioned:false partitionColumns: totalNumberFiles:3 -totalFileSize:7596 -maxFileSize:2532 -minFileSize:2532 +totalFileSize:7617 +maxFileSize:2539 +minFileSize:2539 #### A masked pattern was here #### PREHOOK: query: desc extended src_orc_merge_test_stat @@ -93,7 +93,7 @@ Table Parameters: numFiles 3 numRows 1500 rawDataSize 147000 - totalSize 7596 + totalSize 7617 #### A masked pattern was here #### # Storage Information @@ -144,7 +144,7 @@ Table Parameters: numFiles 1 numRows 1500 rawDataSize 147000 - totalSize 7216 + totalSize 7224 #### A masked pattern was here #### # Storage Information @@ -214,9 +214,9 @@ columns:struct columns { i32 key, string value} partitioned:true partitionColumns:struct partition_columns { string ds} totalNumberFiles:3 -totalFileSize:7596 -maxFileSize:2532 -minFileSize:2532 +totalFileSize:7617 +maxFileSize:2539 +minFileSize:2539 #### A masked pattern was here #### PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011') @@ -243,7 +243,7 @@ Partition Parameters: numFiles 3 numRows 1500 rawDataSize 141000 - totalSize 7596 + totalSize 7617 #### A masked pattern was here #### # Storage Information @@ -290,7 +290,7 @@ Partition Parameters: numFiles 3 numRows 1500 rawDataSize 147000 - totalSize 7596 + totalSize 7617 #### A masked pattern was here #### # Storage Information @@ -345,7 +345,7 @@ Partition Parameters: numFiles 1 numRows 1500 rawDataSize 147000 - totalSize 7216 + totalSize 7224 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out b/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out index c30b58f525ea..1ebc46204dc4 100644 --- a/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out +++ b/ql/src/test/results/clientpositive/llap/autoColumnStats_4.q.out @@ -214,7 +214,7 @@ Table Parameters: numFiles 2 numRows 10 rawDataSize 0 - totalSize 1884 + totalSize 1898 transactional true transactional_properties default #### A masked pattern was here #### @@ -257,7 +257,7 @@ Table Parameters: numFiles 3 numRows 8 rawDataSize 0 - totalSize 2596 + totalSize 2615 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/autoColumnStats_6.q.out b/ql/src/test/results/clientpositive/llap/autoColumnStats_6.q.out index e6ea95c8489e..af8016e2957a 100644 --- a/ql/src/test/results/clientpositive/llap/autoColumnStats_6.q.out +++ b/ql/src/test/results/clientpositive/llap/autoColumnStats_6.q.out @@ -213,7 +213,7 @@ POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=1).value SIMPLE [ POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge2a PARTITION(one=1,two=9,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 351 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 358 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge2a diff --git a/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_2.q.out b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_2.q.out index 915c73e0f63e..d34cb7553a60 100644 --- a/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_2.q.out +++ b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_2.q.out @@ -42,7 +42,7 @@ Table Parameters: numFiles 2 numRows 3 rawDataSize 24 - totalSize 571 + totalSize 585 #### A masked pattern was here #### # Storage Information @@ -87,7 +87,7 @@ Table Parameters: numFiles 2 numRows 3 rawDataSize 36 - totalSize 571 + totalSize 585 #### A masked pattern was here #### # Storage Information @@ -185,7 +185,7 @@ Table Parameters: numFiles 2 numRows 3 rawDataSize 36 - totalSize 571 + totalSize 585 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out b/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out index 5eb0ef9ad252..21676f9e2999 100644 --- a/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out +++ b/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out @@ -42,7 +42,7 @@ Table Parameters: numFiles 1 numRows 1 rawDataSize 170 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information @@ -221,7 +221,7 @@ Table Parameters: numFiles 1 numRows 1 rawDataSize 174 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information @@ -298,7 +298,7 @@ Table Parameters: numPartitions 2 numRows 2 rawDataSize 340 - totalSize 570 + totalSize 584 #### A masked pattern was here #### # Storage Information @@ -540,7 +540,7 @@ Table Parameters: numPartitions 2 numRows 2 rawDataSize 348 - totalSize 570 + totalSize 584 #### A masked pattern was here #### # Storage Information @@ -578,7 +578,7 @@ Partition Parameters: numFiles 1 numRows 1 rawDataSize 174 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information @@ -616,7 +616,7 @@ Partition Parameters: numFiles 1 numRows 1 rawDataSize 174 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information @@ -697,7 +697,7 @@ Table Parameters: numPartitions 2 numRows 2 rawDataSize 340 - totalSize 570 + totalSize 584 #### A masked pattern was here #### # Storage Information @@ -897,7 +897,7 @@ Table Parameters: numPartitions 2 numRows 2 rawDataSize 344 - totalSize 570 + totalSize 584 #### A masked pattern was here #### # Storage Information @@ -935,7 +935,7 @@ Partition Parameters: numFiles 1 numRows 1 rawDataSize 174 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information @@ -973,7 +973,7 @@ Partition Parameters: numFiles 1 numRows 1 rawDataSize 170 - totalSize 285 + totalSize 292 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/ctas.q.out b/ql/src/test/results/clientpositive/llap/ctas.q.out index b6f1d24faf8d..20d587cfa1b8 100644 --- a/ql/src/test/results/clientpositive/llap/ctas.q.out +++ b/ql/src/test/results/clientpositive/llap/ctas.q.out @@ -1183,4 +1183,4 @@ POSTHOOK: Input: default@acid_ctas_part@k=0 POSTHOOK: Input: default@acid_ctas_part@k=10 POSTHOOK: Input: default@acid_ctas_part@k=100 #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"acid_ctas_part\"\n ],\n \"table:alias\": \"acid_ctas_part\",\n \"inputs\": [],\n \"rowCount\": 270.0,\n \"avgRowSize\": 200.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"k\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"ROW__IS__DELETED\"\n }\n ],\n \"partitionColumns\": [\n \"k\"\n ],\n \"colStats\": [\n {\n \"name\": \"value\",\n \"ndv\": 54\n },\n {\n \"name\": \"k\",\n \"ndv\": 3\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"k\",\n \"value\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 270.0\n }\n ]\n}","optimizedSQL":"SELECT `k`, `value`\nFROM `default`.`acid_ctas_part`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"acid_ctas_part","columns:":["value"],"database:":"default","table:":"acid_ctas_part","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"k (type: string), value (type: string)","columnExprMap:":{"_col0":"k","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"acid_ctas_part\"\n ],\n \"table:alias\": \"acid_ctas_part\",\n \"inputs\": [],\n \"rowCount\": 273.0,\n \"avgRowSize\": 200.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"k\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"ROW__IS__DELETED\"\n }\n ],\n \"partitionColumns\": [\n \"k\"\n ],\n \"colStats\": [\n {\n \"name\": \"value\",\n \"ndv\": 54\n },\n {\n \"name\": \"k\",\n \"ndv\": 3\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"k\",\n \"value\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 273.0\n }\n ]\n}","optimizedSQL":"SELECT `k`, `value`\nFROM `default`.`acid_ctas_part`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"acid_ctas_part","columns:":["value"],"database:":"default","table:":"acid_ctas_part","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"k (type: string), value (type: string)","columnExprMap:":{"_col0":"k","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} diff --git a/ql/src/test/results/clientpositive/llap/default_constraint.q.out b/ql/src/test/results/clientpositive/llap/default_constraint.q.out index 48347175aa1c..b12508bed344 100644 --- a/ql/src/test/results/clientpositive/llap/default_constraint.q.out +++ b/ql/src/test/results/clientpositive/llap/default_constraint.q.out @@ -1556,7 +1556,7 @@ Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 numFiles 1 - totalSize 1101 + totalSize 1106 transactional true transactional_properties default #### A masked pattern was here #### @@ -1730,7 +1730,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 2 - totalSize 2202 + totalSize 2212 transactional true transactional_properties default #### A masked pattern was here #### @@ -1808,7 +1808,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 2 - totalSize 2202 + totalSize 2212 transactional true transactional_properties default #### A masked pattern was here #### @@ -1993,7 +1993,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 3 - totalSize 3290 + totalSize 3305 transactional true transactional_properties default #### A masked pattern was here #### @@ -2074,7 +2074,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 3 - totalSize 3290 + totalSize 3305 transactional true transactional_properties default #### A masked pattern was here #### @@ -2152,7 +2152,7 @@ Table Parameters: bucketing_version 2 #### A masked pattern was here #### numFiles 3 - totalSize 3290 + totalSize 3305 transactional true transactional_properties default #### A masked pattern was here #### @@ -2756,7 +2756,7 @@ Table Type: MANAGED_TABLE Table Parameters: bucketing_version 2 numFiles 1 - totalSize 1101 + totalSize 1106 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out b/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out index 030b61d6dc45..96abe6cd4571 100644 --- a/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out +++ b/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out @@ -54,7 +54,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 634 - totalSize 598 + totalSize 603 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out index fa852eb143c8..c0a00d0b60b5 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_opt_vectorization.q.out @@ -1362,7 +1362,7 @@ Partition Parameters: numFiles 2 numRows 32 rawDataSize 640 - totalSize 1432 + totalSize 1446 #### A masked pattern was here #### # Storage Information @@ -1402,7 +1402,7 @@ Partition Parameters: numFiles 2 numRows 6 rawDataSize 120 - totalSize 1136 + totalSize 1150 #### A masked pattern was here #### # Storage Information @@ -1442,7 +1442,7 @@ Partition Parameters: numFiles 2 numRows 14 rawDataSize 280 - totalSize 1250 + totalSize 1264 #### A masked pattern was here #### # Storage Information @@ -1482,7 +1482,7 @@ Partition Parameters: numFiles 2 numRows 6 rawDataSize 120 - totalSize 1136 + totalSize 1150 #### A masked pattern was here #### # Storage Information @@ -1521,7 +1521,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4656 + totalSize 4712 #### A masked pattern was here #### # Storage Information @@ -1560,7 +1560,7 @@ Partition Parameters: numFiles 4 numRows 6 rawDataSize 120 - totalSize 2082 + totalSize 2110 #### A masked pattern was here #### # Storage Information @@ -1599,7 +1599,7 @@ Partition Parameters: numFiles 8 numRows 32 rawDataSize 640 - totalSize 4674 + totalSize 4730 #### A masked pattern was here #### # Storage Information @@ -1638,7 +1638,7 @@ Partition Parameters: numFiles 4 numRows 6 rawDataSize 120 - totalSize 2082 + totalSize 2110 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out index 3f7cfa8ce9fd..86bf490d1971 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out @@ -1373,7 +1373,7 @@ Partition Parameters: numFiles 1 numRows 11 rawDataSize 88 - totalSize 466 + totalSize 473 #### A masked pattern was here #### # Storage Information @@ -1431,7 +1431,7 @@ Partition Parameters: numFiles 1 numRows 13 rawDataSize 104 - totalSize 489 + totalSize 496 #### A masked pattern was here #### # Storage Information @@ -1633,7 +1633,7 @@ Partition Parameters: numFiles 1 numRows 11 rawDataSize 88 - totalSize 466 + totalSize 473 #### A masked pattern was here #### # Storage Information @@ -1691,7 +1691,7 @@ Partition Parameters: numFiles 1 numRows 13 rawDataSize 104 - totalSize 489 + totalSize 496 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/explainanalyze_acid_with_direct_insert.q.out b/ql/src/test/results/clientpositive/llap/explainanalyze_acid_with_direct_insert.q.out index d06bd281d7d8..6b33ee77b2d1 100644 --- a/ql/src/test/results/clientpositive/llap/explainanalyze_acid_with_direct_insert.q.out +++ b/ql/src/test/results/clientpositive/llap/explainanalyze_acid_with_direct_insert.q.out @@ -679,24 +679,24 @@ STAGE PLANS: TableScan alias: analyze_part_table filterExpr: (b = 3) (type: boolean) - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) null sort order: z sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 11 (type: int), 3 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -711,10 +711,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 3 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -818,17 +818,17 @@ STAGE PLANS: TableScan alias: analyze_part_table filterExpr: (b = 1) (type: boolean) - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) null sort order: z sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: may be used (ACID table) Reducer 2 @@ -837,10 +837,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1639,24 +1639,24 @@ STAGE PLANS: TableScan alias: analyze_part_table filterExpr: (b = 3) (type: boolean) - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) null sort order: z sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: 11 (type: int), 3 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1671,10 +1671,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 3 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3/3 Data size: 6374 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3/3 Data size: 6434 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1778,17 +1778,17 @@ STAGE PLANS: TableScan alias: analyze_part_table filterExpr: (b = 1) (type: boolean) - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) null sort order: z sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized, llap LLAP IO: may be used (ACID table) Reducer 2 @@ -1797,10 +1797,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 2/2 Data size: 6316 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2/2 Data size: 6386 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/insert_only_to_acid_convert.q.out b/ql/src/test/results/clientpositive/llap/insert_only_to_acid_convert.q.out index 284e5d419a2c..079b22d84c28 100644 --- a/ql/src/test/results/clientpositive/llap/insert_only_to_acid_convert.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_only_to_acid_convert.q.out @@ -38,7 +38,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 182 - totalSize 310 + totalSize 317 transactional true transactional_properties insert_only #### A masked pattern was here #### @@ -84,7 +84,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 182 - totalSize 310 + totalSize 317 transactional true transactional_properties default #### A masked pattern was here #### @@ -159,7 +159,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 182 - totalSize 310 + totalSize 317 transactional true transactional_properties insert_only #### A masked pattern was here #### @@ -204,7 +204,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 182 - totalSize 310 + totalSize 317 transactional true transactional_properties insert_only #### A masked pattern was here #### @@ -296,7 +296,7 @@ Table Parameters: numPartitions 2 numRows 20 rawDataSize 2008 - totalSize 1920 + totalSize 1948 transactional true transactional_properties insert_only #### A masked pattern was here #### @@ -347,7 +347,7 @@ Table Parameters: numPartitions 2 numRows 20 rawDataSize 2008 - totalSize 1920 + totalSize 1948 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out index a2f11d24d4e7..c17b37147f0a 100644 --- a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out @@ -171,7 +171,7 @@ Table Parameters: numFiles 1 numRows 12288 rawDataSize 0 - totalSize 309566 + totalSize 309571 transactional true transactional_properties default #### A masked pattern was here #### @@ -340,7 +340,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 0 - totalSize 1668 + totalSize 1673 transactional true transactional_properties default #### A masked pattern was here #### @@ -436,7 +436,7 @@ Table Parameters: numFiles 2 numRows 4 rawDataSize 0 - totalSize 3337 + totalSize 3346 transactional true transactional_properties default #### A masked pattern was here #### @@ -528,7 +528,7 @@ Table Parameters: numFiles 3 numRows 12292 rawDataSize 0 - totalSize 312904 + totalSize 312919 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/masking_mv.q.out b/ql/src/test/results/clientpositive/llap/masking_mv.q.out index 01f6b95cb17f..ba4f4b82c741 100644 --- a/ql/src/test/results/clientpositive/llap/masking_mv.q.out +++ b/ql/src/test/results/clientpositive/llap/masking_mv.q.out @@ -160,7 +160,7 @@ Table Parameters: numFiles 1 numRows 500 rawDataSize 2000 - totalSize 976 + totalSize 983 #### A masked pattern was here #### # Storage Information @@ -862,7 +862,7 @@ Table Parameters: numFiles 1 numRows 500 rawDataSize 2000 - totalSize 976 + totalSize 983 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/masking_mv_by_text.q.out b/ql/src/test/results/clientpositive/llap/masking_mv_by_text.q.out index 2f797c8d7706..bd7182b92640 100644 --- a/ql/src/test/results/clientpositive/llap/masking_mv_by_text.q.out +++ b/ql/src/test/results/clientpositive/llap/masking_mv_by_text.q.out @@ -160,7 +160,7 @@ Table Parameters: numFiles 1 numRows 500 rawDataSize 2000 - totalSize 976 + totalSize 983 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out index 132ad39b95e4..65d6ba544faf 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out @@ -53,7 +53,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 1025 - totalSize 511 + totalSize 516 #### A masked pattern was here #### # Storage Information @@ -121,7 +121,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 580 - totalSize 356 + totalSize 362 #### A masked pattern was here #### # Storage Information @@ -273,7 +273,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 5 rawDataSize 1605 -totalSize 727 +totalSize 732 #### A masked pattern was here #### PREHOOK: query: drop materialized view cmv_mat_view_n4 PREHOOK: type: DROP_MATERIALIZED_VIEW @@ -373,4 +373,4 @@ POSTHOOK: Input: default@acid_cmv_part@k=0 POSTHOOK: Input: default@acid_cmv_part@k=10 POSTHOOK: Input: default@acid_cmv_part@k=100 #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"acid_cmv_part\"\n ],\n \"table:alias\": \"acid_cmv_part\",\n \"inputs\": [],\n \"rowCount\": 270.0,\n \"avgRowSize\": 200.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"k\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"ROW__IS__DELETED\"\n }\n ],\n \"partitionColumns\": [\n \"k\"\n ],\n \"colStats\": [\n {\n \"name\": \"value\",\n \"ndv\": 54\n },\n {\n \"name\": \"k\",\n \"ndv\": 3\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"k\",\n \"value\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 270.0\n }\n ]\n}","optimizedSQL":"SELECT `k`, `value`\nFROM `default`.`acid_cmv_part`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"acid_cmv_part","columns:":["value"],"database:":"default","table:":"acid_cmv_part","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"k (type: string), value (type: string)","columnExprMap:":{"_col0":"k","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"acid_cmv_part\"\n ],\n \"table:alias\": \"acid_cmv_part\",\n \"inputs\": [],\n \"rowCount\": 273.0,\n \"avgRowSize\": 200.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"k\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"ROW__IS__DELETED\"\n }\n ],\n \"partitionColumns\": [\n \"k\"\n ],\n \"colStats\": [\n {\n \"name\": \"value\",\n \"ndv\": 54\n },\n {\n \"name\": \"k\",\n \"ndv\": 3\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"k\",\n \"value\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 273.0\n }\n ]\n}","optimizedSQL":"SELECT `k`, `value`\nFROM `default`.`acid_cmv_part`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"acid_cmv_part","columns:":["value"],"database:":"default","table:":"acid_cmv_part","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"k (type: string), value (type: string)","columnExprMap:":{"_col0":"k","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_acid.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_acid.q.out index f68b72535844..23aa956d04e8 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_acid.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_acid.q.out @@ -55,7 +55,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 0 - totalSize 926 + totalSize 931 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out index 9ccf526a515b..2af2b9bc0078 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out @@ -63,7 +63,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 408 -totalSize 468 +totalSize 474 #### A masked pattern was here #### PREHOOK: query: create materialized view if not exists cmv_mat_view2_n4 as select a, c from cmv_basetable_n10 where a = 3 @@ -99,7 +99,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 232 -totalSize 334 +totalSize 340 #### A masked pattern was here #### PREHOOK: query: explain select a, c from cmv_basetable_n10 where a = 3 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out index 4b8cd42ef9d6..1b5b6839dcbd 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out @@ -296,7 +296,7 @@ Table Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1542 + totalSize 1554 transactional true transactional_properties default #### A masked pattern was here #### @@ -540,7 +540,7 @@ Table Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1542 + totalSize 1554 transactional true transactional_properties default #### A masked pattern was here #### @@ -1029,7 +1029,7 @@ Table Parameters: numFiles 3 numRows 3 rawDataSize 0 - totalSize 2301 + totalSize 2318 transactional true transactional_properties default #### A masked pattern was here #### @@ -1345,7 +1345,7 @@ Table Parameters: numFiles 2 numRows 3 rawDataSize 0 - totalSize 1047 + totalSize 1058 transactional true transactional_properties default #### A masked pattern was here #### @@ -1655,7 +1655,7 @@ Table Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1044 + totalSize 1056 transactional true transactional_properties default #### A masked pattern was here #### @@ -2071,7 +2071,7 @@ Table Parameters: numFiles 3 numRows 3 rawDataSize 0 - totalSize 1801 + totalSize 1819 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out index abeab03cf397..bed471893793 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out @@ -393,7 +393,7 @@ Table Parameters: numFiles 2 numRows 5 rawDataSize 0 - totalSize 1505 + totalSize 1516 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_by_text_multi_db.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_by_text_multi_db.q.out index 34d28e3b5edb..b4fd488bc8cf 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_by_text_multi_db.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_by_text_multi_db.q.out @@ -87,7 +87,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 408 -totalSize 468 +totalSize 474 #### A masked pattern was here #### PREHOOK: query: create materialized view if not exists cmv_mat_view2_n2 as select a, c from db1.cmv_basetable_n7 where a = 3 @@ -123,7 +123,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 232 -totalSize 334 +totalSize 340 #### A masked pattern was here #### PREHOOK: query: create database db3 PREHOOK: type: CREATEDATABASE diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out index 7f1db9ffd9ba..bf53cf8cbf47 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out @@ -63,7 +63,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 408 -totalSize 468 +totalSize 474 #### A masked pattern was here #### PREHOOK: query: create materialized view if not exists cmv_mat_view2 as select a, c from cmv_basetable_n0 where a = 3 @@ -99,7 +99,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 232 -totalSize 334 +totalSize 340 #### A masked pattern was here #### PREHOOK: query: explain select a, c from cmv_basetable_n0 where a = 3 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out index 34d28e3b5edb..b4fd488bc8cf 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out @@ -87,7 +87,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 408 -totalSize 468 +totalSize 474 #### A masked pattern was here #### PREHOOK: query: create materialized view if not exists cmv_mat_view2_n2 as select a, c from db1.cmv_basetable_n7 where a = 3 @@ -123,7 +123,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 2 rawDataSize 232 -totalSize 334 +totalSize 340 #### A masked pattern was here #### PREHOOK: query: create database db3 PREHOOK: type: CREATEDATABASE diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out index 87933394f39b..45096f02a774 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out @@ -290,7 +290,7 @@ Table Parameters: numRows 2 rawDataSize 232 rewriting.time.window 5min - totalSize 610 + totalSize 624 #### A masked pattern was here #### # Storage Information @@ -529,7 +529,7 @@ Table Parameters: numRows 2 rawDataSize 232 rewriting.time.window 5min - totalSize 610 + totalSize 624 #### A masked pattern was here #### # Storage Information @@ -873,7 +873,7 @@ Table Parameters: numRows 3 rawDataSize 348 rewriting.time.window 5min - totalSize 640 + totalSize 653 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window_2.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window_2.q.out index e9ddcb0722b9..8eb5d2674c0d 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window_2.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window_2.q.out @@ -96,7 +96,7 @@ Table Parameters: numFiles 2 numRows 2 rawDataSize 232 - totalSize 610 + totalSize 624 #### A masked pattern was here #### # Storage Information @@ -160,7 +160,7 @@ Table Parameters: numFiles 2 numRows 2 rawDataSize 232 - totalSize 610 + totalSize 624 #### A masked pattern was here #### # Storage Information @@ -224,7 +224,7 @@ Table Parameters: numFiles 2 numRows 3 rawDataSize 348 - totalSize 640 + totalSize 653 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out index 6c45f0b7c441..d35e1dc5ae52 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out @@ -75,7 +75,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 580 - totalSize 356 + totalSize 362 #### A masked pattern was here #### # Storage Information @@ -108,7 +108,7 @@ numFiles 1 numFilesErasureCoded 0 numRows 5 rawDataSize 580 -totalSize 356 +totalSize 362 #### A masked pattern was here #### PREHOOK: query: select a, c from cmv_mat_view_n8 PREHOOK: type: QUERY @@ -259,7 +259,7 @@ Table Parameters: numFiles 1 numRows 5 rawDataSize 1025 - totalSize 511 + totalSize 516 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_drop.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_drop.q.out index ce73b68498c7..fb415df88867 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_drop.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_drop.q.out @@ -22,9 +22,9 @@ columns:struct columns { i32 cint, string cstring1} partitioned:false partitionColumns: totalNumberFiles:1 -totalFileSize:47139 -maxFileSize:47139 -minFileSize:47139 +totalFileSize:47146 +maxFileSize:47146 +minFileSize:47146 #### A masked pattern was here #### PREHOOK: query: drop materialized view dmv_mat_view diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out index 9c7eacb07dbd..65562130e795 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partition_cluster.q.out @@ -301,7 +301,7 @@ Table Parameters: numPartitions 32 numRows 55 rawDataSize 9790 - totalSize 10870 + totalSize 11094 #### A masked pattern was here #### # Storage Information @@ -1828,7 +1828,7 @@ Table Parameters: numPartitions 32 numRows 57 rawDataSize 11062 - totalSize 17166 + totalSize 17390 #### A masked pattern was here #### # Storage Information @@ -2629,7 +2629,7 @@ Table Parameters: numPartitions 32 numRows 58 rawDataSize 11257 - totalSize 17169 + totalSize 17393 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out index 17ec0c351f40..9d989b20e379 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out @@ -263,7 +263,7 @@ Table Parameters: numPartitions 32 numRows 55 rawDataSize 5005 - totalSize 7682 + totalSize 7906 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out index f81ec3c9d23b..2eaaf04b6393 100644 --- a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out @@ -102,7 +102,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 53000 - totalSize 3231 + totalSize 3238 #### A masked pattern was here #### # Storage Information @@ -150,7 +150,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 53000 - totalSize 3231 + totalSize 3238 #### A masked pattern was here #### # Storage Information @@ -237,7 +237,7 @@ Table Parameters: numFiles 1 numRows 100 rawDataSize 52600 - totalSize 3231 + totalSize 3238 #### A masked pattern was here #### # Storage Information @@ -346,7 +346,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22150 - totalSize 2131 + totalSize 2138 #### A masked pattern was here #### # Storage Information @@ -387,7 +387,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22250 - totalSize 2143 + totalSize 2150 #### A masked pattern was here #### # Storage Information @@ -440,7 +440,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22150 - totalSize 2131 + totalSize 2138 #### A masked pattern was here #### # Storage Information @@ -481,7 +481,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22250 - totalSize 2143 + totalSize 2150 #### A masked pattern was here #### # Storage Information @@ -578,7 +578,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 21950 - totalSize 2131 + totalSize 2138 #### A masked pattern was here #### # Storage Information @@ -619,7 +619,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22050 - totalSize 2143 + totalSize 2150 #### A masked pattern was here #### # Storage Information @@ -734,7 +734,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22155 - totalSize 5396 + totalSize 5424 #### A masked pattern was here #### # Storage Information @@ -775,7 +775,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22243 - totalSize 5378 + totalSize 5404 #### A masked pattern was here #### # Storage Information @@ -828,7 +828,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22155 - totalSize 5396 + totalSize 5424 #### A masked pattern was here #### # Storage Information @@ -869,7 +869,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22243 - totalSize 5378 + totalSize 5404 #### A masked pattern was here #### # Storage Information @@ -972,7 +972,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 21955 - totalSize 5396 + totalSize 5424 #### A masked pattern was here #### # Storage Information @@ -1013,7 +1013,7 @@ Partition Parameters: numFiles 4 numRows 50 rawDataSize 22043 - totalSize 5378 + totalSize 5404 #### A masked pattern was here #### # Storage Information @@ -1122,7 +1122,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22150 - totalSize 2131 + totalSize 2138 #### A masked pattern was here #### # Storage Information @@ -1175,7 +1175,7 @@ Partition Parameters: numFiles 1 numRows 50 rawDataSize 22150 - totalSize 2131 + totalSize 2138 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/orc_file_dump.q.out b/ql/src/test/results/clientpositive/llap/orc_file_dump.q.out index 467f8e9d9055..f5a27f44a7ce 100644 --- a/ql/src/test/results/clientpositive/llap/orc_file_dump.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_file_dump.q.out @@ -93,7 +93,7 @@ PREHOOK: Input: default@orc_ppd_n0 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.1 Rows: 1049 Compression: ZLIB Compression size: 262144 @@ -270,7 +270,7 @@ Stripes: Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 98 loadFactor: 0.0156 expectedFpp: 5.9604645E-8 Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 102 loadFactor: 0.0163 expectedFpp: 6.9948186E-8 -File length: 32344 bytes +File length: 32351 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -291,7 +291,7 @@ PREHOOK: Input: default@orc_ppd_n0 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.1 Rows: 1049 Compression: ZLIB Compression size: 262144 @@ -468,7 +468,7 @@ Stripes: Entry 1: numHashFunctions: 7 bitCount: 9600 popCount: 174 loadFactor: 0.0181 expectedFpp: 6.426078E-13 Stripe level merge: numHashFunctions: 7 bitCount: 9600 popCount: 181 loadFactor: 0.0189 expectedFpp: 8.4693775E-13 -File length: 36988 bytes +File length: 36995 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -501,7 +501,7 @@ PREHOOK: Input: default@orc_ppd_part@ds=2015/hr=10 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.1 Rows: 1049 Compression: ZLIB Compression size: 262144 @@ -678,7 +678,7 @@ Stripes: Entry 1: numHashFunctions: 4 bitCount: 6272 popCount: 98 loadFactor: 0.0156 expectedFpp: 5.9604645E-8 Stripe level merge: numHashFunctions: 4 bitCount: 6272 popCount: 102 loadFactor: 0.0163 expectedFpp: 6.9948186E-8 -File length: 32344 bytes +File length: 32351 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index 3a270924a3b2..5abe2f747888 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -239,7 +239,7 @@ Table Parameters: orc.bloom.filter.columns * orc.row.index.stride 1000 rawDataSize 1139514 - totalSize 55696 + totalSize 55702 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out index 24b42becd299..fa6f68bb8500 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out @@ -239,7 +239,7 @@ Table Parameters: orc.bloom.filter.columns * orc.row.index.stride 1000 rawDataSize 1139514 - totalSize 55696 + totalSize 55702 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out index 9864e4838cf1..a71d287f2e58 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out @@ -1348,7 +1348,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: orc_llap_nonvector_2 - Statistics: Num rows: 12288 Data size: 4468180 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: 4468240 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/orc_merge1.q.out b/ql/src/test/results/clientpositive/llap/orc_merge1.q.out index e28e2e647b35..c1e508eefd15 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge1.q.out @@ -167,12 +167,12 @@ POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=0).value SIMPLE [(src)s POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 6 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 557 ### HDFS DATE ### hdfs://### HDFS PATH ### -rw-rw-rw- 3 ### USER ### ### GROUP ### 564 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 571 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 570 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 505 ### HDFS DATE ### hdfs://### HDFS PATH ### -rw-rw-rw- 3 ### USER ### ### GROUP ### 563 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 498 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 556 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 480 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 487 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b_n1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -350,7 +350,7 @@ POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=0).value SIMPLE [(src) POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1b_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1362 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1369 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1c_n1 PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -520,7 +520,7 @@ POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=0).value SIMPLE [(src) POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1c_n1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2463 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 2470 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1_n1 WHERE ds='1' diff --git a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out index 7784e6e5aeee..0ee4afa8e4bd 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out @@ -182,7 +182,7 @@ POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=0).value SIMPLE [(src)src. POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1 PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1756 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1763 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1b PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -375,7 +375,7 @@ POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=0).value SIMPLE [(src)src POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1b PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1756 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1763 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge1c PARTITION (ds='1', part) SELECT key, value, PMOD(HASH(key), 2) as part @@ -560,7 +560,7 @@ POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=0).value SIMPLE [(src)src POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge1c PARTITION(ds=1,part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1756 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1763 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1 WHERE ds='1' @@ -672,7 +672,7 @@ POSTHOOK: type: ALTER_PARTITION_MERGE POSTHOOK: Input: default@orcfile_merge1 POSTHOOK: Output: default@orcfile_merge1@ds=1/part=0 Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 1756 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 1763 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge1c WHERE ds='1' @@ -744,7 +744,7 @@ PREHOOK: Input: default@orcfile_merge1@ds=1/part=0 PREHOOK: Output: hdfs://### HDFS PATH ### -- BEGIN ORC FILE DUMP -- Structure for hdfs://### HDFS PATH ### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.1 Rows: 242 Compression: SNAPPY Compression size: 4096 @@ -781,7 +781,7 @@ Stripes: Row group indices for column 2: Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0 -File length: 1756 bytes +File length: 1763 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -795,7 +795,7 @@ PREHOOK: Input: default@orcfile_merge1c@ds=1/part=0 PREHOOK: Output: hdfs://### HDFS PATH ### -- BEGIN ORC FILE DUMP -- Structure for hdfs://### HDFS PATH ### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.1 Rows: 242 Compression: SNAPPY Compression size: 4096 @@ -832,7 +832,7 @@ Stripes: Row group indices for column 2: Entry 0: count: 242 hasNull: false min: val_0 max: val_97 sum: 1646 positions: 0,0,0 -File length: 1756 bytes +File length: 1763 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ diff --git a/ql/src/test/results/clientpositive/llap/orc_merge11.q.out b/ql/src/test/results/clientpositive/llap/orc_merge11.q.out index dbc6d828d5de..3bc63aeede46 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge11.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge11.q.out @@ -72,7 +72,7 @@ PREHOOK: Input: default@orcfile_merge1_n2 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.1 Rows: 50000 Compression: ZLIB Compression size: 4096 @@ -156,7 +156,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,506,294,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,666,54,0,312,64 -File length: 6685 bytes +File length: 6692 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -164,7 +164,7 @@ ________________________________________________________________________________ -- END ORC FILE DUMP -- -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.1 Rows: 50000 Compression: ZLIB Compression size: 4096 @@ -248,7 +248,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,506,294,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,666,54,0,312,64 -File length: 6685 bytes +File length: 6692 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ @@ -277,7 +277,7 @@ PREHOOK: Input: default@orcfile_merge1_n2 #### A masked pattern was here #### -- BEGIN ORC FILE DUMP -- #### A masked pattern was here #### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.1 Rows: 100000 Compression: ZLIB Compression size: 4096 @@ -426,7 +426,7 @@ Stripes: Entry 3: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,506,294,0,232,304 Entry 4: count: 10000 hasNull: false min: 1969-12-31 16:04:10.0 max: 1969-12-31 16:04:10.0 positions: 0,666,54,0,312,64 -File length: 12994 bytes +File length: 13001 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ diff --git a/ql/src/test/results/clientpositive/llap/orc_merge2.q.out b/ql/src/test/results/clientpositive/llap/orc_merge2.q.out index a9708e303cfd..1964f1c03d60 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge2.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge2.q.out @@ -213,7 +213,7 @@ POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=1).value SIMPL POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=7).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge2a_n0 PARTITION(one=1,two=9,three=7).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 351 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 358 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(*) USING 'tr \t _' AS (c) FROM orcfile_merge2a_n0 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge3.q.out b/ql/src/test/results/clientpositive/llap/orc_merge3.q.out index a4da817607a0..82a32433fd0c 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge3.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge3.q.out @@ -168,7 +168,7 @@ POSTHOOK: Output: default@orcfile_merge3b_n0 POSTHOOK: Lineage: orcfile_merge3b_n0.key SIMPLE [(orcfile_merge3a_n0)orcfile_merge3a_n0.FieldSchema(name:key, type:int, comment:null), ] POSTHOOK: Lineage: orcfile_merge3b_n0.value SIMPLE [(orcfile_merge3a_n0)orcfile_merge3a_n0.FieldSchema(name:value, type:string, comment:null), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2574 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 2581 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: SELECT SUM(HASH(c)) FROM ( SELECT TRANSFORM(key, value) USING 'tr \t _' AS (c) FROM orcfile_merge3a_n0 diff --git a/ql/src/test/results/clientpositive/llap/orc_merge4.q.out b/ql/src/test/results/clientpositive/llap/orc_merge4.q.out index 7c28d26b5cfd..8d784f8fae6f 100644 --- a/ql/src/test/results/clientpositive/llap/orc_merge4.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_merge4.q.out @@ -37,7 +37,7 @@ POSTHOOK: Output: default@orcfile_merge3a@ds=1 POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2532 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 2539 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: INSERT OVERWRITE TABLE orcfile_merge3a PARTITION (ds='1') SELECT * FROM src PREHOOK: type: QUERY @@ -63,9 +63,9 @@ POSTHOOK: Output: default@orcfile_merge3a@ds=2 POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: orcfile_merge3a PARTITION(ds=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2532 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 2539 ### HDFS DATE ### hdfs://### HDFS PATH ### Found 1 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 2532 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 2539 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: EXPLAIN INSERT OVERWRITE TABLE orcfile_merge3b SELECT key, value FROM orcfile_merge3a PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index 27346e0a737d..78b873b2200d 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -1913,7 +1913,7 @@ PREHOOK: Output: database:default PREHOOK: Output: default@orc_ppd_1 Stage-1 FILE SYSTEM COUNTERS: HDFS_BYTES_READ: 10583 - HDFS_BYTES_WRITTEN: 1468 + HDFS_BYTES_WRITTEN: 1475 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 HDFS_WRITE_OPS: 3 @@ -1949,7 +1949,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_1 PREHOOK: Output: hdfs://### HDFS PATH ### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1592 + HDFS_BYTES_READ: 1599 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 5 HDFS_LARGE_READ_OPS: 0 diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out index 8f3f96aebcce..c1ce4e06cc52 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out @@ -294,7 +294,7 @@ Table Parameters: numPartitions 2 numRows 8 rawDataSize 1148 - totalSize 852 + totalSize 866 #### A masked pattern was here #### # Storage Information @@ -331,7 +331,7 @@ Partition Parameters: numFiles 1 numRows 4 rawDataSize 400 - totalSize 325 + totalSize 332 #### A masked pattern was here #### # Storage Information @@ -370,7 +370,7 @@ Partition Parameters: numFiles 1 numRows 4 rawDataSize 748 - totalSize 527 + totalSize 534 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out b/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out index b757bbc9d773..d3b3d5f4c7bb 100644 --- a/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/sqlmerge_stats.q.out @@ -91,7 +91,7 @@ Table Parameters: numFiles 1 numRows 1 rawDataSize 0 - totalSize 659 + totalSize 666 transactional true transactional_properties default #### A masked pattern was here #### @@ -447,7 +447,7 @@ Table Parameters: numFiles 4 numRows 2 rawDataSize 0 - totalSize 2703 + totalSize 2730 transactional true transactional_properties default #### A masked pattern was here #### @@ -511,7 +511,7 @@ Table Parameters: numFiles 6 numRows 0 rawDataSize 0 - totalSize 4075 + totalSize 4115 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/stats_cbo_1.q.out b/ql/src/test/results/clientpositive/llap/stats_cbo_1.q.out index 65684ffac394..47e2405c15f9 100644 --- a/ql/src/test/results/clientpositive/llap/stats_cbo_1.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_cbo_1.q.out @@ -175,7 +175,7 @@ STAGE PLANS: TableScan alias: repro filterExpr: i is not null (type: boolean) - Statistics: Num rows: 1 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: i is not null (type: boolean) Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE @@ -202,7 +202,7 @@ STAGE PLANS: TableScan alias: repro filterExpr: i is not null (type: boolean) - Statistics: Num rows: 1 Data size: 1850 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 1920 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: i is not null (type: boolean) Statistics: Num rows: 1 Data size: 84 Basic stats: COMPLETE Column stats: COMPLETE diff --git a/ql/src/test/results/clientpositive/llap/stats_histogram.q.out b/ql/src/test/results/clientpositive/llap/stats_histogram.q.out index ab433500e7b9..cfe43efbe7a4 100644 --- a/ql/src/test/results/clientpositive/llap/stats_histogram.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_histogram.q.out @@ -260,7 +260,7 @@ Table Parameters: numFiles 15 numRows 15 rawDataSize 4287 - totalSize 11029 + totalSize 11134 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/stats_nonpart.q.out b/ql/src/test/results/clientpositive/llap/stats_nonpart.q.out index 3136e352e5b5..4addff2ab25d 100644 --- a/ql/src/test/results/clientpositive/llap/stats_nonpart.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_nonpart.q.out @@ -239,7 +239,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 0 - totalSize 720 + totalSize 726 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/stats_part.q.out b/ql/src/test/results/clientpositive/llap/stats_part.q.out index 5a208e0a37c1..55a7a2fc08be 100644 --- a/ql/src/test/results/clientpositive/llap/stats_part.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_part.q.out @@ -227,7 +227,7 @@ Table Parameters: numPartitions 3 numRows 6 rawDataSize 0 - totalSize 2274 + totalSize 2292 transactional true transactional_properties default #### A masked pattern was here #### @@ -288,7 +288,7 @@ Table Parameters: numPartitions 3 numRows 8 rawDataSize 0 - totalSize 3042 + totalSize 3066 transactional true transactional_properties default #### A masked pattern was here #### @@ -445,7 +445,7 @@ Table Parameters: numPartitions 3 numRows 8 rawDataSize 0 - totalSize 3042 + totalSize 3066 transactional true transactional_properties default #### A masked pattern was here #### @@ -530,7 +530,7 @@ Table Parameters: numPartitions 3 numRows 8 rawDataSize 0 - totalSize 3042 + totalSize 3066 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/stats_part2.q.out b/ql/src/test/results/clientpositive/llap/stats_part2.q.out index e90f93a03c50..ae0565ca816a 100644 --- a/ql/src/test/results/clientpositive/llap/stats_part2.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_part2.q.out @@ -275,7 +275,7 @@ Table Parameters: numPartitions 3 numRows 6 rawDataSize 0 - totalSize 4387 + totalSize 4425 transactional true transactional_properties default #### A masked pattern was here #### @@ -372,7 +372,7 @@ Table Parameters: numPartitions 3 numRows 8 rawDataSize 0 - totalSize 5850 + totalSize 5899 transactional true transactional_properties default #### A masked pattern was here #### @@ -526,7 +526,7 @@ Partition Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1459 + totalSize 1471 #### A masked pattern was here #### # Storage Information @@ -561,7 +561,7 @@ Partition Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1464 + totalSize 1477 #### A masked pattern was here #### # Storage Information @@ -596,7 +596,7 @@ Partition Parameters: numFiles 4 numRows 4 rawDataSize 0 - totalSize 2927 + totalSize 2951 #### A masked pattern was here #### # Storage Information @@ -732,7 +732,7 @@ Partition Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1459 + totalSize 1471 #### A masked pattern was here #### # Storage Information @@ -767,7 +767,7 @@ Partition Parameters: numFiles 4 numRows 2 rawDataSize 0 - totalSize 2919 + totalSize 2944 #### A masked pattern was here #### # Storage Information @@ -802,7 +802,7 @@ Partition Parameters: numFiles 4 numRows 4 rawDataSize 0 - totalSize 2927 + totalSize 2951 #### A masked pattern was here #### # Storage Information @@ -868,7 +868,7 @@ Partition Parameters: numFiles 2 numRows 2 rawDataSize 0 - totalSize 1459 + totalSize 1471 #### A masked pattern was here #### # Storage Information @@ -903,7 +903,7 @@ Partition Parameters: numFiles 4 numRows 2 rawDataSize 0 - totalSize 2919 + totalSize 2944 #### A masked pattern was here #### # Storage Information @@ -938,7 +938,7 @@ Partition Parameters: numFiles 4 numRows 4 rawDataSize 0 - totalSize 2927 + totalSize 2951 #### A masked pattern was here #### # Storage Information @@ -1006,7 +1006,7 @@ Partition Parameters: numFiles 3 numRows 1 rawDataSize 0 - totalSize 2163 + totalSize 2182 #### A masked pattern was here #### # Storage Information @@ -1041,7 +1041,7 @@ Partition Parameters: numFiles 5 numRows 1 rawDataSize 0 - totalSize 3617 + totalSize 3648 #### A masked pattern was here #### # Storage Information @@ -1076,7 +1076,7 @@ Partition Parameters: numFiles 4 numRows 4 rawDataSize 0 - totalSize 2927 + totalSize 2951 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/stats_part_multi_insert_acid.q.out b/ql/src/test/results/clientpositive/llap/stats_part_multi_insert_acid.q.out index c40f7d765208..455ec735ac7d 100644 --- a/ql/src/test/results/clientpositive/llap/stats_part_multi_insert_acid.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_part_multi_insert_acid.q.out @@ -81,7 +81,7 @@ Table Parameters: numPartitions 1 numRows 2 rawDataSize 0 - totalSize 1470 + totalSize 1482 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/stats_sizebug.q.out b/ql/src/test/results/clientpositive/llap/stats_sizebug.q.out index 051ccca3c339..15f2c2d481b7 100644 --- a/ql/src/test/results/clientpositive/llap/stats_sizebug.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_sizebug.q.out @@ -177,7 +177,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 0 - totalSize 720 + totalSize 726 transactional true transactional_properties default #### A masked pattern was here #### @@ -222,7 +222,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 0 - totalSize 720 + totalSize 726 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out index fe2abd5916e2..d1462a2a7ae5 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out @@ -3908,7 +3908,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesnullorc - Statistics: Num rows: 12288 Data size: 9470 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: 9540 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true Select Operator @@ -3916,7 +3916,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [] - Statistics: Num rows: 12288 Data size: 9470 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: 9540 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() Group By Vectorization: diff --git a/ql/src/test/results/clientpositive/row__id.q.out b/ql/src/test/results/clientpositive/row__id.q.out index a1123ac7cfef..9a92f4309e70 100644 --- a/ql/src/test/results/clientpositive/row__id.q.out +++ b/ql/src/test/results/clientpositive/row__id.q.out @@ -72,25 +72,25 @@ STAGE PLANS: Map Operator Tree: TableScan alias: hello_acid - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20134 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID.writeid (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20134 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: bigint) null sort order: z sort order: + - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20134 Basic stats: COMPLETE Column stats: NONE Execution mode: vectorized Reduce Operator Tree: Select Operator expressions: KEY.reducesinkkey0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20134 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20134 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -146,17 +146,17 @@ STAGE PLANS: TableScan alias: hello_acid filterExpr: (ROW__ID.writeid = 3L) (type: boolean) - Statistics: Num rows: 3 Data size: 19944 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 3 Data size: 20134 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (ROW__ID.writeid = 3L) (type: boolean) - Statistics: Num rows: 1 Data size: 6648 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 6711 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID.writeid (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 6648 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 6711 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 6648 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 6711 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out index 57a21e9a8693..0c3f3fe8da3d 100644 --- a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out +++ b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out @@ -372,10 +372,10 @@ POSTHOOK: Lineage: over10k_orc_bucketed_n0.si SIMPLE [(over10k_n9)over10k_n9.Fie POSTHOOK: Lineage: over10k_orc_bucketed_n0.t SIMPLE [(over10k_n9)over10k_n9.FieldSchema(name:t, type:tinyint, comment:null), ] POSTHOOK: Lineage: over10k_orc_bucketed_n0.ts SIMPLE [(over10k_n9)over10k_n9.FieldSchema(name:ts, type:timestamp, comment:null), ] Found 4 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 8746 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7525 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7167 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7058 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 8753 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7530 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7173 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7066 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: insert into over10k_orc_bucketed_n0 select * from over10k_n9 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n9 @@ -396,14 +396,14 @@ POSTHOOK: Lineage: over10k_orc_bucketed_n0.si SIMPLE [(over10k_n9)over10k_n9.Fie POSTHOOK: Lineage: over10k_orc_bucketed_n0.t SIMPLE [(over10k_n9)over10k_n9.FieldSchema(name:t, type:tinyint, comment:null), ] POSTHOOK: Lineage: over10k_orc_bucketed_n0.ts SIMPLE [(over10k_n9)over10k_n9.FieldSchema(name:ts, type:timestamp, comment:null), ] Found 8 items --rw-rw-rw- 3 ### USER ### ### GROUP ### 8746 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 8746 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7525 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7525 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7167 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7167 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7058 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7058 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 8753 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 8753 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7530 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7530 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7173 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7173 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7066 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7066 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: select distinct 7 as seven, INPUT__FILE__NAME from over10k_orc_bucketed_n0 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_orc_bucketed_n0 diff --git a/ql/src/test/results/clientpositive/tez/orc_merge12.q.out b/ql/src/test/results/clientpositive/tez/orc_merge12.q.out index c7615751fd65..d4f53b0ea288 100644 --- a/ql/src/test/results/clientpositive/tez/orc_merge12.q.out +++ b/ql/src/test/results/clientpositive/tez/orc_merge12.q.out @@ -144,7 +144,7 @@ PREHOOK: Input: default@alltypesorc3xcols PREHOOK: Output: hdfs://### HDFS PATH ### -- BEGIN ORC FILE DUMP -- Structure for hdfs://### HDFS PATH ### -File Version: 0.12 with ORC_14 +File Version: 0.12 with ORC_14 by ORC Java 1.8.1 Rows: 24576 Compression: ZLIB Compression size: 131072 @@ -814,7 +814,7 @@ Stripes: Entry 0: count: 6889 hasNull: true true: 3402 positions: 0,0,0,0,0,0,0,0 Entry 1: count: 2284 hasNull: true true: 581 positions: 0,168,8,0,0,520,97,1 -File length: 3004632 bytes +File length: 3004638 bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 57c3258af2a7..ee62b7a78aeb 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -88,7 +88,7 @@ 0.16.0 2.18.0 3.3.3 - 1.6.9 + 1.8.3 3.21.7 1.51.0 1.9.0