Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2819#discussion_r226863659
--- Diff: core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v3/CompressedDimensionChunkFileBasedReaderV3.java ---
@@ -221,49 +229,66 @@ protected DimensionRawColumnChunk getDimensionRawColumnChunk(FileReader fileRead
int offset = (int) rawColumnPage.getOffSet() + dimensionChunksLength
.get(rawColumnPage.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber);
// first read the data and uncompressed it
- return decodeDimension(rawColumnPage, rawData, pageMetadata, offset);
+ return decodeDimension(rawColumnPage, rawData, pageMetadata, offset, vectorInfo);
+ }
+
+ @Override
+ public void decodeColumnPageAndFillVector(DimensionRawColumnChunk dimensionRawColumnChunk,
+ int pageNumber, ColumnVectorInfo vectorInfo) throws IOException, MemoryException {
+ DimensionColumnPage columnPage =
+ decodeColumnPage(dimensionRawColumnChunk, pageNumber, vectorInfo);
+ columnPage.freeMemory();
}
- private ColumnPage decodeDimensionByMeta(DataChunk2 pageMetadata,
- ByteBuffer pageData, int offset, boolean isLocalDictEncodedPage)
+ private ColumnPage decodeDimensionByMeta(DataChunk2 pageMetadata, ByteBuffer pageData, int offset,
+ boolean isLocalDictEncodedPage, ColumnVectorInfo vectorInfo, BitSet nullBitSet)
throws IOException, MemoryException {
List<Encoding> encodings = pageMetadata.getEncoders();
List<ByteBuffer> encoderMetas = pageMetadata.getEncoder_meta();
String compressorName = CarbonMetadataUtil.getCompressorNameFromChunkMeta(
pageMetadata.getChunk_meta());
ColumnPageDecoder decoder = encodingFactory.createDecoder(encodings, encoderMetas,
- compressorName);
- return decoder
- .decode(pageData.array(), offset, pageMetadata.data_page_length, isLocalDictEncodedPage);
+ compressorName, vectorInfo != null);
+ if (vectorInfo != null) {
+ return decoder
+ .decodeAndFillVector(pageData.array(), offset, pageMetadata.data_page_length, vectorInfo,
+ nullBitSet, isLocalDictEncodedPage);
+ } else {
+ return decoder
+ .decode(pageData.array(), offset, pageMetadata.data_page_length, isLocalDictEncodedPage);
+ }
}
protected DimensionColumnPage decodeDimension(DimensionRawColumnChunk rawColumnPage,
- ByteBuffer pageData, DataChunk2 pageMetadata, int offset)
+ ByteBuffer pageData, DataChunk2 pageMetadata, int offset, ColumnVectorInfo vectorInfo)
throws IOException, MemoryException {
List<Encoding> encodings = pageMetadata.getEncoders();
if (CarbonUtil.isEncodedWithMeta(encodings)) {
- ColumnPage decodedPage = decodeDimensionByMeta(pageMetadata, pageData, offset,
- null != rawColumnPage.getLocalDictionary());
- decodedPage.setNullBits(QueryUtil.getNullBitSet(pageMetadata.presence, this.compressor));
int[] invertedIndexes = new int[0];
int[] invertedIndexesReverse = new int[0];
// in case of no dictionary measure data types, if it is included in sort columns
// then inverted index to be uncompressed
+ boolean isExplicitSorted =
+ CarbonUtil.hasEncoding(pageMetadata.encoders, Encoding.INVERTED_INDEX);
+ int dataOffset = offset;
if (encodings.contains(Encoding.INVERTED_INDEX)) {
offset += pageMetadata.data_page_length;
- if (CarbonUtil.hasEncoding(pageMetadata.encoders, Encoding.INVERTED_INDEX)) {
+ if (isExplicitSorted) {
--- End diff --
ok
---
Locked
