Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1265#discussion_r137809952
--- Diff: core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java ---
@@ -17,46 +17,183 @@
package org.apache.carbondata.core.datastore.chunk.store;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.ColumnType;
+import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.util.ByteUtil;
+/**
+ * ColumnPage wrapper for dimension column reader
+ */
public class ColumnPageWrapper implements DimensionColumnDataChunk {
+ private TableSpec.ColumnSpec columnSpec;
private ColumnPage columnPage;
+ private int columnValueSize;
- public ColumnPageWrapper(ColumnPage columnPage) {
+ public ColumnPageWrapper(ColumnPage columnPage, int columnValueSize) {
this.columnPage = columnPage;
+ this.columnValueSize = columnValueSize;
+ this.columnSpec = columnPage.getColumnSpec();
}
@Override
public int fillChunkData(byte[] data, int offset, int columnIndex,
KeyStructureInfo restructuringInfo) {
- throw new UnsupportedOperationException("internal error");
+ int surrogate = columnPage.getInt(columnIndex);
+ ByteBuffer buffer = ByteBuffer.wrap(data);
+ buffer.putInt(offset, surrogate);
+ return columnValueSize;
}
@Override
public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
KeyStructureInfo restructuringInfo) {
- throw new UnsupportedOperationException("internal error");
+ row[columnIndex] = columnPage.getInt(rowId);
+ return columnIndex + 1;
}
@Override
public int fillConvertedChunkData(ColumnVectorInfo[] vectorInfo, int column,
KeyStructureInfo restructuringInfo) {
- throw new UnsupportedOperationException("internal error");
+ // fill the vector with data in column page
+ ColumnVectorInfo columnVectorInfo = vectorInfo[column];
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ fillData(null, columnVectorInfo, vector);
+ return column + 1;
}
@Override
public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, int column,
KeyStructureInfo restructuringInfo) {
- throw new UnsupportedOperationException("internal error");
+ ColumnVectorInfo columnVectorInfo = vectorInfo[column];
+ CarbonColumnVector vector = columnVectorInfo.vector;
+ fillData(rowMapping, columnVectorInfo, vector);
+ return column + 1;
+ }
+
+ /**
+ * Copy columnar data from internal column page to `vector`, row information described by
+ * `columnVectorInfo`
+ */
+ private void fillData(int[] rowMapping, ColumnVectorInfo columnVectorInfo,
--- End diff --
For dictionary column, it still using `IndexStorageCodec` to do encoding and using `decodeDimensionLegacy` to decode, so `FixedLengthDimensionDataChunk` is still required. `VariableLengthDimensionDataChunk` is also required for backward compatiblity to read old data file
---
Locked
