Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1792#discussion_r164681706
--- Diff: processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java ---
@@ -80,255 +63,43 @@ public UnsafeCarbonRowPage(boolean[] noDictionaryDimensionMapping,
this.managerType = MemoryManagerType.UNSAFE_MEMORY_MANAGER;
}
- public int addRow(Object[] row) {
- int size = addRow(row, dataBlock.getBaseOffset() + lastSize);
+ public int addRow(Object[] row, ByteBuffer rowBuffer) {
+ int size = addRow(row, dataBlock.getBaseOffset() + lastSize, rowBuffer);
buffer.set(lastSize);
lastSize = lastSize + size;
return size;
}
- private int addRow(Object[] row, long address) {
- if (row == null) {
- throw new RuntimeException("Row is null ??");
- }
- int dimCount = 0;
- int size = 0;
- Object baseObject = dataBlock.getBaseObject();
- for (; dimCount < noDictionaryDimensionMapping.length; dimCount++) {
- if (noDictionaryDimensionMapping[dimCount]) {
- byte[] col = (byte[]) row[dimCount];
- CarbonUnsafe.getUnsafe()
- .putShort(baseObject, address + size, (short) col.length);
- size += 2;
- CarbonUnsafe.getUnsafe().copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
- address + size, col.length);
- size += col.length;
- } else {
- int value = (int) row[dimCount];
- CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, value);
- size += 4;
- }
- }
-
- // write complex dimensions here.
- for (; dimCount < dimensionSize; dimCount++) {
- byte[] col = (byte[]) row[dimCount];
- CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, (short) col.length);
- size += 2;
- CarbonUnsafe.getUnsafe().copyMemory(col, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
- address + size, col.length);
- size += col.length;
- }
- Arrays.fill(nullSetWords, 0);
- int nullSetSize = nullSetWords.length * 8;
- int nullWordLoc = size;
- size += nullSetSize;
- for (int mesCount = 0; mesCount < measureSize; mesCount++) {
- Object value = row[mesCount + dimensionSize];
- if (null != value) {
- DataType dataType = measureDataType[mesCount];
- if (dataType == DataTypes.BOOLEAN) {
- Boolean bval = (Boolean) value;
- CarbonUnsafe.getUnsafe().putBoolean(baseObject, address + size, bval);
- size += 1;
- } else if (dataType == DataTypes.SHORT) {
- Short sval = (Short) value;
- CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, sval);
- size += 2;
- } else if (dataType == DataTypes.INT) {
- Integer ival = (Integer) value;
- CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, ival);
- size += 4;
- } else if (dataType == DataTypes.LONG) {
- Long val = (Long) value;
- CarbonUnsafe.getUnsafe().putLong(baseObject, address + size, val);
- size += 8;
- } else if (dataType == DataTypes.DOUBLE) {
- Double doubleVal = (Double) value;
- CarbonUnsafe.getUnsafe().putDouble(baseObject, address + size, doubleVal);
- size += 8;
- } else if (DataTypes.isDecimal(dataType)) {
- BigDecimal decimalVal = (BigDecimal) value;
- byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(decimalVal);
- CarbonUnsafe.getUnsafe()
- .putShort(baseObject, address + size, (short) bigDecimalInBytes.length);
- size += 2;
- CarbonUnsafe.getUnsafe()
- .copyMemory(bigDecimalInBytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject,
- address + size, bigDecimalInBytes.length);
- size += bigDecimalInBytes.length;
- } else {
- throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]);
- }
- set(nullSetWords, mesCount);
- } else {
- unset(nullSetWords, mesCount);
- }
- }
- CarbonUnsafe.getUnsafe().copyMemory(nullSetWords, CarbonUnsafe.LONG_ARRAY_OFFSET, baseObject,
- address + nullWordLoc, nullSetSize);
- return size;
+ /**
+ * add row as 3 parts
+ * @param row
+ * @param address
+ * @return
+ */
+ private int addRow(Object[] row, long address, ByteBuffer rowBuffer) {
+ return sortStepRowHandler.writeRawRowAsIntermediateSortTempRowToUnsafeMemory(row,
--- End diff --
Can you add a conversion function to convert row to IntermediateSortTempRow, then add call sortStepRowHandler
---
Locked
