akkio-97 commented on a change in pull request #4073:
URL:
https://github.com/apache/carbondata/pull/4073#discussion_r565936260##########
File path: core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
##########
@@ -141,100 +165,116 @@ public void fillVector(Object valuesToBeConverted, int size,
// for string, varchar, binary, date, decimal types
lengthStoredInBytes = shortSizeInBytes;
}
- byte[] data = (byte[]) valuesToBeConverted;
- if (pageType == DataTypes.BYTE) {
- for (int i = 0; i < size; i++) {
- if (nullBitSet.get(i)) {
- vector.putNull(i);
- } else {
- BigDecimal value = BigDecimal.valueOf(data[i], scale);
- if (value.scale() < newMeasureScale) {
- value = value.setScale(newMeasureScale);
- }
- vector.putDecimal(i, value, precision);
- }
- }
- } else if (pageType == DataTypes.SHORT) {
+ if (this instanceof DecimalUnscaledConverter && scale < newMeasureScale) {
+ scale = newMeasureScale;
+ }
+
+ if (valuesToBeConverted instanceof byte[][]) {
+ byte[][] data = (byte[][]) valuesToBeConverted;
for (int i = 0; i < size; i++) {
if (nullBitSet.get(i)) {
vector.putNull(i);
} else {
- BigDecimal value = BigDecimal
- .valueOf(ByteUtil.toShortLittleEndian(data, i * shortSizeInBytes),
- scale);
+ BigInteger bigInteger = new BigInteger(data[i]);
+ BigDecimal value = new BigDecimal(bigInteger, scale);
if (value.scale() < newMeasureScale) {
value = value.setScale(newMeasureScale);
}
vector.putDecimal(i, value, precision);
}
}
- } else if (pageType == DataTypes.SHORT_INT) {
- int shortIntSizeInBytes = DataTypes.SHORT_INT.getSizeInBytes();
- for (int i = 0; i < size; i++) {
- if (nullBitSet.get(i)) {
- vector.putNull(i);
- } else {
- BigDecimal value = BigDecimal
- .valueOf(ByteUtil.valueOf3Bytes(data, i * shortIntSizeInBytes),
- scale);
- if (value.scale() < newMeasureScale) {
- value = value.setScale(newMeasureScale);
+ } else if (valuesToBeConverted instanceof byte[]) {
+ byte[] data = (byte[]) valuesToBeConverted;
+ if (pageType == DataTypes.BYTE) {
+ for (int i = 0; i < size; i++) {
+ if (nullBitSet.get(i)) {
+ vector.putNull(i);
+ } else {
+ BigDecimal value = BigDecimal.valueOf(data[i], scale);
Review comment:
done
##########
File path: integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkStoreCreatorForPresto.scala
##########
@@ -365,6 +366,15 @@ class SparkStoreCreatorForPresto extends QueryTest with BeforeAndAfterAll{
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO TABLE streaming_table""")
}
+ test("Test decimal unscaled converter") {
+ sql("drop table if exists array_decimal")
+ sql(
+ "CREATE TABLE IF NOT EXISTS array_decimal (salary array<decimal(20,3)>) STORED AS " +
Review comment:
okay
##########
File path: core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
##########
@@ -88,11 +88,34 @@ void fillVector(Object valuesToBeConverted, int size, ColumnVectorInfo info, Bit
DecimalConverterType getDecimalConverterType();
+ default int getPrecisionForDimension(CarbonColumnVector vector) {
Review comment:
okay
##########
File path: core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
##########
@@ -328,9 +328,29 @@ public BigDecimal getDecimal(Object valueToBeConverted) {
public void fillVector(Object valuesToBeConverted, int size,
ColumnVectorInfo vectorInfo, BitSet nullBitSet, DataType pageType) {
CarbonColumnVector vector = getCarbonColumnVector(vectorInfo, nullBitSet);
- //TODO handle complex child
- int precision = vectorInfo.measure.getMeasure().getPrecision();
- int newMeasureScale = vectorInfo.measure.getMeasure().getScale();
+ int precision;
+ int newMeasureScale;
+ if (vectorInfo.measure == null) {
+ // complex primitive decimal flow comes as dimension
+ precision = ((DecimalType) vector.getType()).getPrecision();
+ newMeasureScale = ((DecimalType) vector.getType()).getScale();
+ size = ColumnVectorInfo.getUpdatedPageSizeForChildVector(vectorInfo, size);
+ } else {
+ precision = vectorInfo.measure.getMeasure().getPrecision();
+ newMeasureScale = vectorInfo.measure.getMeasure().getScale();
+ }
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[hidden email]
Locked
