Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2819#discussion_r228040088
--- Diff: core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/AbstractNonDictionaryVectorFiller.java ---
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *
http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.chunk.store.impl.safe;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+public abstract class AbstractNonDictionaryVectorFiller {
+
+ protected int lengthSize;
+ protected int numberOfRows;
+
+ public AbstractNonDictionaryVectorFiller(int lengthSize, int numberOfRows) {
+ this.lengthSize = lengthSize;
+ this.numberOfRows = numberOfRows;
+ }
+
+ public abstract void fillVector(byte[] data, CarbonColumnVector vector, ByteBuffer buffer);
+
+ public int getLengthFromBuffer(ByteBuffer buffer) {
+ return buffer.getShort();
+ }
+}
+
+class NonDictionaryVectorFillerFactory {
+
+ public static AbstractNonDictionaryVectorFiller getVectorFiller(DataType type, int lengthSize,
+ int numberOfRows) {
+ if (type == DataTypes.STRING) {
+ return new StringVectorFiller(lengthSize, numberOfRows);
+ } else if (type == DataTypes.VARCHAR) {
+ return new LongStringVectorFiller(lengthSize, numberOfRows);
+ } else if (type == DataTypes.TIMESTAMP) {
+ return new TimeStampVectorFiller(lengthSize, numberOfRows);
+ } else if (type == DataTypes.BOOLEAN) {
+ return new BooleanVectorFiller(lengthSize, numberOfRows);
+ } else if (type == DataTypes.SHORT) {
+ return new ShortVectorFiller(lengthSize, numberOfRows);
+ } else if (type == DataTypes.INT) {
+ return new IntVectorFiller(lengthSize, numberOfRows);
+ } else if (type == DataTypes.LONG) {
+ return new LongVectorFiller(lengthSize, numberOfRows);
+ } else {
+ throw new UnsupportedOperationException("Not supported datatype : " + type);
+ }
+
+ }
+
+}
+
+class StringVectorFiller extends AbstractNonDictionaryVectorFiller {
+
+ public StringVectorFiller(int lengthSize, int numberOfRows) {
+ super(lengthSize, numberOfRows);
+ }
+
+ @Override
+ public void fillVector(byte[] data, CarbonColumnVector vector, ByteBuffer buffer) {
+ // start position will be used to store the current data position
+ int startOffset = 0;
+ // as first position will be start from length of bytes as data is stored first in the memory
+ // block we need to skip first two bytes this is because first two bytes will be length of the
+ // data which we have to skip
+ int currentOffset = lengthSize;
+ ByteUtil.UnsafeComparer comparator = ByteUtil.UnsafeComparer.INSTANCE;
+ for (int i = 0; i < numberOfRows - 1; i++) {
+ buffer.position(startOffset);
+ startOffset += getLengthFromBuffer(buffer) + lengthSize;
+ int length = startOffset - (currentOffset);
+ if (comparator.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, 0,
+ CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length, data, currentOffset, length)) {
+ vector.putNull(i);
+ } else {
+ vector.putByteArray(i, currentOffset, length, data);
+ }
+ currentOffset = startOffset + lengthSize;
+ }
+ // Handle last row
+ int length = (data.length - currentOffset);
+ if (comparator.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, 0,
+ CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length, data, currentOffset, length)) {
+ vector.putNull(numberOfRows - 1);
+ } else {
+ vector.putByteArray(numberOfRows - 1, currentOffset, length, data);
+ }
+ }
+}
+
+class LongStringVectorFiller extends StringVectorFiller {
--- End diff --
Yes, all these are non public classes, only used in this class
---
Locked
