[jira] [Created] (CARBONDATA-2409) Inappropriate Exception handling while select query with complex Data type

[Akash R Nilugal (Jira)]

Vandana Yadav created CARBONDATA-2409:
-----------------------------------------

             Summary: Inappropriate Exception handling while select query with complex Data type
                 Key: CARBONDATA-2409
                 URL: https://issues.apache.org/jira/browse/CARBONDATA-2409
             Project: CarbonData
          Issue Type: Bug
          Components: data-query
    Affects Versions: 1.4.0
         Environment: spark 2.2
            Reporter: Vandana Yadav
         Attachments: Array.csv

Inappropriate Exception handling while select query with complex Datatype

steps to Reproduce:

1) Create Table:

create table Array_com (CUST_ID string, YEAR int, MONTH int, AGE int, GENDER string, EDUCATED string, IS_MARRIED string, ARRAY_INT array<int>,ARRAY_STRING array<string>,ARRAY_DATE array<timestamp>,CARD_COUNT int,DEBIT_COUNT int, CREDIT_COUNT int, DEPOSIT double, HQ_DEPOSIT double) STORED BY 'org.apache.carbondata.format';

2) Load data into table:

LOAD DATA INPATH 'HDFS_URL/BabuStore/Data/complex/Array.csv' INTO table Array_com options ('DELIMITER'=',', 'QUOTECHAR'='"', 'FILEHEADER'='CUST_ID,YEAR,MONTH,AGE,GENDER,EDUCATED,IS_MARRIED,ARRAY_INT,ARRAY_STRING,ARRAY_DATE,CARD_COUNT,DEBIT_COUNT,CREDIT_COUNT,DEPOSIT,HQ_DEPOSIT','COMPLEX_DELIMITER_LEVEL_1'='$');

3) Execute Query:

SELECT array_int[2], array_date[2],array_string[1], count(distinct array_date[2]) AS count_date,count(distinct array_string[2]) as count_string,count(distinct array_int[0]) as count_int FROM (select * from array_com) SUB_QRY GROUP BY array_int[2], array_date[2],array_string[1] order by array_int[2], array_date[2],array_string[1];

 

4) Expected Result: It should display the correct result as per the select query.

5) Actual Result:

Error: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 455.0 failed 1 times, most recent failure: Lost task 0.0 in stage 455.0 (TID 33517, localhost, executor driver): java.util.concurrent.ExecutionException: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 38, Column 25: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 38, Column 25: Cannot cast "org.apache.spark.unsafe.types.UTF8String" to "int"
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private UnsafeRow result;
/* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 011 */
/* 012 */ public SpecificUnsafeProjection(Object[] references) {
/* 013 */ this.references = references;
/* 014 */ result = new UnsafeRow(7);
/* 015 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 64);
/* 016 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 7);
/* 017 */
/* 018 */ }
/* 019 */
/* 020 */ public void initialize(int partitionIndex) {
/* 021 */
/* 022 */ }
/* 023 */
/* 024 */
/* 025 */ private void apply_1(InternalRow i) {
/* 026 */
/* 027 */
/* 028 */ boolean isNull6 = true;
/* 029 */ UTF8String value6 = null;
/* 030 */
/* 031 */ boolean isNull7 = i.isNullAt(1);
/* 032 */ ArrayData value7 = isNull7 ? null : (i.getArray(1));
/* 033 */ if (!isNull7) {
/* 034 */
/* 035 */
/* 036 */ isNull6 = false; // resultCode could change nullability.
/* 037 */
/* 038 */ final int index2 = (int) ((UTF8String) references[0]);
/* 039 */ if (index2 >= value7.numElements() || index2 < 0 || value7.isNullAt(index2)) {
/* 040 */ isNull6 = true;
/* 041 */ } else {
/* 042 */ value6 = value7.getUTF8String(index2);
/* 043 */ }
/* 044 */
/* 045 */
/* 046 */ }
/* 047 */ if (isNull6) {
/* 048 */ rowWriter.setNullAt(2);
/* 049 */ } else {
/* 050 */ rowWriter.write(2, value6);
/* 051 */ }
/* 052 */
/* 053 */
/* 054 */ boolean isNull9 = true;
/* 055 */ int value9 = -1;
/* 056 */
/* 057 */ boolean isNull10 = i.isNullAt(0);
/* 058 */ ArrayData value10 = isNull10 ? null : (i.getArray(0));
/* 059 */ if (!isNull10) {
/* 060 */
/* 061 */
/* 062 */ isNull9 = false; // resultCode could change nullability.
/* 063 */
/* 064 */ final int index3 = (int) 0;
/* 065 */ if (index3 >= value10.numElements() || index3 < 0 || value10.isNullAt(index3)) {
/* 066 */ isNull9 = true;
/* 067 */ } else {
/* 068 */ value9 = value10.getInt(index3);
/* 069 */ }
/* 070 */
/* 071 */
/* 072 */ }
/* 073 */ if (isNull9) {
/* 074 */ rowWriter.setNullAt(3);
/* 075 */ } else {
/* 076 */ rowWriter.write(3, value9);
/* 077 */ }
/* 078 */
/* 079 */ }
/* 080 */
/* 081 */
/* 082 */ private void apply_0(InternalRow i) {
/* 083 */
/* 084 */
/* 085 */ boolean isNull = true;
/* 086 */ int value = -1;
/* 087 */
/* 088 */ boolean isNull1 = i.isNullAt(0);
/* 089 */ ArrayData value1 = isNull1 ? null : (i.getArray(0));
/* 090 */ if (!isNull1) {
/* 091 */
/* 092 */
/* 093 */ isNull = false; // resultCode could change nullability.
/* 094 */
/* 095 */ final int index = (int) 2;
/* 096 */ if (index >= value1.numElements() || index < 0 || value1.isNullAt(index)) {
/* 097 */ isNull = true;
/* 098 */ } else {
/* 099 */ value = value1.getInt(index);
/* 100 */ }
/* 101 */
/* 102 */
/* 103 */ }
/* 104 */ if (isNull) {
/* 105 */ rowWriter.setNullAt(0);
/* 106 */ } else {
/* 107 */ rowWriter.write(0, value);
/* 108 */ }
/* 109 */
/* 110 */
/* 111 */ boolean isNull3 = true;
/* 112 */ long value3 = -1L;
/* 113 */
/* 114 */ boolean isNull4 = i.isNullAt(2);
/* 115 */ ArrayData value4 = isNull4 ? null : (i.getArray(2));
/* 116 */ if (!isNull4) {
/* 117 */
/* 118 */
/* 119 */ isNull3 = false; // resultCode could change nullability.
/* 120 */
/* 121 */ final int index1 = (int) 2L;
/* 122 */ if (index1 >= value4.numElements() || index1 < 0 || value4.isNullAt(index1)) {
/* 123 */ isNull3 = true;
/* 124 */ } else {
/* 125 */ value3 = value4.getLong(index1);
/* 126 */ }
/* 127 */
/* 128 */
/* 129 */ }
/* 130 */ if (isNull3) {
/* 131 */ rowWriter.setNullAt(1);
/* 132 */ } else {
/* 133 */ rowWriter.write(1, value3);
/* 134 */ }
/* 135 */
/* 136 */ }
/* 137 */
/* 138 */
/* 139 */ private void apply_2(InternalRow i) {
/* 140 */
/* 141 */
/* 142 */ final long value12 = -1L;
/* 143 */ if (true) {
/* 144 */ rowWriter.setNullAt(4);
/* 145 */ } else {
/* 146 */ rowWriter.write(4, value12);
/* 147 */ }
/* 148 */
/* 149 */
/* 150 */ final UTF8String value13 = null;
/* 151 */ if (true) {
/* 152 */ rowWriter.setNullAt(5);
/* 153 */ } else {
/* 154 */ rowWriter.write(5, value13);
/* 155 */ }
/* 156 */
/* 157 */
/* 158 */ rowWriter.write(6, 1);
/* 159 */
/* 160 */ }
/* 161 */
/* 162 */
/* 163 */ // Scala.Function1 need this
/* 164 */ public java.lang.Object apply(java.lang.Object row) {
/* 165 */ return apply((InternalRow) row);
/* 166 */ }
/* 167 */
/* 168 */ public UnsafeRow apply(InternalRow i) {
/* 169 */ holder.reset();
/* 170 */
/* 171 */ rowWriter.zeroOutNullBytes();
/* 172 */ apply_0(i);
/* 173 */ apply_1(i);
/* 174 */ apply_2(i);
/* 175 */ result.setTotalSize(holder.totalSize());
/* 176 */ return result;
/* 177 */ }
/* 178 */ }

at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306)
 at org.spark_project.guava.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293)
 at org.spark_project.guava.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
 at org.spark_project.guava.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135)
 at org.spark_project.guava.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410)
 at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2380)
 at org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
 at org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
 at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
 at org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
 at org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
 at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:946)
 at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:412)
 at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:366)
 at org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:32)
 at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:930)
 at org.apache.spark.sql.catalyst.expressions.UnsafeProjection$.create(Projection.scala:130)
 at org.apache.spark.sql.catalyst.expressions.UnsafeProjection$.create(Projection.scala:140)
 at org.apache.spark.sql.execution.ExpandExec$$anonfun$2.apply(ExpandExec.scala:53)
 at org.apache.spark.sql.execution.ExpandExec$$anonfun$2.apply(ExpandExec.scala:53)
 at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
 at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
 at scala.collection.immutable.List.foreach(List.scala:381)
 at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
 at scala.collection.immutable.List.map(List.scala:285)
 at org.apache.spark.sql.execution.ExpandExec$$anonfun$doExecute$1$$anonfun$3.apply(ExpandExec.scala:59)
 at org.apache.spark.sql.execution.ExpandExec$$anonfun$doExecute$1$$anonfun$3.apply(ExpandExec.scala:58)
 at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:797)
 at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:797)
 at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
 at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
 at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
 at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
 at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
 at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
 at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
 at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
 at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
 at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)
 at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)
 at org.apache.spark.scheduler.Task.run(Task.scala:108)
 at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:338)
 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
 at java.lang.Thread.run(Thread.java:748)
Caused by: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 38, Column 25: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 38, Column 25: Cannot cast "org.apache.spark.unsafe.types.UTF8String" to "int"
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private UnsafeRow result;
/* 009 */ private org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 010 */ private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 011 */
/* 012 */ public SpecificUnsafeProjection(Object[] references) {
/* 013 */ this.references = references;
/* 014 */ result = new UnsafeRow(7);
/* 015 */ this.holder = new org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 64);
/* 016 */ this.rowWriter = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 7);
/* 017 */
/* 018 */ }
/* 019 */
/* 020 */ public void initialize(int partitionIndex) {
/* 021 */
/* 022 */ }
/* 023 */
/* 024 */
/* 025 */ private void apply_1(InternalRow i) {
/* 026 */
/* 027 */
/* 028 */ boolean isNull6 = true;
/* 029 */ UTF8String value6 = null;
/* 030 */
/* 031 */ boolean isNull7 = i.isNullAt(1);
/* 032 */ ArrayData value7 = isNull7 ? null : (i.getArray(1));
/* 033 */ if (!isNull7) {
/* 034 */
/* 035 */
/* 036 */ isNull6 = false; // resultCode could change nullability.
/* 037 */
/* 038 */ final int index2 = (int) ((UTF8String) references[0]);
/* 039 */ if (index2 >= value7.numElements() || index2 < 0 || value7.isNullAt(index2)) {
/* 040 */ isNull6 = true;
/* 041 */ } else {
/* 042 */ value6 = value7.getUTF8String(index2);
/* 043 */ }
/* 044 */
/* 045 */
/* 046 */ }
/* 047 */ if (isNull6) {
/* 048 */ rowWriter.setNullAt(2);
/* 049 */ } else {
/* 050 */ rowWriter.write(2, value6);
/* 051 */ }
/* 052 */
/* 053 */
/* 054 */ boolean isNull9 = true;
/* 055 */ int value9 = -1;
/* 056 */
/* 057 */ boolean isNull10 = i.isNullAt(0);
/* 058 */ ArrayData value10 = isNull10 ? null : (i.getArray(0));
/* 059 */ if (!isNull10) {
/* 060 */
/* 061 */
/* 062 */ isNull9 = false; // resultCode could change nullability.
/* 063 */
/* 064 */ final int index3 = (int) 0;
/* 065 */ if (index3 >= value10.numElements() || index3 < 0 || value10.isNullAt(index3)) {
/* 066 */ isNull9 = true;
/* 067 */ } else {
/* 068 */ value9 = value10.getInt(index3);
/* 069 */ }
/* 070 */
/* 071 */
/* 072 */ }
/* 073 */ if (isNull9) {
/* 074 */ rowWriter.setNullAt(3);
/* 075 */ } else {
/* 076 */ rowWriter.write(3, value9);
/* 077 */ }
/* 078 */
/* 079 */ }
/* 080 */
/* 081 */
/* 082 */ private void apply_0(InternalRow i) {
/* 083 */
/* 084 */
/* 085 */ boolean isNull = true;
/* 086 */ int value = -1;
/* 087 */
/* 088 */ boolean isNull1 = i.isNullAt(0);
/* 089 */ ArrayData value1 = isNull1 ? null : (i.getArray(0));
/* 090 */ if (!isNull1) {
/* 091 */
/* 092 */
/* 093 */ isNull = false; // resultCode could change nullability.
/* 094 */
/* 095 */ final int index = (int) 2;
/* 096 */ if (index >= value1.numElements() || index < 0 || value1.isNullAt(index)) {
/* 097 */ isNull = true;
/* 098 */ } else {
/* 099 */ value = value1.getInt(index);
/* 100 */ }
/* 101 */
/* 102 */
/* 103 */ }
/* 104 */ if (isNull) {
/* 105 */ rowWriter.setNullAt(0);
/* 106 */ } else {
/* 107 */ rowWriter.write(0, value);
/* 108 */ }
/* 109 */
/* 110 */
/* 111 */ boolean isNull3 = true;
/* 112 */ long value3 = -1L;
/* 113 */
/* 114 */ boolean isNull4 = i.isNullAt(2);
/* 115 */ ArrayData value4 = isNull4 ? null : (i.getArray(2));
/* 116 */ if (!isNull4) {
/* 117 */
/* 118 */
/* 119 */ isNull3 = false; // resultCode could change nullability.
/* 120 */
/* 121 */ final int index1 = (int) 2L;
/* 122 */ if (index1 >= value4.numElements() || index1 < 0 || value4.isNullAt(index1)) {
/* 123 */ isNull3 = true;
/* 124 */ } else {
/* 125 */ value3 = value4.getLong(index1);
/* 126 */ }
/* 127 */
/* 128 */
/* 129 */ }
/* 130 */ if (isNull3) {
/* 131 */ rowWriter.setNullAt(1);
/* 132 */ } else {
/* 133 */ rowWriter.write(1, value3);
/* 134 */ }
/* 135 */
/* 136 */ }
/* 137 */
/* 138 */
/* 139 */ private void apply_2(InternalRow i) {
/* 140 */
/* 141 */
/* 142 */ final long value12 = -1L;
/* 143 */ if (true) {
/* 144 */ rowWriter.setNullAt(4);
/* 145 */ } else {
/* 146 */ rowWriter.write(4, value12);
/* 147 */ }
/* 148 */
/* 149 */
/* 150 */ final UTF8String value13 = null;
/* 151 */ if (true) {
/* 152 */ rowWriter.setNullAt(5);
/* 153 */ } else {
/* 154 */ rowWriter.write(5, value13);
/* 155 */ }
/* 156 */
/* 157 */
/* 158 */ rowWriter.write(6, 1);
/* 159 */
/* 160 */ }
/* 161 */
/* 162 */
/* 163 */ // Scala.Function1 need this
/* 164 */ public java.lang.Object apply(java.lang.Object row) {
/* 165 */ return apply((InternalRow) row);
/* 166 */ }
/* 167 */
/* 168 */ public UnsafeRow apply(InternalRow i) {
/* 169 */ holder.reset();
/* 170 */
/* 171 */ rowWriter.zeroOutNullBytes();
/* 172 */ apply_0(i);
/* 173 */ apply_1(i);
/* 174 */ apply_2(i);
/* 175 */ result.setTotalSize(holder.totalSize());
/* 176 */ return result;
/* 177 */ }
/* 178 */ }

at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1010)
 at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1067)
 at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1064)
 at org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
 at org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
 ... 39 more

Driver stacktrace: (state=,code=0)



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)