Apache CarbonData Dev Mailing List archive › Apache CarbonData JIRA issues

[jira] [Updated] (CARBONDATA-4111) Filter query having invalid results after add segment to table having SI with Indexserver

classic Classic list List threaded Threaded
♦
♦
 1 message Options
Akash R Nilugal (Jira)
Reply | Threaded
Open this post in threaded view
♦
♦
|

[jira] [Updated] (CARBONDATA-4111) Filter query having invalid results after add segment to table having SI with Indexserver

Akash R Nilugal (Jira)

     [ https://issues.apache.org/jira/browse/CARBONDATA-4111?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

SHREELEKHYA GAMPA updated CARBONDATA-4111:
------------------------------------------
    Description:
queries to execute:

create table maintable_sdk(a string, b int, c string) stored as carbondata;
 insert into maintable_sdk select 'k',1,'k';
 insert into maintable_sdk select 'l',2,'l';
 CREATE INDEX maintable_si_sdk on table maintable_sdk (c) as 'carbondata';
 alter table maintable_sdk add segment options('path'='hdfs://hacluster/sdkfiles/newsegment/', 'format'='carbon');

spark-sql> select *from maintable_sdk where c='m';
2021-01-27 12:10:54,326 | WARN | IPC Client (653337757) connection to linux-30/10.19.90.30:22900 from [hidden email] | Unexpected error reading responses on connection Thread[IPC Client (653337757) connection to linux-30/10.19.90.30:22900 from [hidden email],5,main] | org.apache.hadoop.ipc.Client.run(Client.java:1113)
java.lang.RuntimeException: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
 at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:135)
 at org.apache.hadoop.io.WritableFactories.newInstance(WritableFactories.java:58)
 at org.apache.hadoop.io.ObjectWritable.readObject(ObjectWritable.java:284)
 at org.apache.hadoop.io.ObjectWritable.readFields(ObjectWritable.java:77)
 at org.apache.hadoop.ipc.RpcWritable$WritableWrapper.readFrom(RpcWritable.java:85)
 at org.apache.hadoop.ipc.RpcWritable$Buffer.getValue(RpcWritable.java:187)
 at org.apache.hadoop.ipc.RpcWritable$Buffer.newInstance(RpcWritable.java:183)
 at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1223)
 at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1107)
Caused by: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
 at java.lang.Class.getConstructor0(Class.java:3082)
 at java.lang.Class.getDeclaredConstructor(Class.java:2178)
 at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:129)
 ... 8 more
2021-01-27 12:10:54,330 | WARN | main | Distributed Segment Pruning failed, initiating embedded pruning | org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin$.getFilteredSegments(BroadCastSIFilterPushJoin.scala:349)
java.lang.reflect.UndeclaredThrowableException
 at com.sun.proxy.$Proxy59.getPrunedSegments(Unknown Source)
 at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin$.getFilteredSegments(BroadCastSIFilterPushJoin.scala:341)
 at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin$.getFilteredSegments(BroadCastSIFilterPushJoin.scala:426)
 at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.partitions$lzycompute(BroadCastSIFilterPushJoin.scala:80)
 at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.partitions(BroadCastSIFilterPushJoin.scala:78)
 at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.inputCopy$lzycompute(BroadCastSIFilterPushJoin.scala:94)
 at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.inputCopy(BroadCastSIFilterPushJoin.scala:93)
 at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.doExecute(BroadCastSIFilterPushJoin.scala:132)
 at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:177)
 at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:173)
 at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:201)
 at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
 at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:198)
 at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:173)
 at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:293)
 at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:342)
 at org.apache.spark.sql.execution.SparkPlan.executeCollectPublic(SparkPlan.scala:372)
 at org.apache.spark.sql.execution.QueryExecution.hiveResultString(QueryExecution.scala:127)
 at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver$$anonfun$run$1.apply(SparkSQLDriver.scala:66)
 at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver$$anonfun$run$1.apply(SparkSQLDriver.scala:66)
 at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1$$anonfun$apply$1.apply(SQLExecution.scala:95)
 at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:144)
 at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:86)
 at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:789)
 at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:63)
 at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:65)
 at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:383)
 at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:406)
 at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:277)
 at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
 at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:882)
 at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:164)
 at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:187)
 at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:89)
 at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:957)
 at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:966)
 at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.io.IOException: DestHost:destPort linux-30:22900 , LocalHost:localPort linux-28/10.19.90.28:0. Failed on local exception: java.io.IOException: Error reading responses
 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
 at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
 at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
 at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
 at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:1014)
 at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:989)
 at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1567)
 at org.apache.hadoop.ipc.Client.call(Client.java:1509)
 at org.apache.hadoop.ipc.Client.call(Client.java:1408)
 at org.apache.hadoop.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:251)
 ... 42 more
Caused by: java.io.IOException: Error reading responses
 at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1114)
Caused by: java.lang.RuntimeException: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
 at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:135)
 at org.apache.hadoop.io.WritableFactories.newInstance(WritableFactories.java:58)
 at org.apache.hadoop.io.ObjectWritable.readObject(ObjectWritable.java:284)
 at org.apache.hadoop.io.ObjectWritable.readFields(ObjectWritable.java:77)
 at org.apache.hadoop.ipc.RpcWritable$WritableWrapper.readFrom(RpcWritable.java:85)
 at org.apache.hadoop.ipc.RpcWritable$Buffer.getValue(RpcWritable.java:187)
 at org.apache.hadoop.ipc.RpcWritable$Buffer.newInstance(RpcWritable.java:183)
 at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1223)
 at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1107)
Caused by: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
 at java.lang.Class.getConstructor0(Class.java:3082)
 at java.lang.Class.getDeclaredConstructor(Class.java:2178)
 at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:129)
 ... 8 more
Time taken: 2.813 seconds
spark-sql> select *from maintable_sdk;
k 1 k
l 2 l
m 3 m
Time taken: 0.569 seconds, Fetched 3 row(s)

  was:
queries to execute:

create table maintable_sdk(a string, b int, c string) stored as carbondata;
 insert into maintable_sdk select 'k',1,'k';
 insert into maintable_sdk select 'l',2,'l';
 CREATE INDEX maintable_si_sdk on table maintable_sdk (c) as 'carbondata';
 alter table maintable_sdk add segment options('path'='hdfs://hacluster/sdkfiles/newsegment/', 'format'='carbon');




> Filter query having invalid results after add segment to table having SI with Indexserver
> -----------------------------------------------------------------------------------------
>
>                 Key: CARBONDATA-4111
>                 URL: https://issues.apache.org/jira/browse/CARBONDATA-4111
>             Project: CarbonData
>          Issue Type: Bug
>            Reporter: SHREELEKHYA GAMPA
>            Priority: Minor
>         Attachments: addseg_si_is.png
>
>
> queries to execute:
> create table maintable_sdk(a string, b int, c string) stored as carbondata;
>  insert into maintable_sdk select 'k',1,'k';
>  insert into maintable_sdk select 'l',2,'l';
>  CREATE INDEX maintable_si_sdk on table maintable_sdk (c) as 'carbondata';
>  alter table maintable_sdk add segment options('path'='hdfs://hacluster/sdkfiles/newsegment/', 'format'='carbon');
> spark-sql> select *from maintable_sdk where c='m';
> 2021-01-27 12:10:54,326 | WARN | IPC Client (653337757) connection to linux-30/10.19.90.30:22900 from [hidden email] | Unexpected error reading responses on connection Thread[IPC Client (653337757) connection to linux-30/10.19.90.30:22900 from [hidden email],5,main] | org.apache.hadoop.ipc.Client.run(Client.java:1113)
> java.lang.RuntimeException: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
>  at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:135)
>  at org.apache.hadoop.io.WritableFactories.newInstance(WritableFactories.java:58)
>  at org.apache.hadoop.io.ObjectWritable.readObject(ObjectWritable.java:284)
>  at org.apache.hadoop.io.ObjectWritable.readFields(ObjectWritable.java:77)
>  at org.apache.hadoop.ipc.RpcWritable$WritableWrapper.readFrom(RpcWritable.java:85)
>  at org.apache.hadoop.ipc.RpcWritable$Buffer.getValue(RpcWritable.java:187)
>  at org.apache.hadoop.ipc.RpcWritable$Buffer.newInstance(RpcWritable.java:183)
>  at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1223)
>  at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1107)
> Caused by: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
>  at java.lang.Class.getConstructor0(Class.java:3082)
>  at java.lang.Class.getDeclaredConstructor(Class.java:2178)
>  at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:129)
>  ... 8 more
> 2021-01-27 12:10:54,330 | WARN | main | Distributed Segment Pruning failed, initiating embedded pruning | org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin$.getFilteredSegments(BroadCastSIFilterPushJoin.scala:349)
> java.lang.reflect.UndeclaredThrowableException
>  at com.sun.proxy.$Proxy59.getPrunedSegments(Unknown Source)
>  at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin$.getFilteredSegments(BroadCastSIFilterPushJoin.scala:341)
>  at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin$.getFilteredSegments(BroadCastSIFilterPushJoin.scala:426)
>  at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.partitions$lzycompute(BroadCastSIFilterPushJoin.scala:80)
>  at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.partitions(BroadCastSIFilterPushJoin.scala:78)
>  at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.inputCopy$lzycompute(BroadCastSIFilterPushJoin.scala:94)
>  at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.inputCopy(BroadCastSIFilterPushJoin.scala:93)
>  at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.doExecute(BroadCastSIFilterPushJoin.scala:132)
>  at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:177)
>  at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:173)
>  at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:201)
>  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
>  at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:198)
>  at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:173)
>  at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:293)
>  at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:342)
>  at org.apache.spark.sql.execution.SparkPlan.executeCollectPublic(SparkPlan.scala:372)
>  at org.apache.spark.sql.execution.QueryExecution.hiveResultString(QueryExecution.scala:127)
>  at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver$$anonfun$run$1.apply(SparkSQLDriver.scala:66)
>  at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver$$anonfun$run$1.apply(SparkSQLDriver.scala:66)
>  at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1$$anonfun$apply$1.apply(SQLExecution.scala:95)
>  at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:144)
>  at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:86)
>  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:789)
>  at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:63)
>  at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:65)
>  at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:383)
>  at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:406)
>  at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:277)
>  at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala)
>  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>  at java.lang.reflect.Method.invoke(Method.java:498)
>  at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
>  at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:882)
>  at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:164)
>  at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:187)
>  at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:89)
>  at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:957)
>  at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:966)
>  at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> Caused by: java.io.IOException: DestHost:destPort linux-30:22900 , LocalHost:localPort linux-28/10.19.90.28:0. Failed on local exception: java.io.IOException: Error reading responses
>  at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
>  at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
>  at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
>  at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:1014)
>  at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:989)
>  at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1567)
>  at org.apache.hadoop.ipc.Client.call(Client.java:1509)
>  at org.apache.hadoop.ipc.Client.call(Client.java:1408)
>  at org.apache.hadoop.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:251)
>  ... 42 more
> Caused by: java.io.IOException: Error reading responses
>  at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1114)
> Caused by: java.lang.RuntimeException: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
>  at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:135)
>  at org.apache.hadoop.io.WritableFactories.newInstance(WritableFactories.java:58)
>  at org.apache.hadoop.io.ObjectWritable.readObject(ObjectWritable.java:284)
>  at org.apache.hadoop.io.ObjectWritable.readFields(ObjectWritable.java:77)
>  at org.apache.hadoop.ipc.RpcWritable$WritableWrapper.readFrom(RpcWritable.java:85)
>  at org.apache.hadoop.ipc.RpcWritable$Buffer.getValue(RpcWritable.java:187)
>  at org.apache.hadoop.ipc.RpcWritable$Buffer.newInstance(RpcWritable.java:183)
>  at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1223)
>  at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1107)
> Caused by: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
>  at java.lang.Class.getConstructor0(Class.java:3082)
>  at java.lang.Class.getDeclaredConstructor(Class.java:2178)
>  at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:129)
>  ... 8 more
> Time taken: 2.813 seconds
> spark-sql> select *from maintable_sdk;
> k 1 k
> l 2 l
> m 3 m
> Time taken: 0.569 seconds, Fetched 3 row(s)



--
This message was sent by Atlassian Jira
(v8.3.4#803005)
Free forum by Nabble Edit this page