ClassNotFound error when insert carbontable from hive table
Locked
ClassNotFound error when insert carbontable from hive table
 
|
Hi dev,
I'm trying to insert data from a hive table to carbon table: cc.sql("insert into carbon_test select * from target_table where pt = '20170101'") Any one knows how to fix this error? [Stage 8:> (0 + 4) / 156]17/08/21 15:59:01 WARN scheduler.TaskSetManager: Lost task 1.0 in stage 8.0 (TID 48, , executor 16): java.lang.ClassNotFoundException: org.apache.carbondata.spark.rdd.CarbonBlockDistinctValuesCombineRDD at org.apache.spark.repl.ExecutorClassLoader.findClass(ExecutorClassLoader.scala:82) at java.lang.ClassLoader.loadClass(ClassLoader.java:425) at java.lang.ClassLoader.loadClass(ClassLoader.java:358) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:270) at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:67) at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1612) at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1517) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1771) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370) at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:75) at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:114) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:85) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) at org.apache.spark.scheduler.Task.run(Task.scala:99) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.ClassNotFoundException: org.apache.carbondata.spark.rdd.CarbonBlockDistinctValuesCombineRDD at java.lang.ClassLoader.findClass(ClassLoader.java:531) at org.apache.spark.util.ParentClassLoader.findClass(ParentClassLoader.scala:26) at java.lang.ClassLoader.loadClass(ClassLoader.java:425) at org.apache.spark.util.ParentClassLoader.loadClass(ParentClassLoader.scala:34) at java.lang.ClassLoader.loadClass(ClassLoader.java:358) at org.apache.spark.util.ParentClassLoader.loadClass(ParentClassLoader.scala:30) at org.apache.spark.repl.ExecutorClassLoader.findClass(ExecutorClassLoader.scala:77) ... 23 more [Stage 8:> (0 + 4) / 156]17/08/21 15:59:02 ERROR scheduler.TaskSetManager: Task 1 in stage 8.0 failed 4 times; aborting job 17/08/21 15:59:02 ERROR util.GlobalDictionaryUtil$: main generate global dictionary failed org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 8.0 failed 4 times, most recent failure: Lost task 1.3 in stage 8.0 (TID 61, scsp00382.saicdt.com, executor 16): java.lang.ClassNotFoundException: org.apache.carbondata.spark.rdd.CarbonBlockDistinctValuesCombineRDD at org.apache.spark.repl.ExecutorClassLoader.findClass(ExecutorClassLoader.scala:82) at java.lang.ClassLoader.loadClass(ClassLoader.java:425) at java.lang.ClassLoader.loadClass(ClassLoader.java:358) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:270) at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:67) at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1612) at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1517) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1771) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370) at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:75) at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:114) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:85) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) at org.apache.spark.scheduler.Task.run(Task.scala:99) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.ClassNotFoundException: org.apache.carbondata.spark.rdd.CarbonBlockDistinctValuesCombineRDD at java.lang.ClassLoader.findClass(ClassLoader.java:531) at org.apache.spark.util.ParentClassLoader.findClass(ParentClassLoader.scala:26) at java.lang.ClassLoader.loadClass(ClassLoader.java:425) at org.apache.spark.util.ParentClassLoader.loadClass(ParentClassLoader.scala:34) at java.lang.ClassLoader.loadClass(ClassLoader.java:358) at org.apache.spark.util.ParentClassLoader.loadClass(ParentClassLoader.scala:30) at org.apache.spark.repl.ExecutorClassLoader.findClass(ExecutorClassLoader.scala:77) ... 23 more Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.org $apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802) at scala.Option.foreach(Option.scala:257) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1944) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958) at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:935) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) at org.apache.spark.rdd.RDD.collect(RDD.scala:934) at org.apache.carbondata.spark.util.GlobalDictionaryUtil$.generateGlobalDictionary(GlobalDictionaryUtil.scala:741) at org.apache.spark.sql.execution.command.LoadTable.run(carbonTableSchema.scala:703) at org.apache.spark.sql.execution.command.LoadTableByInsert.run(carbonTableSchema.scala:338) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56) at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:74) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:135) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:132) at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113) at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:87) at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:87) at org.apache.spark.sql.Dataset.<init>(Dataset.scala:185) at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64) at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592) at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:33) at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:38) at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:40) at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:42) at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:44) at $line30.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:46) at $line30.$read$$iw$$iw$$iw$$iw.<init>(<console>:48) at $line30.$read$$iw$$iw$$iw.<init>(<console>:50) at $line30.$read$$iw$$iw.<init>(<console>:52) at $line30.$read$$iw.<init>(<console>:54) at $line30.$read.<init>(<console>:56) at $line30.$read$.<init>(<console>:60) at $line30.$read$.<clinit>(<console>) at $line30.$eval$.$print$lzycompute(<console>:7) at $line30.$eval$.$print(<console>:6) at $line30.$eval.$print(<console>) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786) at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047) at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638) at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637) at scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31) at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19) at scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637) at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807) at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681) at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395) at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415) at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923) at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97) at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909) at org.apache.spark.repl.Main$.doMain(Main.scala:69) at org.apache.spark.repl.Main$.main(Main.scala:52) at org.apache.spark.repl.Main.main(Main.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:738) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by: java.lang.ClassNotFoundException: org.apache.carbondata.spark.rdd.CarbonBlockDistinctValuesCombineRDD at org.apache.spark.repl.ExecutorClassLoader.findClass(ExecutorClassLoader.scala:82) at java.lang.ClassLoader.loadClass(ClassLoader.java:425) at java.lang.ClassLoader.loadClass(ClassLoader.java:358) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:270) at org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:67) at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1612) at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1517) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1771) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1990) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1798) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370) at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:75) at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:114) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:85) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) at org.apache.spark.scheduler.Task.run(Task.scala:99) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.ClassNotFoundException: org.apache.carbondata.spark.rdd.CarbonBlockDistinctValuesCombineRDD at java.lang.ClassLoader.findClass(ClassLoader.java:531) at org.apache.spark.util.ParentClassLoader.findClass(ParentClassLoader.scala:26) at java.lang.ClassLoader.loadClass(ClassLoader.java:425) at org.apache.spark.util.ParentClassLoader.loadClass(ParentClassLoader.scala:34) at java.lang.ClassLoader.loadClass(ClassLoader.java:358) at org.apache.spark.util.ParentClassLoader.loadClass(ParentClassLoader.scala:30) at org.apache.spark.repl.ExecutorClassLoader.findClass(ExecutorClassLoader.scala:77) ... 23 more 17/08/21 15:59:02 ERROR command.LoadTable: main java.lang.Exception: generate global dictionary failed, null at org.apache.carbondata.spark.util.GlobalDictionaryUtil$.generateGlobalDictionary(GlobalDictionaryUtil.scala:761) at org.apache.spark.sql.execution.command.LoadTable.run(carbonTableSchema.scala:703) at org.apache.spark.sql.execution.command.LoadTableByInsert.run(carbonTableSchema.scala:338) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56) at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:74) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:135) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:132) at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113) at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:87) at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:87) at org.apache.spark.sql.Dataset.<init>(Dataset.scala:185) at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64) at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592) at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:33) at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:38) at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:40) at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:42) at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:44) at $line30.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:46) at $line30.$read$$iw$$iw$$iw$$iw.<init>(<console>:48) at $line30.$read$$iw$$iw$$iw.<init>(<console>:50) at $line30.$read$$iw$$iw.<init>(<console>:52) at $line30.$read$$iw.<init>(<console>:54) at $line30.$read.<init>(<console>:56) at $line30.$read$.<init>(<console>:60) at $line30.$read$.<clinit>(<console>) at $line30.$eval$.$print$lzycompute(<console>:7) at $line30.$eval$.$print(<console>:6) at $line30.$eval.$print(<console>) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786) at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047) at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638) at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637) at scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31) at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19) at scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637) at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807) at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681) at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395) at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415) at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923) at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97) at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909) at org.apache.spark.repl.Main$.doMain(Main.scala:69) at org.apache.spark.repl.Main$.main(Main.scala:52) at org.apache.spark.repl.Main.main(Main.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:738) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 17/08/21 15:59:02 AUDIT command.LoadTable: [scsp00267.saicdt.com][carbon2][Thread-1]Dataload failure for default.crx5_tbox_normal_test. Please check the logs 17/08/21 15:59:02 WARN spark.ExecutorAllocationManager: No stages are running, but numRunningTasks != 0 java.lang.Exception: generate global dictionary failed, null at org.apache.carbondata.spark.util.GlobalDictionaryUtil$.generateGlobalDictionary(GlobalDictionaryUtil.scala:761) at org.apache.spark.sql.execution.command.LoadTable.run(carbonTableSchema.scala:703) at org.apache.spark.sql.execution.command.LoadTableByInsert.run(carbonTableSchema.scala:338) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58) at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56) at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:74) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:135) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:132) at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113) at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:87) at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:87) at org.apache.spark.sql.Dataset.<init>(Dataset.scala:185) at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64) at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592) ... 50 elided Thanks, CaoLu |
Re: ClassNotFound error when insert carbontable from hive table
|
|
This issue had been fixed.
On Mon, Aug 21, 2017 at 4:04 PM, Lu Cao <[hidden email]> wrote: > Hi dev, > > I'm trying to insert data from a hive table to carbon table: > > cc.sql("insert into carbon_test select * from target_table where pt = > '20170101'") > > > Any one knows how to fix this error? > > [Stage 8:> (0 + 4) > / 156]17/08/21 15:59:01 WARN scheduler.TaskSetManager: Lost task 1.0 in > stage 8.0 (TID 48, , executor 16): java.lang.ClassNotFoundException: > org.apache.carbondata.spark.rdd.CarbonBlockDistinctValuesCombineRDD > > at org.apache.spark.repl.ExecutorClassLoader.findClass( > ExecutorClassLoader.scala:82) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > > at java.lang.Class.forName0(Native Method) > > at java.lang.Class.forName(Class.java:270) > > at org.apache.spark.serializer.JavaDeserializationStream$$ > anon$1.resolveClass(JavaSerializer.scala:67) > > at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1612) > > at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1517) > > at java.io.ObjectInputStream.readOrdinaryObject( > ObjectInputStream.java:1771) > > at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > > at java.io.ObjectInputStream.defaultReadFields( > ObjectInputStream.java:1990) > > at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915) > > at java.io.ObjectInputStream.readOrdinaryObject( > ObjectInputStream.java:1798) > > at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > > at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370) > > at org.apache.spark.serializer.JavaDeserializationStream. > readObject(JavaSerializer.scala:75) > > at org.apache.spark.serializer.JavaSerializerInstance. > deserialize(JavaSerializer.scala:114) > > at org.apache.spark.scheduler.ShuffleMapTask.runTask( > ShuffleMapTask.scala:85) > > at org.apache.spark.scheduler.ShuffleMapTask.runTask( > ShuffleMapTask.scala:53) > > at org.apache.spark.scheduler.Task.run(Task.scala:99) > > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) > > at java.util.concurrent.ThreadPoolExecutor.runWorker( > ThreadPoolExecutor.java:1145) > > at java.util.concurrent.ThreadPoolExecutor$Worker.run( > ThreadPoolExecutor.java:615) > > at java.lang.Thread.run(Thread.java:745) > > Caused by: java.lang.ClassNotFoundException: org.apache.carbondata.spark. > rdd.CarbonBlockDistinctValuesCombineRDD > > at java.lang.ClassLoader.findClass(ClassLoader.java:531) > > at org.apache.spark.util.ParentClassLoader.findClass( > ParentClassLoader.scala:26) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > > at org.apache.spark.util.ParentClassLoader.loadClass( > ParentClassLoader.scala:34) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > > at org.apache.spark.util.ParentClassLoader.loadClass( > ParentClassLoader.scala:30) > > at org.apache.spark.repl.ExecutorClassLoader.findClass( > ExecutorClassLoader.scala:77) > > ... 23 more > > > [Stage 8:> (0 + 4) > / 156]17/08/21 15:59:02 ERROR scheduler.TaskSetManager: Task 1 in stage 8.0 > failed 4 times; aborting job > > 17/08/21 15:59:02 ERROR util.GlobalDictionaryUtil$: main generate global > dictionary failed > > org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 > in stage 8.0 failed 4 times, most recent failure: Lost task 1.3 in stage > 8.0 (TID 61, scsp00382.saicdt.com, executor 16): java.lang.ClassNotFoundException: > org.apache.carbondata.spark.rdd.CarbonBlockDistinctValuesCombineRDD > > at org.apache.spark.repl.ExecutorClassLoader.findClass( > ExecutorClassLoader.scala:82) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > > at java.lang.Class.forName0(Native Method) > > at java.lang.Class.forName(Class.java:270) > > at org.apache.spark.serializer.JavaDeserializationStream$$ > anon$1.resolveClass(JavaSerializer.scala:67) > > at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1612) > > at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1517) > > at java.io.ObjectInputStream.readOrdinaryObject( > ObjectInputStream.java:1771) > > at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > > at java.io.ObjectInputStream.defaultReadFields( > ObjectInputStream.java:1990) > > at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915) > > at java.io.ObjectInputStream.readOrdinaryObject( > ObjectInputStream.java:1798) > > at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > > at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370) > > at org.apache.spark.serializer.JavaDeserializationStream. > readObject(JavaSerializer.scala:75) > > at org.apache.spark.serializer.JavaSerializerInstance. > deserialize(JavaSerializer.scala:114) > > at org.apache.spark.scheduler.ShuffleMapTask.runTask( > ShuffleMapTask.scala:85) > > at org.apache.spark.scheduler.ShuffleMapTask.runTask( > ShuffleMapTask.scala:53) > > at org.apache.spark.scheduler.Task.run(Task.scala:99) > > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) > > at java.util.concurrent.ThreadPoolExecutor.runWorker( > ThreadPoolExecutor.java:1145) > > at java.util.concurrent.ThreadPoolExecutor$Worker.run( > ThreadPoolExecutor.java:615) > > at java.lang.Thread.run(Thread.java:745) > > Caused by: java.lang.ClassNotFoundException: org.apache.carbondata.spark. > rdd.CarbonBlockDistinctValuesCombineRDD > > at java.lang.ClassLoader.findClass(ClassLoader.java:531) > > at org.apache.spark.util.ParentClassLoader.findClass( > ParentClassLoader.scala:26) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > > at org.apache.spark.util.ParentClassLoader.loadClass( > ParentClassLoader.scala:34) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > > at org.apache.spark.util.ParentClassLoader.loadClass( > ParentClassLoader.scala:30) > > at org.apache.spark.repl.ExecutorClassLoader.findClass( > ExecutorClassLoader.scala:77) > > ... 23 more > > > Driver stacktrace: > > at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$ > scheduler$DAGScheduler$$failJobAndIndependentStages( > DAGScheduler.scala:1435) > > at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply( > DAGScheduler.scala:1423) > > at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply( > DAGScheduler.scala:1422) > > at scala.collection.mutable.ResizableArray$class.foreach( > ResizableArray.scala:59) > > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) > > at org.apache.spark.scheduler.DAGScheduler.abortStage( > DAGScheduler.scala:1422) > > at org.apache.spark.scheduler.DAGScheduler$$anonfun$ > handleTaskSetFailed$1.apply(DAGScheduler.scala:802) > > at org.apache.spark.scheduler.DAGScheduler$$anonfun$ > handleTaskSetFailed$1.apply(DAGScheduler.scala:802) > > at scala.Option.foreach(Option.scala:257) > > at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed( > DAGScheduler.scala:802) > > at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop. > doOnReceive(DAGScheduler.scala:1650) > > at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop. > onReceive(DAGScheduler.scala:1605) > > at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop. > onReceive(DAGScheduler.scala:1594) > > at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) > > at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628) > > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918) > > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931) > > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1944) > > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958) > > at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:935) > > at org.apache.spark.rdd.RDDOperationScope$.withScope( > RDDOperationScope.scala:151) > > at org.apache.spark.rdd.RDDOperationScope$.withScope( > RDDOperationScope.scala:112) > > at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) > > at org.apache.spark.rdd.RDD.collect(RDD.scala:934) > > at org.apache.carbondata.spark.util.GlobalDictionaryUtil$. > generateGlobalDictionary(GlobalDictionaryUtil.scala:741) > > at org.apache.spark.sql.execution.command.LoadTable. > run(carbonTableSchema.scala:703) > > at org.apache.spark.sql.execution.command.LoadTableByInsert.run( > carbonTableSchema.scala:338) > > at org.apache.spark.sql.execution.command.ExecutedCommandExec. > sideEffectResult$lzycompute(commands.scala:58) > > at org.apache.spark.sql.execution.command.ExecutedCommandExec. > sideEffectResult(commands.scala:56) > > at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute( > commands.scala:74) > > at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > execute$1.apply(SparkPlan.scala:114) > > at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > execute$1.apply(SparkPlan.scala:114) > > at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply( > SparkPlan.scala:135) > > at org.apache.spark.rdd.RDDOperationScope$.withScope( > RDDOperationScope.scala:151) > > at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala: > 132) > > at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113) > > at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute( > QueryExecution.scala:87) > > at org.apache.spark.sql.execution.QueryExecution. > toRdd(QueryExecution.scala:87) > > at org.apache.spark.sql.Dataset.<init>(Dataset.scala:185) > > at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64) > > at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init> > (<console>:33) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:38) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:40) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:42) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:44) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:46) > > at $line30.$read$$iw$$iw$$iw$$iw.<init>(<console>:48) > > at $line30.$read$$iw$$iw$$iw.<init>(<console>:50) > > at $line30.$read$$iw$$iw.<init>(<console>:52) > > at $line30.$read$$iw.<init>(<console>:54) > > at $line30.$read.<init>(<console>:56) > > at $line30.$read$.<init>(<console>:60) > > at $line30.$read$.<clinit>(<console>) > > at $line30.$eval$.$print$lzycompute(<console>:7) > > at $line30.$eval$.$print(<console>:6) > > at $line30.$eval.$print(<console>) > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > > at sun.reflect.NativeMethodAccessorImpl.invoke( > NativeMethodAccessorImpl.java:57) > > at sun.reflect.DelegatingMethodAccessorImpl.invoke( > DelegatingMethodAccessorImpl.java:43) > > at java.lang.reflect.Method.invoke(Method.java:606) > > at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786) > > at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047) > > at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$ > loadAndRunReq$1.apply(IMain.scala:638) > > at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$ > loadAndRunReq$1.apply(IMain.scala:637) > > at scala.reflect.internal.util.ScalaClassLoader$class. > asContext(ScalaClassLoader.scala:31) > > at scala.reflect.internal.util.AbstractFileClassLoader.asContext( > AbstractFileClassLoader.scala:19) > > at scala.tools.nsc.interpreter.IMain$WrappedRequest. > loadAndRunReq(IMain.scala:637) > > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) > > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) > > at scala.tools.nsc.interpreter.ILoop.interpretStartingWith( > ILoop.scala:807) > > at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681) > > at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395) > > at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415) > > at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > apply$mcZ$sp(ILoop.scala:923) > > at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > apply(ILoop.scala:909) > > at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > apply(ILoop.scala:909) > > at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader( > ScalaClassLoader.scala:97) > > at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909) > > at org.apache.spark.repl.Main$.doMain(Main.scala:69) > > at org.apache.spark.repl.Main$.main(Main.scala:52) > > at org.apache.spark.repl.Main.main(Main.scala) > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > > at sun.reflect.NativeMethodAccessorImpl.invoke( > NativeMethodAccessorImpl.java:57) > > at sun.reflect.DelegatingMethodAccessorImpl.invoke( > DelegatingMethodAccessorImpl.java:43) > > at java.lang.reflect.Method.invoke(Method.java:606) > > at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$ > deploy$SparkSubmit$$runMain(SparkSubmit.scala:738) > > at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187) > > at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212) > > at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126) > > at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) > > Caused by: java.lang.ClassNotFoundException: org.apache.carbondata.spark. > rdd.CarbonBlockDistinctValuesCombineRDD > > at org.apache.spark.repl.ExecutorClassLoader.findClass( > ExecutorClassLoader.scala:82) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > > at java.lang.Class.forName0(Native Method) > > at java.lang.Class.forName(Class.java:270) > > at org.apache.spark.serializer.JavaDeserializationStream$$ > anon$1.resolveClass(JavaSerializer.scala:67) > > at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1612) > > at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1517) > > at java.io.ObjectInputStream.readOrdinaryObject( > ObjectInputStream.java:1771) > > at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > > at java.io.ObjectInputStream.defaultReadFields( > ObjectInputStream.java:1990) > > at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1915) > > at java.io.ObjectInputStream.readOrdinaryObject( > ObjectInputStream.java:1798) > > at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > > at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370) > > at org.apache.spark.serializer.JavaDeserializationStream. > readObject(JavaSerializer.scala:75) > > at org.apache.spark.serializer.JavaSerializerInstance. > deserialize(JavaSerializer.scala:114) > > at org.apache.spark.scheduler.ShuffleMapTask.runTask( > ShuffleMapTask.scala:85) > > at org.apache.spark.scheduler.ShuffleMapTask.runTask( > ShuffleMapTask.scala:53) > > at org.apache.spark.scheduler.Task.run(Task.scala:99) > > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) > > at java.util.concurrent.ThreadPoolExecutor.runWorker( > ThreadPoolExecutor.java:1145) > > at java.util.concurrent.ThreadPoolExecutor$Worker.run( > ThreadPoolExecutor.java:615) > > at java.lang.Thread.run(Thread.java:745) > > Caused by: java.lang.ClassNotFoundException: org.apache.carbondata.spark. > rdd.CarbonBlockDistinctValuesCombineRDD > > at java.lang.ClassLoader.findClass(ClassLoader.java:531) > > at org.apache.spark.util.ParentClassLoader.findClass( > ParentClassLoader.scala:26) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > > at org.apache.spark.util.ParentClassLoader.loadClass( > ParentClassLoader.scala:34) > > at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > > at org.apache.spark.util.ParentClassLoader.loadClass( > ParentClassLoader.scala:30) > > at org.apache.spark.repl.ExecutorClassLoader.findClass( > ExecutorClassLoader.scala:77) > > ... 23 more > > 17/08/21 15:59:02 ERROR command.LoadTable: main > > java.lang.Exception: generate global dictionary failed, null > > at org.apache.carbondata.spark.util.GlobalDictionaryUtil$. > generateGlobalDictionary(GlobalDictionaryUtil.scala:761) > > at org.apache.spark.sql.execution.command.LoadTable. > run(carbonTableSchema.scala:703) > > at org.apache.spark.sql.execution.command.LoadTableByInsert.run( > carbonTableSchema.scala:338) > > at org.apache.spark.sql.execution.command.ExecutedCommandExec. > sideEffectResult$lzycompute(commands.scala:58) > > at org.apache.spark.sql.execution.command.ExecutedCommandExec. > sideEffectResult(commands.scala:56) > > at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute( > commands.scala:74) > > at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > execute$1.apply(SparkPlan.scala:114) > > at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > execute$1.apply(SparkPlan.scala:114) > > at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply( > SparkPlan.scala:135) > > at org.apache.spark.rdd.RDDOperationScope$.withScope( > RDDOperationScope.scala:151) > > at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala: > 132) > > at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113) > > at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute( > QueryExecution.scala:87) > > at org.apache.spark.sql.execution.QueryExecution. > toRdd(QueryExecution.scala:87) > > at org.apache.spark.sql.Dataset.<init>(Dataset.scala:185) > > at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64) > > at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init> > (<console>:33) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:38) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:40) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:42) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:44) > > at $line30.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:46) > > at $line30.$read$$iw$$iw$$iw$$iw.<init>(<console>:48) > > at $line30.$read$$iw$$iw$$iw.<init>(<console>:50) > > at $line30.$read$$iw$$iw.<init>(<console>:52) > > at $line30.$read$$iw.<init>(<console>:54) > > at $line30.$read.<init>(<console>:56) > > at $line30.$read$.<init>(<console>:60) > > at $line30.$read$.<clinit>(<console>) > > at $line30.$eval$.$print$lzycompute(<console>:7) > > at $line30.$eval$.$print(<console>:6) > > at $line30.$eval.$print(<console>) > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > > at sun.reflect.NativeMethodAccessorImpl.invoke( > NativeMethodAccessorImpl.java:57) > > at sun.reflect.DelegatingMethodAccessorImpl.invoke( > DelegatingMethodAccessorImpl.java:43) > > at java.lang.reflect.Method.invoke(Method.java:606) > > at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786) > > at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047) > > at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$ > loadAndRunReq$1.apply(IMain.scala:638) > > at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$ > loadAndRunReq$1.apply(IMain.scala:637) > > at scala.reflect.internal.util.ScalaClassLoader$class. > asContext(ScalaClassLoader.scala:31) > > at scala.reflect.internal.util.AbstractFileClassLoader.asContext( > AbstractFileClassLoader.scala:19) > > at scala.tools.nsc.interpreter.IMain$WrappedRequest. > loadAndRunReq(IMain.scala:637) > > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) > > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) > > at scala.tools.nsc.interpreter.ILoop.interpretStartingWith( > ILoop.scala:807) > > at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681) > > at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395) > > at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415) > > at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > apply$mcZ$sp(ILoop.scala:923) > > at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > apply(ILoop.scala:909) > > at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > apply(ILoop.scala:909) > > at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader( > ScalaClassLoader.scala:97) > > at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909) > > at org.apache.spark.repl.Main$.doMain(Main.scala:69) > > at org.apache.spark.repl.Main$.main(Main.scala:52) > > at org.apache.spark.repl.Main.main(Main.scala) > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > > at sun.reflect.NativeMethodAccessorImpl.invoke( > NativeMethodAccessorImpl.java:57) > > at sun.reflect.DelegatingMethodAccessorImpl.invoke( > DelegatingMethodAccessorImpl.java:43) > > at java.lang.reflect.Method.invoke(Method.java:606) > > at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$ > deploy$SparkSubmit$$runMain(SparkSubmit.scala:738) > > at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187) > > at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212) > > at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126) > > at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) > > 17/08/21 15:59:02 AUDIT command.LoadTable: [scsp00267.saicdt.com][carbon2][Thread-1]Dataload > failure for default.crx5_tbox_normal_test. Please check the logs > > 17/08/21 15:59:02 WARN spark.ExecutorAllocationManager: No stages are > running, but numRunningTasks != 0 > > java.lang.Exception: generate global dictionary failed, null > > at org.apache.carbondata.spark.util.GlobalDictionaryUtil$. > generateGlobalDictionary(GlobalDictionaryUtil.scala:761) > > at org.apache.spark.sql.execution.command.LoadTable. > run(carbonTableSchema.scala:703) > > at org.apache.spark.sql.execution.command.LoadTableByInsert.run( > carbonTableSchema.scala:338) > > at org.apache.spark.sql.execution.command.ExecutedCommandExec. > sideEffectResult$lzycompute(commands.scala:58) > > at org.apache.spark.sql.execution.command.ExecutedCommandExec. > sideEffectResult(commands.scala:56) > > at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute( > commands.scala:74) > > at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > execute$1.apply(SparkPlan.scala:114) > > at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > execute$1.apply(SparkPlan.scala:114) > > at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > executeQuery$1.apply(SparkPlan.scala:135) > > at org.apache.spark.rdd.RDDOperationScope$.withScope( > RDDOperationScope.scala:151) > > at org.apache.spark.sql.execution.SparkPlan. > executeQuery(SparkPlan.scala:132) > > at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113) > > at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute( > QueryExecution.scala:87) > > at org.apache.spark.sql.execution.QueryExecution. > toRdd(QueryExecution.scala:87) > > at org.apache.spark.sql.Dataset.<init>(Dataset.scala:185) > > at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64) > > at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592) > > ... 50 elided > > > > Thanks, > > CaoLu > |
Re: ClassNotFound error when insert carbontable from hive table
|
Administrator
|
Hi lionel
Can you share with us how did you fix this issue? Regards Liang
|
Re: ClassNotFound error when insert carbontable from hive table
|
|
Just a configuration error.
The carbondata jar was in wrong place with which configured in spark-defaults.conf, after corrected the path it was fixed. thanks, Lionel On Tue, Aug 22, 2017 at 3:17 PM, Liang Chen <[hidden email]> wrote: > Hi lionel > > Can you share with us how did you fix this issue? > > Regards > Liang > > > lionel061201 wrote > > This issue had been fixed. > > > > On Mon, Aug 21, 2017 at 4:04 PM, Lu Cao < > > > whucaolu@ > > > > wrote: > > > >> Hi dev, > >> > >> I'm trying to insert data from a hive table to carbon table: > >> > >> cc.sql("insert into carbon_test select * from target_table where pt = > >> '20170101'") > >> > >> > >> Any one knows how to fix this error? > >> > >> [Stage 8:> (0 + > 4) > >> / 156]17/08/21 15:59:01 WARN scheduler.TaskSetManager: Lost task 1.0 in > >> stage 8.0 (TID 48, , executor 16): java.lang.ClassNotFoundException: > >> org.apache.carbondata.spark.rdd.CarbonBlockDistinctValuesCombineRDD > >> > >> at org.apache.spark.repl.ExecutorClassLoader.findClass( > >> ExecutorClassLoader.scala:82) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > >> > >> at java.lang.Class.forName0(Native Method) > >> > >> at java.lang.Class.forName(Class.java:270) > >> > >> at org.apache.spark.serializer.JavaDeserializationStream$$ > >> anon$1.resolveClass(JavaSerializer.scala:67) > >> > >> at > >> java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1612) > >> > >> at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1517) > >> > >> at java.io.ObjectInputStream.readOrdinaryObject( > >> ObjectInputStream.java:1771) > >> > >> at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > >> > >> at java.io.ObjectInputStream.defaultReadFields( > >> ObjectInputStream.java:1990) > >> > >> at java.io.ObjectInputStream.readSerialData( > ObjectInputStream.java:1915) > >> > >> at java.io.ObjectInputStream.readOrdinaryObject( > >> ObjectInputStream.java:1798) > >> > >> at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > >> > >> at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370) > >> > >> at org.apache.spark.serializer.JavaDeserializationStream. > >> readObject(JavaSerializer.scala:75) > >> > >> at org.apache.spark.serializer.JavaSerializerInstance. > >> deserialize(JavaSerializer.scala:114) > >> > >> at org.apache.spark.scheduler.ShuffleMapTask.runTask( > >> ShuffleMapTask.scala:85) > >> > >> at org.apache.spark.scheduler.ShuffleMapTask.runTask( > >> ShuffleMapTask.scala:53) > >> > >> at org.apache.spark.scheduler.Task.run(Task.scala:99) > >> > >> at org.apache.spark.executor.Executor$TaskRunner.run( > Executor.scala:322) > >> > >> at java.util.concurrent.ThreadPoolExecutor.runWorker( > >> ThreadPoolExecutor.java:1145) > >> > >> at java.util.concurrent.ThreadPoolExecutor$Worker.run( > >> ThreadPoolExecutor.java:615) > >> > >> at java.lang.Thread.run(Thread.java:745) > >> > >> Caused by: java.lang.ClassNotFoundException: > org.apache.carbondata.spark. > >> rdd.CarbonBlockDistinctValuesCombineRDD > >> > >> at java.lang.ClassLoader.findClass(ClassLoader.java:531) > >> > >> at org.apache.spark.util.ParentClassLoader.findClass( > >> ParentClassLoader.scala:26) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > >> > >> at org.apache.spark.util.ParentClassLoader.loadClass( > >> ParentClassLoader.scala:34) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > >> > >> at org.apache.spark.util.ParentClassLoader.loadClass( > >> ParentClassLoader.scala:30) > >> > >> at org.apache.spark.repl.ExecutorClassLoader.findClass( > >> ExecutorClassLoader.scala:77) > >> > >> ... 23 more > >> > >> > >> [Stage 8:> (0 + > 4) > >> / 156]17/08/21 15:59:02 ERROR scheduler.TaskSetManager: Task 1 in stage > >> 8.0 > >> failed 4 times; aborting job > >> > >> 17/08/21 15:59:02 ERROR util.GlobalDictionaryUtil$: main generate global > >> dictionary failed > >> > >> org.apache.spark.SparkException: Job aborted due to stage failure: > Task 1 > >> in stage 8.0 failed 4 times, most recent failure: Lost task 1.3 in stage > >> 8.0 (TID 61, scsp00382.saicdt.com, executor 16): > >> java.lang.ClassNotFoundException: > >> org.apache.carbondata.spark.rdd.CarbonBlockDistinctValuesCombineRDD > >> > >> at org.apache.spark.repl.ExecutorClassLoader.findClass( > >> ExecutorClassLoader.scala:82) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > >> > >> at java.lang.Class.forName0(Native Method) > >> > >> at java.lang.Class.forName(Class.java:270) > >> > >> at org.apache.spark.serializer.JavaDeserializationStream$$ > >> anon$1.resolveClass(JavaSerializer.scala:67) > >> > >> at > >> java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1612) > >> > >> at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1517) > >> > >> at java.io.ObjectInputStream.readOrdinaryObject( > >> ObjectInputStream.java:1771) > >> > >> at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > >> > >> at java.io.ObjectInputStream.defaultReadFields( > >> ObjectInputStream.java:1990) > >> > >> at java.io.ObjectInputStream.readSerialData( > ObjectInputStream.java:1915) > >> > >> at java.io.ObjectInputStream.readOrdinaryObject( > >> ObjectInputStream.java:1798) > >> > >> at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > >> > >> at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370) > >> > >> at org.apache.spark.serializer.JavaDeserializationStream. > >> readObject(JavaSerializer.scala:75) > >> > >> at org.apache.spark.serializer.JavaSerializerInstance. > >> deserialize(JavaSerializer.scala:114) > >> > >> at org.apache.spark.scheduler.ShuffleMapTask.runTask( > >> ShuffleMapTask.scala:85) > >> > >> at org.apache.spark.scheduler.ShuffleMapTask.runTask( > >> ShuffleMapTask.scala:53) > >> > >> at org.apache.spark.scheduler.Task.run(Task.scala:99) > >> > >> at org.apache.spark.executor.Executor$TaskRunner.run( > Executor.scala:322) > >> > >> at java.util.concurrent.ThreadPoolExecutor.runWorker( > >> ThreadPoolExecutor.java:1145) > >> > >> at java.util.concurrent.ThreadPoolExecutor$Worker.run( > >> ThreadPoolExecutor.java:615) > >> > >> at java.lang.Thread.run(Thread.java:745) > >> > >> Caused by: java.lang.ClassNotFoundException: > org.apache.carbondata.spark. > >> rdd.CarbonBlockDistinctValuesCombineRDD > >> > >> at java.lang.ClassLoader.findClass(ClassLoader.java:531) > >> > >> at org.apache.spark.util.ParentClassLoader.findClass( > >> ParentClassLoader.scala:26) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > >> > >> at org.apache.spark.util.ParentClassLoader.loadClass( > >> ParentClassLoader.scala:34) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > >> > >> at org.apache.spark.util.ParentClassLoader.loadClass( > >> ParentClassLoader.scala:30) > >> > >> at org.apache.spark.repl.ExecutorClassLoader.findClass( > >> ExecutorClassLoader.scala:77) > >> > >> ... 23 more > >> > >> > >> Driver stacktrace: > >> > >> at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$ > >> scheduler$DAGScheduler$$failJobAndIndependentStages( > >> DAGScheduler.scala:1435) > >> > >> at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply( > >> DAGScheduler.scala:1423) > >> > >> at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply( > >> DAGScheduler.scala:1422) > >> > >> at scala.collection.mutable.ResizableArray$class.foreach( > >> ResizableArray.scala:59) > >> > >> at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) > >> > >> at org.apache.spark.scheduler.DAGScheduler.abortStage( > >> DAGScheduler.scala:1422) > >> > >> at org.apache.spark.scheduler.DAGScheduler$$anonfun$ > >> handleTaskSetFailed$1.apply(DAGScheduler.scala:802) > >> > >> at org.apache.spark.scheduler.DAGScheduler$$anonfun$ > >> handleTaskSetFailed$1.apply(DAGScheduler.scala:802) > >> > >> at scala.Option.foreach(Option.scala:257) > >> > >> at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed( > >> DAGScheduler.scala:802) > >> > >> at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop. > >> doOnReceive(DAGScheduler.scala:1650) > >> > >> at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop. > >> onReceive(DAGScheduler.scala:1605) > >> > >> at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop. > >> onReceive(DAGScheduler.scala:1594) > >> > >> at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) > >> > >> at org.apache.spark.scheduler.DAGScheduler.runJob( > DAGScheduler.scala:628) > >> > >> at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918) > >> > >> at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931) > >> > >> at org.apache.spark.SparkContext.runJob(SparkContext.scala:1944) > >> > >> at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958) > >> > >> at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:935) > >> > >> at org.apache.spark.rdd.RDDOperationScope$.withScope( > >> RDDOperationScope.scala:151) > >> > >> at org.apache.spark.rdd.RDDOperationScope$.withScope( > >> RDDOperationScope.scala:112) > >> > >> at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) > >> > >> at org.apache.spark.rdd.RDD.collect(RDD.scala:934) > >> > >> at org.apache.carbondata.spark.util.GlobalDictionaryUtil$. > >> generateGlobalDictionary(GlobalDictionaryUtil.scala:741) > >> > >> at org.apache.spark.sql.execution.command.LoadTable. > >> run(carbonTableSchema.scala:703) > >> > >> at org.apache.spark.sql.execution.command.LoadTableByInsert.run( > >> carbonTableSchema.scala:338) > >> > >> at org.apache.spark.sql.execution.command.ExecutedCommandExec. > >> sideEffectResult$lzycompute(commands.scala:58) > >> > >> at org.apache.spark.sql.execution.command.ExecutedCommandExec. > >> sideEffectResult(commands.scala:56) > >> > >> at org.apache.spark.sql.execution.command. > ExecutedCommandExec.doExecute( > >> commands.scala:74) > >> > >> at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > >> execute$1.apply(SparkPlan.scala:114) > >> > >> at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > >> execute$1.apply(SparkPlan.scala:114) > >> > >> at > >> org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply( > >> SparkPlan.scala:135) > >> > >> at org.apache.spark.rdd.RDDOperationScope$.withScope( > >> RDDOperationScope.scala:151) > >> > >> at org.apache.spark.sql.execution.SparkPlan. > executeQuery(SparkPlan.scala: > >> 132) > >> > >> at org.apache.spark.sql.execution.SparkPlan.execute( > SparkPlan.scala:113) > >> > >> at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute( > >> QueryExecution.scala:87) > >> > >> at org.apache.spark.sql.execution.QueryExecution. > >> toRdd(QueryExecution.scala:87) > >> > >> at org.apache.spark.sql.Dataset. > > <init> > > (Dataset.scala:185) > >> > >> at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64) > >> > >> at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > > <init> > >> ( > > <console> > > :33) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :38) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :40) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :42) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :44) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :46) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :48) > >> > >> at $line30.$read$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :50) > >> > >> at $line30.$read$$iw$$iw. > > <init> > > ( > > <console> > > :52) > >> > >> at $line30.$read$$iw. > > <init> > > ( > > <console> > > :54) > >> > >> at $line30.$read. > > <init> > > ( > > <console> > > :56) > >> > >> at $line30.$read$. > > <init> > > ( > > <console> > > :60) > >> > >> at $line30.$read$. > > <clinit> > > ( > > <console> > > ) > >> > >> at $line30.$eval$.$print$lzycompute( > > <console> > > :7) > >> > >> at $line30.$eval$.$print( > > <console> > > :6) > >> > >> at $line30.$eval.$print( > > <console> > > ) > >> > >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > >> > >> at sun.reflect.NativeMethodAccessorImpl.invoke( > >> NativeMethodAccessorImpl.java:57) > >> > >> at sun.reflect.DelegatingMethodAccessorImpl.invoke( > >> DelegatingMethodAccessorImpl.java:43) > >> > >> at java.lang.reflect.Method.invoke(Method.java:606) > >> > >> at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call( > IMain.scala:786) > >> > >> at scala.tools.nsc.interpreter.IMain$Request.loadAndRun( > IMain.scala:1047) > >> > >> at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$ > >> loadAndRunReq$1.apply(IMain.scala:638) > >> > >> at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$ > >> loadAndRunReq$1.apply(IMain.scala:637) > >> > >> at scala.reflect.internal.util.ScalaClassLoader$class. > >> asContext(ScalaClassLoader.scala:31) > >> > >> at scala.reflect.internal.util.AbstractFileClassLoader.asContext( > >> AbstractFileClassLoader.scala:19) > >> > >> at scala.tools.nsc.interpreter.IMain$WrappedRequest. > >> loadAndRunReq(IMain.scala:637) > >> > >> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) > >> > >> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) > >> > >> at scala.tools.nsc.interpreter.ILoop.interpretStartingWith( > >> ILoop.scala:807) > >> > >> at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681) > >> > >> at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395) > >> > >> at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415) > >> > >> at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > >> apply$mcZ$sp(ILoop.scala:923) > >> > >> at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > >> apply(ILoop.scala:909) > >> > >> at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > >> apply(ILoop.scala:909) > >> > >> at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader( > >> ScalaClassLoader.scala:97) > >> > >> at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909) > >> > >> at org.apache.spark.repl.Main$.doMain(Main.scala:69) > >> > >> at org.apache.spark.repl.Main$.main(Main.scala:52) > >> > >> at org.apache.spark.repl.Main.main(Main.scala) > >> > >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > >> > >> at sun.reflect.NativeMethodAccessorImpl.invoke( > >> NativeMethodAccessorImpl.java:57) > >> > >> at sun.reflect.DelegatingMethodAccessorImpl.invoke( > >> DelegatingMethodAccessorImpl.java:43) > >> > >> at java.lang.reflect.Method.invoke(Method.java:606) > >> > >> at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$ > >> deploy$SparkSubmit$$runMain(SparkSubmit.scala:738) > >> > >> at > >> org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187) > >> > >> at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212) > >> > >> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126) > >> > >> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) > >> > >> Caused by: java.lang.ClassNotFoundException: > org.apache.carbondata.spark. > >> rdd.CarbonBlockDistinctValuesCombineRDD > >> > >> at org.apache.spark.repl.ExecutorClassLoader.findClass( > >> ExecutorClassLoader.scala:82) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > >> > >> at java.lang.Class.forName0(Native Method) > >> > >> at java.lang.Class.forName(Class.java:270) > >> > >> at org.apache.spark.serializer.JavaDeserializationStream$$ > >> anon$1.resolveClass(JavaSerializer.scala:67) > >> > >> at > >> java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1612) > >> > >> at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1517) > >> > >> at java.io.ObjectInputStream.readOrdinaryObject( > >> ObjectInputStream.java:1771) > >> > >> at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > >> > >> at java.io.ObjectInputStream.defaultReadFields( > >> ObjectInputStream.java:1990) > >> > >> at java.io.ObjectInputStream.readSerialData( > ObjectInputStream.java:1915) > >> > >> at java.io.ObjectInputStream.readOrdinaryObject( > >> ObjectInputStream.java:1798) > >> > >> at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1350) > >> > >> at java.io.ObjectInputStream.readObject(ObjectInputStream.java:370) > >> > >> at org.apache.spark.serializer.JavaDeserializationStream. > >> readObject(JavaSerializer.scala:75) > >> > >> at org.apache.spark.serializer.JavaSerializerInstance. > >> deserialize(JavaSerializer.scala:114) > >> > >> at org.apache.spark.scheduler.ShuffleMapTask.runTask( > >> ShuffleMapTask.scala:85) > >> > >> at org.apache.spark.scheduler.ShuffleMapTask.runTask( > >> ShuffleMapTask.scala:53) > >> > >> at org.apache.spark.scheduler.Task.run(Task.scala:99) > >> > >> at org.apache.spark.executor.Executor$TaskRunner.run( > Executor.scala:322) > >> > >> at java.util.concurrent.ThreadPoolExecutor.runWorker( > >> ThreadPoolExecutor.java:1145) > >> > >> at java.util.concurrent.ThreadPoolExecutor$Worker.run( > >> ThreadPoolExecutor.java:615) > >> > >> at java.lang.Thread.run(Thread.java:745) > >> > >> Caused by: java.lang.ClassNotFoundException: > org.apache.carbondata.spark. > >> rdd.CarbonBlockDistinctValuesCombineRDD > >> > >> at java.lang.ClassLoader.findClass(ClassLoader.java:531) > >> > >> at org.apache.spark.util.ParentClassLoader.findClass( > >> ParentClassLoader.scala:26) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:425) > >> > >> at org.apache.spark.util.ParentClassLoader.loadClass( > >> ParentClassLoader.scala:34) > >> > >> at java.lang.ClassLoader.loadClass(ClassLoader.java:358) > >> > >> at org.apache.spark.util.ParentClassLoader.loadClass( > >> ParentClassLoader.scala:30) > >> > >> at org.apache.spark.repl.ExecutorClassLoader.findClass( > >> ExecutorClassLoader.scala:77) > >> > >> ... 23 more > >> > >> 17/08/21 15:59:02 ERROR command.LoadTable: main > >> > >> java.lang.Exception: generate global dictionary failed, null > >> > >> at org.apache.carbondata.spark.util.GlobalDictionaryUtil$. > >> generateGlobalDictionary(GlobalDictionaryUtil.scala:761) > >> > >> at org.apache.spark.sql.execution.command.LoadTable. > >> run(carbonTableSchema.scala:703) > >> > >> at org.apache.spark.sql.execution.command.LoadTableByInsert.run( > >> carbonTableSchema.scala:338) > >> > >> at org.apache.spark.sql.execution.command.ExecutedCommandExec. > >> sideEffectResult$lzycompute(commands.scala:58) > >> > >> at org.apache.spark.sql.execution.command.ExecutedCommandExec. > >> sideEffectResult(commands.scala:56) > >> > >> at org.apache.spark.sql.execution.command. > ExecutedCommandExec.doExecute( > >> commands.scala:74) > >> > >> at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > >> execute$1.apply(SparkPlan.scala:114) > >> > >> at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > >> execute$1.apply(SparkPlan.scala:114) > >> > >> at > >> org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply( > >> SparkPlan.scala:135) > >> > >> at org.apache.spark.rdd.RDDOperationScope$.withScope( > >> RDDOperationScope.scala:151) > >> > >> at org.apache.spark.sql.execution.SparkPlan. > executeQuery(SparkPlan.scala: > >> 132) > >> > >> at org.apache.spark.sql.execution.SparkPlan.execute( > SparkPlan.scala:113) > >> > >> at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute( > >> QueryExecution.scala:87) > >> > >> at org.apache.spark.sql.execution.QueryExecution. > >> toRdd(QueryExecution.scala:87) > >> > >> at org.apache.spark.sql.Dataset. > > <init> > > (Dataset.scala:185) > >> > >> at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64) > >> > >> at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > > <init> > >> ( > > <console> > > :33) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :38) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :40) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :42) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :44) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :46) > >> > >> at $line30.$read$$iw$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :48) > >> > >> at $line30.$read$$iw$$iw$$iw. > > <init> > > ( > > <console> > > :50) > >> > >> at $line30.$read$$iw$$iw. > > <init> > > ( > > <console> > > :52) > >> > >> at $line30.$read$$iw. > > <init> > > ( > > <console> > > :54) > >> > >> at $line30.$read. > > <init> > > ( > > <console> > > :56) > >> > >> at $line30.$read$. > > <init> > > ( > > <console> > > :60) > >> > >> at $line30.$read$. > > <clinit> > > ( > > <console> > > ) > >> > >> at $line30.$eval$.$print$lzycompute( > > <console> > > :7) > >> > >> at $line30.$eval$.$print( > > <console> > > :6) > >> > >> at $line30.$eval.$print( > > <console> > > ) > >> > >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > >> > >> at sun.reflect.NativeMethodAccessorImpl.invoke( > >> NativeMethodAccessorImpl.java:57) > >> > >> at sun.reflect.DelegatingMethodAccessorImpl.invoke( > >> DelegatingMethodAccessorImpl.java:43) > >> > >> at java.lang.reflect.Method.invoke(Method.java:606) > >> > >> at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call( > IMain.scala:786) > >> > >> at scala.tools.nsc.interpreter.IMain$Request.loadAndRun( > IMain.scala:1047) > >> > >> at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$ > >> loadAndRunReq$1.apply(IMain.scala:638) > >> > >> at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$ > >> loadAndRunReq$1.apply(IMain.scala:637) > >> > >> at scala.reflect.internal.util.ScalaClassLoader$class. > >> asContext(ScalaClassLoader.scala:31) > >> > >> at scala.reflect.internal.util.AbstractFileClassLoader.asContext( > >> AbstractFileClassLoader.scala:19) > >> > >> at scala.tools.nsc.interpreter.IMain$WrappedRequest. > >> loadAndRunReq(IMain.scala:637) > >> > >> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) > >> > >> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) > >> > >> at scala.tools.nsc.interpreter.ILoop.interpretStartingWith( > >> ILoop.scala:807) > >> > >> at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681) > >> > >> at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395) > >> > >> at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415) > >> > >> at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > >> apply$mcZ$sp(ILoop.scala:923) > >> > >> at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > >> apply(ILoop.scala:909) > >> > >> at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1. > >> apply(ILoop.scala:909) > >> > >> at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader( > >> ScalaClassLoader.scala:97) > >> > >> at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909) > >> > >> at org.apache.spark.repl.Main$.doMain(Main.scala:69) > >> > >> at org.apache.spark.repl.Main$.main(Main.scala:52) > >> > >> at org.apache.spark.repl.Main.main(Main.scala) > >> > >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > >> > >> at sun.reflect.NativeMethodAccessorImpl.invoke( > >> NativeMethodAccessorImpl.java:57) > >> > >> at sun.reflect.DelegatingMethodAccessorImpl.invoke( > >> DelegatingMethodAccessorImpl.java:43) > >> > >> at java.lang.reflect.Method.invoke(Method.java:606) > >> > >> at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$ > >> deploy$SparkSubmit$$runMain(SparkSubmit.scala:738) > >> > >> at > >> org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187) > >> > >> at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212) > >> > >> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126) > >> > >> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) > >> > >> 17/08/21 15:59:02 AUDIT command.LoadTable: > >> [scsp00267.saicdt.com][carbon2][Thread-1]Dataload > >> failure for default.crx5_tbox_normal_test. Please check the logs > >> > >> 17/08/21 15:59:02 WARN spark.ExecutorAllocationManager: No stages are > >> running, but numRunningTasks != 0 > >> > >> java.lang.Exception: generate global dictionary failed, null > >> > >> at org.apache.carbondata.spark.util.GlobalDictionaryUtil$. > >> generateGlobalDictionary(GlobalDictionaryUtil.scala:761) > >> > >> at org.apache.spark.sql.execution.command.LoadTable. > >> run(carbonTableSchema.scala:703) > >> > >> at org.apache.spark.sql.execution.command.LoadTableByInsert.run( > >> carbonTableSchema.scala:338) > >> > >> at org.apache.spark.sql.execution.command.ExecutedCommandExec. > >> sideEffectResult$lzycompute(commands.scala:58) > >> > >> at org.apache.spark.sql.execution.command.ExecutedCommandExec. > >> sideEffectResult(commands.scala:56) > >> > >> at > >> org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute( > >> commands.scala:74) > >> > >> at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > >> execute$1.apply(SparkPlan.scala:114) > >> > >> at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > >> execute$1.apply(SparkPlan.scala:114) > >> > >> at org.apache.spark.sql.execution.SparkPlan$$anonfun$ > >> executeQuery$1.apply(SparkPlan.scala:135) > >> > >> at org.apache.spark.rdd.RDDOperationScope$.withScope( > >> RDDOperationScope.scala:151) > >> > >> at org.apache.spark.sql.execution.SparkPlan. > >> executeQuery(SparkPlan.scala:132) > >> > >> at > >> org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113) > >> > >> at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute( > >> QueryExecution.scala:87) > >> > >> at org.apache.spark.sql.execution.QueryExecution. > >> toRdd(QueryExecution.scala:87) > >> > >> at org.apache.spark.sql.Dataset. > > <init> > > (Dataset.scala:185) > >> > >> at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64) > >> > >> at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:592) > >> > >> ... 50 elided > >> > >> > >> > >> Thanks, > >> > >> CaoLu > >> > > > > > > -- > View this message in context: http://apache-carbondata-dev- > mailing-list-archive.1130556.n5.nabble.com/ClassNotFound- > error-when-insert-carbontable-from-hive-table-tp20598p20644.html > Sent from the Apache CarbonData Dev Mailing List archive mailing list > archive at Nabble.com. > |
«
Return to Apache CarbonData Dev Mailing List archive
|
1 view|%1 views
| Free forum by Nabble | Edit this page |