http://apache-carbondata-dev-mailing-list-archive.168.s1.nabble.com/error-occur-when-I-load-data-to-s3-tp60717p61265.html
Ok. Let me have a look
> Hi kunalkapoor,
> It seems that error not fixed yet. Do you have any idea?
>
> thanks
> aaron
>
> aaron:2.2.1 aaron$ spark-shell --executor-memory 4g --driver-memory 2g
> Ivy Default Cache set to: /Users/aaron/.ivy2/cache
> The jars for the packages stored in: /Users/aaron/.ivy2/jars
> :: loading settings :: url =
>
> jar:file:/usr/local/Cellar/apache-spark/2.2.1/lib/apache-carbondata-1.5.0-SNAPSHOT-bin-spark2.2.1-hadoop2.7.2.jar!/org/apache/ivy/core/settings/ivysettings.xml
> com.amazonaws#aws-java-sdk added as a dependency
> org.apache.hadoop#hadoop-aws added as a dependency
> com.databricks#spark-avro_2.11 added as a dependency
> :: resolving dependencies :: org.apache.spark#spark-submit-parent;1.0
> confs: [default]
> found com.amazonaws#aws-java-sdk;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-support;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-core;1.10.75.1 in central
> found commons-logging#commons-logging;1.1.3 in central
> found org.apache.httpcomponents#httpclient;4.3.6 in local-m2-cache
> found org.apache.httpcomponents#httpcore;4.3.3 in local-m2-cache
> found commons-codec#commons-codec;1.6 in local-m2-cache
> found com.fasterxml.jackson.core#jackson-databind;2.5.3 in central
> found com.fasterxml.jackson.core#jackson-annotations;2.5.0 in
> central
> found com.fasterxml.jackson.core#jackson-core;2.5.3 in central
> found
> com.fasterxml.jackson.dataformat#jackson-dataformat-cbor;2.5.3 in
> central
> found joda-time#joda-time;2.8.1 in central
> found com.amazonaws#aws-java-sdk-simpledb;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-simpleworkflow;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-storagegateway;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-route53;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-s3;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-kms;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-importexport;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-sts;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-sqs;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-rds;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-redshift;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-elasticbeanstalk;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-glacier;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-sns;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-iam;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-datapipeline;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-elasticloadbalancing;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-emr;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-elasticache;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-elastictranscoder;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-ec2;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-dynamodb;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudtrail;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudwatch;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-logs;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-events;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cognitoidentity;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-cognitosync;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-directconnect;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudformation;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-cloudfront;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-kinesis;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-opsworks;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-ses;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-autoscaling;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudsearch;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudwatchmetrics;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-swf-libraries;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-codedeploy;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-codepipeline;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-config;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-lambda;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-ecs;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-ecr;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudhsm;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-ssm;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-workspaces;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-machinelearning;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-directory;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-efs;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-codecommit;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-devicefarm;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-elasticsearch;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-waf;1.10.75.1 in central
> found
> com.amazonaws#aws-java-sdk-marketplacecommerceanalytics;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-inspector;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-iot;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-api-gateway;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-acm;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-gamelift;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-dms;1.10.75.1 in central
> found
> com.amazonaws#aws-java-sdk-marketplacemeteringservice;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-cognitoidp;1.10.75.1 in central
> found org.apache.hadoop#hadoop-aws;2.7.3 in central
> found org.apache.hadoop#hadoop-common;2.7.3 in local-m2-cache
> found org.apache.hadoop#hadoop-annotations;2.7.3 in local-m2-cache
> found com.google.guava#guava;11.0.2 in local-m2-cache
> found com.google.code.findbugs#jsr305;3.0.0 in local-m2-cache
> found commons-cli#commons-cli;1.2 in local-m2-cache
> found org.apache.commons#commons-math3;3.1.1 in local-m2-cache
> found xmlenc#xmlenc;0.52 in local-m2-cache
> found commons-httpclient#commons-httpclient;3.1 in local-m2-cache
> found commons-io#commons-io;2.4 in local-m2-cache
> found commons-net#commons-net;3.1 in local-m2-cache
> found commons-collections#commons-collections;3.2.2 in
> local-m2-cache
> found javax.servlet#servlet-api;2.5 in local-m2-cache
> found org.mortbay.jetty#jetty;6.1.26 in local-m2-cache
> found org.mortbay.jetty#jetty-util;6.1.26 in local-m2-cache
> found com.sun.jersey#jersey-core;1.9 in local-m2-cache
> found com.sun.jersey#jersey-json;1.9 in local-m2-cache
> found org.codehaus.jettison#jettison;1.1 in local-m2-cache
> found com.sun.xml.bind#jaxb-impl;2.2.3-1 in local-m2-cache
> found javax.xml.bind#jaxb-api;2.2.2 in local-m2-cache
> found javax.xml.stream#stax-api;1.0-2 in local-m2-cache
> found javax.activation#activation;1.1 in local-m2-cache
> found org.codehaus.jackson#jackson-core-asl;1.9.13 in
> local-m2-cache
> found org.codehaus.jackson#jackson-mapper-asl;1.9.13 in
> local-m2-cache
> found org.codehaus.jackson#jackson-jaxrs;1.9.13 in local-m2-cache
> found org.codehaus.jackson#jackson-xc;1.9.13 in local-m2-cache
> found com.sun.jersey#jersey-server;1.9 in local-m2-cache
> found asm#asm;3.2 in local-m2-cache
> found log4j#log4j;1.2.17 in local-m2-cache
> found net.java.dev.jets3t#jets3t;0.9.0 in local-m2-cache
> found com.jamesmurty.utils#java-xmlbuilder;0.4 in local-m2-cache
> found commons-lang#commons-lang;2.6 in central
> found commons-configuration#commons-configuration;1.6 in
> local-m2-cache
> found commons-digester#commons-digester;1.8 in local-m2-cache
> found commons-beanutils#commons-beanutils;1.7.0 in local-m2-cache
> found commons-beanutils#commons-beanutils-core;1.8.0 in
> local-m2-cache
> found org.slf4j#slf4j-api;1.7.10 in local-m2-cache
> found org.apache.avro#avro;1.7.4 in local-m2-cache
> found com.thoughtworks.paranamer#paranamer;2.3 in local-m2-cache
> found org.xerial.snappy#snappy-java;1.0.4.1 in local-m2-cache
> found org.apache.commons#commons-compress;1.4.1 in local-m2-cache
> found org.tukaani#xz;1.0 in local-m2-cache
> found com.google.protobuf#protobuf-java;2.5.0 in central
> found com.google.code.gson#gson;2.2.4 in local-m2-cache
> found org.apache.hadoop#hadoop-auth;2.7.3 in local-m2-cache
> found
> org.apache.directory.server#apacheds-kerberos-codec;2.0.0-M15 in
> local-m2-cache
> found org.apache.directory.server#apacheds-i18n;2.0.0-M15 in
> local-m2-cache
> found org.apache.directory.api#api-asn1-api;1.0.0-M20 in
> local-m2-cache
> found org.apache.directory.api#api-util;1.0.0-M20 in local-m2-cache
> found org.apache.zookeeper#zookeeper;3.4.6 in local-m2-cache
> found org.slf4j#slf4j-log4j12;1.7.10 in local-m2-cache
> found io.netty#netty;3.6.2.Final in local-m2-cache
> found org.apache.curator#curator-framework;2.7.1 in local-m2-cache
> found org.apache.curator#curator-client;2.7.1 in local-m2-cache
> found com.jcraft#jsch;0.1.42 in local-m2-cache
> found org.apache.curator#curator-recipes;2.7.1 in local-m2-cache
> found org.apache.htrace#htrace-core;3.1.0-incubating in
> local-m2-cache
> found javax.servlet.jsp#jsp-api;2.1 in local-m2-cache
> found jline#jline;0.9.94 in local-m2-cache
> found junit#junit;4.11 in local-m2-cache
> found org.hamcrest#hamcrest-core;1.3 in local-m2-cache
> found com.databricks#spark-avro_2.11;4.0.0 in central
> found org.apache.avro#avro;1.7.6 in central
> found org.xerial.snappy#snappy-java;1.0.5 in local-m2-cache
> :: resolution report :: resolve 3300ms :: artifacts dl 303ms
> :: modules in use:
> asm#asm;3.2 from local-m2-cache in [default]
> com.amazonaws#aws-java-sdk;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-acm;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-api-gateway;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-autoscaling;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudformation;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudfront;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudhsm;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudsearch;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudtrail;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudwatch;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudwatchmetrics;1.10.75.1 from
> central in
> [default]
> com.amazonaws#aws-java-sdk-codecommit;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-codedeploy;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-codepipeline;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cognitoidentity;1.10.75.1 from central
> in
> [default]
> com.amazonaws#aws-java-sdk-cognitoidp;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cognitosync;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-config;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-core;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-datapipeline;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-devicefarm;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-directconnect;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-directory;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-dms;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-dynamodb;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-ec2;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-ecr;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-ecs;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-efs;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-elasticache;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-elasticbeanstalk;1.10.75.1 from central
> in
> [default]
> com.amazonaws#aws-java-sdk-elasticloadbalancing;1.10.75.1 from
> central in
> [default]
> com.amazonaws#aws-java-sdk-elasticsearch;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-elastictranscoder;1.10.75.1 from
> central in
> [default]
> com.amazonaws#aws-java-sdk-emr;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-events;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-gamelift;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-glacier;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-iam;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-importexport;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-inspector;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-iot;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-kinesis;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-kms;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-lambda;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-logs;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-machinelearning;1.10.75.1 from central
> in
> [default]
> com.amazonaws#aws-java-sdk-marketplacecommerceanalytics;1.10.75.1
> from
> central in [default]
> com.amazonaws#aws-java-sdk-marketplacemeteringservice;1.10.75.1
> from
> central in [default]
> com.amazonaws#aws-java-sdk-opsworks;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-rds;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-redshift;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-route53;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-s3;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-ses;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-simpledb;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-simpleworkflow;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-sns;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-sqs;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-ssm;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-storagegateway;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-sts;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-support;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-swf-libraries;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-waf;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-workspaces;1.10.75.1 from central in
> [default]
> com.databricks#spark-avro_2.11;4.0.0 from central in [default]
> com.fasterxml.jackson.core#jackson-annotations;2.5.0 from central
> in
> [default]
> com.fasterxml.jackson.core#jackson-core;2.5.3 from central in
> [default]
> com.fasterxml.jackson.core#jackson-databind;2.5.3 from central in
> [default]
> com.fasterxml.jackson.dataformat#jackson-dataformat-cbor;2.5.3
> from central
> in [default]
> com.google.code.findbugs#jsr305;3.0.0 from local-m2-cache in
> [default]
> com.google.code.gson#gson;2.2.4 from local-m2-cache in [default]
> com.google.guava#guava;11.0.2 from local-m2-cache in [default]
> com.google.protobuf#protobuf-java;2.5.0 from central in [default]
> com.jamesmurty.utils#java-xmlbuilder;0.4 from local-m2-cache in
> [default]
> com.jcraft#jsch;0.1.42 from local-m2-cache in [default]
> com.sun.jersey#jersey-core;1.9 from local-m2-cache in [default]
> com.sun.jersey#jersey-json;1.9 from local-m2-cache in [default]
> com.sun.jersey#jersey-server;1.9 from local-m2-cache in [default]
> com.sun.xml.bind#jaxb-impl;2.2.3-1 from local-m2-cache in [default]
> com.thoughtworks.paranamer#paranamer;2.3 from local-m2-cache in
> [default]
> commons-beanutils#commons-beanutils;1.7.0 from local-m2-cache in
> [default]
> commons-beanutils#commons-beanutils-core;1.8.0 from local-m2-cache
> in
> [default]
> commons-cli#commons-cli;1.2 from local-m2-cache in [default]
> commons-codec#commons-codec;1.6 from local-m2-cache in [default]
> commons-collections#commons-collections;3.2.2 from local-m2-cache
> in
> [default]
> commons-configuration#commons-configuration;1.6 from
> local-m2-cache in
> [default]
> commons-digester#commons-digester;1.8 from local-m2-cache in
> [default]
> commons-httpclient#commons-httpclient;3.1 from local-m2-cache in
> [default]
> commons-io#commons-io;2.4 from local-m2-cache in [default]
> commons-lang#commons-lang;2.6 from central in [default]
> commons-logging#commons-logging;1.1.3 from central in [default]
> commons-net#commons-net;3.1 from local-m2-cache in [default]
> io.netty#netty;3.6.2.Final from local-m2-cache in [default]
> javax.activation#activation;1.1 from local-m2-cache in [default]
> javax.servlet#servlet-api;2.5 from local-m2-cache in [default]
> javax.servlet.jsp#jsp-api;2.1 from local-m2-cache in [default]
> javax.xml.bind#jaxb-api;2.2.2 from local-m2-cache in [default]
> javax.xml.stream#stax-api;1.0-2 from local-m2-cache in [default]
> jline#jline;0.9.94 from local-m2-cache in [default]
> joda-time#joda-time;2.8.1 from central in [default]
> junit#junit;4.11 from local-m2-cache in [default]
> log4j#log4j;1.2.17 from local-m2-cache in [default]
> net.java.dev.jets3t#jets3t;0.9.0 from local-m2-cache in [default]
> org.apache.avro#avro;1.7.6 from central in [default]
> org.apache.commons#commons-compress;1.4.1 from local-m2-cache in
> [default]
> org.apache.commons#commons-math3;3.1.1 from local-m2-cache in
> [default]
> org.apache.curator#curator-client;2.7.1 from local-m2-cache in
> [default]
> org.apache.curator#curator-framework;2.7.1 from local-m2-cache in
> [default]
> org.apache.curator#curator-recipes;2.7.1 from local-m2-cache in
> [default]
> org.apache.directory.api#api-asn1-api;1.0.0-M20 from
> local-m2-cache in
> [default]
> org.apache.directory.api#api-util;1.0.0-M20 from local-m2-cache in
> [default]
> org.apache.directory.server#apacheds-i18n;2.0.0-M15 from
> local-m2-cache in
> [default]
> org.apache.directory.server#apacheds-kerberos-codec;2.0.0-M15 from
> local-m2-cache in [default]
> org.apache.hadoop#hadoop-annotations;2.7.3 from local-m2-cache in
> [default]
> org.apache.hadoop#hadoop-auth;2.7.3 from local-m2-cache in
> [default]
> org.apache.hadoop#hadoop-aws;2.7.3 from central in [default]
> org.apache.hadoop#hadoop-common;2.7.3 from local-m2-cache in
> [default]
> org.apache.htrace#htrace-core;3.1.0-incubating from local-m2-cache
> in
> [default]
> org.apache.httpcomponents#httpclient;4.3.6 from local-m2-cache in
> [default]
> org.apache.httpcomponents#httpcore;4.3.3 from local-m2-cache in
> [default]
> org.apache.zookeeper#zookeeper;3.4.6 from local-m2-cache in
> [default]
> org.codehaus.jackson#jackson-core-asl;1.9.13 from local-m2-cache in
> [default]
> org.codehaus.jackson#jackson-jaxrs;1.9.13 from local-m2-cache in
> [default]
> org.codehaus.jackson#jackson-mapper-asl;1.9.13 from local-m2-cache
> in
> [default]
> org.codehaus.jackson#jackson-xc;1.9.13 from local-m2-cache in
> [default]
> org.codehaus.jettison#jettison;1.1 from local-m2-cache in [default]
> org.hamcrest#hamcrest-core;1.3 from local-m2-cache in [default]
> org.mortbay.jetty#jetty;6.1.26 from local-m2-cache in [default]
> org.mortbay.jetty#jetty-util;6.1.26 from local-m2-cache in
> [default]
> org.slf4j#slf4j-api;1.7.10 from local-m2-cache in [default]
> org.slf4j#slf4j-log4j12;1.7.10 from local-m2-cache in [default]
> org.tukaani#xz;1.0 from local-m2-cache in [default]
> org.xerial.snappy#snappy-java;1.0.5 from local-m2-cache in
> [default]
> xmlenc#xmlenc;0.52 from local-m2-cache in [default]
> :: evicted modules:
> commons-codec#commons-codec;1.4 by
> [commons-codec#commons-codec;1.6] in
> [default]
> org.apache.httpcomponents#httpclient;4.2.5 by
> [org.apache.httpcomponents#httpclient;4.3.6] in [default]
> org.apache.httpcomponents#httpcore;4.2.5 by
> [org.apache.httpcomponents#httpcore;4.3.3] in [default]
> org.apache.avro#avro;1.7.4 by [org.apache.avro#avro;1.7.6] in
> [default]
> org.xerial.snappy#snappy-java;1.0.4.1 by
> [org.xerial.snappy#snappy-java;1.0.5] in [default]
> com.fasterxml.jackson.core#jackson-databind;2.2.3 by
> [com.fasterxml.jackson.core#jackson-databind;2.5.3] in [default]
> com.fasterxml.jackson.core#jackson-annotations;2.2.3 by
> [com.fasterxml.jackson.core#jackson-annotations;2.5.0] in [default]
> com.amazonaws#aws-java-sdk;1.7.4 by
> [com.amazonaws#aws-java-sdk;1.10.75.1]
> in [default]
> org.slf4j#slf4j-api;1.7.5 by [org.slf4j#slf4j-api;1.7.10] in
> [default]
> org.slf4j#slf4j-api;1.6.4 by [org.slf4j#slf4j-api;1.7.10] in
> [default]
>
> ---------------------------------------------------------------------
> | | modules || artifacts
> |
> | conf | number| search|dwnlded|evicted||
> number|dwnlded|
>
> ---------------------------------------------------------------------
> | default | 147 | 0 | 0 | 10 || 137 | 0
> |
>
> ---------------------------------------------------------------------
> :: retrieving :: org.apache.spark#spark-submit-parent
> confs: [default]
> 0 artifacts copied, 137 already retrieved (0kB/48ms)
> SLF4J: Class path contains multiple SLF4J bindings.
> SLF4J: Found binding in
>
> [jar:file:/usr/local/Cellar/apache-spark/2.2.1/lib/apache-carbondata-1.5.0-SNAPSHOT-bin-spark2.2.1-hadoop2.7.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: Found binding in
>
> [jar:file:/usr/local/Cellar/apache-spark/2.2.1/jars/slf4j-log4j12-1.7.16.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: See
http://www.slf4j.org/codes.html#multiple_bindings for an
> explanation.
> SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
> Using Spark's default log4j profile:
> org/apache/spark/log4j-defaults.properties
> Setting default log level to "WARN".
> To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use
> setLogLevel(newLevel).
> 18/09/04 10:41:37 WARN NativeCodeLoader: Unable to load native-hadoop
> library for your platform... using builtin-java classes where applicable
> 18/09/04 10:41:37 WARN Utils: Your hostname, aaron.lan.appannie.com
> resolves
> to a loopback address: 127.0.0.1; using 192.168.2.212 instead (on interface
> en0)
> 18/09/04 10:41:37 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to
> another address
> Spark context Web UI available at
http://192.168.2.212:4040> Spark context available as 'sc' (master = local[*], app id =
> local-1536028898657).
> Spark session available as 'spark'.
> Welcome to
> ____ __
> / __/__ ___ _____/ /__
> _\ \/ _ \/ _ `/ __/ '_/
> /___/ .__/\_,_/_/ /_/\_\ version 2.2.1
> /_/
>
> Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java
> 1.8.0_144)
> Type in expressions to have them evaluated.
> Type :help for more information.
>
> scala> import org.apache.spark.sql.SparkSession
> import org.apache.spark.sql.SparkSession
>
> scala> import org.apache.spark.sql.CarbonSession._
> import org.apache.spark.sql.CarbonSession._
>
> scala> import org.apache.spark.sql.catalyst.util._
> import org.apache.spark.sql.catalyst.util._
>
> scala> val carbon =
>
> SparkSession.builder().config(sc.getConf).config("spark.hadoop.fs.s3a.impl",
>
> "org.apache.hadoop.fs.s3a.S3AFileSystem").config("spark.hadoop.fs.s3a.access.key",
> "xxx").config("spark.hadoop.fs.s3a.secret.key",
> "xxx").getOrCreateCarbonSession("hdfs://localhost:9000/usr/carbon-meta")
> 18/09/04 10:41:58 WARN SparkContext: Using an existing SparkContext; some
> configuration may not take effect.
> 18/09/04 10:41:59 WARN CarbonProperties: main The enable off heap sort
> value
> "null" is invalid. Using the default value "true
> 18/09/04 10:41:59 WARN CarbonProperties: main The specified value for
> property sort.inmemory.size.inmbis Invalid. Taking the default value.1024
> 18/09/04 10:42:05 WARN ObjectStore: Failed to get database global_temp,
> returning NoSuchObjectException
> carbon: org.apache.spark.sql.SparkSession =
> org.apache.spark.sql.CarbonSession@2d492d46
>
> scala> carbon.sql("CREATE TABLE IF NOT EXISTS test_s3_table(id string, name
> string, city string, age Int) STORED BY 'carbondata' LOCATION
> 's3a://sdk-demo-sg/'")
> 18/09/04 10:42:52 AUDIT CarbonCreateTableCommand:
> [aaron.lan.appannie.com][aaron][Thread-1]Creating Table with Database name
> [default] and Table name [test_s3_table]
> 18/09/04 10:43:01 WARN HiveExternalCatalog: Couldn't find corresponding
> Hive
> SerDe for data source provider org.apache.spark.sql.CarbonSource.
> Persisting
> data source table `default`.`test_s3_table` into Hive metastore in Spark
> SQL
> specific format, which is NOT compatible with Hive.
> 18/09/04 10:43:01 AUDIT CarbonCreateTableCommand:
> [aaron.lan.appannie.com][aaron][Thread-1]Table created with Database name
> [default] and Table name [test_s3_table]
> res1: org.apache.spark.sql.DataFrame = []
>
> scala> carbon.sql("describe extended test_s3_table").show(200,
> truncate=false)
> 18/09/04 10:50:02 ERROR CarbonUtil: main Unable to unlock Table lock for
> table during table status updation
>
> +------------------------------------+--------------------------------------------------------------------------------+------------------------------------------------------------------------+
> |col_name |data_type
>
> |comment |
>
> +------------------------------------+--------------------------------------------------------------------------------+------------------------------------------------------------------------+
> |id |string
>
> |KEY COLUMN,null |
> |name |string
>
> |KEY COLUMN,null |
> |city |string
>
> |KEY COLUMN,null |
> |age |int
>
> |MEASURE,null |
> | |
>
> | |
> |##Detailed Table Information |
>
> | |
> |Database Name |default
>
> | |
> |Table Name |test_s3_table
>
> | |
> |CARBON Store Path |s3a://sdk-demo-sg/
>
> | |
> |Comment |
>
> | |
> |Table Block Size |1024 MB
>
> | |
> |Table Data Size |0
>
> | |
> |Table Index Size |0
>
> | |
> |Last Update Time |0
>
> | |
> |SORT_SCOPE |LOCAL_SORT
>
> |LOCAL_SORT |
> |CACHE_LEVEL |BLOCK
>
> | |
> |Streaming |false
>
> | |
> |Local Dictionary Enabled |false
>
> | |
> | |
>
> | |
> |##Detailed Column property |
>
> | |
> |ADAPTIVE |
>
> | |
> |SORT_COLUMNS |id,name,city
>
> | |
>
> +------------------------------------+--------------------------------------------------------------------------------+------------------------------------------------------------------------+
>
> scala> carbon.sql("LOAD DATA INPATH
> 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE test_s3_table")
> 18/09/04 10:43:23 ERROR CarbonLoaderUtil: main Unable to unlock Table lock
> for tabledefault.test_s3_table during table status updation
> 18/09/04 10:43:23 ERROR CarbonLoadDataCommand: main
> java.lang.ArrayIndexOutOfBoundsException
> at java.lang.System.arraycopy(Native Method)
> at
> java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> at
> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> at
>
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> at java.io.DataOutputStream.write(DataOutputStream.java:107)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> at
>
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:289)
> at
> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
> at
>
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:264)
> at
>
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:106)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:95)
> at
> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:153)
> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:93)
> at
>
> $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:37)
> at
>
> $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:42)
> at
> $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:44)
> at
> $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:46)
> at
> $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:48)
> at $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:50)
> at $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:52)
> at $line23.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:54)
> at $line23.$read$$iw$$iw$$iw$$iw.<init>(<console>:56)
> at $line23.$read$$iw$$iw$$iw.<init>(<console>:58)
> at $line23.$read$$iw$$iw.<init>(<console>:60)
> at $line23.$read$$iw.<init>(<console>:62)
> at $line23.$read.<init>(<console>:64)
> at $line23.$read$.<init>(<console>:68)
> at $line23.$read$.<clinit>(<console>)
> at $line23.$eval$.$print$lzycompute(<console>:7)
> at $line23.$eval$.$print(<console>:6)
> at $line23.$eval.$print(<console>)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
> at
> scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
> at
>
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
> at
>
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
> at
>
> scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
> at
>
> scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
> at
>
> scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
> at
> scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
> at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
> at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
> at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415)
> at
>
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923)
> at
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
> at
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
> at
>
> scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
> at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
> at org.apache.spark.repl.Main$.doMain(Main.scala:74)
> at org.apache.spark.repl.Main$.main(Main.scala:54)
> at org.apache.spark.repl.Main.main(Main.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
>
> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
> at
> org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
> at
> org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> 18/09/04 10:43:23 AUDIT CarbonLoadDataCommand:
> [aaron.lan.appannie.com][aaron][Thread-1]Dataload failure for
> default.test_s3_table. Please check the logs
> 18/09/04 10:43:23 ERROR CarbonLoadDataCommand: main Got exception
> java.lang.ArrayIndexOutOfBoundsException when processing data. But this
> command does not support undo yet, skipping the undo part.
> java.lang.ArrayIndexOutOfBoundsException
> at java.lang.System.arraycopy(Native Method)
> at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> at
> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> at
>
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> at java.io.DataOutputStream.write(DataOutputStream.java:107)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> at
>
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:289)
> at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
> at
>
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:264)
> at
>
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:106)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:95)
> at
> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:153)
> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:93)
> ... 52 elided
>
> scala>
>
>
>
> --
> Sent from:
>
http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/>
>