[jira] [Resolved] (CARBONDATA-2956) CarbonReader can't support use configuration to read S3 data

[Akash R Nilugal (Jira)]

     [ https://issues.apache.org/jira/browse/CARBONDATA-2956?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

Kunal Kapoor resolved CARBONDATA-2956.
--------------------------------------
    Resolution: Fixed

> CarbonReader can't support use configuration to read S3 data
> ------------------------------------------------------------
>
>                 Key: CARBONDATA-2956
>                 URL: https://issues.apache.org/jira/browse/CARBONDATA-2956
>             Project: CarbonData
>          Issue Type: Bug
>          Components: other
>    Affects Versions: 1.5.0
>            Reporter: xubo245
>            Assignee: xubo245
>            Priority: Major
>             Fix For: 1.5.0
>
>          Time Spent: 1h 40m
>  Remaining Estimate: 0h
>
> Test code:
> {code:java}
> package org.apache.carbondata.examples.sdk;
> import org.apache.carbondata.common.logging.LogService;
> import org.apache.carbondata.common.logging.LogServiceFactory;
> import org.apache.carbondata.core.metadata.datatype.DataTypes;
> import org.apache.carbondata.core.scan.expression.ColumnExpression;
> import org.apache.carbondata.core.scan.expression.LiteralExpression;
> import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
> import org.apache.carbondata.sdk.file.*;
> import org.apache.hadoop.conf.Configuration;
> import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
> import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
> import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
> /**
>  * Example for testing CarbonWriter on S3
>  */
> public class SDKS3Example {
>     public static void main(String[] args) throws Exception {
>         LogService logger = LogServiceFactory.getLogService(SDKS3Example.class.getName());
>         if (args == null || args.length < 3) {
>             logger.error("Usage: java CarbonS3Example: <access-key> <secret-key>"
>                 + "<s3-endpoint> [table-path-on-s3] [rows]");
>             System.exit(0);
>         }
>         String path = "s3a://sdk/WriterOutput";
>         if (args.length > 3) {
>             path=args[3];
>         }
>         int num = 3;
>         if (args.length > 4) {
>             num = Integer.parseInt(args[4]);
>         }
>         Configuration conf = new Configuration();
>         conf.set(ACCESS_KEY,args[0]);
>         conf.set(SECRET_KEY,args[1]);
>         conf.set(ENDPOINT,args[2]);
> //        Field[] fields = new Field[2];
> //        fields[0] = new Field("name", DataTypes.STRING);
> //        fields[1] = new Field("age", DataTypes.INT);
> //        CarbonWriterBuilder builder = CarbonWriter.builder()
> //                .setAccessKey(args[0])
> //                .setSecretKey(args[1])
> //                .setEndPoint(args[2])
> //                .outputPath(path);
> //
> //        CarbonWriter writer = builder.buildWriterForCSVInput(new Schema(fields), conf);
> //
> //        for (int i = 0; i < num; i++) {
> //            writer.write(new String[]{"robot" + (i % 10), String.valueOf(i)});
> //        }
> //        writer.close();
>         // Read data
>         EqualToExpression equalToExpression = new EqualToExpression(
>             new ColumnExpression("name", DataTypes.STRING),
>             new LiteralExpression("robot1", DataTypes.STRING));
>         CarbonReader reader = CarbonReader
>             .builder(path, "_temp")
>             .projection(new String[]{"name", "age"})
>             .filter(equalToExpression)
>             .build(conf);
>         System.out.println("\nData:");
>         int i = 0;
>         while (i < 20 && reader.hasNext()) {
>             Object[] row = (Object[]) reader.readNextRow();
>             System.out.println(row[0] + " " + row[1]);
>             i++;
>         }
>         System.out.println("\nFinished");
>         reader.close();
>     }
> }
> {code}
> Exception:
> {code:java}
> log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell).
> log4j:WARN Please initialize the log4j system properly.
> log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
> Exception in thread "main" com.amazonaws.AmazonClientException: Unable to load AWS credentials from any provider in the chain
> at com.amazonaws.auth.AWSCredentialsProviderChain.getCredentials(AWSCredentialsProviderChain.java:117)
> at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3521)
> at com.amazonaws.services.s3.AmazonS3Client.headBucket(AmazonS3Client.java:1031)
> at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:994)
> at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:297)
> at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2653)
> at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:92)
> at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2687)
> at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2669)
> at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:371)
> at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
> at org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:74)
> at org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:66)
> at org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile.<init>(HDFSCarbonFile.java:41)
> at org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.<init>(S3CarbonFile.java:41)
> at org.apache.carbondata.core.datastore.impl.DefaultFileTypeProvider.getCarbonFile(DefaultFileTypeProvider.java:53)
> at org.apache.carbondata.core.datastore.impl.FileFactory.getCarbonFile(FileFactory.java:99)
> at org.apache.carbondata.core.metadata.schema.table.CarbonTable.buildTable(CarbonTable.java:241)
> at org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:191)
> at org.apache.carbondata.examples.sdk.SDKS3Example.main(SDKS3Example.java:91)
> Process finished with exit code 1
> {code}
> if configure like:
> {code:java}
>  CarbonReader reader = CarbonReader
>             .builder(path, "_temp")
>             .projection(new String[]{"name", "age"})
>             .filter(equalToExpression)
>             .setAccessKey(args[0])
>             .setSecretKey(args[1])
>             .setEndPoint(args[2])
>             .build(new Configuration(false));
> {code}
>    
> Exception2:
> {code:java}
> log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell).
> log4j:WARN Please initialize the log4j system properly.
> log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
> Exception in thread "main" com.amazonaws.AmazonClientException: Unable to execute HTTP request: sdk.obs.cn-north-1.myhwclouds.com: nodename nor servname provided, or not known
> at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:454)
> at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:232)
> at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3528)
> at com.amazonaws.services.s3.AmazonS3Client.headBucket(AmazonS3Client.java:1031)
> at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:994)
> at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:297)
> at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2653)
> at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:92)
> at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2687)
> at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2669)
> at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:371)
> at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
> at org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:74)
> at org.apache.carbondata.core.datastore.filesystem.AbstractDFSCarbonFile.<init>(AbstractDFSCarbonFile.java:66)
> at org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile.<init>(HDFSCarbonFile.java:41)
> at org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.<init>(S3CarbonFile.java:41)
> at org.apache.carbondata.core.datastore.impl.DefaultFileTypeProvider.getCarbonFile(DefaultFileTypeProvider.java:53)
> at org.apache.carbondata.core.datastore.impl.FileFactory.getCarbonFile(FileFactory.java:99)
> at org.apache.carbondata.core.metadata.schema.table.CarbonTable.buildTable(CarbonTable.java:241)
> at org.apache.carbondata.sdk.file.CarbonReaderBuilder.build(CarbonReaderBuilder.java:191)
> at org.apache.carbondata.examples.sdk.SDKS3Example.main(SDKS3Example.java:91)
> Caused by: java.net.UnknownHostException: sdk.obs.cn-north-1.myhwclouds.com: nodename nor servname provided, or not known
> at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
> at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928)
> at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
> at java.net.InetAddress.getAllByName0(InetAddress.java:1276)
> at java.net.InetAddress.getAllByName(InetAddress.java:1192)
> at java.net.InetAddress.getAllByName(InetAddress.java:1126)
> at org.apache.http.impl.conn.SystemDefaultDnsResolver.resolve(SystemDefaultDnsResolver.java:45)
> at org.apache.http.impl.conn.DefaultClientConnectionOperator.resolveHostname(DefaultClientConnectionOperator.java:278)
> at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:162)
> at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
> at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:641)
> at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
> at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
> at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
> at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:384)
> ... 20 more
> Process finished with exit code 1
> {code}

--
This message was sent by Atlassian JIRA
(v7.6.3#76005)