nihal0107 commented on a change in pull request #3819:
URL:
https://github.com/apache/carbondata/pull/3819#discussion_r467936238##########
File path: sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
##########
@@ -660,13 +1102,41 @@ public CarbonWriter build() throws IOException, InvalidLoadOptionException {
// removed from the load. LoadWithoutConverter flag is going to point to the Loader Builder
// which will skip Conversion Step.
loadModel.setLoadWithoutConverterStep(true);
- return new AvroCarbonWriter(loadModel, hadoopConf, this.avroSchema);
+ AvroCarbonWriter avroCarbonWriter = new AvroCarbonWriter(loadModel,
+ hadoopConf, this.avroSchema);
+ if (this.filePath != null && this.filePath.length() != 0) {
+ avroCarbonWriter.setDataFiles(this.dataFiles);
+ }
+ return avroCarbonWriter;
} else if (this.writerType == WRITER_TYPE.JSON) {
loadModel.setJsonFileLoad(true);
- return new JsonCarbonWriter(loadModel, hadoopConf);
+ JsonCarbonWriter jsonCarbonWriter = new JsonCarbonWriter(loadModel, hadoopConf);
+ if (this.filePath != null && this.filePath.length() != 0) {
+ jsonCarbonWriter.setDataFiles(this.dataFiles);
+ }
+ return jsonCarbonWriter;
+ } else if (this.writerType == WRITER_TYPE.PARQUET) {
+ loadModel.setLoadWithoutConverterStep(true);
+ AvroCarbonWriter avroCarbonWriter = new AvroCarbonWriter(loadModel,
+ hadoopConf, this.avroSchema);
+ ParquetCarbonWriter parquetCarbonWriter = new ParquetCarbonWriter(avroCarbonWriter);
+ parquetCarbonWriter.setDataFiles(this.dataFiles);
+ return parquetCarbonWriter;
+ } else if (this.writerType == WRITER_TYPE.ORC) {
+ CSVCarbonWriter csvCarbonWriter = new CSVCarbonWriter(loadModel, hadoopConf);
+ ORCCarbonWriter orcCarbonWriter = new ORCCarbonWriter(csvCarbonWriter);
+ orcCarbonWriter.setDataFiles(this.dataFiles);
+ return orcCarbonWriter;
} else {
// CSV
- return new CSVCarbonWriter(loadModel, hadoopConf);
+ CSVCarbonWriter csvCarbonWriter = new CSVCarbonWriter(loadModel, hadoopConf);
+ if (this.filePath != null && this.filePath.length() != 0) {
+ csvCarbonWriter.setDataFiles(this.dataFiles);
+ if (!this.options.containsKey("fileHeader")) {
Review comment:
Done
##########
File path: sdk/sdk/src/main/java/org/apache/carbondata/sdk/file/JsonCarbonWriter.java
##########
@@ -91,4 +102,39 @@ public void close() throws IOException {
throw new IOException(e);
}
}
+
+ private void loadSingleFile(File file) throws IOException {
+ try {
+ Reader reader = SDKUtil.buildJsonReader(file);
+ JSONParser jsonParser = new JSONParser();
+ Object jsonRecord = jsonParser.parse(reader);
+ if (jsonRecord instanceof JSONArray) {
+ JSONArray jsonArray = (JSONArray) jsonRecord;
+ for (Object record : jsonArray) {
+ this.write(record.toString());
+ }
+ } else {
+ this.write(jsonRecord.toString());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
Review comment:
Done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[hidden email]
Locked
