[GitHub] [carbondata] xubo245 commented on a change in pull request #3819: [CARBONDATA-3855]support carbon SDK to load data from different files

[GitBox]

xubo245 commented on a change in pull request #3819:
URL: https://github.com/apache/carbondata/pull/3819#discussion_r464181249



##########
File path: sdk/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java
##########
@@ -846,4 +849,158 @@ public void testWritingAndReadingArrayStruct() throws IOException {
     }
   }
 
+  @Test
+  public void testCsvLoadAndCarbonReadWithPrimitiveType() throws IOException, InvalidLoadOptionException, InterruptedException {
+    String path = "./testCsvFileLoad";
+    String filePath = "./src/test/resources/file/csv_files/primitive_data.csv";
+    FileUtils.deleteDirectory(new File(path));
+    CarbonWriterBuilder carbonWriterBuilder = new CarbonWriterBuilder();
+    Field fields[] = new Field[4];
+    fields[0] = new Field("id", "INT");
+    fields[1] = new Field("country", "STRING");
+    fields[2] = new Field("name", "STRING");
+    fields[3] = new Field("salary", "INT");
+    CarbonWriter carbonWriter = carbonWriterBuilder.withCsvInput(new Schema(fields)).
+        withCsvPath(filePath).outputPath(path).writtenBy("CSVCarbonWriter").build();
+    carbonWriter.write();
+    carbonWriter.close();
+    File[] dataFiles = new File(path).listFiles();
+    assert (Objects.requireNonNull(dataFiles).length == 2);
+
+    CarbonReader reader = CarbonReader.builder("./testCsvFileLoad", "_temp")
+            .projection(new String[]{"id", "country", "name", "salary"}).build();
+    int rowCount = 0;
+    while (reader.hasNext()) {
+      Object[] row = (Object[]) reader.readNextRow();
+      rowCount++;
+      Assert.assertEquals(row[0], rowCount);
+      Assert.assertEquals(row[1], "china");
+      Assert.assertEquals(row[2], "aaa" + rowCount);
+      Assert.assertEquals(row[3], 14999 + rowCount);
+    }
+    assert (rowCount == 10);
+    FileUtils.deleteDirectory(new File(path));
+  }
+
+  @Test
+  public void testCsvLoadAndCarbonReadWithComplexType() throws IOException, InterruptedException, InvalidLoadOptionException {
+    String path = "./testCsvFileLoad";
+    String filePath = "../../examples/spark/src/main/resources/data.csv";
+    FileUtils.deleteDirectory(new File(path));
+    CarbonWriterBuilder carbonWriterBuilder = new CarbonWriterBuilder();
+    Field fields[] = new Field[11];
+    fields[0] = new Field("shortField", "SHORT");
+    fields[1] = new Field("intField", "INT");
+    fields[2] = new Field("bigintField", "LONG");
+    fields[3] = new Field("doubleField", "DOUBLE");
+    fields[4] = new Field("stringField", "STRING");
+    fields[5] = new Field("timestampfield", "TIMESTAMP");
+    fields[6] = new Field("decimalField", "DECIMAL");
+    fields[7] = new Field("datefield", "DATE");
+    fields[8] = new Field("charField", "VARCHAR");
+    fields[9] = new Field("floatField", "FLOAT");
+
+    StructField[] structFields = new StructField[3];
+    structFields[0] = new StructField("fooField", DataTypes.STRING);
+    structFields[1] = new StructField("barField", DataTypes.STRING);
+    structFields[2] = new StructField("worldField", DataTypes.STRING);
+    Field structType = new Field("structField", "struct", Arrays.asList(structFields));
+
+    fields[10] = structType;
+    Map<String, String> options = new HashMap<>();
+    options.put("timestampformat", "yyyy/MM/dd HH:mm:ss");
+    options.put("dateformat", "yyyy/MM/dd");
+    options.put("complex_delimiter_level_1", "#");
+    CarbonWriter carbonWriter = carbonWriterBuilder.withCsvInput(new Schema(fields)).
+        withCsvPath(filePath).outputPath(path).writtenBy("CSVCarbonWriter").withLoadOptions(options).build();
+    carbonWriter.write();
+    carbonWriter.close();
+    File[] dataFiles = new File(path).listFiles();
+    assert (Objects.requireNonNull(dataFiles).length == 2);
+
+    CarbonReader reader = CarbonReader.builder("./testCsvFileLoad", "_temp")
+            .projection(new String[]{"structfield"}).build();
+    int rowCount = 0;
+    while (reader.hasNext()) {
+      Object[] row = (Object[]) reader.readNextRow();
+      Object[] structCol = (Object[]) row[0];
+      assert (structCol.length == 3);
+      Assert.assertEquals(structCol[0], "'foo'");
+      Assert.assertEquals(structCol[1], "'bar'");
+      Assert.assertEquals(structCol[2], "'world'");
+      rowCount++;
+    }
+    assert (rowCount == 10);
+    FileUtils.deleteDirectory(new File(path));
+  }
+
+  @Test
+  public void testMultipleCsvFileLoad() throws IOException, InvalidLoadOptionException, InterruptedException {
+    String path = "./testCsvFileLoad";
+    String filePath = "./src/test/resources/file/csv_files";
+    FileUtils.deleteDirectory(new File(path));
+    CarbonWriterBuilder carbonWriterBuilder = new CarbonWriterBuilder();
+    Field fields[] = new Field[4];
+    fields[0] = new Field("id", "INT");
+    fields[1] = new Field("country", "STRING");
+    fields[2] = new Field("name", "STRING");
+    fields[3] = new Field("salary", "INT");
+    CarbonWriter carbonWriter = carbonWriterBuilder.withCsvInput(new Schema(fields)).
+        withCsvPath(filePath).outputPath(path).writtenBy("CSVCarbonWriter").build();
+    carbonWriter.write();
+    carbonWriter.close();
+    File[] dataFiles = new File(path).listFiles();
+    assert (Objects.requireNonNull(dataFiles).length == 2);
+
+    CarbonReader reader = CarbonReader.builder("./testCsvFileLoad", "_temp")
+            .projection(new String[]{"id", "country", "name", "salary"}).build();
+    int rowCount = 0;
+    while (reader.hasNext()) {
+      Object[] row = (Object[]) reader.readNextRow();
+      rowCount++;
+      Assert.assertEquals(row[0], rowCount);
+      Assert.assertEquals(row[1], "china");
+      Assert.assertEquals(row[2], "aaa" + rowCount);
+      Assert.assertEquals(row[3], 14999 + rowCount);
+    }
+    assert (rowCount == 30);
+    FileUtils.deleteDirectory(new File(path));
+  }
+
+  @Test
+  public void testSelectedCsvFileLoadInDirectory() throws IOException,
+      InvalidLoadOptionException, InterruptedException {
+    String path = "./testCsvFileLoad";
+    String filePath = "./src/test/resources/file/csv_files";
+    FileUtils.deleteDirectory(new File(path));
+    CarbonWriterBuilder carbonWriterBuilder = new CarbonWriterBuilder();
+    Field fields[] = new Field[4];
+    fields[0] = new Field("id", "INT");
+    fields[1] = new Field("country", "STRING");
+    fields[2] = new Field("name", "STRING");
+    fields[3] = new Field("salary", "INT");
+    List<String> fileList = new ArrayList<>();
+    fileList.add("primitive_data_2.csv");
+    fileList.add("primitive_data_3.csv");
+    CarbonWriter carbonWriter = carbonWriterBuilder.withCsvInput(new Schema(fields)).
+        withCsvPath(filePath, fileList).outputPath(path).writtenBy("CSVCarbonWriter").build();
+    carbonWriter.write();
+    carbonWriter.close();
+    File[] dataFiles = new File(path).listFiles();
+    assert (Objects.requireNonNull(dataFiles).length == 2);
+
+    CarbonReader reader = CarbonReader.builder("./testCsvFileLoad", "_temp")

Review comment:
       pleae close reader  when finihed.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
[hidden email]