Apache CarbonData Dev Mailing List archive › Apache CarbonData JIRA issues

[GitHub] carbondata pull request #2433: [CARBONDATA-2676]Support local Dictionary for...

Classic

List

Threaded

45 messages Options

123

qiuchenjian-2

[GitHub] carbondata pull request #2433: [CARBONDATA-2676]Support local Dictionary for...

Github user BJangir commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/2433#discussion_r200820929

--- Diff: store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java ---
@@ -284,4 +284,137 @@ public void testSchemaPersistence() throws IOException {
FileUtils.deleteDirectory(new File(path));
}

+ @Test
+ public void testLocalDictionarywithTrue() throws Exception {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[3];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("surname", DataTypes.STRING);
+ fields[2] = new Field("age", DataTypes.INT);
+
+ CarbonWriterBuilder builder = CarbonWriter.builder().isTransactionalTable(false).sortBy(new String[]{"name"}).withBlockSize(12).isLocalDictionaryEnabled(true)
+ .uniqueIdentifier(System.currentTimeMillis()).taskNo(System.nanoTime()).outputPath(path);
+ CarbonWriter carbonWriter = builder.buildWriterForCSVInput(new Schema(fields));
+ for (int i = 0; i < 100; i++) {
+ carbonWriter.write(new String[]{"robot" + (i % 10),"robot_surname" + (i % 10), String.valueOf(i)});
+ }
+ carbonWriter.close();
+
+ File segmentFolder = new File(path);
+ Assert.assertTrue(segmentFolder.exists());
+
+ File[] dataFiles = segmentFolder.listFiles(new FileFilter() {
+ @Override public boolean accept(File pathname) {
+ return pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT);
+ }
+ });
+ Assert.assertNotNull(dataFiles);
+ Assert.assertTrue(dataFiles.length > 0);
+
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
+ public void testLocalDictionarywithFalseOption() throws Exception {
+ String path = "./testWriteFiles";
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[3];
+ fields[0] = new Field("name", DataTypes.STRING);
+ fields[1] = new Field("surname", DataTypes.STRING);
+ fields[2] = new Field("age", DataTypes.INT);
+
+ CarbonWriterBuilder builder = CarbonWriter.builder().isTransactionalTable(false).sortBy(new String[]{"name"}).withBlockSize(12).isLocalDictionaryEnabled(false)
+ .uniqueIdentifier(System.currentTimeMillis()).taskNo(System.nanoTime()).outputPath(path);
+ CarbonWriter carbonWriter = builder.buildWriterForCSVInput(new Schema(fields));
+ for (int i = 0; i < 100; i++) {
+ carbonWriter.write(new String[]{"robot" + (i % 10),"robot_surname" + (i % 10), String.valueOf(i)});
+ }
+ carbonWriter.close();
+
+ File segmentFolder = new File(path);
+ Assert.assertTrue(segmentFolder.exists());
+
+ File[] dataFiles = segmentFolder.listFiles(new FileFilter() {
+ @Override public boolean accept(File pathname) {
+ return pathname.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT);
+ }
+ });
+ Assert.assertNotNull(dataFiles);
+ Assert.assertTrue(dataFiles.length > 0);
+
+
+ FileUtils.deleteDirectory(new File(path));
+ }
+
+ @Test
+ public void testLocalDictionarywithThreshold() throws Exception {
--- End diff --

Add Validation whether DataChunk has dictionary or not.

---

qiuchenjian-2