Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1818/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Failed with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2029/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Failed with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10078/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1823/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Success with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2033/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Success with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10081/ --- |
In reply to this post by qiuchenjian-2
Github user KanakaKumar commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2919#discussion_r242577513 --- Diff: store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonSchemaReaderTest.java --- @@ -236,6 +243,113 @@ public void testReadSchemaWithDifferentSchema() { } } + @Test + public void testGetVersionDetailsAndValidate() { + try { + String versionDetails = CarbonSchemaReader + .getVersionDetails(path); + Assert.assertTrue(versionDetails.contains("CarbonSchemaReaderTest in version: ")); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testGetVersionDetailsWithoutValidate() { + try { + String versionDetails = CarbonSchemaReader + .getVersionDetails(path); + Assert.assertTrue(versionDetails.contains("CarbonSchemaReaderTest in version: ")); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testGetVersionDetailsWithCarbonDataFile() { + try { + File[] dataFiles = new File(path).listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(CARBON_DATA_EXT); + } + }); + String versionDetails = CarbonSchemaReader.getVersionDetails(dataFiles[0].getAbsolutePath()); + Assert.assertTrue(versionDetails.contains("CarbonSchemaReaderTest in version: ")); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testGetVersionDetailsWithCarbonIndexFile() { + try { + File[] indexFiles = new File(path).listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(INDEX_FILE_EXT); + } + }); + CarbonSchemaReader.getVersionDetails(indexFiles[0].getAbsolutePath()); + Assert.fail(); + } catch (Throwable e) { + Assert.assertTrue(e.getMessage() + .equalsIgnoreCase("Can't get version details from carbonindex file.")); + } + } + + public void testGetVersionDetailsWithTheSameSchema() { + try { + writeData(); + try { + String versionDetails = CarbonSchemaReader + .getVersionDetails(path); + Assert.assertTrue(versionDetails + .contains("CarbonSchemaReaderTest in version: ")); + } catch (Exception e) { + Assert.fail(); + } + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testGetVersionDetailsWithDifferentSchema() { + try { + int num = 10; + Field[] fields = new Field[2]; + fields[0] = new Field("name", DataTypes.STRING); + fields[1] = new Field("age", DataTypes.INT); + CarbonWriter writer = CarbonWriter + .builder() + .outputPath(path) + .withCsvInput(new Schema(fields)) + .writtenBy("testReadSchemaWithDifferentSchema") + .build(); + + for (int i = 0; i < num; i++) { + writer.write(new String[]{"robot" + (i % 10), String.valueOf(i)}); + } + writer.close(); + try { + CarbonSchemaReader + .getVersionDetails(path); + } catch (Exception e) { + Assert.assertTrue(e.getMessage() + .equalsIgnoreCase("Version details is different between different files.")); + Assert.fail(); --- End diff -- What is the purpose of this test case? "Version details is different between different files" exception we never get now right? If thee is no separate validation required, we can remove this test. --- |
In reply to this post by qiuchenjian-2
Github user KanakaKumar commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2919#discussion_r242577599 --- Diff: store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java --- @@ -241,12 +241,25 @@ private static Schema readSchemaFromIndexFile(String indexFilePath) throws IOExc /** * This method return the version details in formatted string by reading from carbondata file + * default won't validate the version details between different carbondata files. * - * @param dataFilePath - * @return + * @param path carbondata file path or folder path + * @return string with information of who has written this file + * in which carbondata project version * @throws IOException */ - public static String getVersionDetails(String dataFilePath) throws IOException { + public static String getVersionDetails(String path) throws IOException { + if (path.endsWith(INDEX_FILE_EXT)) { + throw new RuntimeException("Can't get version details from carbonindex file."); + } else if (path.endsWith(CARBON_DATA_EXT)) { + return getVersionDetailsFromDataFile(path); + } else { + String indexFilePath = getCarbonFile(path, CARBON_DATA_EXT)[0].getAbsolutePath(); --- End diff -- this is data file --- |
In reply to this post by qiuchenjian-2
Github user KanakaKumar commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2919#discussion_r242578856 --- Diff: docs/sdk-guide.md --- @@ -816,13 +816,15 @@ Find example code at [CarbonReaderExample](https://github.com/apache/carbondata/ * This method return the version details in formatted string by reading from carbondata file * If application name is SDK_1.0.0 and this has written the carbondata file in carbondata 1.6 project version, * then this API returns the String "SDK_1.0.0 in version: 1.6.0-SNAPSHOT" - * @param dataFilePath complete path including carbondata file name + * Default value of validate is false, it won't validate the version details between different carbondata files. --- End diff -- " Default value of validate is false," - This statement is not valid as we don't have validate and no validate flows differently. --- |
In reply to this post by qiuchenjian-2
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2919#discussion_r242583087 --- Diff: store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonSchemaReaderTest.java --- @@ -236,6 +243,113 @@ public void testReadSchemaWithDifferentSchema() { } } + @Test + public void testGetVersionDetailsAndValidate() { + try { + String versionDetails = CarbonSchemaReader + .getVersionDetails(path); + Assert.assertTrue(versionDetails.contains("CarbonSchemaReaderTest in version: ")); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testGetVersionDetailsWithoutValidate() { + try { + String versionDetails = CarbonSchemaReader + .getVersionDetails(path); + Assert.assertTrue(versionDetails.contains("CarbonSchemaReaderTest in version: ")); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testGetVersionDetailsWithCarbonDataFile() { + try { + File[] dataFiles = new File(path).listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(CARBON_DATA_EXT); + } + }); + String versionDetails = CarbonSchemaReader.getVersionDetails(dataFiles[0].getAbsolutePath()); + Assert.assertTrue(versionDetails.contains("CarbonSchemaReaderTest in version: ")); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testGetVersionDetailsWithCarbonIndexFile() { + try { + File[] indexFiles = new File(path).listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(INDEX_FILE_EXT); + } + }); + CarbonSchemaReader.getVersionDetails(indexFiles[0].getAbsolutePath()); + Assert.fail(); + } catch (Throwable e) { + Assert.assertTrue(e.getMessage() + .equalsIgnoreCase("Can't get version details from carbonindex file.")); + } + } + + public void testGetVersionDetailsWithTheSameSchema() { + try { + writeData(); + try { + String versionDetails = CarbonSchemaReader + .getVersionDetails(path); + Assert.assertTrue(versionDetails + .contains("CarbonSchemaReaderTest in version: ")); + } catch (Exception e) { + Assert.fail(); + } + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testGetVersionDetailsWithDifferentSchema() { + try { + int num = 10; + Field[] fields = new Field[2]; + fields[0] = new Field("name", DataTypes.STRING); + fields[1] = new Field("age", DataTypes.INT); + CarbonWriter writer = CarbonWriter + .builder() + .outputPath(path) + .withCsvInput(new Schema(fields)) + .writtenBy("testReadSchemaWithDifferentSchema") + .build(); + + for (int i = 0; i < num; i++) { + writer.write(new String[]{"robot" + (i % 10), String.valueOf(i)}); + } + writer.close(); + try { + CarbonSchemaReader + .getVersionDetails(path); + } catch (Exception e) { + Assert.assertTrue(e.getMessage() + .equalsIgnoreCase("Version details is different between different files.")); + Assert.fail(); --- End diff -- ok, removed. --- |
In reply to this post by qiuchenjian-2
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2919#discussion_r242583526 --- Diff: store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java --- @@ -241,12 +241,25 @@ private static Schema readSchemaFromIndexFile(String indexFilePath) throws IOExc /** * This method return the version details in formatted string by reading from carbondata file + * default won't validate the version details between different carbondata files. * - * @param dataFilePath - * @return + * @param path carbondata file path or folder path + * @return string with information of who has written this file + * in which carbondata project version * @throws IOException */ - public static String getVersionDetails(String dataFilePath) throws IOException { + public static String getVersionDetails(String path) throws IOException { + if (path.endsWith(INDEX_FILE_EXT)) { + throw new RuntimeException("Can't get version details from carbonindex file."); + } else if (path.endsWith(CARBON_DATA_EXT)) { + return getVersionDetailsFromDataFile(path); + } else { + String indexFilePath = getCarbonFile(path, CARBON_DATA_EXT)[0].getAbsolutePath(); --- End diff -- ok, change the name to dataFilePath --- |
In reply to this post by qiuchenjian-2
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2919#discussion_r242583802 --- Diff: docs/sdk-guide.md --- @@ -816,13 +816,15 @@ Find example code at [CarbonReaderExample](https://github.com/apache/carbondata/ * This method return the version details in formatted string by reading from carbondata file * If application name is SDK_1.0.0 and this has written the carbondata file in carbondata 1.6 project version, * then this API returns the String "SDK_1.0.0 in version: 1.6.0-SNAPSHOT" - * @param dataFilePath complete path including carbondata file name + * Default value of validate is false, it won't validate the version details between different carbondata files. --- End diff -- ok, removed --- |
In reply to this post by qiuchenjian-2
Github user KanakaKumar commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2919#discussion_r242588569 --- Diff: store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java --- @@ -241,12 +241,25 @@ private static Schema readSchemaFromIndexFile(String indexFilePath) throws IOExc /** * This method return the version details in formatted string by reading from carbondata file + * default won't validate the version details between different carbondata files. * - * @param dataFilePath - * @return + * @param path carbondata file path or folder path + * @return string with information of who has written this file + * in which carbondata project version * @throws IOException */ - public static String getVersionDetails(String dataFilePath) throws IOException { + public static String getVersionDetails(String path) throws IOException { + if (path.endsWith(INDEX_FILE_EXT)) { + throw new RuntimeException("Can't get version details from carbonindex file."); --- End diff -- RuntimeException is mentioned in the signature. Should throw IOException. Also some places in this class throws CarbonDataLoadingException but if not intentional we can change them to IOException as per signature instead of a subclass of RuntimeException (if it was not intentional) --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1833/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Success with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10090/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Failed with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2041/ --- |
In reply to this post by qiuchenjian-2
Github user xubo245 commented on the issue:
https://github.com/apache/carbondata/pull/2919 retest this please --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1838/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1839/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2919 Build Failed with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2047/ --- |
Free forum by Nabble | Edit this page |