Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/1653 Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/1954/ --- |
In reply to this post by qiuchenjian-2
Github user manishgupta88 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1653#discussion_r156888264 --- Diff: integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala --- @@ -871,6 +871,32 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { throw new MalformedCarbonCommandException(errorMessage) } + // Validate QUOTECHAR length + if (options.exists(_._1.equalsIgnoreCase("QUOTECHAR"))) { + val quoteChar: String = options.get("quotechar").get.head._2 + if (quoteChar.length > 1) { + throw new MalformedCarbonCommandException("Quotation cannot be more than one character.") + } + } + + // Validate COMMENTCHAR length + if (options.exists(_._1.equalsIgnoreCase("COMMENTCHAR"))) { + val commentChar: String = options.get("commentchar").get.head._2 + if (commentChar.length > 1) { + throw new MalformedCarbonCommandException( + "Comment marker cannot be more than one character.") + } + } + + // Validate ESCAPECHAR length + if (options.exists(_._1.equalsIgnoreCase("ESCAPECHAR"))) { + val escapechar: String = options.get("escapechar").get.head._2 + if (escapechar.length > 1) { --- End diff -- \n, \t are valid escape sequences...add a method to validate valid and allowed escape sequences --- |
In reply to this post by qiuchenjian-2
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1653#discussion_r156946110 --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithMalformedCarbonCommandException.scala --- @@ -156,4 +156,40 @@ class TestLoadDataWithMalformedCarbonCommandException extends QueryTest with Bef case _: Throwable => assert(false) } } + + test("test load data with more than one char in quotechar option") { + try { --- End diff -- use intercept[MalformedCarbonCommandException] instead of try catch --- |
In reply to this post by qiuchenjian-2
Github user manishgupta88 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1653#discussion_r156958685 --- Diff: integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala --- @@ -932,6 +958,11 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { } } + def isEscapeSequence(loadOption: String): Boolean = { --- End diff -- Rename it to isValidEscapeSequence and also add one valid test case for validating a correct escape sequence --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/1653 Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/1970/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/1653 Build Success with Spark 2.2.0, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/743/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/1653 Build Success with Spark 2.2.0, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/746/ --- |
In reply to this post by qiuchenjian-2
Github user manishgupta88 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1653#discussion_r157009247 --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadOptions.scala --- @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.spark.testsuite.dataload + +import org.apache.spark.sql.test.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.spark.exception.MalformedCarbonCommandException + +class TestLoadOptions extends QueryTest with BeforeAndAfterAll{ + + override def beforeAll { + sql("drop table if exists TestLoadTableOptions") + sql("CREATE table TestLoadTableOptions (ID int, date String, country String, name String," + + "phonetype String, serialname String, salary int) stored by 'org.apache.carbondata.format'") + + } + + override def afterAll { + sql("drop table if exists TestLoadTableOptions") + } + + + test("test load data with more than one char in quotechar option") { + val errorMessage = intercept[MalformedCarbonCommandException] { + sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE " + + s"TestLoadTableOptions OPTIONS('QUOTECHAR'='\\\\')") + }.getMessage + assert(errorMessage.equals("Quotation cannot be more than one character.")) + } + + test("test load data with more than one char in commentchar option") { + val errorMessage = intercept[MalformedCarbonCommandException] { + sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE " + + s"TestLoadTableOptions OPTIONS('COMMENTCHAR'='##')") + assert(false) + }.getMessage + assert(errorMessage.equals("Comment marker cannot be more than one character.")) + } + + test("test load data with more than one char in escapechar option") { + val errorMessage = intercept[MalformedCarbonCommandException] { + sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE " + + s"TestLoadTableOptions OPTIONS('ESCAPECHAR'='\\\\')") + assert(false) + }.getMessage + assert(errorMessage.equals("Escape character cannot be more than one character.")) + } + + test("test load data with invalid escape sequence in quotechar option") { + val errorMessage = intercept[MalformedCarbonCommandException] { + sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE " + + s"TestLoadTableOptions OPTIONS('QUOTECHAR'='\\y')") + }.getMessage + assert(errorMessage.equals("Quotation cannot be more than one character.")) + } + + test("test load data with with valid escape sequence in quotechar option") { + sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/dataretention1.csv' INTO TABLE " + + s"TestLoadTableOptions OPTIONS('QUOTECHAR'='\\n')") --- End diff -- Replace quotechar with escapechar in above 2 test cases --- |
In reply to this post by qiuchenjian-2
Github user manishgupta88 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1653#discussion_r157009066 --- Diff: integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala --- @@ -871,6 +871,32 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser { throw new MalformedCarbonCommandException(errorMessage) } + // Validate QUOTECHAR length + if (options.exists(_._1.equalsIgnoreCase("QUOTECHAR"))) { + val quoteChar: String = options.get("quotechar").get.head._2 + if (quoteChar.length > 1 && !isValidEscapeSequence(quoteChar)) { --- End diff -- No need to validate quote character for a valid escape sequence --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/1653 Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/1975/ --- |
In reply to this post by qiuchenjian-2
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/1653 SDV Build Fail , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/2292/ --- |
In reply to this post by qiuchenjian-2
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/1653 SDV Build Success , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/2294/ --- |
In reply to this post by qiuchenjian-2
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/1653 SDV Build Success , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/2296/ --- |
In reply to this post by qiuchenjian-2
Github user dhatchayani commented on the issue:
https://github.com/apache/carbondata/pull/1653 retest sdv please --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/1653 Build Success with Spark 2.2.0, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/761/ --- |
In reply to this post by qiuchenjian-2
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/1653 SDV Build Success , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/2306/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/1653 Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/1990/ --- |
In reply to this post by qiuchenjian-2
Github user dhatchayani commented on the issue:
https://github.com/apache/carbondata/pull/1653 retest sdv please --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/1653 Build Success with Spark 2.2.0, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/767/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/1653 Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/1998/ --- |
Free forum by Nabble | Edit this page |