[GitHub] carbondata pull request #2422: [CARBONDATA-2585][CARBONDATA-2586][Local Dict...

[qiuchenjian-2]

Github user praveenmeenakshi56 commented on the issue:

    https://github.com/apache/carbondata/pull/2422
 
    retest this please


---

[qiuchenjian-2]

Github user ravipesala commented on the issue:

    https://github.com/apache/carbondata/pull/2422
 
    SDV Build Success , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/5599/



---

[qiuchenjian-2]

Github user CarbonDataQA commented on the issue:

    https://github.com/apache/carbondata/pull/2422
 
    Build Success with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/5583/



---

[qiuchenjian-2]

Github user CarbonDataQA commented on the issue:

    https://github.com/apache/carbondata/pull/2422
 
    Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/6766/



---

[qiuchenjian-2]

Github user praveenmeenakshi56 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/2422#discussion_r200237535
 
    --- Diff: integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala ---
    @@ -391,7 +495,7 @@ object TableNewProcessor {
         if (dataType == DataTypes.DATE) {
           encoders.add(Encoding.DIRECT_DICTIONARY)
         }
    -    if (dataType == DataTypes.TIMESTAMP && ! highCardinalityDims.contains(colName)) {
    --- End diff --
   
    these are all handled in #2450


---

[qiuchenjian-2]

Github user praveenmeenakshi56 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/2422#discussion_r200237575
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala ---
    @@ -0,0 +1,1183 @@
    +package org.apache.carbondata.spark.testsuite.localdictionary
    +
    +import org.apache.spark.sql.test.util.QueryTest
    +import org.scalatest.BeforeAndAfterAll
    +
    +import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
    +
    +class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfterAll{
    +
    +  override protected def beforeAll(): Unit = {
    +    sql("DROP TABLE IF EXISTS LOCAL1")
    +  }
    +
    +  test("test alter table add column") {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true',
    +        | 'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name')
    +      """.stripMargin)
    +    sql("alter table local1 add columns (alt string) tblproperties('local_dictionary_include'='alt')")
    +    val descLoc = sql("describe formatted local1").collect
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("20000"))
    +    }
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city,alt"))
    +    }
    +  }
    +
    +  test("test alter table add column default configs for local dictionary") {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true',
    +        | 'local_dictionary_threshold'='20000','no_inverted_index'='name')
    +      """.stripMargin)
    +    sql("alter table local1 add columns (alt string)")
    +    val descLoc = sql("describe formatted local1").collect
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("20000"))
    +    }
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name,city,alt"))
    +    }
    +  }
    +
    +  test("test alter table add column where same column is in dictionary include and local dictionary include") {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true',
    +        | 'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name')
    +      """.stripMargin)
    +    val exception = intercept[MalformedCarbonCommandException] {
    +      sql(
    +        "alter table local1 add columns (alt string) tblproperties('local_dictionary_include'='alt','dictionary_include'='alt')")
    +    }
    +    assert(exception.getMessage
    +      .contains(
    +        "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: alt specified in Dictionary " +
    +        "include. Local Dictionary will not be generated for Dictionary include columns. " +
    +        "Please check the DDL."))
    +  }
    +
    +  test("test alter table add column where duplicate columns present in local dictionary include") {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true',
    +        | 'local_dictionary_threshold'='20000','local_dictionary_include'='city','no_inverted_index'='name')
    +      """.stripMargin)
    +    val exception = intercept[MalformedCarbonCommandException] {
    +      sql(
    +        "alter table local1 add columns (alt string) tblproperties('local_dictionary_include'='alt,alt')")
    +    }
    +    assert(exception.getMessage
    +      .contains(
    +        "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE contains Duplicate Columns: alt. " +
    +        "Please check the DDL."))
    +  }
    +
    +  test("test alter table add column where duplicate columns present in local dictionary include/exclude")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true',
    +        | 'local_dictionary_threshold'='20000','local_dictionary_include'='city',
    +        | 'no_inverted_index'='name')
    +      """.stripMargin)
    +    val exception1 = intercept[MalformedCarbonCommandException] {
    +      sql(
    +        "alter table local1 add columns (alt string) tblproperties" +
    +        "('local_dictionary_include'='abc')")
    +    }
    +    assert(exception1.getMessage
    +      .contains(
    +        "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " +
    +        "Please check the DDL."))
    +    val exception2 = intercept[MalformedCarbonCommandException] {
    +      sql(
    +        "alter table local1 add columns (alt string) tblproperties" +
    +        "('local_dictionary_exclude'='abc')")
    +    }
    +    assert(exception2.getMessage
    +      .contains(
    +        "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " +
    +        "Please check the DDL."))
    +  }
    +
    +  test("test alter table add column for datatype validation")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """ | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true',
    +        | 'local_dictionary_include'='city', 'no_inverted_index'='name')
    +      """.stripMargin)
    +    val exception = intercept[MalformedCarbonCommandException] {
    +      sql(
    +        "alter table local1 add columns (alt string,abc int) tblproperties" +
    +        "('local_dictionary_include'='abc')")
    +    }
    +    assert(exception.getMessage
    +      .contains(
    +        "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc is not a String/complex " +
    +        "datatype column. LOCAL_DICTIONARY_COLUMN should be no dictionary string/complex datatype" +
    +        " column.Please check the DDL."))
    +  }
    +
    +  test("test alter table add column where duplicate columns are present in local dictionary include and exclude")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true',
    +        | 'local_dictionary_include'='city', 'no_inverted_index'='name')
    +      """.stripMargin)
    +    val exception = intercept[MalformedCarbonCommandException] {
    +      sql(
    +        "alter table local1 add columns (alt string,abc string) tblproperties" +
    +        "('local_dictionary_include'='abc','local_dictionary_exclude'='alt,abc')")
    +    }
    +    assert(exception.getMessage
    +      .contains(
    +        "Column ambiguity as duplicate column(s):abc is present in LOCAL_DICTIONARY_INCLUDE " +
    +        "and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed."))
    +  }
    +
    +  test("test alter table add column unsupported table property")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true',
    +        | 'local_dictionary_include'='city', 'no_inverted_index'='name')
    +      """.stripMargin)
    +    val exception = intercept[MalformedCarbonCommandException] {
    +      sql(
    +        "alter table local1 add columns (alt string,abc string) tblproperties" +
    +        "('local_dictionary_enable'='abc')")
    +    }
    +    assert(exception.getMessage
    +      .contains(
    +        "Unsupported Table property in add column: local_dictionary_enable"))
    +    val exception1 = intercept[MalformedCarbonCommandException] {
    +      sql(
    +        "alter table local1 add columns (alt string,abc string) tblproperties" +
    +        "('local_dictionary_threshold'='10000')")
    +    }
    +    assert(exception1.getMessage
    +      .contains(
    +        "Unsupported Table property in add column: local_dictionary_threshold"))
    +  }
    +
    +  test("test alter table add column when main table is disabled for local dictionary")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='false',
    +        | 'local_dictionary_include'='city', 'no_inverted_index'='name')
    +      """.stripMargin)
    +    sql(
    +      "alter table local1 add columns (alt string,abc string) tblproperties" +
    +      "('local_dictionary_include'='abc')")
    +    val descLoc = sql("describe formatted local1").collect
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("false"))
    +    }
    +
    +    checkExistence(sql("DESC FORMATTED local1"), false,
    +      "Local Dictionary Include")
    +  }
    +
    +  test("test local dictionary threshold for boundary values") {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_threshold'='300000')
    +      """.stripMargin)
    +    val descLoc = sql("describe formatted local1").collect
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("10000"))
    +    }
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_threshold'='500')
    +      """.stripMargin)
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("10000"))
    +    }
    +  }
    +
    +  test("test alter table add column for local dictionary include and exclude configs")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true',
    +        | 'local_dictionary_include'='city', 'no_inverted_index'='name')
    +      """.stripMargin)
    +    sql(
    +      "alter table local1 add columns (alt string,abc string) tblproperties" +
    +      "('local_dictionary_include'='abc','local_dictionary_exclude'='alt')")
    +    val descLoc = sql("describe formatted local1").collect
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city,abc"))
    +    }
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("alt"))
    +    }
    +  }
    +
    +  test("test local dictionary foer varchar datatype columns") {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_include'='city',
    +        | 'LONG_STRING_COLUMNS'='city')
    +      """.stripMargin)
    +    val descLoc = sql("describe formatted local1").collect
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city"))
    +    }
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("10000"))
    +    }
    +  }
    +
    +  test("test local dictionary describe formatted only with default configs")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata'
    +      """.stripMargin)
    +
    +    val descLoc = sql("describe formatted local1").collect
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("10000"))
    +    }
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name,city"))
    +    }
    +  }
    +
    +  test("test local dictionary for invalid threshold")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_threshold'='300000')
    +      """.stripMargin)
    +
    +    val descLoc = sql("describe formatted local1").collect
    +    descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("10000"))
    +    }
    +  }
    +
    +  test("test alter set for local dictionary enable to disable")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_threshold'='300000')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("10000"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_enable'='false')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("false"))
    +    }
    +    checkExistence(sql("DESC FORMATTED local1"), false,
    +      "Local Dictionary Threshold")
    +  }
    +
    +  test("test alter set for local dictionary _001")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata'
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_threshold'='30000')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("30000"))
    +    }
    +  }
    +
    +
    +  test("test alter set for local dictionary _002")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata'
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_include'='name')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name") && !row.get(1).toString.contains("city"))
    +    }
    +  }
    +
    +  test("test alter set for local dictionary _003")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata'
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_exclude'='name')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name"))
    +    }
    +  }
    +
    +  test("test alter set for local dictionary _004")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_include'='city')")
    +
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city"))
    +    }
    +  }
    +
    +  test("test alter set for local dictionary _005")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_include'='name,city')")
    +
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name,city"))
    +    }
    +  }
    +
    +  test("test alter set for local dictionary _006")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_exclude'='city')")
    +
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city"))
    +    }
    +  }
    +
    +  test("test alter set for local dictionary _007")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    val exception1 = intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_include'='city, ')")
    +    }
    +    assert(exception1.getMessage
    +      .contains(
    +        "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column:  does not exist in table. " +
    +        "Please check the DDL."))
    +  }
    +
    +  test("test alter set for local dictionary _008")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    val exception1 = intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='name')")
    +    }
    +    assert(exception1.getMessage.contains("Column ambiguity as duplicate column(s):name is present in LOCAL_DICTIONARY_INCLUDE and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed."))
    +  }
    +
    +  test("test alter set for local dictionary _009")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    val exception1 = intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='id')")
    +    }
    +    assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE should be no dictionary string/complex datatype column."))
    +  }
    +
    +  test("test alter set for local dictionary _010")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('dictionary_include'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    val exception1 = intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_include'='name')")
    +    }
    +    assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: name specified in Dictionary include. Local Dictionary will not be generated for Dictionary include columns. Please check the DDL."))
    +  }
    +
    +  test("test alter set for local dictionary _011")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_exclude'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_exclude'='city')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city"))
    +    }
    +  }
    +
    +  test("test alter set for local dictionary _012")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_exclude'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    val exception1 = intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='city, ')")
    +    }
    +    assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column:  does not exist in table. Please check the DDL."))
    +  }
    +
    +  test("test alter set for local dictionary _013")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('dictionary_include'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    val exception1 = intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='name')")
    +    }
    +    assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: name specified in Dictionary include. Local Dictionary will not be generated for Dictionary include columns. Please check the DDL."))
    +  }
    +
    +  test("test alter set for local dictionary _014")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    val exception1 = intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='name')")
    +    }
    +    assert(exception1.getMessage.contains("Column ambiguity as duplicate column(s):name is present in LOCAL_DICTIONARY_INCLUDE and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed."))
    +  }
    +
    +  test("test alter set for local dictionary _015")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_enable'='false')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("false"))
    +    }
    +  }
    +
    +  test("test alter set for local dictionary _016")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='false')
    +      """.stripMargin)
    +
    +    sql("alter table local1 set tblproperties('local_dictionary_include'='name')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("false"))
    +    }
    +  }
    +
    +  test("test alter set for local dictionary _017")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name','local_dictionary_exclude'='city')
    +      """.stripMargin)
    +
    +    val exception1 = intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='name')")
    +    }
    +    assert(exception1.getMessage.contains("Column ambiguity as duplicate column(s):name is present in LOCAL_DICTIONARY_INCLUDE and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed."))
    +  }
    +
    +  test("test alter unset for local dictionary _001")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name','local_dictionary_exclude'='city')
    +      """.stripMargin)
    +
    +    sql("alter table local1 unset tblproperties('local_dictionary_exclude')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name"))
    +    }
    +  }
    +
    +  test("test alter unset for local dictionary _002")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name','local_dictionary_exclude'='city')
    +      """.stripMargin)
    +
    +    sql("alter table local1 unset tblproperties('local_dictionary_exclude')")
    +    sql("alter table local1 set tblproperties('local_dictionary_exclude'='city')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city"))
    +    }
    +  }
    +
    +  test("test alter set for local dictionary disable to enable")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='false','local_dictionary_threshold'='300000')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("false"))
    +    }
    +    checkExistence(sql("DESC FORMATTED local1"), false,
    +      "Local Dictionary Threshold")
    +    sql("alter table local1 set tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='30000')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("30000"))
    +    }
    +  }
    +
    +  test("test alter set for local dictionary include valid and invalid")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int,add string)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='city','dictionary_include'='add')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_include'='name')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name"))
    +    }
    +    intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_include'='id')")
    +    }
    +    intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_include'='city,city')")
    +    }
    +    intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_include'='')")
    +    }
    +    intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_include'='add')")
    +    }
    +  }
    +
    +  test("test alter set for local dictionary exclude valid and invalid")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int,add string)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_exclude'='city','dictionary_include'='add')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_exclude'='name')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name"))
    +    }
    +    intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='id')")
    +    }
    +    intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='city,city')")
    +    }
    +    intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='')")
    +    }
    +    intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='add')")
    +    }
    +  }
    +
    +  test("test alter set same column for local dictionary exclude and include")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_exclude'='city')
    +      """.stripMargin)
    +    intercept[Exception] {
    +      sql(
    +        "alter table local1 set tblproperties('local_dictionary_include'='name'," +
    +        "'local_dictionary_exclude'='name')")
    +    }
    +  }
    +
    +  test("test alter set for valid and invalid complex type as include/exclude") {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int,st struct<s_id:int,
    +        | s_name:string,s_city:array<string>>, dcity array<string>)
    +        | STORED BY 'org.apache.carbondata.format'
    +        | tblproperties('local_dictionary_exclude'='name','local_dictionary_include'='city,dcity',
    +        | 'local_dictionary_enable'='true')
    +      """.
    +        stripMargin)
    +    val descFormatted1 = sql("describe formatted local1").collect
    +    descFormatted1.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +    descFormatted1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(
    +        row.get(1).toString.contains("city,dcity") && !row.get(1).toString.contains("name"))
    +    }
    +    intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='dcity')")
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_exclude'='st')")
    +    val descFormatted2 = sql("describe formatted local1").collect
    +    descFormatted2.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("st"))
    +    }
    +    descFormatted2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city,dcity"))
    +    }
    +    sql("alter table local1 set tblproperties('local_dictionary_exclude'='st'," +
    +        "'local_dictionary_include'='dcity')")
    +    val descFormatted3 = sql("describe formatted local1").collect
    +    descFormatted3.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("st"))
    +    }
    +    descFormatted3.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("dcity"))
    +    }
    +  }
    +
    +  test("test alter set for invalid complex type as include/exclude") {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int,st struct<s_id:int,
    +        | s_name:int,s_city:array<int>>, dcity array<int>)
    +        | STORED BY 'org.apache.carbondata.format'
    +        | tblproperties('local_dictionary_exclude'='name',
    +        | 'local_dictionary_enable'='true')
    +      """.
    +        stripMargin)
    +    intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_exclude'='dcity')")
    +    }
    +    intercept[Exception] {
    +      sql("alter table local1 set tblproperties('local_dictionary_include'='st')")
    +    }
    +  }
    +
    +  test("test alter unset for local dictionary disable")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int,add string)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='false')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Enable")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("false"))
    +    }
    +    sql("alter table local1 unset tblproperties('local_dictionary_enable')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Enable")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("true"))
    +    }
    +  }
    +
    +  test("test alter unset for local dictionary enable local dict include")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int,add string)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='city')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city"))
    +    }
    +    sql("alter table local1 unset tblproperties('local_dictionary_include')")
    +    val descLoc2 = sql("describe formatted local1").collect
    +    descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("name,city"))
    +    }
    +  }
    +
    +  test("test alter unset for local dictionary enable local dict exclude")
    +  {
    +    sql("drop table if exists local1")
    +    sql(
    +      """
    +        | CREATE TABLE local1(id int, name string, city string, age int,add string)
    +        | STORED BY 'carbondata' tblproperties('local_dictionary_include'='city',
    +        | 'local_dictionary_exclude'='name')
    +      """.stripMargin)
    +
    +    val descLoc1 = sql("describe formatted local1").collect
    +    descLoc1.find(_.get(0).toString.contains("Local Dictionary Include")) match {
    +      case Some(row) => assert(row.get(1).toString.contains("city"))
    +    }
    +    sql("alter table local1 unset tblproperties('local_dictionary_include')")
    --- End diff --
   
    handled in #2450


---

[qiuchenjian-2]

Github user kunal642 commented on the issue:

    https://github.com/apache/carbondata/pull/2422
 
    LGTM


---

[qiuchenjian-2]

Github user asfgit closed the pull request at:

    https://github.com/apache/carbondata/pull/2422


---