[GitHub] carbondata pull request #1856: [CARBONDATA-2073][CARBONDATA-1516][Tests] Add...

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174056762
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala ---
    @@ -82,149 +92,209 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
            """.stripMargin)
       }
     
    +  override def afterEach(): Unit = {
    +    dropDataMaps("mainTable", "agg1_second", "agg1_minute",
    +      "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
    +  }
    +
       test("test timeseries create table 1") {
         checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_second"), true, "maintable_agg0_second")
    -    sql("drop datamap agg0_second on table mainTable")
    +    sql("DROP DATAMAP agg0_second ON TABLE mainTable")
    --- End diff --
   
    why is this change necessary??


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174058314
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala ---
    @@ -241,12 +311,12 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
         assert(e.getMessage.equals("Only one granularity level can be defined"))
       }
     
    -  test("test timeseries create table 14: Only one granularity level can be defined 2") {
    -    sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
    +  test("test timeseries create table 17: Only one granularity level can be defined 2") {
    +    sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
    --- End diff --
   
    why change just the name of the table??


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174058262
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala ---
    @@ -82,149 +92,209 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
            """.stripMargin)
       }
     
    +  override def afterEach(): Unit = {
    +    dropDataMaps("mainTable", "agg1_second", "agg1_minute",
    +      "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
    +  }
    +
       test("test timeseries create table 1") {
         checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_second"), true, "maintable_agg0_second")
    -    sql("drop datamap agg0_second on table mainTable")
    +    sql("DROP DATAMAP agg0_second ON TABLE mainTable")
       }
     
       test("test timeseries create table 2") {
         checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_hour"), true, "maintable_agg0_hour")
    -    sql("drop datamap agg0_hour on table mainTable")
    +    sql("DROP DATAMAP agg0_hour ON TABLE mainTable")
       }
    +
       test("test timeseries create table 3") {
         checkExistence(sql("DESCRIBE FORMATTED maintable_agg0_day"), true, "maintable_agg0_day")
    -    sql("drop datamap agg0_day on table mainTable")
    +    sql("DROP DATAMAP agg0_day ON TABLE mainTable")
       }
    +
       test("test timeseries create table 4") {
         checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_month"), true, "maintable_agg0_month")
    -    sql("drop datamap agg0_month on table mainTable")
    +    sql("DROP DATAMAP agg0_month ON TABLE mainTable")
       }
    +
       test("test timeseries create table 5") {
         checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_year"), true, "maintable_agg0_year")
    -    sql("drop datamap agg0_year on table mainTable")
    +    sql("DROP DATAMAP agg0_year ON TABLE mainTable")
       }
     
    -  test("test timeseries create table 6") {
    -    intercept[Exception] {
    +  test("test timeseries create table 6: TIMESERIES should define time granularity") {
    +    sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
    +    val e = intercept[MalformedCarbonCommandException] {
           sql(
    -        s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    -           | USING '$timeSeries'
    -           | DMPROPERTIES (
    -           | 'EVENT_TIME'='dataTime',
    -           | 'SEC_GRANULARITY'='1')
    -           | AS SELECT dataTime, SUM(age) FROM mainTable
    -           | GROUP BY dataTime
    +        s"""CREATE DATAMAP agg0_second ON TABLE mainTable USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='dataTime',
    +           |   'SEC_GRANULARITY'='1')
    +           |AS SELECT dataTime, SUM(age) FROM mainTable
    +           |GROUP BY dataTime
             """.stripMargin)
         }
    +    assert(e.getMessage.contains("TIMESERIES should define time granularity"))
       }
     
    -  test("test timeseries create table 7") {
    -    intercept[Exception] {
    +  test("test timeseries create table 7: Granularity only support 1") {
    +    sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
    +    val e = intercept[MalformedDataMapCommandException] {
           sql(
             s"""
                | CREATE DATAMAP agg0_second ON TABLE mainTable
                | USING '$timeSeries'
                | DMPROPERTIES (
                | 'EVENT_TIME'='dataTime',
    -           | 'SECOND_GRANULARITY'='2')
    +           | 'DAY_GRANULARITY'='1.5')
                | AS SELECT dataTime, SUM(age) FROM mainTable
                | GROUP BY dataTime
            """.stripMargin)
         }
    +    assert(e.getMessage.equals("Granularity only support 1"))
       }
     
    -  test("test timeseries create table 8") {
    -    intercept[Exception] {
    +  test("test timeseries create table 8: Granularity only support 1") {
    +    dropDataMaps("mainTable", "agg1_hour")
    +    val e = intercept[MalformedCarbonCommandException] {
           sql(
    -        s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    -           | USING '$timeSeries'
    -           | DMPROPERTIES (
    -           | 'EVENT_TIME'='dataTime',
    -           | 'SECOND_GRANULARITY'='1')
    -           | AS SELECT dataTime, SUM(age) FROM mainTable
    -           | GROUP BY dataTime
    -       """.stripMargin)
    +        s"""CREATE DATAMAP agg1_hour ON TABLE mainTable USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='dataTime',
    +           |   'HOUR_GRANULARITY'='hour=-2')
    +           |AS SELECT dataTime, SUM(age) FROM mainTable
    +           |GROUP BY dataTime
    +        """.stripMargin)
    +    }
    +    assert(e.getMessage.contains("Granularity only support "))
    +    checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "maintable_agg1_hour")
    +  }
    +
    +  test("test timeseries create table 9: SECOND_GRANULARITY is null") {
    +    sql("DROP DATAMAP IF EXISTS agg1 ON TABLE mainTable")
    +    val e = intercept[MalformedCarbonCommandException] {
           sql(
    -        s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    -           | USING '$timeSeries'
    -           | DMPROPERTIES (
    -           | 'EVENT_TIME'='dataTime',
    -           | 'SECOND_GRANULARITY'='1')
    -           | AS SELECT dataTime, SUM(age) FROM mainTable
    -           | GROUP BY dataTime
    -       """.stripMargin)
    +        s"""CREATE DATAMAP agg0_hour ON TABLE mainTable
    +           |USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='dataTime',
    +           |   'HOUR_GRANULARITY'='')
    +           |AS SELECT dataTime, SUM(age) FROM mainTable
    +           |GROUP BY dataTime
    +        """.stripMargin)
    +    }
    +    assert(e.getMessage.contains("Granularity only support 1"))
    +  }
    +
    +  test("test timeseries create table 10: Table already exists in database") {
    +    val e = intercept[MalformedDataMapCommandException] {
    +      sql(
    +        s"""CREATE DATAMAP agg1_hour ON TABLE mainTable USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='dataTime',
    +           |   'HOUR_GRANULARITY'='1')
    +           |AS SELECT dataTime, SUM(age) FROM mainTable
    +           |GROUP BY dataTime
    +        """.stripMargin)
    +      sql(
    +        s"""CREATE DATAMAP agg1_hour ON TABLE mainTable USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='dataTime',
    +           |   'HOUR_GRANULARITY'='1')
    +           |AS SELECT dataTime, SUM(age) FROM mainTable
    +           |GROUP BY dataTime
    +        """.stripMargin)
         }
    +    assert(e.getMessage.contains(
    +      "DataMap name 'agg1_hour' already exist"))
       }
     
    -  test("test timeseries create table 9") {
    -    intercept[Exception] {
    +  test("test timeseries create table 11: don't support create timeseries table on non timestamp") {
    +    sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
    +    val e = intercept[MalformedCarbonCommandException] {
           sql(
             s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    +           | CREATE DATAMAP agg0_hour ON TABLE mainTable
                | USING '$timeSeries'
                | DMPROPERTIES (
                | 'EVENT_TIME'='name',
    -           | 'SECOND_GRANULARITY'='1')
    +           | 'HOUR_GRANULARITY'='1')
                | AS SELECT dataTime, SUM(age) FROM mainTable
                | GROUP BY dataTime
            """.stripMargin)
         }
    +    assert(e.getMessage.equals("Timeseries event time is only supported on Timestamp column"))
       }
     
    -  test("test timeseries create table 10") {
    -    intercept[Exception] {
    +  test("test timeseries create table 12: Time series column dataTime does not exists in select") {
    +    sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
    +    val e = intercept[MalformedCarbonCommandException] {
           sql(
             s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    +           | CREATE DATAMAP agg0_hour ON TABLE mainTable
                | USING '$timeSeries'
                | DMPROPERTIES (
    -           | 'EVENT_TIME'='name',
    -           | 'SECOND_GRANULARITY'='1')
    -           | AS SELECT dataTime, SUM(age) FROM mainTable
    -           | GROUP BY dataTime
    -       """.stripMargin)
    +           | 'EVENT_TIME'='dataTime',
    +           | 'HOUR_GRANULARITY'='1')
    +           | AS SELECT name, SUM(age) FROM mainTable
    +           | GROUP BY name
    +         """.stripMargin)
    +    }
    +    assert(e.getMessage.equals("Time series column dataTime does not exists in select"))
    +  }
    +
    +  test("test timeseries create table 13: don't support create timeseries table on non timestamp") {
    +    sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
    +    val e = intercept[MalformedCarbonCommandException] {
    +      sql(
    +        s"""CREATE DATAMAP agg0_hour ON TABLE mainTable
    +           |USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='name',
    +           |   'HOUR_GRANULARITY'='1')
    +           |AS SELECT name, SUM(age) FROM mainTable
    +           |GROUP BY name
    +        """.stripMargin)
         }
    +    assert(e.getMessage.contains("Timeseries event time is only supported on Timestamp column"))
       }
     
    -  test("test timeseries create table 11: USING") {
    +  test("test timeseries create table 14: USING") {
         val e: Exception = intercept[MalformedDataMapCommandException] {
           sql(
    -        """CREATE DATAMAP agg1 ON TABLE mainTable
    +        """CREATE DATAMAP agg0_hour ON TABLE mainTable
               | USING 'abc'
               | DMPROPERTIES (
               |   'EVENT_TIME'='dataTime',
    -          |   'SECOND_GRANULARITY'='1')
    +          |   'HOUR_GRANULARITY'='1')
               | AS SELECT dataTime, SUM(age) FROM mainTable
               | GROUP BY dataTime
             """.stripMargin)
         }
         assert(e.getMessage.equals("DataMap 'abc' not found"))
       }
     
    -  test("test timeseries create table 12: USING and catch MalformedCarbonCommandException") {
    +  test("test timeseries create table 15: USING and catch MalformedCarbonCommandException") {
         val e: Exception = intercept[MalformedCarbonCommandException] {
           sql(
    -        """CREATE DATAMAP agg1 ON TABLE mainTable
    +        """CREATE DATAMAP agg0_hour ON TABLE mainTable
               | USING 'abc'
               | DMPROPERTIES (
               |   'EVENT_TIME'='dataTime',
    -          |   'SECOND_GRANULARITY'='1')
    +          |   'HOUR_GRANULARITY'='1')
               | AS SELECT dataTime, SUM(age) FROM mainTable
               | GROUP BY dataTime
             """.stripMargin)
         }
         assert(e.getMessage.equals("DataMap 'abc' not found"))
       }
     
    -  test("test timeseries create table 13: Only one granularity level can be defined 1") {
    -    sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
    +  test("test timeseries create table 16: Only one granularity level can be defined 1") {
    +    sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
         val e: Exception = intercept[MalformedCarbonCommandException] {
           sql(
             s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    +           | CREATE DATAMAP agg0_hour ON TABLE mainTable
    --- End diff --
   
    why change just the name of the table??


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174059512
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala ---
    @@ -354,21 +390,137 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
         checkExistence(sql("DESC FORMATTED mainTable_agg1"), true, "maintable_age_sum")
       }
     
    +  test("test timeseries create table 32: should support if not exists, create when same table not exists") {
    +    sql("DROP DATAMAP IF EXISTS agg1_year ON TABLE mainTable")
    +    sql(
    +      s"""
    +         |CREATE DATAMAP if not exists agg1_year ON TABLE mainTable
    +         |USING '$timeSeries'
    +         |DMPROPERTIES (
    +         |   'event_time'='dataTime',
    +         |   'YEAR_GRANULARITY'='1')
    +         |AS SELECT dataTime, SUM(age) FROM mainTable
    +         |GROUP BY dataTime
    +        """.stripMargin)
    +    checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "agg1_year")
    +    checkExistence(sql("DESC FORMATTED mainTable_agg1_year"), true, "maintable_age_sum")
    +  }
    +
       test("test timeseries create table 20: don't support 'create datamap if exists'") {
         val e: Exception = intercept[AnalysisException] {
           sql(
             s"""CREATE DATAMAP IF EXISTS agg2 ON TABLE mainTable
    -          | USING '$timeSeries'
    -          | DMPROPERTIES (
    -          |   'EVENT_TIME'='dataTime',
    -          |   'MONTH_GRANULARITY'='1')
    -          | AS SELECT dataTime, SUM(age) FROM mainTable
    -          | GROUP BY dataTime
    +           | USING '$timeSeries'
    +           | DMPROPERTIES (
    +           |   'EVENT_TIME'='dataTime',
    +           |   'MONTH_GRANULARITY'='1')
    +           | AS SELECT dataTime, SUM(age) FROM mainTable
    +           | GROUP BY dataTime
             """.stripMargin)
         }
         assert(e.getMessage.contains("identifier matching regex"))
       }
     
    +  test("test timeseries create table 26: test different data type") {
    +    sql("drop table if exists dataTable")
    +    sql(
    +      s"""
    +         | CREATE TABLE dataTable(
    +         | shortField SHORT,
    +         | booleanField BOOLEAN,
    +         | intField INT,
    +         | bigintField LONG,
    +         | doubleField DOUBLE,
    +         | stringField STRING,
    +         | decimalField DECIMAL(18,2),
    +         | charField CHAR(5),
    +         | floatField FLOAT,
    +         | dataTime timestamp
    +         | )
    +         | STORED BY 'carbondata'
    +       """.stripMargin)
    +
    +
    +    sql(
    +      s"""CREATE DATAMAP agg0_hour ON TABLE dataTable
    +         | USING '$timeSeries'
    +         | DMPROPERTIES (
    +         |   'event_time'='dataTime',
    +         |   'HOUR_GRANULARITY'='1')
    +         | AS SELECT
    +         |   dataTime,
    +         |   SUM(intField),
    +         |   shortField,
    +         |   booleanField,
    +         |   intField,
    +         |   bigintField,
    +         |   doubleField,
    +         |   stringField,
    +         |   decimalField,
    +         |   charField,
    +         |   floatField
    +         | FROM dataTable
    +         | GROUP BY
    +         |   dataTime,
    +         |   shortField,
    +         |   booleanField,
    +         |   intField,
    +         |   bigintField,
    +         |   doubleField,
    +         |   stringField,
    +         |   decimalField,
    +         |   charField,
    +         |   floatField
    +        """.stripMargin)
    +    checkExistence(sql("SHOW DATAMAP ON TABLE dataTable"), true, "datatable_agg0_hour")
    +    sql("DROP TABLE IF EXISTS dataTable")
    +  }
    +
    +  test("test timeseries create table 27: test data map name") {
    --- End diff --
   
    same as test timeseries create table 32. Please remove. avoid adding duplicate test cases


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174060207
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala ---
    @@ -354,21 +390,137 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
         checkExistence(sql("DESC FORMATTED mainTable_agg1"), true, "maintable_age_sum")
       }
     
    +  test("test timeseries create table 32: should support if not exists, create when same table not exists") {
    +    sql("DROP DATAMAP IF EXISTS agg1_year ON TABLE mainTable")
    +    sql(
    +      s"""
    +         |CREATE DATAMAP if not exists agg1_year ON TABLE mainTable
    +         |USING '$timeSeries'
    +         |DMPROPERTIES (
    +         |   'event_time'='dataTime',
    +         |   'YEAR_GRANULARITY'='1')
    +         |AS SELECT dataTime, SUM(age) FROM mainTable
    +         |GROUP BY dataTime
    +        """.stripMargin)
    +    checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), true, "agg1_year")
    +    checkExistence(sql("DESC FORMATTED mainTable_agg1_year"), true, "maintable_age_sum")
    +  }
    +
       test("test timeseries create table 20: don't support 'create datamap if exists'") {
         val e: Exception = intercept[AnalysisException] {
           sql(
             s"""CREATE DATAMAP IF EXISTS agg2 ON TABLE mainTable
    -          | USING '$timeSeries'
    -          | DMPROPERTIES (
    -          |   'EVENT_TIME'='dataTime',
    -          |   'MONTH_GRANULARITY'='1')
    -          | AS SELECT dataTime, SUM(age) FROM mainTable
    -          | GROUP BY dataTime
    +           | USING '$timeSeries'
    --- End diff --
   
    revert this change. it is unncessary


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174060559
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala ---
    @@ -17,18 +17,28 @@
     package org.apache.carbondata.integration.spark.testsuite.timeseries
     
     import org.apache.spark.sql.AnalysisException
    +import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
     import org.apache.spark.sql.test.util.QueryTest
    -import org.scalatest.BeforeAndAfterAll
    +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
     
     import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
    +import org.apache.carbondata.core.constants.CarbonCommonConstants
     import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
    +import org.apache.carbondata.core.util.CarbonProperties
     
    -class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
    +class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach{
     
       val timeSeries = TIMESERIES.toString
    +  var timestampFormat: String = _
    --- End diff --
   
    no need for this variable. simply set CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT as the timestamp format is after all.


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174060602
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesDropSuite.scala ---
    @@ -16,15 +16,28 @@
      */
     package org.apache.carbondata.integration.spark.testsuite.timeseries
     
    +import org.apache.spark.sql.AnalysisException
     import org.apache.spark.sql.test.util.QueryTest
     import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
     
    -import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
    +import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, NoSuchDataMapException}
    +import org.apache.carbondata.core.constants.CarbonCommonConstants
    +import org.apache.carbondata.core.util.CarbonProperties
    +import org.apache.carbondata.spark.exception.ProcessMetaDataException
     
     class TestTimeSeriesDropSuite extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
     
    +  val timeSeries = "timeseries"
    +  var timestampFormat: String = _
    --- End diff --
   
    no need for this variable. simply set CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT as the timestamp format is after all.


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174060918
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesMatchStrategySuite.scala ---
    @@ -0,0 +1,401 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *    http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.carbondata.integration.spark.testsuite.timeseries
    +
    +import java.sql.Timestamp
    +
    +import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, Row}
    +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
    +import org.apache.spark.sql.execution.datasources.LogicalRelation
    +import org.apache.spark.sql.hive.CarbonRelation
    +import org.apache.spark.sql.test.util.QueryTest
    +import org.apache.spark.util.SparkUtil4Test
    +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
    +
    +import org.apache.carbondata.core.constants.CarbonCommonConstants
    +import org.apache.carbondata.core.util.CarbonProperties
    +
    +class TestTimeSeriesMatchStrategySuite extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
    +
    +  var timestampFormat: String = _
    --- End diff --
   
    no need for this variable. simply set CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT as the timestamp format is after all.


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174063930
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesUnsupportedSuite.scala ---
    @@ -0,0 +1,265 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *    http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package org.apache.carbondata.integration.spark.testsuite.timeseries
    +
    +import java.sql.Timestamp
    +
    +import org.apache.spark.sql.Row
    +import org.apache.spark.sql.test.util.QueryTest
    +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
    +
    +import org.apache.carbondata.core.constants.CarbonCommonConstants
    +import org.apache.carbondata.core.util.CarbonProperties
    +
    +class TestTimeSeriesUnsupportedSuite extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
    --- End diff --
   
    I don't think this Unsupported suite is required. Already have test cases for each scenario. No need to test the same for timeseries. It will increase the build time.


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174064354
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala ---
    @@ -167,59 +170,60 @@ class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
            """.stripMargin)
     
       }
    -  test("test Year level timeseries data validation1 ") {
    -    checkAnswer( sql("select count(*) from table_03_ag1_year"),
    +
    +  test("test timeseries table selection 1: year level timeseries data validation1 ") {
    +    checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_year"),
           Seq(Row(4)))
       }
     
    -  test("test month level timeseries data validation1 ") {
    -    checkAnswer( sql("select count(*) from table_03_ag1_month"),
    +  test("test timeseries table selection 2: month level timeseries data validation1 ") {
    +    checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_month"),
           Seq(Row(4)))
       }
     
    -  test("test day level timeseries data validation1 ") {
    -    checkAnswer( sql("select count(*) from table_03_ag1_day"),
    +  test("test timeseries table selection 3: day level timeseries data validation1 ") {
    +    checkAnswer(sql("SELECT COUNT(*) FROM table_03_ag1_day"),
    --- End diff --
   
    why is this required?


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174064540
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala ---
    @@ -330,16 +366,232 @@ class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
             """.stripMargin)
     
         checkAnswer(sql("select * from maintable_agg0_second"),
    -      Seq(Row(Timestamp.valueOf("2016-02-23 01:01:30.0"), 10),
    -        Row(Timestamp.valueOf("2016-02-23 01:01:40.0"), 20),
    -        Row(Timestamp.valueOf("2016-02-23 01:01:50.0"), 30),
    -        Row(Timestamp.valueOf("2016-02-23 01:02:30.0"), 40),
    -        Row(Timestamp.valueOf("2016-02-23 01:02:40.0"), 50),
    -        Row(Timestamp.valueOf("2016-02-23 01:02:50.0"), 50)))
    +      Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
    +  }
    +
    +  test("test timeseries table selection 14: load data into mainTable after create timeseries datamap ON TABLE and SELECT sub table") {
    +    sql("DROP TABLE IF EXISTS main_table")
    +    sql(
    +      """
    +        | CREATE TABLE main_table(
    +        |   mytime timestamp,
    +        |   name string,
    +        |   age int)
    +        | STORED BY 'org.apache.carbondata.format'
    +      """.stripMargin)
    +    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +    sql(
    +      s"""
    +         | CREATE DATAMAP agg0_second ON TABLE main_table
    +         | USING '$timeSeries'
    +         | DMPROPERTIES (
    +         |   'event_time'='mytime',
    +         |   'SECOND_GRANULARITY'='1')
    +         | AS SELECT mytime, SUM(age)
    +         | FROM main_table
    +         | GROUP BY mytime""".stripMargin)
    +
    +
    +    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +
    +    checkAnswer(sql("SELECT * FROM main_table_agg0_second"),
    +      Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
    +  }
    +
    +  test("test timeseries table selection 15: load data into main_table after create timeseries datamap ON TABLE 1") {
    --- End diff --
   
    this scenario is already covered in the previous tests


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174064625
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala ---
    @@ -330,16 +366,232 @@ class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
             """.stripMargin)
     
         checkAnswer(sql("select * from maintable_agg0_second"),
    -      Seq(Row(Timestamp.valueOf("2016-02-23 01:01:30.0"), 10),
    -        Row(Timestamp.valueOf("2016-02-23 01:01:40.0"), 20),
    -        Row(Timestamp.valueOf("2016-02-23 01:01:50.0"), 30),
    -        Row(Timestamp.valueOf("2016-02-23 01:02:30.0"), 40),
    -        Row(Timestamp.valueOf("2016-02-23 01:02:40.0"), 50),
    -        Row(Timestamp.valueOf("2016-02-23 01:02:50.0"), 50)))
    +      Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
    +  }
    +
    +  test("test timeseries table selection 14: load data into mainTable after create timeseries datamap ON TABLE and SELECT sub table") {
    +    sql("DROP TABLE IF EXISTS main_table")
    +    sql(
    +      """
    +        | CREATE TABLE main_table(
    +        |   mytime timestamp,
    +        |   name string,
    +        |   age int)
    +        | STORED BY 'org.apache.carbondata.format'
    +      """.stripMargin)
    +    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +    sql(
    +      s"""
    +         | CREATE DATAMAP agg0_second ON TABLE main_table
    +         | USING '$timeSeries'
    +         | DMPROPERTIES (
    +         |   'event_time'='mytime',
    +         |   'SECOND_GRANULARITY'='1')
    +         | AS SELECT mytime, SUM(age)
    +         | FROM main_table
    +         | GROUP BY mytime""".stripMargin)
    +
    +
    +    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +
    +    checkAnswer(sql("SELECT * FROM main_table_agg0_second"),
    +      Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
    +  }
    +
    +  test("test timeseries table selection 15: load data into main_table after create timeseries datamap ON TABLE 1") {
    +    sql("DROP TABLE IF EXISTS main_table")
    +    sql(
    +      """
    +        | CREATE TABLE main_table(
    +        |   mytime timestamp,
    +        |   name string,
    +        |   age int)
    +        | STORED BY 'org.apache.carbondata.format'
    +      """.stripMargin)
    +    sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +
    +    sql(
    +      s"""
    +        | CREATE DATAMAP agg0_minute ON TABLE main_table
    +        | USING '$timeSeries'
    +        | DMPROPERTIES (
    +        |   'event_time'='mytime',
    +        |   'MINUTE_GRANULARITY'='1')
    +        | AS SELECT mytime, SUM(age)
    +        | FROM main_table
    +        | GROUP BY mytime""".stripMargin)
    +
    +    sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +    val df = sql(
    +      """
    +        | SELECT
    +        |   timeseries(mytime,'minute') AS minuteLevel,
    +        |   SUM(age) AS SUM
    +        | FROM main_table
    +        | WHERE timeseries(mytime,'minute')>='2016-02-23 09:01:00'
    +        | GROUP BY
    +        |   timeseries(mytime,'minute')
    +        | ORDER BY
    +        |   timeseries(mytime,'minute')
    +      """.stripMargin)
    +
    +    checkAnswer(df,
    +      Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 120),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:00"), 280)))
    +  }
    +
    +  test("test timeseries table selection 16: load data into main_table after create timeseries datamap ON TABLE 2") {
    --- End diff --
   
    duplicate


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174064693
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesDataLoad.scala ---
    @@ -330,16 +366,232 @@ class TestTimeseriesDataLoad extends QueryTest with BeforeAndAfterAll {
             """.stripMargin)
     
         checkAnswer(sql("select * from maintable_agg0_second"),
    -      Seq(Row(Timestamp.valueOf("2016-02-23 01:01:30.0"), 10),
    -        Row(Timestamp.valueOf("2016-02-23 01:01:40.0"), 20),
    -        Row(Timestamp.valueOf("2016-02-23 01:01:50.0"), 30),
    -        Row(Timestamp.valueOf("2016-02-23 01:02:30.0"), 40),
    -        Row(Timestamp.valueOf("2016-02-23 01:02:40.0"), 50),
    -        Row(Timestamp.valueOf("2016-02-23 01:02:50.0"), 50)))
    +      Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
    +  }
    +
    +  test("test timeseries table selection 14: load data into mainTable after create timeseries datamap ON TABLE and SELECT sub table") {
    +    sql("DROP TABLE IF EXISTS main_table")
    +    sql(
    +      """
    +        | CREATE TABLE main_table(
    +        |   mytime timestamp,
    +        |   name string,
    +        |   age int)
    +        | STORED BY 'org.apache.carbondata.format'
    +      """.stripMargin)
    +    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +    sql(
    +      s"""
    +         | CREATE DATAMAP agg0_second ON TABLE main_table
    +         | USING '$timeSeries'
    +         | DMPROPERTIES (
    +         |   'event_time'='mytime',
    +         |   'SECOND_GRANULARITY'='1')
    +         | AS SELECT mytime, SUM(age)
    +         | FROM main_table
    +         | GROUP BY mytime""".stripMargin)
    +
    +
    +    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +
    +    checkAnswer(sql("SELECT * FROM main_table_agg0_second"),
    +      Seq(Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:30.0"), 10),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:40.0"), 20),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:50.0"), 30),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:30.0"), 40),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:40.0"), 50),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:50.0"), 50)))
    +  }
    +
    +  test("test timeseries table selection 15: load data into main_table after create timeseries datamap ON TABLE 1") {
    +    sql("DROP TABLE IF EXISTS main_table")
    +    sql(
    +      """
    +        | CREATE TABLE main_table(
    +        |   mytime timestamp,
    +        |   name string,
    +        |   age int)
    +        | STORED BY 'org.apache.carbondata.format'
    +      """.stripMargin)
    +    sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +
    +    sql(
    +      s"""
    +        | CREATE DATAMAP agg0_minute ON TABLE main_table
    +        | USING '$timeSeries'
    +        | DMPROPERTIES (
    +        |   'event_time'='mytime',
    +        |   'MINUTE_GRANULARITY'='1')
    +        | AS SELECT mytime, SUM(age)
    +        | FROM main_table
    +        | GROUP BY mytime""".stripMargin)
    +
    +    sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +    val df = sql(
    +      """
    +        | SELECT
    +        |   timeseries(mytime,'minute') AS minuteLevel,
    +        |   SUM(age) AS SUM
    +        | FROM main_table
    +        | WHERE timeseries(mytime,'minute')>='2016-02-23 09:01:00'
    +        | GROUP BY
    +        |   timeseries(mytime,'minute')
    +        | ORDER BY
    +        |   timeseries(mytime,'minute')
    +      """.stripMargin)
    +
    +    checkAnswer(df,
    +      Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 120),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:00"), 280)))
    +  }
    +
    +  test("test timeseries table selection 16: load data into main_table after create timeseries datamap ON TABLE 2") {
    +    sql("DROP TABLE IF EXISTS main_table")
    +    sql(
    +      """
    +        | CREATE TABLE main_table(
    +        |   mytime timestamp,
    +        |   name string,
    +        |   age int)
    +        | STORED BY 'org.apache.carbondata.format'
    +      """.stripMargin)
    +
    +    sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +    sql(s"LOAD DATA INPATH '$resourcesPath/timeseriestest.csv' INTO TABLE main_table")
    +    sql(
    +      s"""
    +        | CREATE DATAMAP agg0_minute ON TABLE main_table
    +        | USING '$timeSeries'
    +        | DMPROPERTIES (
    +        |   'event_time'='mytime',
    +        |   'MINUTE_GRANULARITY'='1')
    +        | AS SELECT mytime, SUM(age)
    +        | FROM main_table
    +        | GROUP BY mytime""".stripMargin)
    +
    +
    +    val df = sql(
    +      """
    +        | SELECT
    +        |   timeseries(mytime,'minute') AS minuteLevel,
    +        |   SUM(age) AS SUM
    +        | FROM main_table
    +        | WHERE timeseries(mytime,'minute')>='2016-02-23 09:01:00'
    +        | GROUP BY
    +        |   timeseries(mytime,'minute')
    +        | ORDER BY
    +        |   timeseries(mytime,'minute')
    +      """.stripMargin)
    +
    +    checkAnswer(df,
    +      Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 120),
    +        Row(Timestamp.valueOf("2016-02-23 09:02:00"), 280)))
    +  }
    +
    +  test("test timeseries table selection 17: load data into main_table after create timeseries datamap ON TABLE 3") {
    --- End diff --
   
    duplicate


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174064855
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala ---
    @@ -99,124 +113,763 @@ class TestTimeseriesTableSelection extends QueryTest with BeforeAndAfterAll {
         sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
       }
     
    -  test("test PreAggregate table selection 1") {
    -    val df = sql("select mytime from mainTable group by mytime")
    +  test("test timeseries table selection 1") {
    +    val df = sql("SELECT mytime FROM mainTable GROUP BY mytime")
         preAggTableValidator(df.queryExecution.analyzed, "maintable")
       }
     
    -  test("test PreAggregate table selection 2") {
    -    val df = sql("select timeseries(mytime,'hour') from mainTable group by timeseries(mytime,'hour')")
    +  test("test timeseries table selection 2") {
    +    val df = sql("SELECT TIMESERIES(mytime,'hour') FROM mainTable GROUP BY TIMESERIES(mytime,'hour')")
         preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
       }
     
    -  test("test PreAggregate table selection 3") {
    -    val df = sql("select timeseries(mytime,'milli') from mainTable group by timeseries(mytime,'milli')")
    -    preAggTableValidator(df.queryExecution.analyzed, "maintable")
    +  test("test timeseries table selection 3: No enum constant MILLI") {
    +    val e = intercept[Exception] {
    +      val df = sql(
    +        """
    +          | SELECT TIMESERIES(mytime,'milli')
    +          | FROM mainTable
    +          | GROUP BY TIMESERIES(mytime,'milli')
    +        """.stripMargin)
    +      preAggTableValidator(df.queryExecution.analyzed, "maintable")
    +      df.show()
    +    }
    +    assert(e.getMessage.contains(
    +      "No enum constant org.apache.carbondata.core.preagg.TimeSeriesFunctionEnum.MILLI"))
       }
     
    -  test("test PreAggregate table selection 4") {
    -    val df = sql("select timeseries(mytime,'year') from mainTable group by timeseries(mytime,'year')")
    +  test("test timeseries table selection 4") {
    +    val df = sql("SELECT TIMESERIES(mytime,'year') FROM mainTable GROUP BY TIMESERIES(mytime,'year')")
    --- End diff --
   
    why is this requried?


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174066445
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeseriesTableSelection.scala ---
    @@ -99,124 +113,763 @@ class TestTimeseriesTableSelection extends QueryTest with BeforeAndAfterAll {
         sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/timeseriestest.csv' into table mainTable")
       }
     
    -  test("test PreAggregate table selection 1") {
    -    val df = sql("select mytime from mainTable group by mytime")
    +  test("test timeseries table selection 1") {
    +    val df = sql("SELECT mytime FROM mainTable GROUP BY mytime")
         preAggTableValidator(df.queryExecution.analyzed, "maintable")
       }
     
    -  test("test PreAggregate table selection 2") {
    -    val df = sql("select timeseries(mytime,'hour') from mainTable group by timeseries(mytime,'hour')")
    +  test("test timeseries table selection 2") {
    +    val df = sql("SELECT TIMESERIES(mytime,'hour') FROM mainTable GROUP BY TIMESERIES(mytime,'hour')")
         preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
       }
     
    -  test("test PreAggregate table selection 3") {
    -    val df = sql("select timeseries(mytime,'milli') from mainTable group by timeseries(mytime,'milli')")
    -    preAggTableValidator(df.queryExecution.analyzed, "maintable")
    +  test("test timeseries table selection 3: No enum constant MILLI") {
    +    val e = intercept[Exception] {
    +      val df = sql(
    +        """
    +          | SELECT TIMESERIES(mytime,'milli')
    +          | FROM mainTable
    +          | GROUP BY TIMESERIES(mytime,'milli')
    +        """.stripMargin)
    +      preAggTableValidator(df.queryExecution.analyzed, "maintable")
    +      df.show()
    +    }
    +    assert(e.getMessage.contains(
    +      "No enum constant org.apache.carbondata.core.preagg.TimeSeriesFunctionEnum.MILLI"))
       }
     
    -  test("test PreAggregate table selection 4") {
    -    val df = sql("select timeseries(mytime,'year') from mainTable group by timeseries(mytime,'year')")
    +  test("test timeseries table selection 4") {
    +    val df = sql("SELECT TIMESERIES(mytime,'year') FROM mainTable GROUP BY TIMESERIES(mytime,'year')")
         preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_year")
       }
     
    -  test("test PreAggregate table selection 5") {
    -    val df = sql("select timeseries(mytime,'day') from mainTable group by timeseries(mytime,'day')")
    +  test("test timeseries table selection 5") {
    +    val df = sql("SELECT TIMESERIES(mytime,'day') FROM mainTable GROUP BY TIMESERIES(mytime,'day')")
         preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_day")
       }
     
    -  test("test PreAggregate table selection 6") {
    -    val df = sql("select timeseries(mytime,'month') from mainTable group by timeseries(mytime,'month')")
    +  test("test timeseries table selection 6") {
    +    val df = sql("SELECT TIMESERIES(mytime,'month') FROM mainTable GROUP BY TIMESERIES(mytime,'month')")
         preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_month")
       }
     
    -  test("test PreAggregate table selection 7") {
    -    val df = sql("select timeseries(mytime,'minute') from mainTable group by timeseries(mytime,'minute')")
    +  test("test timeseries table selection 7") {
    +    val df = sql("SELECT TIMESERIES(mytime,'minute') FROM mainTable GROUP BY TIMESERIES(mytime,'minute')")
         preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_minute")
       }
     
    -  test("test PreAggregate table selection 8") {
    -    val df = sql("select timeseries(mytime,'second') from mainTable group by timeseries(mytime,'second')")
    +  test("test timeseries table selection 8") {
    +    val df = sql("SELECT TIMESERIES(mytime,'second') FROM mainTable GROUP BY TIMESERIES(mytime,'second')")
         preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_second")
       }
     
    -  test("test PreAggregate table selection 9") {
    -    val df = sql("select timeseries(mytime,'hour') from mainTable where timeseries(mytime,'hour')='x' group by timeseries(mytime,'hour')")
    -    preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_hour")
    +  test("test timeseries table selection 9") {
    +    val df = sql(
    +      """
    +        | SELECT TIMESERIES(mytime,'hour')
    +        | FROM mainTable
    +        | WHERE TIMESERIES(mytime,'hour')='x'
    +        | GROUP BY TIMESERIES(mytime,'hour')
    +      """.stripMargin)
    +    preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
       }
     
    -  test("test PreAggregate table selection 10") {
    -    val df = sql("select timeseries(mytime,'hour') from mainTable where timeseries(mytime,'hour')='x' group by timeseries(mytime,'hour') order by timeseries(mytime,'hour')")
    -    preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_hour")
    +  test("test timeseries table selection 10") {
    +    val df = sql(
    +      """
    +        | SELECT TIMESERIES(mytime,'hour')
    +        | FROM mainTable
    +        | WHERE TIMESERIES(mytime,'hour')='x'
    +        | GROUP BY TIMESERIES(mytime,'hour')
    +        | ORDER BY TIMESERIES(mytime,'hour')
    +      """.stripMargin)
    +    preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
    +  }
    +
    +  test("test timeseries table selection 11") {
    +    val df = sql(
    +      """
    +        | SELECT TIMESERIES(mytime,'hour'),SUM(age)
    +        | FROM mainTable
    +        | WHERE TIMESERIES(mytime,'hour')='x'
    +        | GROUP BY TIMESERIES(mytime,'hour')
    +        | ORDER BY TIMESERIES(mytime,'hour')
    +      """.stripMargin)
    +    preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
       }
     
    -  test("test PreAggregate table selection 11") {
    -    val df = sql("select timeseries(mytime,'hour'),sum(age) from mainTable where timeseries(mytime,'hour')='x' group by timeseries(mytime,'hour') order by timeseries(mytime,'hour')")
    -    preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_hour")
    +  test("test timeseries table selection 12") {
    +    val df = sql(
    +      """
    +        | SELECT TIMESERIES(mytime,'hour') AS hourlevel,SUM(age) AS SUM
    +        | FROM mainTable
    +        | WHERE TIMESERIES(mytime,'hour')='x'
    +        | GROUP BY TIMESERIES(mytime,'hour')
    +        | ORDER BY TIMESERIES(mytime,'hour')
    +      """.stripMargin)
    +    preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
       }
     
    -  test("test PreAggregate table selection 12") {
    -    val df = sql("select timeseries(mytime,'hour')as hourlevel,sum(age) as sum from mainTable where timeseries(mytime,'hour')='x' group by timeseries(mytime,'hour') order by timeseries(mytime,'hour')")
    -    preAggTableValidator(df.queryExecution.analyzed,"maintable_agg0_hour")
    +  test("test timeseries table selection 13") {
    +    val df = sql(
    +      """
    +        | SELECT TIMESERIES(mytime,'hour')as hourlevel,SUM(age) AS SUM
    +        | FROM mainTable
    +        | WHERE TIMESERIES(mytime,'hour')='x' AND name='vishal'
    +        | GROUP BY TIMESERIES(mytime,'hour')
    +        | ORDER BY TIMESERIES(mytime,'hour')
    +      """.stripMargin)
    +    preAggTableValidator(df.queryExecution.analyzed, "maintable")
       }
     
    -  test("test PreAggregate table selection 13") {
    -    val df = sql("select timeseries(mytime,'hour')as hourlevel,sum(age) as sum from mainTable where timeseries(mytime,'hour')='x' and name='vishal' group by timeseries(mytime,'hour') order by timeseries(mytime,'hour')")
    -    preAggTableValidator(df.queryExecution.analyzed,"maintable")
    +  test("test timeseries table selection 14: TIMESERIES(mytime,'hour') match") {
    +    val df = sql(
    +      """
    +        | SELECT TIMESERIES(mytime,'hour')
    +        | FROM mainTable
    +        | WHERE TIMESERIES(mytime,'hour')='2016-02-23 09:00:00'
    +        | GROUP BY TIMESERIES(mytime,'hour')
    +      """.stripMargin)
    +    preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
    +    checkAnswer(df, Row(Timestamp.valueOf("2016-02-23 09:00:00.0")))
       }
     
    -  test("test timeseries table selection 14: Granularity only support 1 and throw Exception") {
    -    val e = intercept[MalformedCarbonCommandException] {
    -      sql(
    -        s"""
    -           | CREATE DATAMAP agg3_second ON TABLE mainTable
    -           | USING '$timeSeries'
    -           | DMPROPERTIES (
    -           | 'EVENT_TIME'='dataTime',
    -           | 'HOUR_GRANULARITY'='2')
    -           | AS SELECT dataTime, SUM(age) FROM mainTable
    -           | GROUP BY dataTime
    -       """.stripMargin)
    -    }
    -    assert(e.getMessage.contains("Granularity only support 1"))
    -  }
    -
    -  test("test timeseries table selection 15: Granularity only support 1 and throw Exception") {
    -    val e = intercept[MalformedCarbonCommandException] {
    -      sql(
    -        s"""
    -           | CREATE DATAMAP agg3_second ON TABLE mainTable
    -           | USING '$timeSeries'
    -           | DMPROPERTIES (
    -           | 'EVENT_TIME'='dataTime',
    -           | 'HOUR_GRANULARITY'='1.5')
    -           | AS SELECT dataTime, SUM(age) FROM mainTable
    -           | GROUP BY dataTime
    -       """.stripMargin)
    -    }
    -    assert(e.getMessage.contains("Granularity only support 1"))
    -  }
    -
    -  test("test timeseries table selection 16: Granularity only support 1 and throw Exception") {
    -    val e = intercept[MalformedCarbonCommandException] {
    -      sql(
    -        s"""
    -           | CREATE DATAMAP agg3_second ON TABLE mainTable
    -           | USING '$timeSeries'
    -           | DMPROPERTIES (
    -           | 'EVENT_TIME'='dataTime',
    -           | 'HOUR_GRANULARITY'='-1')
    -           | AS SELECT dataTime, SUM(age) FROM mainTable
    -           | GROUP BY dataTime
    -       """.stripMargin)
    -    }
    -    assert(e.getMessage.contains("Granularity only support 1"))
    +  test("test timeseries table selection 15: TIMESERIES(mytime,'hour') not match") {
    +    val df = sql(
    +      """
    +        | SELECT TIMESERIES(mytime,'hour')
    +        | FROM mainTable
    +        | WHERE TIMESERIES(mytime,'hour')='2016-02-23 09:01:00'
    +        | GROUP BY TIMESERIES(mytime,'hour')
    +      """.stripMargin)
    +    preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_hour")
    +    checkExistence(df, false, "2016-02-23 09:00:00", "2016-02-23 09:01:00")
    +  }
    +
    +  test("test timeseries table selection 16: TIMESERIES(mytime,'minute') match") {
    +    checkExistence(sql("SELECT * FROM mainTable"), true,
    +      "2016-02-23 09:01:30", "2016-02-23 09:02:40")
    +    checkExistence(sql("SELECT * FROM mainTable"), false,
    +      "2016-02-23 09:02:00", "2016-02-23 09:01:00")
    +    val df = sql(
    +      """
    +        |SELECT TIMESERIES(mytime,'minute')
    +        |FROM mainTable
    +        |GROUP BY TIMESERIES(mytime,'minute')
    +      """.stripMargin)
    +    preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0_minute")
    +    checkExistence(df, true, "2016-02-23 09:02:00", "2016-02-23 09:01:00")
    +    checkAnswer(df,
    +      Seq(Row(Timestamp.valueOf("2016-02-23 09:02:00.0")),
    +        Row(Timestamp.valueOf("2016-02-23 09:01:00.0"))))
    +
    +    val df2 = sql(
    +      """
    +        | SELECT
    +        |   TIMESERIES(mytime,'minute')as minutelevel,
    +        |   SUM(age) AS SUM
    +        | FROM mainTable
    +        | WHERE TIMESERIES(mytime,'minute')='2016-02-23 09:01:00'
    +        | GROUP BY TIMESERIES(mytime,'minute')
    +        | ORDER BY TIMESERIES(mytime,'minute')
    +      """.stripMargin)
    +    preAggTableValidator(df2.queryExecution.analyzed, "maintable_agg0_minute")
    +    checkAnswer(df2, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 60)))
    +  }
    +
    +  test("test timeseries table selection 17: TIMESERIES(mytime,'minute') not match pre agg") {
    +    val df = sql(
    +      """
    +        | SELECT
    +        |   TIMESERIES(mytime,'minute')as minutelevel,
    +        |   SUM(age) AS SUM
    +        | FROM mainTable
    +        | WHERE TIMESERIES(mytime,'minute')='2016-02-23 09:01:00' AND name='vishal'
    +        | GROUP BY TIMESERIES(mytime,'minute')
    +        | ORDER BY TIMESERIES(mytime,'minute')
    +      """.stripMargin)
    +    checkAnswer(df, Seq(Row(Timestamp.valueOf("2016-02-23 09:01:00"), 10)))
    +    preAggTableValidator(df.queryExecution.analyzed, "maintable")
    +  }
    +
    +  test("test timeseries table selection 18: select with many GROUP BY AND one filter") {
    +    val df = sql(
    +      """
    +        | SELECT
    +        |   TIMESERIES(mytime,'year') AS yearLevel,
    +        |   TIMESERIES(mytime,'month') AS monthLevel,
    +        |   TIMESERIES(mytime,'day') AS dayLevel,
    +        |   TIMESERIES(mytime,'hour') AS hourLevel,
    +        |   TIMESERIES(mytime,'minute') AS minuteLevel,
    +        |   TIMESERIES(mytime,'second') AS secondLevel,
    +        |   SUM(age) AS SUM
    +        | FROM mainTable
    +        | WHERE TIMESERIES(mytime,'minute')='2016-02-23 09:01:00'
    +        | GROUP BY
    +        |   TIMESERIES(mytime,'year'),
    +        |   TIMESERIES(mytime,'month'),
    +        |   TIMESERIES(mytime,'day'),
    +        |   TIMESERIES(mytime,'hour'),
    +        |   TIMESERIES(mytime,'minute'),
    +        |   TIMESERIES(mytime,'second')
    +        | ORDER BY
    +        |   TIMESERIES(mytime,'year'),
    +        |   TIMESERIES(mytime,'month'),
    +        |   TIMESERIES(mytime,'day'),
    +        |   TIMESERIES(mytime,'hour'),
    +        |   TIMESERIES(mytime,'minute'),
    +        |   TIMESERIES(mytime,'second')
    +      """.stripMargin)
    +
    +    checkExistence(df, true,
    +      "2016-01-01 00:00:00",
    +      "2016-02-01 00:00:00",
    +      "2016-02-23 09:00:00",
    +      "2016-02-23 09:01:00",
    +      "2016-02-23 09:01:50",
    +      "30"
    +    )
    +  }
    +
    +  test("test timeseries table selection 19: select with many GROUP BY AND many filter") {
    +    val df = sql(
    +      """
    +        | SELECT
    +        |   TIMESERIES(mytime,'year') AS yearLevel,
    +        |   TIMESERIES(mytime,'month') AS monthLevel,
    +        |   TIMESERIES(mytime,'day') AS dayLevel,
    +        |   TIMESERIES(mytime,'hour') AS hourLevel,
    +        |   TIMESERIES(mytime,'minute') AS minuteLevel,
    +        |   TIMESERIES(mytime,'second') AS secondLevel,
    +        |   SUM(age) AS SUM
    +        | FROM mainTable
    +        | WHERE
    +        |   TIMESERIES(mytime,'second')='2016-02-23 09:01:50' and
    +        |   TIMESERIES(mytime,'minute')='2016-02-23 09:01:00' and
    +        |   TIMESERIES(mytime,'hour')='2016-02-23 09:00:00' and
    +        |   TIMESERIES(mytime,'month')='2016-02-01 00:00:00' and
    +        |   TIMESERIES(mytime,'year')='2016-01-01 00:00:00'
    +        | GROUP BY
    +        |   TIMESERIES(mytime,'year'),
    +        |   TIMESERIES(mytime,'month'),
    +        |   TIMESERIES(mytime,'day'),
    +        |   TIMESERIES(mytime,'hour'),
    +        |   TIMESERIES(mytime,'minute'),
    +        |   TIMESERIES(mytime,'second')
    +        | ORDER BY
    +        |   TIMESERIES(mytime,'year'),
    +        |   TIMESERIES(mytime,'month'),
    +        |   TIMESERIES(mytime,'day'),
    +        |   TIMESERIES(mytime,'hour'),
    +        |   TIMESERIES(mytime,'minute'),
    +        |   TIMESERIES(mytime,'second')
    +      """.stripMargin)
    +
    +    checkExistence(df, true,
    +      "2016-01-01 00:00:00",
    +      "2016-02-01 00:00:00",
    +      "2016-02-23 09:00:00",
    +      "2016-02-23 09:01:00",
    +      "2016-02-23 09:01:50",
    +      "30"
    +    )
    +  }
    +
    +  test("test timeseries table selection 20: filter < AND >") {
    --- End diff --
   
    Filter scenarios are already covered in TestTimeSeriesMatchStrategySuite.scala. Please remove duplicate scenarios


---

[qiuchenjian-2]

Github user kunal642 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174066548
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala ---
    @@ -33,16 +33,15 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
           CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
     
         SparkUtil4Test.createTaskMockUp(sqlContext)
    -    dropTable
         CarbonProperties.getInstance()
           .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
    +    dropTestTables
    --- End diff --
   
    not required


---

[qiuchenjian-2]

Github user xubo245 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174119683
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala ---
    @@ -82,149 +92,209 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
            """.stripMargin)
       }
     
    +  override def afterEach(): Unit = {
    +    dropDataMaps("mainTable", "agg1_second", "agg1_minute",
    +      "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
    +  }
    +
       test("test timeseries create table 1") {
         checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_second"), true, "maintable_agg0_second")
    -    sql("drop datamap agg0_second on table mainTable")
    +    sql("DROP DATAMAP agg0_second ON TABLE mainTable")
       }
     
       test("test timeseries create table 2") {
         checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_hour"), true, "maintable_agg0_hour")
    -    sql("drop datamap agg0_hour on table mainTable")
    +    sql("DROP DATAMAP agg0_hour ON TABLE mainTable")
       }
    +
       test("test timeseries create table 3") {
         checkExistence(sql("DESCRIBE FORMATTED maintable_agg0_day"), true, "maintable_agg0_day")
    -    sql("drop datamap agg0_day on table mainTable")
    +    sql("DROP DATAMAP agg0_day ON TABLE mainTable")
       }
    +
       test("test timeseries create table 4") {
         checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_month"), true, "maintable_agg0_month")
    -    sql("drop datamap agg0_month on table mainTable")
    +    sql("DROP DATAMAP agg0_month ON TABLE mainTable")
       }
    +
       test("test timeseries create table 5") {
         checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_year"), true, "maintable_agg0_year")
    -    sql("drop datamap agg0_year on table mainTable")
    +    sql("DROP DATAMAP agg0_year ON TABLE mainTable")
       }
     
    -  test("test timeseries create table 6") {
    -    intercept[Exception] {
    +  test("test timeseries create table 6: TIMESERIES should define time granularity") {
    +    sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
    +    val e = intercept[MalformedCarbonCommandException] {
           sql(
    -        s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    -           | USING '$timeSeries'
    -           | DMPROPERTIES (
    -           | 'EVENT_TIME'='dataTime',
    -           | 'SEC_GRANULARITY'='1')
    -           | AS SELECT dataTime, SUM(age) FROM mainTable
    -           | GROUP BY dataTime
    +        s"""CREATE DATAMAP agg0_second ON TABLE mainTable USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='dataTime',
    +           |   'SEC_GRANULARITY'='1')
    +           |AS SELECT dataTime, SUM(age) FROM mainTable
    +           |GROUP BY dataTime
             """.stripMargin)
         }
    +    assert(e.getMessage.contains("TIMESERIES should define time granularity"))
       }
     
    -  test("test timeseries create table 7") {
    -    intercept[Exception] {
    +  test("test timeseries create table 7: Granularity only support 1") {
    +    sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
    +    val e = intercept[MalformedDataMapCommandException] {
           sql(
             s"""
                | CREATE DATAMAP agg0_second ON TABLE mainTable
                | USING '$timeSeries'
                | DMPROPERTIES (
                | 'EVENT_TIME'='dataTime',
    -           | 'SECOND_GRANULARITY'='2')
    +           | 'DAY_GRANULARITY'='1.5')
                | AS SELECT dataTime, SUM(age) FROM mainTable
                | GROUP BY dataTime
            """.stripMargin)
         }
    +    assert(e.getMessage.equals("Granularity only support 1"))
       }
     
    -  test("test timeseries create table 8") {
    -    intercept[Exception] {
    +  test("test timeseries create table 8: Granularity only support 1") {
    +    dropDataMaps("mainTable", "agg1_hour")
    +    val e = intercept[MalformedCarbonCommandException] {
           sql(
    -        s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    -           | USING '$timeSeries'
    -           | DMPROPERTIES (
    -           | 'EVENT_TIME'='dataTime',
    -           | 'SECOND_GRANULARITY'='1')
    -           | AS SELECT dataTime, SUM(age) FROM mainTable
    -           | GROUP BY dataTime
    -       """.stripMargin)
    +        s"""CREATE DATAMAP agg1_hour ON TABLE mainTable USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='dataTime',
    +           |   'HOUR_GRANULARITY'='hour=-2')
    +           |AS SELECT dataTime, SUM(age) FROM mainTable
    +           |GROUP BY dataTime
    +        """.stripMargin)
    +    }
    +    assert(e.getMessage.contains("Granularity only support "))
    +    checkExistence(sql("SHOW DATAMAP ON TABLE mainTable"), false, "maintable_agg1_hour")
    +  }
    +
    +  test("test timeseries create table 9: SECOND_GRANULARITY is null") {
    +    sql("DROP DATAMAP IF EXISTS agg1 ON TABLE mainTable")
    +    val e = intercept[MalformedCarbonCommandException] {
           sql(
    -        s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    -           | USING '$timeSeries'
    -           | DMPROPERTIES (
    -           | 'EVENT_TIME'='dataTime',
    -           | 'SECOND_GRANULARITY'='1')
    -           | AS SELECT dataTime, SUM(age) FROM mainTable
    -           | GROUP BY dataTime
    -       """.stripMargin)
    +        s"""CREATE DATAMAP agg0_hour ON TABLE mainTable
    +           |USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='dataTime',
    +           |   'HOUR_GRANULARITY'='')
    +           |AS SELECT dataTime, SUM(age) FROM mainTable
    +           |GROUP BY dataTime
    +        """.stripMargin)
    +    }
    +    assert(e.getMessage.contains("Granularity only support 1"))
    +  }
    +
    +  test("test timeseries create table 10: Table already exists in database") {
    +    val e = intercept[MalformedDataMapCommandException] {
    +      sql(
    +        s"""CREATE DATAMAP agg1_hour ON TABLE mainTable USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='dataTime',
    +           |   'HOUR_GRANULARITY'='1')
    +           |AS SELECT dataTime, SUM(age) FROM mainTable
    +           |GROUP BY dataTime
    +        """.stripMargin)
    +      sql(
    +        s"""CREATE DATAMAP agg1_hour ON TABLE mainTable USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='dataTime',
    +           |   'HOUR_GRANULARITY'='1')
    +           |AS SELECT dataTime, SUM(age) FROM mainTable
    +           |GROUP BY dataTime
    +        """.stripMargin)
         }
    +    assert(e.getMessage.contains(
    +      "DataMap name 'agg1_hour' already exist"))
       }
     
    -  test("test timeseries create table 9") {
    -    intercept[Exception] {
    +  test("test timeseries create table 11: don't support create timeseries table on non timestamp") {
    +    sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
    +    val e = intercept[MalformedCarbonCommandException] {
           sql(
             s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    +           | CREATE DATAMAP agg0_hour ON TABLE mainTable
                | USING '$timeSeries'
                | DMPROPERTIES (
                | 'EVENT_TIME'='name',
    -           | 'SECOND_GRANULARITY'='1')
    +           | 'HOUR_GRANULARITY'='1')
                | AS SELECT dataTime, SUM(age) FROM mainTable
                | GROUP BY dataTime
            """.stripMargin)
         }
    +    assert(e.getMessage.equals("Timeseries event time is only supported on Timestamp column"))
       }
     
    -  test("test timeseries create table 10") {
    -    intercept[Exception] {
    +  test("test timeseries create table 12: Time series column dataTime does not exists in select") {
    +    sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
    +    val e = intercept[MalformedCarbonCommandException] {
           sql(
             s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    +           | CREATE DATAMAP agg0_hour ON TABLE mainTable
                | USING '$timeSeries'
                | DMPROPERTIES (
    -           | 'EVENT_TIME'='name',
    -           | 'SECOND_GRANULARITY'='1')
    -           | AS SELECT dataTime, SUM(age) FROM mainTable
    -           | GROUP BY dataTime
    -       """.stripMargin)
    +           | 'EVENT_TIME'='dataTime',
    +           | 'HOUR_GRANULARITY'='1')
    +           | AS SELECT name, SUM(age) FROM mainTable
    +           | GROUP BY name
    +         """.stripMargin)
    +    }
    +    assert(e.getMessage.equals("Time series column dataTime does not exists in select"))
    +  }
    +
    +  test("test timeseries create table 13: don't support create timeseries table on non timestamp") {
    +    sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
    +    val e = intercept[MalformedCarbonCommandException] {
    +      sql(
    +        s"""CREATE DATAMAP agg0_hour ON TABLE mainTable
    +           |USING '$timeSeries'
    +           |DMPROPERTIES (
    +           |   'event_time'='name',
    +           |   'HOUR_GRANULARITY'='1')
    +           |AS SELECT name, SUM(age) FROM mainTable
    +           |GROUP BY name
    +        """.stripMargin)
         }
    +    assert(e.getMessage.contains("Timeseries event time is only supported on Timestamp column"))
       }
     
    -  test("test timeseries create table 11: USING") {
    +  test("test timeseries create table 14: USING") {
         val e: Exception = intercept[MalformedDataMapCommandException] {
           sql(
    -        """CREATE DATAMAP agg1 ON TABLE mainTable
    +        """CREATE DATAMAP agg0_hour ON TABLE mainTable
               | USING 'abc'
               | DMPROPERTIES (
               |   'EVENT_TIME'='dataTime',
    -          |   'SECOND_GRANULARITY'='1')
    +          |   'HOUR_GRANULARITY'='1')
               | AS SELECT dataTime, SUM(age) FROM mainTable
               | GROUP BY dataTime
             """.stripMargin)
         }
         assert(e.getMessage.equals("DataMap 'abc' not found"))
       }
     
    -  test("test timeseries create table 12: USING and catch MalformedCarbonCommandException") {
    +  test("test timeseries create table 15: USING and catch MalformedCarbonCommandException") {
         val e: Exception = intercept[MalformedCarbonCommandException] {
           sql(
    -        """CREATE DATAMAP agg1 ON TABLE mainTable
    +        """CREATE DATAMAP agg0_hour ON TABLE mainTable
               | USING 'abc'
               | DMPROPERTIES (
               |   'EVENT_TIME'='dataTime',
    -          |   'SECOND_GRANULARITY'='1')
    +          |   'HOUR_GRANULARITY'='1')
               | AS SELECT dataTime, SUM(age) FROM mainTable
               | GROUP BY dataTime
             """.stripMargin)
         }
         assert(e.getMessage.equals("DataMap 'abc' not found"))
       }
     
    -  test("test timeseries create table 13: Only one granularity level can be defined 1") {
    -    sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
    +  test("test timeseries create table 16: Only one granularity level can be defined 1") {
    +    sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
         val e: Exception = intercept[MalformedCarbonCommandException] {
           sql(
             s"""
    -           | CREATE DATAMAP agg0_second ON TABLE mainTable
    +           | CREATE DATAMAP agg0_hour ON TABLE mainTable
    --- End diff --
   
    It's unreasonable to create pre-aggregate with second granularity, second aggregate is used less than hour in reality life.


---

[qiuchenjian-2]

Github user xubo245 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174119980
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala ---
    @@ -241,12 +311,12 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
         assert(e.getMessage.equals("Only one granularity level can be defined"))
       }
     
    -  test("test timeseries create table 14: Only one granularity level can be defined 2") {
    -    sql("DROP DATAMAP IF EXISTS agg0_second ON TABLE mainTable")
    +  test("test timeseries create table 17: Only one granularity level can be defined 2") {
    +    sql("DROP DATAMAP IF EXISTS agg0_hour ON TABLE mainTable")
    --- End diff --
   
    It's unreasonable to create pre-aggregate with second granularity, second aggregate is used less than hour in reality life.


---

[qiuchenjian-2]

Github user xubo245 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174122755
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala ---
    @@ -82,149 +92,209 @@ class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
            """.stripMargin)
       }
     
    +  override def afterEach(): Unit = {
    +    dropDataMaps("mainTable", "agg1_second", "agg1_minute",
    +      "agg1_hour", "agg1_day", "agg1_month", "agg1_year")
    +  }
    +
       test("test timeseries create table 1") {
         checkExistence(sql("DESCRIBE FORMATTED mainTable_agg0_second"), true, "maintable_agg0_second")
    -    sql("drop datamap agg0_second on table mainTable")
    +    sql("DROP DATAMAP agg0_second ON TABLE mainTable")
    --- End diff --
   
    The operation should be upper case and table name should be down case according to SQL syntax, Hive syntax also like this.


---

[qiuchenjian-2]

Github user xubo245 commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/1856#discussion_r174123530
 
    --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/timeseries/TestTimeSeriesCreateTable.scala ---
    @@ -17,18 +17,28 @@
     package org.apache.carbondata.integration.spark.testsuite.timeseries
     
     import org.apache.spark.sql.AnalysisException
    +import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException
     import org.apache.spark.sql.test.util.QueryTest
    -import org.scalatest.BeforeAndAfterAll
    +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
     
     import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
    +import org.apache.carbondata.core.constants.CarbonCommonConstants
     import org.apache.carbondata.core.metadata.schema.datamap.DataMapProvider.TIMESERIES
    +import org.apache.carbondata.core.util.CarbonProperties
     
    -class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll {
    +class TestTimeSeriesCreateTable extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach{
     
       val timeSeries = TIMESERIES.toString
    +  var timestampFormat: String = _
    --- End diff --
   
    @sraghunandan  suggested should not change the CarbonCommonConstants values after running the test case, because other place maybe need this constants value.


---