Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122648924 --- Diff: examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonShowPartitionInfo.scala --- @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import scala.collection.mutable.LinkedHashMap + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.examples.util.ExampleUtils + +object CarbonShowPartitionInfo { --- End diff -- Suggest the name to ShowPartitionInfoExample, because we may unify the management of Examples. This is just my idea, we can easily know this is a point to point test case if its name end with Example. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122650261 --- Diff: examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonShowPartitionInfo.scala --- @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import scala.collection.mutable.LinkedHashMap + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.examples.util.ExampleUtils + +object CarbonShowPartitionInfo { + def main(args: Array[String]) { + + CarbonShowPartitionInfo.extracted("t3", args) + } + def extracted(tableName: String, args: Array[String]): Unit = { + val cc = ExampleUtils.createCarbonContext("CarbonShowPartitionInfo") + val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv" + + // range partition + cc.sql("DROP TABLE IF EXISTS t1") + + cc.sql(""" + | CREATE TABLE IF NOT EXISTS t1 + | ( + | vin String, + | phonenumber Int, + | country String, + | area String + | ) + | PARTITIONED BY (logdate Timestamp) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='RANGE', + | 'RANGE_INFO'='20140101, 2015/01/01 ,2016-01-01') --- End diff -- We will add partition check mechanism later, So change 'RANGE_INFO' = '2014/01/01,2015/01/01,2016/01/01', otherwise it will fail after check mechanism merged. Besides, it will be better for specify the timestamp format suitable for '2014/0101' before running the sql command. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122650800 --- Diff: examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonShowPartitionInfo.scala --- @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import scala.collection.mutable.LinkedHashMap + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.examples.util.ExampleUtils + +object CarbonShowPartitionInfo { + def main(args: Array[String]) { + + CarbonShowPartitionInfo.extracted("t3", args) + } + def extracted(tableName: String, args: Array[String]): Unit = { + val cc = ExampleUtils.createCarbonContext("CarbonShowPartitionInfo") + val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv" + + // range partition + cc.sql("DROP TABLE IF EXISTS t1") + + cc.sql(""" + | CREATE TABLE IF NOT EXISTS t1 + | ( + | vin String, + | phonenumber Int, + | country String, + | area String + | ) --- End diff -- I think key words in sql command and data type should be upper case. for example, "String" -> "STRING" anyway, it's not a big problem. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122650894 --- Diff: examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonShowPartitionInfo.scala --- @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import scala.collection.mutable.LinkedHashMap + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.examples.util.ExampleUtils + +object CarbonShowPartitionInfo { + def main(args: Array[String]) { + --- End diff -- delete extra space line --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122652358 --- Diff: examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonShowPartitionInfo.scala --- @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import scala.collection.mutable.LinkedHashMap + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.examples.util.ExampleUtils + +object CarbonShowPartitionInfo { + def main(args: Array[String]) { + + CarbonShowPartitionInfo.extracted("t3", args) + } + def extracted(tableName: String, args: Array[String]): Unit = { + val cc = ExampleUtils.createCarbonContext("CarbonShowPartitionInfo") + val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv" + + // range partition + cc.sql("DROP TABLE IF EXISTS t1") + + cc.sql(""" + | CREATE TABLE IF NOT EXISTS t1 + | ( + | vin String, + | phonenumber Int, + | country String, + | area String + | ) + | PARTITIONED BY (logdate Timestamp) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='RANGE', + | 'RANGE_INFO'='20140101, 2015/01/01 ,2016-01-01') + """.stripMargin) + cc.sql(s""" + SHOW PARTITIONS t1 + """).show() --- End diff -- Suggest command âSHOW PARTITIONS t1â can be on one line. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122656172 --- Diff: examples/spark/src/main/scala/org/apache/carbondata/examples/CarbonShowPartitionInfo.scala --- @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import scala.collection.mutable.LinkedHashMap + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.examples.util.ExampleUtils + +object CarbonShowPartitionInfo { + def main(args: Array[String]) { + + CarbonShowPartitionInfo.extracted("t3", args) + } + def extracted(tableName: String, args: Array[String]): Unit = { + val cc = ExampleUtils.createCarbonContext("CarbonShowPartitionInfo") + val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv" + + // range partition + cc.sql("DROP TABLE IF EXISTS t1") + + cc.sql(""" + | CREATE TABLE IF NOT EXISTS t1 + | ( + | vin String, + | phonenumber Int, + | country String, + | area String + | ) + | PARTITIONED BY (logdate Timestamp) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='RANGE', + | 'RANGE_INFO'='20140101, 2015/01/01 ,2016-01-01') + """.stripMargin) + cc.sql(s""" + SHOW PARTITIONS t1 + """).show() + + cc.sql(""" + | CREATE TABLE IF NOT EXISTS t3 + | ( + | logdate Timestamp, + | phonenumber Int, + | country String, + | area String + | ) + | PARTITIONED BY (vin String) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='5') + """.stripMargin) + cc.sql(s""" + SHOW PARTITIONS t3 + """).show() + // list partition + cc.sql("DROP TABLE IF EXISTS t5") + + cc.sql(""" + | CREATE TABLE IF NOT EXISTS t5 + | ( + | vin String, + | logdate Timestamp, + | phonenumber Int, + | area String + |) + | PARTITIONED BY (country string) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='LIST', + | 'LIST_INFO'='(China,United States),UK ,japan,(Canada,Russia), South Korea ') + """.stripMargin) + cc.sql(s""" + SHOW PARTITIONS t5 + """).show() + + cc.sql(s"DROP TABLE IF EXISTS partitionDB.$tableName") + cc.sql(s"DROP DATABASE IF EXISTS partitionDB") + cc.sql(s"CREATE DATABASE partitionDB") + cc.sql(s""" + | CREATE TABLE IF NOT EXISTS partitionDB.$tableName + | ( + | logdate Timestamp, + | phonenumber Int, + | country String, + | area String + | ) + | PARTITIONED BY (vin String) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='5') + """.stripMargin) + cc.sql(s""" + SHOW PARTITIONS partitionDB.$tableName + """).show() + + cc.sql(s""" + SHOW PARTITIONS $tableName + """).show() + --- End diff -- Suggest do some clear work, such as drop created tables and database. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122657713 --- Diff: examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonPartitionExample.scala --- @@ -107,7 +105,6 @@ object CarbonPartitionExample { | vin String, | logdate Timestamp, | phonenumber Long, - | country String, --- End diff -- already have PR-1049 to modify this class, you can help to review. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122657979 --- Diff: examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonShowPartitionInfo.scala --- @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import java.io.File + +import org.apache.spark.sql.SparkSession + +object CarbonShowPartitionInfo { --- End diff -- name definition, same problem as spark1.6. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122658089 --- Diff: examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonShowPartitionInfo.scala --- @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import java.io.File + +import org.apache.spark.sql.SparkSession + +object CarbonShowPartitionInfo { + def main(args: Array[String]) { + + CarbonShowPartitionInfo.extracted("t3", args) + } + def extracted(tableName: String, args: Array[String]): Unit = { + val rootPath = new File(this.getClass.getResource("/").getPath + + "../../../..").getCanonicalPath + val storeLocation = s"$rootPath/examples/spark2/target/store" + val warehouse = s"$rootPath/examples/spark2/target/warehouse" + val metastoredb = s"$rootPath/examples/spark2/target" + val testData = s"$rootPath/examples/spark2/src/main/resources/bitmaptest2.csv" + import org.apache.spark.sql.CarbonSession._ + val spark = SparkSession + .builder() + .master("local") + .appName("CarbonDataLoad") + .config("spark.sql.warehouse.dir", warehouse) + .getOrCreateCarbonSession(storeLocation, metastoredb) + + // range partition + spark.sql("DROP TABLE IF EXISTS t1") + // hash partition + spark.sql("DROP TABLE IF EXISTS t3") + // list partition + spark.sql("DROP TABLE IF EXISTS t5") + + spark.sql(""" + | CREATE TABLE IF NOT EXISTS t1 + | ( + | vin String, + | phonenumber Long, + | country String, + | area String + | ) + | PARTITIONED BY (logdate Timestamp) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='RANGE', + | 'RANGE_INFO'='20140101, 2015/01/01 ,2016-01-01') --- End diff -- Same as mentioned before. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122658297 --- Diff: examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonShowPartitionInfo.scala --- @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import java.io.File + +import org.apache.spark.sql.SparkSession + +object CarbonShowPartitionInfo { + def main(args: Array[String]) { + + CarbonShowPartitionInfo.extracted("t3", args) + } + def extracted(tableName: String, args: Array[String]): Unit = { + val rootPath = new File(this.getClass.getResource("/").getPath + + "../../../..").getCanonicalPath + val storeLocation = s"$rootPath/examples/spark2/target/store" + val warehouse = s"$rootPath/examples/spark2/target/warehouse" + val metastoredb = s"$rootPath/examples/spark2/target" + val testData = s"$rootPath/examples/spark2/src/main/resources/bitmaptest2.csv" + import org.apache.spark.sql.CarbonSession._ + val spark = SparkSession + .builder() + .master("local") + .appName("CarbonDataLoad") + .config("spark.sql.warehouse.dir", warehouse) + .getOrCreateCarbonSession(storeLocation, metastoredb) + + // range partition + spark.sql("DROP TABLE IF EXISTS t1") + // hash partition + spark.sql("DROP TABLE IF EXISTS t3") + // list partition + spark.sql("DROP TABLE IF EXISTS t5") + + spark.sql(""" + | CREATE TABLE IF NOT EXISTS t1 + | ( + | vin String, + | phonenumber Long, + | country String, + | area String + | ) + | PARTITIONED BY (logdate Timestamp) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='RANGE', + | 'RANGE_INFO'='20140101, 2015/01/01 ,2016-01-01') + """.stripMargin) + + spark.sql(""" + | CREATE TABLE IF NOT EXISTS t3 + | ( + | logdate Timestamp, + | phonenumber Long, + | country String, + | area String + | ) + | PARTITIONED BY (vin String) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='5') + """.stripMargin) + + spark.sql(""" + | CREATE TABLE IF NOT EXISTS t5 + | ( + | vin String, + | logdate Timestamp, + | phonenumber Long, + | area String + |) + | PARTITIONED BY (country string) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='LIST', + | 'LIST_INFO'='(China,United States),UK ,japan,(Canada,Russia), South Korea ') + """.stripMargin) + + spark.sparkContext.setLogLevel("WARN") + spark.sql(s""" + SHOW PARTITIONS t1 + """).show() + spark.sql(s""" --- End diff -- One line, same problem. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122658340 --- Diff: examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonShowPartitionInfo.scala --- @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.examples + +import java.io.File + +import org.apache.spark.sql.SparkSession + +object CarbonShowPartitionInfo { + def main(args: Array[String]) { + + CarbonShowPartitionInfo.extracted("t3", args) + } + def extracted(tableName: String, args: Array[String]): Unit = { + val rootPath = new File(this.getClass.getResource("/").getPath + + "../../../..").getCanonicalPath + val storeLocation = s"$rootPath/examples/spark2/target/store" + val warehouse = s"$rootPath/examples/spark2/target/warehouse" + val metastoredb = s"$rootPath/examples/spark2/target" + val testData = s"$rootPath/examples/spark2/src/main/resources/bitmaptest2.csv" + import org.apache.spark.sql.CarbonSession._ + val spark = SparkSession + .builder() + .master("local") + .appName("CarbonDataLoad") + .config("spark.sql.warehouse.dir", warehouse) + .getOrCreateCarbonSession(storeLocation, metastoredb) + + // range partition + spark.sql("DROP TABLE IF EXISTS t1") + // hash partition + spark.sql("DROP TABLE IF EXISTS t3") + // list partition + spark.sql("DROP TABLE IF EXISTS t5") + + spark.sql(""" + | CREATE TABLE IF NOT EXISTS t1 + | ( + | vin String, + | phonenumber Long, + | country String, + | area String + | ) + | PARTITIONED BY (logdate Timestamp) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='RANGE', + | 'RANGE_INFO'='20140101, 2015/01/01 ,2016-01-01') + """.stripMargin) + + spark.sql(""" + | CREATE TABLE IF NOT EXISTS t3 + | ( + | logdate Timestamp, + | phonenumber Long, + | country String, + | area String + | ) + | PARTITIONED BY (vin String) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='5') + """.stripMargin) + + spark.sql(""" + | CREATE TABLE IF NOT EXISTS t5 + | ( + | vin String, + | logdate Timestamp, + | phonenumber Long, + | area String + |) + | PARTITIONED BY (country string) + | STORED BY 'carbondata' + | TBLPROPERTIES('PARTITION_TYPE'='LIST', + | 'LIST_INFO'='(China,United States),UK ,japan,(Canada,Russia), South Korea ') + """.stripMargin) + + spark.sparkContext.setLogLevel("WARN") + spark.sql(s""" + SHOW PARTITIONS t1 + """).show() + spark.sql(s""" + SHOW PARTITIONS t3 + """).show() + spark.sql(s""" + SHOW PARTITIONS t5 + """).show() + + // range partition + spark.sql("DROP TABLE IF EXISTS t1") + // hash partition + spark.sql("DROP TABLE IF EXISTS t3") + // list partition + spark.sql("DROP TABLE IF EXISTS t5") + --- End diff -- Suggest close spark session as last. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122658883 --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala --- @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.spark.testsuite.partition + +import java.sql.Timestamp + +import org.apache.spark.sql.Row +import org.apache.spark.sql.common.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties + + +class TestShowPartition extends QueryTest with BeforeAndAfterAll { + override def beforeAll = { + dropTable + + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + + } + + test("show partition table: hash table") { + sql( + """ + | CREATE TABLE hashTable (empname String, designation String, doj Timestamp, + | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, + | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int, + | utilization int,salary int) + | PARTITIONED BY (empno int) + | STORED BY 'org.apache.carbondata.format' + | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3') + """.stripMargin) + sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE hashTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""") + + // EqualTo + checkAnswer(sql("show partitions hashTable"), Seq(Row("HASH PARTITION", "", "3"))) + + sql("drop table hashTable") + } + + test("show partition table: range partition") { + sql( + """ + | CREATE TABLE rangeTable (empno int, empname String, designation String, + | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, + | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int, + | utilization int,salary int) + | PARTITIONED BY (doj Timestamp) + | STORED BY 'org.apache.carbondata.format' + | TBLPROPERTIES('PARTITION_TYPE'='RANGE', + | 'RANGE_INFO'='01-01-2010, 01-01-2015') + """.stripMargin) + sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE rangeTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""") + + // EqualTo + checkAnswer(sql("show partitions rangeTable"), Seq(Row("0", "", "default"), Row("1", "", "< 01-01-2010"), Row("2", "", "< 01-01-2015"))) + sql("drop table rangeTable") + } + + test("show partition table: list partition") { + sql( + """ + | CREATE TABLE listTable (empno int, empname String, designation String, doj Timestamp, + | workgroupcategoryname String, deptno int, deptname String, + | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int, + | utilization int,salary int) + | PARTITIONED BY (workgroupcategory int) + | STORED BY 'org.apache.carbondata.format' + | TBLPROPERTIES('PARTITION_TYPE'='LIST', + | 'LIST_INFO'='0, 1, (2, 3)') + """.stripMargin) + sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE listTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""") + + // EqualTo + checkAnswer(sql("show partitions listTable"), Seq(Row("0", "", "0"), Row("1", "", "1"), Row("2", "", "2, 3"))) + + sql("drop table listTable") + } + test("show partition table: not default db") { + sql(s"CREATE DATABASE if not exists partitionDB") + + sql( + """ + | CREATE TABLE partitionDB.listTable (empno int, empname String, designation String, doj Timestamp, + | workgroupcategoryname String, deptno int, deptname String, + | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int, + | utilization int,salary int) + | PARTITIONED BY (workgroupcategory int) + | STORED BY 'org.apache.carbondata.format' + | TBLPROPERTIES('PARTITION_TYPE'='LIST', + | 'LIST_INFO'='0, 1, (2, 3)') + """.stripMargin) + sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionDB.listTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""") + + // EqualTo + checkAnswer(sql("show partitions partitionDB.listTable"), Seq(Row("0", "", "0"), Row("1", "", "1"), Row("2", "", "2, 3"))) + + sql("drop table partitionDB.listTable") + sql(s"DROP DATABASE partitionDB") --- End diff -- Can add test range partition and hash partition test as well under your created database. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122676528 --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala --- @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.spark.testsuite.partition + +import java.sql.Timestamp + +import org.apache.spark.sql.Row +import org.apache.spark.sql.common.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties + + +class TestShowPartition extends QueryTest with BeforeAndAfterAll { + override def beforeAll = { + dropTable + + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + + } + + test("show partition table: hash table") { + sql( + """ + | CREATE TABLE hashTable (empname String, designation String, doj Timestamp, + | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, + | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int, + | utilization int,salary int) + | PARTITIONED BY (empno int) + | STORED BY 'org.apache.carbondata.format' + | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3') + """.stripMargin) + sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE hashTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""") --- End diff -- This line maybe over 100 chars. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122715807 --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala --- @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.spark.testsuite.partition + +import java.sql.Timestamp + +import org.apache.spark.sql.Row +import org.apache.spark.sql.common.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties + + +class TestShowPartition extends QueryTest with BeforeAndAfterAll { --- End diff -- extra space between âTestShowPartitionâ and âextendâ --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122717016 --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala --- @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.spark.testsuite.partition + +import java.sql.Timestamp + +import org.apache.spark.sql.Row +import org.apache.spark.sql.common.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties + + +class TestShowPartition extends QueryTest with BeforeAndAfterAll { + override def beforeAll = { + dropTable + + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + + } + + test("show partition table: hash table") { + sql( + """ + | CREATE TABLE hashTable (empname String, designation String, doj Timestamp, + | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, + | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int, + | utilization int,salary int) + | PARTITIONED BY (empno int) + | STORED BY 'org.apache.carbondata.format' + | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3') + """.stripMargin) + sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE hashTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""") + + // EqualTo + checkAnswer(sql("show partitions hashTable"), Seq(Row("HASH PARTITION", "", "3"))) + + sql("drop table hashTable") + } + + test("show partition table: range partition") { + sql( + """ + | CREATE TABLE rangeTable (empno int, empname String, designation String, + | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, + | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int, + | utilization int,salary int) + | PARTITIONED BY (doj Timestamp) + | STORED BY 'org.apache.carbondata.format' + | TBLPROPERTIES('PARTITION_TYPE'='RANGE', + | 'RANGE_INFO'='01-01-2010, 01-01-2015') + """.stripMargin) + sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE rangeTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""") + + // EqualTo + checkAnswer(sql("show partitions rangeTable"), Seq(Row("0", "", "default"), Row("1", "", "< 01-01-2010"), Row("2", "", "< 01-01-2015"))) --- End diff -- over 100 chars ï¼ --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122717829 --- Diff: integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestShowPartitions.scala --- @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.spark.testsuite.partition + +import java.sql.Timestamp + +import org.apache.spark.sql.Row +import org.apache.spark.sql.common.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties + + +class TestShowPartition extends QueryTest with BeforeAndAfterAll { + override def beforeAll = { + dropTable + + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + + } + + test("show partition table: hash table") { + sql( + """ + | CREATE TABLE hashTable (empname String, designation String, doj Timestamp, + | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, + | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int, + | utilization int,salary int) + | PARTITIONED BY (empno int) + | STORED BY 'org.apache.carbondata.format' + | TBLPROPERTIES('PARTITION_TYPE'='HASH','NUM_PARTITIONS'='3') + """.stripMargin) + sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE hashTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""") + + // EqualTo + checkAnswer(sql("show partitions hashTable"), Seq(Row("HASH PARTITION", "", "3"))) + + sql("drop table hashTable") + } + + test("show partition table: range partition") { + sql( + """ + | CREATE TABLE rangeTable (empno int, empname String, designation String, + | workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, + | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int, + | utilization int,salary int) + | PARTITIONED BY (doj Timestamp) + | STORED BY 'org.apache.carbondata.format' + | TBLPROPERTIES('PARTITION_TYPE'='RANGE', + | 'RANGE_INFO'='01-01-2010, 01-01-2015') + """.stripMargin) + sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE rangeTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""") + + // EqualTo + checkAnswer(sql("show partitions rangeTable"), Seq(Row("0", "", "default"), Row("1", "", "< 01-01-2010"), Row("2", "", "< 01-01-2015"))) + sql("drop table rangeTable") + } + + test("show partition table: list partition") { + sql( + """ + | CREATE TABLE listTable (empno int, empname String, designation String, doj Timestamp, + | workgroupcategoryname String, deptno int, deptname String, + | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int, + | utilization int,salary int) + | PARTITIONED BY (workgroupcategory int) + | STORED BY 'org.apache.carbondata.format' + | TBLPROPERTIES('PARTITION_TYPE'='LIST', + | 'LIST_INFO'='0, 1, (2, 3)') + """.stripMargin) + sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE listTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""") + + // EqualTo + checkAnswer(sql("show partitions listTable"), Seq(Row("0", "", "0"), Row("1", "", "1"), Row("2", "", "2, 3"))) + + sql("drop table listTable") + } + test("show partition table: not default db") { + sql(s"CREATE DATABASE if not exists partitionDB") + + sql( + """ + | CREATE TABLE partitionDB.listTable (empno int, empname String, designation String, doj Timestamp, + | workgroupcategoryname String, deptno int, deptname String, + | projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int, + | utilization int,salary int) + | PARTITIONED BY (workgroupcategory int) + | STORED BY 'org.apache.carbondata.format' + | TBLPROPERTIES('PARTITION_TYPE'='LIST', + | 'LIST_INFO'='0, 1, (2, 3)') + """.stripMargin) + sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE partitionDB.listTable OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '"')""") + + // EqualTo + checkAnswer(sql("show partitions partitionDB.listTable"), Seq(Row("0", "", "0"), Row("1", "", "1"), Row("2", "", "2, 3"))) + + sql("drop table partitionDB.listTable") + sql(s"DROP DATABASE partitionDB") + } + override def afterAll = { + dropTable + } + + def dropTable = { + sql("drop table if exists hashTable") + sql("drop table if exists rangeTable") + sql("drop table if exists listTable") --- End diff -- whether should take tables under your created database into consideration ? --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/1042 Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder/2595/ --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user asfgit commented on the issue:
https://github.com/apache/carbondata/pull/1042 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/carbondata-pr-spark-1.6/496/ --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122863841 --- Diff: integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/ShowPartitionsCommand.scala --- @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import java.util + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer + +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.hive.CarbonRelation +import org.apache.spark.sql.types._ + +import org.apache.carbondata.common.logging.LogServiceFactory +import org.apache.carbondata.core.metadata.schema.partition.PartitionType + + + +private[sql] case class ShowCarbonPartitionsCommand( + tableIdentifier: TableIdentifier) extends RunnableCommand { + val LOGGER = LogServiceFactory.getLogService(ShowCarbonPartitionsCommand.getClass.getName) + var columnName = "" + override val output: Seq[Attribute] = Seq( + // Column names are based on Hive. + AttributeReference("ID", StringType, nullable = false, + new MetadataBuilder().putString("comment", "partition id").build())(), + AttributeReference("Name", StringType, nullable = false, + new MetadataBuilder().putString("comment", "partition name").build())(), + AttributeReference("Value(" + columnName + "=)", StringType, nullable = true, + new MetadataBuilder().putString("comment", "partition value").build())() + ) + override def run(sparkSession: SparkSession): Seq[Row] = { + val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore --- End diff -- Suggest use case match for different partition type --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
In reply to this post by qiuchenjian-2
Github user chenerlu commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1042#discussion_r122863911 --- Diff: integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/ShowPartitionsCommand.scala --- @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import java.util + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer + +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.hive.CarbonRelation +import org.apache.spark.sql.types._ + +import org.apache.carbondata.common.logging.LogServiceFactory +import org.apache.carbondata.core.metadata.schema.partition.PartitionType + + + +private[sql] case class ShowCarbonPartitionsCommand( + tableIdentifier: TableIdentifier) extends RunnableCommand { + val LOGGER = LogServiceFactory.getLogService(ShowCarbonPartitionsCommand.getClass.getName) + var columnName = "" + override val output: Seq[Attribute] = Seq( + // Column names are based on Hive. + AttributeReference("ID", StringType, nullable = false, + new MetadataBuilder().putString("comment", "partition id").build())(), + AttributeReference("Name", StringType, nullable = false, + new MetadataBuilder().putString("comment", "partition name").build())(), + AttributeReference("Value(" + columnName + "=)", StringType, nullable = true, + new MetadataBuilder().putString("comment", "partition value").build())() + ) + override def run(sparkSession: SparkSession): Seq[Row] = { + val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore + .lookupRelation(tableIdentifier)(sparkSession). + asInstanceOf[CarbonRelation] + val carbonTable = relation.tableMeta.carbonTable + var partitionInfo = carbonTable.getPartitionInfo( + carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier.getTableName) + var partitionType = partitionInfo.getPartitionType + var result = Seq.newBuilder[Row] + columnName = partitionInfo.getColumnSchemaList.get(0).getColumnName + if (PartitionType.RANGE.equals(partitionType)) { + result.+=(RowFactory.create("0", "", "default")) + var id = 1 + // var name = "partition_" --- End diff -- delete useless notes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at [hidden email] or file a JIRA ticket with INFRA. --- |
Free forum by Nabble | Edit this page |