Github user xuchuanyin commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2685#discussion_r214625972
--- Diff: integration/spark-datasource/src/test/scala/org/apache/spark/sql/carbondata/datasource/SparkCarbonDataSourceTest.scala ---
@@ -590,12 +596,60 @@ class SparkCarbonDataSourceTest extends FunSuite with BeforeAndAfterAll {
}
}
+ test("test read using old data") {
+ val store = new StoreCreator(new File(warehouse1).getAbsolutePath,
+ new File(warehouse1 + "../../../../../hadoop/src/test/resources/data.csv").getCanonicalPath,
+ false)
+ store.createCarbonStore()
+ FileFactory.deleteAllFilesOfDir(new File(warehouse1+"/testdb/testtable/Fact/Part0/Segment_0/0"))
+ val dfread = spark.read.format("carbon").load(warehouse1+"/testdb/testtable/Fact/Part0/Segment_0")
+ dfread.show(false)
+ spark.sql("drop table if exists parquet_table")
+ }
+
+ test("test read using different sort order data") {
+ spark.sql("drop table if exists old_comp")
+ FileFactory.deleteAllFilesOfDir(new File(warehouse1+"/testdb"))
+ val store = new StoreCreator(new File(warehouse1).getAbsolutePath,
+ new File(warehouse1 + "../../../../../hadoop/src/test/resources/data.csv").getCanonicalPath,
+ false)
+ store.setSortCOls(new util.ArrayList[String](Seq ("name").asJava))
+ var model = store.createTableAndLoadModel(false)
+ model.setSegmentId("0")
+ store.createCarbonStore(model)
+ FileFactory.deleteAllFilesOfDir(new File(warehouse1+"/testdb/testtable/Fact/Part0/Segment_0/0"))
+ store.setSortCOls(new util.ArrayList[String](Seq ("country,phonetype").asJava))
+ model = store.createTableAndLoadModel(false)
+ model.setSegmentId("1")
+ store.createCarbonStore(model)
+ FileFactory.deleteAllFilesOfDir(new File(warehouse1+"/testdb/testtable/Fact/Part0/Segment_1/0"))
+ store.setSortCOls(new util.ArrayList[String](Seq ("date").asJava))
+ model = store.createTableAndLoadModel(false)
+ model.setSegmentId("2")
+ store.createCarbonStore(model)
+ FileFactory.deleteAllFilesOfDir(new File(warehouse1+"/testdb/testtable/Fact/Part0/Segment_2/0"))
+ store.setSortCOls(new util.ArrayList[String](Seq ("serialname").asJava))
+ model = store.createTableAndLoadModel(false)
+ model.setSegmentId("3")
+ store.createCarbonStore(model)
+ FileFactory.deleteAllFilesOfDir(new File(warehouse1+"/testdb/testtable/Fact/Part0/Segment_3/0"))
+ spark.sql(s"create table old_comp(id int, date string, country string, name string, phonetype string, serialname string, salary int) using carbon options(path='$warehouse1/testdb/testtable/Fact/Part0/', 'sort_columns'='name')")
+ assert(spark.sql("select * from old_comp where country='china'").count() == 3396)
+ assert(spark.sql("select * from old_comp ").count() == 4000)
+ spark.sql("drop table if exists old_comp")
+
+ spark.sql(s"create table old_comp1 using carbon options(path='$warehouse1/testdb/testtable/Fact/Part0/')")
+ assert(spark.sql("select * from old_comp1 where country='china'").count() == 3396)
+ assert(spark.sql("select * from old_comp1 ").count() == 4000)
+ spark.sql("drop table if exists old_comp1")
+ FileFactory.deleteAllFilesOfDir(new File(warehouse1+"/testdb"))
+ }
override protected def beforeAll(): Unit = {
drop
}
- override def afterAll(): Unit = {
--- End diff --
unnecessary modification...
---
Locked
