Apache CarbonData Dev Mailing List archive

[GitHub] [carbondata] akkio-97 opened a new pull request #4121: [CARBONDATA-4170] Support dropping of single and multi-level complex columns(array/struct)

Classic

List

Threaded

64 messages Options

1234

GitBox

[GitHub] [carbondata] Indhumathi27 commented on a change in pull request #4121: [CARBONDATA-4170] Support dropping of parent complex columns(array/struct/map)

Indhumathi27 commented on a change in pull request #4121:
URL: https://github.com/apache/carbondata/pull/4121#discussion_r618943513

##########
File path: integration/spark/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/DropColumnTestCases.scala
##########
@@ -97,13 +102,157 @@ class DropColumnTestCases extends QueryTest with BeforeAndAfterAll {
test_drop_and_compaction()
}

- test("test dropping of complex column should throw exception") {
- sql("drop table if exists maintbl")
- sql("create table maintbl (a string, b string, c struct<si:int>) STORED AS carbondata")
- assert(intercept[ProcessMetaDataException] {
- sql("alter table maintbl drop columns(b,c)").collect()
- }.getMessage.contains("Complex column cannot be dropped"))
- sql("drop table if exists maintbl")
+ def checkSchemaSize(value: Integer): Unit = {
+ val schema = sql("describe alter_com").collect()
+ assert(schema.size.equals(value))
+ }
+
+ def checkDroppedColumnsInSchemaEvolutionEntry(tableName: String, value: Integer): Unit = {
+ val carbonTable = CarbonEnv.getCarbonTable(None, tableName)(sqlContext.sparkSession)
+ val schemaEvolutionList = carbonTable.getTableInfo
+ .getFactTable
+ .getSchemaEvolution()
+ .getSchemaEvolutionEntryList()
+ var droppedColumns = Seq[ColumnSchema]()
+ for (i <- 0 until schemaEvolutionList.size()) {
+ droppedColumns ++=
+ JavaConverters
+ .asScalaIteratorConverter(schemaEvolutionList.get(i).getRemoved.iterator())
+ .asScala
+ .toSeq
+ }
+ assert(droppedColumns.size.equals(value))
+ }
+
+ test("test dropping of array of all primitive types") {
+ import scala.collection.mutable.WrappedArray.make
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql("CREATE TABLE alter_com(intfield int, arr array<int>, arr1 array<short>, " +
+ "arr2 array<int>, arr3 array<long>, arr4 array<double>, arr5 array<decimal(8,2)>, " +
+ "arr6 array<string>, arr7 array<char(5)>, arr8 array<varchar(50)>, arr9 array<boolean>, " +
+ "arr10 array<date>, arr11 array<timestamp>) STORED AS carbondata")
+ sql("insert into alter_com values(1,array(1,5),array(1,5),array(1,2),array(1,2,3)," +
+ "array(1.2d,2.3d),array(4.5,6.7),array('hello','world'),array('a','bcd')," +
+ "array('abcd','efg'),array(true,false),array('2017-02-01','2018-09-11')," +
+ "array('2017-02-01 00:01:00','2018-02-01 02:21:00') )")
+ sql("ALTER TABLE alter_com DROP COLUMNS(arr1,arr2,arr3,arr4,arr5,arr6) ")
+ sql("ALTER TABLE alter_com DROP COLUMNS(arr7,arr8,arr9) ")
+ sql("ALTER TABLE alter_com DROP COLUMNS(arr10,arr11) ")
+ val exception = intercept[Exception] {
+ sql("ALTER TABLE alter_com DROP COLUMNS(arr10,arr10) ")
+ }
+ val exceptionMessage =
+ "arr10 is duplicate. Duplicate columns not allowed"
+ assert(exception.getMessage.contains(exceptionMessage))
+
+ checkSchemaSize(2)
+ checkAnswer(sql("select * from alter_com"), Seq(Row(1, make(Array(1, 5)))))
+ checkDroppedColumnsInSchemaEvolutionEntry("alter_com", 11)
+ // check adding columns with same names again
+ sql(
+ "ALTER TABLE alter_com ADD COLUMNS(arr1 array<short>, arr2 array<int>, arr3 " +
+ "array<long>, arr4 array<double>, arr5 array<decimal(8,2)>, arr6 array<string>, arr7 " +
+ "array<char(5)>, arr8 array<varchar(50)>, arr9 array<boolean>, arr10 array<date>, arr11 " +
+ "array<timestamp> )")
+ val columns = sql("desc table alter_com").collect()
+ assert(columns.size == 13)
+ sql(
+ "insert into alter_com values(2,array(2,5),array(2,5),array(2,2),array(2,2,3),array(2.2d," +
+ "2.3d),array(2.5,6.7),array('hello2','world'),array('a2','bcd'),array('abcd2','efg'),array" +
+ "(true,false), array('2017-02-01','2018-09-11'),array('2017-02-01 00:01:00','2018-02-01 " +
+ "02:21:00') )")
+ checkAnswer(sql(
+ "select * from alter_com"),
+ Seq(Row(1, make(Array(1, 5)), null, null, null, null, null, null, null, null, null, null,
+ null), Row(2,
+ make(Array(2, 5)),
+ make(Array(2, 5)),
+ make(Array(2, 2)),
+ make(Array(2, 2, 3)),
+ make(Array(2.2, 2.3)),
+ make(Array(java.math.BigDecimal.valueOf(2.5).setScale(2),
+ java.math.BigDecimal.valueOf(6.7).setScale(2))),
+ make(Array("hello2", "world")),
+ make(Array("a2", "bcd")),
+ make(Array("abcd2", "efg")),
+ make(Array(true, false)),
+ make(Array(Date.valueOf("2017-02-01"),
+ Date.valueOf("2018-09-11"))),
+ make(Array(Timestamp.valueOf("2017-02-01 00:01:00"),
+ Timestamp.valueOf("2018-02-01 02:21:00")))
+ )))
+ }
+
+ test("test dropping of struct of all primitive types") {
+ sql("DROP TABLE IF EXISTS alter_com")
+ sql("CREATE TABLE alter_com(intField INT,struct1 struct<a:short,b:int,c:long,d:double," +
+ "e:decimal(8,2),f:string,g:char(5),h:varchar(50),i:boolean,j:date,k:timestamp>) " +
+ "STORED AS carbondata")
+ sql("insert into alter_com values(1, named_struct('a',1,'b',2,'c',3,'d',1.23,'e',2.34,'f'," +
+ "'hello','g','abc','h','def','i',true,'j','2017-02-01','k','2018-02-01 02:00:00.0') ) ")
+ sql("ALTER TABLE alter_com DROP COLUMNS(struct1) ")
+ checkSchemaSize(1)
+ checkDroppedColumnsInSchemaEvolutionEntry("alter_com", 1)
+ // check adding column with same name again
+ sql("ALTER TABLE alter_com ADD COLUMNS(structField struct<a:short,b:int,c:long,d:double, " +

Review comment:
here you are not adding column with same name

--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
[hidden email]

GitBox