[jira] [Updated] (CARBONDATA-2960) SDK reader not working without projection columns

[Akash R Nilugal (Jira)]

     [ https://issues.apache.org/jira/browse/CARBONDATA-2960?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

MANISH NALLA updated CARBONDATA-2960:
-------------------------------------
    Description:
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.carbondata.spark.testsuite.createTable.TestCreateDDLForComplexMapType

import org.apache.hadoop.conf.Configuration
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll

class TestCreateDDLForComplexMapType extends QueryTest with BeforeAndAfterAll{
 private val conf: Configuration = new Configuration(false)

 override def beforeAll(): Unit = {
 sql("DROP TABLE IF EXISTS carbon")
 }

 test("Single Map One Level") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField map<STRING,STRING>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()

 assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,string>"))

 }

 test("Single Map One Level 2") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField map<INT,STRING>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()

 sql("insert into carbon values('1:Nalla%2:Singh%1:Gupta%5000:Kumar')")
 sql("select * from carbon").show(false)
// assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,string>"))

 }

 test("Single Map with Two Nested Level") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField map<STRING,map<INT,STRING>>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()
 assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,map<int,string>>"))
 }

 test("Map Type with array type as value") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField map<STRING,array<INT>>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()
 assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,array<int>>"))
 }

 test("Map Type with struct type as value") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField map<STRING,struct<key:INT,val:INT>>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()
 assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,struct<key:int,val:int>>"))
 }

 test("Map Type as child to struct type") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField struct<key:INT,val:map<INT,INT>>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()
 assert(desc(0).get(1).asInstanceOf[String].trim.equals("struct<key:int,val:map<int,int>>"))
 }

 test("Map Type as child to array type") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField array<map<INT,INT>>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()
 assert(desc(0).get(1).asInstanceOf[String].trim.equals("array<map<int,int>>"))
 }

 test("Map Type as child to array<array> type") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField array<array<map<INT,INT>>>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()
 assert(desc(0).get(1).asInstanceOf[String].trim.equals("array<array<map<int,int>>>"))
 }

 test("3 levels") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField array<struct<key:INT,val:INT>>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 /*val desc =*/ sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).show()/*.collect()*/

 sql("INSERT into carbon values('1:3$2:3$4:3')")
// assert(desc(0).get(1).asInstanceOf[String].trim.equals("array<array<map<int,int>>>"))
 }

 test("Test Load data in map") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField map<INT,STRING>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()

 sql("insert into carbon values('1:Nalla$2:Singh$3:Gupta$5000:Kumar')")
 sql("select * from carbon").show(false)
 }

 test("Test Load duplicate keys data in map") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField map<INT,STRING>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()

 sql("insert into carbon values('1:Nalla$2:Singh$1:Gupta$5000:Kumar')")
 sql("select * from carbon").show(false)
 }

 test("Test Load data in map of map") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField map<STRING,map<INT,STRING>>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()
 sql("insert into carbon values('manish:1%nalla#2%gupta$kunal:1%kapoor#2%sharma')")
 sql("select * from carbon").show(false)
 }

 test("Test Load duplicate keys data in map of map") {
 sql("DROP TABLE IF EXISTS carbon")
 sql(
 s"""
 | CREATE TABLE carbon(
 | mapField map<STRING,map<INT,STRING>>
 | )
 | STORED BY 'carbondata'
 """.stripMargin
 )

 val desc = sql(
 s"""
 | Describe Formatted
 | carbon
 """.stripMargin
 ).collect()
 assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,map<int,string>>"))
 sql("insert into carbon values('manish:1%nalla#1%gupta$kunal:1%kapoor#2%sharma')")
 sql("select * from carbon").show(false)
 }
}

> SDK reader not working without projection columns
> -------------------------------------------------
>
>                 Key: CARBONDATA-2960
>                 URL: https://issues.apache.org/jira/browse/CARBONDATA-2960
>             Project: CarbonData
>          Issue Type: Improvement
>            Reporter: MANISH NALLA
>            Assignee: MANISH NALLA
>            Priority: Major
>          Time Spent: 3h 20m
>  Remaining Estimate: 0h
>
> /*
>  * Licensed to the Apache Software Foundation (ASF) under one or more
>  * contributor license agreements. See the NOTICE file distributed with
>  * this work for additional information regarding copyright ownership.
>  * The ASF licenses this file to You under the Apache License, Version 2.0
>  * (the "License"); you may not use this file except in compliance with
>  * the License. You may obtain a copy of the License at
>  *
>  * http://www.apache.org/licenses/LICENSE-2.0
>  *
>  * Unless required by applicable law or agreed to in writing, software
>  * distributed under the License is distributed on an "AS IS" BASIS,
>  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>  * See the License for the specific language governing permissions and
>  * limitations under the License.
>  */
> package org.apache.carbondata.spark.testsuite.createTable.TestCreateDDLForComplexMapType
> import org.apache.hadoop.conf.Configuration
> import org.apache.spark.sql.test.util.QueryTest
> import org.scalatest.BeforeAndAfterAll
> class TestCreateDDLForComplexMapType extends QueryTest with BeforeAndAfterAll{
>  private val conf: Configuration = new Configuration(false)
>  override def beforeAll(): Unit = {
>  sql("DROP TABLE IF EXISTS carbon")
>  }
>  test("Single Map One Level") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField map<STRING,STRING>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,string>"))
>  }
>  test("Single Map One Level 2") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField map<INT,STRING>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  sql("insert into carbon values('1:Nalla%2:Singh%1:Gupta%5000:Kumar')")
>  sql("select * from carbon").show(false)
> // assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,string>"))
>  }
>  test("Single Map with Two Nested Level") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField map<STRING,map<INT,STRING>>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,map<int,string>>"))
>  }
>  test("Map Type with array type as value") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField map<STRING,array<INT>>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,array<int>>"))
>  }
>  test("Map Type with struct type as value") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField map<STRING,struct<key:INT,val:INT>>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,struct<key:int,val:int>>"))
>  }
>  test("Map Type as child to struct type") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField struct<key:INT,val:map<INT,INT>>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  assert(desc(0).get(1).asInstanceOf[String].trim.equals("struct<key:int,val:map<int,int>>"))
>  }
>  test("Map Type as child to array type") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField array<map<INT,INT>>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  assert(desc(0).get(1).asInstanceOf[String].trim.equals("array<map<int,int>>"))
>  }
>  test("Map Type as child to array<array> type") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField array<array<map<INT,INT>>>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  assert(desc(0).get(1).asInstanceOf[String].trim.equals("array<array<map<int,int>>>"))
>  }
>  test("3 levels") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField array<struct<key:INT,val:INT>>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  /*val desc =*/ sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).show()/*.collect()*/
>  sql("INSERT into carbon values('1:3$2:3$4:3')")
> // assert(desc(0).get(1).asInstanceOf[String].trim.equals("array<array<map<int,int>>>"))
>  }
>  test("Test Load data in map") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField map<INT,STRING>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  sql("insert into carbon values('1:Nalla$2:Singh$3:Gupta$5000:Kumar')")
>  sql("select * from carbon").show(false)
>  }
>  test("Test Load duplicate keys data in map") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField map<INT,STRING>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  sql("insert into carbon values('1:Nalla$2:Singh$1:Gupta$5000:Kumar')")
>  sql("select * from carbon").show(false)
>  }
>  test("Test Load data in map of map") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField map<STRING,map<INT,STRING>>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  sql("insert into carbon values('manish:1%nalla#2%gupta$kunal:1%kapoor#2%sharma')")
>  sql("select * from carbon").show(false)
>  }
>  test("Test Load duplicate keys data in map of map") {
>  sql("DROP TABLE IF EXISTS carbon")
>  sql(
>  s"""
>  | CREATE TABLE carbon(
>  | mapField map<STRING,map<INT,STRING>>
>  | )
>  | STORED BY 'carbondata'
>  """.stripMargin
>  )
>  val desc = sql(
>  s"""
>  | Describe Formatted
>  | carbon
>  """.stripMargin
>  ).collect()
>  assert(desc(0).get(1).asInstanceOf[String].trim.equals("map<string,map<int,string>>"))
>  sql("insert into carbon values('manish:1%nalla#1%gupta$kunal:1%kapoor#2%sharma')")
>  sql("select * from carbon").show(false)
>  }
> }

--
This message was sent by Atlassian JIRA
(v7.6.3#76005)