jackylk commented on a change in pull request #3431: [CARBONDATA-3566] Support add segment for partition table
URL:
https://github.com/apache/carbondata/pull/3431#discussion_r345165878
##########
File path: integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonAddLoadCommand.scala
##########
@@ -89,27 +92,118 @@ case class CarbonAddLoadCommand(
if (SegmentStatusManager.isOverwriteInProgressInTable(carbonTable)) {
throw new ConcurrentOperationException(carbonTable, "insert overwrite", "delete segment")
}
- val segmentPath = options.getOrElse(
- "path", throw new UnsupportedOperationException("PATH is manadatory"))
+ var inputPath = options.getOrElse(
+ "path", throw new UnsupportedOperationException("PATH is mandatory"))
- val segSchema = MixedFormatHandler.getSchema(sparkSession, options, segmentPath)
-
- val segCarbonSchema = new Schema(segSchema.fields.map { field =>
+ // infer schema and collect FileStatus for all partitions
+ val (inputPathSchema, lastLevelDirFileMap) =
+ MixedFormatHandler.collectInfo(sparkSession, options, inputPath)
+ var inputPathCarbonFields = inputPathSchema.fields.map { field =>
val dataType = convertSparkToCarbonDataType(field.dataType)
new Field(field.name, dataType)
- })
-
- val tableCarbonSchema = new Schema(tableSchema.fields.map { field =>
+ }
+ val carbonTableSchema = new Schema(tableSchema.fields.map { field =>
Review comment:
carbonTableSchema is used to check against with inputPathTableFields to validate the schema from the files specified by path given by user, so it is required to construct inputPathTableFields
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[hidden email]
With regards,
Apache Git Services
Locked
