Posted by
GitBox on
Nov 26, 2020; 7:07am
URL: http://apache-carbondata-dev-mailing-list-archive.168.s1.nabble.com/GitHub-carbondata-marchpure-opened-a-new-pull-request-4004-WIP-update-benchmark-tp103137p103617.html
Kejian-Li commented on a change in pull request #4004:
URL:
https://github.com/apache/carbondata/pull/4004#discussion_r530811074##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/DeleteExecution.scala
##########
@@ -350,99 +322,67 @@ object DeleteExecution {
resultIter
}
- (res, blockMappingVO)
+ res
}
// all or none : update status file, only if complete delete operation is successful.
def checkAndUpdateStatusFiles(
executorErrors: ExecutionErrors,
- res: Array[List[(SegmentStatus, (SegmentUpdateDetails, ExecutionErrors, Long))]],
carbonTable: CarbonTable,
timestamp: String,
- blockMappingVO: BlockMappingVO,
- isUpdateOperation: Boolean): Seq[Segment] = {
- val blockUpdateDetailsList = new util.ArrayList[SegmentUpdateDetails]()
- val segmentDetails = new util.HashSet[Segment]()
- res.foreach(resultOfSeg => resultOfSeg.foreach(
- resultOfBlock => {
- if (resultOfBlock._1 == SegmentStatus.SUCCESS) {
- blockUpdateDetailsList.add(resultOfBlock._2._1)
- segmentDetails.add(new Segment(resultOfBlock._2._1.getSegmentName))
- // if this block is invalid then decrement block count in map.
- if (CarbonUpdateUtil.isBlockInvalid(resultOfBlock._2._1.getSegmentStatus)) {
- CarbonUpdateUtil.decrementDeletedBlockCount(resultOfBlock._2._1,
- blockMappingVO.getSegmentNumberOfBlockMapping)
- }
- } else {
- // In case of failure , clean all related delete delta files
- CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timestamp)
- val errorMsg =
- "Delete data operation is failed due to failure in creating delete delta file for " +
- "segment : " + resultOfBlock._2._1.getSegmentName + " block : " +
- resultOfBlock._2._1.getBlockName
- executorErrors.failureCauses = resultOfBlock._2._2.failureCauses
- executorErrors.errorMsg = resultOfBlock._2._2.errorMsg
+ isUpdateOperation: Boolean,
+ updateModel: UpdateTableModel,
+ blockUpdateDetailsList: java.util.List[SegmentUpdateDetails],
+ updatedSegmentList: util.Set[String]): Unit = {
- if (executorErrors.failureCauses == FailureCauses.NONE) {
- executorErrors.failureCauses = FailureCauses.EXECUTOR_FAILURE
- executorErrors.errorMsg = errorMsg
- }
- LOGGER.error(errorMsg)
- return Seq.empty[Segment]
- }
- }))
+ val updateSegmentStatus = CarbonUpdateUtil
+ .updateSegmentStatus(blockUpdateDetailsList, carbonTable, timestamp, false)
- val listOfSegmentToBeMarkedDeleted = CarbonUpdateUtil
- .getListOfSegmentsToMarkDeleted(blockMappingVO.getSegmentNumberOfBlockMapping)
+ if (!updateSegmentStatus) {
+ val errorMessage = "Update data operation is failed due to failure " +
+ "in updatetablestatus update."
+ LOGGER.error("Delete data operation is failed due to failure in updatetablestatus update.")
+ executorErrors.failureCauses = FailureCauses.STATUS_FILE_UPDATION_FAILURE
+ executorErrors.errorMsg = errorMessage
+ throw new Exception(executorErrors.errorMsg)
+ }
- val segmentsTobeDeleted = listOfSegmentToBeMarkedDeleted.asScala
+ var newMetaEntry: LoadMetadataDetails = null
+ if (isUpdateOperation && !updateModel.addedLoadDetail.isEmpty
+ && updateModel.addedLoadDetail.isDefined) {
+ newMetaEntry = updateModel.addedLoadDetail.get
+ }
+
+ val updateTableMetadataStatus = CarbonUpdateUtil.updateTableMetadataStatus(updatedSegmentList,
+ carbonTable, timestamp, true, true,
+ new util.ArrayList[Segment](0), newMetaEntry)
// this is delete flow so no need of putting timestamp in the status file.
Review comment:
this line is useless, it should be removed I think.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[hidden email]