Posted by
GitBox on
Dec 29, 2020; 2:04pm
URL: http://apache-carbondata-dev-mailing-list-archive.168.s1.nabble.com/GitHub-carbondata-Indhumathi27-opened-a-new-pull-request-4067-WIP-Fix-concurrency-Load-Compaction-isI-tp105150p105186.html
ajantha-bhat commented on a change in pull request #4067:
URL:
https://github.com/apache/carbondata/pull/4067#discussion_r549716227##########
File path: integration/spark/src/main/scala/org/apache/spark/sql/index/CarbonIndexUtil.scala
##########
@@ -406,158 +407,195 @@ object CarbonIndexUtil {
.asInstanceOf[CarbonRelation]
.carbonTable
- val siTblLoadMetadataDetails: Array[LoadMetadataDetails] =
- SegmentStatusManager.readLoadMetadata(indexTable.getMetadataPath)
var segmentLocks: ListBuffer[ICarbonLock] = ListBuffer.empty
- if (!CarbonInternalLoaderUtil.checkMainTableSegEqualToSISeg(
- mainTableDetails.toArray,
- siTblLoadMetadataDetails)) {
- val indexColumns = indexMetadata.getIndexColumns(secondaryIndexProvider,
- indexTableName)
- val indexModel = IndexModel(Some(carbonTable.getDatabaseName),
- indexMetadata.getParentTableName,
- indexColumns.split(",").toList,
- indexTableName)
-
- // var details = SegmentStatusManager.readLoadMetadata(indexTable.getMetadataPath)
- // If it empty, then no need to do further computations because the
- // tabletstatus might not have been created and hence next load will take care
- if (siTblLoadMetadataDetails.isEmpty) {
- Seq.empty
- }
+ val compactionLock = CarbonLockFactory.getCarbonLockObj(
+ carbonTable.getAbsoluteTableIdentifier,
+ LockUsage.COMPACTION_LOCK)
+ try {
+ // In some cases, SI table segment might be in COMPACTED state and main table
+ // compaction might be still in progress. In those cases, we can try to take compaction lock
+ // on main table and then compare and add SI segments to failedLoads, to avoid repair
+ // SI SUCCESS loads.
+ if (compactionLock.lockWithRetries()) {
+ var mainTableDetails = try {
+ SegmentStatusManager.readTableStatusFile(CarbonTablePath.getTableStatusFilePath(
+ carbonTable.getTablePath))
+ } catch {
+ case _: Exception =>
+ if (!isLoadOrCompaction) {
+ throw new ConcurrentOperationException(carbonTable.getDatabaseName,
+ carbonTable.getTableName, "table status read", "reindex command")
+ }
+ return;
Review comment:
else case better to add a error log
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[hidden email]