[GitHub] [carbondata] Indhumathi27 commented on a change in pull request #3179: [CARBONDATA-3338] Support Incremental DataLoad for MV Datamap[with single parent table]

[GitBox]

Indhumathi27 commented on a change in pull request #3179: [CARBONDATA-3338] Support Incremental DataLoad for MV Datamap[with single parent table]
URL: https://github.com/apache/carbondata/pull/3179#discussion_r280969676
 
 

 ##########
 File path: core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
 ##########
 @@ -106,13 +122,209 @@ public void initData() { }
    * 1. after datamap creation and if `autoRefreshDataMap` is set to true
    * 2. user manually trigger REBUILD DATAMAP command
    */
-  public abstract void rebuild() throws IOException, NoSuchDataMapException;
+  public boolean rebuild() throws IOException, NoSuchDataMapException {
+    if (null == dataMapSchema.getRelationIdentifier()) {
+      return false;
+    }
+    String newLoadName = "";
+    String segmentMap = "";
+    AbsoluteTableIdentifier dataMapTableAbsoluteTableIdentifier = AbsoluteTableIdentifier
+        .from(dataMapSchema.getRelationIdentifier().getTablePath(),
+            dataMapSchema.getRelationIdentifier().getDatabaseName(),
+            dataMapSchema.getRelationIdentifier().getTableName());
+    SegmentStatusManager segmentStatusManager =
+        new SegmentStatusManager(dataMapTableAbsoluteTableIdentifier);
+    Map<String, List<String>> segmentMapping = new HashMap<>();
+    // Acquire table status lock to handle concurrent dataloading
+    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
+    try {
+      if (carbonLock.lockWithRetries()) {
+        LOGGER.info(
+            "Acquired lock for table" + dataMapSchema.getRelationIdentifier().getDatabaseName()
+                + "." + dataMapSchema.getRelationIdentifier().getTableName()
+                + " for table status updation");
+        String dataMapTableMetadataPath =
+            CarbonTablePath.getMetadataPath(dataMapSchema.getRelationIdentifier().getTablePath());
+        LoadMetadataDetails[] loadMetaDataDetails =
+            SegmentStatusManager.readLoadMetadata(dataMapTableMetadataPath);
+        List<LoadMetadataDetails> listOfLoadFolderDetails =
+            new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+        Collections.addAll(listOfLoadFolderDetails, loadMetaDataDetails);
+        if (dataMapSchema.isLazy()) {
+          // check if rebuild to datamap is already in progress and throw exception
+          if (!listOfLoadFolderDetails.isEmpty()) {
+            for (LoadMetadataDetails loadMetaDetail : loadMetaDataDetails) {
+              if ((loadMetaDetail.getSegmentStatus() == SegmentStatus.INSERT_IN_PROGRESS
+                  || loadMetaDetail.getSegmentStatus()
+                  == SegmentStatus.INSERT_OVERWRITE_IN_PROGRESS) && SegmentStatusManager
+                  .isLoadInProgress(dataMapTableAbsoluteTableIdentifier,
+                      loadMetaDetail.getLoadName())) {
+                throw new RuntimeException("Rebuild to datamap " + dataMapSchema.getDataMapName()
+                    + " is already in progress");
+              }
+            }
+          }
+        }
+        boolean isFullRefresh = false;
+        if (null != dataMapSchema.getProperties().get("full_refresh")) {
+          isFullRefresh = Boolean.valueOf(dataMapSchema.getProperties().get("full_refresh"));
+        }
+        if (!isFullRefresh) {
+          if (!getSpecificSegmentsTobeLoaded(segmentMapping, listOfLoadFolderDetails)) return false;
 
 Review comment:
   done

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
[hidden email]


With regards,
Apache Git Services