[GitHub] [carbondata] ravipesala commented on a change in pull request #3179: [CARBONDATA-3338] Support Incremental DataLoad for MV Datamap[with single parent table]

[GitBox]

ravipesala commented on a change in pull request #3179: [CARBONDATA-3338] Support Incremental DataLoad for MV Datamap[with single parent table]
URL: https://github.com/apache/carbondata/pull/3179#discussion_r278124943
 
 

 ##########
 File path: core/src/main/java/org/apache/carbondata/core/datamap/status/DataMapStatusManager.java
 ##########
 @@ -127,4 +150,115 @@ private static DataMapSchema getDataMapSchema(String dataMapName)
     return DataMapStoreManager.getInstance().getDataMapSchema(dataMapName);
   }
 
+  /**
+   * Returns valid segment list for a given RelationIdentifier
+   *
+   * @param relationIdentifier
+   * @return
+   * @throws IOException
+   */
+  public static List<String> getSegmentList(RelationIdentifier relationIdentifier)
+      throws IOException {
+    List<String> segmentList = new ArrayList<>();
+    AbsoluteTableIdentifier absoluteTableIdentifier =
+        AbsoluteTableIdentifier.from(relationIdentifier.getTablePath());
+    List<Segment> validSegments =
+        new SegmentStatusManager(absoluteTableIdentifier).getValidAndInvalidSegments()
+            .getValidSegments();
+    for (Segment segment : validSegments) {
+      segmentList.add(segment.getSegmentNo());
+    }
+    return segmentList;
+  }
+
+  /**
+   * This method will delete segment folders of the mv datamap table and update the
+   * datamapSegmentStatus map in case of Insert-Overwrite/Update operation on main table
+   */
+  public static void cleanMVdatamap(CarbonTable carbonTable) throws IOException {
+    List<DataMapSchema> allDataMapSchemas =
+        DataMapStoreManager.getInstance().getDataMapSchemasOfTable(carbonTable);
+    for (DataMapSchema datamapschema : allDataMapSchemas) {
+      if (datamapschema.getProviderName()
+          .equalsIgnoreCase(DataMapClassProvider.MV.getShortName())) {
+        CarbonTable datamapTable = CarbonTable
+            .buildFromTablePath(datamapschema.getRelationIdentifier().getTableName(),
+                datamapschema.getRelationIdentifier().getDatabaseName(),
+                datamapschema.getRelationIdentifier().getTablePath(),
+                datamapschema.getRelationIdentifier().getTableId());
+        SegmentStatusManager segmentStatusManager =
+            new SegmentStatusManager(datamapTable.getAbsoluteTableIdentifier());
+        ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
+        try {
+          if (carbonLock.lockWithRetries()) {
+            LOGGER.info(
+                "Acquired lock for table" + datamapTable.getDatabaseName() + "." + datamapTable
+                    .getTableName() + " for table status updation");
+            LoadMetadataDetails[] loadMetadataDetails =
+                SegmentStatusManager.readLoadMetadata(datamapTable.getMetadataPath());
+            List<CarbonFile> staleFolders = new ArrayList<>();
+            for (LoadMetadataDetails entry : loadMetadataDetails) {
+              entry.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
+              // For insert overwrite, we will delete the old segment folder immediately
+              // So collect the old segments here
+              String segmentPath = CarbonTablePath
+                  .getSegmentPath(datamapschema.getRelationIdentifier().getTablePath(),
+                      entry.getLoadName());
+              if (FileFactory.isFileExist(segmentPath, FileFactory.getFileType(segmentPath))) {
+                staleFolders.add(FileFactory.getCarbonFile(segmentPath));
+              }
+            }
+            SegmentStatusManager.writeLoadDetailsIntoFile(CarbonTablePath
+                    .getTableStatusFilePath(datamapschema.getRelationIdentifier().getTablePath()),
+                loadMetadataDetails);
+            // Update datamapSegmentStatus map for mv
+            storageProvider.clearSegmentMapping(datamapschema);
 
 Review comment:
   THis method should belong to here

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
[hidden email]


With regards,
Apache Git Services