[GitHub] [carbondata] ravipesala commented on a change in pull request #3179: [CARBONDATA-3338] Support Incremental DataLoad for MV Datamap[with single parent table]

[GitBox]

ravipesala commented on a change in pull request #3179: [CARBONDATA-3338] Support Incremental DataLoad for MV Datamap[with single parent table]
URL: https://github.com/apache/carbondata/pull/3179#discussion_r279242487
 
 

 ##########
 File path: core/src/main/java/org/apache/carbondata/core/datamap/status/DataMapStatusManager.java
 ##########
 @@ -122,9 +136,99 @@ public static void dropDataMap(String dataMapName) throws IOException, NoSuchDat
     }
   }
 
-  private static DataMapSchema getDataMapSchema(String dataMapName)
+  public static DataMapSchema getDataMapSchema(String dataMapName)
       throws IOException, NoSuchDataMapException {
     return DataMapStoreManager.getInstance().getDataMapSchema(dataMapName);
   }
 
+  /**
+   * This method will remove all segments of dataMap table in case of Insert-Overwrite/Update/Delete
+   * operations on main table
+   *
+   * @param allDataMapSchemas of main carbon table
+   * @throws IOException
+   */
+  public static void truncateDataMap(List<DataMapSchema> allDataMapSchemas) throws IOException {
+    for (DataMapSchema datamapschema : allDataMapSchemas) {
+      if (null != datamapschema.getRelationIdentifier() && !datamapschema.isIndexDataMap()) {
+        RelationIdentifier dataMapRelationIdentifier = datamapschema.getRelationIdentifier();
+        SegmentStatusManager segmentStatusManager = new SegmentStatusManager(AbsoluteTableIdentifier
+            .from(dataMapRelationIdentifier.getTablePath(),
+                dataMapRelationIdentifier.getDatabaseName(),
+                dataMapRelationIdentifier.getTableName()));
+        ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
+        try {
+          if (carbonLock.lockWithRetries()) {
+            LOGGER.info(
+                "Acquired lock for table" + dataMapRelationIdentifier.getDatabaseName() + "."
+                    + dataMapRelationIdentifier.getTableName() + " for table status updation");
+            String metaDataPath =
+                CarbonTablePath.getMetadataPath(dataMapRelationIdentifier.getTablePath());
+            LoadMetadataDetails[] loadMetadataDetails =
+                SegmentStatusManager.readLoadMetadata(metaDataPath);
+            for (LoadMetadataDetails entry : loadMetadataDetails) {
+              entry.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE);
+            }
+            SegmentStatusManager.writeLoadDetailsIntoFile(
+                CarbonTablePath.getTableStatusFilePath(dataMapRelationIdentifier.getTablePath()),
+                loadMetadataDetails);
+          } else {
+            LOGGER.error("Not able to acquire the lock for Table status updation for table "
+                + dataMapRelationIdentifier.getDatabaseName() + "." + dataMapRelationIdentifier
+                .getTableName());
+          }
+        } finally {
+          if (carbonLock.unlock()) {
+            LOGGER.info("Table unlocked successfully after table status updation"
+                + dataMapRelationIdentifier.getDatabaseName() + "." + dataMapRelationIdentifier
+                .getTableName());
+          } else {
+            LOGGER.error("Unable to unlock Table lock for table" + dataMapRelationIdentifier
+                .getDatabaseName() + "." + dataMapRelationIdentifier.getTableName()
+                + " during table status updation");
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * This method checks if main table and datamap table are synchronised or not. If synchronised
+   * return true to enable the datamap
+   * @param dataMapSchema of datamap to be disabled or enabled
+   * @return flag to enable or disable datamap
+   * @throws IOException
+   */
+  public static boolean canDataMapBeEnabled(DataMapSchema dataMapSchema) throws IOException {
+    boolean isDataMapInSync = true;
+    String metaDataPath =
+        CarbonTablePath.getMetadataPath(dataMapSchema.getRelationIdentifier().getTablePath());
+    LoadMetadataDetails[] dataMapLoadMetadataDetails =
+        SegmentStatusManager.readLoadMetadata(metaDataPath);
+    Map<String, List<String>> dataMapSegmentMap = new HashMap<>();
+    for (LoadMetadataDetails loadMetadataDetail : dataMapLoadMetadataDetails) {
+      Map<String, List<String>> segmentMap =
+          DataMapSegmentStatusUtil.getSegmentMap(loadMetadataDetail.getExtraInfo());
+      if (dataMapSegmentMap.isEmpty()) {
+        dataMapSegmentMap.putAll(segmentMap);
+      } else {
+        for (Map.Entry<String, List<String>> entry : segmentMap.entrySet()) {
+          dataMapSegmentMap.get(entry.getKey()).addAll(entry.getValue());
 
 Review comment:
   What if the key is not available in the map?

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
[hidden email]


With regards,
Apache Git Services