jackylk commented on a change in pull request #3459: [CARBONDATA-3582] support table status file backup
URL:
https://github.com/apache/carbondata/pull/3459#discussion_r361083964
##########
File path: core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
##########
@@ -525,45 +540,78 @@ private static Integer compareDateValues(Long loadValue, Long userValue) {
}
/**
- * writes load details into a given file at @param dataLoadLocation
+ * Backup the table status file as 'tablestatus.backup' in the same path
*
- * @param dataLoadLocation
- * @param listOfLoadFolderDetailsArray
- * @throws IOException
+ * @param tableStatusPath table status file path
*/
- public static void writeLoadDetailsIntoFile(String dataLoadLocation,
+ private static void backupTableStatus(String tableStatusPath) throws IOException {
+ CarbonFile file = FileFactory.getCarbonFile(tableStatusPath);
+ if (file.exists()) {
+ String backupPath = tableStatusPath + ".backup";
+ String currentContent = readFileAsString(tableStatusPath);
+ if (currentContent != null) {
+ writeStringIntoFile(backupPath, currentContent);
+ }
+ }
+ }
+
+ /**
+ * writes load details to specified path
+ *
+ * @param tableStatusPath path of the table status file
+ * @param listOfLoadFolderDetailsArray segment metadata
+ * @throws IOException if IO errors
+ */
+ public static void writeLoadDetailsIntoFile(
+ String tableStatusPath,
LoadMetadataDetails[] listOfLoadFolderDetailsArray) throws IOException {
- AtomicFileOperations fileWrite =
- AtomicFileOperationFactory.getAtomicFileOperations(dataLoadLocation);
+ // When overwriting table status file, if process crashed, table status file
+ // will be in corrupted state. This can happen in an unstable environment,
+ // like in the cloud. To prevent the table corruption, user can enable following
+ // property to enable backup of the table status before overwriting it.
+ if (tableStatusPath.endsWith(CarbonTablePath.TABLE_STATUS_FILE) &&
+ CarbonProperties.isEnableTableStatusBackup()) {
+ backupTableStatus(tableStatusPath);
+ }
+ String content = new Gson().toJson(listOfLoadFolderDetailsArray);
+ mockForTest();
+ // make the table status file smaller by removing fields that are default value
+ for (LoadMetadataDetails loadMetadataDetails : listOfLoadFolderDetailsArray) {
+ loadMetadataDetails.removeUnnecessaryField();
+ }
+ // If process crashed during following write, table status file need to be
+ // manually recovered.
+ writeStringIntoFile(tableStatusPath, content);
+ }
+
+ // a dummy func for mocking in testcase, which simulates IOException
+ private static void mockForTest() throws IOException {
Review comment:
There is other place we we add util for test like TestUtil.java in SDK module.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[hidden email]
With regards,
Apache Git Services
Locked
