jackylk commented on a change in pull request #3535: [WIP] Refactory data loading for partition table
URL:
https://github.com/apache/carbondata/pull/3535#discussion_r361857293
##########
File path: core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
##########
@@ -2730,39 +2734,66 @@ public static String encodeToString(byte[] bytes) throws UnsupportedEncodingExce
return Base64.decodeBase64(objectString.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
}
+ public static void copyCarbonDataFileToCarbonStorePath(String localFilePath,
+ String carbonDataDirectoryPath, long fileSizeInBytes,
+ OutputFilesInfoHolder outputFilesInfoHolder) throws CarbonDataWriterException {
+ if (carbonDataDirectoryPath.endsWith(".tmp") && localFilePath
+ .endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
+ // for partition case, write carbondata file directly to final path, keep index in temp path.
+ // This can improve the commit job performance on s3a.
+ carbonDataDirectoryPath =
+ carbonDataDirectoryPath.substring(0, carbonDataDirectoryPath.lastIndexOf("/"));
+ if (outputFilesInfoHolder != null) {
+ outputFilesInfoHolder.addToPartitionPath(carbonDataDirectoryPath);
+ }
+ }
+ long targetSize = copyCarbonDataFileToCarbonStorePath(localFilePath, carbonDataDirectoryPath,
+ fileSizeInBytes);
+ if (outputFilesInfoHolder != null) {
+ // Storing the number of files written by each task.
+ outputFilesInfoHolder.incrementCount();
+ // Storing the files written by each task.
+ outputFilesInfoHolder.addToOutputFiles(carbonDataDirectoryPath + localFilePath
+ .substring(localFilePath.lastIndexOf(File.separator)) + ":" + targetSize);
+ }
+ }
+
/**
* This method will copy the given file to carbon store location
*
* @param localFilePath local file name with full path
* @throws CarbonDataWriterException
*/
- public static void copyCarbonDataFileToCarbonStorePath(String localFilePath,
+ public static long copyCarbonDataFileToCarbonStorePath(String localFilePath,
String carbonDataDirectoryPath, long fileSizeInBytes)
throws CarbonDataWriterException {
long copyStartTime = System.currentTimeMillis();
LOGGER.info(String.format("Copying %s to %s, operation id %d", localFilePath,
carbonDataDirectoryPath, copyStartTime));
+ long targetSize = 0;
try {
CarbonFile localCarbonFile = FileFactory.getCarbonFile(localFilePath);
+ long localFileSize = localCarbonFile.getSize();
// the size of local carbon file must be greater than 0
- if (localCarbonFile.getSize() == 0L) {
+ if (localFileSize == 0L) {
LOGGER.error("The size of local carbon file: " + localFilePath + " is 0.");
throw new CarbonDataWriterException("The size of local carbon file is 0.");
}
String carbonFilePath = carbonDataDirectoryPath + localFilePath
.substring(localFilePath.lastIndexOf(File.separator));
copyLocalFileToCarbonStore(carbonFilePath, localFilePath,
CarbonCommonConstants.BYTEBUFFER_SIZE,
- getMaxOfBlockAndFileSize(fileSizeInBytes, localCarbonFile.getSize()));
+ getMaxOfBlockAndFileSize(fileSizeInBytes, localFileSize));
CarbonFile targetCarbonFile = FileFactory.getCarbonFile(carbonFilePath);
// the size of carbon file must be greater than 0
// and the same as the size of local carbon file
- if (targetCarbonFile.getSize() == 0L ||
- (targetCarbonFile.getSize() != localCarbonFile.getSize())) {
+ targetSize = targetCarbonFile.getSize();
+ if (targetSize == 0L ||
+ (targetSize != localFileSize)) {
Review comment:
```suggestion
targetSize != localFileSize) {
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[hidden email]
With regards,
Apache Git Services
Locked
