Github user xuchuanyin commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2824#discussion_r227352023
--- Diff: integration/spark2/src/main/scala/org/apache/spark/sql/execution/datasources/SparkCarbonTableFormat.scala ---
@@ -172,33 +172,8 @@ with Serializable {
dataSchema: StructType,
context: TaskAttemptContext): OutputWriter = {
val model = CarbonTableOutputFormat.getLoadModel(context.getConfiguration)
- val isCarbonUseMultiDir = CarbonProperties.getInstance().isUseMultiTempDir
- var storeLocation: Array[String] = Array[String]()
- val isCarbonUseLocalDir = CarbonProperties.getInstance()
- .getProperty("carbon.use.local.dir", "false").equalsIgnoreCase("true")
-
-
val taskNumber = generateTaskNumber(path, context, model.getSegmentId)
- val tmpLocationSuffix =
- File.separator + "carbon" + System.nanoTime() + File.separator + taskNumber
- if (isCarbonUseLocalDir) {
- val yarnStoreLocations = Util.getConfiguredLocalDirs(SparkEnv.get.conf)
- if (!isCarbonUseMultiDir && null != yarnStoreLocations && yarnStoreLocations.nonEmpty) {
- // use single dir
- storeLocation = storeLocation :+
- (yarnStoreLocations(Random.nextInt(yarnStoreLocations.length)) + tmpLocationSuffix)
- if (storeLocation == null || storeLocation.isEmpty) {
- storeLocation = storeLocation :+
- (System.getProperty("java.io.tmpdir") + tmpLocationSuffix)
- }
- } else {
- // use all the yarn dirs
- storeLocation = yarnStoreLocations.map(_ + tmpLocationSuffix)
- }
- } else {
- storeLocation =
- storeLocation :+ (System.getProperty("java.io.tmpdir") + tmpLocationSuffix)
- }
+ val storeLocation = CommonUtil.getTempStoreLocations(taskNumber)
--- End diff --
yes, I also noticed this.
The suffix has no meanings, just used to separate each thread's output.
I think it's a mistake by hand -- that's why we need to extract these code to avoid problems like this.
---
Locked
