Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2410#discussion_r199047302
--- Diff: hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java ---
@@ -433,46 +434,57 @@ protected Expression getFilterPredicates(Configuration configuration) {
// First prune using default datamap on driver side.
DataMapExprWrapper dataMapExprWrapper = DataMapChooser
.getDefaultDataMap(getOrCreateCarbonTable(job.getConfiguration()), resolver);
- List<ExtendedBlocklet> prunedBlocklets =
+ List<ExtendedBlocklet> finalPrunedBlocklets =
dataMapExprWrapper.prune(segmentIds, partitionsToPrune);
-
ExplainCollector.recordDefaultDataMapPruning(
- dataMapExprWrapper.getDataMapSchema(), prunedBlocklets.size());
+ dataMapExprWrapper.getDataMapSchema(), finalPrunedBlocklets.size());
+ if (finalPrunedBlocklets.size() == 0) {
+ return finalPrunedBlocklets;
+ }
DataMapChooser chooser = new DataMapChooser(getOrCreateCarbonTable(job.getConfiguration()));
// Get the available CG datamaps and prune further.
DataMapExprWrapper cgDataMapExprWrapper = chooser.chooseCGDataMap(resolver);
if (cgDataMapExprWrapper != null) {
// Prune segments from already pruned blocklets
- pruneSegments(segmentIds, prunedBlocklets);
+ pruneSegments(segmentIds, finalPrunedBlocklets);
+ List<ExtendedBlocklet> cgPrunedBlocklets = new ArrayList<>();
// Again prune with CG datamap.
if (distributedCG && dataMapJob != null) {
- prunedBlocklets = DataMapUtil
+ cgPrunedBlocklets = DataMapUtil
.executeDataMapJob(carbonTable, resolver, segmentIds, cgDataMapExprWrapper, dataMapJob,
partitionsToPrune);
} else {
- prunedBlocklets = cgDataMapExprWrapper.prune(segmentIds, partitionsToPrune);
+ cgPrunedBlocklets = cgDataMapExprWrapper.prune(segmentIds, partitionsToPrune);
}
-
+ // since index datamap prune in segment scope,
+ // the result need to intersect with previous pruned result
+ finalPrunedBlocklets = (List) CollectionUtils.intersection(
+ finalPrunedBlocklets, cgPrunedBlocklets);
ExplainCollector.recordCGDataMapPruning(
- cgDataMapExprWrapper.getDataMapSchema(), prunedBlocklets.size());
+ cgDataMapExprWrapper.getDataMapSchema(), finalPrunedBlocklets.size());
+ }
+
+ if (finalPrunedBlocklets.size() == 0) {
+ return finalPrunedBlocklets;
}
// Now try to prune with FG DataMap.
if (isFgDataMapPruningEnable(job.getConfiguration()) && dataMapJob != null) {
DataMapExprWrapper fgDataMapExprWrapper = chooser.chooseFGDataMap(resolver);
if (fgDataMapExprWrapper != null) {
// Prune segments from already pruned blocklets
- pruneSegments(segmentIds, prunedBlocklets);
- prunedBlocklets = DataMapUtil
+ pruneSegments(segmentIds, finalPrunedBlocklets);
+ List<ExtendedBlocklet> fgPrunedBlocklets = DataMapUtil
.executeDataMapJob(carbonTable, resolver, segmentIds, fgDataMapExprWrapper, dataMapJob,
partitionsToPrune);
-
+ finalPrunedBlocklets = (List) CollectionUtils.intersection(
+ finalPrunedBlocklets, fgPrunedBlocklets);
ExplainCollector.recordFGDataMapPruning(
- fgDataMapExprWrapper.getDataMapSchema(), prunedBlocklets.size());
+ fgDataMapExprWrapper.getDataMapSchema(), finalPrunedBlocklets.size());
}
} // TODO: add a else branch to push FGDataMap pruning to reader side
--- End diff --
This TODO can be removed now
---
Locked
