Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1730#discussion_r158956514
--- Diff: integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala ---
@@ -405,13 +407,60 @@ object CarbonFilters {
}
}
+ /**
+ * Fetches partition information from hive
+ * @param partitionFilters
+ * @param sparkSession
+ * @param identifier
+ * @return
+ */
def getPartitions(partitionFilters: Seq[Expression],
sparkSession: SparkSession,
identifier: TableIdentifier): Seq[String] = {
- val partitions =
- sparkSession.sessionState.catalog.listPartitionsByFilter(identifier, partitionFilters)
+ val partitions = {
+ try {
+ if (CarbonProperties.getInstance().
+ getProperty(CarbonCommonConstants.CARBON_READ_PARTITION_HIVE_DIRECT,
+ CarbonCommonConstants.CARBON_READ_PARTITION_HIVE_DIRECT_DEFAULT).toBoolean) {
+ sparkSession.sessionState.catalog.listPartitionsByFilter(identifier, partitionFilters)
+ } else {
+ getPartitionsAlternate(partitionFilters, sparkSession, identifier)
+ }
+ } catch {
+ case e: Exception =>
+ // Get partition information alternatively.
+ getPartitionsAlternate(partitionFilters, sparkSession, identifier)
+ }
+ }
partitions.toList.flatMap { partition =>
partition.spec.seq.map{case (column, value) => column + "=" + value}
}.toSet.toSeq
}
+
+ /**
+ * This is alternate way of getting partition information. It first fetches all partitions from
+ * hive and then apply filter instead of querying hive along with filters.
+ * @param partitionFilters
+ * @param sparkSession
+ * @param identifier
+ * @return
+ */
+ private def getPartitionsAlternate(partitionFilters: Seq[Expression],
--- End diff --
move `partitionFilters` to next line, please follow this in future
---
Locked
