[GitHub] [carbondata] kunal642 commented on a change in pull request #3146: [WIP] Added PreAgg & Bloom Event-Listener for ShowCacheCommmand

[GitBox]

kunal642 commented on a change in pull request #3146: [WIP]  Added PreAgg & Bloom Event-Listener for ShowCacheCommmand
URL: https://github.com/apache/carbondata/pull/3146#discussion_r267206809
 
 

 ##########
 File path: integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/cache/CarbonShowCacheCommand.scala
 ##########
 @@ -61,241 +62,167 @@ case class CarbonShowCacheCommand(tableIdentifier: Option[TableIdentifier])
 
   override protected def opName: String = "SHOW CACHE"
 
-  def showAllTablesCache(sparkSession: SparkSession): Seq[Row] = {
+  def getAllTablesCache(sparkSession: SparkSession): Seq[Row] = {
     val currentDatabase = sparkSession.sessionState.catalog.getCurrentDatabase
     val cache = CacheProvider.getInstance().getCarbonCache()
     if (cache == null) {
       Seq(
-        Row("ALL", "ALL", bytesToDisplaySize(0L),
-          bytesToDisplaySize(0L), bytesToDisplaySize(0L)),
-        Row(currentDatabase, "ALL", bytesToDisplaySize(0L),
-          bytesToDisplaySize(0L), bytesToDisplaySize(0L)))
+        Row("ALL", "ALL", 0L, 0L, 0L),
+        Row(currentDatabase, "ALL", 0L, 0L, 0L))
     } else {
       val carbonTables = CarbonEnv.getInstance(sparkSession).carbonMetaStore
         .listAllTables(sparkSession)
-        .filter { table =>
-        table.getDatabaseName.equalsIgnoreCase(currentDatabase)
-      }
-      val tablePaths = carbonTables
-        .map { table =>
-          (table.getTablePath + CarbonCommonConstants.FILE_SEPARATOR,
-            table.getDatabaseName + "." + table.getTableName)
-      }
-
-      val dictIds = carbonTables
-        .filter(_ != null)
-        .flatMap { table =>
-          table
-            .getAllDimensions
-            .asScala
-            .filter(_.isGlobalDictionaryEncoding)
-            .toArray
-            .map(dim => (dim.getColumnId, table.getDatabaseName + "." + table.getTableName))
+        .filter {
+          table =>
+            table.getDatabaseName.equalsIgnoreCase(currentDatabase)
+        }
+        .filter {
+          carbonTable =>
+            !carbonTable.isChildDataMap
         }
 
-      // all databases
-      var (allIndexSize, allDatamapSize, allDictSize) = (0L, 0L, 0L)
-      // current database
+      // All tables of current database
       var (dbIndexSize, dbDatamapSize, dbDictSize) = (0L, 0L, 0L)
-      val tableMapIndexSize = mutable.HashMap[String, Long]()
-      val tableMapDatamapSize = mutable.HashMap[String, Long]()
-      val tableMapDictSize = mutable.HashMap[String, Long]()
-      val cacheIterator = cache.getCacheMap.entrySet().iterator()
-      while (cacheIterator.hasNext) {
-        val entry = cacheIterator.next()
-        val cache = entry.getValue
-        if (cache.isInstanceOf[BlockletDataMapIndexWrapper]) {
-          // index
-          allIndexSize = allIndexSize + cache.getMemorySize
-          val indexPath = entry.getKey.replace(
-            CarbonCommonConstants.WINDOWS_FILE_SEPARATOR, CarbonCommonConstants.FILE_SEPARATOR)
-          val tablePath = tablePaths.find(path => indexPath.startsWith(path._1))
-          if (tablePath.isDefined) {
-            dbIndexSize = dbIndexSize + cache.getMemorySize
-            val memorySize = tableMapIndexSize.get(tablePath.get._2)
-            if (memorySize.isEmpty) {
-              tableMapIndexSize.put(tablePath.get._2, cache.getMemorySize)
-            } else {
-              tableMapIndexSize.put(tablePath.get._2, memorySize.get + cache.getMemorySize)
-            }
-          }
-        } else if (cache.isInstanceOf[BloomCacheKeyValue.CacheValue]) {
-          // bloom datamap
-          allDatamapSize = allDatamapSize + cache.getMemorySize
-          val shardPath = entry.getKey.replace(CarbonCommonConstants.WINDOWS_FILE_SEPARATOR,
-            CarbonCommonConstants.FILE_SEPARATOR)
-          val tablePath = tablePaths.find(path => shardPath.contains(path._1))
-          if (tablePath.isDefined) {
-            dbDatamapSize = dbDatamapSize + cache.getMemorySize
-            val memorySize = tableMapDatamapSize.get(tablePath.get._2)
-            if (memorySize.isEmpty) {
-              tableMapDatamapSize.put(tablePath.get._2, cache.getMemorySize)
-            } else {
-              tableMapDatamapSize.put(tablePath.get._2, memorySize.get + cache.getMemorySize)
-            }
+      val tableList: Seq[Row] = carbonTables.map {
+        carbonTable =>
+          val tableResult = getTableCache(sparkSession, carbonTable)
+          var (indexSize, datamapSize) = (tableResult(0).getLong(1), 0L)
+          tableResult.drop(2).foreach {
+            row =>
+              indexSize += row.getLong(1)
+              datamapSize += row.getLong(2)
           }
-        } else if (cache.isInstanceOf[AbstractColumnDictionaryInfo]) {
-          // dictionary
-          allDictSize = allDictSize + cache.getMemorySize
-          val dictId = dictIds.find(id => entry.getKey.startsWith(id._1))
-          if (dictId.isDefined) {
-            dbDictSize = dbDictSize + cache.getMemorySize
-            val memorySize = tableMapDictSize.get(dictId.get._2)
-            if (memorySize.isEmpty) {
-              tableMapDictSize.put(dictId.get._2, cache.getMemorySize)
-            } else {
-              tableMapDictSize.put(dictId.get._2, memorySize.get + cache.getMemorySize)
-            }
-          }
-        }
+          val dictSize = tableResult(1).getLong(1)
+
+          dbIndexSize += indexSize
+          dbDictSize += dictSize
+          dbDatamapSize += datamapSize
+
+          (carbonTable.getTableName, currentDatabase, indexSize, datamapSize, dictSize)
+      }.filter {
+        case (_, _, indexSize, datamapSize, dictSize) =>
+          !((indexSize == 0) && (datamapSize == 0) && (dictSize == 0))
+      }.map {
+        case (db, table, indexSize, datamapSize, dictSize) =>
+          Row(db, table, indexSize, datamapSize, dictSize)
       }
-      if (tableMapIndexSize.isEmpty && tableMapDatamapSize.isEmpty && tableMapDictSize.isEmpty) {
-        Seq(
-          Row("ALL", "ALL", bytesToDisplaySize(allIndexSize),
-            bytesToDisplaySize(allDatamapSize), bytesToDisplaySize(allDictSize)),
-          Row(currentDatabase, "ALL", bytesToDisplaySize(0),
-            bytesToDisplaySize(0), bytesToDisplaySize(0)))
-      } else {
-        val tableList = tableMapIndexSize
-          .map(_._1)
-          .toSeq
-          .union(tableMapDictSize.map(_._1).toSeq)
-          .distinct
-          .sorted
-          .map { uniqueName =>
-            val values = uniqueName.split("\\.")
-            val indexSize = tableMapIndexSize.getOrElse(uniqueName, 0L)
-            val datamapSize = tableMapDatamapSize.getOrElse(uniqueName, 0L)
-            val dictSize = tableMapDictSize.getOrElse(uniqueName, 0L)
-            Row(values(0), values(1), bytesToDisplaySize(indexSize),
-              bytesToDisplaySize(datamapSize), bytesToDisplaySize(dictSize))
+
+      // All Databases
+      val tablePaths: Set[String] = carbonTables.map {
 
 Review comment:
   not used. Please remove

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
[hidden email]


With regards,
Apache Git Services