[jira] [Commented] (CARBONDATA-204) Query statistics issue

[Akash R Nilugal (Jira)]

    [ https://issues.apache.org/jira/browse/CARBONDATA-204?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15460922#comment-15460922 ]

ASF GitHub Bot commented on CARBONDATA-204:
-------------------------------------------

Github user Vimal-Das commented on a diff in the pull request:

    https://github.com/apache/incubator-carbondata/pull/123#discussion_r77435578
 
    --- Diff: core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java ---
    @@ -78,106 +82,142 @@ public synchronized void recordStatisticsForDriver(QueryStatistic statistic, Str
        */
       public void logStatisticsAsTableDriver() {
         synchronized (lock) {
    -      String tableInfo = collectDriverStatistics();
    -      if (null != tableInfo) {
    -        LOGGER.statistic(tableInfo);
    +      for (String key: queryStatisticsMap.keySet()) {
    +        // print sql_parse_t,load_meta_t,block_allocation_t,block_identification_t
    +        // or just print block_allocation_t,block_identification_t
    +        if (queryStatisticsMap.get(key).size() >= 2) {
    +          String tableInfo = collectDriverStatistics(key);
    +          if (null != tableInfo) {
    +            LOGGER.statistic(tableInfo);
    +          }
    +        }
    +        // clear timeout query statistics
    +        if(StringUtils.isEmpty(key)) {
    +          queryStatisticsMap.remove(key);
    +        } else {
    +          long interval = System.nanoTime() - Long.parseLong(key);
    +          if (interval > QueryStatisticsConstants.CLEAR_STATISTICS_TIMEOUT) {
    +            queryStatisticsMap.remove(key);
    +          }
    +        }
           }
         }
       }
     
       /**
        * Below method will parse queryStatisticsMap and put time into table
        */
    -  public String collectDriverStatistics() {
    -    for (String key: queryStatisticsMap.keySet()) {
    -      try {
    -        // TODO: get the finished query, and print Statistics
    -        if (queryStatisticsMap.get(key).size() > 3) {
    -          String sql_parse_time = "";
    -          String load_meta_time = "";
    -          String block_allocation_time = "";
    -          String block_identification_time = "";
    -          Double driver_part_time_tmp = 0.0;
    -          String splitChar = " ";
    -          // get statistic time from the QueryStatistic
    -          for (QueryStatistic statistic : queryStatisticsMap.get(key)) {
    -            switch (statistic.getMessage()) {
    -              case QueryStatisticsConstants.SQL_PARSE:
    -                sql_parse_time += statistic.getTimeTaken() + splitChar;
    -                driver_part_time_tmp += statistic.getTimeTaken();
    -                break;
    -              case QueryStatisticsConstants.LOAD_META:
    -                load_meta_time += statistic.getTimeTaken() + splitChar;
    -                driver_part_time_tmp += statistic.getTimeTaken();
    -                break;
    -              case QueryStatisticsConstants.BLOCK_ALLOCATION:
    -                block_allocation_time += statistic.getTimeTaken() + splitChar;
    -                driver_part_time_tmp += statistic.getTimeTaken();
    -                break;
    -              case QueryStatisticsConstants.BLOCK_IDENTIFICATION:
    -                block_identification_time += statistic.getTimeTaken() + splitChar;
    -                driver_part_time_tmp += statistic.getTimeTaken();
    -                break;
    -              default:
    -                break;
    -            }
    -          }
    -          String driver_part_time = driver_part_time_tmp + splitChar;
    -          // structure the query statistics info table
    -          StringBuilder tableInfo = new StringBuilder();
    -          int len1 = 8;
    -          int len2 = 20;
    -          int len3 = 21;
    -          int len4 = 22;
    -          String line = "+" + printLine("-", len1) + "+" + printLine("-", len2) + "+" +
    -              printLine("-", len3) + "+" + printLine("-", len4) + "+";
    -          String line2 = "|" + printLine(" ", len1) + "+" + printLine("-", len2) + "+" +
    -              printLine(" ", len3) + "+" + printLine("-", len4) + "+";
    -          // table header
    -          tableInfo.append(line).append("\n");
    -          tableInfo.append("|" + printLine(" ", (len1 - "Module".length())) + "Module" + "|" +
    -              printLine(" ", (len2 - "Operation Step".length())) + "Operation Step" + "|" +
    -              printLine(" ", (len3 + len4 + 1 - "Query Cost".length())) +
    -              "Query Cost" + "|" + "\n");
    -          // driver part
    -          tableInfo.append(line).append("\n");
    -          tableInfo.append("|" + printLine(" ", len1) + "|" +
    -              printLine(" ", (len2 - "SQL parse".length())) + "SQL parse" + "|" +
    -              printLine(" ", len3) + "|" +
    -              printLine(" ", (len4 - sql_parse_time.length())) + sql_parse_time + "|" + "\n");
    -          tableInfo.append(line2).append("\n");
    -          tableInfo.append("|" +printLine(" ", (len1 - "Driver".length())) + "Driver" + "|" +
    -              printLine(" ", (len2 - "Load meta data".length())) + "Load meta data" + "|" +
    -              printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" +
    -              printLine(" ", (len4 - load_meta_time.length())) +
    -              load_meta_time + "|" + "\n");
    -          tableInfo.append(line2).append("\n");
    -          tableInfo.append("|" +
    -              printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
    -              printLine(" ", (len2 - "Block allocation".length())) +
    -              "Block allocation" + "|" +
    -              printLine(" ", len3) + "|" +
    -              printLine(" ", (len4 - block_allocation_time.length())) +
    -              block_allocation_time + "|" + "\n");
    -          tableInfo.append(line2).append("\n");
    -          tableInfo.append("|" +
    -              printLine(" ", len1) + "|" +
    -              printLine(" ", (len2 - "Block identification".length())) +
    -              "Block identification" + "|" +
    -              printLine(" ", len3) + "|" +
    -              printLine(" ", (len4 - block_identification_time.length())) +
    -              block_identification_time + "|" + "\n");
    -          tableInfo.append(line).append("\n");
    -
    -          // once the statistics be printed, remove it from the map
    -          queryStatisticsMap.remove(key);
    -          // show query statistic as "query id" + "table"
    -          return "Print query statistic for query id: " + key + "\n" + tableInfo.toString();
    +  public String collectDriverStatistics(String key) {
    +    String sql_parse_time = "";
    +    String load_meta_time = "";
    +    String block_allocation_time = "";
    +    String block_identification_time = "";
    +    Double driver_part_time_tmp = 0.0;
    +    Double driver_part_time_tmp2 = 0.0;
    +    String splitChar = " ";
    +    try {
    +      // get statistic time from the QueryStatistic
    +      for (QueryStatistic statistic : queryStatisticsMap.get(key)) {
    --- End diff --
   
    null possibility here also, pass entry.getValue() instead, once you iterate over entrySet() in the calling method

> Query statistics issue
> -----------------------
>
>                 Key: CARBONDATA-204
>                 URL: https://issues.apache.org/jira/browse/CARBONDATA-204
>             Project: CarbonData
>          Issue Type: Bug
>            Reporter: Gin-zhj
>            Assignee: Gin-zhj
>
> Query statistics issue:
> 1. some query statistics that never be printed will be keeped into querystatisticsMap, so it will cause "out of memory" for long time running
> 2. in some sceniaro, the driver can't record "sql_parse_time" , the driver statistics logs will not be output

--
This message was sent by Atlassian JIRA
(v6.3.4#6332)