Posted by
Akash R Nilugal (Jira) on
URL: http://apache-carbondata-dev-mailing-list-archive.168.s1.nabble.com/jira-Commented-CARBONDATA-200-Add-performance-statistics-logs-to-record-the-query-time-taken-by-carbn-tp678.html
[
https://issues.apache.org/jira/browse/CARBONDATA-200?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15454976#comment-15454976 ]
ASF GitHub Bot commented on CARBONDATA-200:
-------------------------------------------
Github user foryou2030 commented on a diff in the pull request:
https://github.com/apache/incubator-carbondata/pull/91#discussion_r77147108
--- Diff: core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/SingleQueryStatisticsRecorder.java ---
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *
http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.querystatistics;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+/**
+ * Class will be used to record and log the query statistics
+ */
+public class SingleQueryStatisticsRecorder implements Serializable {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(SingleQueryStatisticsRecorder.class.getName());
+ /**
+ * serialization version
+ */
+ private static final long serialVersionUID = -1L;
+
+ /**
+ * singleton QueryStatisticsRecorder for driver
+ */
+ private HashMap<String, List<QueryStatistic>> queryStatisticsMap;
+
+ private SingleQueryStatisticsRecorder() {
+ queryStatisticsMap = new HashMap<String, List<QueryStatistic>>();
+ }
+
+ private static SingleQueryStatisticsRecorder carbonLoadStatisticsImplInstance =
+ new SingleQueryStatisticsRecorder();
+
+ public static SingleQueryStatisticsRecorder getInstance() {
+ return carbonLoadStatisticsImplInstance;
+ }
+
+ /**
+ * Below method will be used to add the statistics
+ *
+ * @param statistic
+ */
+ public synchronized void recordStatisticsForDriver(QueryStatistic statistic, String queryId) {
+ // refresh query Statistics Map
+ if (queryStatisticsMap.get(queryId) != null) {
+ queryStatisticsMap.get(queryId).add(statistic);
+ } else {
+ List<QueryStatistic> newQueryStatistics = new ArrayList<QueryStatistic>();
+ newQueryStatistics.add(statistic);
+ queryStatisticsMap.put(queryId, newQueryStatistics);
+ }
+ }
+
+ /**
+ * Below method will be used to show statistic log as table
+ */
+ public void logStatisticsAsTableDriver() {
+ String tableInfo = collectDriverStatistics();
+ if (null != tableInfo) {
+ LOGGER.statistic(tableInfo);
+ }
+ }
+
+ /**
+ * Below method will parse queryStatisticsMap and put time into table
+ */
+ public String collectDriverStatistics() {
+ for (String key: queryStatisticsMap.keySet()) {
+ try {
+ // TODO: get the finished query, and print Statistics
+ if (queryStatisticsMap.get(key).size() > 2) {
+ String sql_parse_time = "";
+ String load_meta_time = "";
+ String block_allocation_time = "";
+ String block_identification_time = "";
+ Double driver_part_time_tmp = 0.0;
+ String splitChar = " ";
+ // get statistic time from the QueryStatistic
+ for (QueryStatistic statistic : queryStatisticsMap.get(key)) {
+ switch (statistic.getMessage()) {
+ case QueryStatisticsConstants.SQL_PARSE:
+ sql_parse_time += statistic.getTimeTaken() + splitChar;
+ driver_part_time_tmp += statistic.getTimeTaken();
+ break;
+ case QueryStatisticsConstants.LOAD_META:
+ load_meta_time += statistic.getTimeTaken() + splitChar;
+ driver_part_time_tmp += statistic.getTimeTaken();
+ break;
+ case QueryStatisticsConstants.BLOCK_ALLOCATION:
+ block_allocation_time += statistic.getTimeTaken() + splitChar;
+ driver_part_time_tmp += statistic.getTimeTaken();
+ break;
+ case QueryStatisticsConstants.BLOCK_IDENTIFICATION:
+ block_identification_time += statistic.getTimeTaken() + splitChar;
+ driver_part_time_tmp += statistic.getTimeTaken();
+ break;
+ default:
+ break;
+ }
+ }
+ String driver_part_time = driver_part_time_tmp + splitChar;
+ // structure the query statistics info table
+ StringBuilder tableInfo = new StringBuilder();
+ int len1 = 8;
+ int len2 = 20;
+ int len3 = 21;
+ int len4 = 22;
+ String line = "+" + printLine("-", len1) + "+" + printLine("-", len2) + "+" +
+ printLine("-", len3) + "+" + printLine("-", len4) + "+";
+ String line2 = "|" + printLine(" ", len1) + "+" + printLine("-", len2) + "+" +
+ printLine(" ", len3) + "+" + printLine("-", len4) + "+";
+ // table header
+ tableInfo.append(line).append("\n");
+ tableInfo.append("|" + printLine(" ", (len1 - "Module".length())) + "Module" + "|" +
+ printLine(" ", (len2 - "Operation Step".length())) + "Operation Step" + "|" +
+ printLine(" ", (len3 + len4 + 1 - "Query Cost".length())) +
+ "Query Cost" + "|" + "\n");
+ // driver part
+ tableInfo.append(line).append("\n");
+ tableInfo.append("|" + printLine(" ", len1) + "|" +
+ printLine(" ", (len2 - "SQL parse".length())) + "SQL parse" + "|" +
+ printLine(" ", len3) + "|" +
+ printLine(" ", (len4 - sql_parse_time.length())) + sql_parse_time + "|" + "\n");
+ tableInfo.append(line2).append("\n");
+ tableInfo.append("|" +printLine(" ", (len1 - "Driver".length())) + "Driver" + "|" +
+ printLine(" ", (len2 - "Load meta data".length())) + "Load meta data" + "|" +
+ printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" +
+ printLine(" ", (len4 - load_meta_time.length())) +
+ load_meta_time + "|" + "\n");
+ tableInfo.append(line2).append("\n");
+ tableInfo.append("|" +
+ printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
+ printLine(" ", (len2 - "Block allocation".length())) +
+ "Block allocation" + "|" +
+ printLine(" ", len3) + "|" +
+ printLine(" ", (len4 - block_allocation_time.length())) +
+ block_allocation_time + "|" + "\n");
+ tableInfo.append(line2).append("\n");
+ tableInfo.append("|" +
+ printLine(" ", len1) + "|" +
+ printLine(" ", (len2 - "Block identification".length())) +
+ "Block identification" + "|" +
+ printLine(" ", len3) + "|" +
+ printLine(" ", (len4 - block_identification_time.length())) +
+ block_identification_time + "|" + "\n");
+ tableInfo.append(line).append("\n");
+
+ // once the statistics be printed, remove it from the map
+ queryStatisticsMap.remove(key);
+ // show query statistic as "query id" + "table"
+ return "Print query statistic for query id: " + key + "\n" + tableInfo.toString();
+ }
+ } catch (Exception ex) {
+ return "Put statistics into table failed, catch exception: " + ex.getMessage();
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Below method will create string like "***********"
+ *
+ * @param a
+ * @param num
+ */
+ public static String printLine(String a, int num)
--- End diff --
handled
> Add performance statistics logs to record the query time taken by carbon
> ------------------------------------------------------------------------
>
> Key: CARBONDATA-200
> URL:
https://issues.apache.org/jira/browse/CARBONDATA-200> Project: CarbonData
> Issue Type: New Feature
> Reporter: Gin-zhj
> Assignee: Gin-zhj
> Priority: Minor
>
> Add performance statistics logs to record the query time taken by carbon,
> show the query statistics like a table, so users can easily analyzing the query performance.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)