[GitHub] [carbondata] VenuReddy2103 commented on a change in pull request #3481: [CARBONDATA-3548]Geospatial Support： add hash id create，query condition analyze and generate hash id list

[GitBox]

VenuReddy2103 commented on a change in pull request #3481: [CARBONDATA-3548]Geospatial Support： add hash id create，query condition analyze and generate hash id list
URL: https://github.com/apache/carbondata/pull/3481#discussion_r361586667
 
 

 ##########
 File path: geo/src/main/java/org/apache/carbondata/geo/GeoHashImpl.java
 ##########
 @@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.geo;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CustomIndex;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.log4j.Logger;
+
+/**
+ * GeoHash custom implementation.
+ * This class extends {@link CustomIndex}. It provides methods to
+ * 1. Extracts the sub-properties of geohash type index handler such as type, source columns,
+ * grid size, origin, min and max longitude and latitude of data. Validates and stores them in
+ * instance.
+ * 2. Generates column value from the longitude and latitude column values.
+ * 3. Query processor to handle the custom UDF filter queries based on longitude and latitude
+ * columns.
+ */
+public class GeoHashImpl extends CustomIndex<List<Long[]>> {
+  private static final Logger LOGGER =
+      LogServiceFactory.getLogService(GeoHashImpl.class.getName());
+
+  // conversion factor of angle to radian
+  private static final double CONVERT_FACTOR = 180.0;
+  // Earth radius
+  private static final double EARTH_RADIUS = 6371004.0;
+  // Latitude of coordinate origin
+  private double oriLatitude;
+  // User defined maximum longitude of map
+  private double userDefineMaxLongitude;
+  // User defined maximum latitude of map
+  private double userDefineMaxLatitude;
+  // User defined map minimum longitude
+  private double userDefineMinLongitude;
+  // User defined map minimum latitude
+  private double userDefineMinLatitude;
+  // The maximum longitude of the completed map after calculation
+  private double CalculateMaxLongitude;
+  // The maximum latitude of the completed map after calculation
+  private double CalculateMaxLatitude;
+  // Grid length is in meters
+  private int gridSize;
+  // cos value of latitude of origin of coordinate
+  private double mCos;
+  // The degree of Y axis corresponding to each grid size length
+  private double deltaY;
+  // Each grid size length should be the degree of X axis
+  private double deltaX;
+  // Degree * coefficient of Y axis corresponding to each grid size length
+  private double deltaYByRatio;
+  // Each grid size length should be X-axis Degree * coefficient
+  private double deltaXByRatio;
+  // The number of knives cut for the whole area (one horizontally and one vertically)
+  // is the depth of quad tree
+  private int cutLevel;
+  // used to convert the latitude and longitude of double type to int type for calculation
+  private int conversionRatio;
+  // * Constant of coefficient
+  private double lon0ByRation;
+  // * Constant of coefficient
+  private double lat0ByRation;
+
+
+  /**
+   * Initialize the geohash index handler instance.
+   * the properties is like that:
+   * TBLPROPERTIES ('INDEX_HANDLER'='mygeohash',
+   * 'INDEX_HANDLER.mygeohash.type'='geohash',
+   * 'INDEX_HANDLER.mygeohash.sourcecolumns'='longitude, latitude',
+   * 'INDEX_HANDLER.mygeohash.gridSize'=''
+   * 'INDEX_HANDLER.mygeohash.minLongitude'=''
+   * 'INDEX_HANDLER.mygeohash.maxLongitude'=''
+   * 'INDEX_HANDLER.mygeohash.minLatitude'=''
+   * 'INDEX_HANDLER.mygeohash.maxLatitude'=''
+   * 'INDEX_HANDLER.mygeohash.orilatitude''')
+   * @param handlerName the class name of generating algorithm
+   * @param properties input properties,please check the describe
+   * @throws Exception
+   */
+  @Override
+  public void init(String handlerName, Map<String, String> properties) throws Exception {
+    String options = properties.get(CarbonCommonConstants.INDEX_HANDLER);
+    if (StringUtils.isEmpty(options)) {
+      throw new MalformedCarbonCommandException(
+              String.format("%s property is invalid.", CarbonCommonConstants.INDEX_HANDLER));
+    }
+    options = options.toLowerCase();
+    if (!options.contains(handlerName.toLowerCase())) {
+      throw new MalformedCarbonCommandException(
+              String.format("%s property is invalid. %s is not present.",
+                      CarbonCommonConstants.INDEX_HANDLER, handlerName));
+    }
+    String commonKey = CarbonCommonConstants.INDEX_HANDLER + CarbonCommonConstants.POINT +
+            handlerName + CarbonCommonConstants.POINT;
+    String TYPE = commonKey + "type";
+    String type = properties.get(TYPE);
+    if (!CarbonCommonConstants.GEOHASH.equalsIgnoreCase(type)) {
+      throw new MalformedCarbonCommandException(
+              String.format("%s property is invalid. %s property must be %s for this class.",
+                      CarbonCommonConstants.INDEX_HANDLER, TYPE, CarbonCommonConstants.GEOHASH));
+    }
+    String SOURCE_COLUMNS = commonKey + "sourcecolumns";
+    String sourceColumnsOption = properties.get(SOURCE_COLUMNS);
+    if (StringUtils.isEmpty(sourceColumnsOption)) {
+      throw new MalformedCarbonCommandException(
+              String.format("%s property is invalid. Must specify %s property.",
+                      CarbonCommonConstants.INDEX_HANDLER, SOURCE_COLUMNS));
+    }
+    if (sourceColumnsOption.split(",").length != 2) {
+      throw new MalformedCarbonCommandException(
+              String.format("%s property is invalid. %s property must have 2 columns.",
+                      CarbonCommonConstants.INDEX_HANDLER, SOURCE_COLUMNS));
+    }
+    String SOURCE_COLUMN_TYPES = commonKey + "sourcecolumntypes";
+    String sourceDataTypes = properties.get(SOURCE_COLUMN_TYPES);
+    String[] srcTypes = sourceDataTypes.split(",");
+    for (String srcdataType : srcTypes) {
+      if (!"bigint".equalsIgnoreCase(srcdataType)) {
+        throw new MalformedCarbonCommandException(
+                String.format("%s property is invalid. %s datatypes must be long.",
+                        CarbonCommonConstants.INDEX_HANDLER, SOURCE_COLUMNS));
+      }
+    }
+    String TARGET_DATA_TYPE = commonKey + "datatype";
+    String dataType = properties.get(TARGET_DATA_TYPE);
+    if (!StringUtils.isEmpty(dataType) && !"long".equalsIgnoreCase(dataType)) {
+      throw new MalformedCarbonCommandException(
+              String.format("%s property is invalid. %s property must be long for this class.",
+                      CarbonCommonConstants.INDEX_HANDLER, TARGET_DATA_TYPE));
+    }
+    // Set the generated column data type as long
+    properties.put(TARGET_DATA_TYPE, "long");
+    String ORIGIN_LATITUDE = commonKey + "originlatitude";
+    String originLatitude = properties.get(ORIGIN_LATITUDE);
+    if (StringUtils.isEmpty(originLatitude)) {
+      throw new MalformedCarbonCommandException(
+              String.format("%s property is invalid. Must specify %s property.",
+                      CarbonCommonConstants.INDEX_HANDLER, ORIGIN_LATITUDE));
+    }
+    String MIN_LONGITUDE = commonKey + "minlongitude";
+    String MAX_LONGITUDE = commonKey + "maxlongitude";
+    String MIN_LATITUDE = commonKey + "minlatitude";
+    String MAX_LATITUDE = commonKey + "maxlatitude";
+    String minLongitude = properties.get(MIN_LONGITUDE);
+    String maxLongitude = properties.get(MAX_LONGITUDE);
+    String minLatitude = properties.get(MIN_LATITUDE);
+    String maxLatitude = properties.get(MAX_LATITUDE);
+    if (StringUtils.isEmpty(minLongitude)) {
+      throw new MalformedCarbonCommandException(
+          String.format("%s property is invalid. Must specify %s property.",
+              CarbonCommonConstants.INDEX_HANDLER, MIN_LONGITUDE));
+    }
+    if (StringUtils.isEmpty(minLatitude)) {
+      throw new MalformedCarbonCommandException(
+          String.format("%s property is invalid. Must specify %s property.",
+              CarbonCommonConstants.INDEX_HANDLER, MIN_LATITUDE));
+    }
+    if (StringUtils.isEmpty(maxLongitude)) {
+      throw new MalformedCarbonCommandException(
+          String.format("%s property is invalid. Must specify %s property.",
+              CarbonCommonConstants.INDEX_HANDLER, MAX_LONGITUDE));
+    }
+    if (StringUtils.isEmpty(maxLatitude)) {
+      throw new MalformedCarbonCommandException(
+          String.format("%s property is invalid. Must specify %s property.",
+              CarbonCommonConstants.INDEX_HANDLER, MAX_LATITUDE));
+    }
+    String GRID_SIZE = commonKey + "gridsize";
+    String gridSize = properties.get(GRID_SIZE);
+    if (StringUtils.isEmpty(gridSize)) {
+      throw new MalformedCarbonCommandException(
+              String.format("%s property is invalid. %s property must be specified.",
+                      CarbonCommonConstants.INDEX_HANDLER, GRID_SIZE));
+    }
+    String CONVERSION_RATIO = commonKey + "conversionratio";
+    String conversionRatio = properties.get(CONVERSION_RATIO);
+    if (StringUtils.isEmpty(conversionRatio)) {
+      throw new MalformedCarbonCommandException(
+              String.format("%s property is invalid. %s property must be specified.",
+                      CarbonCommonConstants.INDEX_HANDLER, CONVERSION_RATIO));
+    }
+
+    // Fill the values to the instance fields
+    this.oriLatitude = Double.valueOf(originLatitude);
+    this.userDefineMaxLongitude = Double.valueOf(maxLongitude);
+    this.userDefineMaxLatitude = Double.valueOf(maxLatitude);
+    this.userDefineMinLongitude = Double.valueOf(minLongitude);
+    this.userDefineMinLatitude = Double.valueOf(minLatitude);
+    this.gridSize = Integer.parseInt(gridSize);
+    this.conversionRatio = Integer.parseInt(conversionRatio);
+    this.lon0ByRation = userDefineMinLongitude * this.conversionRatio;
+    this.lat0ByRation = userDefineMinLatitude * this.conversionRatio;
+    calculateData();
+  }
+
+  /**
+   * Generates the GeoHash ID column value from the given source columns.
+   * @param sources Longitude and Latitude
+   * @return Returns the generated hash id
+   * @throws Exception
+   */
+  @Override
+  public String generate(List<?> sources) throws Exception {
+    if (sources.size() != 2) {
+      throw new RuntimeException("Source columns list must be of size 2.");
+    }
+    if (sources.get(0) == null || sources.get(1) == null) {
+      // Bad record. Just return null
+      return null;
+    }
+    if (!(sources.get(0) instanceof Long) || !(sources.get(1) instanceof Long)) {
+      throw new RuntimeException("Source columns must be of Long type.");
+    }
+    Long longitude = (Long) sources.get(0);
+    Long latitude  = (Long) sources.get(1);
+    // generate the hash code
+    int[] gridPoint = calculateID(longitude, latitude);
+    Long hashId = createHashID(gridPoint[0], gridPoint[1]);
+    return String.valueOf(hashId);
+  }
+
+  /**
+   * Query processor for GeoHash.
+   * example: POLYGON (35 10, 45 45, 15 40, 10 20, 35 10)
+   * so there will be a sample check
+   * @param polygon a group of pints, close out to form an area
+   * @return Returns list of ranges of GeoHash IDs
+   * @throws Exception
+   */
+  @Override
+  public List<Long[]> query(String polygon) throws Exception {
+    if (!validate(polygon)) {
+      return null;
+    } else {
+      String[] pointList = polygon.trim().split(",");
+      List<double[]> queryList = new ArrayList<>();
+      for (String str: pointList) {
+        String[] points = str.split(" ");
+        if (2 != points.length) {
+          throw new RuntimeException("longitude and latitude is a pair need 2 data");
+        } else {
+          try {
+            queryList.add(new double[] {Double.valueOf(points[0]), Double.valueOf(points[1])});
+          } catch (NumberFormatException e) {
+            throw new RuntimeException("can not covert the string data to double", e);
+          }
+        }
+      }
+      List<Long[]> rangeList = getPolygonRangeList(queryList);
+      return rangeList;
+    }
+  }
+
+  private boolean validate(String polygon) throws Exception{
+    String[] pointList = polygon.trim().split(",");
+    if (4 > pointList.length) {
+      throw new RuntimeException("polygon need at least 3 points, really has " + pointList.length);
+    } else {
+      String firstPoint = pointList[0].trim();
+      String lastPoint = pointList[pointList.length - 1].trim();
+      if (!firstPoint.equals(lastPoint)) {
 
 Review comment:
   Please split and check the long and lat values seperately. `validate()` can fail if the number of spaces between long and lat do not match between first and last points.
   If you do this change, you would want to do this first point and last point match check in   `query(String polygon)` itself as you parse the long and lat there.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
[hidden email]


With regards,
Apache Git Services