Apache CarbonData Dev Mailing List archive › Apache CarbonData JIRA issues

[GitHub] carbondata pull request #2914: [WIP][CARBONDATA-3093] Provide property build...

Classic

List

Threaded

94 messages Options

12345

qiuchenjian-2

[GitHub] carbondata pull request #2914: [CARBONDATA-3093] Provide property builder fo...

Github user xubo245 commented on a diff in the pull request:

https://github.com/apache/carbondata/pull/2914#discussion_r241631889

--- Diff: core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---
@@ -40,1372 +41,1024 @@ private CarbonCommonConstants() {
@CarbonProperty
public static final String STORE_LOCATION = "carbon.storelocation";

- /**
- * blocklet size in carbon file
- */
- @CarbonProperty
- public static final String BLOCKLET_SIZE = "carbon.blocklet.size";
-
- /**
- * default blocklet size
- */
- public static final String BLOCKLET_SIZE_DEFAULT_VAL = "120000";
-
- /**
- * min blocklet size
- */
- public static final int BLOCKLET_SIZE_MIN_VAL = 2000;
-
- /**
- * max blocklet size
- */
- public static final int BLOCKLET_SIZE_MAX_VAL = 12000000;
-
- /**
- * min block size in MB
- */
- public static final int BLOCK_SIZE_MIN_VAL = 1;
-
- /**
- * max block size in MB
- */
- public static final int BLOCK_SIZE_MAX_VAL = 2048;
-
- /**
- * carbon properties file path
- */
- @CarbonProperty
- public static final String CARBON_PROPERTIES_FILE_PATH = "carbon.properties.filepath";
-
- /**
- * default carbon properties file path
- */
- public static final String CARBON_PROPERTIES_FILE_PATH_DEFAULT =
- "../../../conf/carbon.properties";
-
- /**
- * CARBON_DDL_BASE_HDFS_URL
- */
- @CarbonProperty
- public static final String CARBON_DDL_BASE_HDFS_URL = "carbon.ddl.base.hdfs.url";
-
- /**
- * CARBON_BADRECORDS_LOCATION
- */
- @CarbonProperty
- public static final String CARBON_BADRECORDS_LOC = "carbon.badRecords.location";
-
- /**
- * CARBON_BADRECORDS_LOCATION_DEFAULT
- */
- public static final String CARBON_BADRECORDS_LOC_DEFAULT_VAL = "";
-
- /**
- * Property for specifying the format of TIMESTAMP data type column.
- * e.g. yyyy/MM/dd HH:mm:ss, or using default value
- */
- @CarbonProperty
- public static final String CARBON_TIMESTAMP_FORMAT = "carbon.timestamp.format";
-
- /**
- * default value
- */
- public static final String CARBON_TIMESTAMP_DEFAULT_FORMAT = "yyyy-MM-dd HH:mm:ss";
-
- /**
- * CARBON_TIMESTAMP
- */
- public static final String CARBON_TIMESTAMP = "dd-MM-yyyy HH:mm:ss";
-
- /**
- * CARBON_TIMESTAMP
- */
- public static final String CARBON_TIMESTAMP_MILLIS = "dd-MM-yyyy HH:mm:ss:SSS";
-
- /**
- * Property for specifying the format of DATE data type column.
- * e.g. yyyy/MM/dd , or using default value
- */
- @CarbonProperty
- public static final String CARBON_DATE_FORMAT = "carbon.date.format";
-
- /**
- * default value
- */
- public static final String CARBON_DATE_DEFAULT_FORMAT = "yyyy-MM-dd";
-
- /**
- * compressor for writing/reading CarbonData file
- */
- @CarbonProperty
- public static final String COMPRESSOR = "carbon.column.compressor";
-
- /**
- * default compressor is snappy
- */
- public static final String DEFAULT_COMPRESSOR = "snappy";
-
- /**
- * ZOOKEEPER_ENABLE_LOCK if this is set to true then zookeeper
- * will be used to handle locking
- * mechanism of carbon
- */
- @CarbonProperty
- public static final String LOCK_TYPE = "carbon.lock.type";
-
- /**
- * ZOOKEEPER_ENABLE_DEFAULT the default value for zookeeper will be true for carbon
- */
- public static final String LOCK_TYPE_DEFAULT = "LOCALLOCK";
-
- /**
- * Specifies the path where the lock files have to be created.
- * By default, lock files are created in table path.
- */
- @CarbonProperty
- public static final String LOCK_PATH = "carbon.lock.path";
-
- public static final String LOCK_PATH_DEFAULT = "";
-
- /**
- * ZOOKEEPER_LOCATION this is the location in zookeeper file system where locks are created.
- * mechanism of carbon
- */
- public static final String ZOOKEEPER_LOCATION = "/CarbonLocks";
-
- /**
- * xxhash algorithm property for hashmap
- */
- @CarbonProperty
- public static final String ENABLE_XXHASH = "carbon.enableXXHash";
-
- /**
- * xxhash algorithm property for hashmap Default value false
- */
- public static final String ENABLE_XXHASH_DEFAULT = "true";
-
- /**
- * System property to enable or disable local dictionary generation
- */
- @CarbonProperty
- public static final String LOCAL_DICTIONARY_SYSTEM_ENABLE = "carbon.local.dictionary.enable";
-
- /**
- * System property to enable or disable decoder based local dictionary fallback
- */
- @CarbonProperty
- public static final String LOCAL_DICTIONARY_DECODER_BASED_FALLBACK =
- "carbon.local.dictionary.decoder.fallback";
-
- /**
- * System property to enable or disable decoder based local dictionary fallback default value
- */
- public static final String LOCAL_DICTIONARY_DECODER_BASED_FALLBACK_DEFAULT = "true";
-
- /**
- * zookeeper url key
- */
- @CarbonProperty
- public static final String ZOOKEEPER_URL = "spark.deploy.zookeeper.url";
-
- /**
- * carbon data file version property
- */
- @CarbonProperty
- public static final String CARBON_DATA_FILE_VERSION = "carbon.data.file.version";
-
- @CarbonProperty
- public static final String SPARK_SCHEMA_STRING_LENGTH_THRESHOLD =
- "spark.sql.sources.schemaStringLengthThreshold";
-
- public static final int SPARK_SCHEMA_STRING_LENGTH_THRESHOLD_DEFAULT = 4000;
-
- @CarbonProperty
- public static final String CARBON_BAD_RECORDS_ACTION = "carbon.bad.records.action";
-
- /**
- * FAIL action will fail the load in case of bad records in loading data
- */
- public static final String CARBON_BAD_RECORDS_ACTION_DEFAULT = "FAIL";
-
- @CarbonProperty
- public static final String ENABLE_HIVE_SCHEMA_META_STORE = "spark.carbon.hive.schema.store";
-
- public static final String ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT = "false";
-
- /**
- * There is more often that in production uses different drivers for load and queries. So in case
- * of load driver user should set this property to enable loader specific clean up.
- */
- @CarbonProperty
- public static final String DATA_MANAGEMENT_DRIVER = "spark.carbon.datamanagement.driver";
-
- public static final String DATA_MANAGEMENT_DRIVER_DEFAULT = "true";
-
- @CarbonProperty
- public static final String CARBON_SESSIONSTATE_CLASSNAME = "spark.carbon.sessionstate.classname";
-
- /**
- * This property will be used to configure the sqlastbuilder class.
- */
- @CarbonProperty
- public static final String CARBON_SQLASTBUILDER_CLASSNAME =
- "spark.carbon.sqlastbuilder.classname";
-
- public static final String CARBON_SQLASTBUILDER_CLASSNAME_DEFAULT =
- "org.apache.spark.sql.hive.CarbonSqlAstBuilder";
-
- @CarbonProperty
- public static final String CARBON_LEASE_RECOVERY_RETRY_COUNT =
- "carbon.lease.recovery.retry.count";
-
- public static final String CARBON_LEASE_RECOVERY_RETRY_COUNT_DEFAULT = "5";
-
- public static final int CARBON_LEASE_RECOVERY_RETRY_COUNT_MIN = 1;
-
- public static final int CARBON_LEASE_RECOVERY_RETRY_COUNT_MAX = 50;
-
- @CarbonProperty
- public static final String CARBON_LEASE_RECOVERY_RETRY_INTERVAL =
- "carbon.lease.recovery.retry.interval";
-
- public static final String CARBON_LEASE_RECOVERY_RETRY_INTERVAL_DEFAULT = "1000";
-
- public static final int CARBON_LEASE_RECOVERY_RETRY_INTERVAL_MIN = 1000;
-
- public static final int CARBON_LEASE_RECOVERY_RETRY_INTERVAL_MAX = 10000;
-
- @CarbonProperty
- public static final String CARBON_SECURE_DICTIONARY_SERVER =
- "carbon.secure.dictionary.server";
-
- public static final String CARBON_SECURE_DICTIONARY_SERVER_DEFAULT = "true";
-
- /**
- * ENABLE_CALCULATE_DATA_INDEX_SIZE
- */
- @CarbonProperty
- public static final String ENABLE_CALCULATE_SIZE = "carbon.enable.calculate.size";
-
- /**
- * default value of ENABLE_CALCULATE_DATA_INDEX_SIZE
- */
- public static final String DEFAULT_ENABLE_CALCULATE_SIZE = "true";
-
- /**
- * this will be used to skip / ignore empty lines while loading
- */
- @CarbonProperty
- public static final String CARBON_SKIP_EMPTY_LINE = "carbon.skip.empty.line";
-
- public static final String CARBON_SKIP_EMPTY_LINE_DEFAULT = "false";
-
- /**
- * Currently the segment lock files are not deleted immediately when unlock,
- * this value indicates the number of hours the segment lock files will be preserved.
- */
- @CarbonProperty
- public static final String CARBON_SEGMENT_LOCK_FILES_PRESERVE_HOURS =
- "carbon.segment.lock.files.preserve.hours";
-
- /**
- * default value is 2 days
- */
- public static final String CARBON_SEGMENT_LOCK_FILES_PRESERVE_HOURS_DEFAULT = "48";
-
- /**
- * The number of invisible segment info which will be preserved in tablestatus file,
- * if it exceeds this value, they will be removed and write to tablestatus.history file.
- */
- @CarbonProperty
- public static final String CARBON_INVISIBLE_SEGMENTS_PRESERVE_COUNT =
- "carbon.invisible.segments.preserve.count";
-
- /**
- * default value is 200, it means that it will preserve 200 invisible segment info
- * in tablestatus file.
- * The size of one segment info is about 500 bytes, so the size of tablestatus file
- * will remain at 100KB.
- */
- public static final String CARBON_INVISIBLE_SEGMENTS_PRESERVE_COUNT_DEFAULT = "200";
-
- /**
- * System older location to store system level data like datamap schema and status files.
- */
- @CarbonProperty
- public static final String CARBON_SYSTEM_FOLDER_LOCATION = "carbon.system.folder.location";
-
- /**
- * It is internal configuration and used only for test purpose.
- * It will merge the carbon index files with in the segment to single segment.
- */
- @CarbonProperty
- public static final String CARBON_MERGE_INDEX_IN_SEGMENT =
- "carbon.merge.index.in.segment";
-
- public static final String CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT = "true";
-
- /**
- * property to be used for specifying the max byte limit for string/varchar data type till
- * where storing min/max in data file will be considered
- */
- @CarbonProperty
- public static final String CARBON_MINMAX_ALLOWED_BYTE_COUNT =
- "carbon.minmax.allowed.byte.count";
-
- public static final String CARBON_MINMAX_ALLOWED_BYTE_COUNT_DEFAULT = "200";
-
- //////////////////////////////////////////////////////////////////////////////////////////
- // Table level property start here
- //////////////////////////////////////////////////////////////////////////////////////////
- // Table level property is the table property for Carbon table
-
- // Flat folder support on table. when it is true all CarbonData files store directly under table
- // path instead of sub folders.
- public static final String FLAT_FOLDER = "flat_folder";
-
- /**
- * FLAT_FOLDER default value
- */
- public static final String DEFAULT_FLAT_FOLDER = "false";
-
- public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
-
- public static final String DICTIONARY_INCLUDE = "dictionary_include";
-
- public static final String LONG_STRING_COLUMNS = "long_string_columns";
-
- /**
- * Table property to enable or disable local dictionary generation
- */
- public static final String LOCAL_DICTIONARY_ENABLE = "local_dictionary_enable";
-
- /**
- * default value for local dictionary generation
- */
- public static final String LOCAL_DICTIONARY_ENABLE_DEFAULT = "true";
-
- /**
- * Threshold value for local dictionary
- */
- public static final String LOCAL_DICTIONARY_THRESHOLD = "local_dictionary_threshold";
-
- /**
- * default value for local dictionary
- */
- public static final String LOCAL_DICTIONARY_THRESHOLD_DEFAULT = "10000";
-
- /**
- * max dictionary threshold
- */
- public static final int LOCAL_DICTIONARY_MAX = 100000;
-
- /**
- * min dictionary threshold
- */
- public static final int LOCAL_DICTIONARY_MIN = 1000;
-
- /**
- * Table property to specify the columns for which local dictionary needs to be generated.
- */
- public static final String LOCAL_DICTIONARY_INCLUDE = "local_dictionary_include";
-
- /**
- * Table property to specify the columns for which local dictionary should not be to be generated.
- */
- public static final String LOCAL_DICTIONARY_EXCLUDE = "local_dictionary_exclude";
-
- /**
- * DMPROPERTY for Index DataMap, like lucene, bloomfilter DataMap,
- * to indicate a list of column name to be indexed
- */
- public static final String INDEX_COLUMNS = "INDEX_COLUMNS";
-
- /**
- * key for dictionary path
- */
- public static final String DICTIONARY_PATH = "dictionary_path";
- public static final String SORT_COLUMNS = "sort_columns";
- public static final String PARTITION_TYPE = "partition_type";
- public static final String NUM_PARTITIONS = "num_partitions";
- public static final String RANGE_INFO = "range_info";
- public static final String LIST_INFO = "list_info";
- public static final String COLUMN_PROPERTIES = "columnproperties";
- // table block size in MB
- public static final String TABLE_BLOCKSIZE = "table_blocksize";
-
- // default block size in MB
- public static final String TABLE_BLOCK_SIZE_DEFAULT = "1024";
-
- // table blocklet size in MB
- public static final String TABLE_BLOCKLET_SIZE = "table_blocklet_size";
-
- // default blocklet size value in MB
- public static final String TABLE_BLOCKLET_SIZE_DEFAULT = "64";
-
- /**
- * set in column level to disable inverted index
- * @Deprecated :This property is deprecated, it is kept just for compatibility
- */
- public static final String NO_INVERTED_INDEX = "no_inverted_index";
- // set in column level to enable inverted index
- public static final String INVERTED_INDEX = "inverted_index";
- // table property name of major compaction size
- public static final String TABLE_MAJOR_COMPACTION_SIZE = "major_compaction_size";
- // table property name of auto load merge
- public static final String TABLE_AUTO_LOAD_MERGE = "auto_load_merge";
- // table property name of compaction level threshold
- public static final String TABLE_COMPACTION_LEVEL_THRESHOLD = "compaction_level_threshold";
- // table property name of preserve segments numbers while compaction
- public static final String TABLE_COMPACTION_PRESERVE_SEGMENTS = "compaction_preserve_segments";
- // table property name of allowed compaction days while compaction
- public static final String TABLE_ALLOWED_COMPACTION_DAYS = "allowed_compaction_days";
-
- /**
- * property to be specified for caching min/max of required columns
- */
- public static final String COLUMN_META_CACHE = "column_meta_cache";
-
- /**
- * property to be specified for caching level (Block/Blocket)
- */
- public static final String CACHE_LEVEL = "cache_level";
-
- /**
- * default value for cache level
- */
- public static final String CACHE_LEVEL_DEFAULT_VALUE = "BLOCK";
-
- //////////////////////////////////////////////////////////////////////////////////////////
- // Data loading parameter start here
- //////////////////////////////////////////////////////////////////////////////////////////
-
- /**
- * Number of cores to be used while loading
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String NUM_CORES_LOADING = "carbon.number.of.cores.while.loading";
-
- /**
- * default value of number of cores to be used
- */
- public static final String NUM_CORES_DEFAULT_VAL = "2";
-
- /**
- * Number of cores to be used while compacting
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String NUM_CORES_COMPACTING = "carbon.number.of.cores.while.compacting";
-
- /**
- * Number of cores to be used while alter partition
- */
- @CarbonProperty
- public static final String NUM_CORES_ALT_PARTITION = "carbon.number.of.cores.while.altPartition";
-
- /**
- * BYTEBUFFER_SIZE
- */
- public static final int BYTEBUFFER_SIZE = 24 * 1024;
-
- /**
- * SORT_TEMP_FILE_LOCATION
- */
- public static final String SORT_TEMP_FILE_LOCATION = "sortrowtmp";
-
- /**
- * SORT_INTERMEDIATE_FILES_LIMIT
- */
- @CarbonProperty
- public static final String SORT_INTERMEDIATE_FILES_LIMIT = "carbon.sort.intermediate.files.limit";
-
- /**
- * SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE
- */
- public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "20";
-
- public static final int SORT_INTERMEDIATE_FILES_LIMIT_MIN = 2;
-
- public static final int SORT_INTERMEDIATE_FILES_LIMIT_MAX = 50;
-
- /**
- * SORT_FILE_WRITE_BUFFER_SIZE
- */
- @CarbonProperty
- public static final String CARBON_SORT_FILE_WRITE_BUFFER_SIZE =
- "carbon.sort.file.write.buffer.size";
-
- /**
- * SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE
- */
- public static final String CARBON_SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE = "16384";
-
- public static final int CARBON_SORT_FILE_WRITE_BUFFER_SIZE_MIN = 10240;
-
- public static final int CARBON_SORT_FILE_WRITE_BUFFER_SIZE_MAX = 10485760;
-
- /**
- * CSV_READ_BUFFER_SIZE
- */
- @CarbonProperty
- public static final String CSV_READ_BUFFER_SIZE = "carbon.csv.read.buffersize.byte";
-
- /**
- * CSV_READ_BUFFER_SIZE
- * default value is 1mb
- */
- public static final String CSV_READ_BUFFER_SIZE_DEFAULT = "1048576";
-
- /**
- * min value for csv read buffer size, 10 kb
- */
- public static final int CSV_READ_BUFFER_SIZE_MIN = 10240;
-
- /**
- * max value for csv read buffer size, 10 mb
- */
- public static final int CSV_READ_BUFFER_SIZE_MAX = 10485760;
-
- /**
- * CARBON_MERGE_SORT_READER_THREAD
- */
- @CarbonProperty
- public static final String CARBON_MERGE_SORT_READER_THREAD = "carbon.merge.sort.reader.thread";
-
- /**
- * CARBON_MERGE_SORT_READER_THREAD DEFAULT value
- */
- public static final String CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE = "3";
-
- /**
- * TIME_STAT_UTIL_TYPE
- */
- @CarbonProperty
- public static final String ENABLE_DATA_LOADING_STATISTICS = "enable.data.loading.statistics";
-
- /**
- * TIME_STAT_UTIL_TYPE_DEFAULT
- */
-
- public static final String ENABLE_DATA_LOADING_STATISTICS_DEFAULT = "false";
-
- /**
- * NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK
- */
- @CarbonProperty
- public static final String NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK =
- "carbon.concurrent.lock.retries";
-
- /**
- * NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK.
- *
- * Because we want concurrent loads to be completed even if they have to wait for the lock
- * therefore taking the default as 100.
- *
- * Example: Concurrent loads will use this to wait to acquire the table status lock.
- */
- public static final int NUMBER_OF_TRIES_FOR_CONCURRENT_LOCK_DEFAULT = 100;
-
- /**
- * MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK
- */
- @CarbonProperty
- public static final String MAX_TIMEOUT_FOR_CONCURRENT_LOCK =
- "carbon.concurrent.lock.retry.timeout.sec";
-
- /**
- * MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK
- *
- * Example: Concurrent loads will use this to wait to acquire the table status lock.
- */
- public static final int MAX_TIMEOUT_FOR_CONCURRENT_LOCK_DEFAULT = 1;
-
- /**
- * NUMBER_OF_TRIES_FOR_CARBON_LOCK
- */
- @CarbonProperty
- public static final String NUMBER_OF_TRIES_FOR_CARBON_LOCK =
- "carbon.lock.retries";
-
- /**
- * NUMBER_OF_TRIES_FOR_CARBON_LOCK
- */
- public static final int NUMBER_OF_TRIES_FOR_CARBON_LOCK_DEFAULT = 3;
-
- /**
- * MAX_TIMEOUT_FOR_CARBON_LOCK
- */
- @CarbonProperty
- public static final String MAX_TIMEOUT_FOR_CARBON_LOCK =
- "carbon.lock.retry.timeout.sec";
-
- /**
- * MAX_TIMEOUT_FOR_CARBON_LOCK
- */
- public static final int MAX_TIMEOUT_FOR_CARBON_LOCK_DEFAULT = 5;
-
- /**
- * CARBON_PREFETCH_BUFFERSIZE
- */
- @CarbonProperty
- public static final String CARBON_PREFETCH_BUFFERSIZE = "carbon.prefetch.buffersize";
-
- /**
- * CARBON_PREFETCH_BUFFERSIZE DEFAULT VALUE
- */
- public static final String CARBON_PREFETCH_BUFFERSIZE_DEFAULT = "1000";
-
- /**
- * CARBON_PREFETCH_IN_MERGE
- */
- public static final boolean CARBON_PREFETCH_IN_MERGE_VALUE = false;
-
- /**
- * ENABLE_AUTO_LOAD_MERGE
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String ENABLE_AUTO_LOAD_MERGE = "carbon.enable.auto.load.merge";
-
- /**
- * DEFAULT value of ENABLE_AUTO_LOAD_MERGE
- */
- public static final String DEFAULT_ENABLE_AUTO_LOAD_MERGE = "false";
-
- /**
- * maximum dictionary chunk size that can be kept in memory while writing dictionary file
- */
- @CarbonProperty
- public static final String DICTIONARY_ONE_CHUNK_SIZE = "carbon.dictionary.chunk.size";
-
- /**
- * dictionary chunk default size
- */
- public static final String DICTIONARY_ONE_CHUNK_SIZE_DEFAULT = "10000";
-
- /**
- * Dictionary Server Worker Threads
- */
- @CarbonProperty
- public static final String DICTIONARY_WORKER_THREADS = "dictionary.worker.threads";
-
- /**
- * Dictionary Server Worker Threads
- */
- public static final String DICTIONARY_WORKER_THREADS_DEFAULT = "1";
-
- /**
- * Size of Major Compaction in MBs
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String CARBON_MAJOR_COMPACTION_SIZE = "carbon.major.compaction.size";
-
- /**
- * By default size of major compaction in MBs.
- */
- public static final String DEFAULT_CARBON_MAJOR_COMPACTION_SIZE = "1024";
-
- /**
- * This property is used to tell how many segments to be preserved from merging.
- */
- @CarbonProperty
- public static final java.lang.String PRESERVE_LATEST_SEGMENTS_NUMBER =
- "carbon.numberof.preserve.segments";
-
- /**
- * If preserve property is enabled then 2 segments will be preserved.
- */
- public static final String DEFAULT_PRESERVE_LATEST_SEGMENTS_NUMBER = "0";
-
- /**
- * This property will determine the loads of how many days can be compacted.
- */
- @CarbonProperty
- public static final java.lang.String DAYS_ALLOWED_TO_COMPACT = "carbon.allowed.compaction.days";
-
- /**
- * Default value of 1 day loads can be compacted
- */
- public static final String DEFAULT_DAYS_ALLOWED_TO_COMPACT = "0";
-
- /**
- * space reserved for writing block meta data in carbon data file
- */
- @CarbonProperty
- public static final String CARBON_BLOCK_META_RESERVED_SPACE =
- "carbon.block.meta.size.reserved.percentage";
-
- /**
- * default value for space reserved for writing block meta data in carbon data file
- */
- public static final String CARBON_BLOCK_META_RESERVED_SPACE_DEFAULT = "10";
-
- /**
- * this variable is to enable/disable prefetch of data during merge sort while
- * reading data from sort temp files
- */
- @CarbonProperty
- public static final String CARBON_MERGE_SORT_PREFETCH = "carbon.merge.sort.prefetch";
-
- public static final String CARBON_MERGE_SORT_PREFETCH_DEFAULT = "true";
-
- /**
- * If we are executing insert into query from source table using select statement
- * & loading the same source table concurrently, when select happens on source table
- * during the data load , it gets new record for which dictionary is not generated,
- * So there will be inconsistency. To avoid this condition we can persist the dataframe
- * into MEMORY_AND_DISK and perform insert into operation. By default this value
- * will be false because no need to persist the dataframe in all cases. If user want
- * to run load and insert queries on source table concurrently then user can enable this flag
- */
- @InterfaceStability.Evolving
- @CarbonProperty
- public static final String CARBON_INSERT_PERSIST_ENABLED = "carbon.insert.persist.enable";
-
- /**
- * by default rdd will not be persisted in the insert case.
- */
- public static final String CARBON_INSERT_PERSIST_ENABLED_DEFAULT = "false";
-
- /**
- * Which storage level to persist dataset when insert into data
- * with 'carbon.insert.persist.enable'='true'
- */
- @InterfaceStability.Evolving
- @CarbonProperty
- public static final String CARBON_INSERT_STORAGE_LEVEL =
- "carbon.insert.storage.level";
-
- /**
- * The default value(MEMORY_AND_DISK) is the same as the default storage level of Dataset.
- * Unlike `RDD.cache()`, the default storage level is set to be `MEMORY_AND_DISK` because
- * recomputing the in-memory columnar representation of the underlying table is expensive.
- *
- * if user's executor has less memory, set the CARBON_INSERT_STORAGE_LEVEL
- * to MEMORY_AND_DISK_SER or other storage level to correspond to different environment.
- * You can get more recommendations about storage level in spark website:
- * http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence.
- */
- public static final String CARBON_INSERT_STORAGE_LEVEL_DEFAULT = "MEMORY_AND_DISK";
-
- /**
- * Number of unmerged segments to be merged.
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String COMPACTION_SEGMENT_LEVEL_THRESHOLD =
- "carbon.compaction.level.threshold";
-
- /**
- * Default count for Number of segments to be merged in levels is 4,3
- */
- public static final String DEFAULT_SEGMENT_LEVEL_THRESHOLD = "4,3";
-
- /**
- * Number of Update Delta files which is the Threshold for IUD compaction.
- * Only accepted Range is 0 - 10000. Outside this range system will pick default value.
- */
- @CarbonProperty
- public static final String UPDATE_DELTAFILE_COUNT_THRESHOLD_IUD_COMPACTION =
- "carbon.horizontal.update.compaction.threshold";
-
- /**
- * Default count of segments which act as a threshold for IUD compaction merge.
- */
- public static final String DEFAULT_UPDATE_DELTAFILE_COUNT_THRESHOLD_IUD_COMPACTION = "1";
-
- /**
- * Number of Delete Delta files which is the Threshold for IUD compaction.
- * Only accepted Range is 0 - 10000. Outside this range system will pick default value.
- */
- @CarbonProperty
- public static final String DELETE_DELTAFILE_COUNT_THRESHOLD_IUD_COMPACTION =
- "carbon.horizontal.delete.compaction.threshold";
-
- /**
- * Default count of segments which act as a threshold for IUD compaction merge.
- */
- public static final String DEFAULT_DELETE_DELTAFILE_COUNT_THRESHOLD_IUD_COMPACTION = "1";
-
- /**
- * @Deprecated : This property has been deprecated.
- * Property for enabling system level compaction lock.1 compaction can run at once.
- */
- @CarbonProperty
- public static final String ENABLE_CONCURRENT_COMPACTION = "carbon.concurrent.compaction";
-
- /**
- * Default value of Property for enabling system level compaction lock.1 compaction can run
- * at once.
- */
- public static final String DEFAULT_ENABLE_CONCURRENT_COMPACTION = "true";
-
- /**
- * This batch size is used to send rows from load step to another step in batches.
- */
- @CarbonProperty
- public static final String DATA_LOAD_BATCH_SIZE = "DATA_LOAD_BATCH_SIZE";
-
- /**
- * Default size of data load batch size.
- */
- public static final String DATA_LOAD_BATCH_SIZE_DEFAULT = "1000";
-
- /**
- * to determine to use the rdd persist or not.
- */
- @InterfaceStability.Evolving
- @CarbonProperty
- public static final String CARBON_UPDATE_PERSIST_ENABLE = "carbon.update.persist.enable";
-
- /**
- * by default rdd will be persisted in the update case.
- */
- public static final String CARBON_UPDATE_PERSIST_ENABLE_DEFAULT = "true";
-
- /**
- * for enabling or disabling Horizontal Compaction.
- */
- @CarbonProperty
- public static final String CARBON_HORIZONTAL_COMPACTION_ENABLE =
- "carbon.horizontal.compaction.enable";
-
- /**
- * Default value for HorizontalCompaction is true.
- */
- public static final String CARBON_HORIZONTAL_COMPACTION_ENABLE_DEFAULT = "true";
-
- /**
- * Which storage level to persist dataset when updating data
- * with 'carbon.update.persist.enable'='true'
- */
- @InterfaceStability.Evolving
- @CarbonProperty
- public static final String CARBON_UPDATE_STORAGE_LEVEL =
- "carbon.update.storage.level";
-
- /**
- * The default value(MEMORY_AND_DISK) is the same as the default storage level of Dataset.
- * Unlike `RDD.cache()`, the default storage level is set to be `MEMORY_AND_DISK` because
- * recomputing the in-memory columnar representation of the underlying table is expensive.
- *
- * if user's executor has less memory, set the CARBON_UPDATE_STORAGE_LEVEL
- * to MEMORY_AND_DISK_SER or other storage level to correspond to different environment.
- * You can get more recommendations about storage level in spark website:
- * http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence.
- */
- public static final String CARBON_UPDATE_STORAGE_LEVEL_DEFAULT = "MEMORY_AND_DISK";
-
- /**
- * to enable unsafe column page
- */
- @CarbonProperty
- public static final String ENABLE_UNSAFE_COLUMN_PAGE = "enable.unsafe.columnpage";
-
- /**
- * default value of ENABLE_UNSAFE_COLUMN_PAGE
- */
- public static final String ENABLE_UNSAFE_COLUMN_PAGE_DEFAULT = "true";
-
- /**
- * to enable offheap sort
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String ENABLE_UNSAFE_SORT = "enable.unsafe.sort";
-
- /**
- * to enable offheap sort
- */
- public static final String ENABLE_UNSAFE_SORT_DEFAULT = "true";
-
- /**
- * to enable offheap sort
- */
- @CarbonProperty(dynamicConfigurable = true)
- public static final String ENABLE_OFFHEAP_SORT = "enable.offheap.sort";
-
- /**
- * to enable offheap sort
- */
- public static final String ENABLE_OFFHEAP_SORT_DEFAULT = "true";
-
- @CarbonProperty
- public static final String ENABLE_INMEMORY_MERGE_SORT = "enable.inmemory.merge.sort";
-
- public static final String ENABLE_INMEMORY_MERGE_SORT_DEFAULT = "false";
-
- @CarbonProperty
- public static final String OFFHEAP_SORT_CHUNK_SIZE_IN_MB = "offheap.sort.chunk.size.inmb";
-
- public static final String OFFHEAP_SORT_CHUNK_SIZE_IN_MB_DEFAULT = "64";
-
- @CarbonProperty
- public static final String UNSAFE_WORKING_MEMORY_IN_MB = "carbon.unsafe.working.memory.in.mb";
-
- public static final String UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT = "512";
-
- @CarbonProperty
- public static final String UNSAFE_DRIVER_WORKING_MEMORY_IN_MB =
- "carbon.unsafe.driver.working.memory.in.mb";
-
- /**
- * Sorts the data in batches and writes the batch data to store with index file.
- */
- @CarbonProperty
- public static final String LOAD_SORT_SCOPE = "carbon.load.sort.scope";
-
- /**
- * If set to BATCH_SORT, the sorting scope is smaller and more index tree will be created,
- * thus loading is faster but query maybe slower.
- * If set to LOCAL_SORT, the sorting scope is bigger and one index tree per data node will be
- * created, thus loading is slower but query is faster.
- * If set to GLOBAL_SORT, the sorting scope is bigger and one index tree per task will be
- * created, thus loading is slower but query is faster.
- */
- public static final String LOAD_SORT_SCOPE_DEFAULT = "LOCAL_SORT";
-
- /**
- * Size of batch data to keep in memory, as a thumb rule it supposed
- * to be less than 45% of sort.inmemory.size.inmb otherwise it may spill intermediate data to disk
- */
- @CarbonProperty
- public static final String LOAD_BATCH_SORT_SIZE_INMB = "carbon.load.batch.sort.size.inmb";
-
- public static final String LOAD_BATCH_SORT_SIZE_INMB_DEFAULT = "0";
-
- /**
- * The Number of partitions to use when shuffling data for sort. If user don't configurate or
- * configurate it less than 1, it uses the number of map tasks as reduce tasks. In general, we
- * recommend 2-3 tasks per CPU core in your cluster.
- */
- @CarbonProperty
- public static final String LOAD_GLOBAL_SORT_PARTITIONS = "carbon.load.global.sort.partitions";
-
- public static final String LOAD_GLOBAL_SORT_PARTITIONS_DEFAULT = "0";
-
- /*
- * carbon dictionary server port
- */
- @CarbonProperty
- public static final String DICTIONARY_SERVER_PORT = "carbon.dictionary.server.port";
-
- /**
- * Default carbon dictionary server port
- */
- public static final String DICTIONARY_SERVER_PORT_DEFAULT = "2030";
- /**
- * whether to prefetch data while loading.
- */
- @CarbonProperty
- public static final String USE_PREFETCH_WHILE_LOADING = "carbon.loading.prefetch";
-
- /**
- * default value for prefetch data while loading.
- */
- public static final String USE_PREFETCH_WHILE_LOADING_DEFAULT = "false";
-
- /**
- * for loading, whether to use yarn's local dir the main purpose is to avoid single disk hot spot
- */
- @CarbonProperty
- public static final String CARBON_LOADING_USE_YARN_LOCAL_DIR = "carbon.use.local.dir";
-
- /**
- * default value for whether to enable carbon use yarn local dir
- */
- public static final String CARBON_LOADING_USE_YARN_LOCAL_DIR_DEFAULT = "true";
+ public static final Property BLOCKLET_SIZE = Property.buildIntProperty()
+ .key("carbon.blocklet.size")
+ .defaultValue(120000)
+ .minValue(2000)
+ .maxValue(12000000)
+ .doc("blocklet size in carbon file")
+ .build();

/**
- * name of compressor to compress sort temp files
+ * min block size in MB
*/
- @CarbonProperty
- public static final String CARBON_SORT_TEMP_COMPRESSOR = "carbon.sort.temp.compressor";
+ public static final int BLOCK_SIZE_MIN_VAL = 1;

/**
- * The optional values are 'SNAPPY','GZIP','BZIP2','LZ4','ZSTD' and empty.
- * Specially, empty means that Carbondata will not compress the sort temp files.
+ * max block size in MB
*/
- public static final String CARBON_SORT_TEMP_COMPRESSOR_DEFAULT = "SNAPPY";
+ public static final int BLOCK_SIZE_MAX_VAL = 2048;

- /**
- * Which storage level to persist rdd when sort_scope=global_sort
- */
- @InterfaceStability.Evolving
- @CarbonProperty
- public static final String CARBON_GLOBAL_SORT_RDD_STORAGE_LEVEL =
- "carbon.global.sort.rdd.storage.level";
+ public static final Property CARBON_PROPERTIES_FILE_PATH = Property.buildStringProperty()
+ .key("carbon.properties.filepath")
+ .defaultValue("../../../conf/carbon.properties")
+ .doc("carbon properties file path")
+ .build();
+
+ public static final Property CARBON_DDL_BASE_HDFS_URL = Property.buildStringProperty()
+ .key("carbon.ddl.base.hdfs.url")
+ .defaultValue("")
+ .doc("CARBON_DDL_BASE_HDFS_URL")
+ .build();
+
+ public static final Property CARBON_BADRECORDS_LOC = Property.buildStringProperty()
+ .key("carbon.badRecords.location")
+ .defaultValue("")
+ .doc("CARBON_BADRECORDS_LOCATION")
+ .build();
+
+ public static final Property CARBON_TIMESTAMP_FORMAT =
+ Property.buildStringProperty()
+ .key("carbon.timestamp.format")
+ .defaultValue("yyyy-MM-dd HH:mm:ss")
+ .doc("Property for specifying the format of TIMESTAMP data type column. " +
+ "e.g. yyyy/MM/dd HH:mm:ss, or using default value")
+ .build();

/**
- * The default value(MEMORY_ONLY) is designed for executors with big memory, if user's executor
- * has less memory, set the CARBON_GLOBAL_SORT_RDD_STORAGE_LEVEL to MEMORY_AND_DISK_SER or
- * other storage level to correspond to different environment.
- * You can get more recommendations about storage level in spark website:
- * http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence.
+ * CARBON_TIMESTAMP
*/
- public static final String CARBON_GLOBAL_SORT_RDD_STORAGE_LEVEL_DEFAULT = "MEMORY_ONLY";
+ public static final String CARBON_TIMESTAMP = "dd-MM-yyyy HH:mm:ss";

/**
- * property for configuring parallelism per segment when doing an update. Increase this
- * value will avoid data screw problem for a large segment.
- * Refer to CarbonData-1373 for more details.
+ * CARBON_TIMESTAMP
*/
- @CarbonProperty
- public static final String CARBON_UPDATE_SEGMENT_PARALLELISM =
- "carbon.update.segment.parallelism";
+ public static final String CARBON_TIMESTAMP_MILLIS = "dd-MM-yyyy HH:mm:ss:SSS";

- /**
- * In default we will not optimize the update
- */
- public static final String CARBON_UPDATE_SEGMENT_PARALLELISM_DEFAULT = "1";

- /**
- * The property to configure the mdt file folder path, earlier it was pointing to the
- * fixed carbon store path. This is needed in case of the federation setup when user removes
- * the fixed store path nameService
- */
- @CarbonProperty
- public static final String CARBON_UPDATE_SYNC_FOLDER = "carbon.update.sync.folder";
+ public static final Property CARBON_DATE_FORMAT = Property.buildStringProperty()
+ .key("carbon.date.format")
+ .defaultValue("yyyy-MM-dd")
+ .doc("Property for specifying the format of DATE data type column." +
+ " e.g. yyyy/MM/dd , or using default value")
+ .build();

- public static final String CARBON_UPDATE_SYNC_FOLDER_DEFAULT = "/tmp/carbondata";
+ public static final Property COMPRESSOR = Property.buildStringProperty()
+ .key("carbon.column.compressor")
+ .defaultValue("snappy")
+ .doc("compressor for writing/reading CarbonData file")
+ .build();

- /**
- * Configures the parser/writer to limit the length of displayed contents being parsed/written
- * in the exception message when an error occurs.
- * Here {@code 0} means no exceptions will include the content being manipulated in their
- * attributes.
- */
- public static final int CARBON_ERROR_CONTENT_LENGTH = 0;
+ public static final Property LOCK_TYPE = Property.buildStringProperty()
+ .key("carbon.lock.type")
+ .defaultValue("LOCALLOCK")
+ .doc("ZOOKEEPER_ENABLE_LOCK if this is set to true then zookeeper will" +
+ " be used to handle locking mechanism of carbon")
+ .build();

- /**
- * if the byte size of streaming segment reach this value,
- * the system will create a new stream segment
- */
- @CarbonProperty
- public static final String HANDOFF_SIZE = "carbon.streaming.segment.max.size";
+ public static final Property LOCK_PATH = Property.buildStringProperty()
+ .key("carbon.lock.path")
+ .defaultValue("")
+ .doc("Specifies the path where the lock files have to be created. " +
+ "By default, lock files are created in table path.")
+ .build();

/**
- * the default handoff size of streaming segment, the unit is byte
+ * ZOOKEEPER_LOCATION this is the location in zookeeper file system where locks are created.
+ * mechanism of carbon
*/
- public static final long HANDOFF_SIZE_DEFAULT = 1024L * 1024 * 1024;
+ public static final String ZOOKEEPER_LOCATION = "/CarbonLocks";
--- End diff --

ok, done

---

qiuchenjian-2