Github user xuchuanyin commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2275#discussion_r187769074
--- Diff: datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapRefresher.java ---
@@ -114,108 +118,36 @@ public void initialize() throws IOException {
indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(analyzer));
}
- private IndexWriter createPageIndexWriter() throws IOException {
- // save index data into ram, write into disk after one page finished
- RAMDirectory ramDir = new RAMDirectory();
- return new IndexWriter(ramDir, new IndexWriterConfig(analyzer));
- }
-
- private void addPageIndex(IndexWriter pageIndexWriter) throws IOException {
-
- Directory directory = pageIndexWriter.getDirectory();
-
- // close ram writer
- pageIndexWriter.close();
-
- // add ram index data into disk
- indexWriter.addIndexes(directory);
-
- // delete this ram data
- directory.close();
- }
-
- @Override
- public void addRow(int blockletId, int pageId, int rowId, Object[] values) throws IOException {
- if (rowId == 0) {
- if (pageIndexWriter != null) {
- addPageIndex(pageIndexWriter);
- }
- pageIndexWriter = createPageIndexWriter();
- }
-
- // create a new document
- Document doc = new Document();
-
- // add blocklet Id
- doc.add(new IntPoint(LuceneDataMapWriter.BLOCKLETID_NAME, (int) values[columnsCount]));
- doc.add(new StoredField(LuceneDataMapWriter.BLOCKLETID_NAME, (int) values[columnsCount]));
-
- // add page id
- doc.add(new IntPoint(LuceneDataMapWriter.PAGEID_NAME, (int) values[columnsCount + 1]));
- doc.add(new StoredField(LuceneDataMapWriter.PAGEID_NAME, (int) values[columnsCount + 1]));
-
- // add row id
- doc.add(new IntPoint(LuceneDataMapWriter.ROWID_NAME, rowId));
- doc.add(new StoredField(LuceneDataMapWriter.ROWID_NAME, rowId));
+ @Override public void addRow(int blockletId, int pageId, int rowId, Object[] values)
+ throws IOException {
// add other fields
+ LuceneDataMapWriter.LuceneColumnKeys columns =
+ new LuceneDataMapWriter.LuceneColumnKeys(columnsCount);
for (int colIdx = 0; colIdx < columnsCount; colIdx++) {
- CarbonColumn column = indexColumns.get(colIdx);
- addField(doc, column.getColName(), column.getDataType(), values[colIdx]);
+ columns.getColValues()[colIdx] = values[colIdx];
+ }
+ if (writeCacheSize > 0) {
+ addToCache(columns, rowId, pageId, blockletId, cache, intBuffer, storeBlockletWise);
+ flushCacheIfCan();
+ } else {
+ addData(columns, rowId, pageId, blockletId, intBuffer, indexWriter, indexColumns,
+ storeBlockletWise);
}
- pageIndexWriter.addDocument(doc);
}
- private boolean addField(Document doc, String fieldName, DataType type, Object value) {
- if (type == DataTypes.STRING) {
- doc.add(new TextField(fieldName, (String) value, Field.Store.NO));
- } else if (type == DataTypes.BYTE) {
- // byte type , use int range to deal with byte, lucene has no byte type
- IntRangeField field =
- new IntRangeField(fieldName, new int[] { Byte.MIN_VALUE }, new int[] { Byte.MAX_VALUE });
- field.setIntValue((int) value);
- doc.add(field);
- } else if (type == DataTypes.SHORT) {
- // short type , use int range to deal with short type, lucene has no short type
- IntRangeField field = new IntRangeField(fieldName, new int[] { Short.MIN_VALUE },
- new int[] { Short.MAX_VALUE });
- field.setShortValue((short) value);
- doc.add(field);
- } else if (type == DataTypes.INT) {
- // int type , use int point to deal with int type
- doc.add(new IntPoint(fieldName, (int) value));
- } else if (type == DataTypes.LONG) {
- // long type , use long point to deal with long type
- doc.add(new LongPoint(fieldName, (long) value));
- } else if (type == DataTypes.FLOAT) {
- doc.add(new FloatPoint(fieldName, (float) value));
- } else if (type == DataTypes.DOUBLE) {
- doc.add(new DoublePoint(fieldName, (double) value));
- } else if (type == DataTypes.DATE) {
- // TODO: how to get data value
- } else if (type == DataTypes.TIMESTAMP) {
- // TODO: how to get
- } else if (type == DataTypes.BOOLEAN) {
- IntRangeField field = new IntRangeField(fieldName, new int[] { 0 }, new int[] { 1 });
- field.setIntValue((boolean) value ? 1 : 0);
- doc.add(field);
- } else {
- LOGGER.error("unsupport data type " + type);
- throw new RuntimeException("unsupported data type " + type);
+ private void flushCacheIfCan() throws IOException {
--- End diff --
better to name it `flushCacheIfPossible`
---
Locked
