Posted by
Akash R Nilugal (Jira) on
Oct 07, 2016; 7:35am
URL: http://apache-carbondata-dev-mailing-list-archive.168.s1.nabble.com/jira-Commented-CARBONDATA-233-bad-record-logger-support-for-non-parseable-numeric-and-timestamp-data-tp1649.html
[
https://issues.apache.org/jira/browse/CARBONDATA-233?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15554396#comment-15554396 ]
ASF GitHub Bot commented on CARBONDATA-233:
-------------------------------------------
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/incubator-carbondata/pull/148#discussion_r82341356
--- Diff: processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordslogger.java ---
@@ -89,41 +95,55 @@ public static String hasBadRecord(String key) {
return badRecordEntry.remove(key);
}
- public void addBadRecordsToBilder(Object[] row, int size, String reason, String valueComparer) {
+ public void addBadRecordsToBuilder(Object[] row, int size, String reason, String valueComparer,
+ boolean badRecordsLogRedirect, boolean badRecordLoggerEnable) {
StringBuilder logStrings = new StringBuilder();
+ size = row.length;
int count = size;
for (int i = 0; i < size; i++) {
if (null == row[i]) {
- logStrings.append(row[i]);
+ char ch = logStrings.length() > 0 ? logStrings.charAt(logStrings.length() - 1) : (char) -1;
+ if (ch == ',') {
+ logStrings = logStrings.deleteCharAt(logStrings.lastIndexOf(","));
+ }
+ break;
} else if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(row[i].toString())) {
logStrings.append(valueComparer);
} else {
logStrings.append(row[i]);
}
if (count > 1) {
- logStrings.append(" , ");
+ logStrings.append(',');
}
count--;
}
-
- logStrings.append("----->");
- if (null != reason) {
- if (reason.indexOf(CarbonCommonConstants.MEMBER_DEFAULT_VAL) > -1) {
- logStrings.append(reason.replace(CarbonCommonConstants.MEMBER_DEFAULT_VAL, valueComparer));
- } else {
- logStrings.append(reason);
- }
+ if (badRecordsLogRedirect) {
+ writeBadRecordsToCSVFile(logStrings);
}
+ if( badRecordLoggerEnable ) {
+ logStrings.append("----->");
+ if (null != reason) {
+ if (reason.indexOf(CarbonCommonConstants.MEMBER_DEFAULT_VAL) > -1) {
+ logStrings
+ .append(reason.replace(CarbonCommonConstants.MEMBER_DEFAULT_VAL, valueComparer));
+ } else {
+ logStrings.append(reason);
+ }
+ }
- writeBadRecordsToFile(logStrings);
+ writeBadRecordsToFile(logStrings);
+ } else {
+ badRecordEntry.put(taskKey, "Partially");
+ }
}
/**
*
*/
private synchronized void writeBadRecordsToFile(StringBuilder logStrings) {
- String filePath = this.storePath + File.separator + this.fileName
- + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
+ String filePath =
--- End diff --
This line should be moved down under `if` condition
> bad record logger support for non parseable numeric and timestamp data
> ----------------------------------------------------------------------
>
> Key: CARBONDATA-233
> URL:
https://issues.apache.org/jira/browse/CARBONDATA-233> Project: CarbonData
> Issue Type: Improvement
> Reporter: Mohammad Shahid Khan
> Assignee: Mohammad Shahid Khan
>
> bad record logger support for non parseable numeric and timestamp data
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)