Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228073382 --- Diff: docs/csdk-guide.md --- @@ -30,106 +30,13 @@ code and without CarbonSession. In the carbon jars package, there exist a carbondata-sdk.jar, including SDK reader for CSDK. ## Quick example -``` -// 1. init JVM -JavaVM *jvm; -JNIEnv *initJVM() { - JNIEnv *env; - JavaVMInitArgs vm_args; - int parNum = 3; - int res; - JavaVMOption options[parNum]; - - options[0].optionString = "-Djava.compiler=NONE"; - options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar"; - options[2].optionString = "-verbose:jni"; - vm_args.version = JNI_VERSION_1_8; - vm_args.nOptions = parNum; - vm_args.options = options; - vm_args.ignoreUnrecognized = JNI_FALSE; - - res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args); - if (res < 0) { - fprintf(stderr, "\nCan't create Java VM\n"); - exit(1); - } - - return env; -} - -// 2. create carbon reader and read data -// 2.1 read data from local disk -/** - * test read data from local disk, without projection - * - * @param env jni env - * @return - */ -bool readFromLocalWithoutProjection(JNIEnv *env) { - - CarbonReader carbonReaderClass; - carbonReaderClass.builder(env, "../resources/carbondata", "test"); - carbonReaderClass.build(); - - while (carbonReaderClass.hasNext()) { - jobjectArray row = carbonReaderClass.readNextRow(); - jsize length = env->GetArrayLength(row); - int j = 0; - for (j = 0; j < length; j++) { - jobject element = env->GetObjectArrayElement(row, j); - char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE); - printf("%s\t", str); - } - printf("\n"); - } - carbonReaderClass.close(); -} - -// 2.2 read data from S3 - -/** - * read data from S3 - * parameter is ak sk endpoint - * - * @param env jni env - * @param argv argument vector - * @return - */ -bool readFromS3(JNIEnv *env, char *argv[]) { - CarbonReader reader; - - char *args[3]; - // "your access key" - args[0] = argv[1]; - // "your secret key" - args[1] = argv[2]; - // "your endPoint" - args[2] = argv[3]; - - reader.builder(env, "s3a://sdk/WriterOutput", "test"); - reader.withHadoopConf(3, args); - reader.build(); - printf("\nRead data from S3:\n"); - while (reader.hasNext()) { - jobjectArray row = reader.readNextRow(); - jsize length = env->GetArrayLength(row); - - int j = 0; - for (j = 0; j < length; j++) { - jobject element = env->GetObjectArrayElement(row, j); - char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE); - printf("%s\t", str); - } - printf("\n"); - } - - reader.close(); -} - -// 3. destory JVM - (jvm)->DestroyJavaVM(); -``` -Find example code at main.cpp of CSDK module + +Please find example code at main.cpp of CSDK module --- End diff -- ok, done --- |
In reply to this post by qiuchenjian-2
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228074939 --- Diff: store/CSDK/CarbonReader.cpp --- @@ -19,15 +19,14 @@ #include <jni.h> jobject CarbonReader::builder(JNIEnv *env, char *path, char *tableName) { - jniEnv = env; jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader"); jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder", --- End diff -- No please handle at once. Because these are same feature. suggest we fix it and make it clean. we should not merge PR which has issues as if some user uses it, he will get jvm crash --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2792 Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1008/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2792 Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1011/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2792 Build Success with Spark 2.3.1, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/9276/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2792 Build Success with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/1224/ --- |
In reply to this post by qiuchenjian-2
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228124647 --- Diff: store/CSDK/src/CarbonReader.cpp --- @@ -19,15 +19,14 @@ #include <jni.h> jobject CarbonReader::builder(JNIEnv *env, char *path, char *tableName) { - jniEnv = env; jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader"); jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder", "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;"); - jstring jpath = env->NewStringUTF(path); + jstring jPath = env->NewStringUTF(path); jstring jtableName = env->NewStringUTF(tableName); jvalue args[2]; - args[0].l = jpath; + args[0].l = jPath; args[1].l = jtableName; carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args); return carbonReaderBuilderObject; --- End diff -- No need to return object because we are storing this variable. make it void. --- |
In reply to this post by qiuchenjian-2
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228126086 --- Diff: store/CSDK/src/CarbonReader.h --- @@ -35,6 +40,13 @@ class CarbonReader { */ jobject carbonReaderObject; +public: + + /** + * jni env + */ + JNIEnv *jniEnv; + /** * create a CarbonReaderBuilder object for building carbonReader, * CarbonReaderBuilder object can configure different parameter --- End diff -- builder, projection, withHaaddopConf -- No need to have return value. They are not used. we can return void --- |
In reply to this post by qiuchenjian-2
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228128521 --- Diff: store/CSDK/src/CarbonReader.cpp --- @@ -79,21 +78,19 @@ jobject CarbonReader::build() { jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build", "()Lorg/apache/carbondata/sdk/file/CarbonReader;"); carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID); + jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); + hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z"); --- End diff -- please move this hasNextID to CarbonReader::hasNext(). if not null then only assign (only one time assignment) Also same comment for readNextRowID. --- |
In reply to this post by qiuchenjian-2
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228130454 --- Diff: store/CSDK/src/CarbonReader.cpp --- @@ -79,21 +78,19 @@ jobject CarbonReader::build() { jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build", "()Lorg/apache/carbondata/sdk/file/CarbonReader;"); carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID); + jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); + hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z"); + readNextRowID = jniEnv->GetMethodID(carbonReader, "readNextRow", + "()Ljava/lang/Object;"); return carbonReaderObject; } jboolean CarbonReader::hasNext() { - jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); - jmethodID hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z"); - unsigned char hasNext = jniEnv->CallBooleanMethod(carbonReaderObject, hasNextID); - return hasNext; + return jniEnv->CallBooleanMethod(carbonReaderObject, hasNextID); } -jobjectArray CarbonReader::readNextRow() { - jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); - jmethodID readNextRow2ID = jniEnv->GetMethodID(carbonReader, "readNextStringRow", "()[Ljava/lang/Object;"); - jobjectArray row = (jobjectArray) jniEnv->CallObjectMethod(carbonReaderObject, readNextRow2ID); - return row; +jobject CarbonReader::readNextRow() { + return (jobject) jniEnv->CallObjectMethod(carbonReaderObject, readNextRowID); } jboolean CarbonReader::close() { --- End diff -- call CallObjectMethod() instead of boolean as return of close is always void(), update header also --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2792 Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1024/ --- |
In reply to this post by qiuchenjian-2
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228186934 --- Diff: store/CSDK/CarbonReader.cpp --- @@ -19,15 +19,14 @@ #include <jni.h> jobject CarbonReader::builder(JNIEnv *env, char *path, char *tableName) { - jniEnv = env; jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader"); jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder", --- End diff -- I throw exception in build and handle it in main test case. --- |
In reply to this post by qiuchenjian-2
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228187015 --- Diff: store/CSDK/CarbonReader.cpp --- @@ -38,9 +37,9 @@ jobject CarbonReader::builder(JNIEnv *env, char *path) { jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader"); --- End diff -- added --- |
In reply to this post by qiuchenjian-2
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228188056 --- Diff: store/CSDK/src/CarbonReader.cpp --- @@ -19,15 +19,14 @@ #include <jni.h> jobject CarbonReader::builder(JNIEnv *env, char *path, char *tableName) { - jniEnv = env; jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader"); jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder", "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;"); - jstring jpath = env->NewStringUTF(path); + jstring jPath = env->NewStringUTF(path); jstring jtableName = env->NewStringUTF(tableName); jvalue args[2]; - args[0].l = jpath; + args[0].l = jPath; args[1].l = jtableName; carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args); return carbonReaderBuilderObject; --- End diff -- ok --- |
In reply to this post by qiuchenjian-2
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228188421 --- Diff: store/CSDK/src/CarbonReader.h --- @@ -35,6 +40,13 @@ class CarbonReader { */ jobject carbonReaderObject; +public: + + /** + * jni env + */ + JNIEnv *jniEnv; + /** * create a CarbonReaderBuilder object for building carbonReader, * CarbonReaderBuilder object can configure different parameter --- End diff -- ok, build --- |
In reply to this post by qiuchenjian-2
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228189848 --- Diff: store/CSDK/src/CarbonReader.cpp --- @@ -79,21 +78,19 @@ jobject CarbonReader::build() { jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build", "()Lorg/apache/carbondata/sdk/file/CarbonReader;"); carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID); + jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); + hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z"); --- End diff -- hasNextId is for method of class, only need one times. If move to haCarbonReader::hasNext(), it will increase read carbon file time --- |
In reply to this post by qiuchenjian-2
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228192284 --- Diff: store/CSDK/src/CarbonReader.cpp --- @@ -79,21 +78,19 @@ jobject CarbonReader::build() { jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build", "()Lorg/apache/carbondata/sdk/file/CarbonReader;"); carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID); + jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); + hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z"); --- End diff -- ok, done --- |
In reply to this post by qiuchenjian-2
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2792#discussion_r228192785 --- Diff: store/CSDK/src/CarbonReader.cpp --- @@ -79,21 +78,19 @@ jobject CarbonReader::build() { jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build", "()Lorg/apache/carbondata/sdk/file/CarbonReader;"); carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID); + jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); + hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z"); + readNextRowID = jniEnv->GetMethodID(carbonReader, "readNextRow", + "()Ljava/lang/Object;"); return carbonReaderObject; } jboolean CarbonReader::hasNext() { - jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); - jmethodID hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z"); - unsigned char hasNext = jniEnv->CallBooleanMethod(carbonReaderObject, hasNextID); - return hasNext; + return jniEnv->CallBooleanMethod(carbonReaderObject, hasNextID); } -jobjectArray CarbonReader::readNextRow() { - jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject); - jmethodID readNextRow2ID = jniEnv->GetMethodID(carbonReader, "readNextStringRow", "()[Ljava/lang/Object;"); - jobjectArray row = (jobjectArray) jniEnv->CallObjectMethod(carbonReaderObject, readNextRow2ID); - return row; +jobject CarbonReader::readNextRow() { + return (jobject) jniEnv->CallObjectMethod(carbonReaderObject, readNextRowID); } jboolean CarbonReader::close() { --- End diff -- ok, done --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2792 Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1025/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2792 Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1026/ --- |
Free forum by Nabble | Edit this page |