Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2589 Build Failed with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/6439/ --- |
In reply to this post by qiuchenjian-2
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2589 SDV Build Fail , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/6106/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2589 Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/7749/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2589 Build Failed with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/6474/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2589 Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/7754/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2589 Build Failed with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/6479/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2589 Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/7755/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2589 Build Failed with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/6480/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2589 Build Failed with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/6481/ --- |
In reply to this post by qiuchenjian-2
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2589 Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/7756/ --- |
In reply to this post by qiuchenjian-2
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2589 SDV Build Fail , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/6138/ --- |
In reply to this post by qiuchenjian-2
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2589 SDV Build Fail , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/6140/ --- |
In reply to this post by qiuchenjian-2
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2589 SDV Build Fail , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/6141/ --- |
In reply to this post by qiuchenjian-2
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2589 SDV Build Fail , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/6142/ --- |
In reply to this post by qiuchenjian-2
Github user ajithme commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2589#discussion_r207501460 --- Diff: store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/ScanDescriptor.java --- @@ -15,23 +15,33 @@ * limitations under the License. */ -package org.apache.carbondata.store.api.descriptor; +package org.apache.carbondata.sdk.store.descriptor; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.util.Objects; +import org.apache.carbondata.common.annotations.InterfaceAudience; +import org.apache.carbondata.common.annotations.InterfaceStability; import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.util.ObjectSerializationUtil; -public class SelectDescriptor { +import org.apache.hadoop.io.Writable; + +@InterfaceAudience.User +@InterfaceStability.Evolving +public class ScanDescriptor implements Writable { private TableIdentifier table; private String[] projection; private Expression filter; private long limit; --- End diff -- Must be Long.MAX_VALUE --- |
In reply to this post by qiuchenjian-2
Github user ajithme commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2589#discussion_r207431095 --- Diff: store/sdk/src/main/java/org/apache/carbondata/sdk/store/ScannerImpl.java --- @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.sdk.store; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.datastore.row.CarbonRow; +import org.apache.carbondata.core.metadata.schema.table.TableInfo; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.hadoop.CarbonInputSplit; +import org.apache.carbondata.hadoop.CarbonMultiBlockSplit; +import org.apache.carbondata.hadoop.api.CarbonInputFormat; +import org.apache.carbondata.sdk.store.conf.StoreConf; +import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor; +import org.apache.carbondata.sdk.store.descriptor.TableIdentifier; +import org.apache.carbondata.sdk.store.exception.CarbonException; +import org.apache.carbondata.sdk.store.service.DataService; +import org.apache.carbondata.sdk.store.service.PruneService; +import org.apache.carbondata.sdk.store.service.ServiceFactory; +import org.apache.carbondata.sdk.store.service.model.PruneRequest; +import org.apache.carbondata.sdk.store.service.model.PruneResponse; +import org.apache.carbondata.sdk.store.service.model.ScanRequest; +import org.apache.carbondata.sdk.store.service.model.ScanResponse; + +import org.apache.hadoop.conf.Configuration; + +class ScannerImpl implements Scanner { + private static final LogService LOGGER = + LogServiceFactory.getLogService(ScannerImpl.class.getCanonicalName()); + + private PruneService pruneService; + private TableInfo tableInfo; + + ScannerImpl(StoreConf conf, TableInfo tableInfo) throws IOException { + this.pruneService = ServiceFactory.createPruneService( + conf.masterHost(), conf.registryServicePort()); --- End diff -- must be prune service port --- |
In reply to this post by qiuchenjian-2
Github user ajithme commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2589#discussion_r207431252 --- Diff: store/sdk/src/main/java/org/apache/carbondata/sdk/store/service/StoreService.java --- @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.sdk.store.service; + +import java.util.List; + +import org.apache.carbondata.common.annotations.InterfaceAudience; +import org.apache.carbondata.core.datastore.row.CarbonRow; +import org.apache.carbondata.core.metadata.schema.table.CarbonTable; +import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor; +import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor; +import org.apache.carbondata.sdk.store.descriptor.TableDescriptor; +import org.apache.carbondata.sdk.store.descriptor.TableIdentifier; +import org.apache.carbondata.sdk.store.exception.CarbonException; + +import org.apache.hadoop.ipc.VersionedProtocol; + +@InterfaceAudience.Internal +public interface StoreService extends VersionedProtocol { + long versionID = 1L; + + void createTable(TableDescriptor descriptor) throws CarbonException; + + void dropTable(TableIdentifier table) throws CarbonException; + + CarbonTable getTable(TableIdentifier table) throws CarbonException; --- End diff -- hadoop RPC need response object to be a org.apache.hadoop.io.serializer.WritableSerialization --- |
In reply to this post by qiuchenjian-2
Github user ajithme commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2589#discussion_r207433215 --- Diff: store/sdk/src/main/java/org/apache/carbondata/sdk/store/ScannerImpl.java --- @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.sdk.store; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.datastore.row.CarbonRow; +import org.apache.carbondata.core.metadata.schema.table.TableInfo; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.hadoop.CarbonInputSplit; +import org.apache.carbondata.hadoop.CarbonMultiBlockSplit; +import org.apache.carbondata.hadoop.api.CarbonInputFormat; +import org.apache.carbondata.sdk.store.conf.StoreConf; +import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor; +import org.apache.carbondata.sdk.store.descriptor.TableIdentifier; +import org.apache.carbondata.sdk.store.exception.CarbonException; +import org.apache.carbondata.sdk.store.service.DataService; +import org.apache.carbondata.sdk.store.service.PruneService; +import org.apache.carbondata.sdk.store.service.ServiceFactory; +import org.apache.carbondata.sdk.store.service.model.PruneRequest; +import org.apache.carbondata.sdk.store.service.model.PruneResponse; +import org.apache.carbondata.sdk.store.service.model.ScanRequest; +import org.apache.carbondata.sdk.store.service.model.ScanResponse; + +import org.apache.hadoop.conf.Configuration; + +class ScannerImpl implements Scanner { + private static final LogService LOGGER = + LogServiceFactory.getLogService(ScannerImpl.class.getCanonicalName()); + + private PruneService pruneService; + private TableInfo tableInfo; + + ScannerImpl(StoreConf conf, TableInfo tableInfo) throws IOException { + this.pruneService = ServiceFactory.createPruneService( + conf.masterHost(), conf.registryServicePort()); + this.tableInfo = tableInfo; + } + + /** + * Trigger a RPC to Carbon Master to do pruning + * @param table table identifier + * @param filterExpression expression of filter predicate given by user + * @return list of ScanUnit + * @throws CarbonException if any error occurs + */ + @Override + public List<ScanUnit> prune(TableIdentifier table, Expression filterExpression) + throws CarbonException { + try { + Configuration configuration = new Configuration(); + CarbonInputFormat.setTableName(configuration, table.getTableName()); --- End diff -- can use CarbonInputFormat.setTableInfo(configuration, tableInfo); else org.apache.carbondata.hadoop.api.CarbonInputFormat#getAbsoluteTableIdentifier will have empty path --- |
In reply to this post by qiuchenjian-2
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2589#discussion_r207698994 --- Diff: store/sdk/src/main/java/org/apache/carbondata/sdk/store/ScannerImpl.java --- @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.sdk.store; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.datastore.row.CarbonRow; +import org.apache.carbondata.core.metadata.schema.table.TableInfo; +import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.hadoop.CarbonInputSplit; +import org.apache.carbondata.hadoop.CarbonMultiBlockSplit; +import org.apache.carbondata.hadoop.api.CarbonInputFormat; +import org.apache.carbondata.sdk.store.conf.StoreConf; +import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor; +import org.apache.carbondata.sdk.store.descriptor.TableIdentifier; +import org.apache.carbondata.sdk.store.exception.CarbonException; +import org.apache.carbondata.sdk.store.service.DataService; +import org.apache.carbondata.sdk.store.service.PruneService; +import org.apache.carbondata.sdk.store.service.ServiceFactory; +import org.apache.carbondata.sdk.store.service.model.PruneRequest; +import org.apache.carbondata.sdk.store.service.model.PruneResponse; +import org.apache.carbondata.sdk.store.service.model.ScanRequest; +import org.apache.carbondata.sdk.store.service.model.ScanResponse; + +import org.apache.hadoop.conf.Configuration; + +class ScannerImpl implements Scanner { + private static final LogService LOGGER = + LogServiceFactory.getLogService(ScannerImpl.class.getCanonicalName()); + + private PruneService pruneService; + private TableInfo tableInfo; + + ScannerImpl(StoreConf conf, TableInfo tableInfo) throws IOException { + this.pruneService = ServiceFactory.createPruneService( + conf.masterHost(), conf.registryServicePort()); + this.tableInfo = tableInfo; + } + + /** + * Trigger a RPC to Carbon Master to do pruning + * @param table table identifier + * @param filterExpression expression of filter predicate given by user + * @return list of ScanUnit + * @throws CarbonException if any error occurs + */ + @Override + public List<ScanUnit> prune(TableIdentifier table, Expression filterExpression) + throws CarbonException { + try { + Configuration configuration = new Configuration(); + CarbonInputFormat.setTableName(configuration, table.getTableName()); --- End diff -- ok --- |
In reply to this post by qiuchenjian-2
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2589#discussion_r207699000 --- Diff: store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/ScanDescriptor.java --- @@ -15,23 +15,33 @@ * limitations under the License. */ -package org.apache.carbondata.store.api.descriptor; +package org.apache.carbondata.sdk.store.descriptor; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.util.Objects; +import org.apache.carbondata.common.annotations.InterfaceAudience; +import org.apache.carbondata.common.annotations.InterfaceStability; import org.apache.carbondata.core.scan.expression.Expression; +import org.apache.carbondata.core.util.ObjectSerializationUtil; -public class SelectDescriptor { +import org.apache.hadoop.io.Writable; + +@InterfaceAudience.User +@InterfaceStability.Evolving +public class ScanDescriptor implements Writable { private TableIdentifier table; private String[] projection; private Expression filter; private long limit; --- End diff -- ok --- |
Free forum by Nabble | Edit this page |