Repository: accumulo-wikisearch Updated Branches: refs/heads/1.8 [created] dd23f1071
ACCUMULO-2446: many updates for Accumulo 1.8.0 Project: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/commit/dd23f107 Tree: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/tree/dd23f107 Diff: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/diff/dd23f107 Branch: refs/heads/1.8 Commit: dd23f1071068625c6958001a1c0bc7ad39e30227 Parents: 9c30660 Author: Mike Miller <mmil...@apache.org> Authored: Thu Dec 1 15:18:30 2016 -0500 Committer: Mike Miller <mmil...@apache.org> Committed: Fri Dec 9 15:22:46 2016 -0500 ---------------------------------------------------------------------- .gitignore | 4 + ingest/bin/ingest.sh | 2 +- ingest/conf/wikipedia.xml.uno | 43 ++++++++ ingest/pom.xml | 57 ++--------- .../wikisearch/ingest/WikipediaIngester.java | 13 ++- .../ingest/WikipediaPartitionedIngester.java | 12 ++- .../output/BufferingRFileRecordWriter.java | 47 ++++----- .../output/SortingRFileOutputFormat.java | 9 +- pom.xml | 102 ++++++------------- query-war/pom.xml | 2 +- query/pom.xml | 95 ++--------------- .../iterator/DefaultIteratorEnvironment.java | 22 ++++ .../examples/wikisearch/query/Query.java | 5 +- 13 files changed, 159 insertions(+), 254 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 698a832..8a09ac0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ .idea **/*.iml **/lib +.project +.settings/ +.classpath + http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/ingest/bin/ingest.sh ---------------------------------------------------------------------- diff --git a/ingest/bin/ingest.sh b/ingest/bin/ingest.sh index 73d582d..f9f3495 100755 --- a/ingest/bin/ingest.sh +++ b/ingest/bin/ingest.sh @@ -38,7 +38,7 @@ LIBJARS=`echo $CLASSPATH | sed 's/^://' | sed 's/:/,/g'` # # Map/Reduce job # -JAR=$SCRIPT_DIR/../lib/wikisearch-ingest-1.5.0.jar +JAR=$SCRIPT_DIR/../lib/wikisearch-ingest-*.jar CONF=$SCRIPT_DIR/../conf/wikipedia.xml HDFS_DATA_DIR=$1 export HADOOP_CLASSPATH=$CLASSPATH http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/ingest/conf/wikipedia.xml.uno ---------------------------------------------------------------------- diff --git a/ingest/conf/wikipedia.xml.uno b/ingest/conf/wikipedia.xml.uno new file mode 100644 index 0000000..093551c --- /dev/null +++ b/ingest/conf/wikipedia.xml.uno @@ -0,0 +1,43 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + <property> + <name>wikipedia.accumulo.zookeepers</name> + <value>localhost:2181</value> + </property> + <property> + <name>wikipedia.accumulo.instance_name</name> + <value>uno</value> + </property> + <property> + <name>wikipedia.accumulo.user</name> + <value>root</value> + </property> + <property> + <name>wikipedia.accumulo.password</name> + <value>secret</value> + </property> + <property> + <name>wikipedia.accumulo.table</name> + <value>wikipedia</value> + </property> + <property> + <name>wikipedia.ingest.partitions</name> + <value>1</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/ingest/pom.xml ---------------------------------------------------------------------- diff --git a/ingest/pom.xml b/ingest/pom.xml index 1d17e27..426cff2 100644 --- a/ingest/pom.xml +++ b/ingest/pom.xml @@ -20,7 +20,7 @@ <parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-wikisearch</artifactId> - <version>1.5.0</version> + <version>1.8.0</version> </parent> <artifactId>wikisearch-ingest</artifactId> <name>wikisearch-ingest</name> @@ -48,6 +48,12 @@ <dependency> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-core</artifactId> + <exclusions> + <exclusion> + <groupId>commons-digester</groupId> + <artifactId>commons-digester</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.lucene</groupId> @@ -101,53 +107,4 @@ </plugin> </plugins> </build> - <profiles> - <!-- profile for building against Hadoop 1.0.x - Activate by not specifying hadoop.profile --> - <profile> - <id>hadoop-1.0</id> - <activation> - <property> - <name>!hadoop.profile</name> - </property> - </activation> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>${version.hadoop}</version> - <exclusions> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> - </profile> - <!-- profile for building against Hadoop 2.0.x - Activate using: mvn -Dhadoop.profile=2.0 --> - <profile> - <id>hadoop-2.0</id> - <activation> - <property> - <name>hadoop.profile</name> - <value>2.0</value> - </property> - </activation> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <version>${version.hadoop}</version> - <exclusions> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-mapper-asl</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> - </profile> - </profiles> </project> http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java ---------------------------------------------------------------------- diff --git a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java index d4fa1c6..1a495ed 100644 --- a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java +++ b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java @@ -29,13 +29,16 @@ import java.util.regex.Pattern; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.IteratorSetting.Column; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.user.SummingCombiner; @@ -135,9 +138,9 @@ public class WikipediaIngester extends Configured implements Tool { conf.set("mapred.map.tasks.speculative.execution", "false"); String tablename = WikipediaConfiguration.getTableName(conf); - - String zookeepers = WikipediaConfiguration.getZookeepers(conf); - String instanceName = WikipediaConfiguration.getInstanceName(conf); + ClientConfiguration clientConfig = new ClientConfiguration(); + clientConfig.setProperty(ClientProperty.INSTANCE_NAME, WikipediaConfiguration.getInstanceName(conf)); + clientConfig.setProperty(ClientProperty.INSTANCE_ZK_HOST, WikipediaConfiguration.getZookeepers(conf)); String user = WikipediaConfiguration.getUser(conf); byte[] password = WikipediaConfiguration.getPassword(conf); @@ -168,8 +171,8 @@ public class WikipediaIngester extends Configured implements Tool { job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Mutation.class); job.setOutputFormatClass(AccumuloOutputFormat.class); - AccumuloOutputFormat.setOutputInfo(job.getConfiguration(), user, password, true, tablename); - AccumuloOutputFormat.setZooKeeperInstance(job.getConfiguration(), instanceName, zookeepers); + AccumuloOutputFormat.setConnectorInfo(job, user, new PasswordToken(password)); + AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig); return job.waitForCompletion(true) ? 0 : 1; } http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java ---------------------------------------------------------------------- diff --git a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java index 59035dc..841f169 100644 --- a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java +++ b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaPartitionedIngester.java @@ -29,13 +29,16 @@ import java.util.regex.Pattern; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.IteratorSetting.Column; import org.apache.accumulo.core.client.TableExistsException; import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.user.SummingCombiner; @@ -236,12 +239,13 @@ public class WikipediaPartitionedIngester extends Configured implements Tool { SortingRFileOutputFormat.setPathName(ingestConf, WikipediaConfiguration.bulkIngestDir(ingestConf)); } else { ingestJob.setOutputFormatClass(AccumuloOutputFormat.class); - String zookeepers = WikipediaConfiguration.getZookeepers(ingestConf); - String instanceName = WikipediaConfiguration.getInstanceName(ingestConf); + ClientConfiguration clientConfig = new ClientConfiguration(); + clientConfig.setProperty(ClientProperty.INSTANCE_NAME, WikipediaConfiguration.getInstanceName(ingestConf)); + clientConfig.setProperty(ClientProperty.INSTANCE_ZK_HOST, WikipediaConfiguration.getZookeepers(ingestConf)); String user = WikipediaConfiguration.getUser(ingestConf); byte[] password = WikipediaConfiguration.getPassword(ingestConf); - AccumuloOutputFormat.setOutputInfo(ingestJob.getConfiguration(), user, password, true, tablename); - AccumuloOutputFormat.setZooKeeperInstance(ingestJob.getConfiguration(), instanceName, zookeepers); + AccumuloOutputFormat.setConnectorInfo(ingestJob, user, new PasswordToken(password)); + AccumuloOutputFormat.setZooKeeperInstance(ingestJob, clientConfig); } return ingestJob.waitForCompletion(true) ? 0 : 1; http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java ---------------------------------------------------------------------- diff --git a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java index 9b663de..aa6e357 100644 --- a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java +++ b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/BufferingRFileRecordWriter.java @@ -22,13 +22,18 @@ import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.data.ColumnUpdate; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.file.FileSKVWriter; -import org.apache.accumulo.core.file.rfile.RFileOperations; +import org.apache.accumulo.examples.wikisearch.ingest.WikipediaConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.io.Text; @@ -37,12 +42,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; final class BufferingRFileRecordWriter extends RecordWriter<Text,Mutation> { private final long maxSize; - private final AccumuloConfiguration acuconf; private final Configuration conf; - private final String filenamePrefix; - private final String taskID; - private final FileSystem fs; - private int fileCount = 0; private long size; private Map<Text,TreeMap<Key,Value>> buffers = new HashMap<Text,TreeMap<Key,Value>>(); @@ -79,31 +79,32 @@ final class BufferingRFileRecordWriter extends RecordWriter<Text,Mutation> { if (buffer.size() == 0) return; - String file = filenamePrefix + "/" + tablename + "/" + taskID + "_" + (fileCount++) + ".rf"; + Connector conn; + try { + conn = WikipediaConfiguration.getConnector(conf); + BatchWriterConfig bwconfig = new BatchWriterConfig(); + BatchWriter writer = conn.createBatchWriter(tablename.toString(), bwconfig); + for (Entry<Key,Value> e : buffer.entrySet()) { + Key k = e.getKey(); + Mutation m = new Mutation(); + m.put(k.getColumnFamily(), k.getColumnQualifier(), e.getValue()); + writer.addMutation(m); + } + writer.close(); + } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } // TODO get the table configuration for the given table? - FileSKVWriter writer = RFileOperations.getInstance().openWriter(file, fs, conf, acuconf); - - // forget locality groups for now, just write everything to the default - writer.startDefaultLocalityGroup(); - - for (Entry<Key,Value> e : buffer.entrySet()) { - writer.append(e.getKey(), e.getValue()); - } - - writer.close(); size -= bufferSize; buffer.clear(); bufferSizes.put(tablename, 0l); } - BufferingRFileRecordWriter(long maxSize, AccumuloConfiguration acuconf, Configuration conf, String filenamePrefix, String taskID, FileSystem fs) { + BufferingRFileRecordWriter(long maxSize, Configuration conf) { this.maxSize = maxSize; - this.acuconf = acuconf; this.conf = conf; - this.filenamePrefix = filenamePrefix; - this.taskID = taskID; - this.fs = fs; } @Override http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java ---------------------------------------------------------------------- diff --git a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java index 1fa8fdc..c54db06 100644 --- a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java +++ b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/output/SortingRFileOutputFormat.java @@ -105,17 +105,10 @@ public class SortingRFileOutputFormat extends OutputFormat<Text,Mutation> { // grab the configuration final Configuration conf = attempt.getConfiguration(); - // create a filename - final String filenamePrefix = getPathName(conf); - final String taskID = attempt.getTaskAttemptID().toString(); // grab the max size final long maxSize = getMaxBufferSize(conf); - // grab the FileSystem - final FileSystem fs = FileSystem.get(conf); - // create a default AccumuloConfiguration - final AccumuloConfiguration acuconf = AccumuloConfiguration.getDefaultConfiguration(); - return new BufferingRFileRecordWriter(maxSize, acuconf, conf, filenamePrefix, taskID, fs); + return new BufferingRFileRecordWriter(maxSize, conf); } } http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2e3cb3f..87fd3b2 100644 --- a/pom.xml +++ b/pom.xml @@ -21,12 +21,12 @@ <parent> <groupId>org.apache</groupId> <artifactId>apache</artifactId> - <version>14</version> + <version>18</version> </parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-wikisearch</artifactId> - <version>1.5.0</version> + <version>1.8.0</version> <packaging>pom</packaging> <name>accumulo-wikisearch</name> @@ -44,7 +44,7 @@ <version.commons-lang>2.4</version.commons-lang> <version.ejb-spec-api>1.0.1.Final</version.ejb-spec-api> <version.guava>11.0.2</version.guava> - <version.hadoop>1.0.4</version.hadoop> + <version.hadoop>2.6.4</version.hadoop> <version.jaxrs>2.1.0.GA</version.jaxrs> <version.junit>4.11</version.junit> <version.kryo>1.04</version.kryo> @@ -55,7 +55,7 @@ <version.lucene-wikipedia>3.0.2</version.lucene-wikipedia> <version.minlog>1.2</version.minlog> <version.protobuf>2.3.0</version.protobuf> - <version.thrift>0.9.0</version.thrift> + <version.thrift>0.9.1</version.thrift> <version.zookeeper>3.3.1</version.zookeeper> </properties> <dependencyManagement> @@ -76,6 +76,16 @@ <version>${version.collections}</version> </dependency> <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>1.1.1</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-math3</artifactId> + <version>3.6.1</version> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>${version.junit}</version> @@ -206,6 +216,21 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>1.7.21</version> + </dependency> + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-core-asl</artifactId> + <version>1.9.13</version> + </dependency> + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + <version>1.9.13</version> + </dependency> </dependencies> </dependencyManagement> <repositories> @@ -351,73 +376,4 @@ </plugin> </plugins> </build> - <profiles> - <profile> - <!-- profile for building against Hadoop 1.0.x - Activate by not specifying hadoop.profile --> - <id>hadoop-1.0</id> - <activation> - <property> - <name>!hadoop.profile</name> - </property> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <version>1.8</version> - <executions> - <execution> - <id>add-test-source</id> - <goals> - <goal>add-test-source</goal> - </goals> - <phase>generate-test-sources</phase> - <configuration> - <sources> - <source>src/test/hadoop1</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - <profile> - <!-- profile for building against Hadoop 2.0.x - Activate using: mvn -Dhadoop.profile=2.0 --> - <id>hadoop-2.0</id> - <activation> - <property> - <name>hadoop.profile</name> - <value>2.0</value> - </property> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <version>1.8</version> - <executions> - <execution> - <id>add-test-source</id> - <goals> - <goal>add-test-source</goal> - </goals> - <phase>generate-test-sources</phase> - <configuration> - <sources> - <source>src/test/hadoop2</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> </project> http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/query-war/pom.xml ---------------------------------------------------------------------- diff --git a/query-war/pom.xml b/query-war/pom.xml index 3ee0bc2..45a1f1b 100644 --- a/query-war/pom.xml +++ b/query-war/pom.xml @@ -20,7 +20,7 @@ <parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-wikisearch</artifactId> - <version>1.5.0</version> + <version>1.8.0</version> </parent> <artifactId>wikisearch-query-war</artifactId> http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/query/pom.xml ---------------------------------------------------------------------- diff --git a/query/pom.xml b/query/pom.xml index e779945..eeb2337 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -20,7 +20,7 @@ <parent> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-wikisearch</artifactId> - <version>1.5.0</version> + <version>1.8.0</version> </parent> <artifactId>wikisearch-query</artifactId> <packaging>ejb</packaging> @@ -51,6 +51,11 @@ <artifactId>commons-collections</artifactId> </dependency> <dependency> + <groupId>commons-configuration</groupId> + <artifactId>commons-configuration</artifactId> + <version>1.6</version> + </dependency> + <dependency> <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> </dependency> @@ -83,21 +88,9 @@ <scope>runtime</scope> </dependency> <dependency> - <groupId>commons-configuration</groupId> - <artifactId>commons-configuration</artifactId> - <version>1.6</version> - <exclusions> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - </exclusions> - <scope>runtime</scope> - </dependency> - <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> - <version>2.1</version> + <version>2.4</version> <scope>runtime</scope> </dependency> <dependency> @@ -142,7 +135,7 @@ <outputDirectory>lib</outputDirectory> <!-- just grab the non-provided runtime dependencies --> <!-- XXX we include guava at the same version as hadoop 2 provides so that we have it on hadoop 1 --> - <includeArtifactIds>commons-io,commons-configuration,commons-lang,commons-codec,protobuf-java,libthrift,zookeeper,hadoop-core,commons-jexl,guava,kryo,asm,minlog,reflectasm,wikisearch-ingest,accumulo-core,accumulo-fate,accumulo-trace</includeArtifactIds> + <includeArtifactIds>commons-io,commons-configuration,commons-lang,commons-codec,protobuf-java,libthrift,zookeeper,hadoop-client,commons-jexl,guava,kryo,asm,minlog,reflectasm,wikisearch-ingest,accumulo-core,accumulo-fate,accumulo-trace</includeArtifactIds> <excludeTransitive>true</excludeTransitive> </configuration> </execution> @@ -178,76 +171,4 @@ </plugin> </plugins> </build> - <profiles> - <!-- profile for building against Hadoop 1.0.x - Activate by not specifying hadoop.profile --> - <profile> - <id>hadoop-1.0</id> - <activation> - <property> - <name>!hadoop.profile</name> - </property> - </activation> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>${version.hadoop}</version> - <exclusions> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> - </profile> - <!-- profile for building against Hadoop 2.0.x - Activate using: mvn -Dhadoop.profile=2.0 --> - <profile> - <id>hadoop-2.0</id> - <activation> - <property> - <name>hadoop.profile</name> - <value>2.0</value> - </property> - </activation> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <version>${version.hadoop}</version> - <exclusions> - <exclusion> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-mapper-asl</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>copy-dependencies</id> - <goals> - <goal>copy-dependencies</goal> - </goals> - <phase>prepare-package</phase> - <configuration> - <outputDirectory>lib</outputDirectory> - <!-- just grab the non-provided runtime dependencies --> - <includeArtifactIds>commons-io,commons-configuration,commons-lang,commons-codec,protobuf-java,libthrift,zookeeper,hadoop-client,hadoop-common,hadoop-hdfs,commons-jexl,guava,kryo,asm,minlog,reflectasm,wikisearch-ingest,accumulo-core,accumulo-fate,accumulo-trace</includeArtifactIds> - <excludeTransitive>false</excludeTransitive> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> </project> http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java b/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java index 6b58d08..f05304a 100644 --- a/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java +++ b/query/src/main/java/org/apache/accumulo/examples/wikisearch/iterator/DefaultIteratorEnvironment.java @@ -18,6 +18,7 @@ package org.apache.accumulo.examples.wikisearch.iterator; import java.io.IOException; +import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; @@ -25,6 +26,7 @@ import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.system.MapFileIterator; +import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -70,5 +72,25 @@ public class DefaultIteratorEnvironment implements IteratorEnvironment { public void registerSideChannel(SortedKeyValueIterator<Key,Value> iter) { throw new UnsupportedOperationException(); } + +@Override +public Authorizations getAuthorizations() { + throw new UnsupportedOperationException(); +} + +@Override +public IteratorEnvironment cloneWithSamplingEnabled() { + throw new UnsupportedOperationException(); +} + +@Override +public boolean isSamplingEnabled() { + throw new UnsupportedOperationException(); +} + +@Override +public SamplerConfiguration getSamplerConfiguration() { + throw new UnsupportedOperationException(); +} } http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/dd23f107/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java b/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java index 024a865..62aa4e2 100644 --- a/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java +++ b/query/src/main/java/org/apache/accumulo/examples/wikisearch/query/Query.java @@ -43,6 +43,7 @@ import javax.xml.transform.stream.StreamSource; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.ZooKeeperInstance; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.examples.wikisearch.logic.ContentLogic; import org.apache.accumulo.examples.wikisearch.logic.QueryLogic; import org.apache.accumulo.examples.wikisearch.sample.Results; @@ -179,7 +180,7 @@ public class Query implements IQuery { Instance instance = new ZooKeeperInstance(this.instanceName, this.zooKeepers); try { log.info("Connecting to [instanceName = " + this.instanceName + ", zookeepers = " + this.zooKeepers + ", username = " + this.username + "]."); - connector = instance.getConnector(this.username, this.password.getBytes()); + connector = instance.getConnector(this.username, new PasswordToken(this.password.getBytes())); } catch (Exception e) { throw new EJBException("Error getting connector from instance", e); } @@ -212,7 +213,7 @@ public class Query implements IQuery { Instance instance = new ZooKeeperInstance(this.instanceName, this.zooKeepers); try { log.info("Connecting to [instanceName = " + this.instanceName + ", zookeepers = " + this.zooKeepers + ", username = " + this.username + "]."); - connector = instance.getConnector(this.username, this.password.getBytes()); + connector = instance.getConnector(this.username, new PasswordToken(this.password.getBytes())); } catch (Exception e) { throw new EJBException("Error getting connector from instance", e); }
