Updated Branches: refs/heads/1.6.0-SNAPSHOT 55e864ce9 -> 4e331fabd
ACCUMULO-2283 removing server-extras Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4e331fab Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4e331fab Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4e331fab Branch: refs/heads/1.6.0-SNAPSHOT Commit: 4e331fabdb7a26bbfd6df7a2aa2bb62e09d52d82 Parents: 55e864c Author: John Vines <vi...@apache.org> Authored: Tue Feb 4 14:33:05 2014 -0500 Committer: John Vines <vi...@apache.org> Committed: Tue Feb 4 14:34:00 2014 -0500 ---------------------------------------------------------------------- assemble/pom.xml | 4 - pom.xml | 13 -- server/extras/pom.xml | 106 ---------- .../apache/accumulo/utils/NamespaceRename.java | 211 ------------------- .../accumulo/utils/metanalysis/FilterMeta.java | 93 -------- .../accumulo/utils/metanalysis/FindTablet.java | 66 ------ .../accumulo/utils/metanalysis/IndexMeta.java | 183 ---------------- .../utils/metanalysis/LogFileInputFormat.java | 116 ---------- .../utils/metanalysis/LogFileOutputFormat.java | 66 ------ .../accumulo/utils/metanalysis/PrintEvents.java | 99 --------- .../utils/metanalysis/package-info.java | 34 --- 11 files changed, 991 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/assemble/pom.xml ---------------------------------------------------------------------- diff --git a/assemble/pom.xml b/assemble/pom.xml index b9d0b3e..9a45198 100644 --- a/assemble/pom.xml +++ b/assemble/pom.xml @@ -90,10 +90,6 @@ </dependency> <dependency> <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-server-extras</artifactId> - </dependency> - <dependency> - <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-start</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index dd13d50..c5e8673 100644 --- a/pom.xml +++ b/pom.xml @@ -90,7 +90,6 @@ <module>server/native</module> <module>server/tracer</module> <module>server/tserver</module> - <module>server/extras</module> </modules> <scm> <connection>scm:git:git://git.apache.org/accumulo.git</connection> @@ -348,18 +347,6 @@ </dependency> <dependency> <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-server-extras</artifactId> - <version>${project.version}</version> - <classifier>el6</classifier> - <type>rpm</type> - </dependency> - <dependency> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-server-extras</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-start</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/pom.xml ---------------------------------------------------------------------- diff --git a/server/extras/pom.xml b/server/extras/pom.xml deleted file mode 100644 index 8654c9f..0000000 --- a/server/extras/pom.xml +++ /dev/null @@ -1,106 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-project</artifactId> - <version>1.6.0-SNAPSHOT</version> - <relativePath>../../pom.xml</relativePath> - </parent> - <artifactId>accumulo-server-extras</artifactId> - <name>Extra Server Utilities</name> - <description>A library of additional tools to work with Apache Accumulo.</description> - <dependencies> - <dependency> - <groupId>com.beust</groupId> - <artifactId>jcommander</artifactId> - </dependency> - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </dependency> - <dependency> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-core</artifactId> - </dependency> - <dependency> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-fate</artifactId> - </dependency> - <dependency> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-server-base</artifactId> - </dependency> - <dependency> - <groupId>org.apache.accumulo</groupId> - <artifactId>accumulo-tserver</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <profiles> - <profile> - <id>rpm</id> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>rpm-maven-plugin</artifactId> - <executions> - <execution> - <id>build-rpm</id> - <goals> - <goal>attached-rpm</goal> - </goals> - <phase>package</phase> - <configuration> - <requires> - <require>jre >= 1.6.0</require> - <require>accumulo-tserver = %{version}-%{release}</require> - </requires> - <mappings> - <mapping> - <directory>%{_javadir}/accumulo</directory> - <artifact /> - </mapping> - </mappings> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> -</project> http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java ---------------------------------------------------------------------- diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java b/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java deleted file mode 100644 index bc0c834..0000000 --- a/server/extras/src/main/java/org/apache/accumulo/utils/NamespaceRename.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.utils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; -import org.apache.accumulo.core.client.MutationsRejectedException; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.master.thrift.MasterGoalState; -import org.apache.accumulo.core.metadata.MetadataTable; -import org.apache.accumulo.core.metadata.RootTable; -import org.apache.accumulo.core.metadata.schema.MetadataSchema; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.tabletserver.log.LogEntry; -import org.apache.accumulo.core.util.CachedConfiguration; -import org.apache.accumulo.core.util.ColumnFQ; -import org.apache.accumulo.core.zookeeper.ZooUtil; -import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy; -import org.apache.accumulo.server.Accumulo; -import org.apache.accumulo.server.client.HdfsZooInstance; -import org.apache.accumulo.server.conf.ServerConfiguration; -import org.apache.accumulo.server.fs.VolumeManager; -import org.apache.accumulo.server.fs.VolumeManagerImpl; -import org.apache.accumulo.server.security.SystemCredentials; -import org.apache.accumulo.server.zookeeper.ZooReaderWriter; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.Text; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; - -public class NamespaceRename { - - static final Logger log = LoggerFactory.getLogger(NamespaceRename.class); - - static class Opts { - @Parameter(names = {"--old", "-o"}, required = true) - String oldName = null; - @Parameter(names = {"--new", "-n"}, required = true) - String newName = null; - } - - /** - * Utility to recovery from a name node restoration at a new location. For example, if you had been using "nn1" and the machine died but you were able to - * restore the service on a different machine, "nn2" you could rewrite the metadata using - * <pre> - * accumulo org.apache.accumulo.server.util.NamespaceRename --old hdfs://nn1:9001 --new hdfs://nn2:9001 - * </pre> - * @param args - * @throws Exception - */ - public static void main(String[] args) throws Exception { - Opts opts = new Opts(); - JCommander cmdline = new JCommander(opts); - cmdline.parse(args); - log.info("Checking current configuration"); - AccumuloConfiguration configuration = ServerConfiguration.getSiteConfiguration(); - checkConfiguration(opts, configuration); - Instance instance = HdfsZooInstance.getInstance(); - log.info("Waiting for HDFS and Zookeeper to be ready"); - VolumeManager fs = VolumeManagerImpl.get(); - Accumulo.waitForZookeeperAndHdfs(fs); - log.info("Putting servers in SAFE_MODE"); - ZooReaderWriter zoo = ZooReaderWriter.getInstance(); - zoo.putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, MasterGoalState.SAFE_MODE.toString().getBytes(), NodeExistsPolicy.OVERWRITE); - log.info("Updating root table write-ahead logs"); - updateZookeeper(opts, instance, zoo); - log.info("Updating file references in the root table"); - updateMetaTable(opts, instance, RootTable.NAME); - log.info("Updating file references in the metadata table"); - updateMetaTable(opts, instance, MetadataTable.NAME); - zoo.putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZMASTER_GOAL_STATE, MasterGoalState.NORMAL.toString().getBytes(), NodeExistsPolicy.OVERWRITE); - log.info("Namespace " + opts.oldName + " has been renamed " + opts.newName); - } - - static final ColumnFQ DIRECTORY_COLUMN = MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN; - - private static void updateMetaTable(Opts opts, Instance instance, String tableName) throws Exception, - MutationsRejectedException { - log.info("Waiting for " + tableName + " to come online"); - Connector conn = getConnector(instance); - Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY); - scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LogColumnFamily.NAME); - scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME); - DIRECTORY_COLUMN.fetch(scanner); - scanner.iterator().hasNext(); - log.info(tableName + " is online"); - BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig()); - for (Entry<Key,Value> entry : scanner) { - Key key = entry.getKey(); - Mutation m = new Mutation(key.getRow()); - if (DIRECTORY_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier())) { - m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), key.getTimestamp()); - String newName = rename(entry.getValue().toString(), opts); - m.put(key.getColumnFamily(), key.getColumnQualifier(), new Value(newName.getBytes())); - bw.addMutation(m); - } else if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME)) { - m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), key.getTimestamp()); - String newName = rename(key.getColumnQualifier().toString(), opts); - m.put(key.getColumnFamily(), new Text(newName), entry.getValue()); - bw.addMutation(m); - } else if (key.getColumnFamily().equals(MetadataSchema.TabletsSection.LogColumnFamily.NAME)) { - m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), key.getTimestamp()); - LogEntry update = convert(LogEntry.fromKeyValue(entry.getKey(), entry.getValue()), opts); - m.put(update.getColumnFamily(), update.getColumnQualifier(), update.getValue()); - bw.addMutation(m); - } - } - bw.close(); - } - - static private LogEntry convert(LogEntry entry, Opts opts) { - entry.filename = rename(entry.filename, opts); - List<String> logSet = new ArrayList<String>(); - for (String log : entry.logSet) { - logSet.add(rename(log, opts)); - } - entry.logSet = logSet; - return entry; - } - - private static Connector getConnector(Instance instance) throws AccumuloException, AccumuloSecurityException { - return instance.getConnector(SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken()); - } - - private static void updateZookeeper(Opts opts, Instance instance, ZooReaderWriter zoo) throws KeeperException, InterruptedException, - IOException { - String root = ZooUtil.getRoot(instance); - String rootTabletLocation = root + RootTable.ZROOT_TABLET_WALOGS; - for (String walogName : zoo.getChildren(rootTabletLocation)) { - LogEntry entry = new LogEntry(); - String logZPath = rootTabletLocation + "/" + walogName; - byte[] data = zoo.getData(logZPath, null); - entry.fromBytes(data); - entry = convert(entry, opts); - zoo.putPersistentData(logZPath, entry.toBytes(), NodeExistsPolicy.OVERWRITE); - } - String dirPath = root + RootTable.ZROOT_TABLET_PATH; - byte[] dir = zoo.getData(dirPath, null); - String newDir = rename(new String(dir), opts); - zoo.putPersistentData(dirPath, newDir.getBytes(), NodeExistsPolicy.OVERWRITE); - } - - private static String rename(String filename, Opts opts) { - if (filename.startsWith(opts.oldName)) - return opts.newName + filename.substring(opts.oldName.length(), filename.length()); - return filename; - } - - private static void checkConfiguration(Opts opts, AccumuloConfiguration configuration) throws IOException { - if (opts.oldName.endsWith("/")) - throw new RuntimeException(opts.oldName + " ends with a slash, do not include it"); - if (opts.newName.endsWith("/")) - throw new RuntimeException(opts.newName + " ends with a slash, do not include it"); - String volumes = configuration.get(Property.INSTANCE_VOLUMES); - if (volumes != null && !volumes.isEmpty()) { - Set<String> volumeSet = new HashSet<String>(Arrays.asList(volumes.split(","))); - if (volumeSet.contains(opts.oldName)) - throw new RuntimeException(Property.INSTANCE_VOLUMES.getKey() + " is set to " + volumes + " which still contains the old name " + opts.oldName); - if (!volumeSet.contains(opts.newName)) - throw new RuntimeException(Property.INSTANCE_VOLUMES.getKey() + " is set to " + volumes + " which does not contain the new name " + opts.oldName); - return; - } else { - String uri = configuration.get(Property.INSTANCE_DFS_URI); - if (uri != null && !uri.isEmpty()) { - if (!uri.startsWith(opts.newName)) - throw new RuntimeException(Property.INSTANCE_DFS_DIR.getKey() + " is set to " + uri + " which is not in " + opts.newName); - return; - } - } - FileSystem fs = FileSystem.get(CachedConfiguration.getInstance()); - if (!fs.getUri().toString().equals(opts.newName)) - throw new RuntimeException("Default filesystem is " + fs.getUri() + " and the new name is " + opts.newName + ". Update your hadoop dfs configuration."); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FilterMeta.java ---------------------------------------------------------------------- diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FilterMeta.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FilterMeta.java deleted file mode 100644 index ac56c99..0000000 --- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FilterMeta.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.utils.metanalysis; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Set; - -import org.apache.accumulo.core.metadata.MetadataTable; -import org.apache.accumulo.core.util.CachedConfiguration; -import org.apache.accumulo.tserver.logger.LogEvents; -import org.apache.accumulo.tserver.logger.LogFileKey; -import org.apache.accumulo.tserver.logger.LogFileValue; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; - -/** - * A map reduce job that takes a set of walogs and filters out all non metadata table events. - */ -public class FilterMeta extends Configured implements Tool { - - public static class FilterMapper extends Mapper<LogFileKey,LogFileValue,LogFileKey,LogFileValue> { - private Set<Integer> tabletIds; - - @Override - protected void setup(Context context) throws java.io.IOException, java.lang.InterruptedException { - tabletIds = new HashSet<Integer>(); - } - - @Override - public void map(LogFileKey key, LogFileValue value, Context context) throws IOException, InterruptedException { - if (key.event == LogEvents.OPEN) { - context.write(key, value); - } else if (key.event == LogEvents.DEFINE_TABLET && key.tablet.getTableId().toString().equals(MetadataTable.ID)) { - tabletIds.add(key.tid); - context.write(key, value); - } else if ((key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) && tabletIds.contains(key.tid)) { - context.write(key, value); - } - } - } - - @Override - public int run(String[] args) throws Exception { - - String jobName = this.getClass().getSimpleName() + "_" + System.currentTimeMillis(); - - @SuppressWarnings("deprecation") - Job job = new Job(getConf(), jobName); - job.setJarByClass(this.getClass()); - - Path paths[] = new Path[args.length - 1]; - for (int i = 0; i < paths.length; i++) { - paths[i] = new Path(args[i]); - } - - job.setInputFormatClass(LogFileInputFormat.class); - LogFileInputFormat.setInputPaths(job, paths); - - job.setOutputFormatClass(LogFileOutputFormat.class); - LogFileOutputFormat.setOutputPath(job, new Path(args[args.length - 1])); - - job.setMapperClass(FilterMapper.class); - - job.setNumReduceTasks(0); - - job.waitForCompletion(true); - return job.isSuccessful() ? 0 : 1; - } - - public static void main(String[] args) throws Exception { - int res = ToolRunner.run(CachedConfiguration.getInstance(), new FilterMeta(), args); - System.exit(res); - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FindTablet.java ---------------------------------------------------------------------- diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FindTablet.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FindTablet.java deleted file mode 100644 index 773c41a..0000000 --- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/FindTablet.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.utils.metanalysis; - -import java.util.Map.Entry; - -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.KeyExtent; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.util.TextUtil; -import org.apache.accumulo.server.cli.ClientOpts; -import org.apache.hadoop.io.Text; - -import com.beust.jcommander.Parameter; - -/** - * Finds tablet creation events. - */ -public class FindTablet { - - static public class Opts extends ClientOpts { - @Parameter(names = {"-r", "--row"}, required = true, description = "find tablets that contain this row") - String row = null; - - @Parameter(names = "--tableId", required = true, description = "table id") - String tableId = null; - } - - public static void main(String[] args) throws Exception { - Opts opts = new Opts(); - opts.parseArgs(FindTablet.class.getName(), args); - - findContainingTablets(opts); - } - - private static void findContainingTablets(Opts opts) throws Exception { - Range range = new KeyExtent(new Text(opts.tableId), null, null).toMetadataRange(); - - Scanner scanner = opts.getConnector().createScanner("createEvents", opts.auths); - scanner.setRange(range); - - Text row = new Text(opts.row); - for (Entry<Key,Value> entry : scanner) { - KeyExtent ke = new KeyExtent(entry.getKey().getRow(), new Value(TextUtil.getBytes(entry.getKey().getColumnFamily()))); - if (ke.contains(row)) { - System.out.println(entry.getKey().getColumnQualifier() + " " + ke + " " + entry.getValue()); - } - } - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/IndexMeta.java ---------------------------------------------------------------------- diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/IndexMeta.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/IndexMeta.java deleted file mode 100644 index 448b871..0000000 --- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/IndexMeta.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.utils.metanalysis; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.client.ClientConfiguration; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.TableExistsException; -import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat; -import org.apache.accumulo.core.data.ColumnUpdate; -import org.apache.accumulo.core.data.KeyExtent; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.metadata.MetadataTable; -import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; -import org.apache.accumulo.core.util.CachedConfiguration; -import org.apache.accumulo.server.cli.ClientOpts; -import org.apache.accumulo.tserver.logger.LogEvents; -import org.apache.accumulo.tserver.logger.LogFileKey; -import org.apache.accumulo.tserver.logger.LogFileValue; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableUtils; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; -import org.apache.log4j.Logger; - -import com.beust.jcommander.Parameter; - -/** - * A map reduce job that takes write ahead logs containing mutations for the metadata table and indexes them into Accumulo tables for analysis. - * - */ - -public class IndexMeta extends Configured implements Tool { - - public static class IndexMapper extends Mapper<LogFileKey,LogFileValue,Text,Mutation> { - private static final Text CREATE_EVENTS_TABLE = new Text("createEvents"); - private static final Text TABLET_EVENTS_TABLE = new Text("tabletEvents"); - private Map<Integer,KeyExtent> tabletIds = new HashMap<Integer,KeyExtent>(); - private String uuid = null; - - @Override - protected void setup(Context context) throws java.io.IOException, java.lang.InterruptedException { - tabletIds = new HashMap<Integer,KeyExtent>(); - uuid = null; - } - - @Override - public void map(LogFileKey key, LogFileValue value, Context context) throws IOException, InterruptedException { - if (key.event == LogEvents.OPEN) { - uuid = key.tserverSession; - } else if (key.event == LogEvents.DEFINE_TABLET) { - if (key.tablet.getTableId().toString().equals(MetadataTable.ID)) { - tabletIds.put(key.tid, new KeyExtent(key.tablet)); - } - } else if ((key.event == LogEvents.MUTATION || key.event == LogEvents.MANY_MUTATIONS) && tabletIds.containsKey(key.tid)) { - for (Mutation m : value.mutations) { - index(context, m, uuid, tabletIds.get(key.tid)); - } - } - } - - void index(Context context, Mutation m, String logFile, KeyExtent metaTablet) throws IOException, InterruptedException { - List<ColumnUpdate> columnsUpdates = m.getUpdates(); - - Text prevRow = null; - long timestamp = 0; - - if (m.getRow().length > 0 && m.getRow()[0] == '~') { - return; - } - - for (ColumnUpdate cu : columnsUpdates) { - if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(new Text(cu.getColumnFamily()), new Text(cu.getColumnQualifier())) && !cu.isDeleted()) { - prevRow = new Text(cu.getValue()); - } - - timestamp = cu.getTimestamp(); - } - - byte[] serMut = WritableUtils.toByteArray(m); - - if (prevRow != null) { - Mutation createEvent = new Mutation(new Text(m.getRow())); - createEvent.put(prevRow, new Text(String.format("%020d", timestamp)), new Value(metaTablet.toString().getBytes(Constants.UTF8))); - context.write(CREATE_EVENTS_TABLE, createEvent); - } - - Mutation tabletEvent = new Mutation(new Text(m.getRow())); - tabletEvent.put(new Text(String.format("%020d", timestamp)), new Text("mut"), new Value(serMut)); - tabletEvent.put(new Text(String.format("%020d", timestamp)), new Text("mtab"), new Value(metaTablet.toString().getBytes(Constants.UTF8))); - tabletEvent.put(new Text(String.format("%020d", timestamp)), new Text("log"), new Value(logFile.getBytes(Constants.UTF8))); - context.write(TABLET_EVENTS_TABLE, tabletEvent); - } - } - - static class Opts extends ClientOpts { - @Parameter(description = "<logfile> { <logfile> ...}") - List<String> logFiles = new ArrayList<String>(); - - public ClientConfiguration getConf() { - return this.getClientConfiguration(); - } - } - - @Override - public int run(String[] args) throws Exception { - Opts opts = new Opts(); - opts.parseArgs(IndexMeta.class.getName(), args); - - String jobName = this.getClass().getSimpleName() + "_" + System.currentTimeMillis(); - - @SuppressWarnings("deprecation") - Job job = new Job(getConf(), jobName); - job.setJarByClass(this.getClass()); - - List<String> logFiles = Arrays.asList(args).subList(4, args.length); - Path paths[] = new Path[logFiles.size()]; - int count = 0; - for (String logFile : logFiles) { - paths[count++] = new Path(logFile); - } - - job.setInputFormatClass(LogFileInputFormat.class); - LogFileInputFormat.setInputPaths(job, paths); - - job.setNumReduceTasks(0); - - job.setOutputFormatClass(AccumuloOutputFormat.class); - AccumuloOutputFormat.setZooKeeperInstance(job, opts.getConf()); - AccumuloOutputFormat.setConnectorInfo(job, opts.principal, opts.getToken()); - AccumuloOutputFormat.setCreateTables(job, false); - - job.setMapperClass(IndexMapper.class); - - Connector conn = opts.getConnector(); - - try { - conn.tableOperations().create("createEvents"); - } catch (TableExistsException tee) { - Logger.getLogger(IndexMeta.class).warn("Table createEvents exists"); - } - - try { - conn.tableOperations().create("tabletEvents"); - } catch (TableExistsException tee) { - Logger.getLogger(IndexMeta.class).warn("Table tabletEvents exists"); - } - - job.waitForCompletion(true); - return job.isSuccessful() ? 0 : 1; - } - - public static void main(String[] args) throws Exception { - int res = ToolRunner.run(CachedConfiguration.getInstance(), new IndexMeta(), args); - System.exit(res); - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileInputFormat.java ---------------------------------------------------------------------- diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileInputFormat.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileInputFormat.java deleted file mode 100644 index 603bf71..0000000 --- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileInputFormat.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.accumulo.utils.metanalysis; - -import java.io.EOFException; -import java.io.IOException; - -import org.apache.accumulo.tserver.logger.LogFileKey; -import org.apache.accumulo.tserver.logger.LogFileValue; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.input.FileSplit; - -/** - * Input format for Accumulo write ahead logs - */ -public class LogFileInputFormat extends FileInputFormat<LogFileKey,LogFileValue> { - - private static class LogFileRecordReader extends RecordReader<LogFileKey,LogFileValue> { - - private FSDataInputStream fsdis; - private LogFileKey key; - private LogFileValue value; - private long length; - - @Override - public void close() throws IOException { - fsdis.close(); - } - - @Override - public LogFileKey getCurrentKey() throws IOException, InterruptedException { - return key; - } - - @Override - public LogFileValue getCurrentValue() throws IOException, InterruptedException { - return value; - } - - @Override - public float getProgress() throws IOException, InterruptedException { - float progress = (length - fsdis.getPos()) / (float) length; - if (progress < 0) - return 0; - return progress; - } - - @Override - public void initialize(InputSplit is, TaskAttemptContext context) throws IOException, InterruptedException { - FileSplit fileSplit = (FileSplit) is; - - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.get(conf); - - key = new LogFileKey(); - value = new LogFileValue(); - - fsdis = fs.open(fileSplit.getPath()); - FileStatus status = fs.getFileStatus(fileSplit.getPath()); - length = status.getLen(); - } - - @Override - public boolean nextKeyValue() throws IOException, InterruptedException { - if (key == null) - return false; - - try { - key.readFields(fsdis); - value.readFields(fsdis); - return true; - } catch (EOFException ex) { - key = null; - value = null; - return false; - } - } - - } - - - @Override - public RecordReader<LogFileKey,LogFileValue> createRecordReader(InputSplit arg0, TaskAttemptContext arg1) throws IOException, InterruptedException { - return new LogFileRecordReader(); - } - - @Override - protected boolean isSplitable(JobContext context, Path filename) { - return false; - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileOutputFormat.java ---------------------------------------------------------------------- diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileOutputFormat.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileOutputFormat.java deleted file mode 100644 index cccb89e..0000000 --- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/LogFileOutputFormat.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.utils.metanalysis; - -import java.io.IOException; - -import org.apache.accumulo.tserver.logger.LogFileKey; -import org.apache.accumulo.tserver.logger.LogFileValue; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; - -/** - * Output format for Accumulo write ahead logs. - */ -public class LogFileOutputFormat extends FileOutputFormat<LogFileKey,LogFileValue> { - - private static class LogFileRecordWriter extends RecordWriter<LogFileKey,LogFileValue> { - - private FSDataOutputStream out; - - public LogFileRecordWriter(Path outputPath) throws IOException { - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.get(conf); - - out = fs.create(outputPath); - } - - @Override - public void close(TaskAttemptContext arg0) throws IOException, InterruptedException { - out.close(); - } - - @Override - public void write(LogFileKey key, LogFileValue val) throws IOException, InterruptedException { - key.write(out); - val.write(out); - } - - } - - @Override - public RecordWriter<LogFileKey,LogFileValue> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { - Path outputPath = getDefaultWorkFile(context, ""); - return new LogFileRecordWriter(outputPath); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/PrintEvents.java ---------------------------------------------------------------------- diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/PrintEvents.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/PrintEvents.java deleted file mode 100644 index fc6c18a..0000000 --- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/PrintEvents.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.utils.metanalysis; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.util.Collections; -import java.util.List; -import java.util.Map.Entry; - -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.data.ColumnUpdate; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.PartialKey; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.server.cli.ClientOpts; -import org.apache.accumulo.tserver.logger.LogFileValue; -import org.apache.hadoop.io.Text; - -import com.beust.jcommander.Parameter; - -/** - * Looks up and prints mutations indexed by IndexMeta - */ -public class PrintEvents { - - static class Opts extends ClientOpts { - @Parameter(names = {"-t", "--tableId"}, description = "table id", required = true) - String tableId; - @Parameter(names = {"-e", "--endRow"}, description = "end row") - String endRow; - @Parameter(names = {"-t", "--time"}, description = "time, in milliseconds", required = true) - long time; - } - - public static void main(String[] args) throws Exception { - Opts opts = new Opts(); - opts.parseArgs(PrintEvents.class.getName(), args); - - Connector conn = opts.getConnector(); - - printEvents(conn, opts.tableId, opts.endRow, opts.time); - } - - private static void printEvents(Connector conn, String tableId, String endRow, Long time) throws Exception { - Scanner scanner = conn.createScanner("tabletEvents", new Authorizations()); - String metaRow = tableId + (endRow == null ? "<" : ";" + endRow); - scanner.setRange(new Range(new Key(metaRow, String.format("%020d", time)), true, new Key(metaRow).followingKey(PartialKey.ROW), false)); - int count = 0; - - String lastLog = null; - - loop1: for (Entry<Key,Value> entry : scanner) { - if (entry.getKey().getColumnQualifier().toString().equals("log")) { - if (lastLog == null || !lastLog.equals(entry.getValue().toString())) - System.out.println("Log : " + entry.getValue()); - lastLog = entry.getValue().toString(); - } else if (entry.getKey().getColumnQualifier().toString().equals("mut")) { - DataInputStream dis = new DataInputStream(new ByteArrayInputStream(entry.getValue().get())); - Mutation m = new Mutation(); - m.readFields(dis); - - LogFileValue lfv = new LogFileValue(); - lfv.mutations = Collections.singletonList(m); - - System.out.println(LogFileValue.format(lfv, 1)); - - List<ColumnUpdate> columnsUpdates = m.getUpdates(); - for (ColumnUpdate cu : columnsUpdates) { - if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(new Text(cu.getColumnFamily()), new Text(cu.getColumnQualifier())) && count > 0) { - System.out.println("Saw change to prevrow, stopping printing events."); - break loop1; - } - } - count++; - } - } - - } -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/4e331fab/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/package-info.java ---------------------------------------------------------------------- diff --git a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/package-info.java b/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/package-info.java deleted file mode 100644 index 3c15884..0000000 --- a/server/extras/src/main/java/org/apache/accumulo/utils/metanalysis/package-info.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Provides programs to analyze metadata mutations written to write ahead logs. - * - * <p> - * These programs can be used when write ahead logs are archived. The best way to find - * which write ahead logs contain metadata mutations is to grep the tablet server logs. - * Grep for events where walogs were added to metadata tablets, then take the unique set - * of walogs. - * - * <p> - * To use these programs, use IndexMeta to index the metadata mutations in walogs into - * Accumulo tables. Then use FindTable and PrintEvents to analyze those indexes. - * FilterMetaiallows filtering walogs down to just metadata events. This is useful for the - * case where the walogs need to be exported from the cluster for analysis. - * - * @since 1.5 - */ -package org.apache.accumulo.utils.metanalysis;
