Author: billie Date: Tue Dec 11 00:01:39 2012 New Revision: 1419924 URL: http://svn.apache.org/viewvc?rev=1419924&view=rev Log: ACCUMULO-876 added maven profiles to assist in compiling against different versions of hadoop
Modified: accumulo/trunk/README accumulo/trunk/core/pom.xml accumulo/trunk/examples/instamo/pom.xml accumulo/trunk/examples/simple/pom.xml accumulo/trunk/fate/pom.xml accumulo/trunk/pom.xml accumulo/trunk/server/pom.xml accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java accumulo/trunk/start/pom.xml accumulo/trunk/test/pom.xml accumulo/trunk/trace/pom.xml Modified: accumulo/trunk/README URL: http://svn.apache.org/viewvc/accumulo/trunk/README?rev=1419924&r1=1419923&r2=1419924&view=diff ============================================================================== --- accumulo/trunk/README (original) +++ accumulo/trunk/README Tue Dec 11 00:01:39 2012 @@ -18,8 +18,14 @@ have maven configured to get Accumulo pr the pom.xml file for the necessary components. Activate the 'docs' profile to build the Accumulo developer and user manual. -Run "mvn package -P assemble" to build a distribution, or run "mvn package -P assemble,docs" -to also build the documentation. +Run "mvn package -P assemble" to build a distribution, or run +"mvn package -P assemble,docs" to also build the documentation. By default, +Accumulo compiles against Hadoop 1.0.4. To compile against a different version +that is compatible with Hadoop 1.0, specify hadoop.version on the command line, +e.g. "-Dhadoop.version=0.20.205.0" or "-Dhadoop.version=1.1.0". To compile +against Hadoop 2.0, specify "-Dhadoop.profile=2.0". By default this uses +2.0.2-alpha. To compile against a different 2.0-compatible version, specify +the profile and version, e.g. "-Dhadoop.profile=2.0 -Dhadoop.version=0.23.5". If you are running on another Unix-like operating system (OSX, etc) then you may wish to build the native libraries. They are not strictly necessary Modified: accumulo/trunk/core/pom.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/core/pom.xml?rev=1419924&r1=1419923&r2=1419924&view=diff ============================================================================== --- accumulo/trunk/core/pom.xml (original) +++ accumulo/trunk/core/pom.xml Tue Dec 11 00:01:39 2012 @@ -73,7 +73,43 @@ </plugin> </plugins> </build> - + + <profiles> + <!-- profile for building against Hadoop 1.0.x + Activate by not specifying hadoop.profile --> + <profile> + <id>hadoop-1.0</id> + <activation> + <property> + <name>!hadoop.profile</name> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </dependency> + </dependencies> + </profile> + <!-- profile for building against Hadoop 2.0.x + Activate using: mvn -Dhadoop.profile=2.0 --> + <profile> + <id>hadoop-2.0</id> + <activation> + <property> + <name>hadoop.profile</name> + <value>2.0</value> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + </dependency> + </dependencies> + </profile> + </profiles> + <dependencies> <dependency> <groupId>org.apache.accumulo</groupId> @@ -88,15 +124,11 @@ <artifactId>cloudtrace</artifactId> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - </dependency> - <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> + <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> </dependency> <dependency> Modified: accumulo/trunk/examples/instamo/pom.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/instamo/pom.xml?rev=1419924&r1=1419923&r2=1419924&view=diff ============================================================================== --- accumulo/trunk/examples/instamo/pom.xml (original) +++ accumulo/trunk/examples/instamo/pom.xml Tue Dec 11 00:01:39 2012 @@ -9,8 +9,53 @@ <url>http://maven.apache.org</url> <properties> <accumulo.version>1.5.0-SNAPSHOT</accumulo.version> + <hadoop-one.version>1.0.4</hadoop-one.version> + <hadoop-two.version>2.0.2-alpha</hadoop-two.version> <maclass>org.apache.accumulo.instamo.MapReduceExample</maclass> </properties> + <profiles> + <!-- profile for building against Hadoop 1.0.x + Activate by not specifying hadoop.profile --> + <profile> + <id>hadoop-1.0</id> + <activation> + <property> + <name>!hadoop.profile</name> + </property> + </activation> + <properties> + <hadoop.version>${hadoop-one.version}</hadoop.version> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + <version>${hadoop.version}</version> + </dependency> + </dependencies> + </profile> + <!-- profile for building against Hadoop 2.0.x + Activate using: mvn -Dhadoop.profile=2.0 --> + <profile> + <id>hadoop-2.0</id> + <activation> + <property> + <name>hadoop.profile</name> + <value>2.0</value> + </property> + </activation> + <properties> + <hadoop.version>${hadoop-two.version}</hadoop.version> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + </dependency> + </dependencies> + </profile> + </profiles> <dependencies> <dependency> <groupId>junit</groupId> @@ -34,11 +79,6 @@ <version>3.3.4</version> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>1.0.1</version> - </dependency> - <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> <version>2.4</version> Modified: accumulo/trunk/examples/simple/pom.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/examples/simple/pom.xml?rev=1419924&r1=1419923&r2=1419924&view=diff ============================================================================== --- accumulo/trunk/examples/simple/pom.xml (original) +++ accumulo/trunk/examples/simple/pom.xml Tue Dec 11 00:01:39 2012 @@ -26,17 +26,49 @@ <modelVersion>4.0.0</modelVersion> <artifactId>examples-simple</artifactId> <name>examples-simple</name> - + + <profiles> + <!-- profile for building against Hadoop 1.0.x + Activate by not specifying hadoop.profile --> + <profile> + <id>hadoop-1.0</id> + <activation> + <property> + <name>!hadoop.profile</name> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </dependency> + </dependencies> + </profile> + <!-- profile for building against Hadoop 2.0.x + Activate using: mvn -Dhadoop.profile=2.0 --> + <profile> + <id>hadoop-2.0</id> + <activation> + <property> + <name>hadoop.profile</name> + <value>2.0</value> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + </dependency> + </dependencies> + </profile> + </profiles> + <dependencies> <dependency> <groupId>org.apache.accumulo</groupId> <artifactId>accumulo-core</artifactId> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - </dependency> - <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> </dependency> Modified: accumulo/trunk/fate/pom.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/fate/pom.xml?rev=1419924&r1=1419923&r2=1419924&view=diff ============================================================================== --- accumulo/trunk/fate/pom.xml (original) +++ accumulo/trunk/fate/pom.xml Tue Dec 11 00:01:39 2012 @@ -51,7 +51,7 @@ </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> + <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> </dependency> </dependencies> Modified: accumulo/trunk/pom.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/pom.xml?rev=1419924&r1=1419923&r2=1419924&view=diff ============================================================================== --- accumulo/trunk/pom.xml (original) +++ accumulo/trunk/pom.xml Tue Dec 11 00:01:39 2012 @@ -498,6 +498,62 @@ </plugins> </reporting> </profile> + <!-- profile for building against Hadoop 1.0.x + Activate by not specifying hadoop.profile --> + <profile> + <id>hadoop-1.0</id> + <activation> + <property> + <name>!hadoop.profile</name> + </property> + </activation> + <properties> + <hadoop.version>1.0.4</hadoop.version> + <slf4j.version>1.4.3</slf4j.version> + </properties> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + </dependencyManagement> + </profile> + <!-- profile for building against Hadoop 2.0.x + Activate using: mvn -Dhadoop.profile=2.0 --> + <profile> + <id>hadoop-2.0</id> + <activation> + <property> + <name>hadoop.profile</name> + <value>2.0</value> + </property> + </activation> + <properties> + <slf4j.version>1.6.1</slf4j.version> + <hadoop.version>2.0.2-alpha</hadoop.version> + <avro.version>1.5.3</avro.version> + </properties> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + <version>${avro.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + </dependencyManagement> + </profile> </profiles> <dependencyManagement> @@ -512,15 +568,9 @@ <!-- provided dependencies needed at runtime --> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - <version>0.20.205.0</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> + <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> - <version>3.3.1</version> + <version>${zookeeper.version}</version> <scope>provided</scope> </dependency> <dependency> @@ -532,7 +582,7 @@ <dependency> <groupId>org.mortbay.jetty</groupId> <artifactId>jetty</artifactId> - <version>[5.1,7.0)</version> + <version>[6.1,7.0)</version> <scope>provided</scope> </dependency> @@ -645,12 +695,12 @@ <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> - <version>1.4.3</version> + <version>${slf4j.version}</version> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> - <version>1.4.3</version> + <version>${slf4j.version}</version> </dependency> <dependency> <groupId>org.apache.commons</groupId> @@ -695,6 +745,7 @@ <properties> <targetJdk>1.6</targetJdk> + <zookeeper.version>3.3.1</zookeeper.version> </properties> </project> Modified: accumulo/trunk/server/pom.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/server/pom.xml?rev=1419924&r1=1419923&r2=1419924&view=diff ============================================================================== --- accumulo/trunk/server/pom.xml (original) +++ accumulo/trunk/server/pom.xml Tue Dec 11 00:01:39 2012 @@ -71,11 +71,7 @@ <artifactId>libthrift</artifactId> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> + <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> </dependency> <dependency> @@ -114,6 +110,10 @@ <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> </dependency> + <dependency> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jetty</artifactId> + </dependency> </dependencies> <profiles> @@ -141,6 +141,43 @@ </plugins> </build> </profile> + <!-- profile for building against Hadoop 1.0.x + Activate by not specifying hadoop.profile --> + <profile> + <id>hadoop-1.0</id> + <activation> + <property> + <name>!hadoop.profile</name> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </dependency> + </dependencies> + </profile> + <!-- profile for building against Hadoop 2.0.x + Activate using: mvn -Dhadoop.profile=2.0 --> + <profile> + <id>hadoop-2.0</id> + <activation> + <property> + <name>hadoop.profile</name> + <value>2.0</value> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + </dependency> + </dependencies> + </profile> </profiles> </project> Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java?rev=1419924&r1=1419923&r2=1419924&view=diff ============================================================================== --- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java (original) +++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/monitor/servlets/DefaultServlet.java Tue Dec 11 00:01:39 2012 @@ -21,7 +21,6 @@ import java.io.FileInputStream; import java.io.FilePermission; import java.io.IOException; import java.io.InputStream; -import java.net.InetSocketAddress; import java.security.AccessControlContext; import java.security.AccessController; import java.security.PermissionCollection; @@ -57,9 +56,6 @@ import org.apache.hadoop.hdfs.Distribute import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.mapred.ClusterStatus; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobTracker; public class DefaultServlet extends BasicServlet { @@ -84,16 +80,16 @@ public class DefaultServlet extends Basi InputStream data = BasicServlet.class.getClassLoader().getResourceAsStream(path); ServletOutputStream out = resp.getOutputStream(); try { - if (data != null) { - byte[] buffer = new byte[1024]; - int n; - while ((n = data.read(buffer)) > 0) - out.write(buffer, 0, n); - } else { - out.write(("could not get resource " + path + "").getBytes()); - } + if (data != null) { + byte[] buffer = new byte[1024]; + int n; + while ((n = data.read(buffer)) > 0) + out.write(buffer, 0, n); + } else { + out.write(("could not get resource " + path + "").getBytes()); + } } finally { - data.close(); + data.close(); } } catch (Throwable t) { log.error(t, t); @@ -117,7 +113,7 @@ public class DefaultServlet extends Basi @Override public IOException run() { - InputStream data = null; + InputStream data = null; try { File file = new File(aHome + path); data = new FileInputStream(file.getAbsolutePath()); @@ -136,7 +132,7 @@ public class DefaultServlet extends Basi } catch (IOException ex) { log.error(ex, ex); } - } + } } } }, acc); @@ -248,10 +244,6 @@ public class DefaultServlet extends Basi sb.append("</td>\n"); sb.append("<td class='noborder'>\n"); - doJobTrackerTable(sb); - sb.append("</td>\n"); - - sb.append("<td class='noborder'>\n"); doZooKeeperTable(sb); sb.append("</td>\n"); @@ -363,34 +355,6 @@ public class DefaultServlet extends Basi sb.append("</table>\n"); } - private void doJobTrackerTable(StringBuilder sb) { - // Job Tracker - Configuration conf = CachedConfiguration.getInstance(); - sb.append("<table>\n"); - try { - InetSocketAddress address = JobTracker.getAddress(conf); - - // No alternative api in hadoop 20 - JobClient jc = new JobClient(new org.apache.hadoop.mapred.JobConf(conf)); - String httpAddress = conf.get("mapred.job.tracker.http.address"); - String port = httpAddress.split(":")[1]; - String href = "http://" + address.getHostName() + ":" + port; - String activeUrl = href + "/machines.jsp?type=active"; - String blacklistUrl = href + "/machines.jsp?type=blacklisted"; - sb.append("<tr><th colspan='2'><a href='" + href + "'>JobTracker</a></th></tr>\n"); - boolean highlight = false; - tableRow(sb, (highlight = !highlight), "Running Jobs", jc.jobsToComplete().length); - ClusterStatus status = jc.getClusterStatus(); - tableRow(sb, (highlight = !highlight), "Map Tasks", status.getMapTasks() + "/" + status.getMaxMapTasks()); - tableRow(sb, (highlight = !highlight), "Reduce Tasks", status.getReduceTasks() + "/" + status.getMaxReduceTasks()); - tableRow(sb, (highlight = !highlight), "<a href='" + activeUrl + "'>Trackers</a>", status.getTaskTrackers()); - tableRow(sb, (highlight = !highlight), "<a href='" + blacklistUrl + "'>Blacklisted</a>", status.getBlacklistedTrackers()); - } catch (Exception ex) { - sb.append("<tr><td colspan='2'><span class='error'>Job Tracker is Down</span></td></tr>\n"); - } - sb.append("</table>\n"); - } - private void doZooKeeperTable(StringBuilder sb) throws IOException { // Zookeepers sb.append("<table>\n"); Modified: accumulo/trunk/start/pom.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/start/pom.xml?rev=1419924&r1=1419923&r2=1419924&view=diff ============================================================================== --- accumulo/trunk/start/pom.xml (original) +++ accumulo/trunk/start/pom.xml Tue Dec 11 00:01:39 2012 @@ -56,7 +56,55 @@ </plugins> </pluginManagement> </build> - + + <profiles> + <!-- profile for building against Hadoop 1.0.x + Activate by not specifying hadoop.profile --> + <profile> + <id>hadoop-1.0</id> + <activation> + <property> + <name>!hadoop.profile</name> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-test</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + <!-- profile for building against Hadoop 2.0.x + Activate using: mvn -Dhadoop.profile=2.0 --> + <profile> + <id>hadoop-2.0</id> + <activation> + <property> + <name>hadoop.profile</name> + <value>2.0</value> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-minicluster</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> + </profiles> + <dependencies> <dependency> <groupId>log4j</groupId> @@ -81,12 +129,6 @@ </dependency> <!-- TEST DEPENDENCIES --> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-test</artifactId> - <version>0.20.205.0</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> Modified: accumulo/trunk/test/pom.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/test/pom.xml?rev=1419924&r1=1419923&r2=1419924&view=diff ============================================================================== --- accumulo/trunk/test/pom.xml (original) +++ accumulo/trunk/test/pom.xml Tue Dec 11 00:01:39 2012 @@ -57,7 +57,43 @@ </plugin> </plugins> </build> - + + <profiles> + <!-- profile for building against Hadoop 1.0.x + Activate by not specifying hadoop.profile --> + <profile> + <id>hadoop-1.0</id> + <activation> + <property> + <name>!hadoop.profile</name> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </dependency> + </dependencies> + </profile> + <!-- profile for building against Hadoop 2.0.x + Activate using: mvn -Dhadoop.profile=2.0 --> + <profile> + <id>hadoop-2.0</id> + <activation> + <property> + <name>hadoop.profile</name> + <value>2.0</value> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + </dependency> + </dependencies> + </profile> + </profiles> + <dependencies> <dependency> <groupId>org.apache.accumulo</groupId> @@ -68,14 +104,9 @@ <artifactId>accumulo-server</artifactId> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> + <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - </dependency> - </dependencies> </project> Modified: accumulo/trunk/trace/pom.xml URL: http://svn.apache.org/viewvc/accumulo/trunk/trace/pom.xml?rev=1419924&r1=1419923&r2=1419924&view=diff ============================================================================== --- accumulo/trunk/trace/pom.xml (original) +++ accumulo/trunk/trace/pom.xml Tue Dec 11 00:01:39 2012 @@ -52,7 +52,7 @@ <dependency> - <groupId>org.apache.hadoop</groupId> + <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> </dependency> <dependency>