#IGNITE-857 WIP
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/81cde9b7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/81cde9b7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/81cde9b7 Branch: refs/heads/ignite-gg-9615 Commit: 81cde9b71b3ab88023f68fa6f3bee86f20442412 Parents: b569191 Author: nikolay tikhonov <ntikho...@gridgain.com> Authored: Wed Jun 3 18:37:02 2015 +0300 Committer: nikolay tikhonov <ntikho...@gridgain.com> Committed: Wed Jun 3 18:37:02 2015 +0300 ---------------------------------------------------------------------- .../apache/ignite/yarn/ApplicationMaster.java | 51 +++++++++++++++----- .../apache/ignite/yarn/IgniteYarnClient.java | 15 ++---- 2 files changed, 45 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/81cde9b7/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java index 9ab70d4..532830c 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java @@ -19,6 +19,9 @@ package org.apache.ignite.yarn; import com.google.common.collect.Lists; import org.apache.hadoop.conf.*; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.*; import org.apache.hadoop.yarn.api.protocolrecords.*; import org.apache.hadoop.yarn.api.records.*; @@ -33,27 +36,33 @@ import java.util.*; * TODO */ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { - Configuration configuration; + YarnConfiguration configuration; NMClient nmClient; - int numContainersToWaitFor = 5; + int numContainersToWaitFor = 1; public ApplicationMaster() { configuration = new YarnConfiguration(); + nmClient = NMClient.createNMClient(); nmClient.init(configuration); nmClient.start(); } + /** {@inheritDoc} */ public void onContainersAllocated(List<Container> containers) { for (Container container : containers) { try { // Launch container by create ContainerLaunchContext - // bin/hadoop fs -rm /user/ntikhonov/*.jar && bin/hadoop fs -copyFromLocal ./ignite-yarn.jar /user/ntikhonov - ContainerLaunchContext ctx = - Records.newRecord(ContainerLaunchContext.class); + ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); + + final LocalResource igniteZip = Records.newRecord(LocalResource.class); + setupAppMasterJar(new Path("/user/ntikhonov/gridgain-community-fabric-1.0.6.zip"), igniteZip, + configuration); + + ctx.setLocalResources(Collections.singletonMap("ignite", igniteZip)); ctx.setCommands( Lists.newArrayList( - "ls " + + "$LOCAL_DIRS/ignite/*/bin/ignite.sh" + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr" )); @@ -65,6 +74,24 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { } } + /** {@inheritDoc} */ + private static void setupAppMasterJar(Path jarPath, LocalResource appMasterJar, YarnConfiguration conf) + throws Exception { + FileSystem fileSystem = FileSystem.get(conf); + jarPath = fileSystem.makeQualified(jarPath); + + FileStatus jarStat = fileSystem.getFileStatus(jarPath); + + appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(jarPath)); + appMasterJar.setSize(jarStat.getLen()); + appMasterJar.setTimestamp(jarStat.getModificationTime()); + appMasterJar.setType(LocalResourceType.ARCHIVE); + appMasterJar.setVisibility(LocalResourceVisibility.APPLICATION); + + System.out.println("Path :" + jarPath); + } + + /** {@inheritDoc} */ public void onContainersCompleted(List<ContainerStatus> statuses) { for (ContainerStatus status : statuses) { System.out.println("[AM] Completed container " + status.getContainerId()); @@ -74,20 +101,21 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { } } + /** {@inheritDoc} */ public void onNodesUpdated(List<NodeReport> updated) { } - public void onReboot() { - } - + /** {@inheritDoc} */ public void onShutdownRequest() { } + /** {@inheritDoc} */ public void onError(Throwable t) { } + /** {@inheritDoc} */ public float getProgress() { - return 0; + return 50; } public boolean doneWithContainers() { @@ -101,7 +129,6 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { public static void main(String[] args) throws Exception { ApplicationMaster master = new ApplicationMaster(); master.runMainLoop(); - } public void runMainLoop() throws Exception { @@ -136,6 +163,8 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { Thread.sleep(100); } + + System.out.println("[AM] unregisterApplicationMaster 0"); // Un-register with ResourceManager rmClient.unregisterApplicationMaster( http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/81cde9b7/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java index e020ef4..092aaa9 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java @@ -74,20 +74,15 @@ public class IgniteYarnClient { final LocalResource igniteZip = Records.newRecord(LocalResource.class); setupAppMasterJar(new Path("/user/ntikhonov/gridgain-community-fabric-1.0.6.zip"), igniteZip, conf); - FileSystem fileSystem = FileSystem.get(conf); - - Path path = fileSystem.makeQualified(new Path("/user/ntikhonov/gridgain-community-fabric-1.0.6/bin/ignite.sh")); - - System.out.println("Path: " + path); - System.out.println("Path URI: " + path.toUri().toString()); - - amContainer.setLocalResources(new HashMap<String, LocalResource>(){{ + amContainer.setLocalResources(new HashMap<String, LocalResource>() {{ put("ignite-yarn.jar", appMasterJar); - put("ignite", igniteZip); + put("gridgain-community-fabric-1.0.6.zip", igniteZip); }}); + + // Setup CLASSPATH for ApplicationMaster - Map<String, String> appMasterEnv = new HashMap<String, String>(); + Map<String, String> appMasterEnv = new HashMap<>(); setupAppMasterEnv(appMasterEnv, conf); amContainer.setEnvironment(appMasterEnv);
