view release on metacpan or  search on metacpan

xgboost/dmlc-core/tracker/yarn/src/main/java/org/apache/hadoop/yarn/dmlc/Client.java  view on Meta::CPAN

                this.jobName = args[++i];
            } else if(args[i].equals("-tempdir")) {
                this.tempdir = args[++i];
            } else if(args[i].equals("-queue")) {
                this.queue = args[++i];
            } else if(args[i].equals("-appcp")) {
                this.appCp = args[++i];
            } else if(args[i].equals("-env")) {
                sargs.append(" ");
                sargs.append(args[i]);
                sargs.append(" ");
                sargs.append(args[i+1]);
                String[] pair = args[++i].split("=", 2);
                env.put(pair[0], (pair.length == 1) ? "" : pair[1]);
            } else {
                sargs.append(" ");
                sargs.append(args[i]);
            }
        }
        this.appArgs = sargs.toString();
    }

    private void run(String[] args) throws Exception {
        if (args.length == 0) {
            System.out.println("Usage: [options] [commands..]");
            System.out.println("options: [-file filename] [-appcp appClasspath]");
            return;
        }
        this.initArgs(args);
        // Create yarnClient
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(conf);
        yarnClient.start();

        // Create application via yarnClient
        YarnClientApplication app = yarnClient.createApplication();

        // Set up the container launch context for the application master
        ContainerLaunchContext amContainer = Records
                .newRecord(ContainerLaunchContext.class);
        ApplicationSubmissionContext appContext = app
                .getApplicationSubmissionContext();
        // Submit application
        ApplicationId appId = appContext.getApplicationId();

        //add ctrl+c signal handler
        CtrlCHandler handler = new CtrlCHandler(appId, yarnClient);
        Signal intSignal = new Signal("INT");
        Signal.handle(intSignal, handler);

        // setup security token
        amContainer.setTokens(this.setupTokens());
        // setup cache-files and environment variables
        amContainer.setLocalResources(this.setupCacheFiles(appId));
        amContainer.setEnvironment(this.getEnvironment());
        String cmd = Environment.JAVA_HOME.$$() + "/bin/java"
                + " -Xmx900m"
                + " org.apache.hadoop.yarn.dmlc.ApplicationMaster"
                + this.cacheFileArg + ' ' + this.appArgs + " 1>"
                + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"
                + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr";

        LOG.debug(cmd);
        amContainer.setCommands(Collections.singletonList(cmd));

        // Set up resource type requirements for ApplicationMaster
        Resource capability = Records.newRecord(Resource.class);
        capability.setMemory(1024);
        capability.setVirtualCores(1);
        LOG.info("jobname=" + this.jobName + ",username=" + this.userName);

        appContext.setApplicationName(jobName + ":DMLC-YARN");
        appContext.setAMContainerSpec(amContainer);
        appContext.setResource(capability);
        appContext.setQueue(queue);
        //appContext.setUser(userName);
        LOG.info("Submitting application " + appId);
        yarnClient.submitApplication(appContext);

        ApplicationReport appReport = yarnClient.getApplicationReport(appId);
        YarnApplicationState appState = appReport.getYarnApplicationState();
        while (appState != YarnApplicationState.FINISHED
                && appState != YarnApplicationState.KILLED
                && appState != YarnApplicationState.FAILED) {
            Thread.sleep(100);
            appReport = yarnClient.getApplicationReport(appId);
            appState = appReport.getYarnApplicationState();
        }

        System.out.println("Application " + appId + " finished with"
                + " state " + appState + " at " + appReport.getFinishTime());
        if (!appReport.getFinalApplicationStatus().equals(
                FinalApplicationStatus.SUCCEEDED)) {
            System.err.println(appReport.getDiagnostics());
            System.out.println("Available queues:");
            for (QueueInfo q : yarnClient.getAllQueues()) {
              System.out.println(q.getQueueName());
            }

            yarnClient.killApplication(appId);
        }
    }

    class CtrlCHandler implements SignalHandler{
        private ApplicationId appId;
        private YarnClient yarnClient;
        public CtrlCHandler(ApplicationId appId, YarnClient yarnClient){
            this.appId = appId;
            this.yarnClient = yarnClient;
        }
        public void handle(Signal signal){
            try{
                yarnClient.killApplication(appId);
            }catch (Exception e){
                System.out.println("yarn client exception");
            }
        }
    }
    public static void main(String[] args) throws Exception {
        new Client().run(args);
    }