Question

我正在尝试在亚马逊ec2上运行mapreduce任务。我设置了所有配置参数，然后调用AmazonElasticMapReduce服务的runFlowJob方法。我想知道有没有办法知道工作是否已经完成以及状态如何。（我需要知道我何时可以从s3中获取mapreduce结果以进行进一步处理）

目前代码只是继续执行，因为对runJobFlow的调用是非阻塞的。



public void startMapReduceTask(String accessKey, String secretKey
        ,String eC2KeyPairName, String endPointURL, String jobName
        ,int numInstances, String instanceType, String placement
        ,String logDirName, String bucketName, String pigScriptName) {
    log.info("Start running MapReduce");

    // config.set
    ClientConfiguration config = new ClientConfiguration();
    AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);

    AmazonElasticMapReduce service = new AmazonElasticMapReduceClient(credentials, config);
    service.setEndpoint(endPointURL);

    JobFlowInstancesConfig conf = new JobFlowInstancesConfig();

    conf.setEc2KeyName(eC2KeyPairName);
    conf.setInstanceCount(numInstances);
    conf.setKeepJobFlowAliveWhenNoSteps(true);
    conf.setMasterInstanceType(instanceType);
    conf.setPlacement(new PlacementType(placement));
    conf.setSlaveInstanceType(instanceType);

    StepFactory stepFactory = new StepFactory();

    StepConfig enableDebugging = new StepConfig()
    .withName("Enable Debugging")
    .withActionOnFailure("TERMINATE_JOB_FLOW")
    .withHadoopJarStep(stepFactory.newEnableDebuggingStep());

    StepConfig installPig = new StepConfig()
    .withName("Install Pig")
    .withActionOnFailure("TERMINATE_JOB_FLOW")
    .withHadoopJarStep(stepFactory.newInstallPigStep());

    StepConfig runPigScript = new StepConfig()
    .withName("Run Pig Script")
    .withActionOnFailure("TERMINATE_JOB_FLOW")
    .withHadoopJarStep(stepFactory.newRunPigScriptStep("s3://" + bucketName + "/" + pigScriptName, ""));

    RunJobFlowRequest request = new RunJobFlowRequest(jobName, conf)
    .withSteps(enableDebugging, installPig, runPigScript)
    .withLogUri("s3n://" + bucketName + "/" + logDirName);

    try {
        RunJobFlowResult res = service.runJobFlow(request);
        log.info("Mapreduce job with id[" + res.getJobFlowId() + "] completed successfully");
    } catch (Exception e) {
        log.error("Caught Exception: ", e);
    }
    log.info("End running MapReduce");      
}

public void startMapReduceTask(String accessKey, String secretKey ,String eC2KeyPairName, String endPointURL, String jobName ,int numInstances, String instanceType, String placement ,String logDirName, String bucketName, String pigScriptName) { log.info("Start running MapReduce"); // config.set ClientConfiguration config = new ClientConfiguration(); AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); AmazonElasticMapReduce service = new AmazonElasticMapReduceClient(credentials, config); service.setEndpoint(endPointURL); JobFlowInstancesConfig conf = new JobFlowInstancesConfig(); conf.setEc2KeyName(eC2KeyPairName); conf.setInstanceCount(numInstances); conf.setKeepJobFlowAliveWhenNoSteps(true); conf.setMasterInstanceType(instanceType); conf.setPlacement(new PlacementType(placement)); conf.setSlaveInstanceType(instanceType); StepFactory stepFactory = new StepFactory(); StepConfig enableDebugging = new StepConfig() .withName("Enable Debugging") .withActionOnFailure("TERMINATE_JOB_FLOW") .withHadoopJarStep(stepFactory.newEnableDebuggingStep()); StepConfig installPig = new StepConfig() .withName("Install Pig") .withActionOnFailure("TERMINATE_JOB_FLOW") .withHadoopJarStep(stepFactory.newInstallPigStep()); StepConfig runPigScript = new StepConfig() .withName("Run Pig Script") .withActionOnFailure("TERMINATE_JOB_FLOW") .withHadoopJarStep(stepFactory.newRunPigScriptStep("s3://" + bucketName + "/" + pigScriptName, "")); RunJobFlowRequest request = new RunJobFlowRequest(jobName, conf) .withSteps(enableDebugging, installPig, runPigScript) .withLogUri("s3n://" + bucketName + "/" + logDirName); try { RunJobFlowResult res = service.runJobFlow(request); log.info("Mapreduce job with id[" + res.getJobFlowId() + "] completed successfully"); } catch (Exception e) { log.error("Caught Exception: ", e); } log.info("End running MapReduce"); }

感谢，

阿维亚德

Answer 1

来自AWS文档：

作业流完成后，群集将停止，HDFS分区将丢失。 为防止数据丢失，请配置作业流程的最后一步，以便在Amazon S3中存储结果。

接着说：

如果JobFlowInstancesDetail : KeepJobFlowAliveWhenNoSteps参数设置为TRUE，则作业流程将转换为WAITING状态，而不是在步骤完成后关闭。

每个作业流程最多允许256步。

对于长时间运行的作业流程，我们建议您定期存储结果。

所以看起来没有办法知道什么时候完成。相反，您需要将数据保存为工作的一部分。

我如何知道亚马逊mapreduce任务何时完成？

1 个答案: