Question

当我尝试使用LocalJobRunner或MiniCluster运行简单的Map-Reduce测试时，我得到的错误如下所示。我无法弄明白为什么。

使用LocalJobRunner进行测试

public class ReduceSideJoinTest
{
    @Test
    public void run() throws Exception
    {
        Path inputPath = new Path("/tmp/mrtest/input");
        Path outputPath = new Path("/tmp/mrtest/output");

        Configuration conf = new Configuration();

        conf.set("mapred.job.tracker", "local");
        conf.set("fs.default.name", "file:///");

        FileSystem fs = FileSystem.get(conf);
        if (fs.exists(outputPath))
        {
            fs.delete(outputPath, true);
        }
        if (fs.exists(inputPath))
        {
            fs.delete(inputPath, true);
        }
        fs.mkdirs(inputPath);

        String input = "foo\tbar";
        DataOutputStream file = fs.create(new Path(inputPath, "part-" + 0));
        file.writeBytes(input);
        file.close();

        Job job = runJob(conf, inputPath, outputPath);
        assertTrue(job.isSuccessful());

        List<String> lines = IOUtils.readLines(fs.open(new Path(outputPath, "part-r-00000")));

        assertEquals(1, lines.size());
        String[] parts = StringUtils.split(lines.get(0), "\t");
        assertEquals("foo", parts[0]);
        assertEquals("bar", parts[1]);
    }

    public Job runJob(Configuration conf, Path inputPath, Path outputPath) throws ClassNotFoundException, IOException,
            InterruptedException
    {
        Job job = new Job(conf);

        job.setInputFormatClass(KeyValueTextInputFormat.class);

        job.setMapOutputKeyClass(Text.class);

        FileInputFormat.setInputPaths(job, inputPath);
        FileOutputFormat.setOutputPath(job, outputPath);

        job.waitForCompletion(false);
        return job;
    }

}

错误

java.io.IOException: Cannot run program "cygpath": CreateProcess error=2, The system cannot find the file specified
    at java.lang.ProcessBuilder.start(ProcessBuilder.java:470)
    at org.apache.hadoop.util.Shell.runCommand(Shell.java:206)
    at org.apache.hadoop.util.Shell.run(Shell.java:188)
    at org.apache.hadoop.fs.FileUtil$CygPathCommand.<init>(FileUtil.java:412)
    at org.apache.hadoop.fs.FileUtil.makeShellPath(FileUtil.java:438)
    at org.apache.hadoop.fs.FileUtil.makeShellPath(FileUtil.java:465)
    at org.apache.hadoop.fs.RawLocalFileSystem.execCommand(RawLocalFileSystem.java:573)
    at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:565)
    at org.apache.hadoop.fs.FilterFileSystem.setPermission(FilterFileSystem.java:428)
    at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:452)
    at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:420)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:806)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:787)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:686)
    at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:675)
    at com.aravind.learning.hadoop.mapred.join.ReduceSideJoinTest.run(ReduceSideJoinTest.java:47)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:44)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:41)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:76)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
    at org.junit.runners.ParentRunner$3.run(ParentRunner.java:193)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:52)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:191)
    at org.junit.runners.ParentRunner.access$000(ParentRunner.java:42)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:184)
    at org.junit.runners.ParentRunner.run(ParentRunner.java:236)
    at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50)
    at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:467)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:683)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:390)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:197)
Caused by: java.io.IOException: CreateProcess error=2, The system cannot find the file specified
    at java.lang.ProcessImpl.create(Native Method)
    at java.lang.ProcessImpl.<init>(ProcessImpl.java:177)
    at java.lang.ProcessImpl.start(ProcessImpl.java:28)
    at java.lang.ProcessBuilder.start(ProcessBuilder.java:452)
    ... 37 more

Answer 1

回答我自己的问题。是的，它需要安装Cygwin。请按照以下步骤

从here
将路径设置为cygwin_install_dir/bin
重新运行测试

LocalJobRunner和MiniCluster是否要求cygwin在Windows 7上运行？

1 个答案: