我已经编写了Map-Reduce代码来将文件从HDFS复制到Local,当我运行Map-Reduce作业时,它会抛出以下错误。
Log Upload Time: Thu Jun 29 10:51:22 +0530 2017
Log Length: 78421677
Showing 1000000 bytes of 78421677 total. Click
here
for the full log.
t org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.<init>(RawLocalFileSystem.java:209)
at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:307)
at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:295)
at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:328)
at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.<init>(ChecksumFileSystem.java:393)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:456)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:435)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:922)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:903)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:800)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:368)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:341)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:292)
at org.apache.hadoop.fs.FileSystem.copyToLocalFile(FileSystem.java:2016)
at org.apache.hadoop.fs.FileSystem.copyToLocalFile(FileSystem.java:1985)
at org.apache.hadoop.fs.FileSystem.copyToLocalFile(FileSystem.java:1961)
at com.mani.pmml_mr.PMMLMapper.map(PMMLMapper.java:64)
at com.mani.pmml_mr.PMMLMapper.map(PMMLMapper.java:35)
at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
EPERM: Operation not permitted
at org.apache.hadoop.io.nativeio.NativeIO$POSIX.chmodImpl(Native Method)
at org.apache.hadoop.io.nativeio.NativeIO$POSIX.chmod(NativeIO.java:230)
at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:708)
at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.<init>(RawLocalFileSystem.java:225)
at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.<init>(RawLocalFileSystem.java:209)
at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:307)
at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:295)
at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:328)
at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.<init>(ChecksumFileSystem.java:393)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:456)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:435)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:922)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:903)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:800)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:368)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:341)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:292)
at org.apache.hadoop.fs.FileSystem.copyToLocalFile(FileSystem.java:2016)
at org.apache.hadoop.fs.FileSystem.copyToLocalFile(FileSystem.java:1985)
at org.apache.hadoop.fs.FileSystem.copyToLocalFile(FileSystem.java:1961)
在我的代码中我使用 copyToLocalFile()方法,但我不确定它为什么会抛出错误。我将完全权限(777)提供给将要复制文件的本地文件夹。但它仍然会引发一些错误。
我注意到的另一个问题是,即使是工作也会抛出错误(我从Hue的Job Browser注意到),仍然在控制台中显示工作成功。
17/06/29 10:51:16 INFO mapreduce.Job: Job job_1495430640647_0231 completed successfully
有人可以帮助我。