Python - Mapreduce - PermissionError:[WinError 5]访问被拒绝

时间:2018-03-03 20:30:33

标签: python python-3.x mapreduce mrjob

我收到此错误,我尝试了管理员权限,以管理员身份打开,UAC关闭..但仍有同样的问题,任何人都可以说出问题是什么?我从终端

传递了2个文件movies2.csv和ratings2.csv
from mrjob.job import MRJob
from mrjob.step import MRStep
from itertools import combinations
import numpy
from scipy import spatial
class movies_count(MRJob):
 #  steps function determine the sequence of operations   
    def steps(self):
        return [
            MRStep(mapper=self.moviedatasplit,
                reducer=self.joinfilereducer),
          MRStep(reducer=self.reducer_moviepairs),
          MRStep(reducer=self.reducer_pairs),
          MRStep(reducer=self.movie_similarity)
    ]

 # Passing two files (movies.csv and ratings.csv) to the first mapper  
    def moviedatasplit(self, _, line):
                dsplit = line.split(",")
                if (len(dsplit) == 3): # movie data
                          yield dsplit[0], dsplit[1]
                else: # rating data
                          yield dsplit[1], (dsplit[0], dsplit[2])
# generating user id as key and movie title, movierating as values with the help of first reducer
    def joinfilereducer(self, _, values):
                movielist = list(values)
                movietitle = movielist[0]
                tuplevalue = movielist[1:]
                for val in tuplevalue:
                        userid = val[0]
                        movierating = val[1]    
                        yield userid, (movietitle, movierating)      
# generating combination of two movies as key and their respective ratings as value for each user id with the second reducer   
    def reducer_moviepairs(self,userid,values):
         for pair1, pair2 in combinations(values,2):
            title1=pair1[0]
            rating1=pair1[1]
            title2=pair2[0]
            rating2=pair2[1]
            yield (title1,title2),(rating1,rating2)
# combining all the ratings for each movie pair by different users with the third reducer     
    def reducer_pairs(self,titles,ratings):
        rating=[]
        for r in ratings:
            rating.append(r)
        yield titles,rating

# finding similarity between movies using statistical coorelation and cosine similarity
    def movie_similarity(self,titles,ratings):     
        rating =list(ratings)
        for ratings in rating:
            n=len(ratings)
        q1=[]
        q2=[]
        for r1 in ratings:
            q1.append((float(r1[0])))
            q2.append((float(r1[1])))

        if(n>3):           
                cor = numpy.corrcoef(q1,q2)[0,1]
                cos_cor = 1-spatial.distance.cosine(q1,q2)
                avg_cor = 0.5*(cor+cos_cor)
                yield titles[0], (titles[1],avg_cor,cor,cos_cor,n)
# main function                               
if __name__ == '__main__':
            movies_count.run()

错误:

No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs found; falling back on auto-configuration
No configs specified for inline runner
No configs specified for inline runner
No configs specified for inline runner
Running step 1 of 4...
Running step 1 of 4...
Running step 1 of 4...
Creating temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579
Creating temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579
Creating temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579
Running step 2 of 4...
Running step 2 of 4...
Running step 2 of 4...
Running step 3 of 4...
Running step 3 of 4...
Running step 3 of 4...
Running step 4 of 4...
Running step 4 of 4...
Running step 4 of 4...
Streaming final output from C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579\output...
Streaming final output from C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579\output...
Streaming final output from C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579\output...
Removing temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579...
Removing temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579...
Removing temp directory C:\Users\SURAJM~1\AppData\Local\Temp\mapreduce.Suraj MS.20180303.201748.144579...
[WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'
Traceback (most recent call last):
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\site-packages\mrjob\runner.py", line 606, in _cleanup_local_tmp
    shutil.rmtree(self._local_tmp_dir)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 494, in rmtree
    return _rmtree_unsafe(path, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 389, in _rmtree_unsafe
    onerror(os.unlink, fullname, sys.exc_info())
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 387, in _rmtree_unsafe
    os.unlink(fullname)
PermissionError: [WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'
[WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'
Traceback (most recent call last):
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\site-packages\mrjob\runner.py", line 606, in _cleanup_local_tmp
    shutil.rmtree(self._local_tmp_dir)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 494, in rmtree
    return _rmtree_unsafe(path, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 389, in _rmtree_unsafe
    onerror(os.unlink, fullname, sys.exc_info())
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 387, in _rmtree_unsafe
    os.unlink(fullname)
PermissionError: [WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'
[WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'
Traceback (most recent call last):
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\site-packages\mrjob\runner.py", line 606, in _cleanup_local_tmp
    shutil.rmtree(self._local_tmp_dir)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 494, in rmtree
    return _rmtree_unsafe(path, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 384, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 389, in _rmtree_unsafe
    onerror(os.unlink, fullname, sys.exc_info())
  File "C:\Program Files (x86)\Microsoft Visual Studio\Shared\Anaconda3_64\lib\shutil.py", line 387, in _rmtree_unsafe
    os.unlink(fullname)
PermissionError: [WinError 5] Access is denied: 'C:\\Users\\SURAJM~1\\AppData\\Local\\Temp\\mapreduce.Suraj MS.20180303.201748.144579\\step\\000\\cache\\mapreduce.py'

1 个答案:

答案 0 :(得分:0)

似乎有许可问题。尝试更改临时文件夹的路径。

在Windows系统中,临时文件夹由

决定
  1. 由TMPDIR环境变量命名的目录。
  2. TEMP环境变量指定的目录。
  3. 由TMP环境变量命名的目录。
  4. 特定于平台的位置 - 按顺序显示目录C:\ TEMP,C:\ TMP,\ TEMP和\ TMP。
  5. 创建/编辑环境变量TMPDIR/TEMP/TMP并将值设置为C:\temp。尝试重新运行该程序。