Question

此函数从文本文件读取并重新格式化内容，然后将内容写入csv。我正在尝试使用线程对for i in lines循环进行多线程处理，这是较大脚本的最长部分，并且占用了大部分运行时间，因为列表lines包含数千个元素。有人可以帮我弄清楚吗？同步而不是并行执行此操作会占用大量时间。我已经看到许多其他类似问题的答案，但是到目前为止，我还没有理解并正确实施这些答案。

 def sheets(i):
        # time format for spreadsheet
        dt_time = datetime.now().strftime('%m/%d|%H:%M')
        # for league name (NFL,NBA,NHL ETC.) in list containing league names
        for league_name in leagues2: 
            league_name = league_name.split('|')[0]
            with open(final_stats_path, 'r+') as lines:
                lines = lines.readlines()
            # i = one long string containg details about the event in the loop, eg. sport, game day, game id, home team name
            for i in lines:
                i = i.split(',')
                minprice = i[6]
                totaltix = i[5]
                event_date = i[2]
                try:
                    dayofweek = datetime.strptime(event_date, '%Y-%m-%d').strftime('%A')
                except:
                    continue
                event_date = i[2][2:]
                event_date = str(event_date).split('-')
                event_date = event_date[1]+'/'+event_date[2]
                sport = i[4]
                event = i[1].replace('Basketball','').replace('\n','')
                away = i[8].replace('Basketball', '').replace('\n','')
                eventid = i[0]
                event_home = i[9].replace('Basketball', '').replace('\n','')
                event = event.split(' at ')[0]
                tixdata = str(totaltix)
                eventid = 'https://pro.stubhub.com/simweb/sim/services/priceanalysis?eventId='+str(eventid)+'&sectionId=0'

                directory = root+'\data'+'\\'+sport+'\\'
                report = directory+'report.xlsx'
                fname = directory+'teams.txt'

                eventleague = sport

                f = open(directory+'acronym.txt', 'r+')
                lines_2 = f.readlines()
                for qt in lines_2:
                    qt = qt.split('-')
                    compare = qt[1]
                    if event_home in compare:
                        event_home = qt[0]
                    else:
                        pass

                troop = []
                d = {
                    'ID' : eventid,
                    'Date' : event_date,
                    'Day' : dayofweek,
                    'Away' : away,
                }

                s = {
                    'time' : tixdata

                }


                numbers = event_home+'.txt'
                numbers_new = 'bk\\bk_'+numbers

                with open(directory+numbers_new, 'a+') as y:
                    pass
                with open(directory+numbers, 'a+') as o:
                    pass



                with open(directory+numbers, 'r+') as g:
                    for row in g:
                        if str(eventid) in row:
                            #print('the event is in the list')
                            row_update = row.replace('}', ", '"+dt_time+"': '"+tixdata+"'}")
                            with open(directory+numbers_new, 'a+') as y:
                                y.write(row_update)
                            break
                    else:
                        with open(directory+numbers, 'a+') as p:
                            #print('the event is not in the list')
                            p.write(str(d)+'\n')
                            with open(directory+numbers_new, 'a+') as n:
                                n.write(str(d)+'\n')


                sizefile = os.path.getsize(directory+numbers_new)
                if sizefile > 0:
                    shutil.copy(directory+numbers_new, directory+numbers)
                    open(directory+numbers_new, 'w').close()
                else:
                    pass
                df = []
                with open(directory+numbers, 'r+') as t:
                    for row in t:
                        b = eval(row)
                        dfs = df.append(b)
                    df = pd.DataFrame(df)
                    yark = list(df.columns)[:-5]
                    zed = ['ID', 'Date', 'Day', 'Away']
                    columns = zed+yark
                    try:
                        df = df[columns]
                    except:
                        pass

                df.index = range(1, 2*len(df)+1, 2)
                df = df.reindex(index=range(2*len(df)))
                writer = pd.ExcelWriter(directory+event_home+'.xlsx', engine='xlsxwriter')
                try:
                    df.to_excel(writer, sheet_name=event_home)
                except:
                    continue
                workbook  = writer.book
                worksheet = writer.sheets[event_home]
                format1 = workbook.add_format({'num_format': '#,##0.00'})
                worksheet.set_column('A:ZZ', 18, format1)
                writer.save()



    if __name__ == "__main__":
        pool = ThreadPool(8)  # Make the Pool of workers
        results = pool.map(sheets) #Open the urls in their own threads
        pool.close() #close the pool and wait for the work to finish 
        pool.join() 

    ##get_numbers()
    ##stats_to_csv()
    ##stats_to_html()
    #sheets()

Answer 1

尝试更改以下行：

results = pool.map(sheets)

收件人：

results = pool.map(sheets,range(8))

多线程列表迭代循环

1 个答案: