我是 python 新手,并尝试并行执行 http post 请求以加快脚本的处理时间
脚本明智地从 Excel 工作表中获取数据并为工作表中的每一行执行 http post。以下是我正在尝试的代码:
def auth():
"Do something"
def read_excel():
xlfile='data.xls'
xls=pandas.ExcelFile(xlfile)
df1=pandas.read_excel(xls,'sheet1')
df2=pandas.read_excel(xls,'sheet2')
return df1,df2
def trigger_job(auth_user,headers,url,record):
payload=request.post(url,auth=(auth_user,auth_pass),headers,record)
'''some code to further process data'''
def template_read(sheet):
with open('templates.json') as templates:
data=json.load(templates)
url=data[sheet]['url']
return url
async def execute_job(df,url,auth_user,auth_pass,headers):
yeslist=["yes","YES"]
recycledf=df[df['Recycle?'].isin(yeslist)]
recycledf=recycledf.drop(['Recycle?'],axis=1)
recycle_json=recycledf.to_json(orient='records')
for record in recycle_json:
trigger_job(auth_user,headers,url,record)
async def main():
sheet1=loop.create_task(execute_job(df1,template_read("sheet1"),auth_user,auth_pass,headers)
sheet2=loop.create_task(execute_job(df2,template_read("sheet2"),auth_user,auth_pass,headers)
await asyncio.wait([sheet1,sheet2])
if __name__=="__main__":
auth_user,auth_pass,headers=auth()
df1,df2=read_excel()
loop= asyncio.geteventloop()
loop.run_until_complete(main())
此代码同步运行 sheet1 和 sheet2 任务。 Sheet1 任务 execute_job 和 for 循环阻塞了 sheet2 任务
虽然我的期望是并行运行 sheet1 任务和 sheet2 两个任务并最终收集结果。
由于我刚开始学习,任何线索都会有所帮助。谢谢