Question

最近，我正在研究gevent演示，我尝试比较gevent和线程之间的效率。一般来说，gevent代码应该比线程代码更有效。但是当我使用time命令来分析程序时，我得到了不寻常的结果（我的命令是time python FILENAME.py 50 1000，最后两个参数意味着池号或线程号，所以我改变下表中的两个数字）。结果显示该线程比gevent代码更有效，所以我想知道为什么会发生这种情况以及我的程序出了什么问题？感谢。

gevent VS thread

enter image description here

我的代码如下（主要思想是使用thread或gevent来发送多HTTP请求）：

******这是线程版本代码******

# _*_ coding: utf-8 _*_
import sys
reload(sys)
sys.setdefaultencoding("utf8")
import requests
import threading
import time
import urllib2

finished = 0


def GetUrl(pagenum):
    url = 'http://opendata.baidu.com/zhaopin/s?p=mini&wd=%B0%D9%B6%C8&pn=' + \
        str(pagenum*20) + '&rn=20'
    return url


def setUrlSet():
    for i in xrange(requestnum):
        urlnum = i % 38
        urlset.append(GetUrl(urlnum))


def GetResponse(pagenum):
    try:
        r = requests.get(urlset[pagenum])
    except Exception, e:
        print e
    pass


def DigJobByPagenum(pagenum, requestnum):
    init_num = pagenum
    print '%d begin' % init_num
    while pagenum < requestnum:
        GetResponse(pagenum)
        pagenum += threadnum
    print '%d over' % init_num


def NormalThread(threadnum):
    startime = time.time()
    print "%s is running..." % threading.current_thread().name
    threads = []
    global finished, requestnum
    for i in xrange(threadnum):
        thread = threading.Thread(target=DigJobByPagenum, args=(i, requestnum))
        threads.append(thread)
    for t in threads:
        t.daemon = True
        t.start()
    for t in threads:
        t.join()
        finished += 1
    endtime = time.time()
    print "%s is stop.The total time is %0.2f" % \
        (threading.current_thread().name, (endtime - startime))


def GetAvageTime(array):
    alltime = 0.0
    for i in array:
        alltime += i
    avageTime = alltime/len(array)
    return avageTime

if __name__ == '__main__':
    threadnum = int(sys.argv[1])
    requestnum = int(sys.argv[2])
    print 'threadnum : %s,requestnum %s ' % (threadnum, requestnum)
    originStartTime = time.time()
    urlset = []
    setUrlSet()
    NormalThread(threadnum)

******这是gevent verison代码******

# _*_ coding: utf-8 _*_
import sys
reload(sys)
sys.setdefaultencoding("utf8")
from gevent import monkey
monkey.patch_all()
import gevent
from gevent import pool
import requests
import time

finished = 0


def GetUrl(pagenum):
    url = 'http://opendata.baidu.com/zhaopin/s?p=mini&wd=%B0%D9%B6%C8&pn=' + \
        str(pagenum*20) + '&rn=20'
    return url


def setUrlSet():
    for i in xrange(requestnum):
        urlnum = i % 38
        urlset.append(GetUrl(urlnum))


def GetResponse(url):
    startime = time.time()
    r = requests.get(url)
    print url
    endtime = time.time()
    spendtime = endtime - startime
    NormalSpendTime.append(spendtime)
    global finished
    finished += 1
    print finished


def GetAvageTime(array):
    alltime = 0.0
    for i in array:
        alltime += i
    avageTime = alltime/len(array)
    return avageTime


def RunAsyncJob():
    jobpool = pool.Pool(concurrent)
    for url in urlset:
        jobpool.spawn(GetResponse, url)
    jobpool.join()
    endtime = time.time()
    allSpendTime = endtime - originStartime
    print 'Total spend time is %0.3f, total request num is %s within %s \
            seconds' % (allSpendTime, finished, timeoutNum)
    print 'Each request time is %0.3f' % (GetAvageTime(NormalSpendTime))


if __name__ == '__main__':
    concurrent = int(sys.argv[1])
    requestnum = int(sys.argv[2])
    timeoutNum = 100
    NormalSpendTime = []
    urlset = []
    urlActionList = []
    setUrlSet()
    originStartime = time.time()
    RunAsyncJob()

Answer 1

尝试

gevent.monkey.patch_all(httplib=True)

默认情况下，gevent似乎没有修补 httplib （看看http://www.gevent.org/gevent.monkey.html：httplib=False）所以你实际上正在阻止请求而你失去了所有优点异步框架。虽然我不确定请求是否使用 httplib 。

如果这不起作用，那么看一下这个lib：

https://github.com/kennethreitz/grequests

Answer 2

回复：httplib=False

您已使用requests库进行网络通话。它有gevent味道，称为grequests：

https://github.com/kennethreitz/grequests

总的来说，如果你的游泳池太小，我不会立即看到很多理由喜欢一种穿线方式。当然真正的线程相对较重（从8MB堆栈开始），但你必须把它与你工作的大小成比例。

我的看法，尝试两个（完成），验证你做得对（做）和让数字进行交谈。

gevent和线程之间的效率比较

2 个答案: