import time
import logging
from functools import reduce
logging.basicConfig(filename='debug.log', level=logging.DEBUG)
def read_large_file(file_object):
"""Uses a generator to read a large file lazily"""
while True:
data = file_object.readline()
if not data:
break
yield data
def process_file_1(file_path):
"""Opens a large file and reads it in"""
try:
with open(file_path) as fp:
for line in read_large_file(fp):
logging.debug(line)
pass
except(IOError, OSError):
print('Error Opening or Processing file')
def process_file_2(file_path):
"""Opens a large file and reads it in"""
try:
with open(path) as file_handler:
while True:
logging.debug(next(file_handler))
except (IOError, OSError):
print("Error opening / processing file")
except StopIteration:
pass
if __name__ == "__main__":
path = "TB_data_dictionary_2016-04-15.csv"
l1 = []
for i in range(1,10):
start = time.clock()
process_file_1(path)
end = time.clock()
diff = (end - start)
l1.append(diff)
avg = reduce(lambda x, y: x + y, l1) / len(l1)
print('processing time (with generators) {}'.format(avg))
l2 = []
for i in range(1,10):
start = time.clock()
process_file_2(path)
end = time.clock()
diff = (end - start)
l2.append(diff)
avg = reduce(lambda x, y: x + y, l2) / len(l2)
print('processing time (with iterators) {}'.format(avg))
计划的输出:
C:\Python34\python.exe C:/pypen/data_structures/generators/generators1.py
processing time (with generators) 0.028033358176432314
processing time (with iterators) 0.02699498330810426
我使用的另一种方法如下。
def wrapper(func, *args, **kwargs):
def wrapped():
return func(*args, **kwargs)
return wrapped
if __name__ == "__main__":
path = "TB_data_dictionary_2016-04-15.csv"
wrapped = wrapper(process_file_1, path)
t = timeit.timeit(wrapped, number=100)
print('processing time (with generators) {}'.format(t))
wrapped = wrapper(process_file_2, path)
t = timeit.timeit(wrapped, number=100)
print('processing time (with iterators) {}'.format(t))
这给了我不同的结果。
C:\Python34\python.exe C:/pypen/data_structures/generators/generators1.py
processing time (with generators) 3.0999817624283916
processing time (with iterators) 3.2149597826018304
我希望使用generators
的实现比使用iterators
的实现更快(这是我在使用timeit时得到的)。
为什么我没有使用其他方法获得相同的结果。