我尝试运行一个演示并发现'pd.read_sql'的结果不能被使用两次! 但它仍在内存中,这真是令人惊讶。我该如何解决?
#coding=utf-6
import pandas as pd
from sqlalchemy import create_engine
engine =
create_engine('mysql+pymysql://root:123457@127.0.0.1:3306/chapter12?
charset=utf8')
sql = pd.read_sql('all_gzdata', engine, chunksize = 10001)
def count109(i):
j = i[['fullURL']][i['fullURLId'].str.contains('109')].copy() #找出类别包含107的网址
j['type'] = None #添加空列
j['type'][j['fullURL'].str.contains('info/.+?/')] = u'知识首页'
j['type'][j['fullURL'].str.contains('info/.+?/.+?')] = u'知识列表页'
j['type'][j['fullURL'].str.contains('/\d+?_*\d+?\.html')] = u'知识内容页'
print('109')
return j['type'].value_counts()
def test(i):
return 3
def test3(i):
return 4
print(sql)
counts5 = [ test(i) for i in sql]
print(counts5)
counts6 = [ test(i) for i in sql]
print(counts6)
print(sql)
print('finish')
结果是:
anakin@anakin:~/Project/chapter12$ python sql_value_count.py
<generator object SQLTable._query_iterator at 0x7f03ce621728>
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
[]
<generator object SQLTable._query_iterator at 0x7f03ce621728>
finish