为了降低内存成本,我使用astype()
指定了我的pandas数据帧的dtypes,如:
df['A'] = df['A'].astype(int8)
然后我使用to_csv()
来存储它,但当我使用read_csv()
再次阅读并检查dtypes
时,我发现它仍然存储在int64
中。
如何保存dtypes,同时将其保存在本地存储中?
答案 0 :(得分:4)
对#Aaron N. Brock的修改也允许parse_dates(而且不更改原始DataFrame):
def to_csv(df, path):
# Prepend dtypes to the top of df
df2 = df.copy()
df2.loc[-1] = df2.dtypes
df2.index = df2.index + 1
df2.sort_index(inplace=True)
# Then save it to a csv
df2.to_csv(path, index=False)
def read_csv(path):
# Read types first line of csv
dtypes = {key:value for (key,value) in pd.read_csv(path,
nrows=1).iloc[0].to_dict().items() if 'date' not in value}
parse_dates = [key for (key,value) in pd.read_csv(path,
nrows=1).iloc[0].to_dict().items() if 'date' in value]
# Read the rest of the lines with the types from above
return pd.read_csv(path, dtype=dtypes, parse_dates=parse_dates, skiprows=[1])
答案 1 :(得分:2)
以下 方式:
import pandas as pd
# Create Example data with types
df = pd.DataFrame({
'words': ['foo', 'bar', 'spam', 'eggs'],
'nums': [1, 2, 3, 4]
}).astype(dtype={
'words': 'object',
'nums': 'int8'
})
def to_csv(df, path):
# Prepend dtypes to the top of df (from https://stackoverflow.com/a/43408736/7607701)
df.loc[-1] = df.dtypes
df.index = df.index + 1
df.sort_index(inplace=True)
# Then save it to a csv
df.to_csv(path, index=False)
def read_csv(path):
# Read types first line of csv
dtypes = pd.read_csv('tmp.csv', nrows=1).iloc[0].to_dict()
# Read the rest of the lines with the types from above
return pd.read_csv('tmp.csv', dtype=dtypes, skiprows=[1])
print('Before: \n{}\n'.format(df.dtypes))
to_csv(df, 'tmp.csv')
df = read_csv('tmp.csv')
print('After: \n{}\n'.format(df.dtypes))
输出:
Before:
nums int8
words object
dtype: object
After:
nums int8 # still int8
words object
dtype: object