pandas内存消耗hdf文件分组

时间:2016-09-21 08:25:05

标签: python pandas ram

我编写了以下脚本,但我遇到内存消耗问题,pandas正在分配超过30 G的ram,其中数据文件的总和大约为18 G

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import time


mean_wo = pd.DataFrame()
mean_w = pd.DataFrame()
std_w = pd.DataFrame()
std_wo = pd.DataFrame()

start_time=time.time() #taking current time as starting time

data_files=['2012.h5','2013.h5','2014.h5','2015.h5', '2016.h5', '2008_2011.h5'] 



for data_file in data_files:
    print data_file
    df = pd.read_hdf(data_file)
    grouped = df.groupby('day')
    mean_wo_tmp=grouped['Significance_without_muons'].agg([np.mean])
    mean_w_tmp=grouped['Significance_with_muons'].agg([np.mean])
    std_wo_tmp=grouped['Significance_without_muons'].agg([np.std])
    std_w_tmp=grouped['Significance_with_muons'].agg([np.std])
    mean_wo = pd.concat([mean_wo, mean_wo_tmp])
    mean_w = pd.concat([mean_w, mean_w_tmp])
    std_w = pd.concat([std_w,std_w_tmp])
    std_wo = pd.concat([std_wo,std_wo_tmp])
    print mean_wo.info()
    print mean_w.info()
    del df, grouped, mean_wo_tmp, mean_w_tmp, std_w_tmp, std_wo_tmp

std_wo=std_wo.reset_index()
std_w=std_w.reset_index()
mean_wo=mean_wo.reset_index()
mean_w=mean_w.reset_index()

#setting the field day as date
std_wo['day']= pd.to_datetime(std_wo['day'], format='%Y-%m-%d')
std_w['day']= pd.to_datetime(std_w['day'], format='%Y-%m-%d')
mean_w['day']= pd.to_datetime(mean_w['day'], format='%Y-%m-%d')
mean_wo['day']= pd.to_datetime(mean_w['day'], format='%Y-%m-%d')

所以有人知道如何减少内存消耗?

干杯,

1 个答案:

答案 0 :(得分:1)

我会做这样的事情 解决方案

data_files=['2012.h5', '2013.h5', '2014.h5', '2015.h5', '2016.h5', '2008_2011.h5'] 
cols = ['Significance_without_muons', 'Significance_with_muons']

np.random.seed([3,1415])
data_df = pd.DataFrame(np.random.rand(1000, 2), columns=cols)
data_df['day'] = np.random.choice(list('ABCDEFG'), 1000)

for fn in data_files:
    data_df.to_hdf(fn, 'day', append=False)

设置

mean_wo_tmp

运行上方解决方案
然后

public Game(MainActivity context){
this.context = context;
}


score = (i+1);
if (score >1){

PackageManager pm = context.getPackageManager();
                        Intent i = new Intent("android.intent.action.MAIN");
                        i.addCategory("android.intent.category.HOME");
                        List<ResolveInfo> lst = pm.queryIntentActivities(i, 0);
                        if (lst != null) {
                            for (ResolveInfo resolveInfo : lst) {
                                try {
                                    ApplicationInfo ai = pm.getApplicationInfo(
                                                    resolveInfo.activityInfo.packageName,0);
                                    if ((ai.flags & ApplicationInfo.FLAG_SYSTEM) != 0) {
                                        Intent goHome = new Intent();
                                        goHome.setFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP);
                                        goHome.setClassName(
                                                resolveInfo.activityInfo.packageName,
                                                resolveInfo.activityInfo.name);
                                        context.startActivity(goHome);
                                        int pid = android.os.Process.myPid();
                                        android.os.Process.killProcess(pid);
                                        break;
                                    }
                                } catch (NameNotFoundException e) {
                                    logMessage(LOG_TYPE_ERROR, "AppConstant-exitFromApp-kill", e.getMessage());
                                }
                            }
                        }

}

enter image description here