我有一个我想要处理的大型数据集（1.2亿条记录）。我的程序目前正在使用谷歌密集哈希，但仍需要29个小时才能完成并使用我的64 GiB服务器的8.5 GiB内存。
请问您有什么建议吗？我是C ++的新手。如果我想用更快的东西替换矢量，那会是什么？
#include <string>
#include <algorithm>
#include <tr1/unordered_map>
#include <iterator>
#include <sstream>
#include <cstring>
#include <iomanip>
#include <fstream>
#include <vector>
#include <iterator>
#include <time.h>
#include <iostream>
#include <iostream>
#include <sparsehash/dense_hash_map>
#include <stdio.h>
#include <string.h>
using google::dense_hash_map;
using std::tr1::hash;
using namespace std;
using std::string;
bool ProcessInput(const string& inChar, vector<string> *invector);
void Processmax( dense_hash_map < string, int>* ins, vector<int> *inc, vector<string> *outs, vector<int> *outc);
int main()
{
time_t start, stop;
time(&start);
ofstream finall;
vector<int> usrsc,artc,tmusrc,tmart2c,atrsc,tmartc;
vector<string> tmart,tmusr,tmart2;
vector< vector<string> > usrlist,artlist;
string x1,x2;
ifstream ifTraceFile;
bool f,f2;
dense_hash_map < string, int > a;
dense_hash_map < string, int > u;
a.set_empty_key(string());
u.set_empty_key(string());
int kl=0;
ifTraceFile.open ("data2.tr", std::ifstream::in);
while (ifTraceFile.good ())
{
ifTraceFile>>x1>> x2;
if (kl==0)
{
a.insert(make_pair(x1,0));
u.insert(make_pair(x2,0));
usrlist.push_back((vector<string>()));
usrlist[0].push_back(x1);
artlist.push_back((vector<string>()));
artlist[0].push_back(x2);
usrsc.push_back(1);
artc.push_back(1);
atrsc.push_back(1);
}
else
{
dense_hash_map < string, int>::iterator itn;
itn=a.find(x1);
if (itn == a.end())
{
a.insert(make_pair(x1,(artlist.size())));
artlist.push_back((vector<string>()));
artlist[(artlist.size()-1)].push_back(x2);
artc.push_back(1);
atrsc.push_back(1);
}
else
{
f=ProcessInput(x2, &artlist[itn->second]);
if(f)
{
artlist[itn->second].push_back(x2);
atrsc[itn->second]+=1;
artc[itn->second]+=1;
}
else
atrsc[itn->second]+=1;
}
dense_hash_map < string, int>::iterator its;
its=u.find(x2);
if (its == u.end())
{
u.insert(make_pair(x2,(usrlist.size())));
usrlist.push_back((vector<string>()));
usrlist[(usrlist.size()-1)].push_back(x1);
usrsc.push_back(1);
}
else
{
f2=ProcessInput(x1, &usrlist[its->second]);
if(f2)
{
usrlist[its->second].push_back(x1);
usrsc[its->second]+=1;
}
}
}
kl++;
}
ifTraceFile.close();
Processmax(&a, &artc, &tmart, &tmartc);
Processmax(&a, &atrsc, &tmart2 ,&tmart2c);
Processmax(&u, &usrsc ,&tmusr, &tmusrc);
int width=15;
cout <<"article has Max. review by users Top 1: "<<tmart.at(0)<<'\t'<<tmartc.at(0)<<endl;
cout <<"article has Max. review by users Top 2: "<<tmart.at(1)<<'\t'<<tmartc.at(1)<<endl;
cout <<"article has Max. review by users Top 3: "<<tmart.at(2)<<'\t'<<tmartc.at(2)<<endl;
cout <<endl;
cout <<"article has Max. review Top 1: "<<tmart2.at(0)<<'\t'<<tmart2c.at(0)<<endl;
cout <<"article has Max. review Top 2: "<<tmart2.at(1)<<'\t'<<tmart2c.at(1)<<endl;
cout <<"article has Max. review Top 3: "<<tmart2.at(2)<<'\t'<<tmart2c.at(2)<<endl;
cout <<endl;
cout <<"user who edited most articles Top 1: "<<tmusr.at(0)<<'\t'<<tmusrc.at(0)<<endl;
cout <<"user who edited most articles Top 2: "<<tmusr.at(1)<<'\t'<<tmusrc.at(1)<<endl;
cout <<"user who edited most articles Top 3: "<<tmusr.at(2)<<'\t'<<tmusrc.at(2)<<endl;
finall.open ("results");
finall << "Q1 results:"<<endl;;
finall <<"article has Max. review Top 1: "<<setw(width)<<tmart2.at(0)<<setw(width)<<tmart2c.at(0)<<endl;
finall <<"article has Max. review Top 2: "<<setw(width)<<tmart2.at(1)<<setw(width)<<tmart2c.at(1)<<endl;
finall <<"article has Max. review Top 3: "<<setw(width)<<tmart2.at(2)<<setw(width)<<tmart2c.at(2)<<endl;
finall<<endl;
finall<<"article has Max. review by users Top 1: "<<setw(width)<<tmart.at(0)<<setw(width)<<tmartc.at(0)<<endl;
finall <<"article has Max. review by users Top 2: "<<setw(width)<<tmart.at(1)<<setw(width)<<tmartc.at(1)<<endl;
finall <<"article has Max. review by users Top 3: "<<setw(width)<<tmart.at(2)<<setw(width)<<tmartc.at(2)<<endl;
finall<<endl;
finall <<"user edited most articles Top 1: "<<setw(width)<<tmusr.at(0)<<setw(width-5)<<tmusrc.at(0)<<endl;
finall <<"user edited most articles Top 2: "<<setw(width)<<tmusr.at(1)<<setw(width-5)<<tmusrc.at(1)<<endl;
finall <<"user edited most articles Top 3: "<<setw(width)<<tmusr.at(2)<<setw(width-5)<<tmusrc.at(2)<<endl;
finall.close ();
time(&stop);
cout<<"Finished in about "<< difftime(stop, start)<< " seconds"<<endl;
return 0;
}
void Processmax( dense_hash_map< string,int >* ins, vector<int> *inc, vector<string> *outs, vector<int> *outc)
{
int index=0;
int l=0;
dense_hash_map < string, int>:: iterator iti;
string value;
while(l!=4)
{
vector<int>::iterator it=max_element(inc->begin(), inc->end());
index = distance(inc->begin(), it);
for (iti = ins->begin(); iti != ins->end(); ++iti)
{
if (iti->second == index)
{
value = iti->first;
break;
}
}
outs->push_back(value);
outc->push_back(inc->at(index));
inc->at(index)=0;
l++;
}
}
bool ProcessInput(const string& inChar, vector<string> *invector)
{
bool index=true;
vector<string>::iterator it=find(invector->begin(), invector->end(), inChar);
if (it!=invector->end())
index=false;
return index;
}
答案 0 :(得分：3)
根据您正在打印的数据判断，您尝试仅列出多个类别中的前三个用户。您只需要存储每个类别中当前的前三个用户，而不是存储所有数据。当新记录到达时，您可以确定它是否替换任何类别中的前三项中的任何一项，如果是，则安排新数据替换相应的旧数据。如果新记录“无趣”，则忽略它。使有趣用户的数量成为计算的参数;求解前N的一般情况，然后将N设为3。
这会将您的存储空间限制在几KiB的最大值。您还可以使用更小的数据结构进行操作，因此它们的速度会更快。您的处理时间应该缩短到读取该文件大小所需的时间，这不是29小时。
答案 1 :(得分：2)
您可能需要执行以下几个简单步骤：
要阅读的一些链接：
http://www.sgi.com/tech/stl/complexity.html
答案 2 :(得分：1)
谢谢你的帮助。我现在可以在10分钟内得到结果。仅!!!!!!!!!
unordered_map < string, unordered_set <string> > a;
unordered_map < string, unordered_set <string> > u;
unordered_map < string, int > artc,usrc,artac;
.....
....
if (true)
{
a[x1].insert(x2);
u[x2].insert(x1);
artc[x1]=a[x1].size();
usrc[x2]=u[x2].size();
artac[x1]++;
}
unordered_map比谷歌密集哈希快100％，并且比谷歌密集的内存少了30％。