Question

我有一个很大的txt文件（100MB，23百万行），我想逐行打开它，并像linux中的GNU shuf命令一样将其洗牌。我在Windows平台上工作，我安装了Visual Studio 2015并开始用c ++编程。我第一次尝试使用我的旧c ++代码，但它太慢了，我切换到boost库。我不得不承认，它真的很快，但我不知道将结果放入数组并将它们混洗（数组必须保持100.000.000索引）。

这就是我尝试

#include <boost/iostreams/device/mapped_file.hpp> // for mmap
#include <algorithm>  // for std::find
#include <iostream>   // for std::cout
#include <cstring>

#include <fstream>
#include <sstream>
#include <string>

int main()
{
    boost::iostreams::mapped_file mmap("input.txt", boost::iostreams::mapped_file::readonly);
    auto f = mmap.const_data();
    auto l = f + mmap.size();

    uintmax_t m_numLines = 0;
    int inc1 = 0;

    char ** ip = NULL;

    boost::array<char, sizeof(int)> send_buf; <-- error here
    /*
    Severity    Code    Description Project File    Line    Suppression State
    Error (active)      namespace "boost" has no member "array" hshuffle    c:\path_to_the\main.cpp 21  
    Severity    Code    Description Project File    Line    Suppression State
    Error (active)      type name is not allowed    hshuffle    c:\path_to_the\main.cpp 21  
    Severity    Code    Description Project File    Line    Suppression State
    Error (active)      identifier "send_buf" is undefined  hshuffle    c:\path_to_the\main.cpp 21  
    Severity    Code    Description Project File    Line    Suppression State
    Error (active)      a value of type "const char *" cannot be assigned to an entity of type "char *" hshuffle    c:\path_to_the\main.cpp 29  
    */

    while (f && f != l)
    {
        if ((f = static_cast<const char*>(memchr(f, '\n', l - f))))
        {
            if ((m_numLines % 1000000) == 0)
            {
                ip[m_numLines] = l;
                std::cout << m_numLines << "\n";
            }


            m_numLines++, f++;
        }
    }

    std::cout << "m_numLines = " << m_numLines << "\n";




    printf("endfille\n");

    char a;
    std::cin >> a;
}

OLD C ++程序

puts("reading ips file [./i]");

if((fp=fopen("i","r")) == NULL)
{ 
   printf("FATAL: Cant find i\n");
   return -1;
}

int increment_ips = 0;
indIP = 0;
while (fgets(nutt,2024,fp))
{
    while (t = strchr (nutt,'\n'))
        *t = ' ';

    temp = strtok (nutt, " ");

    if (temp != NULL) {
        string = strdup (temp);
        indIP++;

        while (temp = strtok (NULL, " "))
        {
            indIP++;
        }
    }

    increment_ips++;
}
fclose(fp);




if((fp=fopen("i","r")) == NULL)
{ 
   printf("FATAL: Cant find i\n");
   return -1;
}

increment_ips = 0;
ip = new char*[indIP];
indIP = 0;

while (fgets(nutt,2024,fp))
{
    while (t = strchr (nutt,'\n'))
        *t = ' ';

    temp = strtok (nutt, " ");

    if (temp != NULL) {
        string = strdup (temp);     
        ip[indIP++]=string;

        while (temp = strtok (NULL, " "))
        {
            string = strdup (temp);

            ip[indIP++]=string;
        }
    }

    increment_ips++;
}
fclose(fp);

// shuffle
printf("Loaded [%d] ips\n",increment_ips);

puts("Shuffeling ips");
srand(time(NULL));
for(int i = 0; i <= increment_ips; i++)
{
    int randnum = rand() % increment_ips + 1;
    char* tempval;
    tempval = ip[i];

    ip[i] = ip[randnum];
    ip[randnum] = tempval;
}
puts("Shuffeled");

任何解决方案？我预感boost因此它非常快。

感谢。

Answer 1

＆＃34; old＆＃34;程序读取输入文件两次，第一次计算空间separeted单词（不是行，似乎）第二次实际存储数据在数组中。使用std::vector std::string，无需预先知道元素的确切数量，可以预留一些空间并让内存管理到标准库。

从C ++ 11开始，它也可以使用std::shuffle来完成OP的需要。但是，很难想象对于如此大的数组（数百万个元素）的Fisher-Yates（或Knuth）混洗算法的缓存友好实现。

我不知道如何将结果放入数组并随机播放

可能的解决方案（没有Boost）可能是：

#include <iostream>
#include <fstream>
#include <vector>
#include <string>
#include <algorithm>
#include <random>

using std::string;
using std::vector;
using std::cout;

int main() {
    // initialize random number generator
    std::random_device rd;
    std::mt19937 g(rd());

    // open input file  
    string file_name{"input.txt"};
    std::ifstream in_file{file_name};
    if ( !in_file ) {
        std::cerr << "Error: Failed to open file \"" << file_name << "\"\n";
        return -1;
    }

    vector<string> words;
    // if you want to avoid too many reallocations:
    const int expected = 100000000;
    words.reserve(expected);

    string word;
    while ( in_file >> word ) {
        words.push_back(word);
    }

    std::cout << "Number of elements read: " << words.size() << '\n';
    std::cout << "Beginning shuffle..." << std::endl;

    std::shuffle(words.begin(),words.end(),g);

    std::cout << "Shuffle done." << std::endl;

    // do whatever you need to do with the shuffled vector...

    return 0;
}

阅读文本文件并随机播放

1 个答案: