Question

我遇到了一些与我试图做的并行I / O有关的问题。代码的输出响应因运行而异。本质上，我正在创建一个生产者/消费者程序，它接收一个目录作为输入，并将该目录完全复制到某个目的地。一个线程将各种文件描述符放在缓冲区（队列或任何其他数据结构）中。代码应该如下运行。

./ task1 threadcount srcdir destdir

基本上，问题是随机的，所以我认为它可能与某种竞争条件有关。我会说每4次一次就能完成这项任务。我只需要一些新鲜的眼睛来抓住我的错误。

标题

注意：下面发布的代码风格非常非常差，并且会得到纠正。我为此道歉。

#include <queue>
#include <pthread.h>
#include <sys/types.h>
#include <dirent.h>
#include <fcntl.h>
#include <unistd.h>
#include <string>
#include <iostream>
#include <unistd.h>
#include <vector>
#include <fstream>
#include <mutex>
#include <thread> 
using namespace std;
const int isFile = 0x8;
/* The task for the consumer thread to execute. */ 
struct task {
    int infd;
    int outfd;
    bool end;
    string filename;
    task& operator =(const task& tsk)
    {
        infd = tsk.infd;
        outfd = tsk.outfd;
        end = tsk.end;
        return *this;
    }

 };
/*The argument for the producer thread */
struct arguments {
    vector<string> files;
    string src_dir;
    string dest_dir;
};
mutex output_lock;
mutex queue_lock;
queue<task> filequeue;

/* Get a list of all the files in a directory */
int get_files(vector<string>& files, string dir_nm ) {
    int file_count = 0;
    DIR* directory;
    struct dirent* dir_str;
    directory = opendir (dir_nm.c_str());
    if(directory == NULL) {
        cout << "Error with directory." << endl;
        return -1; 
    }
    while ((dir_str = readdir(directory)) != NULL) {
        if (dir_str->d_type == isFile) {
            string file_nm(dir_str->d_name);
            files.push_back(file_nm);
        }
    }
    closedir(directory);
    return file_count;
}

void copy(task tsk_to_copy) {
    int src_file = tsk_to_copy.infd;
    int dest_file = tsk_to_copy.outfd;
    char buf[8000];
    if(src_file < 0 || dest_file < 0) {
        cout << "Error with files." << endl;
        return;
    }
    while (true) {
        cout << "The error occurs here. " << endl;
        long int to_write = read(src_file, &buf[0], sizeof(buf));
        if(!to_write) break;
        write(dest_file, &buf[0], to_write);
    }
    cout << "The error occurs here " << endl;
    close(src_file);
    close(dest_file);
}

void create_task(string src, string dest, task& new_task) {
    unlink(dest.c_str());
    int src_file = open(src.c_str(), O_RDONLY);
    int dest_file = creat(dest.c_str(), 0700);
    new_task.infd = src_file;
    new_task.outfd = dest_file;
    new_task.end = false;
}

string pathname(string file_name, string dir) {
    return dir + file_name; 
}

void pdc_thd(void* arg) {
    /*Get arguments from struct */
    arguments* args = (arguments*) arg;
    string src = args->src_dir;
    string dest = args->dest_dir;
    vector<string> files = args->files;
    /* Start the buffer */
    for(int i = 0; i < files.size(); ++i) {
        task new_task;
        create_task(pathname(files[i], src), pathname(files[i], dest), new_task);
        new_task.filename = files[i];
        queue_lock.lock();
        filequeue.push(new_task);
        queue_lock.unlock();
        output_lock.lock();
        cout << "Written file " << files[i] << " to the buffer." << endl;
        output_lock.unlock();
    }
    /*Put a NULL file at the end for the threads to join. */
    queue_lock.lock();
    task final_task;
    final_task.infd = 0;
    final_task.outfd = 0;
    final_task.end = true;
    filequeue.push(final_task);
    queue_lock.unlock();

}


void cons_thd(void* arg) {

    while(true) {
        if(filequeue.empty()) {
            usleep(1);
        }
        else {
            queue_lock.lock();
            task job = filequeue.front();
            if(job.end) {
                queue_lock.unlock();
                break;
            }
            filequeue.pop();
            queue_lock.unlock();
            copy(job);
            output_lock.lock();
            cout << "Finished writing file: " << job.filename << endl;
            output_lock.unlock();
        }
    }
}

主

#include "task1.h"

int main(int argc, char* argv[]) {

    /*Check for valid argument count. */
    if(argc != 4) {
        cout << "Argument error." << endl;
        return -1;
    }
    pthread_attr_t attr;
    void* status;
    pthread_attr_init(&attr);
    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
    /* Get the source and destination directory */
    int total_threads = 1 + atoi(argv[1]);
    string src_dir_nm(argv[2]);
    string dest_dir_nm(argv[3]);
    vector<string> files;
    get_files(files, src_dir_nm);
    thread myThreads[total_threads];
    arguments arg;
    arg.files = files;
    arg.src_dir = src_dir_nm;
    arg.dest_dir = dest_dir_nm;
    myThreads[0] = thread(pdc_thd, &arg);
    for(int i = 1; i < total_threads; ++i) {
        myThreads[i] = thread(cons_thd, &arg);
    }
     for (int i = 0; i < total_threads; i++){
        myThreads[i].join();
    }
    return 0;
}

pthread和标准C ++线程之间存在某种混乱。它们几乎可以在两分钟的编辑中互换。我相信错误的一个地方是写/读，但我不是100％肯定。

Answer 1

您的消费者线程正在调用filequeue.empty而不保留队列锁定。

这可能导致多个线程同时访问filequeue。

pthread segfault做并行i / o

1 个答案: