Question

大家好我正在学习MPI编程，我遇到了这个问题。假设我有一个带有文本的3 000 000行/行的.txt文件（它是一个字典）所以它将是一个数组字符串，如何将它们分块以便由4个处理器处理？即我想让处理器0处理1-750 000行的处理，处理器1处理750 001-1 500 000等等。然后，每个过程都会获取数据的一部分并将其放入一个函数中。 find_some_word。

现在我得到了这个，但我不知道如何从我的大块中获得一个单词。这部分在程序中评论，如何做到

#include <stdio.h>
#include <mpi.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>
#include <stdbool.h>
#include <conio.h>

bool if_anagram(char *a, char *b)
{

    int dl1 = strlen(a), dl2 = strlen(b);


    if(dl1!=dl2) return false;
    if(strcmp(a,b) == 0) return false;

    int licz[0x100]={}; 

    for(int i=0;i<dl1;i++)    
        licz[(unsigned char) a[i]]++; 
    for(int i=0;i<dl1;i++)
        licz[(unsigned char) b[i]]--;       
    for(int i=0;i<256;i++)      
        if(licz[i]!=0) 
            return false; 

  return true; 
} 

void parprocess(MPI_File *in, char *a, const int rank, const int size, const int overlap) {

    MPI_Offset globalstart;
    int mysize;
    char *chunk;

    /* read in relevant chunk of file into "chunk",
     * which starts at location in the file globalstart
     * and has size mysize 
     */
    {
        MPI_Offset globalend;
        MPI_Offset filesize;

        /* figure out who reads what */
        MPI_File_get_size(*in, &filesize);
        filesize--;  /* get rid of text file eof */
        mysize = filesize/size;
        globalstart = rank * mysize;
        globalend   = globalstart + mysize - 1;
        if (rank == size-1) globalend = filesize-1;

        /* add overlap to the end of everyone's chunk except last proc... */
        if (rank != size-1)
            globalend += overlap;

        mysize =  globalend - globalstart + 1;

        /* allocate memory */
        chunk = malloc( (mysize + 1)*sizeof(char));

        /* everyone reads in their part */
        MPI_File_read_at_all(*in, globalstart, chunk, mysize, MPI_CHAR, MPI_STATUS_IGNORE);
        chunk[mysize] = '\0';
    }


    /*
     * everyone calculate what their start and end *really* are by going 
     * from the first newline after start to the first newline after the
     * overlap region starts (eg, after end - overlap + 1)
     */

    int locstart=0, locend=mysize-1;
    if (rank != 0) {
        while(chunk[locstart] != '\n') locstart++;
        locstart++;
    }
    if (rank != size-1) {
        locend-=overlap;
        while(chunk[locend] != '\n') locend++;
    }
    mysize = locend-locstart+1;

    /* "Process" our chunk by replacing non-space characters with '1' for
     * rank 1, '2' for rank 2, etc... 
     */

    char b[101];
    for (int i=locstart; i<=locend; i++) {
                                                // THIS PART
        //fscanf (chunk,"%s",b);                // how get a single word from chunk
        //if(if_anagram(a,b)){
            //printf("Word %s is anagram!\n", b);

    //  It is orginal part  
    //    char c = chunk[i];
     //   chunk[i] = ( isspace(c) ? c : '1' + (char)rank );
    //    printf("Wyraz %d\n", rank);
        }
    //}

}

int main(int argc, char **argv) {

    MPI_File in;
    int rank, size;
    int ierr;
    const int overlap = 100;
    char a[101];


    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    if (argc != 2) {
        if (rank == 0) fprintf(stderr, "Usage: %s infilename\n", argv[0]);
        MPI_Finalize();
        exit(1);
    }

    ierr = MPI_File_open(MPI_COMM_WORLD, argv[1], MPI_MODE_RDONLY, MPI_INFO_NULL, &in);
    if (ierr) {
        if (rank == 0) fprintf(stderr, "%s: Couldn't open file %s\n", argv[0], argv[1]);
        MPI_Finalize();
        exit(2);
    }

        printf("Put word: ");
        scanf ("%s", a);
    printf("\n");

    parprocess(&in, a, rank, size, overlap);


    MPI_File_close(&in);
    MPI_Finalize();
    return 0;
}

Answer 1

关于I / O的好教程： https://www.tacc.utexas.edu/documents/13601/900558/MPI-IO-Final.pdf/eea9d7d3-4b81-471c-b244-41498070e35d

然后必须明白MPI-IO无法读取文件的内容:-)。它只是＆＃34;允许进程跳过文件的一部分（以字节为单位），如果要跳过给定数量的行并且每行具有不同的大小，除了读取整个文件以查找＆＃39;之外，没有其他解决方案。 \ n＆＃39 ;. MPI-IO不是魔术师......

最好的解决方案是创建（一次）一个标题，描述哪些行完成。您可以获得每组10＆＃000; 000行的字节数，然后将其作为文件中的标题。你可以跳过一些街区。显然这样做，你必须阅读整个文件...

C - MPI文本文件打开

1 个答案: