Question

我有以下程序，该程序可以使用openmpi和mpich-3.2.1进行编译和运行，但是在mpich-3.3的MPI_Waitany调用中出现了死锁。该程序必须以偶数个级别运行，其中一半通过互连器发送数据到后半部分。与较大的自定义仿真程序相比，这是一个较小的版本，我尝试给出一个最小的示例。

一个奇怪的部分是将nwork变量增加到2会使死锁消失。

#include <cstdio>
#include <mpi.h>
#include <vector>

const int endMsg = -1;
const int endTag = 424242;

class Work
{
public:
    Work(const MPI_Comm& comm, const MPI_Comm& interComm, int tag) :
        comm(comm), interComm(interComm), tag(tag)
    {
        MPI_Comm_rank(comm, &rank);
    }

    void waitPrevSend()
    {
        printf("[work %d] waiting for previous message\n", tag);
        MPI_Wait(&sizeReq, MPI_STATUS_IGNORE);
        MPI_Wait(&dataReq, MPI_STATUS_IGNORE);
        sizeReq = MPI_REQUEST_NULL;
        dataReq = MPI_REQUEST_NULL;
    }

    void workAndSend()
    {
        waitPrevSend();
        printf("[work %d] creating data\n", tag);
        data.resize(tag + 42, tag);
        sizeInBytes = data.size();
        MPI_Issend(&sizeInBytes, 1, MPI_INT, rank, 2*tag+0, interComm, &sizeReq);
        MPI_Issend(data.data(), data.size(), MPI_BYTE, rank, 2*tag+1, interComm, &dataReq);
        printf("[work %d] has sent %d bytes of data\n", tag, sizeInBytes);
    }


    MPI_Request wait()
    {
        MPI_Request req;
        printf("[work %d] posted recv of size\n", tag);
        MPI_Irecv(&sizeInBytes, 1, MPI_INT, rank, 2*tag+0, interComm, &req);
        return req;
    }

    void recv()
    {
        data.resize(sizeInBytes);
        MPI_Recv(data.data(), data.size(), MPI_BYTE, rank, 2*tag+1, interComm, MPI_STATUS_IGNORE);
        printf("[work %d] has recved %d bytes of data\n", tag, sizeInBytes);
    }


    MPI_Comm comm, interComm;
    int rank;
    int tag;


    MPI_Request sizeReq {MPI_REQUEST_NULL}, dataReq {MPI_REQUEST_NULL};
    std::vector<char> data;
    int sizeInBytes;
};

class Master
{
public:
    Master(const MPI_Comm& comm, const MPI_Comm& interComm) :
        comm(comm), interComm(interComm)
    {
        MPI_Comm_rank(comm, &rank);
    }

    void run(std::vector<Work>& work, int niter)
    {
        for (int i = 0; i < niter; ++i)
            for (auto& w : work)
                w.workAndSend();
        sendEndMsg();
    }

    void sendEndMsg()
    {
        MPI_Ssend(&endMsg, 1, MPI_INT, rank, endTag, interComm);
    }

    MPI_Comm comm, interComm;
    int rank;
};

class Slave
{
public:
    Slave(const MPI_Comm& comm, const MPI_Comm& interComm) :
        comm(comm), interComm(interComm)
    {
        MPI_Comm_rank(comm, &rank);
    }

    void run(std::vector<Work>& work)
    {
        std::vector<MPI_Request> reqs;
        for (auto& w : work)
            reqs.push_back(w.wait());
        reqs.push_back(recvEndMsg());

        while (true)
        {
            int id;
            MPI_Status status;
            printf("waiting for one of %d requests to complete\n", (int) reqs.size());
            MPI_Waitany(reqs.size(), reqs.data(), &id, &status);

            if (id == (int) reqs.size() - 1)
            {
                for (auto& req : reqs)
                {
                    if (req != MPI_REQUEST_NULL)
                    {
                        MPI_Cancel(&req);
                        MPI_Request_free(&req);
                    }
                }
                return;
            }
            else
            {
                work[id].recv();
                reqs[id] = work[id].wait();
            }
        }
    }

    MPI_Request recvEndMsg()
    {
        MPI_Request req;
        int msg;
        MPI_Irecv(&msg, 1, MPI_INT, rank, endTag, interComm, &req);
        return req;
    }

    MPI_Comm comm, interComm;
    int rank;
};


int main(int argc, char **argv)
{
    MPI_Init(&argc, &argv);

    int rank;
    int size;
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    if ((size%2) != 0)
        MPI_Abort(MPI_COMM_WORLD, 1);

    MPI_Comm teamComm, interComm;
    int team = rank % 2;

    MPI_Comm_split(MPI_COMM_WORLD, team, rank, &teamComm);

    const int localLeader  = 0;
    const int remoteLeader = team ? 0 : 1;
    const int tag = 42;
    const int nwork = 1;

    MPI_Intercomm_create(teamComm, localLeader, MPI_COMM_WORLD, remoteLeader, tag, &interComm);

    std::vector<Work> work;
    for (int i = 0; i < nwork; ++i)
        work.emplace_back(Work(teamComm, interComm, i));

    if (team == 0)
    {
        Master master(teamComm, interComm);
        master.run(work, 10);
    }
    else
    {
        Slave slave(teamComm, interComm);
        slave.run(work);
    }

    MPI_Comm_free(&interComm);
    MPI_Comm_free(&teamComm);

    MPI_Finalize();
    return 0;
}

运行

mpirun -n 2 -l ./test_intercomm

仅在mpich-3.3中导致死锁。有什么想法吗？

编辑：我还尝试将停止标记减小到建议的较小值，并且保持相同的行为。上面命令的输出是：

[0] [work 0] waiting for previous message
[0] [work 0] creating data
[0] [work 0] has sent 42 bytes of data
[1] [work 0] posted recv of size
[1] waiting for one of 2 requests to complete
[0] [work 0] waiting for previous message

因此，在等待发送请求完成时，等级1在等待中陷入死锁，等级0陷入死锁（第二个，实际数据，其recv仅在通过waitany后才由等级1发布）。在我看来，MPI_Waitany会阻止所有内容。

Answer 1

这是由MPICH v3.3中的错误引起的。它在提交0f7be7196cc05bf0c908761e148628e88d635190中已修复。将修复程序应用于v3.3可解决死锁。

该修补程序包含在版本3.3.1中，因此您应该升级到该版本。

要提供更多背景信息，提交消息将显示：

testany和waitany函数都跳过非活动或NULL请求在递归到设备层之前。但是，方法发现第一个非NULL请求可能会错误地跳过数组中的第一个请求。要修复，我们初始化第一个非NULL 请求到数组（计数）中的无效索引并设置为有效如果找到一个，则稍后索引。

在mpich-3.3中处于死锁状态，但在其他版本中则没有

1 个答案: