MPI学习--非重复的非阻塞通信之非重复的缓冲通信

本文也是通过一个实例来理解总结一下非阻塞缓冲通信模式。

对比MPI_Bsend(), MPI_Isend(), MPI_Ibsend()

三者均不发生阻塞，执行后都会立即返回，且后台有MPI环境实施消息传输。

  • MPI_Bsend()之所以会立即返回，是因为用户已经定义了缓冲区，调用后会将数据复制到缓冲区中，所以不用等接收端接收便可以返回，但是在执行MPI_Buffer_detach()时才最终确保消息清扫干净，完成传输，也就是说消息传输完毕之前，MPI_Buffer_detach()不会返回。

  • MPI_Isend()本身就是非阻塞通信，所以会立即返回，交给MPI环境后台来实施消息传递。但是标准非阻塞通信需要调用MPI_Wait()等机制来等待传输完成。

  • MPI_Ibsend()更像是前两者的结合体，但是要注意的是发送完成并不代表消息传递完成。

下面通过一个例子来看一下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
#include <stdio.h>
#include <stdlib.h>
#include "mpi.h"

#define DATALEN 6553600

int main(int argc, char ** argv)
{
    MPI_Status sstatus, rstatus;
    MPI_Request sreq, rreq;
    int a[DATALEN], b[DATALEN];
    char * buf;
    int i, j, rank, size, errs = 0;
    int sflag, rflag, other;
    int s1, bufsize;

    MPI_Init(&argc, &argv);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    MPI_Pack_size(DATALEN, MPI_INT, MPI_COMM_WORLD, &s1);
    bufsize = MPI_BSEND_OVERHEAD + s1;
    buf = (char *)malloc(bufsize);

    MPI_Buffer_attach(buf, bufsize);

    if (size != 2)
    {
        fprintf(stderr, "number of processes must be 2");
        MPI_Finalize();
        fflush(stderr);
    }

    if (rank == 0)
    {
        other = 1;
    }
    else
    {
        other = 0;
    }

    sflag = 0;
    rflag = 0;
    for (i = 0; i < DATALEN; ++i)
    {
        a[i] = (rank + DATALEN*i) + i;
    }

    if (rank == 0)
    {
        fprintf(stderr, "proc %d: before ibsend...%f\n", rank, MPI_Wtime());
        MPI_Ibsend(a, DATALEN, MPI_INT, other, 111, MPI_COMM_WORLD, &sreq);
        fprintf(stderr, "proc %d: after ibsend...%f\n", rank, MPI_Wtime());

        while (!sflag)
        {
            fprintf(stderr, "proc %d: before Ibsend test, sflag = %d...%f\n", rank, sflag, MPI_Wtime());
            MPI_Test(&sreq, &sflag, &sstatus);
            fprintf(stderr, "proc %d: after Ibsend test, sflag = %d...%f\n", rank, sflag, MPI_Wtime());
        }
    }

    if (rank == 1)
    {
        for (i = 0; i < 9999999; ++i)
        {
            MPI_Wtime();
        }
        fprintf(stderr, "proc %d: before Irecv...%f\n", rank, MPI_Wtime());
        MPI_Irecv(b, DATALEN, MPI_INT, other, 111, MPI_COMM_WORLD, &rreq);
        fprintf(stderr, "proc %d: after Irecv...%f\n", rank, MPI_Wtime());

        while (!rflag)
        {
            fprintf(stderr, "proc %d: before Irecv test, rflag = %d...%f\n", rank, rflag, MPI_Wtime());
            MPI_Test(&rreq, &rflag, &rstatus);
            fprintf(stderr, "proc %d: after Irecv test, rflag = %d...%f\n", rank, rflag, MPI_Wtime());
        }

        fprintf(stderr, "proc: %d, received ...\n", rank);
        for (i = 0; i < 2; ++i)
        {
            fprintf(stderr, "b[%d]=%d\n", i, b[i]);
        }
    }

    fprintf(stderr, "proc: %d before Buffer_detach...%f\n", rank, MPI_Wtime());
    MPI_Buffer_detach(&buf, &bufsize);
    fprintf(stderr, "proc: %d after Buffer_detach...%f\n", rank, MPI_Wtime());
    
    free(buf);
    
    MPI_Finalize();
        
    return 0;
}

执行结果：
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
[zjshao@master 2-2]$ mpicc ibsendrecv.c -o ibsendrecv.x
[zjshao@master 2-2]$ mpiexec -n 2 -host node01 ibsendrecv.x
proc 0: before ibsend...1468070428.066756
proc 0: after ibsend...1468070428.113055
proc 0: before Ibsend test, sflag = 0...1468070428.113082
proc 0: after Ibsend test, sflag = 1...1468070428.113117
proc: 0 before Buffer_detach...1468070428.113122
proc 1: before Irecv...1468070428.559447
proc 1: after Irecv...1468070428.559488
proc 1: before Irecv test, rflag = 0...1468070428.559491
proc 1: after Irecv test, rflag = 0...1468070428.559572
proc 1: before Irecv test, rflag = 0...1468070428.559575
proc 1: after Irecv test, rflag = 0...1468070428.559581
proc 1: before Irecv test, rflag = 0...1468070428.559583
proc 1: after Irecv test, rflag = 0...1468070428.559588
proc 1: before Irecv test, rflag = 0...1468070428.559590
proc 1: after Irecv test, rflag = 0...1468070428.559596
proc 1: before Irecv test, rflag = 0...1468070428.559598
proc 1: after Irecv test, rflag = 0...1468070428.559604
proc 1: before Irecv test, rflag = 0...1468070428.559606
proc 1: after Irecv test, rflag = 0...1468070428.559612
proc 1: before Irecv test, rflag = 0...1468070428.559614
proc 1: after Irecv test, rflag = 0...1468070428.559619
proc 1: before Irecv test, rflag = 0...1468070428.559622
proc 1: after Irecv test, rflag = 0...1468070428.559627
proc 1: before Irecv test, rflag = 0...1468070428.559629
proc 1: after Irecv test, rflag = 0...1468070428.559635
proc 1: before Irecv test, rflag = 0...1468070428.559637
proc 1: after Irecv test, rflag = 0...1468070428.559642
proc 1: before Irecv test, rflag = 0...1468070428.559645
proc 1: after Irecv test, rflag = 0...1468070428.559650
proc 1: before Irecv test, rflag = 0...1468070428.559652
proc 1: after Irecv test, rflag = 0...1468070428.559658
proc 1: before Irecv test, rflag = 0...1468070428.559660
proc 1: after Irecv test, rflag = 0...1468070428.559665
proc 1: before Irecv test, rflag = 0...1468070428.559668
proc 1: after Irecv test, rflag = 1...1468070428.600704
proc: 1, received ...
b[0]=0
b[1]=6553601
proc: 1 before Buffer_detach...1468070428.600724
proc: 0 after Buffer_detach...1468070428.600687
proc: 1 after Buffer_detach...1468070428.600730
[zjshao@master 2-2]$

  1. 从执行的结果可以看出进程0调用MPI_Ibsend()之后立即返回，而且返回之后进行MPI_Test()的竟然是True。这表明缓冲发送已完成，但这并不是消息传递的完成，因为这其实是把数据复制到了缓冲区中。
  2. 之后进程0立即调用MPI_Buffer_detach()却没有返回，所以到等待消息传递完成才行。
  3. 之后进程1开始进行接收消息，调用MPI_Irecv()还是会立即返回，但我们的数据量比较大，消息并不能解析完，这个时候进行检测显示接收没有完成，于是就有了下面这一大串的输出。其实如果我们把这一部分测试的代码替换成其他的代码，就可以体现出非阻塞的优势了。
  4. 数据传输完之后，便可以进行缓冲区的detach了。

Comments

▲