MPI-IO 写入磁盘时存在MPI_File_write_at_all或MPI_File_write

问：

我正在尝试使用 MPI-IO 从 C++ 中的缓冲区将数据写入磁盘。

为了简洁和 MWE，每个等级在其缓冲区中保留 2 个整数。我希望排名 0 首先将其 2 个元素写入文件中，占据 2 个整数，然后排名第 1 将其 2 个元素写入文件中，然后占据文件中接下来的 2 个整数。

我发现我可以通过两种方式做到这一点（我施加的一个硬性限制是我使用集体调用）：

设置文件视图（每个进程不同）和使用MPI_File_write_all(...)
从一开始就使用MPI_File_write_at_all(...)

在我的 MWE 中，我尝试了这两种方法。

我首先介绍 2）并展示我得到的东西：

方法二）：写

#include <iostream>
#include <complex>

#include <mpi.h>


int main(int argc, char** argv) {

    MPI_Init(&argc, &argv);

    int rank; // the "ID" of this MPI process 
    int size; // total no of MPI processes

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    int * buffer_for_Psi_grid_i_slices = new int [2];

    buffer_for_Psi_grid_i_slices[0] = rank + 3; // rank 0 has 3, rank 1 has 4
    buffer_for_Psi_grid_i_slices[1] = rank + 5; // rank 0 has 5, rank 1 has 6
    std::cout << "rank " << rank << " has Psi[0] = " << buffer_for_Psi_grid_i_slices[0] << " and Psi[1] = " << buffer_for_Psi_grid_i_slices[1] << std::endl;

    MPI_Status status;
    MPI_File fh;
    int MPI_error_value;

    MPI_error_value = MPI_File_open(MPI_COMM_WORLD,    "Psi_grid_from_all_ranks.dat",   (MPI_MODE_CREATE | MPI_MODE_WRONLY),   MPI_INFO_NULL,      &fh);
    if (MPI_error_value != MPI_SUCCESS) {
        std::cout << "rank " << rank << ": ERROR in ``MPI_File_open`` with value = " << MPI_error_value << std::endl;
    }
    std::cout << "rank " << rank << ": we opened the file!" << std::endl;

    MPI_Offset offset = MPI_Offset(rank) * 2; // in which units is this? A: in ``etype`` units
    std::cout << "rank " << rank << ": offset = " << offset << std::endl;
    
    MPI_error_value = MPI_File_write_at_all(fh, offset, &buffer_for_Psi_grid_i_slices[0], 2, MPI_INT, &status);

    if (MPI_error_value != MPI_SUCCESS) {
        std::cout << "rank " << rank << ": ERROR in ``MPI_File_write_all`` with value = " << MPI_error_value << std::endl;
    }
    std::cout << "rank " << rank << ": we wrote_all in the file!" << std::endl;

    MPI_error_value = MPI_Barrier(MPI_COMM_WORLD); // TODO: needed here or not ?
    
    if (MPI_error_value != MPI_SUCCESS) {
        std::cout << "rank " << rank << ": ERROR in ``MPI_Barrier`` with value = " << MPI_error_value << std::endl;
    }    
    std::cout << "rank " << rank << ": we applied the BARRIER!" << std::endl;

    MPI_error_value = MPI_File_close(&fh);
    
    if (MPI_error_value != MPI_SUCCESS) {
        std::cout << "rank " << rank << ": ERROR in ``MPI_File_close`` with value = " << MPI_error_value << std::endl;
    }

    MPI_Finalize();
    return 0;
}

这将生成一个名为 which 的文件，当检查输出时：Psi_grid_from_all_ranks.dat$ od -i Psi_grid_from_all_ranks.dat

0000000      262147      393216           0
0000012

我得到：$ od -d Psi_grid_from_all_ranks.dat

0000000     3     4     0     6     0
0000012

相当接近，但现在我想要的。

我期望获得线性文件为：3 5 4 6。这就是我的目标。

问：我如何使用上述方法 2）获得它？

为了完整起见，我尝试使用方法 1）获取此输出。

虽然我确实从中获得了预期的输出，但我无法在另一个 .cpp 中读取文件的内容，我详细介绍了：od -i Psi_grid_from_all_ranks.dat

方法一）：写作：

#include <iostream>
#include <complex>

#include <mpi.h>

int main(int argc, char** argv) {

    MPI_Init(&argc, &argv);

    int rank; // the "ID" of this MPI process 
    int size; // total no of MPI processes

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    int * buffer_for_Psi_grid_i_slices = new int [2];

    buffer_for_Psi_grid_i_slices[0] = rank + 3; // rank 0 has 3, rank 1 has 4
    buffer_for_Psi_grid_i_slices[1] = rank + 5; // rank 0 has 5, rank 1 has 6
    std::cout << "rank " << rank << " has Psi[0] = " << buffer_for_Psi_grid_i_slices[0] << " and Psi[1] = " << buffer_for_Psi_grid_i_slices[1] << std::endl;

    MPI_Status status;
    MPI_File fh;
    int MPI_error_value;

    MPI_error_value = MPI_File_open(MPI_COMM_WORLD,    "Psi_grid_from_all_ranks.dat",   (MPI_MODE_CREATE | MPI_MODE_WRONLY),   MPI_INFO_NULL,      &fh);
    if (MPI_error_value != MPI_SUCCESS) {
        std::cout << "rank " << rank << ": ERROR in ``MPI_File_open`` with value = " << MPI_error_value << std::endl;
    }
    std::cout << "rank " << rank << ": we opened the file!" << std::endl;

    // disp shall be in BYTES
    MPI_Offset disp = MPI_Offset(rank) * (2) * sizeof(int);

    MPI_error_value = MPI_File_set_view(fh,        disp,         MPI_INT,  MPI_INT,    "native",        MPI_INFO_NULL);

    if (MPI_error_value != MPI_SUCCESS) {
        std::cout << "rank " << rank << ": ERROR in ``MPI_File_set_view`` with value = " << MPI_error_value << std::endl;
    }
    std::cout << "rank " << rank << ": we set the view on the file!" << std::endl;

    MPI_error_value = MPI_File_write_all(fh,           &buffer_for_Psi_grid_i_slices[0],          (2),                                 MPI_INT,         &status);

    MPI_error_value = MPI_Barrier(MPI_COMM_WORLD); // TODO: needed here or not ?

    MPI_error_value = MPI_File_close(&fh);
    
    if (MPI_error_value != MPI_SUCCESS) {
        std::cout << "rank " << rank << ": ERROR in ``MPI_File_close`` with value = " << MPI_error_value << std::endl;
    }

    MPI_Finalize();
    return 0;
}

方法1）：读取和失败：

#include <iostream>
#include <complex>

#include <mpi.h>



int main(int argc, char** argv) {


    MPI_Init(&argc, &argv);

    int rank; // the "ID" of this MPI process 
    int size; // total no of MPI processes

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    int * buffer_for_Psi_grid_i_slices = new int [2];

    MPI_Status status;
    MPI_File fh;
    int MPI_error_value;

    MPI_error_value = MPI_File_open(MPI_COMM_WORLD,    "Psi_grid_from_all_ranks.dat",   (MPI_MODE_CREATE | MPI_MODE_WRONLY),   MPI_INFO_NULL,      &fh);
    if (MPI_error_value != MPI_SUCCESS) {
        std::cout << "rank " << rank << ": ERROR in ``MPI_File_open`` with value = " << MPI_error_value << std::endl;
    }
    std::cout << "rank " << rank << ": we opened the file!" << std::endl;

    MPI_Offset disp = MPI_Offset(rank) * (2) * sizeof(int);

    MPI_error_value = MPI_File_set_view(fh,        disp,        MPI_INT,  MPI_INT,    "native",  MPI_INFO_NULL);

    
    MPI_error_value = MPI_File_read_all(        fh,        &buffer_for_Psi_grid_i_slices[0],  (2),   MPI_INT,       &status);

    for (int i = 0; i <= 1; i++) {
        std::cout << "rank " << rank << ": Psi_grid[i] for i = " << i << " = " << buffer_for_Psi_grid_i_slices[i] << std::endl;
    }

    MPI_error_value = MPI_File_close(&fh);
    
    if (MPI_error_value != MPI_SUCCESS) {
        std::cout << "rank " << rank << ": ERROR in ``MPI_File_close`` with value = " << MPI_error_value << std::endl;
    }

    MPI_Finalize();
    return 0;

}

这将产生以下输出：

rank 0: we opened the file!
rank 0: Psi_grid[i] for i = 0 = 341273760
rank 0: Psi_grid[i] for i = 1 = 32595
rank 1: we opened the file!
rank 1: Psi_grid[i] for i = 0 = -27030512
rank 1: Psi_grid[i] for i = 1 = 32541

完全错误！即使在终端中，方法 1）的输出上的命令也从写入逻辑产生正确的结果：$ od -i Psi_grid_from_all_ranks.dat

0000000           3           5           4           6
0000020

编辑：

在您的帮助下，我设法制作了该方法，以便能够生成一个文件，当检查时产生：MPI_File_write_at_all(...)od -d

od -d Psi_grid_from_all_ranks.dat 
0000000     3     0     5     0     4     0     6     0
0000020

我现在发布读数，它仍然没有用我期望的值填充缓冲区。.cppMPI_File_read_at_all()

#include <iostream>
#include <complex>

#include <mpi.h>
 
int main(int argc, char** argv) {


    MPI_Init(&argc, &argv);

    int rank; // the "ID" of this MPI process 
    int size; // total no of MPI processes

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    int * buffer_for_Psi_grid_i_slices = new int [2];

    MPI_Status status;
    MPI_File fh;
    int MPI_error_value;


    MPI_error_value = MPI_File_open(MPI_COMM_WORLD,    "Psi_grid_from_all_ranks.dat",   (MPI_MODE_CREATE | MPI_MODE_WRONLY),   MPI_INFO_NULL,      &fh);
    if (MPI_error_value != MPI_SUCCESS) {
        std::cout << "rank " << rank << ": ERROR in ``MPI_File_open`` with value = " << MPI_error_value << std::endl;
    }
    std::cout << "rank " << rank << ": we opened the file!" << std::endl;



    
    MPI_Offset offset = MPI_Offset(rank) * 2 * sizeof(int);

    MPI_error_value = MPI_File_read_at_all(fh, offset, &buffer_for_Psi_grid_i_slices[0], 2, MPI_INT, &status);
    

    MPI_Barrier(MPI_COMM_WORLD);

    for (int i = 0; i <= 1; i++) {
        std::cout << "rank " << rank << ": Psi_grid[i] for i = " << i << " = " << buffer_for_Psi_grid_i_slices[i] << std::endl;
    }

    MPI_error_value = MPI_File_close(&fh);
    
    if (MPI_error_value != MPI_SUCCESS) {
        std::cout << "rank " << rank << ": ERROR in ``MPI_File_close`` with value = " << MPI_error_value << std::endl;
    }

    MPI_Finalize();
    return 0;

}

这将产生：

mpirun -np 2 debug_MPI_read.x
rank 0: we opened the file!
rank 0: Psi_grid[i] for i = 0 = 1031781536
rank 0: Psi_grid[i] for i = 1 = 32566
rank 1: we opened the file!
rank 1: Psi_grid[i] for i = 0 = 1931947024
rank 1: Psi_grid[i] for i = 1 = 32669

C++ 文件并行处理 MPI 分布式计算

@GillesGouaillardet，感谢您的帮助和耐心。我还是不明白。如果我使用，我的方法 2），我不是明确告诉计算机使用参数在文件中的某些特定点写入数据吗？使用时需要设置视图吗？如果我设置了视图，那么这个东西没有指定相对于视图的点？在我上面的方法 2）中，我想在文件中指定一个相对于第 0 个字节的位置。MPI_File_write_at_all()offsetwrite_atatat

0赞 velenos14 9/4/2023

@GillesGouaillardet，我想我理解了你说的话，我得到了方法 2）（即我发布的第一个代码片段）。请耐心等待我一会儿。

0赞 velenos14 9/4/2023

@GillesGouaillardet，虽然在检查文件时可以进行写入，但我无法读取文件的内容。如果您还有时间，请参阅问题末尾的部分。谢谢MPI_File_write_at()od.cppEDIT

1赞 Gilles Gouaillardet 9/4/2023

不要打开你的文件，只在你稍后阅读时才写！

答： 暂无答案

上一个：找不到 freeglut.dll，如教程中建议的那样

下一个：C++ 从文件中读取数据并转换为类的模板类型

MPI-IO 写入磁盘时存在MPI_File_write_at_all或MPI_File_write_all误解？

MPI-IO writing to disk with MPI_File_write_at_all or MPI_File_write_all misunderstandings?

评论