DOLFINx
DOLFINx C++ interface
MPI.h
1// Copyright (C) 2007-2014 Magnus Vikstrøm and Garth N. Wells
2//
3// This file is part of DOLFINx (https://www.fenicsproject.org)
4//
5// SPDX-License-Identifier: LGPL-3.0-or-later
6
7#pragma once
8
9#include <array>
10#include <cassert>
11#include <complex>
12#include <cstdint>
13#include <dolfinx/common/Timer.h>
14#include <dolfinx/common/log.h>
15#include <dolfinx/graph/AdjacencyList.h>
16#include <numeric>
17#include <set>
18#include <type_traits>
19#include <utility>
20#include <vector>
21#include <xtl/xspan.hpp>
22
23#define MPICH_IGNORE_CXX_SEEK 1
24#include <mpi.h>
25
27namespace dolfinx::MPI
28{
29
31enum class tag : int
32{
33 consensus_pcx,
34 consensus_pex
35};
36
39class Comm
40{
41public:
43 explicit Comm(MPI_Comm comm, bool duplicate = true);
44
46 Comm(const Comm& comm) noexcept;
47
49 Comm(Comm&& comm) noexcept;
50
51 // Disable copy assignment operator
52 Comm& operator=(const Comm& comm) = delete;
53
55 Comm& operator=(Comm&& comm) noexcept;
56
58 ~Comm();
59
61 MPI_Comm comm() const noexcept;
62
63private:
64 // MPI communicator
65 MPI_Comm _comm;
66};
67
69int rank(MPI_Comm comm);
70
73int size(MPI_Comm comm);
74
82constexpr std::array<std::int64_t, 2> local_range(int rank, std::int64_t N,
83 int size)
84{
85 assert(rank >= 0);
86 assert(N >= 0);
87 assert(size > 0);
88
89 // Compute number of items per rank and remainder
90 const std::int64_t n = N / size;
91 const std::int64_t r = N % size;
92
93 // Compute local range
94 if (rank < r)
95 return {rank * (n + 1), rank * (n + 1) + n + 1};
96 else
97 return {rank * n + r, rank * n + r + n};
98}
99
106constexpr int index_owner(int size, std::size_t index, std::size_t N)
107{
108 assert(index < N);
109
110 // Compute number of items per rank and remainder
111 const std::size_t n = N / size;
112 const std::size_t r = N % size;
113
114 if (index < r * (n + 1))
115 {
116 // First r ranks own n + 1 indices
117 return index / (n + 1);
118 }
119 else
120 {
121 // Remaining ranks own n indices
122 return r + (index - r * (n + 1)) / n;
123 }
124}
125
130std::array<std::vector<int>, 2> neighbors(MPI_Comm comm);
131
155std::vector<int> compute_graph_edges_pcx(MPI_Comm comm,
156 const xtl::span<const int>& edges);
157
182std::vector<int> compute_graph_edges_nbx(MPI_Comm comm,
183 const xtl::span<const int>& edges);
184
204template <typename T>
205std::pair<std::vector<std::int32_t>, std::vector<T>>
206distribute_to_postoffice(MPI_Comm comm, const xtl::span<const T>& x,
207 std::array<std::int64_t, 2> shape,
208 std::int64_t rank_offset);
209
231template <typename T>
232std::vector<T> distribute_from_postoffice(
233 MPI_Comm comm, const xtl::span<const std::int64_t>& indices,
234 const xtl::span<const T>& x, std::array<std::int64_t, 2> shape,
235 std::int64_t rank_offset);
236
260template <typename T>
261std::vector<T> distribute_data(MPI_Comm comm,
262 const xtl::span<const std::int64_t>& indices,
263 const xtl::span<const T>& x, int shape1);
264
273template <typename T>
275neighbor_all_to_all(MPI_Comm comm, const graph::AdjacencyList<T>& send_data);
276
277template <typename T>
278struct dependent_false : std::false_type
279{
280};
281
283template <typename T>
284constexpr MPI_Datatype mpi_type()
285{
286 if constexpr (std::is_same<T, float>::value)
287 return MPI_FLOAT;
288 else if constexpr (std::is_same<T, double>::value)
289 return MPI_DOUBLE;
290 else if constexpr (std::is_same<T, std::complex<double>>::value)
291 return MPI_C_DOUBLE_COMPLEX;
292 else if constexpr (std::is_same<T, std::complex<float>>::value)
293 return MPI_C_FLOAT_COMPLEX;
294 else if constexpr (std::is_same<T, short int>::value)
295 return MPI_SHORT;
296 else if constexpr (std::is_same<T, int>::value)
297 return MPI_INT;
298 else if constexpr (std::is_same<T, unsigned int>::value)
299 return MPI_UNSIGNED;
300 else if constexpr (std::is_same<T, long int>::value)
301 return MPI_LONG;
302 else if constexpr (std::is_same<T, unsigned long>::value)
303 return MPI_UNSIGNED_LONG;
304 else if constexpr (std::is_same<T, long long>::value)
305 return MPI_LONG_LONG;
306 else if constexpr (std::is_same<T, unsigned long long>::value)
307 return MPI_UNSIGNED_LONG_LONG;
308 else if constexpr (std::is_same<T, bool>::value)
309 return MPI_C_BOOL;
310 else if constexpr (std::is_same<T, std::int8_t>::value)
311 return MPI_INT8_T;
312 else
313 // Issue compile time error
314 static_assert(!std::is_same<T, T>::value);
315}
316
317//---------------------------------------------------------------------------
318template <typename T>
319std::pair<std::vector<std::int32_t>, std::vector<T>>
320distribute_to_postoffice(MPI_Comm comm, const xtl::span<const T>& x,
321 std::array<std::int64_t, 2> shape,
322 std::int64_t rank_offset)
323{
324 const int size = dolfinx::MPI::size(comm);
325 const int rank = dolfinx::MPI::rank(comm);
326 assert(x.size() % shape[1] == 0);
327 const std::int32_t shape0_local = x.size() / shape[1];
328
329 LOG(2) << "Sending data to post offices (distribute_to_postoffice)";
330
331 // Post office ranks will receive data from this rank
332 std::vector<int> row_to_dest(shape0_local);
333 for (std::int32_t i = 0; i < shape0_local; ++i)
334 {
335 int dest = MPI::index_owner(size, i + rank_offset, shape[0]);
336 row_to_dest[i] = dest;
337 }
338
339 // Build list of (dest, positions) for each row that doesn't belong to
340 // this rank, then sort
341 std::vector<std::array<std::int32_t, 2>> dest_to_index;
342 dest_to_index.reserve(shape0_local);
343 for (std::int32_t i = 0; i < shape0_local; ++i)
344 {
345 std::size_t idx = i + rank_offset;
346 if (int dest = MPI::index_owner(size, idx, shape[0]); dest != rank)
347 dest_to_index.push_back({dest, i});
348 }
349 std::sort(dest_to_index.begin(), dest_to_index.end());
350
351 // Build list of neighbour src ranks and count number of items (rows
352 // of x) to receive from each src post office (by neighbourhood rank)
353 std::vector<int> dest;
354 std::vector<std::int32_t> num_items_per_dest,
355 pos_to_neigh_rank(shape0_local, -1);
356 {
357 auto it = dest_to_index.begin();
358 while (it != dest_to_index.end())
359 {
360 const int neigh_rank = dest.size();
361
362 // Store global rank
363 dest.push_back((*it)[0]);
364
365 // Find iterator to next global rank
366 auto it1
367 = std::find_if(it, dest_to_index.end(),
368 [r = dest.back()](auto& idx) { return idx[0] != r; });
369
370 // Store number of items for current rank
371 num_items_per_dest.push_back(std::distance(it, it1));
372
373 // Map from local x index to local destination rank
374 for (auto e = it; e != it1; ++e)
375 pos_to_neigh_rank[(*e)[1]] = neigh_rank;
376
377 // Advance iterator
378 it = it1;
379 }
380 }
381
382 // Determine source ranks
383 const std::vector<int> src = MPI::compute_graph_edges_nbx(comm, dest);
384 LOG(INFO)
385 << "Number of neighbourhood source ranks in distribute_to_postoffice: "
386 << src.size();
387
388 // Create neighbourhood communicator for sending data to post offices
389 MPI_Comm neigh_comm;
390 MPI_Dist_graph_create_adjacent(comm, src.size(), src.data(), MPI_UNWEIGHTED,
391 dest.size(), dest.data(), MPI_UNWEIGHTED,
392 MPI_INFO_NULL, false, &neigh_comm);
393
394 // Compute send displacements
395 std::vector<std::int32_t> send_disp = {0};
396 std::partial_sum(num_items_per_dest.begin(), num_items_per_dest.end(),
397 std::back_insert_iterator(send_disp));
398
399 // Pack send buffers
400 std::vector<T> send_buffer_data(shape[1] * send_disp.back());
401 std::vector<std::int64_t> send_buffer_index(send_disp.back());
402 {
403 std::vector<std::int32_t> send_offsets = send_disp;
404 for (std::int32_t i = 0; i < shape0_local; ++i)
405 {
406 if (int neigh_dest = pos_to_neigh_rank[i]; neigh_dest != -1)
407 {
408 std::size_t pos = send_offsets[neigh_dest];
409 send_buffer_index[pos] = i + rank_offset;
410 std::copy_n(std::next(x.begin(), i * shape[1]), shape[1],
411 std::next(send_buffer_data.begin(), shape[1] * pos));
412 ++send_offsets[neigh_dest];
413 }
414 }
415 }
416
417 // Send number of items to post offices (destination) that I will be
418 // sending
419 std::vector<int> num_items_recv(src.size());
420 num_items_per_dest.reserve(1);
421 num_items_recv.reserve(1);
422 MPI_Neighbor_alltoall(num_items_per_dest.data(), 1, MPI_INT,
423 num_items_recv.data(), 1, MPI_INT, neigh_comm);
424
425 // Prepare receive displacement and buffers
426 std::vector<std::int32_t> recv_disp(num_items_recv.size() + 1, 0);
427 std::partial_sum(num_items_recv.begin(), num_items_recv.end(),
428 std::next(recv_disp.begin()));
429
430 // Send/receive global indices
431 std::vector<std::int64_t> recv_buffer_index(recv_disp.back());
432 MPI_Neighbor_alltoallv(send_buffer_index.data(), num_items_per_dest.data(),
433 send_disp.data(), MPI_INT64_T,
434 recv_buffer_index.data(), num_items_recv.data(),
435 recv_disp.data(), MPI_INT64_T, neigh_comm);
436
437 // Send/receive data (x)
438 MPI_Datatype compound_type;
439 MPI_Type_contiguous(shape[1], dolfinx::MPI::mpi_type<T>(), &compound_type);
440 MPI_Type_commit(&compound_type);
441 std::vector<T> recv_buffer_data(shape[1] * recv_disp.back());
442 MPI_Neighbor_alltoallv(send_buffer_data.data(), num_items_per_dest.data(),
443 send_disp.data(), compound_type,
444 recv_buffer_data.data(), num_items_recv.data(),
445 recv_disp.data(), compound_type, neigh_comm);
446 MPI_Type_free(&compound_type);
447 MPI_Comm_free(&neigh_comm);
448
449 LOG(2) << "Completed send data to post offices.";
450
451 // Convert to local indices
452 const std::int64_t r0 = MPI::local_range(rank, shape[0], size)[0];
453 std::vector<std::int32_t> index_local(recv_buffer_index.size());
454 std::transform(recv_buffer_index.cbegin(), recv_buffer_index.cend(),
455 index_local.begin(), [r0](auto idx) { return idx - r0; });
456
457 return {index_local, recv_buffer_data};
458};
459//---------------------------------------------------------------------------
460template <typename T>
462 MPI_Comm comm, const xtl::span<const std::int64_t>& indices,
463 const xtl::span<const T>& x, std::array<std::int64_t, 2> shape,
464 std::int64_t rank_offset)
465{
466 common::Timer timer("Distribute row-wise data (scalable)");
467 assert(shape[1] > 0);
468
469 const int size = dolfinx::MPI::size(comm);
470 const int rank = dolfinx::MPI::rank(comm);
471 assert(x.size() % shape[1] == 0);
472 const std::int64_t shape0_local = x.size() / shape[1];
473
474 // 0. Send x data to/from post offices
475
476 // Send receive x data to post office (only for rows that need to be
477 // communicated)
478 auto [post_indices, post_x] = MPI::distribute_to_postoffice(
479 comm, x, {shape[0], shape[1]}, rank_offset);
480 assert(post_indices.size() == post_x.size() / shape[1]);
481
482 // 1. Send request to post office ranks for data
483
484 // Build list of (src, global index, global, index positions) for each
485 // entry in 'indices' that doesn't belong to this rank, then sort
486 std::vector<std::tuple<int, std::int64_t, std::int32_t>> src_to_index;
487 for (std::size_t i = 0; i < indices.size(); ++i)
488 {
489 std::size_t idx = indices[i];
490 if (int src = MPI::index_owner(size, idx, shape[0]); src != rank)
491 src_to_index.push_back({src, idx, i});
492 }
493 std::sort(src_to_index.begin(), src_to_index.end());
494
495 // Build list is neighbour src ranks and count number of items (rows
496 // of x) to receive from each src post office (by neighbourhood rank)
497 std::vector<std::int32_t> num_items_per_src;
498 std::vector<int> src;
499 {
500 auto it = src_to_index.begin();
501 while (it != src_to_index.end())
502 {
503 src.push_back(std::get<0>(*it));
504 auto it1 = std::find_if(it, src_to_index.end(),
505 [r = src.back()](auto& idx)
506 { return std::get<0>(idx) != r; });
507 num_items_per_src.push_back(std::distance(it, it1));
508 it = it1;
509 }
510 }
511
512 // Determine 'delivery' destination ranks (ranks that want data from
513 // me)
514 const std::vector<int> dest
516 LOG(INFO) << "Neighbourhood destination ranks from post office in "
517 "distribute_data (rank, num dests, num dests/mpi_size): "
518 << rank << ", " << dest.size() << ", "
519 << static_cast<double>(dest.size()) / size;
520
521 // Create neighbourhood communicator for sending data to post offices
522 // (src), and receiving data form my send my post office
523 MPI_Comm neigh_comm0;
524 MPI_Dist_graph_create_adjacent(comm, dest.size(), dest.data(), MPI_UNWEIGHTED,
525 src.size(), src.data(), MPI_UNWEIGHTED,
526 MPI_INFO_NULL, false, &neigh_comm0);
527
528 // Communicate number of requests to each source
529 std::vector<int> num_items_recv(dest.size());
530 num_items_per_src.reserve(1);
531 num_items_recv.reserve(1);
532 MPI_Neighbor_alltoall(num_items_per_src.data(), 1, MPI_INT,
533 num_items_recv.data(), 1, MPI_INT, neigh_comm0);
534
535 // Prepare send/receive displacements
536 std::vector<std::int32_t> send_disp = {0};
537 std::partial_sum(num_items_per_src.begin(), num_items_per_src.end(),
538 std::back_insert_iterator(send_disp));
539 std::vector<std::int32_t> recv_disp = {0};
540 std::partial_sum(num_items_recv.begin(), num_items_recv.end(),
541 std::back_insert_iterator(recv_disp));
542
543 // Pack my requested indices (global) in send buffer ready to send to
544 // post offices
545 assert(send_disp.back() == (int)src_to_index.size());
546 std::vector<std::int64_t> send_buffer_index(src_to_index.size());
547 std::transform(src_to_index.cbegin(), src_to_index.cend(),
548 send_buffer_index.begin(),
549 [](auto& x) { return std::get<1>(x); });
550
551 // Prepare the receive buffer
552 std::vector<std::int64_t> recv_buffer_index(recv_disp.back());
553 MPI_Neighbor_alltoallv(send_buffer_index.data(), num_items_per_src.data(),
554 send_disp.data(), MPI_INT64_T,
555 recv_buffer_index.data(), num_items_recv.data(),
556 recv_disp.data(), MPI_INT64_T, neigh_comm0);
557
558 MPI_Comm_free(&neigh_comm0);
559
560 // 2. Send data (rows of x) back to requesting ranks (transpose of the
561 // preceding communication pattern operation)
562
563 // Build map from local index to post_indices position. Set to -1 for
564 // data that was already on this rank and was therefore was not
565 // sent/received via a postoffice.
566 const std::array<std::int64_t, 2> postoffice_range
567 = MPI::local_range(rank, shape[0], size);
568 std::vector<std::int32_t> post_indices_map(
569 postoffice_range[1] - postoffice_range[0], -1);
570 for (std::size_t i = 0; i < post_indices.size(); ++i)
571 {
572 assert(post_indices[i] < (int)post_indices_map.size());
573 post_indices_map[post_indices[i]] = i;
574 }
575
576 // Build send buffer
577 std::vector<T> send_buffer_data(shape[1] * recv_disp.back());
578 for (std::size_t p = 0; p < recv_disp.size() - 1; ++p)
579 {
580 int offset = recv_disp[p];
581 for (std::int32_t i = recv_disp[p]; i < recv_disp[p + 1]; ++i)
582 {
583 std::int64_t index = recv_buffer_index[i];
584 if (index >= rank_offset and index < (rank_offset + shape0_local))
585 {
586 // I already had this index before any communication
587 std::int32_t local_index = index - rank_offset;
588 std::copy_n(std::next(x.begin(), shape[1] * local_index), shape[1],
589 std::next(send_buffer_data.begin(), shape[1] * offset));
590 }
591 else
592 {
593 // Take from my 'post bag'
594 auto local_index = index - postoffice_range[0];
595 std::int32_t pos = post_indices_map[local_index];
596 assert(pos != -1);
597 std::copy_n(std::next(post_x.begin(), shape[1] * pos), shape[1],
598 std::next(send_buffer_data.begin(), shape[1] * offset));
599 }
600
601 ++offset;
602 }
603 }
604
605 MPI_Dist_graph_create_adjacent(comm, src.size(), src.data(), MPI_UNWEIGHTED,
606 dest.size(), dest.data(), MPI_UNWEIGHTED,
607 MPI_INFO_NULL, false, &neigh_comm0);
608
609 MPI_Datatype compound_type0;
610 MPI_Type_contiguous(shape[1], dolfinx::MPI::mpi_type<T>(), &compound_type0);
611 MPI_Type_commit(&compound_type0);
612
613 std::vector<T> recv_buffer_data(shape[1] * send_disp.back());
614 MPI_Neighbor_alltoallv(send_buffer_data.data(), num_items_recv.data(),
615 recv_disp.data(), compound_type0,
616 recv_buffer_data.data(), num_items_per_src.data(),
617 send_disp.data(), compound_type0, neigh_comm0);
618
619 MPI_Type_free(&compound_type0);
620 MPI_Comm_free(&neigh_comm0);
621
622 std::vector<std::int32_t> index_pos_to_buffer(indices.size(), -1);
623 for (std::size_t i = 0; i < src_to_index.size(); ++i)
624 index_pos_to_buffer[std::get<2>(src_to_index[i])] = i;
625
626 // Extra data to return
627 std::vector<T> x_new(shape[1] * indices.size());
628 for (std::size_t i = 0; i < indices.size(); ++i)
629 {
630 const std::int64_t index = indices[i];
631 if (index >= rank_offset and index < (rank_offset + shape0_local))
632 {
633 // Had data from the start in x
634 auto local_index = index - rank_offset;
635 std::copy_n(std::next(x.begin(), shape[1] * local_index), shape[1],
636 std::next(x_new.begin(), shape[1] * i));
637 }
638 else
639 {
640 if (int src = MPI::index_owner(size, index, shape[0]); src == rank)
641 {
642 // In my post office bag
643 auto local_index = index - postoffice_range[0];
644 std::int32_t pos = post_indices_map[local_index];
645 assert(pos != -1);
646 std::copy_n(std::next(post_x.begin(), shape[1] * pos), shape[1],
647 std::next(x_new.begin(), shape[1] * i));
648 }
649 else
650 {
651 // In my received post
652 std::int32_t pos = index_pos_to_buffer[i];
653 assert(pos != -1);
654 std::copy_n(std::next(recv_buffer_data.begin(), shape[1] * pos),
655 shape[1], std::next(x_new.begin(), shape[1] * i));
656 }
657 }
658 }
659
660 return x_new;
661}
662//---------------------------------------------------------------------------
663template <typename T>
664std::vector<T> distribute_data(MPI_Comm comm,
665 const xtl::span<const std::int64_t>& indices,
666 const xtl::span<const T>& x, int shape1)
667{
668 assert(shape1 > 0);
669 assert(x.size() % shape1 == 0);
670 const std::int64_t shape0_local = x.size() / shape1;
671
672 std::int64_t shape0(0), rank_offset(0);
673 MPI_Allreduce(&shape0_local, &shape0, 1, MPI_INT64_T, MPI_SUM, comm);
674 MPI_Exscan(&shape0_local, &rank_offset, 1, MPI_INT64_T, MPI_SUM, comm);
675
676 return distribute_from_postoffice(comm, indices, x, {shape0, shape1},
677 rank_offset);
678}
679//---------------------------------------------------------------------------
680template <typename T>
682neighbor_all_to_all(MPI_Comm comm, const graph::AdjacencyList<T>& send_data)
683{
684 // Get neighbor processes
685 int indegree(-1), outdegree(-2), weighted(-1);
686 MPI_Dist_graph_neighbors_count(comm, &indegree, &outdegree, &weighted);
687
688 // Allocate memory (add '1' to handle empty case as OpenMPI fails for
689 // null pointers
690 std::vector<int> send_sizes(outdegree, 0);
691 std::vector<int> recv_sizes(indegree);
692 std::adjacent_difference(std::next(send_data.offsets().begin()),
693 send_data.offsets().end(), send_sizes.begin());
694 // Get receive sizes
695 send_sizes.reserve(1);
696 recv_sizes.reserve(1);
697 MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT, recv_sizes.data(), 1,
698 MPI_INT, comm);
699
700 // Work out recv offsets
701 std::vector<int> recv_offsets(indegree + 1);
702 recv_offsets[0] = 0;
703 std::partial_sum(recv_sizes.begin(), recv_sizes.end(),
704 std::next(recv_offsets.begin(), 1));
705
706 std::vector<T> recv_data(recv_offsets[recv_offsets.size() - 1]);
707 MPI_Neighbor_alltoallv(
708 send_data.array().data(), send_sizes.data(), send_data.offsets().data(),
709 dolfinx::MPI::mpi_type<T>(), recv_data.data(), recv_sizes.data(),
710 recv_offsets.data(), dolfinx::MPI::mpi_type<T>(), comm);
711
712 return graph::AdjacencyList<T>(std::move(recv_data), std::move(recv_offsets));
713}
714//---------------------------------------------------------------------------
715
716} // namespace dolfinx::MPI
A duplicate MPI communicator and manage lifetime of the communicator.
Definition: MPI.h:40
Comm(MPI_Comm comm, bool duplicate=true)
Duplicate communicator and wrap duplicate.
Definition: MPI.cpp:12
~Comm()
Destructor (frees wrapped communicator)
Definition: MPI.cpp:39
MPI_Comm comm() const noexcept
Return the underlying MPI_Comm object.
Definition: MPI.cpp:73
A timer can be used for timing tasks. The basic usage is.
Definition: Timer.h:31
This class provides a static adjacency list data structure. It is commonly used to store directed gra...
Definition: AdjacencyList.h:46
const std::vector< T > & array() const
Return contiguous array of links for all nodes (const version)
Definition: AdjacencyList.h:148
const std::vector< std::int32_t > & offsets() const
Offset for each node in array() (const version)
Definition: AdjacencyList.h:154
MPI support functionality.
Definition: MPI.h:28
std::vector< T > distribute_data(MPI_Comm comm, const xtl::span< const std::int64_t > &indices, const xtl::span< const T > &x, int shape1)
Distribute rows of a rectangular data array to ranks where they are required (scalable version).
Definition: MPI.h:664
std::vector< int > compute_graph_edges_pcx(MPI_Comm comm, const xtl::span< const int > &edges)
Determine incoming graph edges using the PCX consensus algorithm.
Definition: MPI.cpp:91
std::vector< T > distribute_from_postoffice(MPI_Comm comm, const xtl::span< const std::int64_t > &indices, const xtl::span< const T > &x, std::array< std::int64_t, 2 > shape, std::int64_t rank_offset)
Distribute rows of a rectangular data array from post office ranks to ranks where they are required.
Definition: MPI.h:461
constexpr int index_owner(int size, std::size_t index, std::size_t N)
Return which rank owns index in global range [0, N - 1] (inverse of MPI::local_range).
Definition: MPI.h:106
std::array< std::vector< int >, 2 > neighbors(MPI_Comm comm)
Return list of neighbors ranks (sources and destinations) for a neighborhood communicator.
Definition: MPI.cpp:224
int size(MPI_Comm comm)
Return size of the group (number of processes) associated with the communicator.
Definition: MPI.cpp:83
int rank(MPI_Comm comm)
Return process rank for the communicator.
Definition: MPI.cpp:75
std::vector< int > compute_graph_edges_nbx(MPI_Comm comm, const xtl::span< const int > &edges)
Determine incoming graph edges using the NBX consensus algorithm.
Definition: MPI.cpp:151
constexpr std::array< std::int64_t, 2 > local_range(int rank, std::int64_t N, int size)
Return local range for the calling process, partitioning the global [0, N - 1] range across all ranks...
Definition: MPI.h:82
std::pair< std::vector< std::int32_t >, std::vector< T > > distribute_to_postoffice(MPI_Comm comm, const xtl::span< const T > &x, std::array< std::int64_t, 2 > shape, std::int64_t rank_offset)
Distribute row data to 'post office' ranks.
Definition: MPI.h:320
graph::AdjacencyList< T > neighbor_all_to_all(MPI_Comm comm, const graph::AdjacencyList< T > &send_data)
Send in_values[n0] to neighbor process n0 and receive values from neighbor process n1 in out_values[n...
Definition: MPI.h:682
constexpr MPI_Datatype mpi_type()
MPI Type.
Definition: MPI.h:284
tag
MPI communication tags.
Definition: MPI.h:32
Definition: MPI.h:279