b2comm_MPI.H

//------------------------------------------------------------------------
// b2comm_MPI.H
//
// written by Harald Klimach <harald.klimach@dlr.de>
//
// (c) 2021 Deutsches Zentrum für Luft- und Raumfahrt (DLR) e.V.
//          Linder Höhe, 51147 Köln
//
// All Rights Reserved.  Proprietary source code.  The contents of
// this file may not be disclosed to third parties, copied or
// duplicated in any form, in whole or in part, without the prior
// written permission of DLR.
//
//------------------------------------------------------------------------
 
#ifndef B2COMM_MPI_H_
#define B2COMM_MPI_H_
 
//------------------------------------------------------------------------
//
// \file
// MPI communicator encapsulation. Provides the communication via MPI.
//
//------------------------------------------------------------------------
 
#include "b2ppconfig.h"
#include "utils/b2exception.H"
 
#ifdef HAVE_MPI
 
#include <mpi.h>
 
#include "utils/b2comm.H"
 
namespace b2000 {
 
// Communication implementation via MPI.
//
// If MPI is available and activated via cmakes USE_MPI option, this class will
// be used to realize the communication between multiple processes.
class B2mpi : public Communicator {
public:
    // A main constructor that initializes MPI.
    // May only be called once in the program execution!
    // This constructor also splits off any processes in MPI_COMM_WORLD,
    // that may run a different program and don't call this constructor.
    B2mpi(int argc, char* argv[]);
 
    explicit B2mpi() = default;
 
    // Constructor from an existing MPI communicator.
    explicit B2mpi(MPI_Comm comm);
 
    // Constructor that splits some processes off from MPI_COMM_WORLD.
    // Processes that should partake in the new communicator need to
    // indicate that and pass true.
    explicit B2mpi(bool participate);
 
    ~B2mpi();
 
    static constexpr bool supported = true;
 
    // Rank of the running process
    int rank() const override { return rank_; }
 
    // Number of processes in this communicator
    int NumRanks() const override { return size_; }
 
    // Is the running process root of the communicator?
    bool IsRoot() const override { return 0 == rank_; }
 
    // Additional command line options for the communication
    void CmdOpts(b2000::cmdline::CmdLine& cmdline) override {
        // No additional command line options necessary for the MPI
        // communication.
    }
 
    // Fortran communicator as needed by MUMPS
    int GetFcomm() const override;
 
    // Wrap MPI abort
    void Abort(int errorcode) override { MPI_Abort(comm_, errorcode); }
 
    // Wrap MPI barrier
    void Barrier() override {
        int error = MPI_Barrier(comm_);
        check_mpi_error(error);
    }
 
    // Receive a message from rank source with tag within the communicator.
    void Recv(char buffer[], uint64_t length, int source, int tag) {
        const int buflen = static_cast<int>(length);
        int error = MPI_Recv(buffer, buflen, MPI_BYTE, source, tag, comm_, &status_);
        check_mpi_error(error);
    }
 
    // Send a message to rank target with tag within the communicator.
    void Send(char buffer[], uint64_t length, int target, int tag) {
        const int buflen = static_cast<int>(length);
        int error = MPI_Send(buffer, buflen, MPI_BYTE, target, tag, comm_);
        check_mpi_error(error);
    }
 
    // Wrap MPI bcast
    void Bcast(uint64_t buffer[], uint64_t length, int root = 0) override {
        const int buflen = static_cast<int>(length);
        int error = MPI_Bcast(buffer, buflen, MPI_UNSIGNED_LONG, root, comm_);
        check_mpi_error(error);
    }
 
    void Bcast(int buffer[], uint64_t length, int root = 0) override {
        int buflen = static_cast<int>(length);
        int error = MPI_Bcast(buffer, buflen, MPI_INT, root, comm_);
        check_mpi_error(error);
    }
 
    void Bcast(double buffer[], uint64_t length, int root = 0) override {
        int buflen = static_cast<int>(length);
        int error = MPI_Bcast(buffer, buflen, MPI_DOUBLE, root, comm_);
        check_mpi_error(error);
    }
 
    // Probe an incoming message for its size in bytes (before calling
    // the corresponding receive).
    uint64_t ProbeByteCount(int source, int tag) override {
        int message_size;
        int error = MPI_Probe(source, tag, comm_, &status_);
        if (0 == error) { error = MPI_Get_count(&status_, MPI_BYTE, &message_size); }
        check_mpi_error(error);
        return static_cast<uint64_t>(message_size);
    }
 
private:
    // Color to use to identify B2000 processes.
    static constexpr int kMpiColor = 2000;
 
    // Color to use to identify shared processes in the FSI context.
    static constexpr int kFsiColor = 3000;
 
    // Identification of the running process
    int rank_ = 0;
 
    // Size of the Communicator
    int size_ = 0;
 
    // Is this the communicator of the main program?
    bool ismain_ = false;
 
    // The MPI communicator to encapsulate
    MPI_Comm comm_ = MPI_COMM_NULL;
 
    // Hold the status of the last MPI call that returned a status.
    MPI_Status status_;
 
    // Check for MPI errors and throw an exception if the MPI call
    // was not successful.
    //
    // Note: usually MPI errors are fatal and the program aborts
    //       when encountering errors likely before running into this
    //       handler. To ensure that this is utilized we'd need to set
    //       a proper MPI Error handler, but this changes the semantic
    //       and would result in ignoring the default MPI configuration.
    void check_mpi_error(int errorcode) {
        if (errorcode != MPI_SUCCESS) {
            char errormessage[MPI_MAX_ERROR_STRING + 1];
            int resultlen;
            MPI_Error_string(errorcode, errormessage, &resultlen);
            errormessage[resultlen] = '\0';
            Exception() << "MPI Error: " << errormessage << THROW;
        }
    }
 
};  // class B2mpi
 
}  // namespace b2000
 
#else  // HAVE_MPI
 
namespace b2000 {
 
// Placeholder when MPI is *NOT* available.
class B2mpi : public Communicator {
public:
    static constexpr bool supported = false;
 
    explicit B2mpi() { UnimplementedError() << THROW; }
    B2mpi(int argc, char* argv[]) { UnimplementedError() << THROW; }
    explicit B2mpi(bool participate) { UnimplementedError() << THROW; }
 
    int rank() const override { return 0; }
    int NumRanks() const override { return 0; }
    bool IsRoot() const override { return false; }
 
    void CmdOpts(b2000::cmdline::CmdLine& cmdline) override {}
 
    int GetFcomm() const override { return 0; }
    void Abort(int errorcode) override {}
    void Barrier() override {}
 
    void Recv(char buffer[], uint64_t length, int source, int tag) override {}
    void Send(char buffer[], uint64_t length, int target, int tag) override {}
 
    void Bcast(uint64_t buffer[], uint64_t length, int root = 0) override {}
    void Bcast(int buffer[], uint64_t length, int root = 0) override {}
    void Bcast(double buffer[], uint64_t length, int root = 0) override {}
 
    uint64_t ProbeByteCount(int source, int tag) override { return 0; }
};  // class B2mpi
 
}  // namespace b2000
 
#endif  // HAVE_MPI
 
#endif  // B2COMM_MPI_H_