ENH: centralized MeshObject handler for multi-world connections

src/meshTools/Make/files

@@ -298,6 +298,8 @@ $(ACMICycPatches)/cyclicACMIPointPatchField/cyclicACMIPointPatchFields.C
 PeriodicAMICycPatches=$(AMI)/patches/cyclicPeriodicAMI
 $(PeriodicAMICycPatches)/cyclicPeriodicAMIPolyPatch/cyclicPeriodicAMIPolyPatch.C
 
+multiWorld/multiWorldConnectionsObject.C
+
 mappedPatches/mappedPolyPatch/mappedPatchBase.C
 mappedPatches/mappedPolyPatch/mappedPolyPatch.C
 mappedPatches/mappedPolyPatch/mappedWallPolyPatch.C

src/meshTools/multiWorld/multiWorldConnectionsObject.C

+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2021 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "multiWorldConnectionsObject.H"
+#include "Pstream.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+namespace Foam
+{
+    defineTypeNameAndDebug(multiWorldConnections, 0);
+}
+
+
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// Combine world-to-world connections.
+// Forward connection = 1, Backward connection = 2, Both = 3
+struct worldConnectBitOrEq
+{
+    void operator()(EdgeMap<unsigned>& a, const EdgeMap<unsigned>& b) const
+    {
+        forAllConstIters(b, iter)
+        {
+            a(iter.key()) |= iter.val();
+        }
+    }
+};
+
+
+static void printDOT(Ostream& os, const EdgeMap<unsigned>& connections)
+{
+    os << nl << "// Multiworld communication graph:" << nl;
+    os.beginBlock("graph");
+
+    // Graph Nodes == worlds
+    label worldi = 0;
+    for (const word& worldName : UPstream::allWorlds())
+    {
+        os.indent();
+        os  << worldi << " [xlabel=" << worldi
+            << ",label=\"" << worldName << "\"]" << nl;
+
+        ++worldi;
+    }
+    os  << nl;
+
+    // Graph Edges == connections
+    for (const edge& connect : connections.sortedToc())
+    {
+        os.indent();
+        os  << connect.first() << " -- " << connect.second();
+
+        // Mismatched forward/backward connections?
+        if (connections.lookup(connect, 0u) != 3u)
+        {
+            os << " [style=dashed] // mismatched?";
+        }
+        os << nl;
+    }
+
+    os.endBlock();
+
+    os << "// end graph" << nl;
+}
+
+} // End namespace Foam
+
+
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+Foam::edge Foam::multiWorldConnections::worldPair(const label otherWorld)
+{
+    if (otherWorld < 0 || !Pstream::parRun())
+    {
+        Perr<< "ignore: no world or non-parallel" << endl;
+        return edge(-1, -1);
+    }
+    else if (UPstream::allWorlds().size() <= otherWorld)
+    {
+        Perr<< "ignore: invalid world: " << otherWorld << endl;
+        return edge(-1, -1);
+    }
+
+    const label thisWorldID = UPstream::myWorldID();
+
+    // The worlds (sorted)
+    return edge(thisWorldID, otherWorld, true);
+}
+
+
+Foam::edge Foam::multiWorldConnections::worldPair(const word& otherWorld)
+{
+    if (otherWorld.empty() || !Pstream::parRun())
+    {
+        Perr<< "ignore: no world or non-parallel" << endl;
+        return edge(-1, -1);
+    }
+
+    const label thisWorldID = UPstream::myWorldID();
+    const label otherWorldID = UPstream::allWorlds().find(otherWorld);
+
+    if (otherWorldID < 0)
+    {
+        FatalErrorInFunction
+            << "Cannot find world " << otherWorld
+            << " in set of worlds " << flatOutput(UPstream::allWorlds())
+            << exit(FatalError);
+    }
+
+    // The worlds (sorted)
+    return edge(thisWorldID, otherWorldID, true);
+}
+
+
+Foam::label Foam::multiWorldConnections::createCommunicator(const edge& worlds)
+{
+    // Fallback: do not create, just use local world
+    label comm = UPstream::worldComm;
+
+    if (!worlds.valid())
+    {
+        return comm;
+    }
+
+    const labelList& worldIDs = UPstream::worldIDs();
+
+    DynamicList<label> subRanks(worldIDs.size());
+    forAll(worldIDs, proci)
+    {
+        if (worlds.found(worldIDs[proci]))
+        {
+            subRanks.append(proci);
+        }
+    }
+
+    // Allocate new communicator with parent 0 (= world)
+    comm = UPstream::allocateCommunicator(0, subRanks, true);
+
+    if (debug & 2)
+    {
+        Pout<< "multiWorld::communicator :"
+            << " between " << UPstream::allWorlds()[worlds.first()]
+            << " and " << UPstream::allWorlds()[worlds.second()]
+            << " sub-ranks: " << subRanks
+            << " comm:" << comm << endl;
+    }
+
+    return comm;
+}
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::multiWorldConnections::multiWorldConnections(const Time& runTime)
+:
+    MeshObjectType(runTime)
+{}
+
+
+// * * * * * * * * * * * * * * * * Selectors * * * * * * * * * * * * * * * * //
+
+const Foam::multiWorldConnections&
+Foam::multiWorldConnections::New(const Time& runTime)
+{
+    return MeshObjectType::New(runTime);
+}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::multiWorldConnections::~multiWorldConnections()
+{}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+bool Foam::multiWorldConnections::empty() const noexcept
+{
+    return table_.empty();
+}
+
+
+Foam::label Foam::multiWorldConnections::size() const noexcept
+{
+    return table_.size();
+}
+
+
+void Foam::multiWorldConnections::createComms()
+{
+    // Need new communicator(s)
+
+    const label thisWorldID = UPstream::myWorldID();
+
+    EdgeMap<unsigned> allConnections;
+    forAllConstIters(table_, iter)
+    {
+        const edge& connect = iter.key();
+
+        allConnections.insert
+        (
+            connect,
+            (connect.first() == thisWorldID ? 1u : 2u)
+        );
+    }
+
+
+    // Use MPI_COMM_WORLD
+    const label oldWorldComm(Pstream::worldComm);
+    const label oldWarnComm(Pstream::warnComm);
+    Pstream::worldComm = 0;
+    Pstream::warnComm = Pstream::worldComm;
+
+    if (Pstream::parRun())
+    {
+        Pstream::combineGather
+        (
+            allConnections,
+            worldConnectBitOrEq()
+        );
+        Pstream::scatter(allConnections);
+    }
+
+    // Check for mismatched connections
+    label brokenConnections = 0;
+
+    forAllConstIters(allConnections, iter)
+    {
+        // Mismatched forward/backward connections?
+        if (iter.val() != 3u)
+        {
+            ++brokenConnections;
+        }
+    }
+
+    if (brokenConnections)
+    {
+        Pstream::warnComm = oldWarnComm;
+        Pstream::worldComm = oldWorldComm;
+
+        FatalErrorInFunction
+            << "Has " << brokenConnections
+            << " broken world-world connections";
+
+        printDOT(FatalError, allConnections);
+
+        FatalError << exit(FatalError);
+    }
+    else
+    {
+        // NOTE: process in sorted order to ensure proper
+        // synchronization on all worlds and all processors
+
+        for (const edge& connect : allConnections.sortedToc())
+        {
+            // Process known connections without communicators.
+            // - create a communicator and cache its value
+
+            auto iter = table_.find(connect);
+            if (iter.found() && iter.val() == -1)
+            {
+                iter.val() = createCommunicator(connect);
+            }
+        }
+
+        Pstream::warnComm = oldWarnComm;
+        Pstream::worldComm = oldWorldComm;
+    }
+
+    if (debug)
+    {
+        printDOT(Info, allConnections);
+    }
+}
+
+
+bool Foam::multiWorldConnections::addConnectionById(const label otherWorld)
+{
+    // The worlds (sorted)
+    edge worlds(worldPair(otherWorld));
+
+    if (!worlds.valid())
+    {
+        return false;
+    }
+
+    const bool added = table_.insert(worlds, -1);
+
+    Pout<< (added ? "Add" : "Existing") << " connection from "
+        << UPstream::myWorld() << " to " << otherWorld << nl;
+
+    return added;
+}
+
+
+bool Foam::multiWorldConnections::addConnectionByName(const word& otherWorld)
+{
+    // The worlds (sorted)
+    edge worlds(worldPair(otherWorld));
+
+    if (!worlds.valid())
+    {
+        return false;
+    }
+
+    const bool added = table_.insert(worlds, -1);
+
+    Pout<< (added ? "Add" : "Existing") << " connection from "
+        << UPstream::myWorld() << " to " << otherWorld << nl;
+
+    return added;
+}
+
+
+Foam::label Foam::multiWorldConnections::getCommById
+(
+    const label otherWorldID
+) const
+{
+    // Default: use local world
+    label comm = UPstream::worldComm;
+
+    // The communication worlds (sorted)
+    edge worlds(worldPair(otherWorldID));
+
+    if (!worlds.valid())
+    {
+        return comm;
+    }
+
+    const auto iter = table_.cfind(worlds);
+
+    if (!iter.found())
+    {
+        FatalErrorInFunction
+            << "No connection registered for worlds " << worlds
+            << exit(FatalError);
+    }
+
+    // Get cached value, or allocate ALL known communicators
+    comm = iter.val();
+
+    if (comm == -1)
+    {
+        // Need new communicator(s)
+        const_cast<multiWorldConnections&>(*this).createComms();
+
+        // Retrieve from table cache
+        comm = table_.lookup(worlds, UPstream::worldComm);
+    }
+
+    return comm;
+}
+
+
+Foam::label Foam::multiWorldConnections::getCommByName
+(
+    const word& otherWorld
+) const
+{
+    // Default: use local world
+    label comm = UPstream::worldComm;
+
+    // The communication worlds (sorted)
+    edge worlds(worldPair(otherWorld));
+
+    if (!worlds.valid())
+    {
+        return comm;
+    }
+
+    const auto iter = table_.cfind(worlds);
+
+    if (!iter.found())
+    {
+        FatalErrorInFunction
+            << "No connection registered for worlds " << worlds
+            << exit(FatalError);
+    }
+
+    // Get cached value, or allocate ALL known communicators
+    comm = iter.val();
+
+    if (comm == -1)
+    {
+        // Need new communicator(s)
+        const_cast<multiWorldConnections&>(*this).createComms();
+
+        // Retrieve from table cache
+        comm = table_.lookup(worlds, UPstream::worldComm);
+    }
+
+    return comm;
+}
+
+
+Foam::labelList Foam::multiWorldConnections::comms() const
+{
+    labelList list(table_.size());
+
+    if (list.empty())
+    {
+        // Default: use local world
+        list.resize(1, UPstream::worldComm);
+    }
+    else
+    {
+        forAllConstIters(table_, iter)
+        {
+            if (iter.val() == -1)
+            {
+                // Need new communicator(s)
+                const_cast<multiWorldConnections&>(*this).createComms();
+                break;
+            }
+        }
+
+        // Retrieve values from table cache
+        label i = 0;
+
+        forAllConstIters(table_, iter)
+        {
+            list[i] = iter.val();
+            ++i;
+        }
+
+        Foam::sort(list);  // Consistent order!
+    }
+
+    return list;
+}
+
+
+// ************************************************************************* //

src/meshTools/multiWorld/multiWorldConnectionsObject.H

+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2021 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::multiWorldConnections
+
+Description
+    Centralized handling of multi-world MPI connections.
+
+Note
+    This class may move to static only or a singleton in the future.
+
+SourceFiles
+    multiWorldConnectionsObject.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef meshObjects_multiWorldConnections_H
+#define meshObjects_multiWorldConnections_H
+
+#include "MeshObject.H"
+#include "EdgeMap.H"
+#include "Time.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                    Class multiWorldConnections Declaration
+\*---------------------------------------------------------------------------*/
+
+class multiWorldConnections
+:
+    public MeshObject
+    <
+        Time,
+        TopologicalMeshObject,
+        multiWorldConnections
+    >
+{
+    // Private Typedefs
+
+        typedef MeshObject
+        <
+            Time,
+            TopologicalMeshObject,
+            multiWorldConnections
+        > MeshObjectType;
+
+
+    // Private Data
+
+        //- Table of world/world connections to communicator label
+        EdgeMap<label> table_;
+
+
+    // Private Member Functions
+
+        //- Pairing from myWorld to other world by ID
+        //  \return invalid on any error or if non-parallel
+        static edge worldPair(const label otherWorld);
+
+        //- Pairing from myWorld to other world by NAME.
+        //  \return invalid on any error or if non-parallel
+        static edge worldPair(const word& otherWorld);
+
+        //- Allocate a communicator between myWorld and other world
+        static label createCommunicator(const edge& worlds);
+
+
+public:
+
+    //- Run-time type information
+    TypeName("multiWorld");
+
+
+    // Constructors
+
+        //- Construct
+        explicit multiWorldConnections(const Time& runTime);
+
+
+    // Selectors
+
+        //- Access mesh object
+        static const multiWorldConnections& New(const Time& runTime);
+
+
+    //- Destructor
+    ~multiWorldConnections();
+
+
+    // Member Functions
+
+        //- True if no world-to-world connections are defined
+        bool empty() const noexcept;
+
+        //- Number of world-to-world connections defined.
+        label size() const noexcept;
+
+        //- Create all communicators.
+        //- Low-level, not normally called directly
+        void createComms();
+
+        //- Define a connection from myWorld to other world by ID
+        bool addConnectionById(const label otherWorld);
+
+        //- Define a connection from myWorld to other world by NAME
+        bool addConnectionByName(const word& otherWorld);
+
+        //- Get communicator for myWorld to other world connection by ID.
+        //  Uses cached value, or creates a new communicator
+        label getCommById(const label otherWorld) const;
+
+        //- Get communicator for myWorld to other world connection by NAME.
+        //  Uses cached value, or creates a new communicator
+        label getCommByName(const word& otherWorld) const;
+
+        //- Get communicators used for myWorld to other worlds in sorted order.
+        //  \return worldComm if no world-to-world communicators in use.
+        labelList comms() const;
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //