ENH: add simple profiling of MPI communications

src/OpenFOAM/Make/files

@@ -11,6 +11,7 @@ global/profiling/profiling.C
 global/profiling/profilingInformation.C
 global/profiling/profilingSysInfo.C
 global/profiling/profilingTrigger.C
+global/profiling/profilingPstream.C
 global/etcFiles/etcFiles.C
 global/version/foamVersion.C
 

src/OpenFOAM/global/profiling/profilingPstream.C

+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2019 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "profilingPstream.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+Foam::autoPtr<Foam::cpuTime> Foam::profilingPstream::timer_(nullptr);
+
+Foam::autoPtr<Foam::cpuTime> Foam::profilingPstream::suspend_(nullptr);
+
+Foam::FixedList<Foam::scalar, 5> Foam::profilingPstream::times_(Zero);
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::profilingPstream::profilingPstream()
+{
+    enable();
+}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::profilingPstream::~profilingPstream()
+{
+    disable();
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+void Foam::profilingPstream::enable()
+{
+    if (timer_.valid())
+    {
+        timer_->resetCpuTime(); // Not really needed ...
+    }
+    else if (suspend_.valid())
+    {
+        suspend_.swap(timer_);
+        timer_->resetCpuTime(); // Not really needed ...
+    }
+    else
+    {
+        timer_.reset(new cpuTime);
+        times_ = Zero;
+    }
+
+    suspend_.clear();
+}
+
+
+void Foam::profilingPstream::disable()
+{
+    timer_.clear();
+    suspend_.clear();
+}
+
+
+void Foam::profilingPstream::suspend()
+{
+    suspend_.clear();
+    suspend_.swap(timer_);
+}
+
+
+void Foam::profilingPstream::resume()
+{
+    if (suspend_.valid())
+    {
+        timer_.clear();
+        timer_.swap(suspend_);
+    }
+}
+
+
+// ************************************************************************* //

src/OpenFOAM/global/profiling/profilingPstream.H

+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2019 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::profilingPstream
+
+Description
+    Timers and values for simple (simplistic) mpi-profiling. The entire
+    class behaves as a singleton.
+
+SourceFiles
+    profilingPstream.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef profilingPstream_H
+#define profilingPstream_H
+
+#include "autoPtr.H"
+#include "cpuTime.H"
+#include "scalar.H"
+#include "FixedList.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                      Class profilingPstream Declaration
+\*---------------------------------------------------------------------------*/
+
+class profilingPstream
+{
+    //- Timer to use
+    static autoPtr<cpuTime> timer_;
+
+    //- Stash for timer during suspend
+    static autoPtr<cpuTime> suspend_;
+
+    //- The timing values
+    static FixedList<scalar, 5> times_;
+
+
+public:
+
+    //- Enumeration within times array
+    enum timingType
+    {
+        GATHER = 0,
+        SCATTER,
+        REDUCE,
+        WAIT,
+        ALL_TO_ALL
+    };
+
+public:
+
+    // Constructors
+
+        //- Construct and enable global timer
+        profilingPstream();
+
+
+    //- Destructor - remove global timer
+    ~profilingPstream();
+
+
+    // Member Functions
+
+        //- Create timer for measuring communication, or reset existing
+        static void enable();
+
+        //- Remove timer for measuring communication activity
+        static void disable();
+
+        //- Suspend use of timer (if active)
+        static void suspend();
+
+        //- Resume use of timer (if previously active)
+        static void resume();
+
+        //- Timer is active
+        inline static bool active()
+        {
+            return timer_.valid();
+        }
+
+        //- Access to the timing information
+        inline static FixedList<scalar, 5>& times()
+        {
+            return times_;
+        }
+
+        //- Access to the timing information
+        inline static scalar times(const enum timingType idx)
+        {
+            return times_[idx];
+        }
+
+        //- Update timer prior to measurement
+        inline static void beginTiming()
+        {
+            if (timer_.valid())
+            {
+                (void) timer_->cpuTimeIncrement();
+            }
+        }
+
+        //- Add time increment
+        inline static void addTime(const enum timingType idx)
+        {
+            if (timer_.valid())
+            {
+                times_[idx] += timer_->cpuTimeIncrement();
+            }
+        }
+
+        //- Add time increment to gatherTime
+        inline static void addGatherTime()
+        {
+            addTime(GATHER);
+        }
+
+        //- Add time increment to scatterTime
+        inline static void addScatterTime()
+        {
+            addTime(SCATTER);
+        }
+
+        //- Add time increment to reduceTime
+        inline static void addReduceTime()
+        {
+            addTime(REDUCE);
+        }
+
+        //- Add time increment to waitTime
+        inline static void addWaitTime()
+        {
+            addTime(WAIT);
+        }
+
+        //- Add time increment to allToAllTime
+        inline static void addAllToAllTime()
+        {
+            addTime(ALL_TO_ALL);
+        }
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //

src/Pstream/mpi/UIPread.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           |
+    \\  /    A nd           | Copyright (C) 2019 OpenCFD Ltd.
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
                             | Copyright (C) 2011-2017 OpenFOAM Foundation
@@ -30,6 +30,7 @@ Description
 
 #include "UIPstream.H"
 #include "PstreamGlobals.H"
+#include "profilingPstream.H"
 #include "IOstreams.H"
 
 #include <mpi.h>
@@ -85,6 +86,8 @@ Foam::UIPstream::UIPstream
         // and set it
         if (!wantedSize)
         {
+            profilingPstream::beginTiming();
+
             MPI_Probe
             (
                 fromProcNo_,
@@ -94,6 +97,8 @@ Foam::UIPstream::UIPstream
             );
             MPI_Get_count(&status, MPI_BYTE, &messageSize_);
 
+            profilingPstream::addWaitTime();
+
             externalBuf_.setCapacity(messageSize_);
             wantedSize = messageSize_;
 
@@ -186,6 +191,8 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
         // and set it
         if (!wantedSize)
         {
+            profilingPstream::beginTiming();
+
             MPI_Probe
             (
                 fromProcNo_,
@@ -195,6 +202,8 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
             );
             MPI_Get_count(&status, MPI_BYTE, &messageSize_);
 
+            profilingPstream::addWaitTime();
+
             externalBuf_.setCapacity(messageSize_);
             wantedSize = messageSize_;
 
@@ -257,6 +266,8 @@ Foam::label Foam::UIPstream::read
         error::printStack(Pout);
     }
 
+    profilingPstream::beginTiming();
+
     if (commsType == commsTypes::blocking || commsType == commsTypes::scheduled)
     {
         MPI_Status status;
@@ -282,6 +293,7 @@ Foam::label Foam::UIPstream::read
             return 0;
         }
 
+        profilingPstream::addScatterTime();
 
         // Check size of message read
 
@@ -332,6 +344,8 @@ Foam::label Foam::UIPstream::read
             return 0;
         }
 
+        profilingPstream::addScatterTime();
+
         if (debug)
         {
             Pout<< "UIPstream::read : started read from:" << fromProcNo
@@ -346,15 +360,12 @@ Foam::label Foam::UIPstream::read
         // Assume the message is completely received.
         return bufSize;
     }
-    else
-    {
-        FatalErrorInFunction
-            << "Unsupported communications type "
-            << int(commsType)
-            << Foam::abort(FatalError);
 
-        return 0;
-    }
+    FatalErrorInFunction
+        << "Unsupported communications type " << int(commsType)
+        << Foam::abort(FatalError);
+
+    return 0;
 }
 
 

src/Pstream/mpi/UOPwrite.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           |
+    \\  /    A nd           | Copyright (C) 2019 OpenCFD Ltd.
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
                             | Copyright (C) 2011-2017 OpenFOAM Foundation
@@ -30,6 +30,7 @@ Description
 
 #include "UOPstream.H"
 #include "PstreamGlobals.H"
+#include "profilingPstream.H"
 
 #include <mpi.h>
 
@@ -70,6 +71,8 @@ bool Foam::UOPstream::write
 
     bool transferFailed = true;
 
+    profilingPstream::beginTiming();
+
     if (commsType == commsTypes::blocking)
     {
         transferFailed = MPI_Bsend
@@ -82,6 +85,8 @@ bool Foam::UOPstream::write
             PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
         );
 
+        profilingPstream::addGatherTime();
+
         if (debug)
         {
             Pout<< "UOPstream::write : finished write to:" << toProcNo
@@ -102,6 +107,8 @@ bool Foam::UOPstream::write
             PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
         );
 
+        profilingPstream::addWaitTime();
+
         if (debug)
         {
             Pout<< "UOPstream::write : finished write to:" << toProcNo
@@ -125,6 +132,8 @@ bool Foam::UOPstream::write
             &request
         );
 
+        profilingPstream::addWaitTime();
+
         if (debug)
         {
             Pout<< "UOPstream::write : started write to:" << toProcNo

src/Pstream/mpi/UPstream.C

@@ -28,6 +28,7 @@ License
 #include "Pstream.H"
 #include "PstreamReduceOps.H"
 #include "PstreamGlobals.H"
+#include "profilingPstream.H"
 #include "SubList.H"
 #include "allReduce.H"
 #include "int.H"
@@ -501,6 +502,8 @@ void Foam::UPstream::allToAll
     }
     else
     {
+        profilingPstream::beginTiming();
+
         if
         (
             MPI_Alltoall
@@ -522,6 +525,8 @@ void Foam::UPstream::allToAll
                 << " on communicator " << communicator
                 << Foam::abort(FatalError);
         }
+
+        profilingPstream::addAllToAllTime();
     }
 }
 
@@ -572,6 +577,8 @@ void Foam::UPstream::allToAll
     }
     else
     {
+        profilingPstream::beginTiming();
+
         if
         (
             MPI_Alltoallv
@@ -594,6 +601,8 @@ void Foam::UPstream::allToAll
                 << " communicator " << communicator
                 << Foam::abort(FatalError);
         }
+
+        profilingPstream::addAllToAllTime();
     }
 }
 
@@ -634,6 +643,8 @@ void Foam::UPstream::gather
     }
     else
     {
+        profilingPstream::beginTiming();
+
         if
         (
             MPI_Gatherv
@@ -656,6 +667,8 @@ void Foam::UPstream::gather
                 << " communicator " << communicator
                 << Foam::abort(FatalError);
         }
+
+        profilingPstream::addGatherTime();
     }
 }
 
@@ -693,6 +706,8 @@ void Foam::UPstream::scatter
     }
     else
     {
+        profilingPstream::beginTiming();
+
         if
         (
             MPI_Scatterv
@@ -715,6 +730,8 @@ void Foam::UPstream::scatter
                 << " communicator " << communicator
                 << Foam::abort(FatalError);
         }
+
+        profilingPstream::addScatterTime();
     }
 }
 
@@ -869,6 +886,8 @@ void Foam::UPstream::waitRequests(const label start)
             start
         );
 
+        profilingPstream::beginTiming();
+
         if
         (
             MPI_Waitall
@@ -883,6 +902,8 @@ void Foam::UPstream::waitRequests(const label start)
                 << "MPI_Waitall returned with error" << Foam::endl;
         }
 
+        profilingPstream::addWaitTime();
+
         resetRequests(start);
     }
 
@@ -911,6 +932,8 @@ void Foam::UPstream::waitRequest(const label i)
             << Foam::abort(FatalError);
     }
 
+    profilingPstream::beginTiming();
+
     if
     (
         MPI_Wait
@@ -924,6 +947,8 @@ void Foam::UPstream::waitRequest(const label i)
             << "MPI_Wait returned with error" << Foam::endl;
     }
 
+    profilingPstream::addWaitTime();
+
     if (debug)
     {
         Pout<< "UPstream::waitRequest : finished wait for request:" << i

src/Pstream/mpi/allReduceTemplates.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           |
+    \\  /    A nd           | Copyright (C) 2019 OpenCFD Ltd.
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
                             | Copyright (C) 2012-2015 OpenFOAM Foundation
@@ -26,6 +26,7 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "allReduce.H"
+#include "profilingPstream.H"
 
 // * * * * * * * * * * * * * * * Global Functions  * * * * * * * * * * * * * //
 
@@ -46,6 +47,8 @@ void Foam::allReduce
         return;
     }
 
+    profilingPstream::beginTiming();
+
     if (UPstream::nProcs(communicator) <= UPstream::nProcsSimpleSum)
     {
         if (UPstream::master(communicator))
@@ -167,6 +170,8 @@ void Foam::allReduce
         );
         Value = sum;
     }
+
+    profilingPstream::addReduceTime();
 }
 
 

src/functionObjects/utilities/Make/files

@@ -12,6 +12,8 @@ vtkWrite/vtkWriteUpdate.C
 
 removeRegisteredObject/removeRegisteredObject.C
 
+parProfiling/parProfiling.C
+
 solverInfo/solverInfo.C
 
 runTimeControl/runTimeControl.C

src/functionObjects/utilities/parProfiling/parProfiling.C

+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2019 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "parProfiling.H"
+#include "addToRunTimeSelectionTable.H"
+#include "UPstream.H"
+#include "Pstream.H"
+#include "PstreamReduceOps.H"
+#include "profilingPstream.H"
+#include "Tuple2.H"
+#include "FixedList.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+namespace Foam
+{
+namespace functionObjects
+{
+    defineTypeNameAndDebug(parProfiling, 0);
+
+    addToRunTimeSelectionTable
+    (
+        functionObject,
+        parProfiling,
+        dictionary
+    );