Commit 828f8e85 authored by Mark Olesen's avatar Mark Olesen Committed by Andrew Heather
Browse files

ENH: add simple profiling of MPI communications

parent 53be1998
......@@ -11,6 +11,7 @@ global/profiling/profiling.C
global/profiling/profilingInformation.C
global/profiling/profilingSysInfo.C
global/profiling/profilingTrigger.C
global/profiling/profilingPstream.C
global/etcFiles/etcFiles.C
global/version/foamVersion.C
......
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "profilingPstream.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
Foam::autoPtr<Foam::cpuTime> Foam::profilingPstream::timer_(nullptr);
Foam::autoPtr<Foam::cpuTime> Foam::profilingPstream::suspend_(nullptr);
Foam::FixedList<Foam::scalar, 5> Foam::profilingPstream::times_(Zero);
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::profilingPstream::profilingPstream()
{
enable();
}
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::profilingPstream::~profilingPstream()
{
disable();
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::profilingPstream::enable()
{
if (timer_.valid())
{
timer_->resetCpuTime(); // Not really needed ...
}
else if (suspend_.valid())
{
suspend_.swap(timer_);
timer_->resetCpuTime(); // Not really needed ...
}
else
{
timer_.reset(new cpuTime);
times_ = Zero;
}
suspend_.clear();
}
void Foam::profilingPstream::disable()
{
timer_.clear();
suspend_.clear();
}
void Foam::profilingPstream::suspend()
{
suspend_.clear();
suspend_.swap(timer_);
}
void Foam::profilingPstream::resume()
{
if (suspend_.valid())
{
timer_.clear();
timer_.swap(suspend_);
}
}
// ************************************************************************* //
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::profilingPstream
Description
Timers and values for simple (simplistic) mpi-profiling. The entire
class behaves as a singleton.
SourceFiles
profilingPstream.C
\*---------------------------------------------------------------------------*/
#ifndef profilingPstream_H
#define profilingPstream_H
#include "autoPtr.H"
#include "cpuTime.H"
#include "scalar.H"
#include "FixedList.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam
{
/*---------------------------------------------------------------------------*\
Class profilingPstream Declaration
\*---------------------------------------------------------------------------*/
class profilingPstream
{
//- Timer to use
static autoPtr<cpuTime> timer_;
//- Stash for timer during suspend
static autoPtr<cpuTime> suspend_;
//- The timing values
static FixedList<scalar, 5> times_;
public:
//- Enumeration within times array
enum timingType
{
GATHER = 0,
SCATTER,
REDUCE,
WAIT,
ALL_TO_ALL
};
public:
// Constructors
//- Construct and enable global timer
profilingPstream();
//- Destructor - remove global timer
~profilingPstream();
// Member Functions
//- Create timer for measuring communication, or reset existing
static void enable();
//- Remove timer for measuring communication activity
static void disable();
//- Suspend use of timer (if active)
static void suspend();
//- Resume use of timer (if previously active)
static void resume();
//- Timer is active
inline static bool active()
{
return timer_.valid();
}
//- Access to the timing information
inline static FixedList<scalar, 5>& times()
{
return times_;
}
//- Access to the timing information
inline static scalar times(const enum timingType idx)
{
return times_[idx];
}
//- Update timer prior to measurement
inline static void beginTiming()
{
if (timer_.valid())
{
(void) timer_->cpuTimeIncrement();
}
}
//- Add time increment
inline static void addTime(const enum timingType idx)
{
if (timer_.valid())
{
times_[idx] += timer_->cpuTimeIncrement();
}
}
//- Add time increment to gatherTime
inline static void addGatherTime()
{
addTime(GATHER);
}
//- Add time increment to scatterTime
inline static void addScatterTime()
{
addTime(SCATTER);
}
//- Add time increment to reduceTime
inline static void addReduceTime()
{
addTime(REDUCE);
}
//- Add time increment to waitTime
inline static void addWaitTime()
{
addTime(WAIT);
}
//- Add time increment to allToAllTime
inline static void addAllToAllTime()
{
addTime(ALL_TO_ALL);
}
};
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#endif
// ************************************************************************* //
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
| Copyright (C) 2011-2017 OpenFOAM Foundation
......@@ -30,6 +30,7 @@ Description
#include "UIPstream.H"
#include "PstreamGlobals.H"
#include "profilingPstream.H"
#include "IOstreams.H"
#include <mpi.h>
......@@ -85,6 +86,8 @@ Foam::UIPstream::UIPstream
// and set it
if (!wantedSize)
{
profilingPstream::beginTiming();
MPI_Probe
(
fromProcNo_,
......@@ -94,6 +97,8 @@ Foam::UIPstream::UIPstream
);
MPI_Get_count(&status, MPI_BYTE, &messageSize_);
profilingPstream::addWaitTime();
externalBuf_.setCapacity(messageSize_);
wantedSize = messageSize_;
......@@ -186,6 +191,8 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
// and set it
if (!wantedSize)
{
profilingPstream::beginTiming();
MPI_Probe
(
fromProcNo_,
......@@ -195,6 +202,8 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
);
MPI_Get_count(&status, MPI_BYTE, &messageSize_);
profilingPstream::addWaitTime();
externalBuf_.setCapacity(messageSize_);
wantedSize = messageSize_;
......@@ -257,6 +266,8 @@ Foam::label Foam::UIPstream::read
error::printStack(Pout);
}
profilingPstream::beginTiming();
if (commsType == commsTypes::blocking || commsType == commsTypes::scheduled)
{
MPI_Status status;
......@@ -282,6 +293,7 @@ Foam::label Foam::UIPstream::read
return 0;
}
profilingPstream::addScatterTime();
// Check size of message read
......@@ -332,6 +344,8 @@ Foam::label Foam::UIPstream::read
return 0;
}
profilingPstream::addScatterTime();
if (debug)
{
Pout<< "UIPstream::read : started read from:" << fromProcNo
......@@ -346,15 +360,12 @@ Foam::label Foam::UIPstream::read
// Assume the message is completely received.
return bufSize;
}
else
{
FatalErrorInFunction
<< "Unsupported communications type "
<< int(commsType)
<< Foam::abort(FatalError);
return 0;
}
FatalErrorInFunction
<< "Unsupported communications type " << int(commsType)
<< Foam::abort(FatalError);
return 0;
}
......
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
| Copyright (C) 2011-2017 OpenFOAM Foundation
......@@ -30,6 +30,7 @@ Description
#include "UOPstream.H"
#include "PstreamGlobals.H"
#include "profilingPstream.H"
#include <mpi.h>
......@@ -70,6 +71,8 @@ bool Foam::UOPstream::write
bool transferFailed = true;
profilingPstream::beginTiming();
if (commsType == commsTypes::blocking)
{
transferFailed = MPI_Bsend
......@@ -82,6 +85,8 @@ bool Foam::UOPstream::write
PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
);
profilingPstream::addGatherTime();
if (debug)
{
Pout<< "UOPstream::write : finished write to:" << toProcNo
......@@ -102,6 +107,8 @@ bool Foam::UOPstream::write
PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
);
profilingPstream::addWaitTime();
if (debug)
{
Pout<< "UOPstream::write : finished write to:" << toProcNo
......@@ -125,6 +132,8 @@ bool Foam::UOPstream::write
&request
);
profilingPstream::addWaitTime();
if (debug)
{
Pout<< "UOPstream::write : started write to:" << toProcNo
......
......@@ -28,6 +28,7 @@ License
#include "Pstream.H"
#include "PstreamReduceOps.H"
#include "PstreamGlobals.H"
#include "profilingPstream.H"
#include "SubList.H"
#include "allReduce.H"
#include "int.H"
......@@ -501,6 +502,8 @@ void Foam::UPstream::allToAll
}
else
{
profilingPstream::beginTiming();
if
(
MPI_Alltoall
......@@ -522,6 +525,8 @@ void Foam::UPstream::allToAll
<< " on communicator " << communicator
<< Foam::abort(FatalError);
}
profilingPstream::addAllToAllTime();
}
}
......@@ -572,6 +577,8 @@ void Foam::UPstream::allToAll
}
else
{
profilingPstream::beginTiming();
if
(
MPI_Alltoallv
......@@ -594,6 +601,8 @@ void Foam::UPstream::allToAll
<< " communicator " << communicator
<< Foam::abort(FatalError);
}
profilingPstream::addAllToAllTime();
}
}
......@@ -634,6 +643,8 @@ void Foam::UPstream::gather
}
else
{
profilingPstream::beginTiming();
if
(
MPI_Gatherv
......@@ -656,6 +667,8 @@ void Foam::UPstream::gather
<< " communicator " << communicator
<< Foam::abort(FatalError);
}
profilingPstream::addGatherTime();
}
}
......@@ -693,6 +706,8 @@ void Foam::UPstream::scatter
}
else
{
profilingPstream::beginTiming();
if
(
MPI_Scatterv
......@@ -715,6 +730,8 @@ void Foam::UPstream::scatter
<< " communicator " << communicator
<< Foam::abort(FatalError);
}
profilingPstream::addScatterTime();
}
}
......@@ -869,6 +886,8 @@ void Foam::UPstream::waitRequests(const label start)
start
);
profilingPstream::beginTiming();
if
(
MPI_Waitall
......@@ -883,6 +902,8 @@ void Foam::UPstream::waitRequests(const label start)
<< "MPI_Waitall returned with error" << Foam::endl;
}
profilingPstream::addWaitTime();
resetRequests(start);
}
......@@ -911,6 +932,8 @@ void Foam::UPstream::waitRequest(const label i)
<< Foam::abort(FatalError);
}
profilingPstream::beginTiming();
if
(
MPI_Wait
......@@ -924,6 +947,8 @@ void Foam::UPstream::waitRequest(const label i)
<< "MPI_Wait returned with error" << Foam::endl;
}
profilingPstream::addWaitTime();
if (debug)
{
Pout<< "UPstream::waitRequest : finished wait for request:" << i
......
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
| Copyright (C) 2012-2015 OpenFOAM Foundation
......@@ -26,6 +26,7 @@ License
\*---------------------------------------------------------------------------*/
#include "allReduce.H"
#include "profilingPstream.H"
// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * //
......@@ -46,6 +47,8 @@ void Foam::allReduce
return;
}
profilingPstream::beginTiming();
if (UPstream::nProcs(communicator) <= UPstream::nProcsSimpleSum)
{
if (UPstream::master(communicator))
......@@ -167,6 +170,8 @@ void Foam::allReduce
);
Value = sum;
}
profilingPstream::addReduceTime();
}
......
......@@ -12,6 +12,8 @@ vtkWrite/vtkWriteUpdate.C
removeRegisteredObject/removeRegisteredObject.C
parProfiling/parProfiling.C
solverInfo/solverInfo.C
runTimeControl/runTimeControl.C
......
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "parProfiling.H"
#include "addToRunTimeSelectionTable.H"
#include "UPstream.H"
#include "Pstream.H"
#include "PstreamReduceOps.H"
#include "profilingPstream.H"
#include "Tuple2.H"
#include "FixedList.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
namespace Foam
{
namespace functionObjects
{
defineTypeNameAndDebug(parProfiling, 0);
addToRunTimeSelectionTable
(
functionObject,
parProfiling,
dictionary
);