diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C
index 79da7ae93b47eca540f218f240ca458fa3c098c0..cc80139f0e68f4c65b2d932962718b4a84fcb231 100644
--- a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C
+++ b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C
@@ -47,7 +47,7 @@ void Foam::masterOFstream::checkWrite
return;
}
- mkDir(fName.path());
+ Foam::mkDir(fName.path());
OFstream os
(
@@ -89,19 +89,23 @@ void Foam::masterOFstream::checkWrite
void Foam::masterOFstream::commit()
{
- if (Pstream::parRun())
+ if (UPstream::parRun())
{
- List<fileName> filePaths(Pstream::nProcs());
- filePaths[Pstream::myProcNo()] = pathName_;
- Pstream::gatherList(filePaths);
+ List<fileName> filePaths(UPstream::nProcs(comm_));
+ filePaths[UPstream::myProcNo(comm_)] = pathName_;
+ Pstream::gatherList(filePaths, UPstream::msgType(), comm_);
- bool uniform = fileOperation::uniformFile(filePaths);
+ bool uniform =
+ (
+ UPstream::master(comm_)
+ && fileOperation::uniformFile(filePaths)
+ );
- Pstream::broadcast(uniform);
+ Pstream::broadcast(uniform, comm_);
if (uniform)
{
- if (Pstream::master() && writeOnProc_)
+ if (UPstream::master(comm_) && writeOnProc_)
{
checkWrite(pathName_, this->str());
}
@@ -111,36 +115,38 @@ void Foam::masterOFstream::commit()
}
// Different files
- PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
+ PstreamBuffers pBufs(comm_, UPstream::commsTypes::nonBlocking);
- // Send my (valid) buffer to master
- if (!Pstream::master())
+ if (!UPstream::master(comm_))
{
if (writeOnProc_)
{
+ // Send buffer to master
string s(this->str());
- UOPstream os(Pstream::masterNo(), pBufs);
+ UOPstream os(UPstream::masterNo(), pBufs);
os.write(s.data(), s.length());
}
- this->reset();
+ this->reset(); // Done with contents
}
pBufs.finishedGathers();
- if (Pstream::master())
+
+ if (UPstream::master(comm_))
{
- // Write (valid) master data
if (writeOnProc_)
{
- checkWrite(filePaths[Pstream::masterNo()], this->str());
+ // Write master data
+ checkWrite(filePaths[UPstream::masterNo()], this->str());
}
- this->reset();
+ this->reset(); // Done with contents
+
// Allocate large enough to read without resizing
List<char> buf(pBufs.maxRecvCount());
- for (const int proci : Pstream::subProcs())
+ for (const int proci : UPstream::subProcs(comm_))
{
const std::streamsize count(pBufs.recvDataCount(proci));
@@ -170,6 +176,7 @@ void Foam::masterOFstream::commit()
Foam::masterOFstream::masterOFstream
(
IOstreamOption::atomicType atomic,
+ const label comm,
const fileName& pathName,
IOstreamOption streamOpt,
IOstreamOption::appendType append,
@@ -181,7 +188,8 @@ Foam::masterOFstream::masterOFstream
atomic_(atomic),
compression_(streamOpt.compression()),
append_(append),
- writeOnProc_(writeOnProc)
+ writeOnProc_(writeOnProc),
+ comm_(comm)
{}
diff --git a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H
index 4abeaf17e8cd6077d6cb647c30f0429e1669723b..c139e00e0de7a1cbe3693c1b0723dc5d78b88186 100644
--- a/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H
+++ b/src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017 OpenFOAM Foundation
- Copyright (C) 2020-2022 OpenCFD Ltd.
+ Copyright (C) 2020-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -30,6 +30,9 @@ Class
Description
Master-only drop-in replacement for OFstream.
+ Called on all processors (of the provided communicator).
+ Sends files to the master and writes them there.
+
SourceFiles
masterOFstream.C
@@ -39,6 +42,7 @@ SourceFiles
#define Foam_masterOFstream_H
#include "StringStream.H"
+#include "UPstream.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@@ -70,6 +74,9 @@ class masterOFstream
//- Should file be written (on this processor)
const bool writeOnProc_;
+ //- Communicator
+ const label comm_;
+
// Private Member Functions
@@ -92,17 +99,62 @@ public:
// Constructors
- //- Construct with specified atomic behaviour (with worldComm)
+ //- Construct with specified atomic behaviour and communicator
//- from pathname, stream option, optional append
masterOFstream
(
IOstreamOption::atomicType atomic,
+ const label comm,
const fileName& pathname,
IOstreamOption streamOpt = IOstreamOption(),
IOstreamOption::appendType append = IOstreamOption::NON_APPEND,
const bool writeOnProc = true
);
+ //- Construct with specified communicator
+ //- from pathname, stream option, optional append
+ masterOFstream
+ (
+ const label comm,
+ const fileName& pathname,
+ IOstreamOption streamOpt = IOstreamOption(),
+ IOstreamOption::appendType append = IOstreamOption::NON_APPEND,
+ const bool writeOnProc = true
+ )
+ :
+ masterOFstream
+ (
+ IOstreamOption::NON_ATOMIC,
+ comm,
+ pathname,
+ streamOpt,
+ append,
+ writeOnProc
+ )
+ {}
+
+ //- Construct with specified atomic behaviour (with worldComm)
+ //- from pathname, stream option, optional append
+ masterOFstream
+ (
+ IOstreamOption::atomicType atomic,
+ const fileName& pathname,
+ IOstreamOption streamOpt = IOstreamOption(),
+ IOstreamOption::appendType append = IOstreamOption::NON_APPEND,
+ const bool writeOnProc = true
+ )
+ :
+ masterOFstream
+ (
+ atomic,
+ UPstream::worldComm,
+ pathname,
+ streamOpt,
+ append,
+ writeOnProc
+ )
+ {}
+
//- Construct (with worldComm)
//- from pathname, stream option, optional append
explicit masterOFstream
@@ -116,6 +168,7 @@ public:
masterOFstream
(
IOstreamOption::NON_ATOMIC,
+ UPstream::worldComm,
pathname,
streamOpt,
append,
diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
index 191ae1f83a70cc6286c17142d2f136ac4e01e8b8..1c846ae5d8702f110ab3edfb5d9efa492d760f43 100644
--- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
@@ -50,6 +50,12 @@ namespace fileOperations
collatedFileOperation,
word
);
+ addToRunTimeSelectionTable
+ (
+ fileOperation,
+ collatedFileOperation,
+ comm
+ );
float collatedFileOperation::maxThreadFileBufferSize
(
@@ -146,6 +152,7 @@ void Foam::fileOperations::collatedFileOperation::printBanner
}
}
+ //- fileModificationChecking already set by base class (masterUncollated)
// if (IOobject::fileModificationChecking == IOobject::timeStampMaster)
// {
// WarningInFunction
@@ -159,29 +166,6 @@ void Foam::fileOperations::collatedFileOperation::printBanner
}
-bool Foam::fileOperations::collatedFileOperation::isMasterRank
-(
- const label proci
-)
-const
-{
- if (Pstream::parRun())
- {
- return Pstream::master(comm_);
- }
- else if (ioRanks_.size())
- {
- // Found myself in IO rank
- return ioRanks_.found(proci);
- }
- else
- {
- // Assume all in single communicator
- return proci == 0;
- }
-}
-
-
bool Foam::fileOperations::collatedFileOperation::appendObject
(
const regIOobject& io,
@@ -207,10 +191,10 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
<< exit(FatalError);
}
- const bool isMaster = isMasterRank(proci);
+ const bool isIOmaster = fileOperation::isIOrank(proci);
// Update meta-data for current state
- if (isMaster)
+ if (isIOmaster)
{
const_cast<regIOobject&>(io).updateMetaData();
}
@@ -227,7 +211,7 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
// UNCOMPRESSED (binary only)
IOstreamOption(IOstreamOption::BINARY, streamOpt.version()),
// Append on sub-ranks
- (isMaster ? IOstreamOption::NON_APPEND : IOstreamOption::APPEND)
+ (isIOmaster ? IOstreamOption::NON_APPEND : IOstreamOption::APPEND)
);
if (!os.good())
@@ -237,7 +221,7 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
<< exit(FatalIOError);
}
- if (isMaster)
+ if (isIOmaster)
{
decomposedBlockData::writeHeader(os, streamOpt, io);
}
@@ -248,14 +232,44 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
streamOpt,
io,
proci,
- // With FoamFile header on master?
- isMaster
+ isIOmaster // With FoamFile header on master
);
return (blockOffset >= 0) && os.good();
}
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// Construction helper: self/world/local communicator and IO ranks
+static Tuple2<label, labelList> getCommPattern()
+{
+ // Default is COMM_WORLD (single master)
+ Tuple2<label, labelList> commAndIORanks
+ (
+ UPstream::worldComm,
+ fileOperation::getGlobalIORanks()
+ );
+
+ if (UPstream::parRun() && commAndIORanks.second().size() > 1)
+ {
+ // Multiple masters: ranks for my IO range
+ commAndIORanks.first() = UPstream::allocateCommunicator
+ (
+ UPstream::worldComm,
+ fileOperation::subRanks(commAndIORanks.second())
+ );
+ }
+
+ return commAndIORanks;
+}
+
+} // End namespace Foam
+
+
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
void Foam::fileOperations::collatedFileOperation::init(bool verbose)
@@ -276,21 +290,12 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
:
masterUncollatedFileOperation
(
- (
- fileOperation::getGlobalIORanks().size()
- ? UPstream::allocateCommunicator
- (
- UPstream::worldComm,
- subRanks(UPstream::nProcs())
- )
- : UPstream::worldComm
- ),
- false
+ getCommPattern(),
+ false, // distributedRoots
+ false // verbose
),
managedComm_(getManagedComm(comm_)), // Possibly locally allocated
- writer_(mag(maxThreadFileBufferSize), comm_),
- nProcs_(Pstream::nProcs()),
- ioRanks_(fileOperation::getGlobalIORanks())
+ writer_(mag(maxThreadFileBufferSize), comm_)
{
init(verbose);
}
@@ -298,16 +303,19 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
Foam::fileOperations::collatedFileOperation::collatedFileOperation
(
- const label comm,
- const labelList& ioRanks,
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
bool verbose
)
:
- masterUncollatedFileOperation(comm, false),
+ masterUncollatedFileOperation
+ (
+ commAndIORanks,
+ distributedRoots,
+ false // verbose
+ ),
managedComm_(-1), // Externally managed
- writer_(mag(maxThreadFileBufferSize), comm),
- nProcs_(Pstream::nProcs()),
- ioRanks_(ioRanks)
+ writer_(mag(maxThreadFileBufferSize), comm_)
{
init(verbose);
}
@@ -392,6 +400,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
// Note: currently still NON_ATOMIC (Dec-2022)
masterOFstream os
(
+ comm_,
pathName,
streamOpt,
IOstreamOption::NON_APPEND,
@@ -423,7 +432,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
mkDir(path);
fileName pathName(path/io.name());
- if (io.global())
+ if (io.global() || io.globalObject())
{
if (debug)
{
@@ -436,6 +445,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
// Note: currently still NON_ATOMIC (Dec-2022)
masterOFstream os
(
+ comm_,
pathName,
streamOpt,
IOstreamOption::NON_APPEND,
@@ -459,7 +469,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
return ok;
}
- else if (!Pstream::parRun())
+ else if (!UPstream::parRun())
{
// Special path for e.g. decomposePar. Append to
// processorsDDD/ file
@@ -502,7 +512,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
bool ok = os.good();
- if (Pstream::master(comm_))
+ if (UPstream::master(comm_))
{
// Suppress comment banner
const bool old = IOobject::bannerEnabled(false);
@@ -548,13 +558,13 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
const fileName& fName
) const
{
- if (Pstream::parRun())
+ if (UPstream::parRun())
{
const List<int>& procs(UPstream::procID(comm_));
- word procDir(processorsBaseDir+Foam::name(Pstream::nProcs()));
+ word procDir(processorsBaseDir+Foam::name(nProcs_));
- if (procs.size() != Pstream::nProcs())
+ if (procs.size() != nProcs_)
{
procDir +=
+ "_"
@@ -616,16 +626,4 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
}
-void Foam::fileOperations::collatedFileOperation::setNProcs(const label nProcs)
-{
- nProcs_ = nProcs;
-
- if (debug)
- {
- Pout<< "collatedFileOperation::setNProcs :"
- << " Setting number of processors to " << nProcs_ << endl;
- }
-}
-
-
// ************************************************************************* //
diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H
index bfec2a9e322f33b9dd3077a7cdb15221175b454b..d2c9ad9888fc813e973a65bbc1bd5e7c1d37005d 100644
--- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017 OpenFOAM Foundation
- Copyright (C) 2019-2022 OpenCFD Ltd.
+ Copyright (C) 2019-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -86,24 +86,12 @@ protected:
//- Threaded writer
mutable OFstreamCollator writer_;
- // For non-parallel operation
-
- //- Number of processors (overall)
- label nProcs_;
-
- //- Ranks of IO handlers
- const labelList ioRanks_;
-
// Protected Member Functions
//- Print banner information, optionally with io ranks
void printBanner(const bool withRanks = false) const;
- //- Is proci master of communicator (in parallel) or master of
- //- the io ranks (non-parallel)
- bool isMasterRank(const label proci) const;
-
//- Append to processorsNN/ file
bool appendObject
(
@@ -115,8 +103,8 @@ protected:
public:
- //- Runtime type information
- TypeName("collated");
+ //- Runtime type information
+ TypeName("collated");
// Static Data
@@ -132,11 +120,11 @@ public:
//- Default construct
explicit collatedFileOperation(bool verbose = false);
- //- Construct from user communicator
- collatedFileOperation
+ //- Construct from communicator with specified io-ranks
+ explicit collatedFileOperation
(
- const label comm,
- const labelList& ioRanks,
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
bool verbose = false
);
@@ -170,6 +158,7 @@ public:
const bool writeOnProc = true
) const;
+
// Other
//- Forcibly wait until all output done. Flush any cached data
@@ -180,10 +169,6 @@ public:
//- Actual name of processors dir
virtual word processorsDir(const fileName&) const;
-
- //- Set number of processor directories/results.
- //- Only used in decomposePar
- virtual void setNProcs(const label nProcs);
};
diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C
index 1bbf105fe8879645ed8057b9813d1a2f424b0d4e..f7ede70f30fcafdfb3de65f04c5b0387851eda00 100644
--- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C
@@ -28,7 +28,6 @@ License
#include "hostCollatedFileOperation.H"
#include "addToRunTimeSelectionTable.H"
-#include "bitSet.H"
/* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */
@@ -43,6 +42,12 @@ namespace fileOperations
hostCollatedFileOperation,
word
);
+ addToRunTimeSelectionTable
+ (
+ fileOperation,
+ hostCollatedFileOperation,
+ comm
+ );
// Register initialisation routine. Signals need for threaded mpi and
// handles command line arguments
@@ -57,25 +62,41 @@ namespace fileOperations
}
-// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
-Foam::labelList Foam::fileOperations::hostCollatedFileOperation::subRanks
-(
- const label n
-)
+namespace Foam
{
- labelList mainIOranks(fileOperation::getGlobalIORanks());
+// Construction helper: self/world/local communicator and IO ranks
+static Tuple2<label, labelList> getCommPattern()
+{
+ // Default is COMM_WORLD (single master)
+ Tuple2<label, labelList> commAndIORanks
+ (
+ UPstream::worldComm,
+ fileOperation::getGlobalIORanks()
+ );
- if (mainIOranks.empty())
+ if (commAndIORanks.second().empty())
{
- mainIOranks = fileOperation::getGlobalHostIORanks();
+ // Default: one master per host
+ commAndIORanks.second() = fileOperation::getGlobalHostIORanks();
}
- labelRange subRange = fileOperation::subRanks(mainIOranks);
+ if (UPstream::parRun() && commAndIORanks.second().size() > 1)
+ {
+ // Multiple masters: ranks for my IO range
+ commAndIORanks.first() = UPstream::allocateCommunicator
+ (
+ UPstream::worldComm,
+ fileOperation::subRanks(commAndIORanks.second())
+ );
+ }
- return identity(subRange);
+ return commAndIORanks;
}
+} // End namespace Foam
+
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
@@ -97,13 +118,9 @@ Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation
:
collatedFileOperation
(
- UPstream::allocateCommunicator
- (
- UPstream::worldComm,
- subRanks(UPstream::nProcs())
- ),
- (UPstream::parRun() ? labelList() : getGlobalIORanks()),
- false // verbose
+ getCommPattern(),
+ false, // distributedRoots
+ false // verbose
),
managedComm_(getManagedComm(comm_)) // Possibly locally allocated
{
@@ -111,6 +128,25 @@ Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation
}
+Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation
+(
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
+ bool verbose
+)
+:
+ collatedFileOperation
+ (
+ commAndIORanks,
+ distributedRoots,
+ false // verbose
+ ),
+ managedComm_(-1) // Externally managed
+{
+ init(verbose);
+}
+
+
void Foam::fileOperations::hostCollatedFileOperation::storeComm() const
{
// From externally -> locally managed
diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H
index f6ea90cf1732724b419c1ff8ccd857aae2357001..23840308c82539a8169b558a3be569b0414bbe45 100644
--- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H
@@ -96,8 +96,6 @@ class hostCollatedFileOperation
//- Any initialisation steps after constructing
void init(bool verbose);
- //- Get the list of processors part of this set
- static labelList subRanks(const label n);
public:
@@ -110,6 +108,14 @@ public:
//- Default construct
explicit hostCollatedFileOperation(bool verbose = false);
+ //- Construct from communicator with specified io-ranks
+ explicit hostCollatedFileOperation
+ (
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
+ bool verbose = false
+ );
+
//- Destructor
virtual ~hostCollatedFileOperation();
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
index 996c7d6e8f615fd8dc6491882b499ef0fe307ce9..08710c72fa321b3c714bb0ea3e062078f00d7ac6 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
@@ -27,16 +27,12 @@ License
\*---------------------------------------------------------------------------*/
#include "fileOperation.H"
-#include "regIOobject.H"
-#include "argList.H"
-#include "HashSet.H"
#include "objectRegistry.H"
-#include "decomposedBlockData.H"
-#include "polyMesh.H"
+#include "labelIOList.H"
#include "registerSwitch.H"
+#include "stringOps.H"
#include "Time.H"
-#include "ITstream.H"
-#include <cerrno>
+#include "OSspecific.H" // for Foam::isDir etc
#include <cinttypes>
/* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */
@@ -45,6 +41,7 @@ namespace Foam
{
defineTypeNameAndDebug(fileOperation, 0);
defineRunTimeSelectionTable(fileOperation, word);
+ defineRunTimeSelectionTable(fileOperation, comm);
word fileOperation::defaultFileHandler
(
@@ -80,6 +77,9 @@ Foam::fileOperation::pathTypeNames_
Foam::word Foam::fileOperation::processorsBaseDir = "processors";
+//- Caching (e.g. of time directories) - enabled by default
+int Foam::fileOperation::cacheLevel_(1);
+
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
@@ -425,6 +425,13 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
// find the corresponding actual processor directory (e.g. 'processors4')
// and index (2)
+ // Behaviour affected by
+ // - UPstream::parRun()
+ // - syncPar : usually true, only uncollated does false. Determines
+ // if directory status gets synchronised
+ // - distributed() : different processors have different roots
+ // - fileModificationChecking : (uncollated only) do IO on master only
+
fileName path, pDir, local;
procRangeType group;
label numProcs;
@@ -435,11 +442,14 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
{
const fileName procPath(path/pDir);
- const auto iter = procsDirs_.cfind(procPath);
-
- if (iter.good())
+ if (cacheLevel() > 0)
{
- return iter.val();
+ const auto iter = procsDirs_.cfind(procPath);
+
+ if (iter.good())
+ {
+ return iter.val();
+ }
}
DynamicList<dirIndex> procDirs;
@@ -453,7 +463,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
const bool readDirMasterOnly
(
- Pstream::parRun() && !distributed()
+ UPstream::parRun() && !distributed()
&&
(
IOobject::fileModificationChecking == IOobject::timeStampMaster
@@ -469,7 +479,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
// Parallel and non-distributed
// Read on master only and send to subProcs
- if (Pstream::master(comm_))
+ if (UPstream::master(UPstream::worldComm))
{
dirEntries = Foam::readDir(path, fileName::Type::DIRECTORY);
@@ -478,7 +488,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
<< " names to sub-processes" << endl;
}
- Pstream::broadcast(dirEntries, comm_);
+ Pstream::broadcast(dirEntries, UPstream::worldComm);
}
else
{
@@ -534,7 +544,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
pathTypeIdx.second() = proci;
}
}
- else if (group.found(proci))
+ else if (group.contains(proci))
{
// "processorsNN_start-end"
// - save the local proc offset
@@ -564,7 +574,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
<< " detected:" << procDirs << endl;
}
- if (Pstream::parRun() && (!distributed() || syncPar))
+ if (UPstream::parRun() && (!distributed() || syncPar))
{
reduce(procDirsStatus, bitOrOp<unsigned>()); // worldComm
@@ -595,7 +605,8 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
if
(
pathTypeIdx.first() == pathType::PROCBASEOBJECT
- && proci < nProcs
+ // Do not restrict to currently used processors
+ // && proci < nProcs
)
{
pathTypeIdx.second() = proci;
@@ -634,12 +645,12 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
}
}
}
- else if (!Pstream::parRun())
+ else if (!UPstream::parRun())
{
// Serial: use the number of decompositions (if found)
if (nProcs)
{
- const_cast<fileOperation&>(*this).setNProcs(nProcs);
+ const_cast<fileOperation&>(*this).nProcs(nProcs);
}
}
@@ -648,10 +659,17 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
if (procDirsStatus & 2u)
{
- procsDirs_.insert(procPath, procDirs);
+ if (cacheLevel() > 0)
+ {
+ procsDirs_.insert(procPath, procDirs);
- // Make sure to return a reference
- return procsDirs_[procPath];
+ // Make sure to return a reference
+ return procsDirs_[procPath];
+ }
+ else
+ {
+ return refPtr<dirIndexList>::New(procDirs);
+ }
}
}
@@ -681,8 +699,11 @@ bool Foam::fileOperation::exists(IOobject& io) const
else
{
ok =
+ (
isFile(objPath)
- && io.typeHeaderOk<IOList<label>>(false);// object with local scope
+ // object with local scope
+ && io.typeHeaderOk<labelIOList>(false)
+ );
}
if (!ok)
@@ -700,8 +721,11 @@ bool Foam::fileOperation::exists(IOobject& io) const
else
{
ok =
+ (
isFile(originalPath)
- && io.typeHeaderOk<IOList<label>>(false);
+ // object with local scope
+ && io.typeHeaderOk<labelIOList>(false)
+ );
}
}
}
@@ -712,14 +736,30 @@ bool Foam::fileOperation::exists(IOobject& io) const
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
+Foam::fileOperation::fileOperation
+(
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots
+)
+:
+ comm_(commAndIORanks.first()),
+ nProcs_(UPstream::nProcs(UPstream::worldComm)),
+ distributed_(distributedRoots),
+ ioRanks_(commAndIORanks.second())
+{}
+
+
Foam::fileOperation::fileOperation
(
const label comm,
+ const labelUList& ioRanks,
const bool distributedRoots
)
:
comm_(comm),
- distributed_(distributedRoots)
+ nProcs_(UPstream::nProcs(UPstream::worldComm)),
+ distributed_(distributedRoots),
+ ioRanks_(ioRanks)
{}
@@ -920,7 +960,7 @@ void Foam::fileOperation::updateStates
const bool syncPar
) const
{
- monitor().updateStates(masterOnly, Pstream::parRun());
+ monitor().updateStates(masterOnly, UPstream::parRun());
}
@@ -1195,10 +1235,6 @@ Foam::fileNameList Foam::fileOperation::readObjects
}
-void Foam::fileOperation::setNProcs(const label nProcs)
-{}
-
-
Foam::label Foam::fileOperation::nProcs
(
const fileName& dir,
@@ -1206,7 +1242,7 @@ Foam::label Foam::fileOperation::nProcs
) const
{
label nProcs = 0;
- if (Pstream::master(comm_))
+ if (UPstream::master(comm_))
{
fileNameList dirNames(Foam::readDir(dir, fileName::Type::DIRECTORY));
@@ -1259,6 +1295,23 @@ void Foam::fileOperation::flush() const
}
+void Foam::fileOperation::sync()
+{
+ if (debug)
+ {
+ Pout<< "fileOperation::sync : parallel synchronisation"
+ << endl;
+ }
+
+ Pstream::broadcasts
+ (
+ UPstream::worldComm,
+ nProcs_,
+ procsDirs_
+ );
+}
+
+
Foam::fileName Foam::fileOperation::processorsCasePath
(
const IOobject& io,
@@ -1491,9 +1544,17 @@ Foam::Ostream& Foam::operator<<
const auto& fp = *iproxy;
os << "fileHandler:" << fp.type()
- // << " nProcs:" << fp.nProcs()
+ << " nProcs:" << fp.nProcs()
<< " comm:" << fp.comm()
- << " distributed:" << fp.distributed() << nl;
+ << " distributed:" << fp.distributed()
+ << " ioranks: " << flatOutput(fp.ioRanks())
+ << " ranks: ";
+
+ if (fp.comm() >= 0)
+ {
+ os << flatOutput(UPstream::procID(fp.comm()));
+ }
+ os << nl;
return os;
}
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
index 721d2ea2c014e63b4b219dee56ba0193fb18dbdb..6c78934f843937c07312c1bada11bef758602ead 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
@@ -80,6 +80,7 @@ Description
#include "fileNameList.H"
#include "instantList.H"
#include "refPtr.H"
+#include "bitSet.H"
#include "Enum.H"
#include "Tuple2.H"
#include "InfoProxy.H"
@@ -144,14 +145,30 @@ public:
protected:
+ // Protected Static Data
+
+ //- Cache level (eg, for caching time directories). Default: 1
+ static int cacheLevel_;
+
+
// Protected Data
//- Communicator to use
mutable label comm_;
+ //- Overall number of processors.
+ // Used to synthesise processor directory naming:
+ // - parallel: UPstream::nProcs(UPstream::worldComm)
+ // - non-parallel: detected from processor dir naming ('processorsNN')
+ label nProcs_;
+
//- Distributed roots (parallel run)
mutable bool distributed_;
+ //- The list of IO ranks (global ranks)
+ // Primarily for additional bookkeeping in non-parallel
+ const labelList ioRanks_;
+
//- Detected processors directories
mutable HashTable<dirIndexList> procsDirs_;
@@ -195,6 +212,15 @@ protected:
//- Is either a directory (empty name()) or a file
bool exists(IOobject& io) const;
+
+ //- Is proci a master rank in the communicator (in parallel)
+ //- or a master rank in the IO ranks (non-parallel)
+ bool isIOrank(const label proci) const;
+
+ //- Helper: output which ranks are IO
+ void printRanks() const;
+
+
//- Construction helper: check for locally allocated communicator
static inline label getManagedComm(const label communicator)
{
@@ -235,13 +261,23 @@ public:
// Constructors
- //- Construct from communicator, optionally with distributed roots
+ //- Construct from communicator,
+ //- optionally with specified io-ranks and/or distributed roots
explicit fileOperation
(
const label comm,
+ const labelUList& ioRanks = labelUList::null(),
+ const bool distributedRoots = false
+ );
+
+ //- Construct from communicator with specified io-ranks
+ explicit fileOperation
+ (
+ const Tuple2<label, labelList>& commAndIORanks,
const bool distributedRoots = false
);
+
//- Clone fileHandler
/// virtual autoPtr<fileOperation> clone() const = 0;
@@ -259,6 +295,19 @@ public:
(verbose)
);
+ declareRunTimeSelectionTable
+ (
+ autoPtr,
+ fileOperation,
+ comm,
+ (
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
+ bool verbose
+ ),
+ (commAndIORanks, distributedRoots, verbose)
+ );
+
// Selectors
@@ -270,6 +319,16 @@ public:
bool verbose = false
);
+ //- Select fileHandler-type.
+ //- Uses defaultFileHandler if the handlerType is empty.
+ static autoPtr<fileOperation> New
+ (
+ const word& handlerType,
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
+ bool verbose = false
+ );
+
//- Destructor
virtual ~fileOperation() = default;
@@ -331,6 +390,20 @@ public:
// Static Functions
+ //- Return cache level
+ static int cacheLevel() noexcept
+ {
+ return cacheLevel_;
+ }
+
+ //- Set cache level (0 = off). \return the previous value
+ static int cacheLevel(int level) noexcept
+ {
+ int old(cacheLevel_);
+ cacheLevel_ = level;
+ return old;
+ }
+
//- Sort directory entries according to time value,
// with "constant" appearing first (if it exists)
static instantList sortTimes
@@ -380,6 +453,10 @@ public:
return old;
}
+ //- The list of IO ranks (global ranks)
+ // Primarily for additional bookkeeping in non-parallel
+ const labelList& ioRanks() const noexcept { return ioRanks_; }
+
//- Return info proxy,
//- used to print information to a stream
InfoProxy<fileOperation> info() const noexcept { return *this; }
@@ -691,12 +768,23 @@ public:
return processorsBaseDir;
}
- //- Set number of processor directories/results. Only used in
- // decomposePar
- virtual void setNProcs(const label nProcs);
+ //- Overall number of processors,
+ //- from UPstream::nProcs() or detected from directories/results.
+ label nProcs() const noexcept { return nProcs_; }
+
+ //- Set number of processor directories/results.
+ // Used to cache format of e.g. processorsDDD.
+ // Returns old number of processors.
+ // Only used in decomposePar
+ label nProcs(const label numProcs) noexcept
+ {
+ label old(nProcs_);
+ nProcs_ = numProcs;
+ return old;
+ }
- //- Get number of processor directories/results. Used for e.g.
- // reconstructPar, argList checking
+ //- Get number of processor directories/results.
+ // Used for e.g. reconstructPar, argList checking
virtual label nProcs
(
const fileName& dir,
@@ -724,11 +812,14 @@ public:
//- Forcibly wait until all output done. Flush any cached data
virtual void flush() const;
+ //- Forcibly parallel sync
+ virtual void sync();
+
//- Generate path (like io.path) from root+casename with any
// 'processorXXX' replaced by procDir (usually 'processsors')
fileName processorsCasePath
(
- const IOobject&,
+ const IOobject& io,
const word& procDir
) const;
@@ -736,7 +827,7 @@ public:
// 'processorXXX' replaced by procDir (usually 'processsors')
fileName processorsPath
(
- const IOobject&,
+ const IOobject& io,
const word& instance,
const word& procDir
) const;
@@ -778,6 +869,12 @@ public:
//- Get list of global IO ranks from FOAM_IORANKS env variable.
//- If set, these correspond to the IO master ranks.
static labelList getGlobalIORanks();
+
+
+ // Housekeeping
+
+ //- Same as nProcs
+ label setNProcs(label numProcs) { return nProcs(numProcs); }
};
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C
index ef7da7d6ddf9f726bbfa86e23aadfb387d7d25e8..dee4b1c4aaa2c3db6d41a77c179ae732ae16f8b3 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C
@@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
- Copyright (C) 2022 OpenCFD Ltd.
+ Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -148,11 +148,12 @@ Foam::fileOperation::New
<< abort(FatalError);
}
+ // Forward to self
return fileOperation::New(fileOperation::defaultFileHandler, verbose);
}
DebugInFunction
- << "Constructing fileHandler" << endl;
+ << "Constructing fileHandler: " << handlerType << endl;
auto* ctorPtr = wordConstructorTable(handlerType);
@@ -170,6 +171,56 @@ Foam::fileOperation::New
}
+Foam::autoPtr<Foam::fileOperation>
+Foam::fileOperation::New
+(
+ const word& handlerType,
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
+ bool verbose
+)
+{
+ if (handlerType.empty())
+ {
+ if (fileOperation::defaultFileHandler.empty())
+ {
+ FatalErrorInFunction
+ << "defaultFileHandler name is undefined" << nl
+ << abort(FatalError);
+ }
+
+ // Forward to self
+ return fileOperation::New
+ (
+ fileOperation::defaultFileHandler,
+ commAndIORanks,
+ distributedRoots,
+ verbose
+ );
+ }
+
+ DebugInFunction
+ << "Constructing fileHandler: " << handlerType << endl;
+
+ auto* ctorPtr = commConstructorTable(handlerType);
+
+ if (!ctorPtr)
+ {
+ FatalErrorInLookup
+ (
+ "fileHandler",
+ handlerType,
+ *commConstructorTablePtr_
+ ) << abort(FatalError);
+ }
+
+ return autoPtr<fileOperation>
+ (
+ ctorPtr(commAndIORanks, distributedRoots, verbose)
+ );
+}
+
+
// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * //
Foam::autoPtr<Foam::fileOperation>
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C
index e0a79e83c213148e5b9138afae6b711d26d22998..318afc65e50d8314a68eeb0f532da9e50592c8f9 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C
@@ -243,7 +243,6 @@ Foam::labelList Foam::fileOperation::getGlobalIORanks
}
-#if 0 // FUTURE
bool Foam::fileOperation::isIOrank(const label proci) const
{
return
@@ -313,7 +312,6 @@ void Foam::fileOperation::printRanks() const
<< ')' << nl;
}
}
-#endif
// ************************************************************************* //
diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
index 6aa0893614c0fb5af796b7b3b26d7ca529803f8e..314cd76cf7a00a7f1988d841c8df7eb4a9d347c1 100644
--- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
@@ -39,7 +39,6 @@ License
#include "dummyISstream.H"
#include "SubList.H"
#include "unthreadedInitialise.H"
-#include "bitSet.H"
/* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */
@@ -54,6 +53,12 @@ namespace fileOperations
masterUncollatedFileOperation,
word
);
+ addToRunTimeSelectionTable
+ (
+ fileOperation,
+ masterUncollatedFileOperation,
+ comm
+ );
float masterUncollatedFileOperation::maxMasterFileBufferSize
(
@@ -80,26 +85,6 @@ namespace fileOperations
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
-Foam::labelList Foam::fileOperations::masterUncollatedFileOperation::subRanks
-(
- const label n
-)
-{
- labelList mainIOranks(fileOperation::getGlobalIORanks());
-
- if (mainIOranks.empty())
- {
- return identity(n);
- }
- else
- {
- labelRange subRange = fileOperation::subRanks(mainIOranks);
-
- return identity(subRange);
- }
-}
-
-
Foam::word
Foam::fileOperations::masterUncollatedFileOperation::findInstancePath
(
@@ -121,7 +106,7 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstancePath
}
}
- return word::null;
+ return word();
}
@@ -131,14 +116,15 @@ Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
const bool checkGlobal,
const bool isFile,
const IOobject& io,
+ const dirIndexList& pDirs,
const bool search,
pathType& searchType,
word& procsDir,
word& newInstancePath
) const
{
- procsDir = word::null;
- newInstancePath = word::null;
+ procsDir.clear();
+ newInstancePath.clear();
if (io.instance().isAbsolute())
{
@@ -152,7 +138,7 @@ Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
else
{
searchType = fileOperation::NOTFOUND;
- return fileName::null;
+ return fileName();
}
}
else
@@ -169,9 +155,7 @@ Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
// 2. Check processors/
if (io.time().processorCase())
{
- refPtr<dirIndexList> pDirs(lookupProcessorsPath(io.objectPath()));
-
- for (const dirIndex& dirIdx : pDirs())
+ for (const dirIndex& dirIdx : pDirs)
{
const fileName& pDir = dirIdx.first();
fileName objPath =
@@ -239,12 +223,7 @@ Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
if (newInstancePath.size() && newInstancePath != io.instance())
{
// 1. Try processors equivalent
- refPtr<dirIndexList> pDirs
- (
- lookupProcessorsPath(io.objectPath())
- );
-
- for (const dirIndex& dirIdx : pDirs())
+ for (const dirIndex& dirIdx : pDirs)
{
const fileName& pDir = dirIdx.first();
@@ -295,10 +274,11 @@ Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
}
}
}
-
- searchType = fileOperation::NOTFOUND;
- return fileName::null;
}
+
+ // Nothing found
+ searchType = fileOperation::NOTFOUND;
+ return fileName();
}
@@ -338,7 +318,7 @@ Foam::fileOperations::masterUncollatedFileOperation::localObjectPath
// Uncollated type, e.g. processor1
const word procName
(
- "processor" + Foam::name(Pstream::myProcNo(Pstream::worldComm))
+ "processor" + Foam::name(Pstream::myProcNo(UPstream::worldComm))
);
return
processorsPath
@@ -395,7 +375,7 @@ Foam::fileOperations::masterUncollatedFileOperation::localObjectPath
const word procName
(
"processor"
- +Foam::name(Pstream::myProcNo(Pstream::worldComm))
+ + Foam::name(Pstream::myProcNo(UPstream::worldComm))
);
return
processorsPath
@@ -432,14 +412,14 @@ Foam::fileOperations::masterUncollatedFileOperation::localObjectPath
case fileOperation::NOTFOUND:
{
- return fileName::null;
+ return fileName();
}
break;
default:
{
NotImplemented;
- return fileName::null;
+ return fileName();
}
}
}
@@ -458,6 +438,8 @@ void Foam::fileOperations::masterUncollatedFileOperation::readAndSend
{
FatalIOErrorInFunction(filePath)
<< "Cannot open file " << filePath
+ //<< " using communicator " << pBufs.comm()
+ //<< " ioRanks:" << UPstream::procID(pBufs.comm())
<< exit(FatalIOError);
}
@@ -529,11 +511,11 @@ Foam::fileOperations::masterUncollatedFileOperation::read
{
autoPtr<ISstream> isPtr;
- // const bool uniform = uniformFile(filePaths);
+ // const bool uniform = fileOperation::uniformFile(filePaths);
PstreamBuffers pBufs(comm, UPstream::commsTypes::nonBlocking);
- if (Pstream::master(comm))
+ if (UPstream::master(comm))
{
if (uniform)
{
@@ -543,6 +525,8 @@ Foam::fileOperations::masterUncollatedFileOperation::read
{
FatalIOErrorInFunction(filePaths[0])
<< "cannot find file " << io.objectPath()
+ << " fileHandler : comm:" << comm
+ << " ioRanks:" << UPstream::procID(comm)
<< exit(FatalIOError);
}
@@ -575,6 +559,8 @@ Foam::fileOperations::masterUncollatedFileOperation::read
{
FatalIOErrorInFunction(filePaths[0])
<< "cannot find file " << io.objectPath()
+ << " fileHandler : comm:" << comm
+ << " ioRanks:" << UPstream::procID(comm)
<< exit(FatalIOError);
}
@@ -586,7 +572,10 @@ Foam::fileOperations::masterUncollatedFileOperation::read
{
FatalIOErrorInFunction(*isPtr)
<< "problem while reading header for object "
- << io.name() << exit(FatalIOError);
+ << io.name()
+ << " fileHandler : comm:" << comm
+ << " ioRanks:" << UPstream::procID(comm)
+ << exit(FatalIOError);
}
}
@@ -612,7 +601,7 @@ Foam::fileOperations::masterUncollatedFileOperation::read
}
}
- pBufs.finishedSends();
+ pBufs.finishedScatters();
// isPtr will be valid on master and will be the unbuffered
// IFstream. Else the information is in the PstreamBuffers (and
@@ -650,7 +639,10 @@ Foam::fileOperations::masterUncollatedFileOperation::read
{
FatalIOErrorInFunction(*isPtr)
<< "problem while reading header for object "
- << io.name() << exit(FatalIOError);
+ << io.name()
+ << " fileHandler : comm:" << comm
+ << " ioRanks:" << UPstream::procID(comm)
+ << exit(FatalIOError);
}
}
else
@@ -659,11 +651,41 @@ Foam::fileOperations::masterUncollatedFileOperation::read
}
}
-
return isPtr;
}
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// Construction helper: self/world/local communicator and IO ranks
+static Tuple2<label, labelList> getCommPattern()
+{
+ // Default is COMM_WORLD (single master)
+ Tuple2<label, labelList> commAndIORanks
+ (
+ UPstream::worldComm,
+ fileOperation::getGlobalIORanks()
+ );
+
+ if (UPstream::parRun() && commAndIORanks.second().size() > 1)
+ {
+ // Multiple masters: ranks for my IO range
+ commAndIORanks.first() = UPstream::allocateCommunicator
+ (
+ UPstream::worldComm,
+ fileOperation::subRanks(commAndIORanks.second())
+ );
+ }
+
+ return commAndIORanks;
+}
+
+} // End namespace Foam
+
+
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
void Foam::fileOperations::masterUncollatedFileOperation::init(bool verbose)
@@ -708,11 +730,7 @@ masterUncollatedFileOperation
:
fileOperation
(
- UPstream::allocateCommunicator
- (
- UPstream::worldComm,
- subRanks(UPstream::nProcs())
- )
+ getCommPattern()
),
managedComm_(getManagedComm(comm_)) // Possibly locally allocated
{
@@ -723,11 +741,12 @@ masterUncollatedFileOperation
Foam::fileOperations::masterUncollatedFileOperation::
masterUncollatedFileOperation
(
- const label comm,
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
bool verbose
)
:
- fileOperation(comm),
+ fileOperation(commAndIORanks, distributedRoots),
managedComm_(-1) // Externally managed
{
init(verbose);
@@ -896,7 +915,7 @@ time_t Foam::fileOperations::masterUncollatedFileOperation::lastModified
fName,
lastModifiedOp(followLink),
Pstream::msgType(),
- comm_
+ UPstream::worldComm
);
}
@@ -912,7 +931,7 @@ double Foam::fileOperations::masterUncollatedFileOperation::highResLastModified
fName,
highResLastModifiedOp(followLink),
Pstream::msgType(),
- comm_
+ UPstream::worldComm
);
}
@@ -1050,16 +1069,20 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
{
Pout<< "masterUncollatedFileOperation::filePath :"
<< " objectPath:" << io.objectPath()
- << " checkGlobal:" << checkGlobal << endl;
+ << " checkGlobal:" << checkGlobal
+ << " parRun:" << Pstream::parRun()
+ << " localmaster:" << Pstream::master(comm_) << endl;
}
// Now that we have an IOobject path use it to detect & cache
// processor directory naming
- (void)lookupProcessorsPath(io.objectPath());
+ const refPtr<dirIndexList> pDirs(lookupProcessorsPath(io.objectPath()));
// Trigger caching of times
- (void)findTimes(io.time().path(), io.time().constant());
-
+ if (cacheLevel() > 0)
+ {
+ (void)findTimes(io.time().path(), io.time().constant());
+ }
// Determine master filePath and scatter
@@ -1070,7 +1093,8 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
if (Pstream::master(comm_))
{
- const bool oldParRun(Pstream::parRun(false));
+ const bool oldParRun = UPstream::parRun(false);
+ const int oldCache = fileOperation::cacheLevel(0);
// All masters search locally. Note that global objects might
// fail (except on master). This gets handled later on (in PARENTOBJECT)
@@ -1080,13 +1104,15 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
checkGlobal,
true,
io,
+ pDirs,
search,
searchType,
procsDir,
newInstancePath
);
- Pstream::parRun(oldParRun);
+ fileOperation::cacheLevel(oldCache);
+ UPstream::parRun(oldParRun);
if (debug)
{
@@ -1098,10 +1124,11 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
}
}
- // Scatter the information about where the master found the object
+ // Broadcast information about where the master found the object
// Note: use the worldComm to make sure all processors decide
// the same type. Only procsDir is allowed to differ; searchType
// and instance have to be same
+ if (UPstream::parRun())
{
int masterType(searchType);
Pstream::broadcasts(UPstream::worldComm, masterType, newInstancePath);
@@ -1177,7 +1204,8 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
if (debug)
{
Pout<< "masterUncollatedFileOperation::filePath :"
- << " Returning from file searching:" << endl
+ << " Returning from file searching using type "
+ << fileOperation::pathTypeNames_[searchType] << endl
<< " objectPath:" << io.objectPath() << endl
<< " filePath :" << objPath << endl << endl;
}
@@ -1196,12 +1224,20 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
{
Pout<< "masterUncollatedFileOperation::dirPath :"
<< " objectPath:" << io.objectPath()
- << " checkGlobal:" << checkGlobal << endl;
+ << " checkGlobal:" << checkGlobal
+ << " parRun:" << Pstream::parRun()
+ << " localmaster:" << Pstream::master(comm_) << endl;
}
// Now that we have an IOobject path use it to detect & cache
// processor directory naming
- (void)lookupProcessorsPath(io.objectPath());
+ const refPtr<dirIndexList> pDirs(lookupProcessorsPath(io.objectPath()));
+
+ // Trigger caching of times
+ if (cacheLevel() > 0)
+ {
+ (void)findTimes(io.time().path(), io.time().constant());
+ }
// Determine master dirPath and broadcast
@@ -1210,31 +1246,50 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
word procsDir;
word newInstancePath;
+ // Local IO node searches for file
if (Pstream::master(comm_))
{
- const bool oldParRun(Pstream::parRun(false));
+ const bool oldParRun = UPstream::parRun(false);
+ const int oldCache = fileOperation::cacheLevel(0);
objPath = filePathInfo
(
checkGlobal,
false,
io,
+ pDirs,
search,
searchType,
procsDir,
newInstancePath
);
- Pstream::parRun(oldParRun);
+ fileOperation::cacheLevel(oldCache);
+ UPstream::parRun(oldParRun);
+
+ if (debug)
+ {
+ Pout<< "masterUncollatedFileOperation::dirPath :"
+ << " master objPath:" << objPath
+ << " searchType:" << fileOperation::pathTypeNames_[searchType]
+ << " procsDir:" << procsDir << " instance:" << newInstancePath
+ << endl;
+ }
}
+
+ // Broadcast information about where the master found the object
+ // Note: use the worldComm to make sure all processors decide
+ // the same type. Only procsDir is allowed to differ; searchType
+ // and instance have to be same
+ if (UPstream::parRun())
{
int masterType(searchType);
- // Future?: comm_,
Pstream::broadcasts(UPstream::worldComm, masterType, newInstancePath);
searchType = pathType(masterType);
}
+
if
(
checkGlobal
@@ -1250,6 +1305,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
}
else
{
+ // Broadcast local processors dir amongst all local nodes
Pstream::broadcast(procsDir, comm_);
// Use the master type to determine if additional information is
@@ -1304,7 +1360,8 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
if (debug)
{
Pout<< "masterUncollatedFileOperation::dirPath :"
- << " Returning from file searching:" << endl
+ << " Returning from directory searching using type "
+ << fileOperation::pathTypeNames_[searchType] << endl
<< " objectPath:" << io.objectPath() << endl
<< " filePath :" << objPath << endl << endl;
}
@@ -1387,24 +1444,28 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
// parent directory in case of parallel)
- refPtr<dirIndexList> pDirs(lookupProcessorsPath(io.objectPath()));
+ const refPtr<dirIndexList> pDirs(lookupProcessorsPath(io.objectPath()));
word foundInstance;
// if (Pstream::master(comm_))
if (Pstream::master(UPstream::worldComm))
{
- const bool oldParRun(Pstream::parRun(false));
+ const bool oldParRun = UPstream::parRun(false);
+ const int oldCache = fileOperation::cacheLevel(0);
+
if (exists(pDirs, io))
{
foundInstance = io.instance();
}
- Pstream::parRun(oldParRun);
+ fileOperation::cacheLevel(oldCache);
+ UPstream::parRun(oldParRun);
}
+
// Do parallel early exit to avoid calling time.times()
- // Pstream::broadcast(foundInstance, comm_);
Pstream::broadcast(foundInstance, UPstream::worldComm);
+
if (!foundInstance.empty())
{
io.instance() = foundInstance;
@@ -1429,7 +1490,8 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
// if (Pstream::master(comm_))
if (Pstream::master(UPstream::worldComm))
{
- const bool oldParRun(Pstream::parRun(false));
+ const bool oldParRun = UPstream::parRun(false);
+ const int oldCache = fileOperation::cacheLevel(0);
label instIndex = ts.size()-1;
@@ -1540,11 +1602,12 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
}
}
+ fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun); // Restore parallel state
}
- // Pstream::broadcast(foundInstance, comm_);
Pstream::broadcast(foundInstance, UPstream::worldComm);
+
io.instance() = foundInstance;
@@ -1603,16 +1666,17 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects
}
fileNameList objectNames;
- newInstance = word::null;
+ newInstance.clear();
// Note: readObjects uses WORLD to make sure order of objects is the
// same everywhere
- if (Pstream::master()) // comm_))
+ if (Pstream::master(UPstream::worldComm))
{
// Avoid fileOperation::readObjects from triggering parallel ops
// (through call to filePath which triggers parallel )
const bool oldParRun = UPstream::parRun(false);
+ const int oldCache = fileOperation::cacheLevel(0);
//- Use non-time searching version
objectNames = fileOperation::readObjects
@@ -1656,10 +1720,10 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects
}
}
+ fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun); // Restore parallel state
}
- // Future? comm_
Pstream::broadcasts(UPstream::worldComm, newInstance, objectNames);
if (debug)
@@ -1689,16 +1753,27 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
<< " filePath :" << fName << endl;
}
+ // We assume if filePath is the same
+ // - headerClassName
+ // - note
+ // are also the same, independent of where the file came from.
+
// Get filePaths on world master
- fileNameList filePaths(Pstream::nProcs(Pstream::worldComm));
- filePaths[Pstream::myProcNo(Pstream::worldComm)] = fName;
- Pstream::gatherList(filePaths, Pstream::msgType(), Pstream::worldComm);
- bool uniform = uniformFile(filePaths);
+ fileNameList filePaths(Pstream::nProcs(UPstream::worldComm));
+ filePaths[UPstream::myProcNo(UPstream::worldComm)] = fName;
+ Pstream::gatherList(filePaths, UPstream::msgType(), UPstream::worldComm);
+
+ bool uniform
+ (
+ UPstream::master(UPstream::worldComm)
+ && fileOperation::uniformFile(filePaths)
+ );
+
Pstream::broadcast(uniform, UPstream::worldComm);
if (uniform)
{
- if (Pstream::master(Pstream::worldComm))
+ if (Pstream::master(UPstream::worldComm))
{
if (!fName.empty())
{
@@ -1722,12 +1797,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
}
else
{
- if (Pstream::nProcs(comm_) != Pstream::nProcs(Pstream::worldComm))
+ if (Pstream::nProcs(comm_) != Pstream::nProcs(UPstream::worldComm))
{
- // Re-gather file paths on local master
- filePaths.resize(Pstream::nProcs(comm_));
- filePaths[Pstream::myProcNo(comm_)] = fName;
- Pstream::gatherList(filePaths, Pstream::msgType(), comm_);
+ // Assume if different nprocs the communicators are also
+ // different. Re-gather file paths on local master
+ filePaths.resize(UPstream::nProcs(comm_));
+ filePaths[UPstream::myProcNo(comm_)] = fName;
+ Pstream::gatherList(filePaths, UPstream::msgType(), comm_);
}
// Intermediate storage arrays (master only)
@@ -1798,7 +1874,8 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
if (debug)
{
Pout<< "masterUncollatedFileOperation::readHeader :" << " ok:" << ok
- << " class:" << io.headerClassName() << endl;
+ << " class:" << io.headerClassName()
+ << " for file:" << fName << endl;;
}
return ok;
}
@@ -1818,6 +1895,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
Pout<< "masterUncollatedFileOperation::readStream :"
<< " object : " << io.name()
<< " global : " << io.global()
+ << " globalObject : " << io.globalObject()
<< " fName : " << fName << " readOnProc:" << readOnProc << endl;
}
@@ -1828,7 +1906,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
// Detect collated format. This could be done on the local communicator
// but we do it on the master node only for now.
- if (UPstream::master()) // comm_))
+ if (UPstream::master(UPstream::worldComm))
{
if (!fName.empty())
{
@@ -1871,7 +1949,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
}
}
- Pstream::broadcast(isCollated); //, comm_);
+ Pstream::broadcast(isCollated, UPstream::worldComm);
if (isCollated)
{
@@ -1904,6 +1982,8 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
FatalIOErrorInFunction(*isPtr)
<< "Could not detect processor number"
<< " from objectPath:" << io.objectPath()
+ << " fileHandler : comm:" << comm_
+ << " ioRanks:" << flatOutput(ioRanks_)
<< exit(FatalIOError);
}
@@ -1954,7 +2034,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
// Get size of file to determine communications type
bool bigSize = false;
- if (Pstream::master()) //, comm_))
+ if (Pstream::master(UPstream::worldComm))
{
// TBD: handle multiple masters?
bigSize =
@@ -1965,7 +2045,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
}
// Reduce (not broadcast)
// - if we have multiple master files (FUTURE)
- Pstream::reduceOr(bigSize); //, comm_);
+ Pstream::reduceOr(bigSize, UPstream::worldComm);
const UPstream::commsTypes myCommsType
(
@@ -1994,23 +2074,32 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
<< " starting separated input from " << fName << endl;
}
- if (io.global())
+ if (io.global() || io.globalObject())
{
// Use worldComm. Note: should not really need to gather filePaths
// since we enforce sending from master anyway ...
- fileNameList filePaths(Pstream::nProcs());
- filePaths[Pstream::myProcNo()] = fName;
- Pstream::gatherList(filePaths);
+ fileNameList filePaths(Pstream::nProcs(UPstream::worldComm));
+ filePaths[Pstream::myProcNo(UPstream::worldComm)] = fName;
+ Pstream::gatherList
+ (
+ filePaths,
+ Pstream::msgType(),
+ UPstream::worldComm
+ );
boolList procValid
(
- UPstream::listGatherValues<bool>(readOnProc)
+ UPstream::listGatherValues<bool>
+ (
+ readOnProc,
+ UPstream::worldComm
+ )
);
// NB: local proc validity information required on sub-ranks too!
- procValid.resize(Pstream::nProcs());
- procValid[Pstream::myProcNo()] = readOnProc;
+ procValid.resize(Pstream::nProcs(UPstream::worldComm));
+ procValid[Pstream::myProcNo(UPstream::worldComm)] = readOnProc;
- return read(io, Pstream::worldComm, true, filePaths, procValid);
+ return read(io, UPstream::worldComm, true, filePaths, procValid);
}
else
{
@@ -2028,7 +2117,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
procValid[Pstream::myProcNo(comm_)] = readOnProc;
// Uniform in local comm
- const bool uniform = uniformFile(filePaths);
+ const bool uniform = fileOperation::uniformFile(filePaths);
return read(io, comm_, uniform, filePaths, procValid);
}
@@ -2046,12 +2135,17 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
{
bool ok = true;
- if (io.globalObject())
+ if (io.global() || io.globalObject())
{
if (debug)
{
Pout<< "masterUncollatedFileOperation::read :"
- << " Reading global object " << io.name() << endl;
+ << " Reading global object " << io.name()
+ << " worldComm:" << UPstream::worldComm
+ << " Pstream::myProcNo:"
+ << Pstream::myProcNo(UPstream::worldComm)
+ << " amMaster:" << Pstream::master(UPstream::worldComm)
+ << endl;
}
bool ok = false;
@@ -2059,10 +2153,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
{
// Do master-only reading always.
const bool oldParRun = UPstream::parRun(false);
+ const int oldCache = fileOperation::cacheLevel(0);
- ok = io.readData(io.readStream(typeName));
+ auto& is = io.readStream(typeName);
+ ok = io.readData(is);
io.close();
+ fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun); // Restore parallel state
}
@@ -2112,6 +2209,14 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
io.close();
}
+ if (debug)
+ {
+ Pout<< "masterUncollatedFileOperation::read :"
+ << " Read object:" << io.name()
+ << " isGlobal:" << (io.global() || io.globalObject())
+ << " status:" << ok << endl;
+ }
+
return ok;
}
@@ -2170,39 +2275,48 @@ Foam::instantList Foam::fileOperations::masterUncollatedFileOperation::findTimes
if (debug)
{
Pout<< "masterUncollatedFileOperation::findTimes :"
- << " Found " << iter.val()->size() << " cached times" << endl;
+ << " Found " << iter.val()->size() << " cached times" << nl
+ << " for directory:" << directory << endl;
}
return *(iter.val());
}
else
{
instantList times;
- if (Pstream::master()) // comm_))
+ if (Pstream::master(UPstream::worldComm))
{
// Do master-only reading always.
const bool oldParRun = UPstream::parRun(false);
+ const int oldCache = fileOperation::cacheLevel(0);
times = fileOperation::findTimes(directory, constantName);
+ fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun); // Restore parallel state
}
- Pstream::broadcast(times); //, comm_);
+
+ Pstream::broadcast(times, UPstream::worldComm);
if (debug)
{
Pout<< "masterUncollatedFileOperation::findTimes :"
- << " Caching times:" << times << nl
+ << " Found times:" << flatOutput(times) << nl
<< " for directory:" << directory << endl;
}
- // Note: do we also cache if no times have been found since it might
- // indicate a directory that is being filled later on ...
-
- auto* tPtr = new DynamicList<instant>(std::move(times));
+ // Caching
+ // - cache values even if no times were found since it might
+ // indicate a directory that is being filled later on ...
+ if (cacheLevel() > 0)
+ {
+ auto* tPtr = new DynamicList<instant>(std::move(times));
+ times_.set(directory, tPtr);
- times_.set(directory, tPtr);
+ return *tPtr;
+ }
- return *tPtr;
+ // Times found (not cached)
+ return times;
}
}
@@ -2220,7 +2334,7 @@ void Foam::fileOperations::masterUncollatedFileOperation::setTime
// Mutable access to instant list for modification and sorting
// - cannot use auto type deduction here
- HashPtrTable<DynamicList<instant>>::iterator iter = times_.find(tm.path());
+ auto iter = times_.find(tm.path());
if (iter.good())
{
@@ -2287,20 +2401,16 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
// Insert logic of filePath. We assume that if a file is absolute
// on the master it is absolute also on the sub-ranks etc.
- fileNameList filePaths(Pstream::nProcs(Pstream::worldComm));
- filePaths[Pstream::myProcNo(Pstream::worldComm)] = filePath;
- Pstream::gatherList(filePaths, Pstream::msgType(), Pstream::worldComm);
+ fileNameList filePaths(Pstream::nProcs(comm_));
+ filePaths[Pstream::myProcNo(comm_)] = filePath;
+ Pstream::gatherList(filePaths, Pstream::msgType(), comm_);
- PstreamBuffers pBufs
- (
- Pstream::commsTypes::nonBlocking,
- Pstream::msgType(),
- Pstream::worldComm
- );
+ PstreamBuffers pBufs(comm_, Pstream::commsTypes::nonBlocking);
- if (Pstream::master(Pstream::worldComm))
+ if (Pstream::master(comm_))
{
- const bool uniform = uniformFile(filePaths);
+ // Same filename on the IO node -> same file
+ const bool uniform = fileOperation::uniformFile(filePaths);
if (uniform)
{
@@ -2313,14 +2423,21 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
readAndSend
(
filePath,
- identity(Pstream::nProcs(Pstream::worldComm)-1, 1),
+ identity(Pstream::nProcs(comm_)-1, 1),
pBufs
);
}
else
{
- for (const int proci : Pstream::subProcs(Pstream::worldComm))
+ for (const int proci : Pstream::subProcs(comm_))
{
+ if (debug)
+ {
+ Pout<< "masterUncollatedFileOperation::NewIFstream :"
+ << " Opening local file " << filePath
+ << " for rank " << proci << endl;
+ }
+
readAndSend
(
filePaths[proci],
@@ -2334,7 +2451,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
pBufs.finishedSends();
- if (Pstream::master(Pstream::worldComm))
+ if (Pstream::master(comm_))
{
// Read myself
isPtr.reset(new IFstream(filePaths[Pstream::masterNo()]));
@@ -2394,6 +2511,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewOFstream
(
new masterOFstream
(
+ comm_,
pathName,
streamOpt,
IOstreamOption::NON_APPEND,
@@ -2417,6 +2535,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewOFstream
new masterOFstream
(
atomic,
+ comm_,
pathName,
streamOpt,
IOstreamOption::NON_APPEND,
@@ -2433,17 +2552,93 @@ void Foam::fileOperations::masterUncollatedFileOperation::flush() const
}
+void Foam::fileOperations::masterUncollatedFileOperation::sync()
+{
+ if (debug)
+ {
+ Pout<< "masterUncollatedFileOperation::sync :"
+ << " syncing information across processors" << endl;
+ }
+
+ fileOperation::sync();
+
+
+ wordList timeNames;
+ List<DynamicList<instant>> instants;
+
+ if (Pstream::master(UPstream::worldComm))
+ {
+ timeNames.resize(times_.size());
+ instants.resize(times_.size());
+
+ // Flatten into two lists to preserve key/val pairing
+ label i = 0;
+ forAllConstIters(times_, iter)
+ {
+ timeNames[i] = iter.key();
+ instants[i] = std::move(*(iter.val()));
+ ++i;
+ }
+ }
+
+ Pstream::broadcasts(UPstream::worldComm, timeNames, instants);
+
+ times_.clear();
+ forAll(timeNames, i)
+ {
+ fileName dir(timeNames[i]);
+ auto ptr = autoPtr<DynamicList<instant>>::New(std::move(instants[i]));
+
+ if (Pstream::parRun() && !Pstream::master(UPstream::worldComm))
+ {
+ // Replace processor0 ending with processorDDD
+ fileName path;
+ fileName pDir;
+ fileName local;
+ procRangeType group;
+ label numProcs;
+ const label proci = splitProcessorPath
+ (
+ dir,
+ path,
+ pDir,
+ local,
+ group,
+ numProcs
+ );
+
+ //Pout<< "**sync : From dir : " << dir << nl
+ // << " path : " << path << nl
+ // << " pDir : " << pDir << nl
+ // << " local: " << local << nl
+ // << " proci: " << proci << nl
+ // << endl;
+
+ const label myProci = Pstream::myProcNo(UPstream::worldComm);
+
+ if (proci != -1 && proci != myProci)
+ {
+ dir = path/"processor" + Foam::name(myProci);
+ }
+ }
+
+ times_.insert(dir, ptr);
+ }
+}
+
+
Foam::label Foam::fileOperations::masterUncollatedFileOperation::addWatch
(
const fileName& fName
) const
{
label watchFd = -1;
- if (Pstream::master()) // comm_))
+ if (!UPstream::parRun() || Pstream::master(UPstream::worldComm))
{
watchFd = monitor().addWatch(fName);
}
- Pstream::broadcast(watchFd); //, comm_);
+
+ Pstream::broadcast(watchFd, UPstream::worldComm);
return watchFd;
}
@@ -2454,11 +2649,12 @@ bool Foam::fileOperations::masterUncollatedFileOperation::removeWatch
) const
{
bool ok = false;
- if (Pstream::master()) // comm_))
+ if (!UPstream::parRun() || Pstream::master(UPstream::worldComm))
{
ok = monitor().removeWatch(watchIndex);
}
- Pstream::broadcast(ok); //, comm_);
+
+ Pstream::broadcast(ok, UPstream::worldComm);
return ok;
}
@@ -2471,7 +2667,7 @@ Foam::label Foam::fileOperations::masterUncollatedFileOperation::findWatch
{
label index = -1;
- if (Pstream::master()) // comm_))
+ if (!UPstream::parRun() || Pstream::master(UPstream::worldComm))
{
forAll(watchIndices, i)
{
@@ -2482,7 +2678,8 @@ Foam::label Foam::fileOperations::masterUncollatedFileOperation::findWatch
}
}
}
- Pstream::broadcast(index); //, comm_);
+
+ Pstream::broadcast(index, UPstream::worldComm);
return index;
}
@@ -2530,11 +2727,12 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::getFile
) const
{
fileName fName;
- if (Pstream::master()) // comm_))
+ if (!UPstream::parRun() || Pstream::master(UPstream::worldComm))
{
fName = monitor().getFile(watchIndex);
}
- Pstream::broadcast(fName); //, comm_);
+
+ Pstream::broadcast(fName, UPstream::worldComm);
return fName;
}
@@ -2545,7 +2743,7 @@ void Foam::fileOperations::masterUncollatedFileOperation::updateStates
const bool syncPar
) const
{
- if (Pstream::master()) // comm_))
+ if (!UPstream::parRun() || Pstream::master(UPstream::worldComm))
{
monitor().updateStates(true, false);
}
@@ -2559,11 +2757,12 @@ Foam::fileOperations::masterUncollatedFileOperation::getState
) const
{
unsigned int state = fileMonitor::UNMODIFIED;
- if (Pstream::master()) // comm_))
+ if (!UPstream::parRun() || Pstream::master(UPstream::worldComm))
{
state = monitor().getState(watchFd);
}
- Pstream::broadcast(state); //, comm_);
+
+ Pstream::broadcast(state, UPstream::worldComm);
return fileMonitor::fileState(state);
}
@@ -2573,7 +2772,7 @@ void Foam::fileOperations::masterUncollatedFileOperation::setUnmodified
const label watchFd
) const
{
- if (Pstream::master()) // comm_))
+ if (!UPstream::parRun() || Pstream::master(UPstream::worldComm))
{
monitor().setUnmodified(watchFd);
}
diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H
index 0b4848488e3d0ae6f378b338be8e23ba24042c01..6ddda2dcdda8328e5bf2971fef67f2bfc96a5e2d 100644
--- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H
+++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H
@@ -392,9 +392,6 @@ protected:
// Private Member Functions
- //- Get the list of processors that are part of this communicator
- static labelList subRanks(const label n);
-
template<class Type>
Type scatterList(const UList<Type>&, const int, const label comm) const;
@@ -436,6 +433,7 @@ protected:
const bool checkGlobal,
const bool isFile,
const IOobject& io,
+ const dirIndexList& pDirs,
const bool search,
pathType& searchType,
word& processorsDir,
@@ -494,8 +492,13 @@ public:
//- Default construct
explicit masterUncollatedFileOperation(bool verbose = false);
- //- Construct from communicator
- masterUncollatedFileOperation(const label comm, bool verbose);
+ //- Construct from communicator with specified io-ranks
+ explicit masterUncollatedFileOperation
+ (
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
+ bool verbose = false
+ );
//- Destructor
@@ -773,6 +776,9 @@ public:
//- Forcibly wait until all output done. Flush any cached data
virtual void flush() const;
+ //- Forcibly parallel sync
+ virtual void sync();
+
//- Return cached times
const HashPtrTable<DynamicList<instant>>& times() const noexcept
{
diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/hostUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/hostUncollatedFileOperation.C
new file mode 100644
index 0000000000000000000000000000000000000000..ab7b28b175d30ec9a6c2321068ef55d2dc1a4e9f
--- /dev/null
+++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/hostUncollatedFileOperation.C
@@ -0,0 +1,175 @@
+/*---------------------------------------------------------------------------*\
+ ========= |
+ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
+ \\ / O peration |
+ \\ / A nd | www.openfoam.com
+ \\/ M anipulation |
+-------------------------------------------------------------------------------
+ Copyright (C) 2022-2023 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+ This file is part of OpenFOAM.
+
+ OpenFOAM is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "hostUncollatedFileOperation.H"
+#include "addToRunTimeSelectionTable.H"
+
+/* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */
+
+namespace Foam
+{
+namespace fileOperations
+{
+ defineTypeNameAndDebug(hostUncollatedFileOperation, 0);
+ addToRunTimeSelectionTable
+ (
+ fileOperation,
+ hostUncollatedFileOperation,
+ word
+ );
+ addToRunTimeSelectionTable
+ (
+ fileOperation,
+ hostUncollatedFileOperation,
+ comm
+ );
+
+ // Register initialisation routine. Signals need for threaded mpi and
+ // handles command line arguments
+ addNamedToRunTimeSelectionTable
+ (
+ fileOperationInitialise,
+ hostUncollatedFileOperationInitialise,
+ word,
+ hostUncollated
+ );
+}
+}
+
+
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// Construction helper: self/world/local communicator and IO ranks
+static Tuple2<label, labelList> getCommPattern()
+{
+ // Default is COMM_WORLD (single master)
+ Tuple2<label, labelList> commAndIORanks
+ (
+ UPstream::worldComm,
+ fileOperation::getGlobalIORanks()
+ );
+
+ if (commAndIORanks.second().empty())
+ {
+ // Default: one master per host
+ commAndIORanks.second() = fileOperation::getGlobalHostIORanks();
+ }
+
+ if (UPstream::parRun() && commAndIORanks.second().size() > 1)
+ {
+ // Multiple masters: ranks for my IO range
+ commAndIORanks.first() = UPstream::allocateCommunicator
+ (
+ UPstream::worldComm,
+ fileOperation::subRanks(commAndIORanks.second())
+ );
+ }
+
+ return commAndIORanks;
+}
+
+} // End namespace Foam
+
+
+// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
+
+void Foam::fileOperations::hostUncollatedFileOperation::init(bool verbose)
+{
+ verbose = (verbose && Foam::infoDetailLevel > 0);
+
+ if (verbose)
+ {
+ DetailInfo
+ << "I/O : " << this->type() << nl;
+
+ if (ioRanks_.size())
+ {
+ fileOperation::printRanks();
+ }
+ }
+}
+
+
+Foam::fileOperations::hostUncollatedFileOperation::hostUncollatedFileOperation
+(
+ bool verbose
+)
+:
+ masterUncollatedFileOperation
+ (
+ getCommPattern(),
+ false, // distributedRoots
+ false // verbose
+ ),
+ managedComm_(getManagedComm(comm_)) // Possibly locally allocated
+{
+ init(verbose);
+}
+
+
+Foam::fileOperations::hostUncollatedFileOperation::hostUncollatedFileOperation
+(
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
+ bool verbose
+)
+:
+ masterUncollatedFileOperation
+ (
+ commAndIORanks,
+ distributedRoots,
+ false // verbose
+ ),
+ managedComm_(-1) // Externally managed
+{
+ init(verbose);
+}
+
+
+void Foam::fileOperations::hostUncollatedFileOperation::storeComm() const
+{
+ // From externally -> locally managed
+ managedComm_ = getManagedComm(comm_);
+}
+
+
+// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
+
+Foam::fileOperations::hostUncollatedFileOperation::
+~hostUncollatedFileOperation()
+{
+ // Wait for any outstanding file operations
+ flush();
+
+ UPstream::freeCommunicator(managedComm_);
+}
+
+
+// ************************************************************************* //
diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/hostUncollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/hostUncollatedFileOperation.H
new file mode 100644
index 0000000000000000000000000000000000000000..c76baf57164bc67dbe048fb055649fa639af7441
--- /dev/null
+++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/hostUncollatedFileOperation.H
@@ -0,0 +1,130 @@
+/*---------------------------------------------------------------------------*\
+ ========= |
+ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
+ \\ / O peration |
+ \\ / A nd | www.openfoam.com
+ \\/ M anipulation |
+-------------------------------------------------------------------------------
+ Copyright (C) 2022-2023 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+ This file is part of OpenFOAM.
+
+ OpenFOAM is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
+
+Class
+ Foam::fileOperations::hostUncollatedFileOperation
+
+Description
+ Version of masterUncollated with host-based IO ranks
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef Foam_fileOperations_hostUncollatedFileOperation_H
+#define Foam_fileOperations_hostUncollatedFileOperation_H
+
+#include "masterUncollatedFileOperation.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+namespace fileOperations
+{
+
+/*---------------------------------------------------------------------------*\
+ Class hostUncollatedFileOperation Declaration
+\*---------------------------------------------------------------------------*/
+
+class hostUncollatedFileOperation
+:
+ public masterUncollatedFileOperation
+{
+ // Private Data
+
+ //- Communicator allocated/managed by us
+ mutable label managedComm_;
+
+
+ // Private Member Functions
+
+ //- Any initialisation steps after constructing
+ void init(bool verbose);
+
+public:
+
+ //- Runtime type information
+ TypeName("hostUncollated");
+
+
+ // Constructors
+
+ //- Default construct
+ explicit hostUncollatedFileOperation(bool verbose = false);
+
+ //- Construct from communicator with specified io-ranks
+ explicit hostUncollatedFileOperation
+ (
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
+ bool verbose = false
+ );
+
+
+ //- Destructor
+ virtual ~hostUncollatedFileOperation();
+
+
+ // Member Functions
+
+ //- Transfer ownership of communicator to this fileOperation.
+ //- Use with caution
+ virtual void storeComm() const;
+};
+
+
+/*---------------------------------------------------------------------------*\
+ Class hostUncollatedFileOperationInitialise Declaration
+\*---------------------------------------------------------------------------*/
+
+class hostUncollatedFileOperationInitialise
+:
+ public masterUncollatedFileOperationInitialise
+{
+public:
+
+ // Constructors
+
+ //- Construct from components
+ hostUncollatedFileOperationInitialise(int& argc, char**& argv)
+ :
+ masterUncollatedFileOperationInitialise(argc, argv)
+ {}
+
+
+ //- Destructor
+ virtual ~hostUncollatedFileOperationInitialise() = default;
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace fileOperations
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
index 0988f3b2515b0604f20195f38eeb163648c7da1e..2b380cc1f673dea25791e312be3b5a94393d2e58 100644
--- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
@@ -41,7 +41,18 @@ namespace Foam
namespace fileOperations
{
defineTypeNameAndDebug(uncollatedFileOperation, 0);
- addToRunTimeSelectionTable(fileOperation, uncollatedFileOperation, word);
+ addToRunTimeSelectionTable
+ (
+ fileOperation,
+ uncollatedFileOperation,
+ word
+ );
+ addToRunTimeSelectionTable
+ (
+ fileOperation,
+ uncollatedFileOperation,
+ comm
+ );
// Mark as not needing threaded mpi
addNamedToRunTimeSelectionTable
@@ -159,7 +170,7 @@ Foam::fileName Foam::fileOperations::uncollatedFileOperation::filePathInfo
}
}
- return fileName::null;
+ return fileName();
}
@@ -174,6 +185,37 @@ Foam::fileOperations::uncollatedFileOperation::lookupProcessorsPath
}
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// Construction helper: self/world/local communicator and IO ranks
+static Tuple2<label, labelList> getCommPattern()
+{
+ // Default is COMM_SELF (only involves itself)
+ Tuple2<label, labelList> commAndIORanks
+ (
+ UPstream::selfComm,
+ fileOperation::getGlobalIORanks()
+ );
+
+ if (UPstream::parRun() && commAndIORanks.second().size() > 1)
+ {
+ // Multiple masters: ranks for my IO range
+ commAndIORanks.first() = UPstream::allocateCommunicator
+ (
+ UPstream::worldComm,
+ fileOperation::subRanks(commAndIORanks.second())
+ );
+ }
+
+ return commAndIORanks;
+}
+
+} // End namespace Foam
+
+
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
void Foam::fileOperations::uncollatedFileOperation::init(bool verbose)
@@ -193,7 +235,24 @@ Foam::fileOperations::uncollatedFileOperation::uncollatedFileOperation
bool verbose
)
:
- fileOperation(UPstream::worldComm),
+ fileOperation
+ (
+ getCommPattern()
+ ),
+ managedComm_(getManagedComm(comm_)) // Possibly locally allocated
+{
+ init(verbose);
+}
+
+
+Foam::fileOperations::uncollatedFileOperation::uncollatedFileOperation
+(
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
+ bool verbose
+)
+:
+ fileOperation(commAndIORanks, distributedRoots),
managedComm_(-1) // Externally managed
{
init(verbose);
@@ -211,6 +270,9 @@ void Foam::fileOperations::uncollatedFileOperation::storeComm() const
Foam::fileOperations::uncollatedFileOperation::~uncollatedFileOperation()
{
+ // Wait for any outstanding file operations
+ flush();
+
UPstream::freeCommunicator(managedComm_);
}
diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H
index a58d60a58d91856f122b2a8ac48e567afacb53f3..41ddb96b5fb1dff4f6a799783e19ae88876758eb 100644
--- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H
+++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H
@@ -64,6 +64,7 @@ class uncollatedFileOperation
//- Any initialisation steps after constructing
void init(bool verbose);
+
protected:
// Protected Member Functions
@@ -90,8 +91,8 @@ protected:
public:
- //- Runtime type information
- TypeName("uncollated");
+ //- Runtime type information
+ TypeName("uncollated");
// Constructors
@@ -99,6 +100,14 @@ public:
//- Default construct
explicit uncollatedFileOperation(bool verbose = false);
+ //- Construct from communicator with specified io-ranks
+ explicit uncollatedFileOperation
+ (
+ const Tuple2<label, labelList>& commAndIORanks,
+ const bool distributedRoots,
+ bool verbose = false
+ );
+
//- Destructor
virtual ~uncollatedFileOperation();