From 627d79dba68d9436a8ce3e4df5e92f40d976cfed Mon Sep 17 00:00:00 2001
From: Mark Olesen <Mark.Olesen@esi-group.com>
Date: Wed, 9 Dec 2020 14:15:24 +0100
Subject: [PATCH] ENH: reduce use of readdir on individual processors (#1946)
- implicitly enabled when timeStampMaster (default) is used
for the fileModificationChecking
- When running with non-distributed roots (eg, NFS-share) read for
processor directories on master only and send to sub-processes
instead individual reads.
- If disabled (old default, or when running with distributed roots),
uses the regular fileHandler readDir, which may perform readDir
on each processor. Potentially slow startup times on large systems.
Improvements based on analysis from T.Aoyagi(RIST), A.Azami(RIST)
---
src/OpenFOAM/db/Time/TimeIO.C | 11 +++-
src/OpenFOAM/global/argList/argList.C | 8 +++
.../fileOperation/fileOperation.C | 63 ++++++++++++++-----
3 files changed, 67 insertions(+), 15 deletions(-)
diff --git a/src/OpenFOAM/db/Time/TimeIO.C b/src/OpenFOAM/db/Time/TimeIO.C
index 1b9232392ce..1a57de375c9 100644
--- a/src/OpenFOAM/db/Time/TimeIO.C
+++ b/src/OpenFOAM/db/Time/TimeIO.C
@@ -166,8 +166,17 @@ void Foam::Time::readDict()
}
controlDict_.watchIndices().clear();
+ // The new handler, with verbosity
+ autoPtr<fileOperation> newHandler =
+ fileOperation::New(fileHandlerName, true);
+
+ if (TimePaths::distributed() && newHandler)
+ {
+ newHandler->distributed(true);
+ }
+
// Installing the new handler
- Foam::fileHandler(fileOperation::New(fileHandlerName, true));
+ Foam::fileHandler(std::move(newHandler));
// Reinstall old watches
fileHandler().addWatches(controlDict_, oldWatched);
diff --git a/src/OpenFOAM/global/argList/argList.C b/src/OpenFOAM/global/argList/argList.C
index 735a2487c97..2e1d2fc48d8 100644
--- a/src/OpenFOAM/global/argList/argList.C
+++ b/src/OpenFOAM/global/argList/argList.C
@@ -1414,6 +1414,14 @@ void Foam::argList::parse
case_ = globalCase_; // Redundant, but extra safety?
}
+ // If needed, adjust fileHandler for distributed roots
+ if (parRunControl_.distributed())
+ {
+ if (fileOperation::fileHandlerPtr_)
+ {
+ fileOperation::fileHandlerPtr_->distributed(true);
+ }
+ }
// Keep or discard slave and root information for reporting:
if (Pstream::master() && parRunControl_.parRun())
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
index 4ee216eb7b3..fa5345096d7 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
@@ -361,21 +361,62 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
return iter.val();
}
+ DynamicList<dirIndex> procDirs;
+ fileNameList dirEntries;
+
// Read all directories to see any beginning with processor
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- DynamicList<dirIndex> procDirs;
-
// Note: use parallel synchronised reading so cache will be same
// order on all processors
- fileNameList dirNames(readDir(path, fileName::Type::DIRECTORY));
+
+ const bool readDirMasterOnly
+ (
+ regIOobject::fileModificationChecking == IOobject::timeStampMaster
+ || regIOobject::fileModificationChecking == IOobject::inotifyMaster
+ );
+
+ // As byproduct of the above selection, we exclude masterUncollated
+ // from using read/send, but that doesn't matter since that is what
+ // its own internals for readDir() do anyhow.
+
+ if (readDirMasterOnly && Pstream::parRun() && !distributed())
+ {
+ // Non-distributed.
+ // Read on master only and send to subProcs
+
+ if (Pstream::master())
+ {
+ dirEntries = Foam::readDir(path, fileName::Type::DIRECTORY);
+
+ DebugInfo
+ << "readDir on master: send " << dirEntries.size()
+ << " names to sub-processes" << endl;
+ }
+
+ Pstream::scatter(dirEntries, Pstream::msgType(), comm_);
+ }
+ else
+ {
+ // Serial or distributed roots.
+ // Handle readDir() with virtual method
+
+ if (debug)
+ {
+ Pout<< "readDir without special master/send treatment"
+ << endl;
+ }
+
+ dirEntries = readDir(path, fileName::Type::DIRECTORY);
+ }
+
// Extract info from processorsDDD or processorDDD:
// - highest processor number
// - directory+offset containing data for proci
label maxProc = -1;
- for (const fileName& dirN : dirNames)
+ for (const fileName& dirN : dirEntries)
{
// Analyse directory name
fileName rp, rd, rl;
@@ -775,16 +816,11 @@ Foam::instantList Foam::fileOperation::findTimes
<< directory << endl;
}
- // Read directory entries into a list
- fileNameList dirEntries
- (
- Foam::readDir
- (
- directory,
- fileName::DIRECTORY
- )
- );
+ // Note: do NOT use master-only reading here (as per lookupProcessorsPath)
+ // since this routine is called on an individual processorN directory
+ // Read directory entries into a list
+ fileNameList dirEntries(Foam::readDir(directory, fileName::DIRECTORY));
instantList times = sortTimes(dirEntries, constantName);
@@ -1269,7 +1305,6 @@ Foam::label Foam::fileOperation::splitProcessorPath
// We are done!
break;
}
-
}
if (pos != string::npos)
--
GitLab