Commit 627d79db authored by Mark Olesen's avatar Mark Olesen
Browse files

ENH: reduce use of readdir on individual processors (#1946)

- implicitly enabled when timeStampMaster (default) is used
  for the fileModificationChecking

- When running with non-distributed roots (eg, NFS-share) read for
  processor directories on master only and send to sub-processes
  instead individual reads.

- If disabled (old default, or when running with distributed roots),
  uses the regular fileHandler readDir, which may perform readDir
  on each processor. Potentially slow startup times on large systems.

Improvements based on analysis from T.Aoyagi(RIST), A.Azami(RIST)
parent c51bfdcd
......@@ -166,8 +166,17 @@ void Foam::Time::readDict()
}
controlDict_.watchIndices().clear();
// The new handler, with verbosity
autoPtr<fileOperation> newHandler =
fileOperation::New(fileHandlerName, true);
if (TimePaths::distributed() && newHandler)
{
newHandler->distributed(true);
}
// Installing the new handler
Foam::fileHandler(fileOperation::New(fileHandlerName, true));
Foam::fileHandler(std::move(newHandler));
// Reinstall old watches
fileHandler().addWatches(controlDict_, oldWatched);
......
......@@ -1414,6 +1414,14 @@ void Foam::argList::parse
case_ = globalCase_; // Redundant, but extra safety?
}
// If needed, adjust fileHandler for distributed roots
if (parRunControl_.distributed())
{
if (fileOperation::fileHandlerPtr_)
{
fileOperation::fileHandlerPtr_->distributed(true);
}
}
// Keep or discard slave and root information for reporting:
if (Pstream::master() && parRunControl_.parRun())
......
......@@ -361,21 +361,62 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
return iter.val();
}
DynamicList<dirIndex> procDirs;
fileNameList dirEntries;
// Read all directories to see any beginning with processor
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
DynamicList<dirIndex> procDirs;
// Note: use parallel synchronised reading so cache will be same
// order on all processors
fileNameList dirNames(readDir(path, fileName::Type::DIRECTORY));
const bool readDirMasterOnly
(
regIOobject::fileModificationChecking == IOobject::timeStampMaster
|| regIOobject::fileModificationChecking == IOobject::inotifyMaster
);
// As byproduct of the above selection, we exclude masterUncollated
// from using read/send, but that doesn't matter since that is what
// its own internals for readDir() do anyhow.
if (readDirMasterOnly && Pstream::parRun() && !distributed())
{
// Non-distributed.
// Read on master only and send to subProcs
if (Pstream::master())
{
dirEntries = Foam::readDir(path, fileName::Type::DIRECTORY);
DebugInfo
<< "readDir on master: send " << dirEntries.size()
<< " names to sub-processes" << endl;
}
Pstream::scatter(dirEntries, Pstream::msgType(), comm_);
}
else
{
// Serial or distributed roots.
// Handle readDir() with virtual method
if (debug)
{
Pout<< "readDir without special master/send treatment"
<< endl;
}
dirEntries = readDir(path, fileName::Type::DIRECTORY);
}
// Extract info from processorsDDD or processorDDD:
// - highest processor number
// - directory+offset containing data for proci
label maxProc = -1;
for (const fileName& dirN : dirNames)
for (const fileName& dirN : dirEntries)
{
// Analyse directory name
fileName rp, rd, rl;
......@@ -775,16 +816,11 @@ Foam::instantList Foam::fileOperation::findTimes
<< directory << endl;
}
// Read directory entries into a list
fileNameList dirEntries
(
Foam::readDir
(
directory,
fileName::DIRECTORY
)
);
// Note: do NOT use master-only reading here (as per lookupProcessorsPath)
// since this routine is called on an individual processorN directory
// Read directory entries into a list
fileNameList dirEntries(Foam::readDir(directory, fileName::DIRECTORY));
instantList times = sortTimes(dirEntries, constantName);
......@@ -1269,7 +1305,6 @@ Foam::label Foam::fileOperation::splitProcessorPath
// We are done!
break;
}
}
if (pos != string::npos)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment