From 35c884971a9ecb1c057cd159958af4ed7ea8d0f0 Mon Sep 17 00:00:00 2001
From: Mark Olesen <Mark.Olesen@esi-group.com>
Date: Tue, 12 Dec 2023 15:58:31 +0100
Subject: [PATCH] BUG: redistributePar blocks with -ioRanks (#3005)
---
.../redistributePar/redistributePar.C | 111 ++++++++++--------
1 file changed, 62 insertions(+), 49 deletions(-)
diff --git a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
index 8324c2134eb..a2f3227bdbf 100644
--- a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
+++ b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
@@ -112,7 +112,7 @@ using namespace Foam;
// Use -verbose -verbose to switch on debug info. TBD.
int debug(::Foam::debug::debugSwitch("redistributePar", 0));
-#define InfoOrPout (::debug ? Pout : Info())
+#define InfoOrPout (::debug ? Pout : Info.stream())
// Allocate a new file handler on valid processors only
@@ -1216,7 +1216,10 @@ int main(int argc, char *argv[])
#include "addOverwriteOption.H"
argList::addBoolOption("decompose", "Decompose case");
argList::addBoolOption("reconstruct", "Reconstruct case");
- argList::addVerboseOption("Additional verbosity");
+ argList::addVerboseOption
+ (
+ "Additional verbosity. (Can be used multiple times for debugging)"
+ );
argList::addDryRunOption
(
"Test without writing the decomposition. "
@@ -1267,9 +1270,9 @@ int main(int argc, char *argv[])
// - reconstruct - reads parallel, write on master only and to parent
// directory
- // Detect if running data-distributed (processors cannot see all other
- // processors' files)
- const bool nfs = !fileHandler().distributed();
+ // Running data-distributed?
+ // (processors cannot see all other processors' files)
+ const bool hasDistributedFiles(fileHandler().distributed());
// Set up loose processorsXXX directory matching (for collated) so e.g.
@@ -1278,21 +1281,6 @@ int main(int argc, char *argv[])
// matching.
fileOperation::nProcsFilter(0);
-
- // Read handler on processors with a volMesh
- refPtr<fileOperation> volMeshReadHandler;
-
- // Read handler on processors with an areaMesh
- refPtr<fileOperation> areaMeshReadHandler;
-
- // Handler for master-only operation (read/writing from/to undecomposed)
- refPtr<fileOperation> masterOnlyHandler;
- if (UPstream::master(UPstream::worldComm))
- {
- masterOnlyHandler = fileOperation::NewUncollated();
- }
-
-
// Need this line since we don't include "setRootCase.H"
#include "foamDlOpenLibs.H"
@@ -1306,19 +1294,36 @@ int main(int argc, char *argv[])
bool decompose = args.found("decompose");
bool overwrite = args.found("overwrite");
- if (optVerbose)
+ // Field restrictions...
+ const wordRes selectedFields;
+ const wordRes selectedLagrangianFields;
+
+
+ if (!UPstream::parRun())
{
- if (optVerbose == 2)
- {
- WarningInFunction
- << "-verbose -verbose switches on full debugging"
- << nl << endl;
- ::debug = 1;
- }
+ FatalErrorInFunction
+ << ": This utility can only be run parallel"
+ << exit(FatalError);
+ }
+ if (decompose && reconstruct)
+ {
+ FatalErrorInFunction
+ << "Cannot specify both -decompose and -reconstruct"
+ << exit(FatalError);
+ }
+
+ if (optVerbose)
+ {
// Report on output
faMeshDistributor::verbose_ = 1;
parPointFieldDistributor::verbose_ = 1;
+
+ if (optVerbose > 1)
+ {
+ Info<< "Additional debugging enabled" << nl << endl;
+ ::debug = 1;
+ }
}
// Disable NaN setting and floating point error trapping. This is to avoid
@@ -1332,19 +1337,33 @@ int main(int argc, char *argv[])
// fvMeshDistribute.
Foam::sigFpe::unset(true);
- const wordRes selectedFields;
- const wordRes selectedLagrangianFields;
+ // File handlers (read/write)
+ // ==========================
+
+ // Read handler on processors with a volMesh
+ refPtr<fileOperation> volMeshReadHandler;
+
+ // Read handler on processors with an areaMesh
+ refPtr<fileOperation> areaMeshReadHandler;
+
+ // Handler for master-only operation (read/writing from/to undecomposed)
+ // - only the 'real' master, not io-rank masters
+ refPtr<fileOperation> masterOnlyHandler;
+
+ if (UPstream::master(UPstream::worldComm))
+ {
+ const bool oldParRun = UPstream::parRun(false);
+
+ masterOnlyHandler = fileOperation::NewUncollated();
+
+ UPstream::parRun(oldParRun);
+ }
if (decompose)
{
- InfoOrPout<< "Decomposing case (like decomposePar)" << nl << endl;
- if (reconstruct)
- {
- FatalErrorInFunction
- << "Cannot specify both -decompose and -reconstruct"
- << exit(FatalError);
- }
+ InfoOrPout<< "Decomposing case (like decomposePar)"
+ << nl << endl;
}
else if (reconstruct)
{
@@ -1374,19 +1393,13 @@ int main(int argc, char *argv[])
if ((decompose || reconstruct) && !overwrite)
{
overwrite = true;
- WarningInFunction
- << "Working in -decompose or -reconstruct mode:"
- " automatically implies -overwrite" << nl << endl;
- }
- if (!Pstream::parRun())
- {
- FatalErrorInFunction
- << ": This utility can only be run parallel"
- << exit(FatalError);
+ Warning
+ << nl << " "
+ << "Added implicit -overwrite for decompose/reconstruct modes"
+ << nl << endl;
}
-
if
(
fileHandler().ioRanks().contains(UPstream::myProcNo())
@@ -1401,7 +1414,7 @@ int main(int argc, char *argv[])
}
- if (!nfs)
+ if (hasDistributedFiles)
{
InfoOrPout<< "Detected multiple roots i.e. non-nfs running"
<< nl << endl;
@@ -1498,7 +1511,7 @@ int main(int argc, char *argv[])
// This will read the same controlDict but might have a different
// set of times so enforce same times
- if (!nfs)
+ if (hasDistributedFiles)
{
InfoOrPout<< "Creating time directories for undecomposed Time"
<< " on all processors" << nl << endl;
--
GitLab