diff --git a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
index 2bf6e7d01ccc58ed3b46c5ee787f3a1b0e5294ee..6fc60079e7de5ab7430d5e530aca4080d9535c51 100644
--- a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
+++ b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
- Copyright (C) 2015-2022 OpenCFD Ltd.
+ Copyright (C) 2015-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -395,6 +395,31 @@ Foam::label Foam::ptscotchDecomp::decompose
}
else
{
+ // Check for optional domains/weights
+ List<SCOTCH_Num> domains;
+ List<scalar> dWeights;
+ if
+ (
+ coeffsDict_.readIfPresent
+ (
+ "domains",
+ domains,
+ keyType::LITERAL
+ )
+ && coeffsDict_.readIfPresent
+ (
+ "domainWeights",
+ dWeights,
+ keyType::LITERAL
+ )
+ )
+ {
+ WarningInFunction
+ << "Ignoring multi-level decomposition since"
+ << " not supported by ptscotch."
+ << " It is supported by scotch" << endl;
+ }
+
if (debug & 2)
{
Pout<< "SCOTCH_archCmplt" << endl;
diff --git a/src/parallel/decompose/scotchDecomp/scotchDecomp.C b/src/parallel/decompose/scotchDecomp/scotchDecomp.C
index 514542e6b32999d8948d0e471452d934bdf02e17..a16695326be73e63f7f3a2babac5670c2e23a192 100644
--- a/src/parallel/decompose/scotchDecomp/scotchDecomp.C
+++ b/src/parallel/decompose/scotchDecomp/scotchDecomp.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
- Copyright (C) 2015-2021 OpenCFD Ltd.
+ Copyright (C) 2015-2021,2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -313,6 +313,12 @@ Foam::label Foam::scotchDecomp::decomposeSerial
);
List<SCOTCH_Num> procWeights;
+
+ // Do optional tree-like/multi-level decomposition by specifying
+ // domains/weights
+ List<SCOTCH_Num> domains;
+ List<scalar> dWeights;
+
if
(
coeffsDict_.readIfPresent("processorWeights", procWeights)
@@ -337,6 +343,71 @@ Foam::label Foam::scotchDecomp::decomposeSerial
"SCOTCH_archCmpltw"
);
}
+ else if
+ (
+ coeffsDict_.readIfPresent("domains", domains, keyType::LITERAL)
+ && coeffsDict_.readIfPresent("domainWeights", dWeights, keyType::LITERAL)
+ )
+ {
+ // multi-level
+
+ label nTotal = 1;
+ for (const label n : domains)
+ {
+ nTotal *= n;
+ }
+
+ if (nTotal < nDomains())
+ {
+ const label sz = domains.size();
+ domains.setSize(sz+1);
+ dWeights.setSize(sz+1);
+ for (label i = sz-1; i >= 0; i--)
+ {
+ domains[i+1] = domains[i];
+ dWeights[i+1] = dWeights[i];
+ }
+
+ if (nDomains() % nTotal)
+ {
+ FatalErrorInFunction
+ << "Top level decomposition specifies " << nDomains()
+ << " domains which is not equal to the product of"
+ << " all sub domains " << nTotal
+ << exit(FatalError);
+ }
+
+ domains[0] = nDomains() / nTotal;
+ dWeights[0] = scalar(1);
+ }
+
+
+ // Note: min, not gMin since routine runs on master only.
+ const scalar minWeights = min(dWeights);
+
+ // Convert to integers.
+ List<SCOTCH_Num> weights(dWeights.size());
+
+ forAll(weights, i)
+ {
+ weights[i] = static_cast<SCOTCH_Num>
+ (
+ (dWeights[i]/minWeights - 1) + 1
+ );
+ }
+
+ check
+ (
+ SCOTCH_archTleaf
+ (
+ &archdat,
+ SCOTCH_Num(domains.size()),
+ domains.cdata(),
+ weights.cdata()
+ ),
+ "SCOTCH_archTleaf"
+ );
+ }
else
{
check
diff --git a/src/parallel/decompose/scotchDecomp/scotchDecomp.H b/src/parallel/decompose/scotchDecomp/scotchDecomp.H
index 051a8ecf995b29cebfc730121387b1f6d6db3262..6ce9588c97c2a54d2b41220b9ecca00353e6e334 100644
--- a/src/parallel/decompose/scotchDecomp/scotchDecomp.H
+++ b/src/parallel/decompose/scotchDecomp/scotchDecomp.H
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
- Copyright (C) 2017-2021 OpenCFD Ltd.
+ Copyright (C) 2017-2021,2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -201,6 +201,39 @@ Description
</ol>
+
+ Also support for multi-level decomposition by specifying inter-level
+ communication weights:
+
+ \verbatim
+ numberOfSubdomains 2048;
+ coeffs
+ {
+ // Divide into 64 clusters, each of 32 cores
+ domains (64 32);
+ // Inside a cluster the communication weight is 1% of that inbetween
+ // clusters
+ domainWeights (1 0.01);
+ }
+ \endverbatim
+
+ Alternatively the first-level decomposition can be left out by assuming
+ weights are 1 for that level:
+
+ \verbatim
+ numberOfSubdomains 2048;
+ coeffs
+ {
+ // Divide into 2048/32 clusters
+ domains (32);
+ // Inside a cluster the communication weight is 1% of that inbetween
+ // clusters
+ domainWeights (0.01);
+ }
+ \endverbatim
+
+
+
Note
\c gpart can be found in the current search path by adding the respective
\c bin folder from the Scotch installation, namely by running the following