diff --git a/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C b/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C
index 1155a1728c016a9c6ed5941e16285e94aebe3c79..63b23ce1cbbc19da10d30be36dcb4c9c1af9e2b3 100644
--- a/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C
+++ b/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C
@@ -54,12 +54,6 @@ namespace Foam
}
-// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
-
-void Foam::ptscotchDecomp::graphPath(const polyMesh& unused) const {}
-void Foam::ptscotchDecomp::check(const int retVal, const char* str) {}
-
-
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
Foam::label Foam::ptscotchDecomp::decompose
@@ -77,25 +71,6 @@ Foam::label Foam::ptscotchDecomp::decompose
}
-// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
-
-Foam::label Foam::ptscotchDecomp::decompose
-(
- const label adjncySize,
- const label adjncy[],
- const label xadjSize,
- const label xadj[],
- const List<scalar>& cWeights,
- labelList& finalDecomp
-) const
-{
- FatalErrorInFunction
- << notImplementedMessage << exit(FatalError);
-
- return -1;
-}
-
-
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::ptscotchDecomp::ptscotchDecomp
diff --git a/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C b/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C
index 535a9fa4815190899d2bad6fd9b355385733a50b..9259f8a85d1881f60805b0eb2ce5336de26e3d92 100644
--- a/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C
+++ b/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C
@@ -53,12 +53,6 @@ namespace Foam
);
}
-// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
-
-void Foam::scotchDecomp::graphPath(const polyMesh& unused) const {}
-
-void Foam::scotchDecomp::check(const int retVal, const char* str) {}
-
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
diff --git a/src/parallel/decompose/decompose/decompositionInformation.C b/src/parallel/decompose/decompose/decompositionInformation.C
index 0324ea602279e51b5ce33086bf896fc86df7679d..444560b47c0aabd1ff173141ccc11d6857c21393 100644
--- a/src/parallel/decompose/decompose/decompositionInformation.C
+++ b/src/parallel/decompose/decompose/decompositionInformation.C
@@ -41,15 +41,15 @@ void Foam::decompositionInformation::populate
nDomains_ = nDomain;
distrib_.clear();
- distrib_.setSize(nDomain);
+ distrib_.resize(nDomain);
for (labelList& subdist : distrib_)
{
subdist.clear();
- subdist.setSize(nDomain, Zero);
+ subdist.resize(nDomain, Zero);
}
- const label nCells = xadj.size()-1;
+ const label nCells = max(0, xadj.size()-1);
for (label celli = 0; celli < nCells; ++celli)
{
const label ownProc = decomp[celli];
@@ -167,7 +167,7 @@ void Foam::decompositionInformation::printSummary(Ostream& os) const
void Foam::decompositionInformation::printDetails(Ostream& os) const
{
os << "Decomposition details with (proc faces) "
- "for each processor connection"<< nl << nl;
+ "for each processor connection" << nl << nl;
forAll(distrib_, ownProc)
{
@@ -195,7 +195,7 @@ void Foam::decompositionInformation::printDetails(Ostream& os) const
// Second pass with details:
if (facesCount)
{
- Info<< " ";
+ os << ' ';
forAll(subdist, neiProc)
{
@@ -203,7 +203,7 @@ void Foam::decompositionInformation::printDetails(Ostream& os) const
if (n && ownProc != neiProc)
{
- os << " (" << neiProc << " " << n << ")";
+ os << " (" << neiProc << ' ' << n << ')';
}
}
}
diff --git a/src/parallel/decompose/kahipDecomp/Make/options b/src/parallel/decompose/kahipDecomp/Make/options
index 29522afef0c4cad3b39cc63d6ffda2e099a9372a..3210de9a2fb388f67c5f0d4cbaaeb6d5837a8348 100644
--- a/src/parallel/decompose/kahipDecomp/Make/options
+++ b/src/parallel/decompose/kahipDecomp/Make/options
@@ -7,5 +7,6 @@ EXE_INC = \
* openmp link dependency.
*/
LIB_LIBS = \
- -ldecompositionMethods \
- -L$(KAHIP_LIB_DIR) $(LINK_OPENMP) -lkahip
+ -L$(FOAM_LIBBIN) -ldecompositionMethods \
+ -L$(KAHIP_LIB_DIR) $(LINK_OPENMP) \
+ -lkahip$(KAHIP_LIBNAME_SUFFIX)
diff --git a/src/parallel/decompose/kahipDecomp/kahipDecomp.C b/src/parallel/decompose/kahipDecomp/kahipDecomp.C
index e372d68dab8b496deaae1d11bc9e71c98717348c..bc3990d10722332eba12bd16d1e03f5e6483594a 100644
--- a/src/parallel/decompose/kahipDecomp/kahipDecomp.C
+++ b/src/parallel/decompose/kahipDecomp/kahipDecomp.C
@@ -94,37 +94,72 @@ Foam::label Foam::kahipDecomp::decomposeSerial
}
#endif
- int numCells = xadj.size()-1;
+ int numCells = max(0, (xadj.size()-1));
- // Cell weights (so on the vertices of the dual)
- List<int> cellWeights;
+ // Addressing
+ ConstPrecisionAdaptor<int, label, List> adjncy_param(adjncy);
+ ConstPrecisionAdaptor<int, label, List> xadj_param(xadj);
+
+ // Output: cell -> processor addressing
+ decomp.resize(numCells);
+ decomp = 0;
+ PrecisionAdaptor<int, label, List> decomp_param(decomp, false);
+
+ // Avoid potential nullptr issues with zero-sized arrays
+ labelList adjncy_dummy, xadj_dummy, decomp_dummy;
+ if (!numCells)
+ {
+ adjncy_dummy.resize(1, 0);
+ adjncy_param.set(adjncy_dummy);
+
+ xadj_dummy.resize(2, 0);
+ xadj_param.set(xadj_dummy);
+
+ decomp_dummy.resize(1, 0);
+ decomp_param.clear(); // Avoid propagating spurious values
+ decomp_param.set(decomp_dummy);
+ }
+
+
+ // Graph
+ // ~~~~~
// Check for externally provided cellweights and if so initialise weights
+
+ bool hasWeights = !cWeights.empty();
+
// Note: min, not gMin since routine runs on master only.
- const scalar minWeights = min(cWeights);
+ const scalar minWeights = hasWeights ? min(cWeights) : scalar(1);
- if (!cWeights.empty())
+ if (minWeights <= 0)
{
- if (minWeights <= 0)
- {
- WarningInFunction
- << "Illegal minimum weight " << minWeights
- << endl;
- }
+ hasWeights = false;
+ WarningInFunction
+ << "Illegal minimum weight " << minWeights
+ << " ... ignoring"
+ << endl;
+ }
+ else if (hasWeights && (cWeights.size() != numCells))
+ {
+ FatalErrorInFunction
+ << "Number of cell weights " << cWeights.size()
+ << " does not equal number of cells " << numCells
+ << exit(FatalError);
+ }
- if (cWeights.size() != numCells)
- {
- FatalErrorInFunction
- << "Number of cell weights " << cWeights.size()
- << " does not equal number of cells " << numCells
- << exit(FatalError);
- }
+ // Cell weights (so on the vertices of the dual)
+ List<int> cellWeights;
+ if (hasWeights)
+ {
// Convert to integers.
- cellWeights.setSize(cWeights.size());
+ cellWeights.resize(cWeights.size());
forAll(cellWeights, i)
{
- cellWeights[i] = int(cWeights[i]/minWeights);
+ cellWeights[i] = static_cast<int>
+ (
+ cWeights[i]/minWeights
+ );
}
}
@@ -215,20 +250,12 @@ Foam::label Foam::kahipDecomp::decomposeSerial
// Output: number of cut edges
int edgeCut = 0;
- // Addressing
- ConstPrecisionAdaptor<int, label, List> xadj_param(xadj);
- ConstPrecisionAdaptor<int, label, List> adjncy_param(adjncy);
-
- // Output: cell -> processor addressing
- decomp.resize(numCells);
- PrecisionAdaptor<int, label, List> decomp_param(decomp, false);
-
#if 0 // WIP: #ifdef KAFFPA_CPP_INTERFACE
kaffpa_cpp
(
&numCells, // num vertices in graph
- (cellWeights.size() ? cellWeights.begin() : nullptr), // vertex wts
+ (cellWeights.empty() ? nullptr : cellWeights.data()), // vertex wts
xadj_param.constCast().data(), // indexing into adjncy
nullptr, // edge wts
adjncy_param.constCast().data(), // neighbour info
@@ -245,7 +272,7 @@ Foam::label Foam::kahipDecomp::decomposeSerial
kaffpa
(
&numCells, // num vertices in graph
- (cellWeights.size() ? cellWeights.begin() : nullptr), // vertex wts
+ (cellWeights.empty() ? nullptr : cellWeights.data()), // vertex wts
xadj_param.constCast().data(), // indexing into adjncy
nullptr, // edge wts
adjncy_param.constCast().data(), // neighbour info
diff --git a/src/parallel/decompose/metisDecomp/Make/options b/src/parallel/decompose/metisDecomp/Make/options
index 013a158dc943ff058594d0762b25bb980215c0fd..a6c3421ba0100b5a9bcdc823f1d8b135a762bab6 100644
--- a/src/parallel/decompose/metisDecomp/Make/options
+++ b/src/parallel/decompose/metisDecomp/Make/options
@@ -3,5 +3,6 @@ EXE_INC = \
-I$(LIB_SRC)/parallel/decompose/decompositionMethods/lnInclude
LIB_LIBS = \
- -ldecompositionMethods \
- -L$(METIS_LIB_DIR) -lmetis
+ -L$(FOAM_LIBBIN) -ldecompositionMethods \
+ -L$(METIS_LIB_DIR) \
+ -lmetis$(METIS_LIBNAME_SUFFIX)
diff --git a/src/parallel/decompose/metisDecomp/metisDecomp.C b/src/parallel/decompose/metisDecomp/metisDecomp.C
index bacb89ac0e6651665de57e48d53677f742388837..41fc7258c53674f922e5ebdf2ca743ab674cfe95 100644
--- a/src/parallel/decompose/metisDecomp/metisDecomp.C
+++ b/src/parallel/decompose/metisDecomp/metisDecomp.C
@@ -81,7 +81,7 @@ Foam::label Foam::metisDecomp::decomposeSerial
const dictionary* coeffsDictPtr = decompDict_.findDict("metisCoeffs");
- idx_t numCells = xadj.size()-1;
+ idx_t numCells = max(0, (xadj.size()-1));
// Decomposition options
List<idx_t> options(METIS_NOPTIONS);
@@ -89,7 +89,7 @@ Foam::label Foam::metisDecomp::decomposeSerial
// Processor weights initialised with no size, only used if specified in
// a file
- Field<real_t> processorWeights;
+ Field<real_t> procWeights;
// Cell weights (so on the vertices of the dual)
List<idx_t> cellWeights;
@@ -164,18 +164,17 @@ Foam::label Foam::metisDecomp::decomposeSerial
<< nl << endl;
}
- if (coeffDict.readIfPresent("processorWeights", processorWeights))
+ if (coeffDict.readIfPresent("processorWeights", procWeights))
{
- processorWeights /= sum(processorWeights);
-
- if (processorWeights.size() != nDomains_)
+ if (procWeights.size() != nDomains_)
{
- FatalErrorInFunction
- << "Number of processor weights "
- << processorWeights.size()
- << " does not equal number of domains " << nDomains_
- << exit(FatalError);
+ FatalIOErrorInFunction(coeffDict)
+ << "processorWeights (" << procWeights.size()
+ << ") != number of domains (" << nDomains_ << ")" << nl
+ << exit(FatalIOError);
}
+
+ procWeights /= sum(procWeights);
}
}
@@ -190,6 +189,26 @@ Foam::label Foam::metisDecomp::decomposeSerial
decomp.resize(numCells);
PrecisionAdaptor<idx_t, label, List> decomp_param(decomp, false);
+ // Avoid potential nullptr issues with zero-sized arrays
+ labelList adjncy_dummy, xadj_dummy, decomp_dummy;
+ if (!numCells)
+ {
+ adjncy_dummy.resize(1, 0);
+ adjncy_param.set(adjncy_dummy);
+
+ xadj_dummy.resize(2, 0);
+ xadj_param.set(xadj_dummy);
+
+ decomp_dummy.resize(1, 0);
+ decomp_param.clear(); // Avoid propagating spurious values
+ decomp_param.set(decomp_dummy);
+ }
+
+
+ //
+ // Decompose
+ //
+
// Output: number of cut edges
idx_t edgeCut = 0;
@@ -205,7 +224,7 @@ Foam::label Foam::metisDecomp::decomposeSerial
nullptr, // vsize: total communication vol
faceWeights.data(), // edge wts
&nProcs, // nParts
- processorWeights.data(), // tpwgts
+ procWeights.data(), // tpwgts
nullptr, // ubvec: processor imbalance (default)
options.data(),
&edgeCut,
@@ -224,7 +243,7 @@ Foam::label Foam::metisDecomp::decomposeSerial
nullptr, // vsize: total communication vol
faceWeights.data(), // edge wts
&nProcs, // nParts
- processorWeights.data(), // tpwgts
+ procWeights.data(), // tpwgts
nullptr, // ubvec: processor imbalance (default)
options.data(),
&edgeCut,
diff --git a/src/parallel/decompose/ptscotchDecomp/Make/options b/src/parallel/decompose/ptscotchDecomp/Make/options
index 31071eb9dda1e1aec8c95db9bbfc45e3a4f6ab39..e74f3e797b985bceb0bd3d9fd5de1c09deda676d 100644
--- a/src/parallel/decompose/ptscotchDecomp/Make/options
+++ b/src/parallel/decompose/ptscotchDecomp/Make/options
@@ -18,8 +18,8 @@ LIB_LIBS = \
-L$(FOAM_LIBBIN) -ldecompositionMethods \
-L$(PTSCOTCH_LIB_DIR) \
-L$(SCOTCH_LIB_DIR) \
- -lscotch \
- -lptscotch
+ -lscotch$(SCOTCH_LIBNAME_SUFFIX) \
+ -lptscotch$(SCOTCH_LIBNAME_SUFFIX)
/* errexit, except for windows compile (already in library) */
ifeq (,$(findstring windows,$(WM_OSTYPE)))
diff --git a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
index b0f3f8dc4585639e415c588acfa1648cdab9dd5d..9f73443ff8e39d599c0fc53d45f167995512046f 100644
--- a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
+++ b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
@@ -28,10 +28,11 @@ License
#include "ptscotchDecomp.H"
#include "addToRunTimeSelectionTable.H"
+#include "floatScalar.H"
#include "Time.H"
+#include "PrecisionAdaptor.H"
#include "OFstream.H"
-#include "globalIndex.H"
-#include "SubField.H"
+#include <limits>
// Avoid too many warnings from mpi.h
#pragma GCC diagnostic ignored "-Wold-style-cast"
@@ -49,11 +50,11 @@ License
#include <fenv.h>
#endif
-// Provide a clear error message if we have a size mismatch
+// Error if we attempt narrowing
static_assert
(
- sizeof(Foam::label) == sizeof(SCOTCH_Num),
- "sizeof(Foam::label) == sizeof(SCOTCH_Num), check your scotch headers"
+ sizeof(Foam::label) <= sizeof(SCOTCH_Num),
+ "SCOTCH_Num is too small for Foam::label, check your scotch headers"
);
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@@ -70,250 +71,73 @@ namespace Foam
}
-// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
-void Foam::ptscotchDecomp::graphPath(const polyMesh& mesh) const
+namespace Foam
{
- graphPath_ = mesh.time().path()/mesh.name();
-}
-
-void Foam::ptscotchDecomp::check(const int retVal, const char* str)
+// Check and print error message
+static inline void check(const int retVal, const char* what)
{
if (retVal)
{
FatalErrorInFunction
- << "Call to scotch routine " << str << " failed.\n"
+ << "Call to scotch routine " << what
+ << " failed (" << retVal << ")\n"
<< exit(FatalError);
}
}
+// The mesh-relative graph path/name (without extension)
+static inline Foam::fileName getGraphPathBase(const polyMesh& mesh)
+{
+ return mesh.time().path()/mesh.name();
+}
+
-////- Does prevention of 0 cell domains and calls ptscotch.
-//Foam::label Foam::ptscotchDecomp::decomposeZeroDomains
-//(
-// const labelList& initadjncy,
-// const labelList& initxadj,
-// const List<scalar>& initcWeights,
-// labelList& finalDecomp
-//) const
-//{
-// globalIndex globalCells(initxadj.size()-1);
-//
-// bool hasZeroDomain = false;
-// for (const int proci : Pstream::allProcs())
-// {
-// if (globalCells.localSize(proci) == 0)
-// {
-// hasZeroDomain = true;
-// break;
-// }
-// }
-//
-// if (!hasZeroDomain)
-// {
-// return decompose
-// (
-// initadjncy,
-// initxadj,
-// initcWeights,
-// finalDecomp
-// );
-// }
-//
-//
-// if (debug)
-// {
-// Info<< "ptscotchDecomp : have graphs with locally 0 cells."
-// << " trickling down." << endl;
-// }
-//
-// // Make sure every domain has at least one cell
-// // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-// // (scotch does not like zero sized domains)
-// // Trickle cells from processors that have them up to those that
-// // don't.
-//
-//
-// // Number of cells to send to the next processor
-// // (is same as number of cells next processor has to receive)
-// List<label> nSendCells(Pstream::nProcs(), Zero);
-//
-// for (label proci = nSendCells.size()-1; proci >=1; proci--)
-// {
-// label nLocalCells = globalCells.localSize(proci);
-// if (nLocalCells-nSendCells[proci] < 1)
-// {
-// nSendCells[proci-1] = nSendCells[proci]-nLocalCells+1;
-// }
-// }
-//
-// // First receive (so increasing the sizes of all arrays)
-//
-// Field<int> xadj(initxadj);
-// Field<int> adjncy(initadjncy);
-// scalarField cWeights(initcWeights);
-//
-// if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
-// {
-// // Receive cells from previous processor
-// IPstream fromPrevProc(Pstream::commsTypes::blocking,
-// Pstream::myProcNo()-1);
-//
-// Field<int> prevXadj(fromPrevProc);
-// Field<int> prevAdjncy(fromPrevProc);
-// scalarField prevCellWeights(fromPrevProc);
-//
-// if (prevXadj.size() != nSendCells[Pstream::myProcNo()-1])
-// {
-// FatalErrorInFunction
-// << "Expected from processor " << Pstream::myProcNo()-1
-// << " connectivity for " << nSendCells[Pstream::myProcNo()-1]
-// << " nCells but only received " << prevXadj.size()
-// << abort(FatalError);
-// }
-//
-// // Insert adjncy
-// prepend(prevAdjncy, adjncy);
-// // Adapt offsets and prepend xadj
-// xadj += prevAdjncy.size();
-// prepend(prevXadj, xadj);
-// // Weights
-// prepend(prevCellWeights, cWeights);
-// }
-//
-//
-// // Send to my next processor
-//
-// if (nSendCells[Pstream::myProcNo()] > 0)
-// {
-// // Send cells to next processor
-// OPstream toNextProc(Pstream::commsTypes::blocking,
-// Pstream::myProcNo()+1);
-//
-// label nCells = nSendCells[Pstream::myProcNo()];
-// label startCell = xadj.size()-1 - nCells;
-// label startFace = xadj[startCell];
-// label nFaces = adjncy.size()-startFace;
-//
-// // Send for all cell data: last nCells elements
-// // Send for all face data: last nFaces elements
-// toNextProc
-// << Field<int>::subField(xadj, nCells, startCell)-startFace
-// << Field<int>::subField(adjncy, nFaces, startFace)
-// <<
-// (
-// cWeights.size()
-// ? static_cast<const scalarField&>
-// (
-// scalarField::subField(cWeights, nCells, startCell)
-// )
-// : scalarField(0)
-// );
-//
-// // Remove data that has been sent
-// if (cWeights.size())
-// {
-// cWeights.setSize(cWeights.size()-nCells);
-// }
-// adjncy.setSize(adjncy.size()-nFaces);
-// xadj.setSize(xadj.size() - nCells);
-// }
-//
-//
-// // Do decomposition as normal. Sets finalDecomp.
-// label result = decompose(adjncy, xadj, cWeights, finalDecomp);
-//
-//
-// if (debug)
-// {
-// Info<< "ptscotchDecomp : have graphs with locally 0 cells."
-// << " trickling up." << endl;
-// }
-//
-//
-// // If we sent cells across make sure we undo it
-// // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-//
-// // Receive back from next processor if I sent something
-// if (nSendCells[Pstream::myProcNo()] > 0)
-// {
-// IPstream fromNextProc(Pstream::commsTypes::blocking,
-// Pstream::myProcNo()+1);
-//
-// List<label> nextFinalDecomp(fromNextProc);
-//
-// if (nextFinalDecomp.size() != nSendCells[Pstream::myProcNo()])
-// {
-// FatalErrorInFunction
-// << "Expected from processor " << Pstream::myProcNo()+1
-// << " decomposition for " << nSendCells[Pstream::myProcNo()]
-// << " nCells but only received " << nextFinalDecomp.size()
-// << abort(FatalError);
-// }
-//
-// append(nextFinalDecomp, finalDecomp);
-// }
-//
-// // Send back to previous processor.
-// if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
-// {
-// OPstream toPrevProc(Pstream::commsTypes::blocking,
-// Pstream::myProcNo()-1);
-//
-// label nToPrevious = nSendCells[Pstream::myProcNo()-1];
-//
-// toPrevProc <<
-// SubList<label>
-// (
-// finalDecomp,
-// nToPrevious,
-// finalDecomp.size()-nToPrevious
-// );
-//
-// // Remove locally what has been sent
-// finalDecomp.setSize(finalDecomp.size()-nToPrevious);
-// }
-// return result;
-//}
+} // End namespace Foam
+// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
+
Foam::label Foam::ptscotchDecomp::decompose
(
const labelList& adjncy,
const labelList& xadj,
const List<scalar>& cWeights,
- labelList& finalDecomp
+ labelList& decomp
) const
{
- List<label> dummyAdjncy;
- List<label> dummyXadj;
+ const SCOTCH_Num numCells = max(0, (xadj.size()-1));
- return decompose
- (
- adjncy.size(),
- (adjncy.size() ? adjncy.begin() : dummyAdjncy.begin()),
- xadj.size(),
- (xadj.size() ? xadj.begin() : dummyXadj.begin()),
- cWeights,
- finalDecomp
- );
-}
+ // Addressing
+ ConstPrecisionAdaptor<SCOTCH_Num, label, List> adjncy_param(adjncy);
+ ConstPrecisionAdaptor<SCOTCH_Num, label, List> xadj_param(xadj);
+ // Output: cell -> processor addressing
+ decomp.resize(numCells);
+ decomp = 0;
+ PrecisionAdaptor<SCOTCH_Num, label, List> decomp_param(decomp, false);
-Foam::label Foam::ptscotchDecomp::decompose
-(
- const label adjncySize,
- const label adjncy[],
- const label xadjSize,
- const label xadj[],
- const List<scalar>& cWeights,
- labelList& finalDecomp
-) const
-{
- if (debug)
+ // Avoid potential nullptr issues with zero-sized arrays
+ labelList adjncy_dummy, xadj_dummy, decomp_dummy;
+ if (!numCells)
+ {
+ adjncy_dummy.resize(1, 0);
+ adjncy_param.set(adjncy_dummy);
+
+ xadj_dummy.resize(2, 0);
+ xadj_param.set(xadj_dummy);
+
+ decomp_dummy.resize(1, 0);
+ decomp_param.clear(); // Avoid propagating spurious values
+ decomp_param.set(decomp_dummy);
+ }
+
+
+ if (debug & 2)
{
- Pout<< "ptscotchDecomp : entering with xadj:" << xadjSize << endl;
+ Pout<< "ptscotchDecomp : " << numCells << " cells" << endl;
}
// Dump graph
@@ -327,36 +151,36 @@ Foam::label Foam::ptscotchDecomp::decompose
Pout<< "Dumping Scotch graph file to " << str.name() << endl
<< "Use this in combination with dgpart." << endl;
- globalIndex globalCells(xadjSize-1);
+ const label numConnect = adjncy.size();
+ const label nTotCells = returnReduce(numCells, sumOp<label>());
+ const label nTotConnect = returnReduce(numConnect, sumOp<label>());
+
+ // Version 2 = Distributed graph file (.dgrf)
+ str << "2" << nl;
- // Distributed graph file (.grf)
- const label version = 2;
- str << version << nl;
- // Number of files (procglbnbr)
- str << Pstream::nProcs();
- // My file number (procloc)
- str << ' ' << Pstream::myProcNo() << nl;
+ // Number of files (procglbnbr), my file number (procloc)
+ str << Pstream::nProcs() << ' ' << Pstream::myProcNo() << nl;
- // Total number of vertices (vertglbnbr)
- str << globalCells.size();
+ // Total number of vertices (vertglbnbr),
// Total number of connections (edgeglbnbr)
- str << ' ' << returnReduce(xadj[xadjSize-1], sumOp<label>()) << nl;
- // Local number of vertices (vertlocnbr)
- str << xadjSize-1;
+ str << nTotCells << ' ' << nTotConnect << nl;
+
+ // Local number of vertices (vertlocnbr),
// Local number of connections (edgelocnbr)
- str << ' ' << xadj[xadjSize-1] << nl;
+ str << numCells << ' ' << numConnect << nl;
+
// Numbering starts from 0
- label baseval = 0;
- // 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeighs
- str << baseval << ' ' << "000" << nl;
- for (label celli = 0; celli < xadjSize-1; celli++)
+ // 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeights
+ str << "0 000" << nl;
+
+ for (label celli = 0; celli < numCells; ++celli)
{
- const label start = xadj[celli];
+ const label beg = xadj[celli];
const label end = xadj[celli+1];
- str << end-start; // size
+ str << (end-beg); // size
- for (label i = start; i < end; i++)
+ for (label i = beg; i < end; ++i)
{
str << ' ' << adjncy[i];
}
@@ -373,110 +197,120 @@ Foam::label Foam::ptscotchDecomp::decompose
// Default.
SCOTCH_Strat stradat;
- check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit");
+ check
+ (
+ SCOTCH_stratInit(&stradat),
+ "SCOTCH_stratInit"
+ );
string strategy;
if (coeffsDict_.readIfPresent("strategy", strategy))
{
- if (debug)
- {
- Info<< "ptscotchDecomp : Using strategy " << strategy << endl;
- }
+ DebugInfo
+ << "ptscotchDecomp : Using strategy " << strategy << endl;
+
SCOTCH_stratDgraphMap(&stradat, strategy.c_str());
//fprintf(stdout, "S\tStrat=");
//SCOTCH_stratSave(&stradat, stdout);
//fprintf(stdout, "\n");
}
+
// Graph
// ~~~~~
- List<label> velotab;
-
-
// Check for externally provided cellweights and if so initialise weights
- const scalar minWeights = gMin(cWeights);
- const scalar maxWeights = gMax(cWeights);
+ bool hasWeights = returnReduce(!cWeights.empty(), orOp<bool>());
- if (maxWeights > minWeights)
- {
- if (minWeights <= 0)
- {
- WarningInFunction
- << "Illegal minimum weight " << minWeights
- << endl;
- }
+ const scalar minWeights = hasWeights ? gMin(cWeights) : scalar(1);
- if (cWeights.size() != xadjSize-1)
- {
- FatalErrorInFunction
- << "Number of cell weights " << cWeights.size()
- << " does not equal number of cells " << xadjSize-1
- << exit(FatalError);
- }
+ if (minWeights <= 0)
+ {
+ hasWeights = false;
+ WarningInFunction
+ << "Illegal minimum weight " << minWeights
+ << " ... ignoring"
+ << endl;
+ }
+ else if (hasWeights && (cWeights.size() != numCells))
+ {
+ FatalErrorInFunction
+ << "Number of cell weights " << cWeights.size()
+ << " does not equal number of cells " << numCells
+ << exit(FatalError);
}
- scalar velotabSum = gSum(cWeights)/minWeights;
- scalar rangeScale(1.0);
+ List<SCOTCH_Num> velotab;
- if (Pstream::master())
+ if (hasWeights)
{
- if (velotabSum > scalar(labelMax - 1))
+ scalar rangeScale(1);
+
+ const scalar velotabSum = gSum(cWeights)/minWeights;
+
+ const scalar upperRange = static_cast<scalar>
+ (
+ std::numeric_limits<SCOTCH_Num>::max()-1
+ );
+
+ if (velotabSum > upperRange)
{
// 0.9 factor of safety to avoid floating point round-off in
// rangeScale tipping the subsequent sum over the integer limit.
- rangeScale = 0.9*scalar(labelMax - 1)/velotabSum;
+ rangeScale = 0.9*upperRange/velotabSum;
WarningInFunction
- << "Sum of weights has overflowed integer: " << velotabSum
- << ", compressing weight scale by a factor of " << rangeScale
- << endl;
+ << "Sum of weights overflows SCOTCH_Num: " << velotabSum
+ << ", compressing by factor " << rangeScale << endl;
}
- }
- Pstream::scatter(rangeScale);
-
- if (maxWeights > minWeights)
- {
if (cWeights.size())
{
// Convert to integers.
- velotab.setSize(cWeights.size());
+ velotab.resize(cWeights.size());
forAll(velotab, i)
{
- velotab[i] = int((cWeights[i]/minWeights - 1)*rangeScale) + 1;
+ velotab[i] = static_cast<SCOTCH_Num>
+ (
+ ((cWeights[i]/minWeights - 1)*rangeScale) + 1
+ );
}
}
else
{
- // Locally zero cells but not globally. Make sure we have
- // some size so .begin() does not return null pointer. Data
- // itself is never used.
- velotab.setSize(1);
- velotab[0] = 1;
+ // Locally zero cells but not globally.
+ // Provide some size to avoid null pointer.
+ velotab.resize(1, 1);
}
}
- if (debug)
+ //
+ // Decomposition graph
+ //
+
+ if (debug & 2)
{
Pout<< "SCOTCH_dgraphInit" << endl;
}
SCOTCH_Dgraph grafdat;
- check(SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD), "SCOTCH_dgraphInit");
-
+ check
+ (
+ SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD),
+ "SCOTCH_dgraphInit"
+ );
- if (debug)
+ if (debug & 2)
{
Pout<< "SCOTCH_dgraphBuild with:" << nl
- << "xadjSize-1 : " << xadjSize-1 << nl
- << "xadj : " << name(xadj) << nl
- << "velotab : " << name(velotab.begin()) << nl
- << "adjncySize : " << adjncySize << nl
- << "adjncy : " << name(adjncy) << nl
+ << "numCells : " << numCells << nl
+ << "xadj : " << name(xadj_param().cdata()) << nl
+ << "velotab : " << name(velotab.cdata()) << nl
+ << "adjncySize : " << adjncy_param().size() << nl
+ << "adjncy : " << name(adjncy_param().cdata()) << nl
<< endl;
}
@@ -484,80 +318,84 @@ Foam::label Foam::ptscotchDecomp::decompose
(
SCOTCH_dgraphBuild
(
- &grafdat, // grafdat
- 0, // baseval, c-style numbering
- xadjSize-1, // vertlocnbr, nCells
- xadjSize-1, // vertlocmax
- const_cast<SCOTCH_Num*>(xadj),
+ &grafdat, // Graph to build
+ 0, // Base for indexing (C-style)
+
+ numCells, // vertlocnbr [== nCells]
+ numCells, // vertlocmax
+
+ xadj_param.constCast().data(),
// vertloctab, start index per cell into
// adjncy
- const_cast<SCOTCH_Num*>(xadj+1),// vendloctab, end index ,,
+ (xadj_param.constCast().data()+1),
+ // vendloctab, end index ,,
- const_cast<SCOTCH_Num*>(velotab.begin()),// veloloctab, vtx weights
- nullptr, // vlblloctab
+ velotab.data(), // veloloctab, vtx weights
+ nullptr, // vlblloctab
- adjncySize, // edgelocnbr, number of arcs
- adjncySize, // edgelocsiz
- const_cast<SCOTCH_Num*>(adjncy), // edgeloctab
- nullptr, // edgegsttab
- nullptr // edlotab, edge weights
+ adjncy.size(), // edgelocnbr, number of arcs
+ adjncy.size(), // edgelocsiz
+ adjncy_param.constCast().data(), // edgeloctab
+ nullptr, // edgegsttab
+ nullptr // edlotab, edge weights
),
"SCOTCH_dgraphBuild"
);
-
- if (debug)
+ if (debug & 2)
{
Pout<< "SCOTCH_dgraphCheck" << endl;
}
- check(SCOTCH_dgraphCheck(&grafdat), "SCOTCH_dgraphCheck");
+ check
+ (
+ SCOTCH_dgraphCheck(&grafdat),
+ "SCOTCH_dgraphCheck"
+ );
// Architecture
// ~~~~~~~~~~~~
// (fully connected network topology since using switch)
- if (debug)
+ if (debug & 2)
{
Pout<< "SCOTCH_archInit" << endl;
}
SCOTCH_Arch archdat;
- check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
+ check
+ (
+ SCOTCH_archInit(&archdat),
+ "SCOTCH_archInit"
+ );
- List<label> processorWeights;
+ List<SCOTCH_Num> procWeights;
if
(
- coeffsDict_.readIfPresent("processorWeights", processorWeights)
- && processorWeights.size()
+ coeffsDict_.readIfPresent("processorWeights", procWeights)
+ && !procWeights.empty()
)
{
- if (debug)
- {
- Info<< "ptscotchDecomp : Using procesor weights "
- << processorWeights
- << endl;
- }
-
- if (processorWeights.size() != nDomains_)
+ if (procWeights.size() != nDomains_)
{
FatalIOErrorInFunction(coeffsDict_)
- << "processorWeights not the same size"
- << " as the wanted number of domains " << nDomains_
+ << "processorWeights (" << procWeights.size()
+ << ") != number of domains (" << nDomains_ << ")" << nl
<< exit(FatalIOError);
}
+ DebugInfo
+ << "ptscotchDecomp : Using procesor weights "
+ << procWeights << endl;
+
check
(
- SCOTCH_archCmpltw
- (
- &archdat, nDomains_, processorWeights.begin()
- ),
+ SCOTCH_archCmpltw(&archdat, nDomains_, procWeights.cdata()),
"SCOTCH_archCmpltw"
);
}
else
{
- if (debug)
+ if (debug & 2)
{
Pout<< "SCOTCH_archCmplt" << endl;
}
@@ -579,12 +417,7 @@ Foam::label Foam::ptscotchDecomp::decompose
);
#endif
-
- // Note: always provide allocated storage even if local size 0
- finalDecomp.setSize(max(1, xadjSize-1));
- finalDecomp = 0;
-
- if (debug)
+ if (debug & 2)
{
Pout<< "SCOTCH_dgraphMap" << endl;
}
@@ -595,7 +428,7 @@ Foam::label Foam::ptscotchDecomp::decompose
&grafdat,
&archdat,
&stradat, // const SCOTCH_Strat *
- finalDecomp.begin() // parttab
+ decomp_param.ref().data() // parttab
),
"SCOTCH_graphMap"
);
@@ -604,9 +437,6 @@ Foam::label Foam::ptscotchDecomp::decompose
feenableexcept(oldExcepts);
#endif
- // See above note to have size 1. Undo.
- finalDecomp.setSize(xadjSize-1);
-
//check
//(
// SCOTCH_dgraphPart
@@ -614,21 +444,19 @@ Foam::label Foam::ptscotchDecomp::decompose
// &grafdat,
// nDomains_, // partnbr
// &stradat, // const SCOTCH_Strat *
- // finalDecomp.begin() // parttab
+ // decomp_param.ref().data() // parttab
// ),
// "SCOTCH_graphPart"
//);
- if (debug)
+ if (debug & 2)
{
Pout<< "SCOTCH_dgraphExit" << endl;
}
- // Release storage for graph
- SCOTCH_dgraphExit(&grafdat);
- // Release storage for strategy
- SCOTCH_stratExit(&stradat);
- // Release storage for network topology
- SCOTCH_archExit(&archdat);
+
+ SCOTCH_dgraphExit(&grafdat); // Release storage for graph
+ SCOTCH_stratExit(&stradat); // Release storage for strategy
+ SCOTCH_archExit(&archdat); // Release storage for network topology
return 0;
}
@@ -657,7 +485,7 @@ Foam::labelList Foam::ptscotchDecomp::decompose
) const
{
// Where to write graph
- graphPath(mesh);
+ graphPath_ = getGraphPathBase(mesh);
if (points.size() != mesh.nCells())
{
@@ -707,7 +535,7 @@ Foam::labelList Foam::ptscotchDecomp::decompose
) const
{
// Where to write graph
- graphPath(mesh);
+ graphPath_ = getGraphPathBase(mesh);
if (agglom.size() != mesh.nCells())
{
diff --git a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.H b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.H
index 26e3d84c36b8db118d2c64c45cc4e643a587798e..6467fd86e06f1388c2a4d5766fd07cc31862dfe6 100644
--- a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.H
+++ b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.H
@@ -81,33 +81,15 @@ class ptscotchDecomp
// Private Member Functions
- //- Set graph path and name (mutable)
- void graphPath(const polyMesh& mesh) const;
-
- //- Check and print error message
- static void check(const int, const char*);
-
//- Decompose. Handles size 0 arrays
label decompose
(
const labelList& adjncy,
const labelList& xadj,
const List<scalar>& cWeights,
- labelList& finalDecomp
- ) const;
-
- //- Low level decompose
- label decompose
- (
- const label adjncySize,
- const label adjncy[],
- const label xadjSize,
- const label xadj[],
- const List<scalar>& cWeights,
- labelList& finalDecomp
+ labelList& decomp
) const;
-
//- No copy construct
ptscotchDecomp(const ptscotchDecomp&) = delete;
diff --git a/src/parallel/decompose/scotchDecomp/Make/options b/src/parallel/decompose/scotchDecomp/Make/options
index 67239cf898929ad150b6c2d035715b51f220c60a..ba446efa71e8bc7f2657ffce2c3ac73b2216ffa1 100644
--- a/src/parallel/decompose/scotchDecomp/Make/options
+++ b/src/parallel/decompose/scotchDecomp/Make/options
@@ -3,9 +3,9 @@ EXE_INC = \
-I$(LIB_SRC)/parallel/decompose/decompositionMethods/lnInclude
LIB_LIBS = \
- -ldecompositionMethods \
+ -L$(FOAM_LIBBIN) -ldecompositionMethods \
-L$(SCOTCH_LIB_DIR) \
- -lscotch
+ -lscotch$(SCOTCH_LIBNAME_SUFFIX)
/* errexit, except for windows compile (already in library) */
ifeq (,$(findstring windows,$(WM_OSTYPE)))
diff --git a/src/parallel/decompose/scotchDecomp/scotchDecomp.C b/src/parallel/decompose/scotchDecomp/scotchDecomp.C
index 034b82ba029dfcf9bda534320a6d2458c9afd815..514542e6b32999d8948d0e471452d934bdf02e17 100644
--- a/src/parallel/decompose/scotchDecomp/scotchDecomp.C
+++ b/src/parallel/decompose/scotchDecomp/scotchDecomp.C
@@ -30,7 +30,9 @@ License
#include "addToRunTimeSelectionTable.H"
#include "floatScalar.H"
#include "Time.H"
+#include "PrecisionAdaptor.H"
#include "OFstream.H"
+#include <limits>
// Probably not needed, but in case we pickup a ptscotch.h ...
#define MPICH_SKIP_MPICXX
@@ -47,14 +49,13 @@ License
#include <fenv.h>
#endif
-// Provide a clear error message if we have a size mismatch
+// Error if we attempt narrowing
static_assert
(
- sizeof(Foam::label) == sizeof(SCOTCH_Num),
- "sizeof(Foam::label) == sizeof(SCOTCH_Num), check your scotch headers"
+ sizeof(Foam::label) <= sizeof(SCOTCH_Num),
+ "SCOTCH_Num is too small for Foam::label, check your scotch headers"
);
-
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
namespace Foam
@@ -69,25 +70,34 @@ namespace Foam
}
-// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
-void Foam::scotchDecomp::graphPath(const polyMesh& mesh) const
+namespace Foam
{
- graphPath_ = mesh.time().path()/mesh.name() + ".grf";
-}
-
-void Foam::scotchDecomp::check(const int retVal, const char* str)
+// Check and print error message
+static inline void check(const int retVal, const char* what)
{
if (retVal)
{
FatalErrorInFunction
- << "Call to scotch routine " << str << " failed.\n"
+ << "Call to scotch routine " << what
+ << " failed (" << retVal << ")\n"
<< exit(FatalError);
}
}
+// The mesh-relative graph path/name (without extension)
+static inline Foam::fileName getGraphPathBase(const polyMesh& mesh)
+{
+ return mesh.time().path()/mesh.name();
+}
+
+
+} // End namespace Foam
+
+
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
Foam::label Foam::scotchDecomp::decomposeSerial
@@ -98,36 +108,62 @@ Foam::label Foam::scotchDecomp::decomposeSerial
labelList& decomp
) const
{
+ const SCOTCH_Num numCells = max(0, (xadj.size()-1));
+
+ // Addressing
+ ConstPrecisionAdaptor<SCOTCH_Num, label, List> adjncy_param(adjncy);
+ ConstPrecisionAdaptor<SCOTCH_Num, label, List> xadj_param(xadj);
+
+ // Output: cell -> processor addressing
+ decomp.resize(numCells);
+ decomp = 0;
+ PrecisionAdaptor<SCOTCH_Num, label, List> decomp_param(decomp, false);
+
+ // Avoid potential nullptr issues with zero-sized arrays
+ labelList adjncy_dummy, xadj_dummy, decomp_dummy;
+ if (!numCells)
+ {
+ adjncy_dummy.resize(1, 0);
+ adjncy_param.set(adjncy_dummy);
+
+ xadj_dummy.resize(2, 0);
+ xadj_param.set(xadj_dummy);
+
+ decomp_dummy.resize(1, 0);
+ decomp_param.clear(); // Avoid propagating spurious values
+ decomp_param.set(decomp_dummy);
+ }
+
+
// Dump graph
if (coeffsDict_.getOrDefault("writeGraph", false))
{
- OFstream str(graphPath_);
+ OFstream str(graphPath_ + ".grf");
Info<< "Dumping Scotch graph file to " << str.name() << nl
<< "Use this in combination with gpart." << endl;
- const label version = 0;
- str << version << nl;
- // Number of vertices
- str << xadj.size()-1 << ' ' << adjncy.size() << nl;
+ const label numConnect = adjncy.size();
- // Numbering starts from 0
- const label baseval = 0;
+ // Version 0 = Graph file (.grf)
+ str << "0" << nl;
- // Has weights?
- const label hasEdgeWeights = 0;
- const label hasVertexWeights = 0;
- const label numericflag = 10*hasEdgeWeights+hasVertexWeights;
- str << baseval << ' ' << numericflag << nl;
+ // Number of vertices,
+ // number of edges (connections)
+ str << numCells << ' ' << numConnect << nl;
+
+ // Numbering starts from 0
+ // 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeights
+ str << "0 000" << nl;
- for (label celli = 1; celli < xadj.size(); ++celli)
+ for (label celli = 0; celli < numCells; ++celli)
{
- const label start = xadj[celli-1];
- const label end = xadj[celli];
+ const label beg = xadj[celli];
+ const label end = xadj[celli+1];
- str << end-start; // size
+ str << (end-beg); // size
- for (label i = start; i < end; ++i)
+ for (label i = beg; i < end; ++i)
{
str << ' ' << adjncy[i];
}
@@ -135,6 +171,7 @@ Foam::label Foam::scotchDecomp::decomposeSerial
}
}
+
// Make repeatable
SCOTCH_randomReset();
@@ -143,15 +180,17 @@ Foam::label Foam::scotchDecomp::decomposeSerial
// Default.
SCOTCH_Strat stradat;
- check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit");
+ check
+ (
+ SCOTCH_stratInit(&stradat),
+ "SCOTCH_stratInit"
+ );
string strategy;
if (coeffsDict_.readIfPresent("strategy", strategy))
{
- if (debug)
- {
- Info<< "scotchDecomp : Using strategy " << strategy << endl;
- }
+ DebugInfo << "scotchDecomp : Using strategy " << strategy << endl;
+
SCOTCH_stratGraphMap(&stradat, strategy.c_str());
//fprintf(stdout, "S\tStrat=");
//SCOTCH_stratSave(&stradat, stdout);
@@ -162,74 +201,104 @@ Foam::label Foam::scotchDecomp::decomposeSerial
// Graph
// ~~~~~
- labelList velotab;
-
// Check for externally provided cellweights and if so initialise weights
+
+ bool hasWeights = !cWeights.empty();
+
// Note: min, not gMin since routine runs on master only.
- const scalar minWeights = min(cWeights);
- if (!cWeights.empty())
+ const scalar minWeights = hasWeights ? min(cWeights) : scalar(1);
+
+ if (minWeights <= 0)
{
- if (minWeights <= 0)
- {
- WarningInFunction
- << "Illegal minimum weight " << minWeights
- << endl;
- }
+ hasWeights = false;
+ WarningInFunction
+ << "Illegal minimum weight " << minWeights
+ << " ... ignoring"
+ << endl;
+ }
+ else if (hasWeights && (cWeights.size() != numCells))
+ {
+ FatalErrorInFunction
+ << "Number of cell weights " << cWeights.size()
+ << " does not equal number of cells " << numCells
+ << exit(FatalError);
+ }
- if (cWeights.size() != xadj.size()-1)
- {
- FatalErrorInFunction
- << "Number of cell weights " << cWeights.size()
- << " does not equal number of cells " << xadj.size()-1
- << exit(FatalError);
- }
- scalar velotabSum = sum(cWeights)/minWeights;
+ List<SCOTCH_Num> velotab;
+
+ if (hasWeights)
+ {
+ scalar rangeScale(1);
- scalar rangeScale(1.0);
+ const scalar velotabSum = sum(cWeights)/minWeights;
- if (velotabSum > scalar(labelMax - 1))
+ const scalar upperRange = static_cast<scalar>
+ (
+ std::numeric_limits<SCOTCH_Num>::max()-1
+ );
+
+ if (velotabSum > upperRange)
{
// 0.9 factor of safety to avoid floating point round-off in
// rangeScale tipping the subsequent sum over the integer limit.
- rangeScale = 0.9*scalar(labelMax - 1)/velotabSum;
+ rangeScale = 0.9*upperRange/velotabSum;
WarningInFunction
- << "Sum of weights has overflowed integer: " << velotabSum
- << ", compressing weight scale by a factor of " << rangeScale
- << endl;
+ << "Sum of weights overflows SCOTCH_Num: " << velotabSum
+ << ", compressing by factor " << rangeScale << endl;
}
- // Convert to integers.
- velotab.setSize(cWeights.size());
-
- forAll(velotab, i)
{
- velotab[i] = int((cWeights[i]/minWeights - 1)*rangeScale) + 1;
+ // Convert to integers.
+ velotab.resize(cWeights.size());
+
+ forAll(velotab, i)
+ {
+ velotab[i] = static_cast<SCOTCH_Num>
+ (
+ ((cWeights[i]/minWeights - 1)*rangeScale) + 1
+ );
+ }
}
}
+ //
+ // Decomposition graph
+ //
+
SCOTCH_Graph grafdat;
- check(SCOTCH_graphInit(&grafdat), "SCOTCH_graphInit");
+ check
+ (
+ SCOTCH_graphInit(&grafdat),
+ "SCOTCH_graphInit"
+ );
check
(
SCOTCH_graphBuild
(
- &grafdat,
- 0, // baseval, c-style numbering
- xadj.size()-1, // vertnbr, nCells
- xadj.begin(), // verttab, start index per cell into adjncy
- &xadj[1], // vendtab, end index ,,
- velotab.begin(), // velotab, vertex weights
- nullptr, // vlbltab
- adjncy.size(), // edgenbr, number of arcs
- adjncy.begin(), // edgetab
- nullptr // edlotab, edge weights
+ &grafdat, // Graph to build
+ 0, // Base for indexing (C-style)
+
+ numCells, // Number of vertices [== nCells]
+ xadj_param().cdata(), // verttab, start index per cell into adjncy
+ nullptr, // vendtab, end index (nullptr == automatic)
+
+ velotab.cdata(), // velotab, vertex weights
+ nullptr, // Vertex labels (nullptr == ignore)
+
+ adjncy.size(), // Number of graph edges
+ adjncy_param().cdata(), // Edge array
+ nullptr // Edge weights (nullptr == ignore)
),
"SCOTCH_graphBuild"
);
- check(SCOTCH_graphCheck(&grafdat), "SCOTCH_graphCheck");
+ check
+ (
+ SCOTCH_graphCheck(&grafdat),
+ "SCOTCH_graphCheck"
+ );
// Architecture
@@ -237,34 +306,34 @@ Foam::label Foam::scotchDecomp::decomposeSerial
// (fully connected network topology since using switch)
SCOTCH_Arch archdat;
- check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
+ check
+ (
+ SCOTCH_archInit(&archdat),
+ "SCOTCH_archInit"
+ );
- labelList processorWeights;
+ List<SCOTCH_Num> procWeights;
if
(
- coeffsDict_.readIfPresent("processorWeights", processorWeights)
- && processorWeights.size()
+ coeffsDict_.readIfPresent("processorWeights", procWeights)
+ && !procWeights.empty()
)
{
- if (debug)
- {
- Info<< "scotchDecomp : Using procesor weights " << processorWeights
- << endl;
- }
- if (processorWeights.size() != nDomains_)
+ if (procWeights.size() != nDomains_)
{
FatalIOErrorInFunction(coeffsDict_)
- << "processorWeights not the same size"
- << " as the wanted number of domains " << nDomains_
+ << "processorWeights (" << procWeights.size()
+ << ") != number of domains (" << nDomains_ << ")" << nl
<< exit(FatalIOError);
}
+ DebugInfo
+ << "scotchDecomp : Using procesor weights "
+ << procWeights << endl;
+
check
(
- SCOTCH_archCmpltw
- (
- &archdat, nDomains_, processorWeights.begin()
- ),
+ SCOTCH_archCmpltw(&archdat, nDomains_, procWeights.cdata()),
"SCOTCH_archCmpltw"
);
}
@@ -328,16 +397,14 @@ Foam::label Foam::scotchDecomp::decomposeSerial
);
#endif
- decomp.setSize(xadj.size()-1);
- decomp = 0;
check
(
SCOTCH_graphMap
(
&grafdat,
&archdat,
- &stradat, // const SCOTCH_Strat *
- decomp.begin() // parttab
+ &stradat, // const SCOTCH_Strat *
+ decomp_param.ref().data() // parttab
),
"SCOTCH_graphMap"
);
@@ -346,25 +413,22 @@ Foam::label Foam::scotchDecomp::decomposeSerial
feenableexcept(oldExcepts);
#endif
- //decomp.setSize(xadj.size()-1);
+ //decomp.resize(numCells);
//check
//(
// SCOTCH_graphPart
// (
// &grafdat,
- // nDomains_, // partnbr
- // &stradat, // const SCOTCH_Strat *
- // decomp.begin() // parttab
+ // nDomains_, // partnbr
+ // &stradat, // const SCOTCH_Strat *
+ // decomp_param.ref().data() // parttab
// ),
// "SCOTCH_graphPart"
//);
- // Release storage for graph
- SCOTCH_graphExit(&grafdat);
- // Release storage for strategy
- SCOTCH_stratExit(&stradat);
- // Release storage for network topology
- SCOTCH_archExit(&archdat);
+ SCOTCH_graphExit(&grafdat); // Release storage for graph
+ SCOTCH_stratExit(&stradat); // Release storage for strategy
+ SCOTCH_archExit(&archdat); // Release storage for network topology
return 0;
}
@@ -392,7 +456,7 @@ Foam::labelList Foam::scotchDecomp::decompose
) const
{
// Where to write graph
- graphPath(mesh);
+ graphPath_ = getGraphPathBase(mesh);
return metisLikeDecomp::decompose
(
@@ -412,7 +476,7 @@ Foam::labelList Foam::scotchDecomp::decompose
) const
{
// Where to write graph
- graphPath(mesh);
+ graphPath_ = getGraphPathBase(mesh);
return metisLikeDecomp::decompose
(
diff --git a/src/parallel/decompose/scotchDecomp/scotchDecomp.H b/src/parallel/decompose/scotchDecomp/scotchDecomp.H
index 6b61c5a66a2eb4becdc7ea92ff63081297e1ee91..051a8ecf995b29cebfc730121387b1f6d6db3262 100644
--- a/src/parallel/decompose/scotchDecomp/scotchDecomp.H
+++ b/src/parallel/decompose/scotchDecomp/scotchDecomp.H
@@ -238,15 +238,6 @@ class scotchDecomp
mutable fileName graphPath_;
- // Private Member Functions
-
- //- Set graph path and name (mutable)
- void graphPath(const polyMesh& mesh) const;
-
- //- Check and print error message
- static void check(const int, const char*);
-
-
protected:
// Protected Member Functions
diff --git a/wmake/scripts/have_kahip b/wmake/scripts/have_kahip
index 04ccc8643e19361c3c97a1787a9f4e2c9f3a7660..540540b55e231a8924a82cc7002088bf1a985e9d 100644
--- a/wmake/scripts/have_kahip
+++ b/wmake/scripts/have_kahip
@@ -5,7 +5,7 @@
# \\ / A nd | www.openfoam.com
# \\/ M anipulation |
#------------------------------------------------------------------------------
-# Copyright (C) 2018-2020 OpenCFD Ltd.
+# Copyright (C) 2018-2021 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@@ -27,6 +27,7 @@
# KAHIP_ARCH_PATH
# KAHIP_INC_DIR
# KAHIP_LIB_DIR
+# KAHIP_LIBNAME_SUFFIX [optional]
#
#------------------------------------------------------------------------------
. ${WM_PROJECT_DIR:?}/wmake/scripts/sysFunctions # General system functions
@@ -37,7 +38,7 @@
no_kahip()
{
unset HAVE_KAHIP KAHIP_ARCH_PATH KAHIP_INC_DIR KAHIP_LIB_DIR
- unset KAHIP_VERSION
+ unset KAHIP_VERSION KAHIP_LIBNAME_SUFFIX
}
@@ -48,6 +49,10 @@ echo_kahip()
echo "root=\"$KAHIP_ARCH_PATH\""
echo "include=\"$KAHIP_INC_DIR\""
echo "library=\"$KAHIP_LIB_DIR\""
+ if [ -n "$KAHIP_LIBNAME_SUFFIX" ]
+ then
+ echo "libsuffix=\"$KAHIP_LIBNAME_SUFFIX\""
+ fi
}
@@ -100,9 +105,9 @@ search_kahip()
# ----------------------------------
# kahip itself is 32-bit int, but our interface handles some
- # 64-bit conversion (mesh size).
+ # 64-bit conversion.
- echo "kahip (label=32) - $prefix"
+ echo "kahip (int32) - $prefix"
export HAVE_KAHIP=true
export KAHIP_ARCH_PATH="$prefix"
export KAHIP_INC_DIR="${header%/*}" # Basename
@@ -153,8 +158,10 @@ no_kahip
# Test/query
case "$1" in
--test)
+-test | -debug-test)
+ [ "$1" = "-debug-test" ] && set -x
have_kahip
+ [ "$1" = "-debug-test" ] && set +x
echo_kahip
;;
-query)
diff --git a/wmake/scripts/have_metis b/wmake/scripts/have_metis
index 7b545d9064f6d9beaf3ee2891cb91b3246893178..1b6eb54f3a063d80ee1f242da6b81033c933d832 100644
--- a/wmake/scripts/have_metis
+++ b/wmake/scripts/have_metis
@@ -5,7 +5,7 @@
# \\ / A nd | www.openfoam.com
# \\/ M anipulation |
#------------------------------------------------------------------------------
-# Copyright (C) 2018-2020 OpenCFD Ltd.
+# Copyright (C) 2018-2021 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@@ -27,6 +27,7 @@
# METIS_ARCH_PATH
# METIS_INC_DIR
# METIS_LIB_DIR
+# METIS_LIBNAME_SUFFIX [optional]
#
#------------------------------------------------------------------------------
. ${WM_PROJECT_DIR:?}/wmake/scripts/sysFunctions # General system functions
@@ -37,7 +38,7 @@
no_metis()
{
unset HAVE_METIS METIS_ARCH_PATH METIS_INC_DIR METIS_LIB_DIR
- unset METIS_VERSION
+ unset METIS_VERSION METIS_LIBNAME_SUFFIX
}
@@ -48,6 +49,10 @@ echo_metis()
echo "root=\"$METIS_ARCH_PATH\""
echo "include=\"$METIS_INC_DIR\""
echo "library=\"$METIS_LIB_DIR\""
+ if [ -n "$METIS_LIBNAME_SUFFIX" ]
+ then
+ echo "libsuffix=\"$METIS_LIBNAME_SUFFIX\""
+ fi
}
@@ -73,7 +78,7 @@ search_metis()
elif hasAbsdir "$prefix"
then
header=$(findFirstFile "$prefix/include/$incName")
- library=$(findExtLib "$libName")
+ library="-extlib" # Delay search...
elif isSystem "$prefix"
then
header=$(findSystemInclude -name="$incName")
@@ -89,6 +94,27 @@ search_metis()
return 2
}
+ # ----------------------------------
+ # Extract IDXTYPEWIDTH from metis.h: regex as per ThirdParty makeMETIS
+ # - ensure consistent size between OpenFOAM and metis header
+
+ local label
+ if [ -f "$header" ]
+ then
+ label=$(sed -ne \
+ 's/^.*#define *IDXTYPEWIDTH *\([1-9][0-9]\).*/\1/p' \
+ "$header")
+ fi
+ : "${label:=unknown}" # Safety
+
+ # Transform (32 | 64) -> (int32 | int64)
+ case "$label" in (32|64) label="int${label}" ;; esac
+
+ if [ "$library" = "-extlib" ]
+ then
+ library=$(findExtLib "${libName}-${label}" "$libName")
+ fi
+
# Library
[ -n "$library" ] \
|| library=$(findLibrary -prefix="$prefix" -name="$libName") \
@@ -97,23 +123,21 @@ search_metis()
return 2
}
- # ----------------------------------
-
- local label
-
- # Ensure consistent sizes between OpenFOAM and metis header
- # Extract IDXTYPEWIDTH from metis.h: regex as per ThirdParty Allwmake
- label=$(sed -ne \
- 's/^.*#define *IDXTYPEWIDTH *\([1-9][0-9]\).*/\1/p' \
- "$header")
- : "${label:=unknown}"
+ # Library name suffix (-int32 | -int64)
+ case "${library##*/}" in
+ (*-int32.*) export METIS_LIBNAME_SUFFIX="-int32" ;;
+ (*-int64.*) export METIS_LIBNAME_SUFFIX="-int64" ;;
+ esac
# OK
- echo "metis (label=$label) - $prefix"
+ echo "metis (${label}) - $prefix"
export HAVE_METIS=true
export METIS_ARCH_PATH="$prefix"
export METIS_INC_DIR="${header%/*}" # Basename
export METIS_LIB_DIR="${library%/*}" # Basename
+
+ ##echo "DEBUG: header=$header" 1>&2
+ ##echo "DEBUG: library=$library" 1>&2
}
@@ -160,8 +184,10 @@ no_metis
# Test/query
case "$1" in
--test)
+-test | -debug-test)
+ [ "$1" = "-debug-test" ] && set -x
have_metis
+ [ "$1" = "-debug-test" ] && set +x
echo_metis
;;
-query)
diff --git a/wmake/scripts/have_scotch b/wmake/scripts/have_scotch
index fdae90a59b810c527ff85eb31cf96dab84cdad81..82adee73645f72db6836d71a39934baaedea6bd1 100644
--- a/wmake/scripts/have_scotch
+++ b/wmake/scripts/have_scotch
@@ -5,7 +5,7 @@
# \\ / A nd | www.openfoam.com
# \\/ M anipulation |
#------------------------------------------------------------------------------
-# Copyright (C) 2018-2020 OpenCFD Ltd.
+# Copyright (C) 2018-2021 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@@ -27,6 +27,7 @@
# SCOTCH_ARCH_PATH
# SCOTCH_INC_DIR
# SCOTCH_LIB_DIR
+# SCOTCH_LIBNAME_SUFFIX [optional]
#
# Functions provided [Must call have_scotch first]
# have_ptscotch, search_ptscotch
@@ -70,6 +71,10 @@
# when MPI_ARCH_PATH=/usr/lib64/openmpi
# and mpicc --showme:compile -> -I/usr/include/openmpi-x86_64
#
+# NB: for RedHat (8+) /usr/include/scotch.h is a wrapper that includes
+# /usr/include/scotch-64.h (LP64 mode)
+# or /usr/include/scotch-32.h (ILP32 mode)
+# In either case, int is 32 bits
#
# openSUSE
# --------
@@ -90,7 +95,7 @@
no_scotch()
{
unset HAVE_SCOTCH SCOTCH_ARCH_PATH SCOTCH_INC_DIR SCOTCH_LIB_DIR
- unset SCOTCH_VERSION
+ unset SCOTCH_VERSION SCOTCH_LIBNAME_SUFFIX
unset HAVE_PTSCOTCH PTSCOTCH_ARCH_PATH PTSCOTCH_INC_DIR PTSCOTCH_LIB_DIR
}
@@ -102,11 +107,19 @@ echo_scotch()
echo "root=\"$SCOTCH_ARCH_PATH\""
echo "include=\"$SCOTCH_INC_DIR\""
echo "library=\"$SCOTCH_LIB_DIR\""
+ if [ -n "$SCOTCH_LIBNAME_SUFFIX" ]
+ then
+ echo "libsuffix=\"$SCOTCH_LIBNAME_SUFFIX\""
+ fi
echo
echo "ptscotch=${HAVE_PTSCOTCH:-false}"
echo "root=\"$PTSCOTCH_ARCH_PATH\""
echo "include=\"$PTSCOTCH_INC_DIR\""
echo "library=\"$PTSCOTCH_LIB_DIR\""
+ if [ -n "$SCOTCH_LIBNAME_SUFFIX" ]
+ then
+ echo "libsuffix=\"$SCOTCH_LIBNAME_SUFFIX\""
+ fi
}
@@ -137,7 +150,7 @@ search_scotch()
"$prefix/include/scotch/$incName" \
"$prefix/include/$incName" \
)
- library=$(findExtLib "$libName")
+ library="-extlib" # Delay search...
elif isSystem "$prefix"
then
header=$(findFirstFile \
@@ -161,6 +174,41 @@ search_scotch()
return 2
}
+ # ----------------------------------
+ # Extract 'typedef int64_t SCOTCH_Num' etc from header file
+ # - ensure consistent size between OpenFOAM and scotch header
+ # - for some systems, scotch.h includes scotch-64.h (for example).
+
+ local label
+ for hdr in \
+ "$header" \
+ "${header%/*}"/scotch-64.h \
+ "${header%/*}"/scotch-32.h \
+ ;
+ do
+ if [ -f "$hdr" ]
+ then
+ label=$(sed -ne \
+ 's/^.*typedef *\([^ ]*\) *SCOTCH_Num.*/\1/p' \
+ "$hdr")
+
+ if [ -n "$label" ]
+ then
+ header="$hdr" # Appears successful
+ break
+ fi
+ fi
+ done
+ : "${label:=unknown}" # Safety
+
+ # Transform (int32_t | int64_t) -> (int32 | int64)
+ case "$label" in (int32_t | int64_t) label="${label%_t}" ;; esac
+
+ if [ "$library" = "-extlib" ]
+ then
+ library=$(findExtLib "${libName}-${label}" "$libName")
+ fi
+
# Library
[ -n "$library" ] \
|| library=$(findLibrary -prefix="$prefix" -name="$libName" -local="$localDir") \
@@ -171,16 +219,6 @@ search_scotch()
# ----------------------------------
- local label
-
- # Ensure consistent sizes between OpenFOAM and scotch header
- # extract 'typedef int64_t SCOTCH_Num' or equivalent
- label=$(sed -ne \
- 's/^.*typedef *\([^ ]*\) *SCOTCH_Num.*/\1/p' \
- "$header")
- : "${label:=unknown}"
-
-
# No SCOTCH_VERSION set? Try to obtain from header
# extract #define SCOTCH_VERSION, SCOTCH_RELEASE, SCOTCH_PATCHLEVEL
[ -n "$SCOTCH_VERSION" ] || \
@@ -197,8 +235,12 @@ search_scotch()
)
: "${SCOTCH_VERSION:=scotch}" # Failsafe value
+ # Accept widening of OpenFOAM label to scotch label (SCOTCH_Num)
+ # but reject narrowing here instead of in the code
case "$WM_LABEL_SIZE:$label" in
- (32:int32_t | 32:int | 64:int64_t | 64:long)
+ ( 32:int32 | 32:int \
+ | 32:int64 | 32:long \
+ | 64:int64 | 64:long )
;;
(*)
@@ -211,13 +253,22 @@ search_scotch()
;;
esac
+ # Library name suffix (-int32 | -int64)
+ case "${library##*/}" in
+ (*-int32.*) export SCOTCH_LIBNAME_SUFFIX="-int32" ;;
+ (*-int64.*) export SCOTCH_LIBNAME_SUFFIX="-int64" ;;
+ esac
+
# OK
- echo "scotch (label=$label) - $prefix"
+ echo "scotch ($label) - $prefix"
export HAVE_SCOTCH=true
export SCOTCH_ARCH_PATH="$prefix"
export SCOTCH_INC_DIR="${header%/*}" # Basename
export SCOTCH_LIB_DIR="${library%/*}" # Basename
export SCOTCH_VERSION
+
+ ##echo "DEBUG: header=$header" 1>&2
+ ##echo "DEBUG: library=$library" 1>&2
}
@@ -256,7 +307,11 @@ search_ptscotch()
"$prefix/include/$mpiName/$incName" \
"$prefix/include/${mpiName}-$(uname -m)/$incName" \
)
- library="$(findExtLib $FOAM_MPI/$libName $libName)"
+ library=$(findExtLib \
+ "$FOAM_MPI/$libName${SCOTCH_LIBNAME_SUFFIX}" \
+ "$libName${SCOTCH_LIBNAME_SUFFIX}" \
+ "$FOAM_MPI/$libName${SCOTCH_LIBNAME_SUFFIX}" \
+ )
elif isSystem "$prefix"
then
header=$(findFirstFile \
@@ -302,6 +357,9 @@ search_ptscotch()
export PTSCOTCH_ARCH_PATH="$prefix"
export PTSCOTCH_INC_DIR="${header%/*}" # Basename
export PTSCOTCH_LIB_DIR="${library%/*}" # Basename
+
+ ##echo "DEBUG: header=$header" 1>&2
+ ##echo "DEBUG: library=$library" 1>&2
}
@@ -366,8 +424,10 @@ no_scotch
# Test/query
case "$1" in
--test)
+-test | -debug-test)
+ [ "$1" = "-debug-test" ] && set -x
have_scotch && have_ptscotch
+ [ "$1" = "-debug-test" ] && set +x
echo_scotch
;;
-query)