Commit 549d26f3 authored by mattijs's avatar mattijs
Browse files

ENH: GAMGAgglomeration: moved lduMesh agglomeration into GAMGAgglomeration

parent d9db5dc4
......@@ -255,4 +255,101 @@ const Foam::lduInterfacePtrsList& Foam::GAMGAgglomeration::interfaceLevel
}
void Foam::GAMGAgglomeration::gatherMeshes
(
const labelList& procIDs,
const label allMeshComm
) const
{
const lduMesh& coarsestMesh = meshLevels_.last();
label coarseComm = coarsestMesh.comm();
label oldWarn = UPstream::warnComm;
UPstream::warnComm = coarseComm;
Pout<< "GAMGAgglomeration : gathering coarsestmesh (level="
<< meshLevels_.size()-1
<< ") using communicator " << coarseComm << endl;
lduPrimitiveMesh::gather(coarsestMesh, procIDs, otherMeshes_);
Pout<< "** Own Mesh " << coarsestMesh.info() << endl;
forAll(otherMeshes_, i)
{
Pout<< "** otherMesh " << i << " "
<< otherMeshes_[i].info()
<< endl;
}
Pout<< endl;
if (Pstream::myProcNo(coarseComm) == procIDs[0])
{
// Agglomerate all addressing
// ~~~~~~~~~~~~~~~~~~~~~~~~~~
allMeshPtr_.reset
(
new lduPrimitiveMesh
(
allMeshComm,
procIDs,
coarsestMesh,
otherMeshes_,
cellOffsets_,
faceMap_,
boundaryMap_,
boundaryFaceMap_
)
);
}
UPstream::warnComm = oldWarn;
}
const Foam::lduPrimitiveMesh& Foam::GAMGAgglomeration::allMesh() const
{
if (!allMeshPtr_.valid())
{
FatalErrorIn("GAMGAgglomeration::allMesh() const")
<< "Processor-agglomerated mesh not constructed. Did you call"
<< " gatherMeshes?"
<< exit(FatalError);
}
return allMeshPtr_();
}
const Foam::labelList& Foam::GAMGAgglomeration::cellOffsets() const
{
return cellOffsets_;
}
const Foam::labelListList& Foam::GAMGAgglomeration::faceMap() const
{
return faceMap_;
}
const Foam::labelListList& Foam::GAMGAgglomeration::boundaryMap() const
{
return boundaryMap_;
}
const Foam::labelListListList& Foam::GAMGAgglomeration::boundaryFaceMap() const
{
return boundaryFaceMap_;
}
const Foam::PtrList<Foam::lduMesh>& Foam::GAMGAgglomeration::otherMeshes() const
{
return otherMeshes_;
}
// ************************************************************************* //
......@@ -99,6 +99,30 @@ protected:
// Warning: Needs to be deleted explicitly.
PtrList<lduInterfacePtrsList> interfaceLevels_;
//- Processor agglomeration
//- Combined coarsest meshes for all processors
mutable autoPtr<lduPrimitiveMesh> allMeshPtr_;
//- Per remote processor the local equivalent. TBD: hack. Can be
// deleted once lduMatrices get received and merged into single
// one without constructing local ones
mutable PtrList<lduMesh> otherMeshes_;
//- Mapping from processor to coarsestAllMesh cells
mutable labelList cellOffsets_;
//- Mapping from processor to coarsestAllMesh face
mutable labelListList faceMap_;
//- Mapping from processor to coarsestAllMesh boundary
mutable labelListList boundaryMap_;
//- Mapping from processor to coarsestAllMesh boundary face
mutable labelListListList boundaryFaceMap_;
//- Assemble coarse mesh addressing
void agglomerateLduAddressing(const label fineLevelIndex);
......@@ -250,6 +274,38 @@ public:
const Field<Type>& cf,
const label coarseLevelIndex
) const;
// Procesor agglomeration
//- Collect and combine processor meshes into allMesh. Pass in
// communicator for combined mesh.
void gatherMeshes
(
const labelList& procIDs,
const label allMeshComm
) const;
//- Combined mesh
const lduPrimitiveMesh& allMesh() const;
//- Mapping from processor to coarsestAllMesh cells
const labelList& cellOffsets() const;
//- Mapping from processor to coarsestAllMesh face
const labelListList& faceMap() const;
//- Mapping from processor to coarsestAllMesh boundary
const labelListList& boundaryMap() const;
//- Mapping from processor to coarsestAllMesh boundary face
const labelListListList& boundaryFaceMap() const;
//- Per remote processor the local equivalent. TBD: hack. Can be
// deleted once lduMatrices get received and merged into single
// one without constructing local ones
const PtrList<lduMesh>& otherMeshes() const;
};
......
......@@ -131,20 +131,6 @@ class GAMGSolver
//- LU decompsed coarsest matrix
autoPtr<LUscalarMatrix> coarsestLUMatrixPtr_;
// Processor addressing agglomeration - to be moved into
// GAMGAgglomeration
//- Combined coarsest meshes for all processors
autoPtr<lduPrimitiveMesh> allMeshPtr_;
//- Mapping from processor to coarsestAllMesh
labelList cellOffsets_;
labelListList faceMap_;
labelListList boundaryMap_;
labelListListList boundaryFaceMap_;
// Processor matrix agglomeration
//- Combined coarsest level matrix for all processors
......@@ -187,7 +173,8 @@ class GAMGSolver
void gatherMatrices
(
const labelList& procIDs,
const PtrList<lduMesh>& procMeshes,
const lduMesh& myMesh,
const PtrList<lduMesh>& otherMeshes,
const label meshComm,
const lduMatrix& mat,
......@@ -200,14 +187,22 @@ class GAMGSolver
PtrList<FieldField<Field, scalar> >& otherIntCoeffs,
PtrList<lduInterfaceFieldPtrsList>& otherInterfaces
) const;
//- Gather field in allMatrix form
void gatherField
void gatherMatrices
(
const label meshComm,
const labelList& procIDs,
const scalarField& coarsestSource,
scalarField& allSource
const lduMesh& dummyMesh,
const label meshComm,
const lduMatrix& mat,
const FieldField<Field, scalar>& interfaceBouCoeffs,
const FieldField<Field, scalar>& interfaceIntCoeffs,
const lduInterfaceFieldPtrsList& interfaces,
PtrList<lduMatrix>& otherMats,
PtrList<FieldField<Field, scalar> >& otherBouCoeffs,
PtrList<FieldField<Field, scalar> >& otherIntCoeffs,
List<boolList>& otherTransforms,
List<List<int> >& otherRanks
) const;
//- Interpolate the correction after injected prolongation
......
......@@ -27,71 +27,10 @@ License
#include "ICCG.H"
#include "BICCG.H"
#include "SubField.H"
#include "globalIndex.H"
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::GAMGSolver::gatherField
(
const label meshComm,
const labelList& procIDs,
const scalarField& coarsestSource,
scalarField& allSource
) const
{
if (Pstream::myProcNo(meshComm) == procIDs[0])
{
// Master.
const lduMesh& allMesh = allMeshPtr_();
allSource.setSize(allMesh.lduAddr().size());
SubList<scalar>
(
allSource,
coarsestSource.size(),
0
).assign(coarsestSource);
for (label i = 1; i < procIDs.size(); i++)
{
IPstream fromSlave
(
Pstream::scheduled,
procIDs[i],
0, // bufSize
Pstream::msgType(),
meshComm
);
SubList<scalar> slaveSlots
(
allSource,
cellOffsets_[i+1]-cellOffsets_[i],
cellOffsets_[i]
);
UList<scalar>& l = slaveSlots;
fromSlave >> l;
}
}
else
{
// Send to master
OPstream toMaster
(
Pstream::scheduled,
procIDs[0],
0,
Pstream::msgType(),
meshComm
);
toMaster << coarsestSource;
}
}
Foam::solverPerformance Foam::GAMGSolver::solve
(
scalarField& psi,
......@@ -519,7 +458,9 @@ void Foam::GAMGSolver::solveCoarsestLevel
const List<int>& procIDs = UPstream::procID(coarseComm);
scalarField allSource;
gatherField
const globalIndex cellOffsets(agglomeration_.cellOffsets());
cellOffsets.gather
(
coarseComm,
procIDs,
......@@ -578,55 +519,25 @@ void Foam::GAMGSolver::solveCoarsestLevel
}
UPstream::warnComm = oldWarn;
}
// Distribute correction
coarsestCorrField = SubList<scalar>
(
allCorrField,
coarsestSource.size(),
0
);
for (label i=1; i < procIDs.size(); i++)
{
Pout<< "** master sending to " << procIDs[i] << endl;
OPstream toSlave
(
Pstream::scheduled,
procIDs[i],
0, // bufSize
Pstream::msgType(),
coarseComm
);
toSlave << SubList<scalar>
(
allCorrField,
cellOffsets_[i+1]-cellOffsets_[i],
cellOffsets_[i]
);
}
}
else
{
Pout<< "** slave receiving from " << procIDs[0] << endl;
IPstream fromMaster
(
Pstream::scheduled,
procIDs[0],
0, // bufSize
Pstream::msgType(),
coarseComm
);
fromMaster >> coarsestCorrField;
}
// Scatter to all processors
coarsestCorrField.setSize(coarsestSource.size());
cellOffsets.scatter
(
coarseComm,
procIDs,
allCorrField,
coarsestCorrField
);
if (debug >= 2)
{
coarseSolverPerf.print(Info(coarseComm));
}
Pout<< "procAgglom: coarsestSource :" << coarsestSource << endl;
Pout<< "procAgglom: coarsestCorrField:" << coarsestCorrField << endl;
}
else
{
......@@ -672,6 +583,9 @@ void Foam::GAMGSolver::solveCoarsestLevel
{
coarseSolverPerf.print(Info(coarseComm));
}
Pout<< "GC: coarsestSource :" << coarsestSource << endl;
Pout<< "GC: coarsestCorrField:" << coarsestCorrField << endl;
}
UPstream::warnComm = oldWarn;
......
......@@ -91,6 +91,36 @@ Foam::Ostream& Foam::operator<<(Ostream& os, const InfoProxy<lduMesh>& ip)
}
}
// Print actual contents
{
const labelList& l = addr.lowerAddr();
const labelList& u = addr.upperAddr();
forAll(l, faceI)
{
Pout<< " face:" << faceI << " l:" << l[faceI]
<< " u:" << u[faceI] << endl;
}
forAll(interfaces, i)
{
if (interfaces.set(i))
{
const labelUList& faceCells = addr.patchAddr(i);
if (faceCells.size())
{
Pout<< " patch:" << i
<< " type:" << interfaces[i].type() << endl;
forAll(faceCells, i)
{
Pout<< " " << i << " own:" << faceCells[i]
<< endl;
}
}
}
}
}
os.check("Ostream& operator<<(Ostream&, const lduMesh&");
return os;
......
......@@ -217,22 +217,12 @@ Foam::lduPrimitiveMesh::lduPrimitiveMesh
}
//Foam::lduPrimitiveMesh::lduPrimitiveMesh(Istream& is)
//:
// lduAddressing(readLabel(is)),
// lowerAddr_(is),
// upperAddr_(is),
// patchAddr_(is),
// interfaces_(is),
// patchSchedule_(is),
// comm_(readLabel(is))
//{}
Foam::lduPrimitiveMesh::lduPrimitiveMesh
(
const label comm,
const labelList& procIDs,
const PtrList<lduMesh>& procMeshes,
const lduMesh& myMesh,
const PtrList<lduMesh>& otherMeshes,
labelList& cellOffsets,
labelListList& faceMap,
......@@ -240,7 +230,7 @@ Foam::lduPrimitiveMesh::lduPrimitiveMesh
labelListListList& boundaryFaceMap
)
:
lduAddressing(size(procMeshes)),
lduAddressing(myMesh.lduAddr().size() + size(otherMeshes)),
lowerAddr_(0),
upperAddr_(0),
patchAddr_(0),
......@@ -261,40 +251,49 @@ Foam::lduPrimitiveMesh::lduPrimitiveMesh
<< procIDs[i-1]
<< exit(FatalError);
}
if (procMeshes[i].comm() != procMeshes[i-1].comm())
}
const label currentComm = myMesh.comm();
forAll(otherMeshes, i)
{
if (otherMeshes[i].comm() != currentComm)
{
FatalErrorIn
(
"lduPrimitiveMesh::lduPrimitiveMesh(..)"
) << "Communicator " << procMeshes[i].comm() << " at index " << i
) << "Communicator " << otherMeshes[i].comm()
<< " at index " << i
<< " differs from that of predecessor "
<< procMeshes[i-1].comm()
<< currentComm
<< exit(FatalError);
}
}
const label currentComm = procMeshes[0].comm();
const label nMeshes = otherMeshes.size()+1;
// Cells get added in order.
cellOffsets.setSize(procMeshes.size()+1);
cellOffsets.setSize(nMeshes+1);
cellOffsets[0] = 0;
forAll(procMeshes, procMeshI)
for (label procMeshI = 0; procMeshI < nMeshes; procMeshI++)
{
const lduMesh& mesh = procMeshes[procMeshI];
const lduMesh& procMesh = mesh(myMesh, otherMeshes, procMeshI);
cellOffsets[procMeshI+1] =
cellOffsets[procMeshI]
+ mesh.lduAddr().size();
+ procMesh.lduAddr().size();
}
// Faces initially get added in order, sorted later
labelList internalFaceOffsets(procMeshes.size()+1);
labelList internalFaceOffsets(nMeshes+1);
internalFaceOffsets[0] = 0;
forAll(procMeshes, procMeshI)
for (label procMeshI = 0; procMeshI < nMeshes; procMeshI++)
{
const lduMesh& mesh = procMeshes[procMeshI];
const lduMesh& procMesh = mesh(myMesh, otherMeshes, procMeshI);
internalFaceOffsets[procMeshI+1] =
internalFaceOffsets[procMeshI]
+ mesh.lduAddr().lowerAddr().size();
+ procMesh.lduAddr().lowerAddr().size();
}
// Count how faces get added. Interfaces inbetween get merged.
......@@ -304,7 +303,7 @@ Foam::lduPrimitiveMesh::lduPrimitiveMesh
// - interface in procMesh
EdgeMap<labelPairList> mergedMap
(
2*procMeshes[0].interfaces().size()
2*myMesh.interfaces().size()
);
// Unmerged interfaces: map from two processors back to
......@@ -312,18 +311,18 @@ Foam::lduPrimitiveMesh::lduPrimitiveMesh
// - interface in procMesh
EdgeMap<labelPairList> unmergedMap(mergedMap.size());
boundaryMap.setSize(procMeshes.size());
boundaryFaceMap.setSize(procMeshes.size());
boundaryMap.setSize(nMeshes);
boundaryFaceMap.setSize(nMeshes);
label nBoundaryFaces = 0;
label nInterfaces = 0;
labelList nCoupledFaces(procMeshes.size(), 0);
labelList nCoupledFaces(nMeshes, 0);
forAll(procMeshes, procMeshI)
for (label procMeshI = 0; procMeshI < nMeshes; procMeshI++)
{
const lduInterfacePtrsList interfaces =
procMeshes[procMeshI].interfaces();
mesh(myMesh, otherMeshes, procMeshI).interfaces();
// Inialise all boundaries as merged
boundaryMap[procMeshI].setSize(interfaces.size(), -1);
......@@ -449,7 +448,7 @@ Foam::lduPrimitiveMesh::lduPrimitiveMesh
label procMeshI = elems[i][0];
label interfaceI = elems[i][1];
const lduInterfacePtrsList interfaces =
procMeshes[procMeshI].interfaces();
mesh(myMesh, otherMeshes, procMeshI).interfaces();
const processorLduInterface& pldui =
refCast<const processorLduInterface>
(
......@@ -476,7 +475,7 @@ Foam::lduPrimitiveMesh::lduPrimitiveMesh
label procMeshI = elems[i][0];
label interfaceI = elems[i][1];
const lduInterfacePtrsList interfaces =
procMeshes[procMeshI].interfaces();
mesh(myMesh, otherMeshes, procMeshI).interfaces();
const processorLduInterface& pldui =
refCast<const processorLduInterface>
(
......@@ -495,12 +494,13 @@ Foam::lduPrimitiveMesh::lduPrimitiveMesh
// Adapt faceOffsets for internal interfaces
labelList faceOffsets(procMeshes.size()+1);
labelList faceOffsets(nMeshes+1);
faceOffsets[0] = 0;
faceMap.setSize(procMeshes.size());
forAll(procMeshes, procMeshI)
faceMap.setSize(nMeshes);
for (label procMeshI = 0; procMeshI < nMeshes; procMeshI++)
{
label nInternal = procMeshes[procMeshI].lduAddr().lowerAddr().size();
const lduMesh& procMesh = mesh(myMesh, otherMeshes, procMeshI);
label nInternal = procMesh.lduAddr().lowerAddr().size();
faceOffsets[procMeshI+1] =