Commit 436902f5 authored by mattijs's avatar mattijs
Browse files

ENH: parallel: overhaul of parallel mapping

- redistributePar to have almost (complete) functionality of decomposePar+reconstructPar
- low-level distributed Field mapping
- support for mapping surfaceFields (including flipping faces)
- support for decomposing/reconstructing refinement data
parent 1c066c14
......@@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
......@@ -202,6 +202,7 @@ void subsetSurfaceFields
(
const fvMesh& mesh,
const fvMesh& subMesh,
const labelList& cellMap,
const labelList& faceMap,
const labelHashSet& addedPatches
)
......@@ -225,6 +226,7 @@ void subsetSurfaceFields
fld,
subMesh,
patchMap,
cellMap,
faceMap
)
);
......@@ -830,6 +832,7 @@ void createAndWriteRegion
(
mesh,
newMesh(),
map().cellMap(),
map().faceMap(),
addedPatches
);
......@@ -837,6 +840,7 @@ void createAndWriteRegion
(
mesh,
newMesh(),
map().cellMap(),
map().faceMap(),
addedPatches
);
......@@ -844,6 +848,7 @@ void createAndWriteRegion
(
mesh,
newMesh(),
map().cellMap(),
map().faceMap(),
addedPatches
);
......@@ -851,6 +856,7 @@ void createAndWriteRegion
(
mesh,
newMesh(),
map().cellMap(),
map().faceMap(),
addedPatches
);
......@@ -858,6 +864,7 @@ void createAndWriteRegion
(
mesh,
newMesh(),
map().cellMap(),
map().faceMap(),
addedPatches
);
......
......@@ -2,6 +2,7 @@ EXE_INC = \
-I$(LIB_SRC)/parallel/decompose/decompose/lnInclude \
-I$(LIB_SRC)/parallel/decompose/decompositionMethods/lnInclude \
-I$(LIB_SRC)/finiteVolume/lnInclude \
-I$(LIB_SRC)/dynamicMesh/lnInclude \
-I$(LIB_SRC)/lagrangian/basic/lnInclude \
-I$(LIB_SRC)/meshTools/lnInclude \
-I$(LIB_SRC)/regionModels/regionModel/lnInclude
......@@ -12,5 +13,6 @@ EXE_LIBS = \
-lgenericPatchFields \
-ldecompositionMethods -L$(FOAM_LIBBIN)/dummy -lmetisDecomp -lscotchDecomp \
-llagrangian \
-ldynamicMesh \
-lmeshTools \
-lregionModels
......@@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
......@@ -96,6 +96,7 @@ Usage
#include "fvFieldDecomposer.H"
#include "pointFieldDecomposer.H"
#include "lagrangianFieldDecomposer.H"
#include "decompositionModel.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
......@@ -142,6 +143,12 @@ int main(int argc, char *argv[])
);
argList::noParallel();
Foam::argList::addOption
(
"decomposeParDict",
"file",
"read decomposePar dictionary from specified location"
);
#include "addRegionOption.H"
argList::addBoolOption
(
......@@ -199,6 +206,17 @@ int main(int argc, char *argv[])
instantList times = timeSelector::selectIfPresent(runTime, args);
// Allow override of decomposeParDict location
fileName decompDictFile;
if (args.optionReadIfPresent("decomposeParDict", decompDictFile))
{
if (isDir(decompDictFile))
{
decompDictFile = decompDictFile/"decomposeParDict";
}
}
wordList regionNames;
wordList regionDirs;
if (allRegions)
......@@ -260,21 +278,27 @@ int main(int argc, char *argv[])
++nProcs;
}
// get requested numberOfSubdomains
// get requested numberOfSubdomains. Note: have no mesh yet so
// cannot use decompositionModel::New
const label nDomains = readLabel
(
IOdictionary
(
IOobject
decompositionModel::selectIO
(
"decomposeParDict",
runTime.time().system(),
regionDir, // use region if non-standard
runTime,
IOobject::MUST_READ_IF_MODIFIED,
IOobject::NO_WRITE,
false
IOobject
(
"decomposeParDict",
runTime.time().system(),
regionDir, // use region if non-standard
runTime,
IOobject::MUST_READ_IF_MODIFIED,
IOobject::NO_WRITE,
false
),
decompDictFile
)
).lookup("numberOfSubdomains")
);
......@@ -288,8 +312,7 @@ int main(int argc, char *argv[])
<< nProcs << " domains"
<< nl
<< "instead of " << nDomains
<< " domains as specified in decomposeParDict"
<< nl
<< " domains as specified in decomposeParDict" << nl
<< exit(FatalError);
}
}
......@@ -351,7 +374,8 @@ int main(int argc, char *argv[])
IOobject::NO_READ,
IOobject::NO_WRITE,
false
)
),
decompDictFile
);
// Decompose the mesh
......@@ -447,15 +471,15 @@ int main(int argc, char *argv[])
// Construct the vol fields
// ~~~~~~~~~~~~~~~~~~~~~~~~
PtrList<volScalarField> volScalarFields;
readFields(mesh, objects, volScalarFields);
readFields(mesh, objects, volScalarFields, false);
PtrList<volVectorField> volVectorFields;
readFields(mesh, objects, volVectorFields);
readFields(mesh, objects, volVectorFields, false);
PtrList<volSphericalTensorField> volSphericalTensorFields;
readFields(mesh, objects, volSphericalTensorFields);
readFields(mesh, objects, volSphericalTensorFields, false);
PtrList<volSymmTensorField> volSymmTensorFields;
readFields(mesh, objects, volSymmTensorFields);
readFields(mesh, objects, volSymmTensorFields, false);
PtrList<volTensorField> volTensorFields;
readFields(mesh, objects, volTensorFields);
readFields(mesh, objects, volTensorFields, false);
// Construct the dimensioned fields
......@@ -476,15 +500,15 @@ int main(int argc, char *argv[])
// Construct the surface fields
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
PtrList<surfaceScalarField> surfaceScalarFields;
readFields(mesh, objects, surfaceScalarFields);
readFields(mesh, objects, surfaceScalarFields, false);
PtrList<surfaceVectorField> surfaceVectorFields;
readFields(mesh, objects, surfaceVectorFields);
readFields(mesh, objects, surfaceVectorFields, false);
PtrList<surfaceSphericalTensorField> surfaceSphericalTensorFields;
readFields(mesh, objects, surfaceSphericalTensorFields);
readFields(mesh, objects, surfaceSphericalTensorFields, false);
PtrList<surfaceSymmTensorField> surfaceSymmTensorFields;
readFields(mesh, objects, surfaceSymmTensorFields);
readFields(mesh, objects, surfaceSymmTensorFields, false);
PtrList<surfaceTensorField> surfaceTensorFields;
readFields(mesh, objects, surfaceTensorFields);
readFields(mesh, objects, surfaceTensorFields, false);
// Construct the point fields
......@@ -492,15 +516,15 @@ int main(int argc, char *argv[])
const pointMesh& pMesh = pointMesh::New(mesh);
PtrList<pointScalarField> pointScalarFields;
readFields(pMesh, objects, pointScalarFields);
readFields(pMesh, objects, pointScalarFields, false);
PtrList<pointVectorField> pointVectorFields;
readFields(pMesh, objects, pointVectorFields);
readFields(pMesh, objects, pointVectorFields, false);
PtrList<pointSphericalTensorField> pointSphericalTensorFields;
readFields(pMesh, objects, pointSphericalTensorFields);
readFields(pMesh, objects, pointSphericalTensorFields, false);
PtrList<pointSymmTensorField> pointSymmTensorFields;
readFields(pMesh, objects, pointSymmTensorFields);
readFields(pMesh, objects, pointSymmTensorFields, false);
PtrList<pointTensorField> pointTensorFields;
readFields(pMesh, objects, pointTensorFields);
readFields(pMesh, objects, pointTensorFields, false);
// Construct the Lagrangian fields
......@@ -820,16 +844,6 @@ int main(int argc, char *argv[])
processorDb.setTime(runTime);
// remove files remnants that can cause horrible problems
// - mut and nut are used to mark the new turbulence models,
// their existence prevents old models from being upgraded
{
fileName timeDir(processorDb.path()/processorDb.timeName());
rm(timeDir/"mut");
rm(timeDir/"nut");
}
// read the mesh
if (!procMeshList.set(procI))
{
......
......@@ -17,11 +17,61 @@ FoamFile
numberOfSubdomains 2;
// Optional decomposition constraints
//constraints
//{
// preserveBaffles
// {
// //- Keep owner and neighbour of baffles on same processor (i.e.
// // keep it detectable as a baffle). Baffles are two boundary face
// // sharing the same points
// type preserveBaffles;
// }
// preserveFaceZones
// {
// //- Keep owner and neighbour on same processor for faces in zones
// type preserveFaceZones;
// zones (".*");
// }
// preservePatches
// {
// //- Keep owner and neighbour on same processor for faces in patches
// // (only makes sense for cyclic patches. Not suitable for e.g.
// // cyclicAMI since these are not coupled on the patch level. Use
// // singleProcessorFaceSets for those)
// type preservePatches;
// patches (".*");
// }
// singleProcessorFaceSets
// {
// //- Keep all of faceSet on a single processor. This puts all cells
// // connected with a point, edge or face on the same processor.
// // (just having face connected cells might not guarantee a balanced
// // decomposition)
// // The processor can be -1 (the decompositionMethod chooses the
// // processor for a good load balance) or explicitly provided (upsets
// // balance)
// type singleProcessorFaceSets;
// singleProcessorFaceSets ((f1 -1));
// }
// refinementHistory
// {
// //- Decompose cells such that all cell originating from single cell
// // end up on same processor
// type refinementHistory;
// }
//}
// Deprecated form of specifying decomposition constraints:
//- Keep owner and neighbour on same processor for faces in zones:
// preserveFaceZones (heater solid1 solid3);
//- Keep owner and neighbour on same processor for faces in patches:
// (makes sense only for cyclic patches)
// (makes sense only for cyclic patches. Not suitable for e.g. cyclicAMI
// since these are not coupled on the patch level. Use
// singleProcessorFaceSets for those)
//preservePatches (cyclic_half0 cyclic_half1);
//- Keep all of faceSet on a single processor. This puts all cells
......@@ -32,12 +82,13 @@ numberOfSubdomains 2;
// for a good load balance) or explicitly provided (upsets balance).
//singleProcessorFaceSets ((f0 -1));
//- Keep owner and neighbour of baffles on same processor (i.e. keep it
// detectable as a baffle). Baffles are two boundary face sharing the
// same points.
//preserveBaffles true;
//- Use the volScalarField named here as a weight for each cell in the
// decomposition. For example, use a particle population field to decompose
// for a balanced number of particles in a lagrangian simulation.
......
......@@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation
\\/ M anipulation |
\\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
......@@ -31,14 +31,14 @@ License
#include "fvMesh.H"
#include "OSspecific.H"
#include "Map.H"
#include "globalMeshData.H"
#include "DynamicList.H"
#include "fvFieldDecomposer.H"
#include "IOobjectList.H"
#include "cellSet.H"
#include "faceSet.H"
#include "pointSet.H"
#include "uniformDimensionedFields.H"
#include "decompositionModel.H"
#include "hexRef8Data.H"
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
......@@ -69,7 +69,12 @@ void Foam::domainDecomposition::mark
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::domainDecomposition::domainDecomposition(const IOobject& io)
// from components
Foam::domainDecomposition::domainDecomposition
(
const IOobject& io,
const fileName& decompDictFile
)
:
fvMesh(io),
facesInstancePointsPtr_
......@@ -90,18 +95,18 @@ Foam::domainDecomposition::domainDecomposition(const IOobject& io)
)
: NULL
),
decompositionDict_
decompDictFile_(decompDictFile),
nProcs_
(
IOobject
readInt
(
"decomposeParDict",
time().system(),
*this,
IOobject::MUST_READ_IF_MODIFIED,
IOobject::NO_WRITE
decompositionModel::New
(
*this,
decompDictFile
).lookup("numberOfSubdomains")
)
),
nProcs_(readInt(decompositionDict_.lookup("numberOfSubdomains"))),
distributed_(false),
cellToProc_(nCells()),
procPointAddressing_(nProcs_),
......@@ -115,7 +120,11 @@ Foam::domainDecomposition::domainDecomposition(const IOobject& io)
procProcessorPatchSubPatchIDs_(nProcs_),
procProcessorPatchSubPatchStarts_(nProcs_)
{
decompositionDict_.readIfPresent("distributed", distributed_);
decompositionModel::New
(
*this,
decompDictFile
).readIfPresent("distributed", distributed_);
}
......@@ -195,57 +204,20 @@ bool Foam::domainDecomposition::writeDecomposition(const bool decomposeSets)
}
autoPtr<labelIOList> cellLevelPtr;
{
IOobject io
(
"cellLevel",
facesInstance(),
polyMesh::meshSubDir,
*this,
IOobject::MUST_READ,
IOobject::NO_WRITE
);
if (io.headerOk())
{
Info<< "Reading hexRef8 data : " << io.name() << endl;
cellLevelPtr.reset(new labelIOList(io));
}
}
autoPtr<labelIOList> pointLevelPtr;
{
IOobject io
(
"pointLevel",
facesInstance(),
polyMesh::meshSubDir,
*this,
IOobject::MUST_READ,
IOobject::NO_WRITE
);
if (io.headerOk())
{
Info<< "Reading hexRef8 data : " << io.name() << endl;
pointLevelPtr.reset(new labelIOList(io));
}
}
autoPtr<uniformDimensionedScalarField> level0EdgePtr;
{
IOobject io
// Load refinement data (if any)
hexRef8Data baseMeshData
(
IOobject
(
"level0Edge",
"dummy",
facesInstance(),
polyMesh::meshSubDir,
*this,
IOobject::MUST_READ,
IOobject::NO_WRITE
);
if (io.headerOk())
{
Info<< "Reading hexRef8 data : " << io.name() << endl;
level0EdgePtr.reset(new uniformDimensionedScalarField(io));
}
}
IOobject::READ_IF_PRESENT,
IOobject::NO_WRITE,
false
)
);
......@@ -816,8 +788,8 @@ bool Foam::domainDecomposition::writeDecomposition(const bool decomposeSets)
}
}
// Set the precision of the points data to 10
IOstream::defaultPrecision(10);
// Set the precision of the points data to be min 10
IOstream::defaultPrecision(max(10u, IOstream::defaultPrecision()));
procMesh.write();
......@@ -887,64 +859,23 @@ bool Foam::domainDecomposition::writeDecomposition(const bool decomposeSets)
}
// hexRef8 data
if (cellLevelPtr.valid())
{
labelIOList
(
IOobject
(
cellLevelPtr().name(),
facesInstance(),
polyMesh::meshSubDir,
procMesh,
IOobject::NO_READ,
IOobject::AUTO_WRITE
),
UIndirectList<label>
(
cellLevelPtr(),
procCellAddressing_[procI]
)()
).write();
}
if (pointLevelPtr.valid())
{
labelIOList
(
IOobject
(
pointLevelPtr().name(),
facesInstance(),
polyMesh::meshSubDir,
procMesh,
IOobject::NO_READ,
IOobject::AUTO_WRITE
),
UIndirectList<label>
(
pointLevelPtr(),
procPointAddressing_[procI]
)()
).write();
}
if (level0EdgePtr.valid())
{
uniformDimensionedScalarField
// Optional hexRef8 data
hexRef8Data
(
IOobject
(
IOobject
(
level0EdgePtr().name(),
facesInstance(),
polyMesh::meshSubDir,
procMesh,
IOobject::NO_READ,
IOobject::AUTO_WRITE
),
level0EdgePtr()
).write();
}
"dummy",
facesInstance(),
polyMesh::meshSubDir,
procMesh,
IOobject::NO_READ,
IOobject::NO_WRITE,
false
),
baseMeshData,
procCellAddressing_[procI],
procPointAddressing_[procI]
).write();
// Statistics
......
......@@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox