Commit b29f2a61 authored by Mark Olesen's avatar Mark Olesen
Browse files

BUG: missing parallel-aware for metis-like decomposition

parent 9b2a2551
......@@ -8,5 +8,6 @@ EXE_INC = \
EXE_LIBS = \
-ldecompose \
-ldecompositionMethods \
-L$(FOAM_LIBBIN)/dummy -lmetisDecomp -lscotchDecomp -lkahipDecomp \
-L$(FOAM_LIBBIN)/dummy \
-lkahipDecomp -lmetisDecomp -lscotchDecomp \
-lregionModels
......@@ -28,5 +28,6 @@ EXE_LIBS = \
-lmeshTools \
-lsampling \
-ldecompositionMethods \
-L$(FOAM_LIBBIN)/dummy -lscotchDecomp -lptscotchDecomp \
-L$(FOAM_LIBBIN)/dummy \
-lkahipDecomp -lmetisDecomp -lptscotchDecomp -lscotchDecomp \
-ldynamicMesh
......@@ -31,5 +31,6 @@ EXE_LIBS = \
-lsampling \
-ldecompositionMethods \
-ldecompose \
-L$(FOAM_LIBBIN)/dummy -lptscotchDecomp -lscotchDecomp \
-L$(FOAM_LIBBIN)/dummy \
-lkahipDecomp -lmetisDecomp -lptscotchDecomp -lscotchDecomp \
-ldynamicMesh
......@@ -18,6 +18,8 @@ EXE_LIBS = \
-L$(FASTDUALOCTREE_SRC_PATH) -lperf_main \
-lGL \
-lconformalVoronoiMesh \
-ldecompositionMethods -L$(FOAM_LIBBIN)/dummy -lscotchDecomp \
-ldecompositionMethods \
-L$(FOAM_LIBBIN)/dummy \
-lkahipDecomp -lmetisDecomp -lscotchDecomp \
-lmeshTools \
-ldynamicMesh
......@@ -31,4 +31,5 @@ EXE_LIBS = \
-lsampling \
-ldynamicMesh \
-ldecompositionMethods \
-L$(FOAM_LIBBIN)/dummy -lscotchDecomp -lptscotchDecomp
-L$(FOAM_LIBBIN)/dummy \
-lkahipDecomp -lmetisDecomp -lptscotchDecomp -lscotchDecomp
......@@ -13,9 +13,8 @@ EXE_INC = \
EXE_LIBS = \
-lfiniteVolume \
-ldecompositionMethods \
-L$(FOAM_LIBBIN)/dummy -lptscotchDecomp \
/* note: scotch < 6.0 does not like both scotch and ptscotch together */ \
-lscotchDecomp \
-L$(FOAM_LIBBIN)/dummy \
-lkahipDecomp -lmetisDecomp -lptscotchDecomp -lscotchDecomp \
-lmeshTools \
-ldynamicMesh \
-ldecompose \
......
......@@ -18,4 +18,5 @@ EXE_LIBS = \
-lreconstruct \
$(LINK_FLAGS) \
-ldecompositionMethods \
-L$(FOAM_LIBBIN)/dummy -lmetisDecomp -lscotchDecomp -lkahipDecomp
-L$(FOAM_LIBBIN)/dummy \
-lkahipDecomp -lmetisDecomp -lscotchDecomp
......@@ -12,7 +12,8 @@ EXE_LIBS = \
-ldecompose \
-lgenericPatchFields \
-ldecompositionMethods \
-L$(FOAM_LIBBIN)/dummy -lmetisDecomp -lscotchDecomp -lkahipDecomp \
-L$(FOAM_LIBBIN)/dummy \
-lkahipDecomp -lmetisDecomp -lscotchDecomp \
-llagrangian \
-ldynamicMesh \
-lregionModels
......@@ -11,7 +11,8 @@ EXE_LIBS = \
-lfiniteVolume \
-lgenericPatchFields \
-ldecompositionMethods \
-L$(FOAM_LIBBIN)/dummy -lptscotchDecomp -lscotchDecomp \
-L$(FOAM_LIBBIN)/dummy \
-lkahipDecomp -lmetisDecomp -lptscotchDecomp -lscotchDecomp \
-ldecompose \
-lmeshTools \
-llagrangian \
......
......@@ -181,7 +181,7 @@ Foam::labelList Foam::metisLikeDecomp::decompose
mesh,
identity(mesh.nCells()),
mesh.nCells(),
false,
true,
cellCells
);
......@@ -214,7 +214,14 @@ Foam::labelList Foam::metisLikeDecomp::decompose
// xadj(celli) : start of information in adjncy for celli
CompactListList<label> cellCells;
calcCellCells(mesh, agglom, agglomPoints.size(), false, cellCells);
calcCellCells
(
mesh,
agglom,
agglomPoints.size(),
true,
cellCells
);
// Decompose using default weights
labelList decomp;
......
......@@ -92,7 +92,9 @@ Foam::label Foam::kahipDecomp::decomposeSerial
List<int> cellWeights;
// Check for externally provided cellweights and if so initialise weights
const scalar minWeights = gMin(cWeights);
// Note: min, not gMin since routine runs on master only.
const scalar minWeights = min(cWeights);
if (!cWeights.empty())
{
if (minWeights <= 0)
......
......@@ -77,10 +77,11 @@ Foam::label Foam::metisDecomp::decomposeSerial
// Face weights (so on the edges of the dual)
List<label> faceWeights;
// Check for externally provided cellweights and if so initialise weights
const scalar minWeights = gMin(cWeights);
if (cWeights.size() > 0)
// Note: min, not gMin since routine runs on master only.
const scalar minWeights = min(cWeights);
if (!cWeights.empty())
{
if (minWeights <= 0)
{
......
......@@ -12,20 +12,21 @@ cd ${0%/*} || exit 1 # Run from this directory
# Agglomerate patch faces
for i in air
do
faceAgglomerate -region $i -dict constant/viewFactorsDict > log.faceAgglomerate.$i 2>&1
runApplication -s $i \
faceAgglomerate -region $i -dict constant/viewFactorsDict
done
# Generate view factors
for i in air
do
viewFactorsGen -region $i > log.viewFactorsGen.$i 2>&1
runApplication -s $i \
viewFactorsGen -region $i
done
runApplication $(getApplication)
echo
echo "creating files for paraview post-processing"
echo "Creating files for paraview post-processing"
echo
paraFoam -touchAll
......
#!/bin/sh
cd ${0%/*} || exit 1 # Run from this directory
# Source tutorial run functions
. $WM_PROJECT_DIR/bin/tools/RunFunctions
cd ${0%/*} || exit 1 # Run from this directory
. $WM_PROJECT_DIR/bin/tools/RunFunctions # Tutorial run functions
# Setup case
./Allrun.pre
......@@ -15,18 +13,18 @@ runApplication decomposePar -allRegions -constant
# Agglomerate patch faces
for i in air
do
runParallel -s $i -np 4 \
runParallel -s $i \
faceAgglomerate -region $i -dict constant/viewFactorsDict
done
# Generate view factors
for i in air
do
runParallel -s $i -np 4 \
runParallel -s $i \
viewFactorsGen -region $i
done
# set the initial fields
# Set the initial fields
restore0Dir
runParallel $(getApplication)
......@@ -35,7 +33,7 @@ runParallel $(getApplication)
runApplication reconstructPar -allRegions
echo
echo "creating files for paraview post-processing"
echo "Creating files for paraview post-processing"
echo
paraFoam -touchAll
......
#!/bin/sh
cd ${0%/*} || exit 1 # Run from this directory
# Source tutorial run functions
. $WM_PROJECT_DIR/bin/tools/RunFunctions
cd ${0%/*} || exit 1 # Run from this directory
. $WM_PROJECT_DIR/bin/tools/RunFunctions # Tutorial run functions
runApplication blockMesh
runApplication topoSet
runApplication splitMeshRegions -cellZones -overwrite
# Remove unwanted region
rm -r 0/domain3
rm -r constant/domain3
rm -r system/domain3
rm -rf 0/domain3 constant/domain3 system/domain3
# remove fluid fields from solid regions (important for post-processing)
# Remove fluid fields from solid regions (important for post-processing)
for i in solid floor
do
rm -f 0*/$i/{rho,mut,alphat,epsilon,k,U,p_rgh,qr,G,IDefault}
done
# set the initial fields
# Set the initial fields
restore0Dir
for i in air solid floor
do
runApplication -s $i changeDictionary -region $i -subDict dictionaryReplacement
runApplication -s $i changeDictionary \
-region $i -subDict dictionaryReplacement
done
# -----------------------------------------------------------------------------
......@@ -4,11 +4,9 @@ cd ${0%/*} || exit 1 # Run from this directory
# Source tutorial run functions
. $WM_PROJECT_DIR/bin/tools/RunFunctions
# Setup case
./Allrun.pre
#-- Run in parallel
# Decompose
......@@ -17,14 +15,14 @@ runApplication decomposePar -allRegions
# Agglomerate patch faces
for i in bottomAir topAir
do
runParallel -s $i -np 4 \
runParallel -s $i \
faceAgglomerate -region $i -dict constant/viewFactorsDict
done
# Generate view factors
for i in bottomAir topAir
do
runParallel -s $i -np 4 \
runParallel -s $i \
viewFactorsGen -region $i
done
......@@ -34,9 +32,8 @@ runParallel $(getApplication)
# Reconstruct
runApplication reconstructPar -allRegions
echo
echo "creating files for paraview post-processing"
echo "Creating files for paraview post-processing"
echo
paraFoam -touchAll
......
#!/bin/sh
cd ${0%/*} || exit 1 # Run from this directory
# Source tutorial clean functions
. $WM_PROJECT_DIR/bin/tools/CleanFunctions
cd ${0%/*} || exit 1 # Run from this directory
. $WM_PROJECT_DIR/bin/tools/CleanFunctions # Tutorial clean functions
cleanCase
# Restore default dictionaries
cp system/decomposeParDict-2 system/decomposeParDict
cp system/controlDict-startTime system/controlDict
# -----------------------------------------------------------------------------
#!/bin/sh
cd ${0%/*} || exit 1 # Run from this directory
# Source tutorial run functions
. $WM_PROJECT_DIR/bin/tools/RunFunctions
cd ${0%/*} || exit 1 # Run from this directory
. $WM_PROJECT_DIR/bin/tools/RunFunctions # Tutorial run functions
runApplication blockMesh
#cp system/decomposeParDict-2 system/decomposeParDict
#runApplication decomposePar
decompDict5="-decomposeParDict system/decomposeParDict-5"
# redistributePar to do decomposition
runParallel -s decompose redistributePar -decompose
......@@ -25,15 +22,12 @@ runParallel -s random icoFoam
cp system/controlDict-latestTime system/controlDict
# Redistribute to 5 processors
runParallel -s 5 -np 5 redistributePar \
-decomposeParDict system/decomposeParDict-5 -cellDist
runParallel -s 5 $decompDict5 redistributePar -cellDist
# Run a bit more
runParallel -s 5 -np 5 icoFoam \
-decomposeParDict system/decomposeParDict-5
runParallel -s 5 $decompDict5 icoFoam
# Reconstruct mesh and results
runParallel -s reconstruct -np 5 redistributePar -reconstruct
runParallel -s reconstruct -np 5 redistributePar -reconstruct
# -----------------------------------------------------------------------------
......@@ -43,13 +43,14 @@ numberOfSubdomains 2;
// for a balanced number of particles in a lagrangian simulation.
// weightField dsmcRhoNMean;
method scotch;
//method hierarchical;
// method simple;
// method metis;
// method manual;
// method multiLevel;
// method structured; // does 2D decomposition of structured mesh
method scotch;
// method kahip;
// method metis;
// method hierarchical;
// method simple;
// method manual;
// method multiLevel;
// method structured; // does 2D decomposition of structured mesh
multiLevelCoeffs
{
......
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: plus |
| \\ / A nd | Web: www.OpenFOAM.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
note "mesh decomposition control dictionary";
object decomposeParDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
numberOfSubdomains 2;
//- Keep owner and neighbour on same processor for faces in zones:
// preserveFaceZones (heater solid1 solid3);
//- Keep owner and neighbour on same processor for faces in patches:
// (makes sense only for cyclic patches)
//preservePatches (cyclic_half0 cyclic_half1);
//- Keep all of faceSet on a single processor. This puts all cells
// connected with a point, edge or face on the same processor.
// (just having face connected cells might not guarantee a balanced
// decomposition)
// The processor can be -1 (the decompositionMethod chooses the processor
// for a good load balance) or explicitly provided (upsets balance).
//singleProcessorFaceSets ((f0 -1));
//- Keep owner and neighbour of baffles on same processor (i.e. keep it
// detectable as a baffle). Baffles are two boundary face sharing the
// same points.
//preserveBaffles true;
//- Use the volScalarField named here as a weight for each cell in the
// decomposition. For example, use a particle population field to decompose
// for a balanced number of particles in a lagrangian simulation.
// weightField dsmcRhoNMean;
method scotch;
//method hierarchical;
// method simple;
// method metis;
// method manual;
// method multiLevel;
// method structured; // does 2D decomposition of structured mesh
multiLevelCoeffs
{
// Decomposition methods to apply in turn. This is like hierarchical but
// fully general - every method can be used at every level.
level0
{
numberOfSubdomains 64;
//method simple;
//simpleCoeffs
//{
// n (2 1 1);
// delta 0.001;
//}
method scotch;
}
level1
{
numberOfSubdomains 4;
method scotch;
}
}
// Desired output
simpleCoeffs
{
n (2 1 1);
delta 0.001;
}
hierarchicalCoeffs
{
n (1 2 1);
delta 0.001;
order xyz;
}
metisCoeffs
{
/*
processorWeights
(
1
1
1
1
);
*/
}
scotchCoeffs
{
//processorWeights
//(
// 1
// 1
// 1
// 1
//);
//writeGraph true;
//strategy "b";
}
manualCoeffs
{
dataFile "decompositionData";
}
structuredCoeffs
{
// Patches to do 2D decomposition on. Structured mesh only; cells have
// to be in 'columns' on top of patches.
patches (movingWall);
// Method to use on the 2D subset
method scotch;
}
//// Is the case distributed? Note: command-line argument -roots takes
//// precedence
//distributed yes;
//// Per slave (so nProcs-1 entries) the directory above the case.
//roots
//(
// "/tmp"
// "/tmp"
//);
// ************************************************************************* //
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment