diff --git a/etc/caseDicts/annotated/snappyHexMeshDict b/etc/caseDicts/annotated/snappyHexMeshDict index 501b5089805ae8b9c5228fec62759b7604389c08..de5177d5b34629d514541c25cc6b005495b1c789 100644 --- a/etc/caseDicts/annotated/snappyHexMeshDict +++ b/etc/caseDicts/annotated/snappyHexMeshDict @@ -100,8 +100,8 @@ castellatedMeshControls // ~~~~~~~~~~~~~~~~~~~~~ // If local number of cells is >= maxLocalCells on any processor - // switches from from refinement followed by balancing - // (current method) to (weighted) balancing before refinement. + // algorithm switches from refinement followed by (weighted) balancing + // to (weighted) balancing followed by refinement. maxLocalCells 100000; // Overall cell limit (approximately). Refinement will stop immediately @@ -123,9 +123,24 @@ castellatedMeshControls // Allow a certain level of imbalance during refining // (since balancing is quite expensive) // Expressed as fraction of perfect balance (= overall number of cells / - // nProcs). 0=balance always. + // nProcs). 0=balance always. Can be triggered by using maxCellUnbalance maxLoadUnbalance 0.10; + // Trigger value after the balancing (based on maxLoadUnbalance) is + // considered since balancing is quite expensive for high core counts. + // Avoid balancing in first refinement iterations as refinement is cheap. + // The trigger value is used for two checks: + // a: if on any processor the number of new added cells <= maxCellUnbalance + // the balancing is skipped + // b: if on any processor the value + // (new cell count of proc - idealCellCount) <= maxCellUnbalance + // the balancing is skipped + // Set to -1 to deactivate + maxCellUnbalance 200000; + + // Force a balancing step after the mesh is completely refined + balanceAtEnd false; + // Number of buffer layers between different levels. // 1 means normal 2:1 refinement restriction, larger means slower // refinement. diff --git a/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H b/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H index 2e985707b7b4364291c3fe435e8f36797ef5af9e..fce8d76e65c3d50154f5dbcb9d772db91930437f 100644 --- a/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H +++ b/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2015-2022 OpenCFD Ltd. + Copyright (C) 2015-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -1271,6 +1271,17 @@ public: //- Refine some cells autoPtr<mapPolyMesh> refine(const labelList& cellsToRefine); + //- Balance the mesh + autoPtr<mapDistributePolyMesh> balance + ( + const string& msg, + decompositionMethod& decomposer, + fvMeshDistribute& distributor, + labelList& cellsToRefine, + const scalar maxLoadUnbalance, + const label maxCellUnbalance + ); + //- Refine some cells and rebalance autoPtr<mapDistributePolyMesh> refineAndBalance ( @@ -1278,7 +1289,8 @@ public: decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& cellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ); //- Balance before refining some cells @@ -1288,7 +1300,8 @@ public: decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& cellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ); //- Calculate list of cells to directionally refine diff --git a/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C b/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C index 312cfa7df3f1b71999b141f0a6f2f0007b4977bc..5747706166906600d7333d1075cfce18d17b913d 100644 --- a/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C +++ b/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2015-2022 OpenCFD Ltd. + Copyright (C) 2015-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -2598,6 +2598,120 @@ Foam::autoPtr<Foam::mapPolyMesh> Foam::meshRefinement::refine } +// Load balancing +Foam::autoPtr<Foam::mapDistributePolyMesh> Foam::meshRefinement::balance +( + const string& msg, + decompositionMethod& decomposer, + fvMeshDistribute& distributor, + labelList& cellsToRefine, + const scalar maxLoadUnbalance, + const label maxCellUnbalance +) +{ + autoPtr<mapDistributePolyMesh> distMap; + + if (Pstream::nProcs() > 1) + { + // First check if we need to balance at all. Precalculate number of + // cells after refinement and see what maximum difference is. + const scalar nNewCells = + scalar(mesh_.nCells() + 7*cellsToRefine.size()); + const scalar nNewCellsAll = returnReduce(nNewCells, sumOp<scalar>()); + const scalar nIdealNewCells = nNewCellsAll / Pstream::nProcs(); + const scalar unbalance = returnReduce + ( + mag(1.0-nNewCells/nIdealNewCells), + maxOp<scalar>() + ); + + // Trigger the balancing to avoid too early balancing for better + // scaling performance. + const scalar nNewCellsOnly = scalar(7*cellsToRefine.size()); + + const label maxNewCells = + label(returnReduce(nNewCellsOnly, maxOp<scalar>())); + + const label maxDeltaCells = + label(mag(returnReduce(nNewCells, maxOp<scalar>())-nIdealNewCells)); + + // New trigger to avoid too early balancing + // 1. Check if globally one proc exceeds the maxCellUnbalance value + // related to the new added cells at the refinement loop + // 2. Check if globally one proc exceeds the maxCellUnbalance based on + // the average cell count a proc should have + if + ( + (maxNewCells <= maxCellUnbalance) + && (maxDeltaCells <= maxCellUnbalance) + ) + { + Info<< "Skipping balancing since trigger value not reached:" << "\n" + << " Trigger cell count: " << maxCellUnbalance << nl + << " Max new cell count in proc: " << maxNewCells << nl + << " Max difference between new cells and balanced: " + << maxDeltaCells + << " Max load unbalance " << maxLoadUnbalance + << nl <<endl; + } + else + { + if (unbalance <= maxLoadUnbalance) + { + Info<< "Skipping balancing since max unbalance " << unbalance + << " is less than allowable " << maxLoadUnbalance + << endl; + } + else + { + scalarField cellWeights(mesh_.nCells(), 1); + forAll(cellsToRefine, i) + { + cellWeights[cellsToRefine[i]] += 7; + } + + distMap = balance + ( + false, //keepZoneFaces + false, //keepBaffles + cellWeights, + decomposer, + distributor + ); + + // Update cells to refine + distMap().distributeCellIndices(cellsToRefine); + + Info<< "Balanced mesh in = " + << mesh_.time().cpuTimeIncrement() << " s" << endl; + + printMeshInfo(debug, "After balancing " + msg); + + + if (debug&meshRefinement::MESH) + { + Pout<< "Writing balanced " << msg + << " mesh to time " << timeName() << endl; + write + ( + debugType(debug), + writeType(writeLevel() | WRITEMESH), + mesh_.time().path()/timeName() + ); + Pout<< "Dumped debug data in = " + << mesh_.time().cpuTimeIncrement() << " s" << endl; + + // test all is still synced across proc patches + checkData(); + } + } + } + } + + return distMap; +} + + // Do refinement of consistent set of cells followed by truncation and // load balancing. Foam::autoPtr<Foam::mapDistributePolyMesh> @@ -2607,10 +2721,13 @@ Foam::meshRefinement::refineAndBalance decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& cellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ) { - // Do all refinement + // Refinement + // ~~~~~~~~~~ + refine(cellsToRefine); if (debug&meshRefinement::MESH) @@ -2638,63 +2755,17 @@ Foam::meshRefinement::refineAndBalance // Load balancing // ~~~~~~~~~~~~~~ - autoPtr<mapDistributePolyMesh> distMap; - - if (Pstream::nProcs() > 1) - { - scalar nIdealCells = - mesh_.globalData().nTotalCells() - / Pstream::nProcs(); + labelList noCellsToRefine; - scalar unbalance = returnReduce - ( - mag(1.0-mesh_.nCells()/nIdealCells), - maxOp<scalar>() - ); - - if (unbalance <= maxLoadUnbalance) - { - Info<< "Skipping balancing since max unbalance " << unbalance - << " is less than allowable " << maxLoadUnbalance - << endl; - } - else - { - scalarField cellWeights(mesh_.nCells(), 1); - - distMap = balance - ( - false, //keepZoneFaces - false, //keepBaffles - cellWeights, - decomposer, - distributor - ); - - Info<< "Balanced mesh in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - - printMeshInfo(debug, "After balancing " + msg); - - - if (debug&meshRefinement::MESH) - { - Pout<< "Writing balanced " << msg - << " mesh to time " << timeName() << endl; - write - ( - debugType(debug), - writeType(writeLevel() | WRITEMESH), - mesh_.time().path()/timeName() - ); - Pout<< "Dumped debug data in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - - // test all is still synced across proc patches - checkData(); - } - } - } + auto distMap = balance + ( + msg, + decomposer, + distributor, + noCellsToRefine, // mesh is already refined; no need to predict + maxLoadUnbalance, + maxCellUnbalance + ); return distMap; } @@ -2708,7 +2779,8 @@ Foam::meshRefinement::balanceAndRefine decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& initCellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ) { labelList cellsToRefine(initCellsToRefine); @@ -2737,90 +2809,24 @@ Foam::meshRefinement::balanceAndRefine //} + // Load balancing // ~~~~~~~~~~~~~~ - autoPtr<mapDistributePolyMesh> distMap; - - if (Pstream::nProcs() > 1) - { - // First check if we need to balance at all. Precalculate number of - // cells after refinement and see what maximum difference is. - scalar nNewCells = scalar(mesh_.nCells() + 7*cellsToRefine.size()); - scalar nIdealNewCells = - returnReduce(nNewCells, sumOp<scalar>()) - / Pstream::nProcs(); - scalar unbalance = returnReduce - ( - mag(1.0-nNewCells/nIdealNewCells), - maxOp<scalar>() - ); - - if (unbalance <= maxLoadUnbalance) - { - Info<< "Skipping balancing since max unbalance " << unbalance - << " is less than allowable " << maxLoadUnbalance - << endl; - } - else - { - scalarField cellWeights(mesh_.nCells(), 1); - forAll(cellsToRefine, i) - { - cellWeights[cellsToRefine[i]] += 7; - } - - distMap = balance - ( - false, //keepZoneFaces - false, //keepBaffles - cellWeights, - decomposer, - distributor - ); - - // Update cells to refine - distMap().distributeCellIndices(cellsToRefine); - - Info<< "Balanced mesh in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - } - - //{ - // globalIndex globalCells(mesh_.nCells()); - // - // Info<< "** Distribution after balancing:" << endl; - // for (const int procI : Pstream::allProcs()) - // { - // Info<< " " << procI << '\t' - // << globalCells.localSize(procI) << endl; - // } - // Info<< endl; - //} - - printMeshInfo(debug, "After balancing " + msg); - - if (debug&meshRefinement::MESH) - { - Pout<< "Writing balanced " << msg - << " mesh to time " << timeName() << endl; - write - ( - debugType(debug), - writeType(writeLevel() | WRITEMESH), - mesh_.time().path()/timeName() - ); - Pout<< "Dumped debug data in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - - // test all is still synced across proc patches - checkData(); - } - } + auto distMap = balance + ( + msg, + decomposer, + distributor, + cellsToRefine, + maxLoadUnbalance, + maxCellUnbalance + ); // Refinement // ~~~~~~~~~~ + // Note: uses updated cellsToRefine refine(cellsToRefine); diff --git a/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C b/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C index e76bf72f9142768a43b75a1bc20a2b166e52a976..08f6507114a60a9fc02653e2544ba59811c31763 100644 --- a/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C +++ b/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2015 OpenFOAM Foundation - Copyright (C) 2015-2020,2022 OpenCFD Ltd. + Copyright (C) 2015-2020,2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -85,6 +85,10 @@ Foam::refinementParameters::refinementParameters dict.getOrDefault("useTopologicalSnapDetection", true) ), maxLoadUnbalance_(dict.getOrDefault<scalar>("maxLoadUnbalance", 0)), + maxCellUnbalance_ + ( + meshRefinement::get<label>(dict, "maxCellUnbalance", 0) + ), handleSnapProblems_ ( dict.getOrDefault<Switch>("handleSnapProblems", true) @@ -97,6 +101,10 @@ Foam::refinementParameters::refinementParameters nFilterIter_(dict.getOrDefault<label>("nFilterIter", 2)), minCellFraction_(dict.getOrDefault<scalar>("minCellFraction", 0)), nMinCells_(dict.getOrDefault<label>("nMinCells", 0)), + balanceAtEnd_ + ( + dict.getOrDefault("balanceAtEnd", false) + ), dryRun_(dryRun) { point locationInMesh; diff --git a/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.H b/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.H index 117ab78d5542e0596b1c0d16e6baeede4f8a6424..3a7abce76dfb0a3fe26be8c4c1a97199a3563ec5 100644 --- a/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.H +++ b/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2014 OpenFOAM Foundation - Copyright (C) 2015-2020,2022 OpenCFD Ltd. + Copyright (C) 2015-2020,2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -108,6 +108,9 @@ class refinementParameters //- Allowed load unbalance const scalar maxLoadUnbalance_; + //- Trigger cell count to start balancing + const label maxCellUnbalance_; + const Switch handleSnapProblems_; const Switch interfaceRefine_; @@ -120,6 +123,9 @@ class refinementParameters const label nMinCells_; + //- Force final balancing after castellation + const bool balanceAtEnd_; + const bool dryRun_; @@ -228,6 +234,12 @@ public: return maxLoadUnbalance_; } + //- Trigger cell count to start balancing + label maxCellUnbalance() const + { + return maxCellUnbalance_; + } + bool handleSnapProblems() const { return handleSnapProblems_; @@ -266,6 +278,12 @@ public: return nMinCells_; } + //- Force final balancing after castallation + bool balanceAtEnd() const + { + return balanceAtEnd_; + } + // Other diff --git a/src/mesh/snappyHexMesh/snappyHexMeshDriver/snappyRefineDriver.C b/src/mesh/snappyHexMesh/snappyHexMeshDriver/snappyRefineDriver.C index c56eb54fad96e57547907a918268f7f04e86a8d5..4832b633391bb216aef2656fdfd93acae0117220 100644 --- a/src/mesh/snappyHexMesh/snappyHexMeshDriver/snappyRefineDriver.C +++ b/src/mesh/snappyHexMesh/snappyHexMeshDriver/snappyRefineDriver.C @@ -179,7 +179,8 @@ Foam::label Foam::snappyRefineDriver::featureEdgeRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -190,7 +191,8 @@ Foam::label Foam::snappyRefineDriver::featureEdgeRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -313,7 +315,8 @@ Foam::label Foam::snappyRefineDriver::smallFeatureRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -324,7 +327,8 @@ Foam::label Foam::snappyRefineDriver::smallFeatureRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -490,7 +494,8 @@ Foam::label Foam::snappyRefineDriver::surfaceOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -501,7 +506,8 @@ Foam::label Foam::snappyRefineDriver::surfaceOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -742,7 +748,8 @@ Foam::label Foam::snappyRefineDriver::gapOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -753,7 +760,8 @@ Foam::label Foam::snappyRefineDriver::gapOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -967,7 +975,8 @@ Foam::label Foam::snappyRefineDriver::bigGapOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -978,7 +987,8 @@ Foam::label Foam::snappyRefineDriver::bigGapOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -1090,6 +1100,22 @@ Foam::label Foam::snappyRefineDriver::danglingCellRefine { Info<< "Stopping refining since too few cells selected." << nl << endl; + + if (refineParams.balanceAtEnd()) + { + Info<< "Final mesh balancing" << endl; + + meshRefiner_.balance + ( + "", + decomposer_, + distributor_, + cellsToRefine, + 0, + -1 + ); + } + break; } @@ -1108,7 +1134,8 @@ Foam::label Foam::snappyRefineDriver::danglingCellRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -1119,7 +1146,8 @@ Foam::label Foam::snappyRefineDriver::danglingCellRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -1453,7 +1481,8 @@ Foam::label Foam::snappyRefineDriver::refinementInterfaceRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -1464,7 +1493,8 @@ Foam::label Foam::snappyRefineDriver::refinementInterfaceRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -1701,7 +1731,8 @@ Foam::label Foam::snappyRefineDriver::boundaryRefinementInterfaceRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -1712,7 +1743,8 @@ Foam::label Foam::snappyRefineDriver::boundaryRefinementInterfaceRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -1961,7 +1993,8 @@ Foam::label Foam::snappyRefineDriver::shellRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -1972,7 +2005,8 @@ Foam::label Foam::snappyRefineDriver::shellRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } }