From ac80242ab7a53a09eb52ea9f20f1323a4d4282cb Mon Sep 17 00:00:00 2001 From: Holzmann <> Date: Tue, 30 May 2023 16:05:17 +0100 Subject: [PATCH] ENH: snappyHexMesh: enable late balancing. Fixes #2792 --- etc/caseDicts/annotated/snappyHexMeshDict | 21 +- .../meshRefinement/meshRefinement.H | 19 +- .../meshRefinement/meshRefinementRefine.C | 280 +++++++++--------- .../refinementParameters.C | 10 +- .../refinementParameters.H | 20 +- .../snappyHexMeshDriver/snappyRefineDriver.C | 70 +++-- 6 files changed, 257 insertions(+), 163 deletions(-) diff --git a/etc/caseDicts/annotated/snappyHexMeshDict b/etc/caseDicts/annotated/snappyHexMeshDict index 501b5089805..de5177d5b34 100644 --- a/etc/caseDicts/annotated/snappyHexMeshDict +++ b/etc/caseDicts/annotated/snappyHexMeshDict @@ -100,8 +100,8 @@ castellatedMeshControls // ~~~~~~~~~~~~~~~~~~~~~ // If local number of cells is >= maxLocalCells on any processor - // switches from from refinement followed by balancing - // (current method) to (weighted) balancing before refinement. + // algorithm switches from refinement followed by (weighted) balancing + // to (weighted) balancing followed by refinement. maxLocalCells 100000; // Overall cell limit (approximately). Refinement will stop immediately @@ -123,9 +123,24 @@ castellatedMeshControls // Allow a certain level of imbalance during refining // (since balancing is quite expensive) // Expressed as fraction of perfect balance (= overall number of cells / - // nProcs). 0=balance always. + // nProcs). 0=balance always. Can be triggered by using maxCellUnbalance maxLoadUnbalance 0.10; + // Trigger value after the balancing (based on maxLoadUnbalance) is + // considered since balancing is quite expensive for high core counts. + // Avoid balancing in first refinement iterations as refinement is cheap. + // The trigger value is used for two checks: + // a: if on any processor the number of new added cells <= maxCellUnbalance + // the balancing is skipped + // b: if on any processor the value + // (new cell count of proc - idealCellCount) <= maxCellUnbalance + // the balancing is skipped + // Set to -1 to deactivate + maxCellUnbalance 200000; + + // Force a balancing step after the mesh is completely refined + balanceAtEnd false; + // Number of buffer layers between different levels. // 1 means normal 2:1 refinement restriction, larger means slower // refinement. diff --git a/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H b/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H index 2e985707b7b..fce8d76e65c 100644 --- a/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H +++ b/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2015-2022 OpenCFD Ltd. + Copyright (C) 2015-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -1271,6 +1271,17 @@ public: //- Refine some cells autoPtr<mapPolyMesh> refine(const labelList& cellsToRefine); + //- Balance the mesh + autoPtr<mapDistributePolyMesh> balance + ( + const string& msg, + decompositionMethod& decomposer, + fvMeshDistribute& distributor, + labelList& cellsToRefine, + const scalar maxLoadUnbalance, + const label maxCellUnbalance + ); + //- Refine some cells and rebalance autoPtr<mapDistributePolyMesh> refineAndBalance ( @@ -1278,7 +1289,8 @@ public: decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& cellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ); //- Balance before refining some cells @@ -1288,7 +1300,8 @@ public: decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& cellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ); //- Calculate list of cells to directionally refine diff --git a/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C b/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C index 312cfa7df3f..57477061669 100644 --- a/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C +++ b/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2015-2022 OpenCFD Ltd. + Copyright (C) 2015-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -2598,6 +2598,120 @@ Foam::autoPtr<Foam::mapPolyMesh> Foam::meshRefinement::refine } +// Load balancing +Foam::autoPtr<Foam::mapDistributePolyMesh> Foam::meshRefinement::balance +( + const string& msg, + decompositionMethod& decomposer, + fvMeshDistribute& distributor, + labelList& cellsToRefine, + const scalar maxLoadUnbalance, + const label maxCellUnbalance +) +{ + autoPtr<mapDistributePolyMesh> distMap; + + if (Pstream::nProcs() > 1) + { + // First check if we need to balance at all. Precalculate number of + // cells after refinement and see what maximum difference is. + const scalar nNewCells = + scalar(mesh_.nCells() + 7*cellsToRefine.size()); + const scalar nNewCellsAll = returnReduce(nNewCells, sumOp<scalar>()); + const scalar nIdealNewCells = nNewCellsAll / Pstream::nProcs(); + const scalar unbalance = returnReduce + ( + mag(1.0-nNewCells/nIdealNewCells), + maxOp<scalar>() + ); + + // Trigger the balancing to avoid too early balancing for better + // scaling performance. + const scalar nNewCellsOnly = scalar(7*cellsToRefine.size()); + + const label maxNewCells = + label(returnReduce(nNewCellsOnly, maxOp<scalar>())); + + const label maxDeltaCells = + label(mag(returnReduce(nNewCells, maxOp<scalar>())-nIdealNewCells)); + + // New trigger to avoid too early balancing + // 1. Check if globally one proc exceeds the maxCellUnbalance value + // related to the new added cells at the refinement loop + // 2. Check if globally one proc exceeds the maxCellUnbalance based on + // the average cell count a proc should have + if + ( + (maxNewCells <= maxCellUnbalance) + && (maxDeltaCells <= maxCellUnbalance) + ) + { + Info<< "Skipping balancing since trigger value not reached:" << "\n" + << " Trigger cell count: " << maxCellUnbalance << nl + << " Max new cell count in proc: " << maxNewCells << nl + << " Max difference between new cells and balanced: " + << maxDeltaCells + << " Max load unbalance " << maxLoadUnbalance + << nl <<endl; + } + else + { + if (unbalance <= maxLoadUnbalance) + { + Info<< "Skipping balancing since max unbalance " << unbalance + << " is less than allowable " << maxLoadUnbalance + << endl; + } + else + { + scalarField cellWeights(mesh_.nCells(), 1); + forAll(cellsToRefine, i) + { + cellWeights[cellsToRefine[i]] += 7; + } + + distMap = balance + ( + false, //keepZoneFaces + false, //keepBaffles + cellWeights, + decomposer, + distributor + ); + + // Update cells to refine + distMap().distributeCellIndices(cellsToRefine); + + Info<< "Balanced mesh in = " + << mesh_.time().cpuTimeIncrement() << " s" << endl; + + printMeshInfo(debug, "After balancing " + msg); + + + if (debug&meshRefinement::MESH) + { + Pout<< "Writing balanced " << msg + << " mesh to time " << timeName() << endl; + write + ( + debugType(debug), + writeType(writeLevel() | WRITEMESH), + mesh_.time().path()/timeName() + ); + Pout<< "Dumped debug data in = " + << mesh_.time().cpuTimeIncrement() << " s" << endl; + + // test all is still synced across proc patches + checkData(); + } + } + } + } + + return distMap; +} + + // Do refinement of consistent set of cells followed by truncation and // load balancing. Foam::autoPtr<Foam::mapDistributePolyMesh> @@ -2607,10 +2721,13 @@ Foam::meshRefinement::refineAndBalance decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& cellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ) { - // Do all refinement + // Refinement + // ~~~~~~~~~~ + refine(cellsToRefine); if (debug&meshRefinement::MESH) @@ -2638,63 +2755,17 @@ Foam::meshRefinement::refineAndBalance // Load balancing // ~~~~~~~~~~~~~~ - autoPtr<mapDistributePolyMesh> distMap; - - if (Pstream::nProcs() > 1) - { - scalar nIdealCells = - mesh_.globalData().nTotalCells() - / Pstream::nProcs(); + labelList noCellsToRefine; - scalar unbalance = returnReduce - ( - mag(1.0-mesh_.nCells()/nIdealCells), - maxOp<scalar>() - ); - - if (unbalance <= maxLoadUnbalance) - { - Info<< "Skipping balancing since max unbalance " << unbalance - << " is less than allowable " << maxLoadUnbalance - << endl; - } - else - { - scalarField cellWeights(mesh_.nCells(), 1); - - distMap = balance - ( - false, //keepZoneFaces - false, //keepBaffles - cellWeights, - decomposer, - distributor - ); - - Info<< "Balanced mesh in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - - printMeshInfo(debug, "After balancing " + msg); - - - if (debug&meshRefinement::MESH) - { - Pout<< "Writing balanced " << msg - << " mesh to time " << timeName() << endl; - write - ( - debugType(debug), - writeType(writeLevel() | WRITEMESH), - mesh_.time().path()/timeName() - ); - Pout<< "Dumped debug data in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - - // test all is still synced across proc patches - checkData(); - } - } - } + auto distMap = balance + ( + msg, + decomposer, + distributor, + noCellsToRefine, // mesh is already refined; no need to predict + maxLoadUnbalance, + maxCellUnbalance + ); return distMap; } @@ -2708,7 +2779,8 @@ Foam::meshRefinement::balanceAndRefine decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& initCellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ) { labelList cellsToRefine(initCellsToRefine); @@ -2737,90 +2809,24 @@ Foam::meshRefinement::balanceAndRefine //} + // Load balancing // ~~~~~~~~~~~~~~ - autoPtr<mapDistributePolyMesh> distMap; - - if (Pstream::nProcs() > 1) - { - // First check if we need to balance at all. Precalculate number of - // cells after refinement and see what maximum difference is. - scalar nNewCells = scalar(mesh_.nCells() + 7*cellsToRefine.size()); - scalar nIdealNewCells = - returnReduce(nNewCells, sumOp<scalar>()) - / Pstream::nProcs(); - scalar unbalance = returnReduce - ( - mag(1.0-nNewCells/nIdealNewCells), - maxOp<scalar>() - ); - - if (unbalance <= maxLoadUnbalance) - { - Info<< "Skipping balancing since max unbalance " << unbalance - << " is less than allowable " << maxLoadUnbalance - << endl; - } - else - { - scalarField cellWeights(mesh_.nCells(), 1); - forAll(cellsToRefine, i) - { - cellWeights[cellsToRefine[i]] += 7; - } - - distMap = balance - ( - false, //keepZoneFaces - false, //keepBaffles - cellWeights, - decomposer, - distributor - ); - - // Update cells to refine - distMap().distributeCellIndices(cellsToRefine); - - Info<< "Balanced mesh in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - } - - //{ - // globalIndex globalCells(mesh_.nCells()); - // - // Info<< "** Distribution after balancing:" << endl; - // for (const int procI : Pstream::allProcs()) - // { - // Info<< " " << procI << '\t' - // << globalCells.localSize(procI) << endl; - // } - // Info<< endl; - //} - - printMeshInfo(debug, "After balancing " + msg); - - if (debug&meshRefinement::MESH) - { - Pout<< "Writing balanced " << msg - << " mesh to time " << timeName() << endl; - write - ( - debugType(debug), - writeType(writeLevel() | WRITEMESH), - mesh_.time().path()/timeName() - ); - Pout<< "Dumped debug data in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - - // test all is still synced across proc patches - checkData(); - } - } + auto distMap = balance + ( + msg, + decomposer, + distributor, + cellsToRefine, + maxLoadUnbalance, + maxCellUnbalance + ); // Refinement // ~~~~~~~~~~ + // Note: uses updated cellsToRefine refine(cellsToRefine); diff --git a/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C b/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C index e76bf72f914..08f6507114a 100644 --- a/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C +++ b/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2015 OpenFOAM Foundation - Copyright (C) 2015-2020,2022 OpenCFD Ltd. + Copyright (C) 2015-2020,2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -85,6 +85,10 @@ Foam::refinementParameters::refinementParameters dict.getOrDefault("useTopologicalSnapDetection", true) ), maxLoadUnbalance_(dict.getOrDefault<scalar>("maxLoadUnbalance", 0)), + maxCellUnbalance_ + ( + meshRefinement::get<label>(dict, "maxCellUnbalance", 0) + ), handleSnapProblems_ ( dict.getOrDefault<Switch>("handleSnapProblems", true) @@ -97,6 +101,10 @@ Foam::refinementParameters::refinementParameters nFilterIter_(dict.getOrDefault<label>("nFilterIter", 2)), minCellFraction_(dict.getOrDefault<scalar>("minCellFraction", 0)), nMinCells_(dict.getOrDefault<label>("nMinCells", 0)), + balanceAtEnd_ + ( + dict.getOrDefault("balanceAtEnd", false) + ), dryRun_(dryRun) { point locationInMesh; diff --git a/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.H b/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.H index 117ab78d554..3a7abce76df 100644 --- a/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.H +++ b/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2014 OpenFOAM Foundation - Copyright (C) 2015-2020,2022 OpenCFD Ltd. + Copyright (C) 2015-2020,2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -108,6 +108,9 @@ class refinementParameters //- Allowed load unbalance const scalar maxLoadUnbalance_; + //- Trigger cell count to start balancing + const label maxCellUnbalance_; + const Switch handleSnapProblems_; const Switch interfaceRefine_; @@ -120,6 +123,9 @@ class refinementParameters const label nMinCells_; + //- Force final balancing after castellation + const bool balanceAtEnd_; + const bool dryRun_; @@ -228,6 +234,12 @@ public: return maxLoadUnbalance_; } + //- Trigger cell count to start balancing + label maxCellUnbalance() const + { + return maxCellUnbalance_; + } + bool handleSnapProblems() const { return handleSnapProblems_; @@ -266,6 +278,12 @@ public: return nMinCells_; } + //- Force final balancing after castallation + bool balanceAtEnd() const + { + return balanceAtEnd_; + } + // Other diff --git a/src/mesh/snappyHexMesh/snappyHexMeshDriver/snappyRefineDriver.C b/src/mesh/snappyHexMesh/snappyHexMeshDriver/snappyRefineDriver.C index c56eb54fad9..4832b633391 100644 --- a/src/mesh/snappyHexMesh/snappyHexMeshDriver/snappyRefineDriver.C +++ b/src/mesh/snappyHexMesh/snappyHexMeshDriver/snappyRefineDriver.C @@ -179,7 +179,8 @@ Foam::label Foam::snappyRefineDriver::featureEdgeRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -190,7 +191,8 @@ Foam::label Foam::snappyRefineDriver::featureEdgeRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -313,7 +315,8 @@ Foam::label Foam::snappyRefineDriver::smallFeatureRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -324,7 +327,8 @@ Foam::label Foam::snappyRefineDriver::smallFeatureRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -490,7 +494,8 @@ Foam::label Foam::snappyRefineDriver::surfaceOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -501,7 +506,8 @@ Foam::label Foam::snappyRefineDriver::surfaceOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -742,7 +748,8 @@ Foam::label Foam::snappyRefineDriver::gapOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -753,7 +760,8 @@ Foam::label Foam::snappyRefineDriver::gapOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -967,7 +975,8 @@ Foam::label Foam::snappyRefineDriver::bigGapOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -978,7 +987,8 @@ Foam::label Foam::snappyRefineDriver::bigGapOnlyRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -1090,6 +1100,22 @@ Foam::label Foam::snappyRefineDriver::danglingCellRefine { Info<< "Stopping refining since too few cells selected." << nl << endl; + + if (refineParams.balanceAtEnd()) + { + Info<< "Final mesh balancing" << endl; + + meshRefiner_.balance + ( + "", + decomposer_, + distributor_, + cellsToRefine, + 0, + -1 + ); + } + break; } @@ -1108,7 +1134,8 @@ Foam::label Foam::snappyRefineDriver::danglingCellRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -1119,7 +1146,8 @@ Foam::label Foam::snappyRefineDriver::danglingCellRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -1453,7 +1481,8 @@ Foam::label Foam::snappyRefineDriver::refinementInterfaceRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -1464,7 +1493,8 @@ Foam::label Foam::snappyRefineDriver::refinementInterfaceRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -1701,7 +1731,8 @@ Foam::label Foam::snappyRefineDriver::boundaryRefinementInterfaceRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -1712,7 +1743,8 @@ Foam::label Foam::snappyRefineDriver::boundaryRefinementInterfaceRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } @@ -1961,7 +1993,8 @@ Foam::label Foam::snappyRefineDriver::shellRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } else @@ -1972,7 +2005,8 @@ Foam::label Foam::snappyRefineDriver::shellRefine decomposer_, distributor_, cellsToRefine, - refineParams.maxLoadUnbalance() + refineParams.maxLoadUnbalance(), + refineParams.maxCellUnbalance() ); } } -- GitLab