ENH: implemented a new caching algorithm (WIP)

src/petsc/Make/files

@@ -2,5 +2,6 @@ csrMatrix/solvers/petscSolver.C
 utils/petscUtils.C
 utils/petscControls.C
 utils/petscLinearSolverContexts.C
+utils/petscCacheManager.C
 
 LIB = $(FOAM_SITE_LIBBIN)/libpetscFoam

src/petsc/csrMatrix/solvers/petscSolver.C

@@ -120,6 +120,10 @@ Foam::solverPerformance Foam::petscSolver::solve
         petscLinearSolverContexts::New(fvm);
 
     petscLinearSolverContext& ctx = contexts.getContext(eqName_);
+    
+    dictionary petscDictCaching = petscDict_.subOrEmptyDict("caching");
+    ctx.caching.init(petscDictCaching);
+    ctx.caching.eventBegin();
 
     Mat& Amat = ctx.Amat;
     KSP& ksp  = ctx.ksp;
@@ -141,20 +145,10 @@ Foam::solverPerformance Foam::petscSolver::solve
         buildKsp(Amat, ksp);
     }
 
-    const bool caching
-    (
-        petscDict_.lookupOrDefault("caching", true)
-    );
-
-    if (!caching && needsCacheUpdate)
+    if (ctx.caching.needsMatrixUpdate() && needsCacheUpdate)
     {
-        if (debug)
-        {
-            Info<< "Updating cached PETSc Linear Solver " << eqName_ << nl;
-        }
-
         buildMat(Amat);
-        updateKsp(Amat, ksp);
+        updateKsp(ksp, Amat, !ctx.caching.needsPrecondUpdate());
     }
 
     // Solver name from petsc dictionary
@@ -221,6 +215,8 @@ Foam::solverPerformance Foam::petscSolver::solve
         solverPerf.finalResidual() = rnorm;
     }
 
+    ctx.caching.eventEnd();
+
     // Return solver performance to OpenFOAM
     return solverPerf;
 }
@@ -228,10 +224,30 @@ Foam::solverPerformance Foam::petscSolver::solve
 
 void Foam::petscSolver::updateKsp
 (
+    KSP& ksp,
     Mat& Amat,
-    KSP& ksp
+    const bool reuse
 ) const
 {
+    if (debug)
+    {
+        Info<< "Cache-Hit: updating PETSc Amat " << eqName_ << nl;
+    }
+
+    if (reuse)
+    {
+        KSPSetReusePreconditioner(ksp, PETSC_TRUE);
+    }
+    else
+    {
+        if (debug)
+        {
+            Info<< "Cache-Hit: updating PETSc Pmat " << eqName_ << nl;
+        }
+
+        KSPSetReusePreconditioner(ksp, PETSC_FALSE);
+    }
+
     KSPSetOperators(ksp, Amat, Amat);
 }
 

src/petsc/csrMatrix/solvers/petscSolver.H

@@ -104,7 +104,7 @@ class petscSolver
         void buildKsp(Mat& Amat, KSP& ksp) const;
 
         //- Update the ksp_ krylov solver context
-        void updateKsp(Mat& Amat, KSP& ksp) const;
+        void updateKsp(KSP& ksp, Mat& Amat, const bool reuse) const;
 
         //- No copy construct
         petscSolver(const petscSolver&) = delete;

src/petsc/utils/petscCacheManager.C

+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2019 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "petscCacheManager.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+namespace Foam
+{
+    defineTypeNameAndDebug(petscCacheManager, 0);
+}
+
+// ************************************************************************* //

src/petsc/utils/petscCacheManager.H

+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2019 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::petscCacheManager
+
+Description
+    The manager class for the caching of matrix and preconditioner.
+
+    An Example is:
+
+    ...
+    petsc
+    {
+        ...
+        caching
+        {
+            matrixCaching
+            {
+                update always;
+            }
+
+            preconditionerCaching
+            {
+                update periodic;
+
+                periodicCoeffs
+                {
+                    frequency 3;
+                }
+            }
+        }
+    }
+    ...
+
+SourceFiles
+    petscCacheManager.H
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef petscCacheManager_H
+#define petscCacheManager_H
+
+#include "solverPerformance.H"
+#include "dictionary.H"
+
+#include "petsctime.h"
+#include "petscmath.h"
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+                     Class petscCacheManager Declaration
+\*---------------------------------------------------------------------------*/
+
+
+struct Caching
+{
+    Caching():
+       updateFrequency{1}, 
+       update{"never"} {}
+
+    void init(const dictionary& petscDict, const word key)
+    {
+        dictionary cachingDict = petscDict.subOrEmptyDict(key);
+        update = cachingDict.lookupOrDefault("update", word("never"));
+
+        if (update == "periodic")
+        {
+            dictionary cachingPeriodicCoeffsDict = cachingDict.subOrEmptyDict("periodicCoeffs");
+            updateFrequency = cachingPeriodicCoeffsDict.lookupOrDefault("frequency", 1);
+        }
+    }
+
+    int updateFrequency;  
+    word update;
+};
+
+
+class petscCacheManager
+{
+    // Private Data
+
+        int iter;
+        PetscLogDouble time;
+        Caching matrixCaching;
+        Caching precondCaching;
+
+public:
+
+    // Public Members
+
+    TypeName("petscCacheManager");
+
+    // Constructors
+
+        //- Construct null
+        petscCacheManager()
+        {
+            iter = 0;
+        }
+
+    //- Destructor
+    virtual ~petscCacheManager()
+    {
+    }
+
+    void init(const dictionary& petscDict)
+    {
+        matrixCaching.init(petscDict, "matrixCaching");
+        precondCaching.init(petscDict, "preconditionerCaching");
+    }
+
+    // Member Functions
+    bool needsMatrixUpdate() const
+    {
+        return needsUpdate(matrixCaching);
+    }
+
+    bool needsPrecondUpdate() const
+    {
+        return needsUpdate(precondCaching);
+    }
+
+    void eventBegin()
+    {
+        PetscTime(&time);
+        return;
+    }
+
+    void eventEnd()
+    {
+        PetscTimeSubtract(&time);
+        double dtime = fabs(static_cast<double>(time));
+        // Info << " delta time: " << dtime << nl;
+        iter++;
+        return;
+    }
+
+private:
+
+    bool needsUpdate(Caching caching) const
+    {
+        if (caching.update == "never")
+        {
+            return false;
+        }
+        else if (caching.update == "always")
+        {
+            return true;
+        }
+        else if (caching.update == "periodic")
+        {
+            if (iter % caching.updateFrequency)
+            {
+                return false;
+            }
+            else
+            {
+                return true;
+            } 
+        }
+        else if (caching.update == "adaptive")
+        {
+            // to be implemented
+            return false;
+        }
+
+        // default
+        return false;
+    }
+};
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //

src/petsc/utils/petscLinearSolverContext.H

@@ -38,6 +38,7 @@ SourceFiles
 #define petscLinearSolverContext_H
 
 #include "solverPerformance.H"
+#include "petscCacheManager.H"
 
 #include "petscvec.h"
 #include "petscmat.h"
@@ -66,6 +67,7 @@ public:
         KSP ksp;
 
         solverPerformance* solverPerf;
+        petscCacheManager caching; 
 
         // Auxiliary vectors for the OpenFOAM-PETSc L1-norm
         Vec AdotPsi;