CUDA ERROR in hypre memory.c when running pitzDaily tutorial in OpenFOAM v2206

Environments

OpenFoam is built from v2206 source code

petsc is built from release branch

nanami@nanami-Mi-Gaming-Laptop-15-6:~/petsc$ git remote -v
commit 54490b9db9480b2dcbe1dd1894a61fa0da0ab46e (HEAD -> release, origin/release)
Merge: 5c04a22c60 f319fef0c3
Author: Satish Balay <balay@mcs.anl.gov>
Date:   Fri Aug 12 15:11:56 2022 +0000

    Merge branch 'jolivet/update-slepc-release' into 'release'

    configure: SLEPc 3.17.2

    See merge request petsc/petsc!5522

petsc build config:

./configure --with-debugging=0 --COPTFLAGS=-O3 --CXXOPTFLAGS=-O3 --FOPTFLAGS=-O --download-hypre --with-cuda --with-fc=0 

simpleFoam command output

nanami@nanami-Mi-Gaming-Laptop-15-6:~/OpenFOAM-v2206/modules/external-solver/tutorials/incompressible/simpleFoam/pitzDaily/run$ simpleFoam
/*---------------------------------------------------------------------------*\
| =========                 |                                                 |
| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
|  \\    /   O peration     | Version:  2206                                  |
|   \\  /    A nd           | Website:  www.openfoam.com                      |
|    \\/     M anipulation  |                                                 |
\*---------------------------------------------------------------------------*/
Build  : _76d719d1e6-20220624 OPENFOAM=2206 version=v2206
Arch   : "LSB;label=32;scalar=64"
Exec   : simpleFoam
Date   : Sep 06 2022
Time   : 14:45:27
Host   : nanami-Mi-Gaming-Laptop-15-6
PID    : 37192
I/O    : uncollated
Case   : /home/nanami/OpenFOAM-v2206/modules/external-solver/tutorials/incompressible/simpleFoam/pitzDaily/run
nProcs : 1
trapFpe: Floating point exception trapping enabled (FOAM_SIGFPE).
fileModificationChecking : Monitoring run-time modified files using timeStampMaster (fileModificationSkew 5, maxFileModificationPolls 20)
allowSystemOperations : Allowing user-supplied system call operations

// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
Create time

Create mesh for time = 0


SIMPLE: convergence criteria
    field p      tolerance 0.01
    field U      tolerance 0.001
    field "(k|epsilon|omega|f|v2)"       tolerance 0.001

Reading field p

Reading field U

Reading/calculating face flux field phi

Selecting incompressible transport model Newtonian
Selecting turbulence model type RAS
Selecting RAS turbulence model kEpsilon
RAS
{
    RASModel        kEpsilon;
    turbulence      on;
    printCoeffs     on;
    Cmu             0.09;
    C1              1.44;
    C2              1.92;
    C3              0;
    sigmak          1;
    sigmaEps        1.3;
}

No MRF models present

No finite volume options present

Starting time loop

streamLine streamlines:
    Employing velocity field U
    automatic track length specified through number of sub cycles : 5

Time = 1

Initializing PETSc
PETSc-bicg:  Solving for Ux, Initial residual = 1, Final residual = 0.0165669, No Iterations 1
PETSc-bicg:  Solving for Uy, Initial residual = 1, Final residual = 0.0143104, No Iterations 1
CUDA ERROR (code = 1, invalid argument) at memory.c:716

memory.c

nanami@nanami-Mi-Gaming-Laptop-15-6:~/petsc/arch-linux-c-opt/externalpackages/git.hypre/src$ cat -n utilities/memory.c | grep 716 -A10 -B10
   706  #if defined(HYPRE_DEVICE_OPENMP_ALLOC)
   707        omp_target_memcpy(dst, src, size, 0, 0, hypre__offload_device_num, hypre__offload_device_num);
   708  #else
   709        HYPRE_OMPOffload(hypre__offload_device_num, src, size, "update", "from");
   710        memcpy(dst, src, size);
   711        HYPRE_OMPOffload(hypre__offload_device_num, dst, size, "update", "to");
   712  #endif
   713  #endif
   714
   715  #if defined(HYPRE_USING_CUDA)
   716        HYPRE_CUDA_CALL( cudaMemcpy(dst, src, size, cudaMemcpyDeviceToDevice) );
   717  #endif
   718
   719  #if defined(HYPRE_USING_HIP)
   720        HYPRE_HIP_CALL( hipMemcpy(dst, src, size, hipMemcpyDeviceToDevice) );
   721  #endif
   722
   723  #if defined(HYPRE_USING_SYCL)
   724        HYPRE_SYCL_CALL( q->memcpy(dst, src, size).wait() );
   725  #endif
   726        return;

Backtrace

backtrace() returned 14 addresses
/home/nanami/petsc/arch-linux-c-opt/lib/libHYPRE-2.24.0.so(+0x52f4b1) [0x7f8f627544b1]
/home/nanami/petsc/arch-linux-c-opt/lib/libHYPRE-2.24.0.so(hypre_IJVectorSetAddValuesParDevice+0x138) [0x7f8f6253d218]
/home/nanami/petsc/arch-linux-c-opt/lib/libHYPRE-2.24.0.so(hypre_IJVectorSetAddValuesParDevice+0x373) [0x7f8f6253d453]
/home/nanami/petsc/arch-linux-c-opt/lib/libpetsc.so.3.17(VecHYPRE_IJVectorCopy+0x77) [0x7f8f65c4b8e7]
/home/nanami/petsc/arch-linux-c-opt/lib/libpetsc.so.3.17(+0xf343fa) [0x7f8f664273fa]
/home/nanami/petsc/arch-linux-c-opt/lib/libpetsc.so.3.17(PCSetCoordinates+0x51) [0x7f8f66436331]
/home/nanami/OpenFOAM/nanami-v2206/platforms/linux64GccDPInt32Opt/lib/libpetscFoam.so(_ZNK4Foam11petscSolver11scalarSolveERNS_5FieldIdEERKS2_h+0x1403) [0x7f8f66ff72e3]
/home/nanami/OpenFOAM/nanami-v2206/platforms/linux64GccDPInt32Opt/lib/libpetscFoam.so(_ZNK4Foam11petscSolver5solveERNS_5FieldIdEERKS2_h+0x48) [0x7f8f66ff5d58]
/home/nanami/OpenFOAM-v2206/platforms/linux64GccDPInt32Opt/lib/libfiniteVolume.so(_ZN4Foam8fvMatrixIdE15solveSegregatedERKNS_10dictionaryE+0x625) [0x7f8f6decb0d5]
/home/nanami/OpenFOAM-v2206/platforms/linux64GccDPInt32Opt/lib/libfiniteVolume.so(_ZN4Foam8fvMatrixIdE24solveSegregatedOrCoupledERKNS_10dictionaryE+0x498) [0x7f8f6d653728]
/home/nanami/OpenFOAM-v2206/platforms/linux64GccDPInt32Opt/lib/libfiniteVolume.so(_ZNK4Foam6fvMesh5solveERNS_8fvMatrixIdEERKNS_10dictionaryE+0x28) [0x7f8f6d5ec2e8]
simpleFoam(+0x2a60f) [0x56548867460f]
/usr/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xf3) [0x7f8f6a2160b3]
simpleFoam(+0x2beee) [0x565488675eee]
dst: 0x7f8edda36000, src: 0x56549067c680, size: 97800

Seems the src param of cudaMemcpy(dst, src, size, cudaMemcpyDeviceToDevice) call is not a valid cuda device mapped mem address. What should I do for further trouble shooting? Thanks.