Skip to content
Snippets Groups Projects
Commit fca698cd authored by mattijs's avatar mattijs Committed by Andrew Heather
Browse files

ENH: cyclicAMI: add non-blocking for GAMG. See #3052

parent 88b3fb7c
Branches
Tags
No related merge requests found
Showing with 426 additions and 57 deletions
...@@ -109,6 +109,18 @@ Foam::processorGAMGInterfaceField::processorGAMGInterfaceField ...@@ -109,6 +109,18 @@ Foam::processorGAMGInterfaceField::processorGAMGInterfaceField
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::processorGAMGInterfaceField::ready() const
{
const bool ok = UPstream::finishedRequest(recvRequest_);
if (ok)
{
recvRequest_ = -1;
if (UPstream::finishedRequest(sendRequest_)) sendRequest_ = -1;
}
return ok;
}
void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
( (
solveScalarField&, solveScalarField&,
......
...@@ -162,6 +162,9 @@ public: ...@@ -162,6 +162,9 @@ public:
// Interface matrix update // Interface matrix update
//- Are all (receive) data available?
virtual bool ready() const;
//- Initialise neighbour matrix update //- Initialise neighbour matrix update
virtual void initInterfaceMatrixUpdate virtual void initInterfaceMatrixUpdate
( (
......
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2013 OpenFOAM Foundation Copyright (C) 2013 OpenFOAM Foundation
Copyright (C) 2019 OpenCFD Ltd. Copyright (C) 2019,2023 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
...@@ -67,7 +67,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField ...@@ -67,7 +67,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, fineInterface), GAMGInterfaceField(GAMGCp, fineInterface),
cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)), cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
doTransform_(false), doTransform_(false),
rank_(0) rank_(0),
sendRequests_(0),
recvRequests_(0)
{ {
const cyclicAMILduInterfaceField& p = const cyclicAMILduInterfaceField& p =
refCast<const cyclicAMILduInterfaceField>(fineInterface); refCast<const cyclicAMILduInterfaceField>(fineInterface);
...@@ -87,7 +89,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField ...@@ -87,7 +89,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, doTransform, rank), GAMGInterfaceField(GAMGCp, doTransform, rank),
cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)), cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
doTransform_(doTransform), doTransform_(doTransform),
rank_(rank) rank_(rank),
sendRequests_(0),
recvRequests_(0)
{} {}
...@@ -100,7 +104,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField ...@@ -100,7 +104,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, is), GAMGInterfaceField(GAMGCp, is),
cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)), cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
doTransform_(readBool(is)), doTransform_(readBool(is)),
rank_(readLabel(is)) rank_(readLabel(is)),
sendRequests_(0),
recvRequests_(0)
{} {}
...@@ -114,7 +120,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField ...@@ -114,7 +120,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, local), GAMGInterfaceField(GAMGCp, local),
cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)), cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
doTransform_(false), doTransform_(false),
rank_(0) rank_(0),
sendRequests_(0),
recvRequests_(0)
{ {
const auto& p = refCast<const cyclicACMILduInterfaceField>(local); const auto& p = refCast<const cyclicACMILduInterfaceField>(local);
...@@ -123,13 +131,108 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField ...@@ -123,13 +131,108 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
} }
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::cyclicACMIGAMGInterfaceField::~cyclicACMIGAMGInterfaceField() bool Foam::cyclicACMIGAMGInterfaceField::ready() const
{} {
if
(
UPstream::finishedRequests
(
recvRequests_.start(),
recvRequests_.size()
)
)
{
recvRequests_.clear();
if
(
UPstream::finishedRequests
(
sendRequests_.start(),
sendRequests_.size()
)
)
{
sendRequests_.clear();
}
return true;
}
return false;
}
void Foam::cyclicACMIGAMGInterfaceField::initInterfaceMatrixUpdate
(
solveScalarField& result,
const bool add,
const lduAddressing& lduAddr,
const label patchId,
const solveScalarField& psiInternal,
const scalarField& coeffs,
const direction cmpt,
const Pstream::commsTypes commsType
) const
{
const auto& AMI =
(
cyclicACMIInterface_.owner()
? cyclicACMIInterface_.AMI()
: cyclicACMIInterface_.neighbPatch().AMI()
);
if (AMI.distributed())
{
DebugPout<< "cyclicACMIFvPatchField::initInterfaceMatrixUpdate() :"
<< " interface:" << cyclicACMIInterface_.index()
<< " size:" << cyclicACMIInterface_.size()
<< " owner:" << cyclicACMIInterface_.owner()
<< " AMI distributed:" << AMI.distributed()
<< endl;
// Start sending
if (commsType != UPstream::commsTypes::nonBlocking)
{
FatalErrorInFunction
<< "Can only evaluate distributed AMI with nonBlocking"
<< exit(FatalError);
}
// Get neighbouring field
const labelList& nbrFaceCells =
lduAddr.patchAddr(cyclicACMIInterface_.neighbPatchID());
solveScalarField pnf(psiInternal, nbrFaceCells);
// Transform according to the transformation tensors
transformCoupleField(pnf, cmpt);
const auto& map =
(
cyclicACMIInterface_.owner()
? AMI.tgtMap()
: AMI.srcMap()
);
// Insert send/receive requests (non-blocking). See e.g.
// cyclicAMIPolyPatchTemplates.C
const label oldWarnComm = UPstream::warnComm;
UPstream::warnComm = AMI.comm();
map.send
(
pnf,
sendRequests_,
scalarSendBufs_,
recvRequests_,
scalarRecvBufs_
);
UPstream::warnComm = oldWarnComm;
}
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
( (
...@@ -143,30 +246,73 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix ...@@ -143,30 +246,73 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
const Pstream::commsTypes const Pstream::commsTypes
) const ) const
{ {
// Get neighbouring field const labelUList& faceCells = lduAddr.patchAddr(patchId);
const labelList& nbrFaceCells =
lduAddr.patchAddr const auto& AMI =
(
cyclicACMIInterface_.owner()
? cyclicACMIInterface_.AMI()
: cyclicACMIInterface_.neighbPatch().AMI()
);
DebugPout<< "cyclicACMIGAMGInterfaceField::updateInterfaceMatrix() :"
<< " interface:" << cyclicACMIInterface_.index()
<< " size:" << cyclicACMIInterface_.size()
<< " owner:" << cyclicACMIInterface_.owner()
<< " AMI distributed:" << AMI.distributed()
<< endl;
if (AMI.distributed())
{
const auto& map =
( (
cyclicACMIInterface_.neighbPatchID() cyclicACMIInterface_.owner()
? AMI.tgtMap()
: AMI.srcMap()
); );
solveScalarField pnf(psiInternal, nbrFaceCells); // Receive (= copy) data from buffers into work. TBD: receive directly
// into slices of work.
solveScalarField work;
map.receive(recvRequests_, scalarRecvBufs_, work);
// Transform according to the transformation tensors solveScalarField pnf(faceCells.size(), Zero);
transformCoupleField(pnf, cmpt); AMI.weightedSum
(
cyclicACMIInterface_.owner(),
work,
pnf, // result
solveScalarField::null()
);
if (cyclicACMIInterface_.owner()) // Add result using coefficients
{ this->addToInternalField(result, !add, faceCells, coeffs, pnf);
pnf = cyclicACMIInterface_.AMI().interpolateToSource(pnf);
} }
else else
{ {
pnf = cyclicACMIInterface_.neighbPatch().AMI().interpolateToTarget(pnf); // Get neighbouring field
} const labelList& nbrFaceCells =
lduAddr.patchAddr(cyclicACMIInterface_.neighbPatchID());
const labelUList& faceCells = lduAddr.patchAddr(patchId); solveScalarField pnf(psiInternal, nbrFaceCells);
// Transform according to the transformation tensors
transformCoupleField(pnf, cmpt);
this->addToInternalField(result, !add, faceCells, coeffs, pnf); if (cyclicACMIInterface_.owner())
{
pnf = AMI.interpolateToSource(pnf);
}
else
{
pnf = AMI.interpolateToTarget(pnf);
}
const labelUList& faceCells = lduAddr.patchAddr(patchId);
this->addToInternalField(result, !add, faceCells, coeffs, pnf);
}
} }
......
...@@ -69,6 +69,21 @@ class cyclicACMIGAMGInterfaceField ...@@ -69,6 +69,21 @@ class cyclicACMIGAMGInterfaceField
int rank_; int rank_;
// Sending and receiving (distributed AMI)
//- Current range of send requests (non-blocking)
mutable labelRange sendRequests_;
//- Current range of recv requests (non-blocking)
mutable labelRange recvRequests_;
//- Scalar send buffers
mutable PtrList<List<solveScalar>> scalarSendBufs_;
//- Scalar receive buffers
mutable PtrList<List<solveScalar>> scalarRecvBufs_;
// Private Member Functions // Private Member Functions
//- No copy construct //- No copy construct
...@@ -139,7 +154,7 @@ public: ...@@ -139,7 +154,7 @@ public:
//- Destructor //- Destructor
virtual ~cyclicACMIGAMGInterfaceField(); virtual ~cyclicACMIGAMGInterfaceField() = default;
// Member Functions // Member Functions
...@@ -155,6 +170,22 @@ public: ...@@ -155,6 +170,22 @@ public:
// Interface matrix update // Interface matrix update
//- Are all (receive) data available?
virtual bool ready() const;
//- Initialise neighbour matrix update
virtual void initInterfaceMatrixUpdate
(
solveScalarField& result,
const bool add,
const lduAddressing& lduAddr,
const label patchId,
const solveScalarField& psiInternal,
const scalarField& coeffs,
const direction cmpt,
const Pstream::commsTypes commsType
) const;
//- Update result field based on interface functionality //- Update result field based on interface functionality
virtual void updateInterfaceMatrix virtual void updateInterfaceMatrix
( (
......
...@@ -67,7 +67,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField ...@@ -67,7 +67,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, fineInterface), GAMGInterfaceField(GAMGCp, fineInterface),
cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)), cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
doTransform_(false), doTransform_(false),
rank_(0) rank_(0),
sendRequests_(0),
recvRequests_(0)
{ {
const cyclicAMILduInterfaceField& p = const cyclicAMILduInterfaceField& p =
refCast<const cyclicAMILduInterfaceField>(fineInterface); refCast<const cyclicAMILduInterfaceField>(fineInterface);
...@@ -87,7 +89,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField ...@@ -87,7 +89,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, doTransform, rank), GAMGInterfaceField(GAMGCp, doTransform, rank),
cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)), cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
doTransform_(doTransform), doTransform_(doTransform),
rank_(rank) rank_(rank),
sendRequests_(0),
recvRequests_(0)
{} {}
...@@ -100,7 +104,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField ...@@ -100,7 +104,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, is), GAMGInterfaceField(GAMGCp, is),
cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)), cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
doTransform_(readBool(is)), doTransform_(readBool(is)),
rank_(readLabel(is)) rank_(readLabel(is)),
sendRequests_(0),
recvRequests_(0)
{} {}
...@@ -114,7 +120,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField ...@@ -114,7 +120,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, local), GAMGInterfaceField(GAMGCp, local),
cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)), cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
doTransform_(false), doTransform_(false),
rank_(0) rank_(0),
sendRequests_(0), // assume no requests in flight for input field
recvRequests_(0)
{ {
const auto& p = refCast<const cyclicAMILduInterfaceField>(local); const auto& p = refCast<const cyclicAMILduInterfaceField>(local);
...@@ -123,15 +131,41 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField ...@@ -123,15 +131,41 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
} }
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::cyclicAMIGAMGInterfaceField::~cyclicAMIGAMGInterfaceField() bool Foam::cyclicAMIGAMGInterfaceField::ready() const
{} {
if
(
UPstream::finishedRequests
(
recvRequests_.start(),
recvRequests_.size()
)
)
{
recvRequests_.clear();
if
(
UPstream::finishedRequests
(
sendRequests_.start(),
sendRequests_.size()
)
)
{
sendRequests_.clear();
}
return true;
}
return false;
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix void Foam::cyclicAMIGAMGInterfaceField::initInterfaceMatrixUpdate
( (
solveScalarField& result, solveScalarField& result,
const bool add, const bool add,
...@@ -140,50 +174,162 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix ...@@ -140,50 +174,162 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
const solveScalarField& psiInternal, const solveScalarField& psiInternal,
const scalarField& coeffs, const scalarField& coeffs,
const direction cmpt, const direction cmpt,
const Pstream::commsTypes const Pstream::commsTypes commsType
) const ) const
{ {
// Get neighbouring field const auto& AMI =
(
cyclicAMIInterface_.owner()
? cyclicAMIInterface_.AMI()
: cyclicAMIInterface_.neighbPatch().AMI()
);
const label oldWarnComm = UPstream::warnComm; if (AMI.distributed())
{
//DebugPout<< "cyclicAMIFvPatchField::initInterfaceMatrixUpdate() :"
// << " interface:" << cyclicAMIInterface_.index()
// << " size:" << cyclicAMIInterface_.size()
// << " owner:" << cyclicAMIInterface_.owner()
// << " AMI distributed:" << AMI.distributed()
// << " AMI low-weight:" << AMI.applyLowWeightCorrection()
// << endl;
// Start sending
if (commsType != UPstream::commsTypes::nonBlocking)
{
FatalErrorInFunction
<< "Can only evaluate distributed AMI with nonBlocking"
<< exit(FatalError);
}
// Get neighbouring field
const labelList& nbrFaceCells =
lduAddr.patchAddr(cyclicAMIInterface_.neighbPatchID());
solveScalarField pnf(psiInternal, nbrFaceCells);
// Transform according to the transformation tensors
transformCoupleField(pnf, cmpt);
const auto& map =
(
cyclicAMIInterface_.owner()
? AMI.tgtMap()
: AMI.srcMap()
);
const labelList& nbrFaceCells = // Insert send/receive requests (non-blocking). See e.g.
lduAddr.patchAddr // cyclicAMIPolyPatchTemplates.C
const label oldWarnComm = UPstream::warnComm;
UPstream::warnComm = AMI.comm();
map.send
( (
cyclicAMIInterface_.neighbPatchID() pnf,
sendRequests_,
scalarSendBufs_,
recvRequests_,
scalarRecvBufs_
); );
UPstream::warnComm = oldWarnComm;
}
}
void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
(
solveScalarField& result,
const bool add,
const lduAddressing& lduAddr,
const label patchId,
const solveScalarField& psiInternal,
const scalarField& coeffs,
const direction cmpt,
const Pstream::commsTypes commsType
) const
{
const labelUList& faceCells = lduAddr.patchAddr(patchId);
const auto& AMI =
(
cyclicAMIInterface_.owner()
? cyclicAMIInterface_.AMI()
: cyclicAMIInterface_.neighbPatch().AMI()
);
solveScalarField pnf(psiInternal, nbrFaceCells); solveScalarField defaultValues;
if (AMI.applyLowWeightCorrection())
{
defaultValues = solveScalarField(psiInternal, faceCells);
}
// Transform according to the transformation tensors //DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :"
transformCoupleField(pnf, cmpt); // << " interface:" << cyclicAMIInterface_.index()
// << " size:" << cyclicAMIInterface_.size()
// << " owner:" << cyclicAMIInterface_.owner()
// << " AMI distributed:" << AMI.distributed()
// << " AMI low-weight:" << AMI.applyLowWeightCorrection()
// << endl;
if (cyclicAMIInterface_.owner()) if (AMI.distributed())
{ {
const auto& AMI = cyclicAMIInterface_.AMI(); if (commsType != UPstream::commsTypes::nonBlocking)
{
FatalErrorInFunction
<< "Can only evaluate distributed AMI with nonBlocking"
<< exit(FatalError);
}
const auto& map =
(
cyclicAMIInterface_.owner()
? AMI.tgtMap()
: AMI.srcMap()
);
// Switch on warning if using wrong communicator. Can be removed if // Receive (= copy) data from buffers into work. TBD: receive directly
// sure all is correct // into slices of work.
UPstream::warnComm = AMI.comm(); solveScalarField work;
map.receive(recvRequests_, scalarRecvBufs_, work);
pnf = AMI.interpolateToSource(pnf); solveScalarField pnf(faceCells.size(), Zero);
AMI.weightedSum
(
cyclicAMIInterface_.owner(),
work,
pnf, // result
defaultValues
);
// Add result using coefficients
this->addToInternalField(result, !add, faceCells, coeffs, pnf);
} }
else else
{ {
const auto& AMI = cyclicAMIInterface_.neighbPatch().AMI(); // Get neighbouring field
const labelList& nbrFaceCells =
lduAddr.patchAddr(cyclicAMIInterface_.neighbPatchID());
solveScalarField work(psiInternal, nbrFaceCells);
// Transform according to the transformation tensors
transformCoupleField(work, cmpt);
// Switch on warning if using wrong communicator. Can be removed if // Switch on warning if using wrong communicator. Can be removed if
// sure all is correct // sure all is correct
UPstream::warnComm = AMI.comm(); UPstream::warnComm = AMI.comm();
pnf = AMI.interpolateToTarget(pnf); solveScalarField pnf(faceCells.size(), Zero);
} AMI.weightedSum
(
const labelUList& faceCells = lduAddr.patchAddr(patchId); cyclicAMIInterface_.owner(),
work,
this->addToInternalField(result, !add, faceCells, coeffs, pnf); pnf, // result
defaultValues
);
UPstream::warnComm = oldWarnComm; // Add result using coefficients
this->addToInternalField(result, !add, faceCells, coeffs, pnf);
}
} }
......
...@@ -68,6 +68,21 @@ class cyclicAMIGAMGInterfaceField ...@@ -68,6 +68,21 @@ class cyclicAMIGAMGInterfaceField
int rank_; int rank_;
// Sending and receiving (distributed AMI)
//- Current range of send requests (non-blocking)
mutable labelRange sendRequests_;
//- Current range of recv requests (non-blocking)
mutable labelRange recvRequests_;
//- Scalar send buffers
mutable PtrList<List<solveScalar>> scalarSendBufs_;
//- Scalar receive buffers
mutable PtrList<List<solveScalar>> scalarRecvBufs_;
// Private Member Functions // Private Member Functions
//- No copy construct //- No copy construct
...@@ -138,7 +153,7 @@ public: ...@@ -138,7 +153,7 @@ public:
//- Destructor //- Destructor
virtual ~cyclicAMIGAMGInterfaceField(); virtual ~cyclicAMIGAMGInterfaceField() = default;
// Member Functions // Member Functions
...@@ -154,6 +169,22 @@ public: ...@@ -154,6 +169,22 @@ public:
// Interface matrix update // Interface matrix update
//- Are all (receive) data available?
virtual bool ready() const;
//- Initialise neighbour matrix update
virtual void initInterfaceMatrixUpdate
(
solveScalarField& result,
const bool add,
const lduAddressing& lduAddr,
const label patchId,
const solveScalarField& psiInternal,
const scalarField& coeffs,
const direction cmpt,
const Pstream::commsTypes commsType
) const;
//- Update result field based on interface functionality //- Update result field based on interface functionality
virtual void updateInterfaceMatrix virtual void updateInterfaceMatrix
( (
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment