meshToMeshParallelOps.C 28 KB
Newer Older
1
2
3
4
/*---------------------------------------------------------------------------*\
  =========                 |
  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
   \\    /   O peration     |
OpenFOAM bot's avatar
OpenFOAM bot committed
5
6
7
8
    \\  /    A nd           | Copyright (C) 2015-2018 OpenCFD Ltd.
     \\/     M anipulation  |
-------------------------------------------------------------------------------
                            | Copyright (C) 2012-2017 OpenFOAM Foundation
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
-------------------------------------------------------------------------------
License
    This file is part of OpenFOAM.

    OpenFOAM is free software: you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    for more details.

    You should have received a copy of the GNU General Public License
    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.

\*---------------------------------------------------------------------------*/

28
#include "meshToMesh.H"
29
30
31
32
33
34
#include "OFstream.H"
#include "Time.H"
#include "globalIndex.H"
#include "mergePoints.H"
#include "processorPolyPatch.H"
#include "SubField.H"
35
#include "AABBTree.H"
36
#include "cellBox.H"
37
38
39

// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //

40
Foam::label Foam::meshToMesh::calcDistribution
41
42
43
44
45
(
    const polyMesh& src,
    const polyMesh& tgt
) const
{
46
    label proci = 0;
47
48
49

    if (Pstream::parRun())
    {
50
        List<label> cellsPresentOnProc(Pstream::nProcs(), Zero);
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
        if ((src.nCells() > 0) || (tgt.nCells() > 0))
        {
            cellsPresentOnProc[Pstream::myProcNo()] = 1;
        }
        else
        {
            cellsPresentOnProc[Pstream::myProcNo()] = 0;
        }

        Pstream::gatherList(cellsPresentOnProc);
        Pstream::scatterList(cellsPresentOnProc);

        label nHaveCells = sum(cellsPresentOnProc);

        if (nHaveCells > 1)
        {
67
            proci = -1;
68
69
            if (debug)
            {
70
                InfoInFunction
71
72
73
74
75
                    << "Meshes split across multiple processors" << endl;
            }
        }
        else if (nHaveCells == 1)
        {
76
            proci = cellsPresentOnProc.find(1);
77
78
            if (debug)
            {
79
                InfoInFunction
80
                    << "Meshes local to processor" << proci << endl;
81
82
83
84
            }
        }
    }

85
    return proci;
86
87
88
}


89
Foam::label Foam::meshToMesh::calcOverlappingProcs
90
(
91
    const List<treeBoundBoxList>& procBb,
92
93
94
95
96
97
98
99
    const boundBox& bb,
    boolList& overlaps
) const
{
    overlaps = false;

    label nOverlaps = 0;

100
    forAll(procBb, proci)
101
    {
102
        const treeBoundBoxList& bbp = procBb[proci];
103

104
        for (const treeBoundBox& b : bbp)
105
        {
106
            if (b.overlaps(bb))
107
            {
108
                overlaps[proci] = true;
109
                ++nOverlaps;
110
111
                break;
            }
112
113
114
115
116
117
118
        }
    }

    return nOverlaps;
}


119
Foam::autoPtr<Foam::mapDistribute> Foam::meshToMesh::calcProcMap
120
121
122
123
124
(
    const polyMesh& src,
    const polyMesh& tgt
) const
{
125
    switch (procMapMethod_)
126
    {
127
128
129
130
131
132
133
        case procMapMethod::pmLOD:
        {
            Info<< "meshToMesh: Using processorLOD method" << endl;

            // Create processor map of overlapping faces. This map gets
            // (possibly remote) cells from the tgt mesh such that they
            // (together) cover all of the src mesh
134
135
            const label nGlobalSrcCells = src.globalData().nTotalCells();
            const label cellsPerBox = max(1, 0.001*nGlobalSrcCells);
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
            typename processorLODs::cellBox boxLOD
            (
                src.cells(),
                src.faces(),
                src.points(),
                tgt.cells(),
                tgt.faces(),
                tgt.points(),
                cellsPerBox,
                src.nCells()
            );

            return boxLOD.map();
            break;
        }
        default:
        {
            Info<< "meshToMesh: Using AABBTree method" << endl;
154

155
156
            // get decomposition of cells on src mesh
            List<treeBoundBoxList> procBb(Pstream::nProcs());
157

158
159
160
161
162
163
164
165
166
167
168
169
170
            if (src.nCells() > 0)
            {
                procBb[Pstream::myProcNo()] = AABBTree<labelList>
                (
                    src.cellPoints(),
                    src.points(),
                    false
                ).boundBoxes();
            }
            else
            {
                procBb[Pstream::myProcNo()] = treeBoundBoxList();
            }
171
172


173
174
            Pstream::gatherList(procBb);
            Pstream::scatterList(procBb);
175
176


177
178
179
180
181
182
183
184
185
186
            if (debug)
            {
                InfoInFunction
                    << "Determining extent of src mesh per processor:" << nl
                    << "\tproc\tbb" << endl;
                forAll(procBb, proci)
                {
                    Info<< '\t' << proci << '\t' << procBb[proci] << endl;
                }
            }
187
188


189
190
191
192
            // determine which cells of tgt mesh overlaps src mesh per proc
            const cellList& cells = tgt.cells();
            const faceList& faces = tgt.faces();
            const pointField& points = tgt.points();
193

194
            labelListList sendMap;
195
196

            {
197
198
199
200
201
                // per processor indices into all segments to send
                List<DynamicList<label>> dynSendMap(Pstream::nProcs());
                label iniSize = floor(tgt.nCells()/Pstream::nProcs());

                forAll(dynSendMap, proci)
202
                {
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
                    dynSendMap[proci].setCapacity(iniSize);
                }

                // work array - whether src processor bb overlaps the tgt cell
                // bounds
                boolList procBbOverlaps(Pstream::nProcs());
                forAll(cells, celli)
                {
                    const cell& c = cells[celli];

                    // determine bounding box of tgt cell
                    boundBox cellBb(boundBox::invertedBox);
                    forAll(c, facei)
                    {
                        const face& f = faces[c[facei]];
                        forAll(f, fp)
                        {
                            cellBb.add(points, f);
                        }
                    }

                    // find the overlapping tgt cells on each src processor
                    (void)calcOverlappingProcs(procBb, cellBb, procBbOverlaps);

                    forAll(procBbOverlaps, proci)
                    {
                        if (procBbOverlaps[proci])
                        {
                            dynSendMap[proci].append(celli);
                        }
                    }
234
235
                }

236
237
238
239
240
241
242
                // convert dynamicList to labelList
                sendMap.setSize(Pstream::nProcs());
                forAll(sendMap, proci)
                {
                    sendMap[proci].transfer(dynSendMap[proci]);
                }
            }
243

244
245
            // debug printing
            if (debug)
246
            {
247
248
249
250
                Pout<< "Of my " << cells.size()
                    << " target cells I need to send to:" << nl
                    << "\tproc\tcells" << endl;
                forAll(sendMap, proci)
251
                {
252
253
                    Pout<< '\t' << proci << '\t' << sendMap[proci].size()
                        << endl;
254
255
256
257
                }
            }


258
259
260
261
262
263
264
265
266
267
            // send over how many tgt cells I need to receive from each
            // processor
            labelListList sendSizes(Pstream::nProcs());
            sendSizes[Pstream::myProcNo()].setSize(Pstream::nProcs());
            forAll(sendMap, proci)
            {
                sendSizes[Pstream::myProcNo()][proci] = sendMap[proci].size();
            }
            Pstream::gatherList(sendSizes);
            Pstream::scatterList(sendSizes);
268
269


270
271
            // determine order of receiving
            labelListList constructMap(Pstream::nProcs());
272

273
274
275
276
277
278
279
            label segmentI = 0;
            forAll(constructMap, proci)
            {
                // what I need to receive is what other processor is sending
                // to me
                label nRecv = sendSizes[proci][Pstream::myProcNo()];
                constructMap[proci].setSize(nRecv);
280

281
282
283
284
285
                for (label i = 0; i < nRecv; i++)
                {
                    constructMap[proci][i] = segmentI++;
                }
            }
286

287
288
289
290
291
292
            return autoPtr<mapDistribute>::New
            (
                segmentI,       // size after construction
                std::move(sendMap),
                std::move(constructMap)
            );
293

294
            break;
295
296
297
298
299
        }
    }
}


300
void Foam::meshToMesh::distributeCells
301
302
303
304
305
306
307
308
309
310
311
312
313
314
(
    const mapDistribute& map,
    const polyMesh& tgtMesh,
    const globalIndex& globalI,
    List<pointField>& points,
    List<label>& nInternalFaces,
    List<faceList>& faces,
    List<labelList>& faceOwner,
    List<labelList>& faceNeighbour,
    List<labelList>& cellIDs,
    List<labelList>& nbrProcIDs,
    List<labelList>& procLocalFaceIDs
) const
{
315
    PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335

    points.setSize(Pstream::nProcs());
    nInternalFaces.setSize(Pstream::nProcs(), 0);
    faces.setSize(Pstream::nProcs());
    faceOwner.setSize(Pstream::nProcs());
    faceNeighbour.setSize(Pstream::nProcs());
    cellIDs.setSize(Pstream::nProcs());

    nbrProcIDs.setSize(Pstream::nProcs());;
    procLocalFaceIDs.setSize(Pstream::nProcs());;


    for (label domain = 0; domain < Pstream::nProcs(); domain++)
    {
        const labelList& sendElems = map.subMap()[domain];

        if (sendElems.size())
        {
            // reverse cell map
            labelList reverseCellMap(tgtMesh.nCells(), -1);
336
            forAll(sendElems, subCelli)
337
            {
338
                reverseCellMap[sendElems[subCelli]] = subCelli;
339
340
341
342
343
344
345
346
347
348
349
350
            }

            DynamicList<face> subFaces(tgtMesh.nFaces());
            DynamicList<label> subFaceOwner(tgtMesh.nFaces());
            DynamicList<label> subFaceNeighbour(tgtMesh.nFaces());

            DynamicList<label> subNbrProcIDs(tgtMesh.nFaces());
            DynamicList<label> subProcLocalFaceIDs(tgtMesh.nFaces());

            label nInternal = 0;

            // internal faces
351
            forAll(tgtMesh.faceNeighbour(), facei)
352
            {
353
354
                label own = tgtMesh.faceOwner()[facei];
                label nbr = tgtMesh.faceNeighbour()[facei];
355
356
357
358
359
360
361
362
363
                label subOwn = reverseCellMap[own];
                label subNbr = reverseCellMap[nbr];

                if (subOwn != -1 && subNbr != -1)
                {
                    nInternal++;

                    if (subOwn < subNbr)
                    {
364
                        subFaces.append(tgtMesh.faces()[facei]);
365
366
367
368
369
370
371
                        subFaceOwner.append(subOwn);
                        subFaceNeighbour.append(subNbr);
                        subNbrProcIDs.append(-1);
                        subProcLocalFaceIDs.append(-1);
                    }
                    else
                    {
372
                        subFaces.append(tgtMesh.faces()[facei].reverseFace());
373
374
375
376
377
378
379
380
381
                        subFaceOwner.append(subNbr);
                        subFaceNeighbour.append(subOwn);
                        subNbrProcIDs.append(-1);
                        subProcLocalFaceIDs.append(-1);
                    }
                }
            }

            // boundary faces for new region
382
            forAll(tgtMesh.faceNeighbour(), facei)
383
            {
384
385
                label own = tgtMesh.faceOwner()[facei];
                label nbr = tgtMesh.faceNeighbour()[facei];
386
387
388
389
390
                label subOwn = reverseCellMap[own];
                label subNbr = reverseCellMap[nbr];

                if (subOwn != -1 && subNbr == -1)
                {
391
                    subFaces.append(tgtMesh.faces()[facei]);
392
393
394
395
396
397
398
                    subFaceOwner.append(subOwn);
                    subFaceNeighbour.append(subNbr);
                    subNbrProcIDs.append(-1);
                    subProcLocalFaceIDs.append(-1);
                }
                else if (subOwn == -1 && subNbr != -1)
                {
399
                    subFaces.append(tgtMesh.faces()[facei].reverseFace());
400
401
402
403
404
405
406
407
                    subFaceOwner.append(subNbr);
                    subFaceNeighbour.append(subOwn);
                    subNbrProcIDs.append(-1);
                    subProcLocalFaceIDs.append(-1);
                }
            }

            // boundary faces of existing region
408
            forAll(tgtMesh.boundaryMesh(), patchi)
409
            {
410
                const polyPatch& pp = tgtMesh.boundaryMesh()[patchi];
411

412
                label nbrProci = -1;
413
414
415
416
417
418
419

                // store info for faces on processor patches
                if (isA<processorPolyPatch>(pp))
                {
                    const processorPolyPatch& ppp =
                        dynamic_cast<const processorPolyPatch&>(pp);

420
                    nbrProci = ppp.neighbProcNo();
421
422
423
424
                }

                forAll(pp, i)
                {
425
426
                    label facei = pp.start() + i;
                    label own = tgtMesh.faceOwner()[facei];
427
428
429

                    if (reverseCellMap[own] != -1)
                    {
430
                        subFaces.append(tgtMesh.faces()[facei]);
431
432
                        subFaceOwner.append(reverseCellMap[own]);
                        subFaceNeighbour.append(-1);
433
                        subNbrProcIDs.append(nbrProci);
434
435
436
437
438
439
440
441
                        subProcLocalFaceIDs.append(i);
                    }
                }
            }

            // reverse point map
            labelList reversePointMap(tgtMesh.nPoints(), -1);
            DynamicList<point> subPoints(tgtMesh.nPoints());
442
            forAll(subFaces, subFacei)
443
            {
444
                face& f = subFaces[subFacei];
445
446
                forAll(f, fp)
                {
447
448
                    label pointi = f[fp];
                    if (reversePointMap[pointi] == -1)
449
                    {
450
451
                        reversePointMap[pointi] = subPoints.size();
                        subPoints.append(tgtMesh.points()[pointi]);
452
453
                    }

454
                    f[fp] = reversePointMap[pointi];
455
456
457
458
                }
            }

            // tgt cells into global numbering
459
460
461
            labelList globalElems(globalI.toGlobal(sendElems));

            if (debug > 1)
462
            {
463
                forAll(sendElems, i)
464
465
466
                {
                    Pout<< "tgtProc:" << Pstream::myProcNo()
                        << " sending tgt cell " << sendElems[i]
467
                        << "[" << globalElems[i] << "]"
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
                        << " to srcProc " << domain << endl;
                }
            }

            // pass data
            if (domain == Pstream::myProcNo())
            {
                // allocate my own data
                points[Pstream::myProcNo()] = subPoints;
                nInternalFaces[Pstream::myProcNo()] = nInternal;
                faces[Pstream::myProcNo()] = subFaces;
                faceOwner[Pstream::myProcNo()] = subFaceOwner;
                faceNeighbour[Pstream::myProcNo()] = subFaceNeighbour;
                cellIDs[Pstream::myProcNo()] = globalElems;
                nbrProcIDs[Pstream::myProcNo()] = subNbrProcIDs;
                procLocalFaceIDs[Pstream::myProcNo()] = subProcLocalFaceIDs;
            }
            else
            {
                // send data to other processor domains
                UOPstream toDomain(domain, pBufs);

                toDomain
                    << subPoints
                    << nInternal
                    << subFaces
                    << subFaceOwner
                    << subFaceNeighbour
                    << globalElems
                    << subNbrProcIDs
                    << subProcLocalFaceIDs;
            }
        }
    }

    // Start receiving
    pBufs.finishedSends();

    // Consume
    for (label domain = 0; domain < Pstream::nProcs(); domain++)
    {
        const labelList& recvElems = map.constructMap()[domain];

        if (domain != Pstream::myProcNo() && recvElems.size())
        {
            UIPstream str(domain, pBufs);

            str >> points[domain]
                >> nInternalFaces[domain]
                >> faces[domain]
                >> faceOwner[domain]
                >> faceNeighbour[domain]
                >> cellIDs[domain]
                >> nbrProcIDs[domain]
                >> procLocalFaceIDs[domain];
        }

        if (debug)
        {
            Pout<< "Target mesh send sizes[" << domain << "]"
                << ": points="<< points[domain].size()
                << ", faces=" << faces[domain].size()
                << ", nInternalFaces=" << nInternalFaces[domain]
                << ", faceOwn=" << faceOwner[domain].size()
                << ", faceNbr=" << faceNeighbour[domain].size()
                << ", cellIDs=" << cellIDs[domain].size() << endl;
        }
    }
}


539
void Foam::meshToMesh::distributeAndMergeCells
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
(
    const mapDistribute& map,
    const polyMesh& tgt,
    const globalIndex& globalI,
    pointField& tgtPoints,
    faceList& tgtFaces,
    labelList& tgtFaceOwners,
    labelList& tgtFaceNeighbours,
    labelList& tgtCellIDs
) const
{
    // Exchange per-processor data
    List<pointField> allPoints;
    List<label> allNInternalFaces;
    List<faceList> allFaces;
    List<labelList> allFaceOwners;
    List<labelList> allFaceNeighbours;
    List<labelList> allTgtCellIDs;

    // Per target mesh face the neighbouring proc and index in
    // processor patch (all -1 for normal boundary face)
    List<labelList> allNbrProcIDs;
    List<labelList> allProcLocalFaceIDs;

    distributeCells
    (
        map,
        tgt,
        globalI,
        allPoints,
        allNInternalFaces,
        allFaces,
        allFaceOwners,
        allFaceNeighbours,
        allTgtCellIDs,
        allNbrProcIDs,
        allProcLocalFaceIDs
    );

    // Convert lists into format that can be used to generate a valid polyMesh
    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    //
    // Points and cells are collected into single flat lists:
    // - i.e. proc0, proc1 ... procN
    //
    // Faces need to be sorted after collection to that internal faces are
    // contiguous, followed by all boundary faces
    //
    // Processor patch faces between included cells on neighbouring processors
    // are converted into internal faces
    //
    // Face list structure:
    // - Per processor:
    //   - internal faces
    //   - processor faces that have been converted into internal faces
    // - Followed by all boundary faces
    //   - from 'normal' boundary faces
    //   - from singularly-sided processor patch faces


    // Number of internal+coupled faces
    labelList allNIntCoupledFaces(allNInternalFaces);

    // Starting offset for points
    label nPoints = 0;
605
    labelList pointOffset(Pstream::nProcs(), Zero);
606
    forAll(allPoints, proci)
607
    {
608
609
        pointOffset[proci] = nPoints;
        nPoints += allPoints[proci].size();
610
611
612
613
    }

    // Starting offset for cells
    label nCells = 0;
614
    labelList cellOffset(Pstream::nProcs(), Zero);
615
    forAll(allTgtCellIDs, proci)
616
    {
617
618
        cellOffset[proci] = nCells;
        nCells += allTgtCellIDs[proci].size();
619
620
621
622
    }

    // Count any coupled faces
    typedef FixedList<label, 3> label3;
623
    typedef HashTable<label, label3, label3::Hash<>> procCoupleInfo;
624
625
    procCoupleInfo procFaceToGlobalCell;

626
    forAll(allNbrProcIDs, proci)
627
    {
628
629
        const labelList& nbrProci = allNbrProcIDs[proci];
        const labelList& localFacei = allProcLocalFaceIDs[proci];
630

631
        forAll(nbrProci, i)
632
        {
633
            if (nbrProci[i] != -1 && localFacei[i] != -1)
634
635
            {
                label3 key;
636
637
638
                key[0] = min(proci, nbrProci[i]);
                key[1] = max(proci, nbrProci[i]);
                key[2] = localFacei[i];
639

640
                const auto fnd = procFaceToGlobalCell.cfind(key);
641

642
                if (!fnd.found())
643
644
645
646
647
                {
                    procFaceToGlobalCell.insert(key, -1);
                }
                else
                {
648
                    if (debug > 1)
649
650
651
652
653
654
                    {
                        Pout<< "Additional internal face between procs:"
                            << key[0] << " and " << key[1]
                            << " across local face " << key[2] << endl;
                    }

655
                    allNIntCoupledFaces[proci]++;
656
657
658
659
660
661
662
663
664
                }
            }
        }
    }


    // Starting offset for internal faces
    label nIntFaces = 0;
    label nFacesTotal = 0;
665
    labelList internalFaceOffset(Pstream::nProcs(), Zero);
666
    forAll(allNIntCoupledFaces, proci)
667
668
    {
        label nCoupledFaces =
669
            allNIntCoupledFaces[proci] - allNInternalFaces[proci];
670

671
672
673
        internalFaceOffset[proci] = nIntFaces;
        nIntFaces += allNIntCoupledFaces[proci];
        nFacesTotal += allFaceOwners[proci].size() - nCoupledFaces;
674
675
676
677
678
679
680
681
682
    }

    tgtPoints.setSize(nPoints);
    tgtFaces.setSize(nFacesTotal);
    tgtFaceOwners.setSize(nFacesTotal);
    tgtFaceNeighbours.setSize(nFacesTotal);
    tgtCellIDs.setSize(nCells);

    // Insert points
683
    forAll(allPoints, proci)
684
    {
685
686
        const pointField& pts = allPoints[proci];
        SubList<point>(tgtPoints, pts.size(), pointOffset[proci]) = pts;
687
688
689
    }

    // Insert cellIDs
690
    forAll(allTgtCellIDs, proci)
691
    {
692
693
        const labelList& cellIDs = allTgtCellIDs[proci];
        SubList<label>(tgtCellIDs, cellIDs.size(), cellOffset[proci]) = cellIDs;
694
695
696
697
    }


    // Insert internal faces (from internal faces)
698
    forAll(allFaces, proci)
699
    {
700
701
702
        const faceList& fcs = allFaces[proci];
        const labelList& faceOs = allFaceOwners[proci];
        const labelList& faceNs = allFaceNeighbours[proci];
703
704
705
706

        SubList<face> slice
        (
            tgtFaces,
707
708
            allNInternalFaces[proci],
            internalFaceOffset[proci]
709
        );
710
        slice = SubList<face>(fcs, allNInternalFaces[proci]);
711
712
        forAll(slice, i)
        {
713
            add(slice[i], pointOffset[proci]);
714
715
716
717
718
        }

        SubField<label> ownSlice
        (
            tgtFaceOwners,
719
720
            allNInternalFaces[proci],
            internalFaceOffset[proci]
721
        );
722
723
        ownSlice = SubField<label>(faceOs, allNInternalFaces[proci]);
        add(ownSlice, cellOffset[proci]);
724
725
726
727

        SubField<label> nbrSlice
        (
            tgtFaceNeighbours,
728
729
            allNInternalFaces[proci],
            internalFaceOffset[proci]
730
        );
731
732
        nbrSlice = SubField<label>(faceNs, allNInternalFaces[proci]);
        add(nbrSlice, cellOffset[proci]);
733

734
        internalFaceOffset[proci] += allNInternalFaces[proci];
735
736
737
738
    }


    // Insert internal faces (from coupled face-pairs)
739
    forAll(allNbrProcIDs, proci)
740
    {
741
742
743
744
        const labelList& nbrProci = allNbrProcIDs[proci];
        const labelList& localFacei = allProcLocalFaceIDs[proci];
        const labelList& faceOs = allFaceOwners[proci];
        const faceList& fcs = allFaces[proci];
745

746
        forAll(nbrProci, i)
747
        {
748
            if (nbrProci[i] != -1 && localFacei[i] != -1)
749
750
            {
                label3 key;
751
752
753
                key[0] = min(proci, nbrProci[i]);
                key[1] = max(proci, nbrProci[i]);
                key[2] = localFacei[i];
754

755
                auto fnd = procFaceToGlobalCell.find(key);
756

757
                if (fnd.found())
758
                {
759
760
                    label tgtFacei = fnd();
                    if (tgtFacei == -1)
761
762
                    {
                        // on first visit store the new cell on this side
763
                        fnd() = cellOffset[proci] + faceOs[i];
764
765
766
767
                    }
                    else
                    {
                        // get owner and neighbour in new cell numbering
768
                        label newOwn = cellOffset[proci] + faceOs[i];
769
                        label newNbr = fnd();
770
                        label tgtFacei = internalFaceOffset[proci]++;
771

772
                        if (debug > 1)
773
                        {
774
775
                            Pout<< "    proc " << proci
                                << "\tinserting face:" << tgtFacei
776
777
778
779
780
781
782
783
                                << " connection between owner " << newOwn
                                << " and neighbour " << newNbr
                                << endl;
                        }

                        if (newOwn < newNbr)
                        {
                            // we have correct orientation
784
785
786
                            tgtFaces[tgtFacei] = fcs[i];
                            tgtFaceOwners[tgtFacei] = newOwn;
                            tgtFaceNeighbours[tgtFacei] = newNbr;
787
788
789
790
                        }
                        else
                        {
                            // reverse orientation
791
792
793
                            tgtFaces[tgtFacei] = fcs[i].reverseFace();
                            tgtFaceOwners[tgtFacei] = newNbr;
                            tgtFaceNeighbours[tgtFacei] = newOwn;
794
795
                        }

796
                        add(tgtFaces[tgtFacei], pointOffset[proci]);
797
798
799
800
801
802
803
804
805
806

                        // mark with unique value
                        fnd() = -2;
                    }
                }
            }
        }
    }


807
    forAll(allNbrProcIDs, proci)
808
    {
809
810
811
812
813
        const labelList& nbrProci = allNbrProcIDs[proci];
        const labelList& localFacei = allProcLocalFaceIDs[proci];
        const labelList& faceOs = allFaceOwners[proci];
        const labelList& faceNs = allFaceNeighbours[proci];
        const faceList& fcs = allFaces[proci];
814

815
        forAll(nbrProci, i)
816
817
        {
            // coupled boundary face
818
            if (nbrProci[i] != -1 && localFacei[i] != -1)
819
820
            {
                label3 key;
821
822
823
                key[0] = min(proci, nbrProci[i]);
                key[1] = max(proci, nbrProci[i]);
                key[2] = localFacei[i];
824

825
                label tgtFacei = procFaceToGlobalCell[key];
826

827
                if (tgtFacei == -1)
828
                {
829
                    FatalErrorInFunction
830
831
832
                        << "Unvisited " << key
                        << abort(FatalError);
                }
833
                else if (tgtFacei != -2)
834
                {
835
836
                    label newOwn = cellOffset[proci] + faceOs[i];
                    label tgtFacei = nIntFaces++;
837

838
                    if (debug > 1)
839
                    {
840
841
                        Pout<< "    proc " << proci
                            << "\tinserting boundary face:" << tgtFacei
842
843
844
845
                            << " from coupled face " << key
                            << endl;
                    }

846
847
                    tgtFaces[tgtFacei] = fcs[i];
                    add(tgtFaces[tgtFacei], pointOffset[proci]);
848

849
850
                    tgtFaceOwners[tgtFacei] = newOwn;
                    tgtFaceNeighbours[tgtFacei] = -1;
851
852
853
854
855
856
857
858
859
                }
            }
            // normal boundary face
            else
            {
                label own = faceOs[i];
                label nbr = faceNs[i];
                if ((own != -1) && (nbr == -1))
                {
860
861
                    label newOwn = cellOffset[proci] + faceOs[i];
                    label tgtFacei = nIntFaces++;
862

863
864
                    tgtFaces[tgtFacei] = fcs[i];
                    add(tgtFaces[tgtFacei], pointOffset[proci]);
865

866
867
                    tgtFaceOwners[tgtFacei] = newOwn;
                    tgtFaceNeighbours[tgtFacei] = -1;
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
                }
            }
        }
    }


    if (debug)
    {
        // only merging points in debug mode

        labelList oldToNew;
        pointField newTgtPoints;
        bool hasMerged = mergePoints
        (
            tgtPoints,
            SMALL,
            false,
            oldToNew,
            newTgtPoints
        );

        if (hasMerged)
        {
            if (debug)
            {
                Pout<< "Merged from " << tgtPoints.size()
                    << " down to " << newTgtPoints.size() << " points" << endl;
            }

            tgtPoints.transfer(newTgtPoints);
            forAll(tgtFaces, i)
            {
                inplaceRenumber(oldToNew, tgtFaces[i]);
            }
        }
    }
}


// ************************************************************************* //