Date: 2016-07-24T16:55:44+00:00

Actual source code: mpiadj.c
petsc-3.7.3 2016-07-24
Report Typos and Errors

      2: /*
      3:     Defines the basic matrix operations for the ADJ adjacency list matrix data-structure.
      4: */
      5: #include <../src/mat/impls/adj/mpi/mpiadj.h>    /*I "petscmat.h" I*/
      6: #include <petscsf.h>

     10: PetscErrorCode MatView_MPIAdj_ASCII(Mat A,PetscViewer viewer)
     11: {
     12:   Mat_MPIAdj        *a = (Mat_MPIAdj*)A->data;
     13:   PetscErrorCode    ierr;
     14:   PetscInt          i,j,m = A->rmap->n;
     15:   const char        *name;
     16:   PetscViewerFormat format;

     19:   PetscObjectGetName((PetscObject)A,&name);
     20:   PetscViewerGetFormat(viewer,&format);
     21:   if (format == PETSC_VIEWER_ASCII_INFO) {
     22:     return(0);
     23:   } else if (format == PETSC_VIEWER_ASCII_MATLAB) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_SUP,"MATLAB format not supported");
     24:   else {
     25:     PetscViewerASCIIUseTabs(viewer,PETSC_FALSE);
     26:     PetscViewerASCIIPushSynchronized(viewer);
     27:     for (i=0; i<m; i++) {
     28:       PetscViewerASCIISynchronizedPrintf(viewer,"row %D:",i+A->rmap->rstart);
     29:       for (j=a->i[i]; j<a->i[i+1]; j++) {
     30:         PetscViewerASCIISynchronizedPrintf(viewer," %D ",a->j[j]);
     31:       }
     32:       PetscViewerASCIISynchronizedPrintf(viewer,"\n");
     33:     }
     34:     PetscViewerASCIIUseTabs(viewer,PETSC_TRUE);
     35:     PetscViewerFlush(viewer);
     36:     PetscViewerASCIIPopSynchronized(viewer);
     37:   }
     38:   return(0);
     39: }

     43: PetscErrorCode MatView_MPIAdj(Mat A,PetscViewer viewer)
     44: {
     46:   PetscBool      iascii;

     49:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
     50:   if (iascii) {
     51:     MatView_MPIAdj_ASCII(A,viewer);
     52:   }
     53:   return(0);
     54: }

     58: PetscErrorCode MatDestroy_MPIAdj(Mat mat)
     59: {
     60:   Mat_MPIAdj     *a = (Mat_MPIAdj*)mat->data;

     64: #if defined(PETSC_USE_LOG)
     65:   PetscLogObjectState((PetscObject)mat,"Rows=%D, Cols=%D, NZ=%D",mat->rmap->n,mat->cmap->n,a->nz);
     66: #endif
     67:   PetscFree(a->diag);
     68:   if (a->freeaij) {
     69:     if (a->freeaijwithfree) {
     70:       if (a->i) free(a->i);
     71:       if (a->j) free(a->j);
     72:     } else {
     73:       PetscFree(a->i);
     74:       PetscFree(a->j);
     75:       PetscFree(a->values);
     76:     }
     77:   }
     78:   PetscFree(a->rowvalues);
     79:   PetscFree(mat->data);
     80:   PetscObjectChangeTypeName((PetscObject)mat,0);
     81:   PetscObjectComposeFunction((PetscObject)mat,"MatMPIAdjSetPreallocation_C",NULL);
     82:   PetscObjectComposeFunction((PetscObject)mat,"MatMPIAdjCreateNonemptySubcommMat_C",NULL);
     83:   return(0);
     84: }

     88: PetscErrorCode MatSetOption_MPIAdj(Mat A,MatOption op,PetscBool flg)
     89: {
     90:   Mat_MPIAdj     *a = (Mat_MPIAdj*)A->data;

     94:   switch (op) {
     95:   case MAT_SYMMETRIC:
     96:   case MAT_STRUCTURALLY_SYMMETRIC:
     97:   case MAT_HERMITIAN:
     98:     a->symmetric = flg;
     99:     break;
    100:   case MAT_SYMMETRY_ETERNAL:
    101:     break;
    102:   default:
    103:     PetscInfo1(A,"Option %s ignored\n",MatOptions[op]);
    104:     break;
    105:   }
    106:   return(0);
    107: }

    110: /*
    111:      Adds diagonal pointers to sparse matrix structure.
    112: */

    116: PetscErrorCode MatMarkDiagonal_MPIAdj(Mat A)
    117: {
    118:   Mat_MPIAdj     *a = (Mat_MPIAdj*)A->data;
    120:   PetscInt       i,j,m = A->rmap->n;

    123:   PetscMalloc1(m,&a->diag);
    124:   PetscLogObjectMemory((PetscObject)A,m*sizeof(PetscInt));
    125:   for (i=0; i<A->rmap->n; i++) {
    126:     for (j=a->i[i]; j<a->i[i+1]; j++) {
    127:       if (a->j[j] == i) {
    128:         a->diag[i] = j;
    129:         break;
    130:       }
    131:     }
    132:   }
    133:   return(0);
    134: }

    138: PetscErrorCode MatGetRow_MPIAdj(Mat A,PetscInt row,PetscInt *nz,PetscInt **idx,PetscScalar **v)
    139: {
    140:   Mat_MPIAdj *a = (Mat_MPIAdj*)A->data;

    144:   row -= A->rmap->rstart;
    145:   if (row < 0 || row >= A->rmap->n) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Row out of range");
    146:   *nz = a->i[row+1] - a->i[row];
    147:   if (v) {
    148:     PetscInt j;
    149:     if (a->rowvalues_alloc < *nz) {
    150:       PetscFree(a->rowvalues);
    151:       a->rowvalues_alloc = PetscMax(a->rowvalues_alloc*2, *nz);
    152:       PetscMalloc1(a->rowvalues_alloc,&a->rowvalues);
    153:     }
    154:     for (j=0; j<*nz; j++){
    155:       a->rowvalues[j] = a->values ? a->values[a->i[row]+j]:1.0;
    156:     }
    157:     *v = (*nz) ? a->rowvalues : NULL;
    158:   }
    159:   if (idx) *idx = (*nz) ? a->j + a->i[row] : NULL;
    160:   return(0);
    161: }

    165: PetscErrorCode MatRestoreRow_MPIAdj(Mat A,PetscInt row,PetscInt *nz,PetscInt **idx,PetscScalar **v)
    166: {
    168:   return(0);
    169: }

    173: PetscErrorCode MatEqual_MPIAdj(Mat A,Mat B,PetscBool * flg)
    174: {
    175:   Mat_MPIAdj     *a = (Mat_MPIAdj*)A->data,*b = (Mat_MPIAdj*)B->data;
    177:   PetscBool      flag;

    180:   /* If the  matrix dimensions are not equal,or no of nonzeros */
    181:   if ((A->rmap->n != B->rmap->n) ||(a->nz != b->nz)) {
    182:     flag = PETSC_FALSE;
    183:   }

    185:   /* if the a->i are the same */
    186:   PetscMemcmp(a->i,b->i,(A->rmap->n+1)*sizeof(PetscInt),&flag);

    188:   /* if a->j are the same */
    189:   PetscMemcmp(a->j,b->j,(a->nz)*sizeof(PetscInt),&flag);

    191:   MPIU_Allreduce(&flag,flg,1,MPIU_BOOL,MPI_LAND,PetscObjectComm((PetscObject)A));
    192:   return(0);
    193: }

    197: PetscErrorCode MatGetRowIJ_MPIAdj(Mat A,PetscInt oshift,PetscBool symmetric,PetscBool blockcompressed,PetscInt *m,const PetscInt *inia[],const PetscInt *inja[],PetscBool  *done)
    198: {
    199:   PetscInt   i;
    200:   Mat_MPIAdj *a   = (Mat_MPIAdj*)A->data;
    201:   PetscInt   **ia = (PetscInt**)inia,**ja = (PetscInt**)inja;

    204:   *m    = A->rmap->n;
    205:   *ia   = a->i;
    206:   *ja   = a->j;
    207:   *done = PETSC_TRUE;
    208:   if (oshift) {
    209:     for (i=0; i<(*ia)[*m]; i++) {
    210:       (*ja)[i]++;
    211:     }
    212:     for (i=0; i<=(*m); i++) (*ia)[i]++;
    213:   }
    214:   return(0);
    215: }

    219: PetscErrorCode MatRestoreRowIJ_MPIAdj(Mat A,PetscInt oshift,PetscBool symmetric,PetscBool blockcompressed,PetscInt *m,const PetscInt *inia[],const PetscInt *inja[],PetscBool  *done)
    220: {
    221:   PetscInt   i;
    222:   Mat_MPIAdj *a   = (Mat_MPIAdj*)A->data;
    223:   PetscInt   **ia = (PetscInt**)inia,**ja = (PetscInt**)inja;

    226:   if (ia && a->i != *ia) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_WRONGSTATE,"ia passed back is not one obtained with MatGetRowIJ()");
    227:   if (ja && a->j != *ja) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_WRONGSTATE,"ja passed back is not one obtained with MatGetRowIJ()");
    228:   if (oshift) {
    229:     if (!ia) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_WRONGSTATE,"If oshift then you must passed in inia[] argument");
    230:     if (!ja) SETERRQ(PetscObjectComm((PetscObject)A),PETSC_ERR_ARG_WRONGSTATE,"If oshift then you must passed in inja[] argument");
    231:     for (i=0; i<=(*m); i++) (*ia)[i]--;
    232:     for (i=0; i<(*ia)[*m]; i++) {
    233:       (*ja)[i]--;
    234:     }
    235:   }
    236:   return(0);
    237: }

    241: PetscErrorCode  MatConvertFrom_MPIAdj(Mat A,MatType type,MatReuse reuse,Mat *newmat)
    242: {
    243:   Mat               B;
    244:   PetscErrorCode    ierr;
    245:   PetscInt          i,m,N,nzeros = 0,*ia,*ja,len,rstart,cnt,j,*a;
    246:   const PetscInt    *rj;
    247:   const PetscScalar *ra;
    248:   MPI_Comm          comm;

    251:   MatGetSize(A,NULL,&N);
    252:   MatGetLocalSize(A,&m,NULL);
    253:   MatGetOwnershipRange(A,&rstart,NULL);

    255:   /* count the number of nonzeros per row */
    256:   for (i=0; i<m; i++) {
    257:     MatGetRow(A,i+rstart,&len,&rj,NULL);
    258:     for (j=0; j<len; j++) {
    259:       if (rj[j] == i+rstart) {len--; break;}    /* don't count diagonal */
    260:     }
    261:     nzeros += len;
    262:     MatRestoreRow(A,i+rstart,&len,&rj,NULL);
    263:   }

    265:   /* malloc space for nonzeros */
    266:   PetscMalloc1(nzeros+1,&a);
    267:   PetscMalloc1(N+1,&ia);
    268:   PetscMalloc1(nzeros+1,&ja);

    270:   nzeros = 0;
    271:   ia[0]  = 0;
    272:   for (i=0; i<m; i++) {
    273:     MatGetRow(A,i+rstart,&len,&rj,&ra);
    274:     cnt  = 0;
    275:     for (j=0; j<len; j++) {
    276:       if (rj[j] != i+rstart) { /* if not diagonal */
    277:         a[nzeros+cnt]    = (PetscInt) PetscAbsScalar(ra[j]);
    278:         ja[nzeros+cnt++] = rj[j];
    279:       }
    280:     }
    281:     MatRestoreRow(A,i+rstart,&len,&rj,&ra);
    282:     nzeros += cnt;
    283:     ia[i+1] = nzeros;
    284:   }

    286:   PetscObjectGetComm((PetscObject)A,&comm);
    287:   MatCreate(comm,&B);
    288:   MatSetSizes(B,m,PETSC_DETERMINE,PETSC_DETERMINE,N);
    289:   MatSetType(B,type);
    290:   MatMPIAdjSetPreallocation(B,ia,ja,a);

    292:   if (reuse == MAT_INPLACE_MATRIX) {
    293:     MatHeaderReplace(A,&B);
    294:   } else {
    295:     *newmat = B;
    296:   }
    297:   return(0);
    298: }

    300: /* -------------------------------------------------------------------*/
    301: static struct _MatOps MatOps_Values = {0,
    302:                                        MatGetRow_MPIAdj,
    303:                                        MatRestoreRow_MPIAdj,
    304:                                        0,
    305:                                 /* 4*/ 0,
    306:                                        0,
    307:                                        0,
    308:                                        0,
    309:                                        0,
    310:                                        0,
    311:                                 /*10*/ 0,
    312:                                        0,
    313:                                        0,
    314:                                        0,
    315:                                        0,
    316:                                 /*15*/ 0,
    317:                                        MatEqual_MPIAdj,
    318:                                        0,
    319:                                        0,
    320:                                        0,
    321:                                 /*20*/ 0,
    322:                                        0,
    323:                                        MatSetOption_MPIAdj,
    324:                                        0,
    325:                                 /*24*/ 0,
    326:                                        0,
    327:                                        0,
    328:                                        0,
    329:                                        0,
    330:                                 /*29*/ 0,
    331:                                        0,
    332:                                        0,
    333:                                        0,
    334:                                        0,
    335:                                 /*34*/ 0,
    336:                                        0,
    337:                                        0,
    338:                                        0,
    339:                                        0,
    340:                                 /*39*/ 0,
    341:                                        MatGetSubMatrices_MPIAdj,
    342:                                        0,
    343:                                        0,
    344:                                        0,
    345:                                 /*44*/ 0,
    346:                                        0,
    347:                                        MatShift_Basic,
    348:                                        0,
    349:                                        0,
    350:                                 /*49*/ 0,
    351:                                        MatGetRowIJ_MPIAdj,
    352:                                        MatRestoreRowIJ_MPIAdj,
    353:                                        0,
    354:                                        0,
    355:                                 /*54*/ 0,
    356:                                        0,
    357:                                        0,
    358:                                        0,
    359:                                        0,
    360:                                 /*59*/ 0,
    361:                                        MatDestroy_MPIAdj,
    362:                                        MatView_MPIAdj,
    363:                                        MatConvertFrom_MPIAdj,
    364:                                        0,
    365:                                 /*64*/ 0,
    366:                                        0,
    367:                                        0,
    368:                                        0,
    369:                                        0,
    370:                                 /*69*/ 0,
    371:                                        0,
    372:                                        0,
    373:                                        0,
    374:                                        0,
    375:                                 /*74*/ 0,
    376:                                        0,
    377:                                        0,
    378:                                        0,
    379:                                        0,
    380:                                 /*79*/ 0,
    381:                                        0,
    382:                                        0,
    383:                                        0,
    384:                                        0,
    385:                                 /*84*/ 0,
    386:                                        0,
    387:                                        0,
    388:                                        0,
    389:                                        0,
    390:                                 /*89*/ 0,
    391:                                        0,
    392:                                        0,
    393:                                        0,
    394:                                        0,
    395:                                 /*94*/ 0,
    396:                                        0,
    397:                                        0,
    398:                                        0,
    399:                                        0,
    400:                                 /*99*/ 0,
    401:                                        0,
    402:                                        0,
    403:                                        0,
    404:                                        0,
    405:                                /*104*/ 0,
    406:                                        0,
    407:                                        0,
    408:                                        0,
    409:                                        0,
    410:                                /*109*/ 0,
    411:                                        0,
    412:                                        0,
    413:                                        0,
    414:                                        0,
    415:                                /*114*/ 0,
    416:                                        0,
    417:                                        0,
    418:                                        0,
    419:                                        0,
    420:                                /*119*/ 0,
    421:                                        0,
    422:                                        0,
    423:                                        0,
    424:                                        0,
    425:                                /*124*/ 0,
    426:                                        0,
    427:                                        0,
    428:                                        0,
    429:                                        MatGetSubMatricesMPI_MPIAdj,
    430:                                /*129*/ 0,
    431:                                        0,
    432:                                        0,
    433:                                        0,
    434:                                        0,
    435:                                /*134*/ 0,
    436:                                        0,
    437:                                        0,
    438:                                        0,
    439:                                        0,
    440:                                /*139*/ 0,
    441:                                        0,
    442:                                        0
    443: };

    447: static PetscErrorCode  MatMPIAdjSetPreallocation_MPIAdj(Mat B,PetscInt *i,PetscInt *j,PetscInt *values)
    448: {
    449:   Mat_MPIAdj     *b = (Mat_MPIAdj*)B->data;
    451: #if defined(PETSC_USE_DEBUG)
    452:   PetscInt ii;
    453: #endif

    456:   PetscLayoutSetUp(B->rmap);
    457:   PetscLayoutSetUp(B->cmap);

    459: #if defined(PETSC_USE_DEBUG)
    460:   if (i[0] != 0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"First i[] index must be zero, instead it is %D\n",i[0]);
    461:   for (ii=1; ii<B->rmap->n; ii++) {
    462:     if (i[ii] < 0 || i[ii] < i[ii-1]) SETERRQ4(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"i[%D]=%D index is out of range: i[%D]=%D",ii,i[ii],ii-1,i[ii-1]);
    463:   }
    464:   for (ii=0; ii<i[B->rmap->n]; ii++) {
    465:     if (j[ii] < 0 || j[ii] >= B->cmap->N) SETERRQ2(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Column index %D out of range %D\n",ii,j[ii]);
    466:   }
    467: #endif
    468:   B->preallocated = PETSC_TRUE;

    470:   b->j      = j;
    471:   b->i      = i;
    472:   b->values = values;

    474:   b->nz        = i[B->rmap->n];
    475:   b->diag      = 0;
    476:   b->symmetric = PETSC_FALSE;
    477:   b->freeaij   = PETSC_TRUE;

    479:   MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);
    480:   MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);
    481:   return(0);
    482: }

    484: static PetscErrorCode MatGetSubMatrix_MPIAdj_data(Mat,IS,IS, PetscInt **,PetscInt **,PetscInt **);
    485: static PetscErrorCode MatGetSubMatrices_MPIAdj_Private(Mat,PetscInt,const IS[],const IS[],PetscBool,MatReuse,Mat **);

    489: PetscErrorCode MatGetSubMatricesMPI_MPIAdj(Mat mat,PetscInt n, const IS irow[],const IS icol[],MatReuse scall,Mat *submat[])
    490: {
    491:   PetscErrorCode     ierr;
    492:   /*get sub-matrices across a sub communicator */
    494:   MatGetSubMatrices_MPIAdj_Private(mat,n,irow,icol,PETSC_TRUE,scall,submat);
    495:   return(0);
    496: }

    501: PetscErrorCode MatGetSubMatrices_MPIAdj(Mat mat,PetscInt n,const IS irow[],const IS icol[],MatReuse scall,Mat *submat[])
    502: {
    503:   PetscErrorCode     ierr;

    506:   /*get sub-matrices based on PETSC_COMM_SELF */
    507:   MatGetSubMatrices_MPIAdj_Private(mat,n,irow,icol,PETSC_FALSE,scall,submat);
    508:   return(0);
    509: }

    513: static PetscErrorCode MatGetSubMatrices_MPIAdj_Private(Mat mat,PetscInt n,const IS irow[],const IS icol[],PetscBool subcomm,MatReuse scall,Mat *submat[])
    514: {
    515:   PetscInt           i,irow_n,icol_n,*sxadj,*sadjncy,*svalues;
    516:   PetscInt          *indices,nindx,j,k,loc;
    517:   PetscMPIInt        issame;
    518:   const PetscInt    *irow_indices,*icol_indices;
    519:   MPI_Comm           scomm_row,scomm_col,scomm_mat;
    520:   PetscErrorCode     ierr;

    523:   nindx = 0;
    524:   /*
    525:    * Estimate a maximum number for allocating memory
    526:    */
    527:   for(i=0; i<n; i++){
    528:     ISGetLocalSize(irow[i],&irow_n);
    529:     ISGetLocalSize(icol[i],&icol_n);
    530:     nindx = nindx>(irow_n+icol_n)? nindx:(irow_n+icol_n);
    531:   }
    532:   PetscCalloc1(nindx,&indices);
    533:   /* construct a submat */
    534:   for(i=0; i<n; i++){
    535:         /*comms */
    536:     if(subcomm){
    537:           PetscObjectGetComm((PetscObject)irow[i],&scomm_row);
    538:           PetscObjectGetComm((PetscObject)icol[i],&scomm_col);
    539:           MPI_Comm_compare(scomm_row,scomm_col,&issame);
    540:           if(issame != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"row index set must have the same comm as the col index set\n");
    541:           MPI_Comm_compare(scomm_row,PETSC_COMM_SELF,&issame);
    542:           if(issame == MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP," can not use PETSC_COMM_SELF as comm when extracting a parallel submatrix\n");
    543:         }else{
    544:           scomm_row = PETSC_COMM_SELF;
    545:         }
    546:         /*get sub-matrix data*/
    547:         sxadj=0; sadjncy=0; svalues=0;
    548:     MatGetSubMatrix_MPIAdj_data(mat,irow[i],icol[i],&sxadj,&sadjncy,&svalues);
    549:     ISGetLocalSize(irow[i],&irow_n);
    550:     ISGetLocalSize(icol[i],&icol_n);
    551:     ISGetIndices(irow[i],&irow_indices);
    552:     PetscMemcpy(indices,irow_indices,sizeof(PetscInt)*irow_n);
    553:     ISRestoreIndices(irow[i],&irow_indices);
    554:     ISGetIndices(icol[i],&icol_indices);
    555:     PetscMemcpy(indices+irow_n,icol_indices,sizeof(PetscInt)*icol_n);
    556:     ISRestoreIndices(icol[i],&icol_indices);
    557:     nindx = irow_n+icol_n;
    558:     PetscSortRemoveDupsInt(&nindx,indices);
    559:     /* renumber columns */
    560:     for(j=0; j<irow_n; j++){
    561:       for(k=sxadj[j]; k<sxadj[j+1]; k++){
    562:             PetscFindInt(sadjncy[k],nindx,indices,&loc);
    563: #if PETSC_USE_DEBUG
    564:             if(loc<0) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"can not find col %d \n",sadjncy[k]);
    565: #endif
    566:         sadjncy[k] = loc;
    567:       }
    568:     }
    569:     if(scall==MAT_INITIAL_MATRIX){
    570:       MatCreateMPIAdj(scomm_row,irow_n,icol_n,sxadj,sadjncy,svalues,submat[i]);
    571:     }else{
    572:        Mat                sadj = *(submat[i]);
    573:        Mat_MPIAdj         *sa  = (Mat_MPIAdj*)((sadj)->data);
    574:        PetscObjectGetComm((PetscObject)sadj,&scomm_mat);
    575:        MPI_Comm_compare(scomm_row,scomm_mat,&issame);
    576:        if(issame != MPI_IDENT) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_INCOMP,"submatrix  must have the same comm as the col index set\n");
    577:        PetscMemcpy(sa->i,sxadj,sizeof(PetscInt)*(irow_n+1));
    578:        PetscMemcpy(sa->j,sadjncy,sizeof(PetscInt)*sxadj[irow_n]);
    579:        if(svalues){PetscMemcpy(sa->values,svalues,sizeof(PetscInt)*sxadj[irow_n]);}
    580:        PetscFree(sxadj);
    581:        PetscFree(sadjncy);
    582:        if(svalues) {PetscFree(svalues);}
    583:     }
    584:   }
    585:   PetscFree(indices);
    586:   return(0);
    587: }

    592: /*
    593:  * The interface should be easy to use for both MatGetSubMatrix (parallel sub-matrix) and MatGetSubMatrices (sequential sub-matrices)
    594:  * */
    595: static PetscErrorCode MatGetSubMatrix_MPIAdj_data(Mat adj,IS irows, IS icols, PetscInt **sadj_xadj,PetscInt **sadj_adjncy,PetscInt **sadj_values)
    596: {
    597:   PetscInt                 nlrows_is,icols_n,i,j,nroots,nleaves,owner,rlocalindex,*ncols_send,*ncols_recv;
    598:   PetscInt           nlrows_mat,*adjncy_recv,Ncols_recv,Ncols_send,*xadj_recv,*values_recv;
    599:   PetscInt          *ncols_recv_offsets,loc,rnclos,*sadjncy,*sxadj,*svalues,isvalue;
    600:   const PetscInt    *irows_indices,*icols_indices,*xadj, *adjncy;
    601:   Mat_MPIAdj        *a = (Mat_MPIAdj*)adj->data;
    602:   PetscLayout        rmap;
    603:   MPI_Comm           comm;
    604:   PetscSF            sf;
    605:   PetscSFNode       *iremote;
    606:   PetscBool          done;
    607:   PetscErrorCode     ierr;

    610:   /* communicator */
    611:   PetscObjectGetComm((PetscObject)adj,&comm);
    612:   /* Layouts */
    613:   MatGetLayouts(adj,&rmap,PETSC_NULL);
    614:   /* get rows information */
    615:   ISGetLocalSize(irows,&nlrows_is);
    616:   ISGetIndices(irows,&irows_indices);
    617:   PetscCalloc1(nlrows_is,&iremote);
    618:   /* construct sf graph*/
    619:   nleaves = nlrows_is;
    620:   for(i=0; i<nlrows_is; i++){
    621:         owner = -1;
    622:         rlocalindex = -1;
    623:     PetscLayoutFindOwnerIndex(rmap,irows_indices[i],&owner,&rlocalindex);
    624:     iremote[i].rank  = owner;
    625:     iremote[i].index = rlocalindex;
    626:   }
    627:   MatGetRowIJ(adj,0,PETSC_FALSE,PETSC_FALSE,&nlrows_mat,&xadj,&adjncy,&done);
    628:   PetscCalloc4(nlrows_mat,&ncols_send,nlrows_is,&xadj_recv,nlrows_is+1,&ncols_recv_offsets,nlrows_is,&ncols_recv);
    629:   nroots = nlrows_mat;
    630:   for(i=0; i<nlrows_mat; i++){
    631:         ncols_send[i] = xadj[i+1]-xadj[i];
    632:   }
    633:   PetscSFCreate(comm,&sf);
    634:   PetscSFSetGraph(sf,nroots,nleaves,PETSC_NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);
    635:   PetscSFSetType(sf,PETSCSFBASIC);
    636:   PetscSFSetFromOptions(sf);
    637:   PetscSFBcastBegin(sf,MPIU_INT,ncols_send,ncols_recv);
    638:   PetscSFBcastEnd(sf,MPIU_INT,ncols_send,ncols_recv);
    639:   PetscSFBcastBegin(sf,MPIU_INT,xadj,xadj_recv);
    640:   PetscSFBcastEnd(sf,MPIU_INT,xadj,xadj_recv);
    641:   PetscSFDestroy(&sf);
    642:   Ncols_recv =0;
    643:   for(i=0; i<nlrows_is; i++){
    644:          Ncols_recv             += ncols_recv[i];
    645:          ncols_recv_offsets[i+1] = ncols_recv[i]+ncols_recv_offsets[i];
    646:   }
    647:   Ncols_send = 0;
    648:   for(i=0; i<nlrows_mat; i++){
    649:         Ncols_send += ncols_send[i];
    650:   }
    651:   PetscCalloc1(Ncols_recv,&iremote);
    652:   PetscCalloc1(Ncols_recv,&adjncy_recv);
    653:   nleaves = Ncols_recv;
    654:   Ncols_recv = 0;
    655:   for(i=0; i<nlrows_is; i++){
    656:     PetscLayoutFindOwner(rmap,irows_indices[i],&owner);
    657:     for(j=0; j<ncols_recv[i]; j++){
    658:       iremote[Ncols_recv].rank    = owner;
    659:       iremote[Ncols_recv++].index = xadj_recv[i]+j;
    660:     }
    661:   }
    662:   ISRestoreIndices(irows,&irows_indices);
    663:   /*if we need to deal with edge weights ???*/
    664:   if(a->values){isvalue=1;}else{isvalue=0;}
    665:   /*involve a global communication */
    666:   /*MPIU_Allreduce(&isvalue,&isvalue,1,MPIU_INT,MPI_SUM,comm);*/
    667:   if(isvalue){PetscCalloc1(Ncols_recv,&values_recv);}
    668:   nroots = Ncols_send;
    669:   PetscSFCreate(comm,&sf);
    670:   PetscSFSetGraph(sf,nroots,nleaves,PETSC_NULL,PETSC_OWN_POINTER,iremote,PETSC_OWN_POINTER);
    671:   PetscSFSetType(sf,PETSCSFBASIC);
    672:   PetscSFSetFromOptions(sf);
    673:   PetscSFBcastBegin(sf,MPIU_INT,adjncy,adjncy_recv);
    674:   PetscSFBcastEnd(sf,MPIU_INT,adjncy,adjncy_recv);
    675:   if(isvalue){
    676:         PetscSFBcastBegin(sf,MPIU_INT,a->values,values_recv);
    677:         PetscSFBcastEnd(sf,MPIU_INT,a->values,values_recv);
    678:   }
    679:   PetscSFDestroy(&sf);
    680:   MatRestoreRowIJ(adj,0,PETSC_FALSE,PETSC_FALSE,&nlrows_mat,&xadj,&adjncy,&done);
    681:   ISGetLocalSize(icols,&icols_n);
    682:   ISGetIndices(icols,&icols_indices);
    683:   rnclos = 0;
    684:   for(i=0; i<nlrows_is; i++){
    685:     for(j=ncols_recv_offsets[i]; j<ncols_recv_offsets[i+1]; j++){
    686:       PetscFindInt(adjncy_recv[j], icols_n, icols_indices, &loc);
    687:       if(loc<0){
    688:         adjncy_recv[j] = -1;
    689:         if(isvalue) values_recv[j] = -1;
    690:         ncols_recv[i]--;
    691:       }else{
    692:             rnclos++;
    693:       }
    694:     }
    695:   }
    696:   ISRestoreIndices(icols,&icols_indices);
    697:   PetscCalloc1(rnclos,&sadjncy);
    698:   if(isvalue) {PetscCalloc1(rnclos,&svalues);}
    699:   PetscCalloc1(nlrows_is+1,&sxadj);
    700:   rnclos = 0;
    701:   for(i=0; i<nlrows_is; i++){
    702:         for(j=ncols_recv_offsets[i]; j<ncols_recv_offsets[i+1]; j++){
    703:           if(adjncy_recv[j]<0) continue;
    704:           sadjncy[rnclos] = adjncy_recv[j];
    705:           if(isvalue) svalues[rnclos] = values_recv[j];
    706:           rnclos++;
    707:         }
    708:   }
    709:   for(i=0; i<nlrows_is; i++){
    710:         sxadj[i+1] = sxadj[i]+ncols_recv[i];
    711:   }
    712:   if(sadj_xadj)  { *sadj_xadj = sxadj;}else    { PetscFree(sxadj);}
    713:   if(sadj_adjncy){ *sadj_adjncy = sadjncy;}else{ PetscFree(sadjncy);}
    714:   if(sadj_values){
    715:         if(isvalue) *sadj_values = svalues; else *sadj_values=0;
    716:   }else{
    717:         if(isvalue) {PetscFree(svalues);}
    718:   }
    719:   PetscFree4(ncols_send,xadj_recv,ncols_recv_offsets,ncols_recv);
    720:   PetscFree(adjncy_recv);
    721:   if(isvalue) {PetscFree(values_recv);}
    722:   return(0);
    723: }

    728: static PetscErrorCode MatMPIAdjCreateNonemptySubcommMat_MPIAdj(Mat A,Mat *B)
    729: {
    730:   Mat_MPIAdj     *a = (Mat_MPIAdj*)A->data;
    732:   const PetscInt *ranges;
    733:   MPI_Comm       acomm,bcomm;
    734:   MPI_Group      agroup,bgroup;
    735:   PetscMPIInt    i,rank,size,nranks,*ranks;

    738:   *B    = NULL;
    739:   PetscObjectGetComm((PetscObject)A,&acomm);
    740:   MPI_Comm_size(acomm,&size);
    741:   MPI_Comm_size(acomm,&rank);
    742:   MatGetOwnershipRanges(A,&ranges);
    743:   for (i=0,nranks=0; i<size; i++) {
    744:     if (ranges[i+1] - ranges[i] > 0) nranks++;
    745:   }
    746:   if (nranks == size) {         /* All ranks have a positive number of rows, so we do not need to create a subcomm; */
    747:     PetscObjectReference((PetscObject)A);
    748:     *B   = A;
    749:     return(0);
    750:   }

    752:   PetscMalloc1(nranks,&ranks);
    753:   for (i=0,nranks=0; i<size; i++) {
    754:     if (ranges[i+1] - ranges[i] > 0) ranks[nranks++] = i;
    755:   }
    756:   MPI_Comm_group(acomm,&agroup);
    757:   MPI_Group_incl(agroup,nranks,ranks,&bgroup);
    758:   PetscFree(ranks);
    759:   MPI_Comm_create(acomm,bgroup,&bcomm);
    760:   MPI_Group_free(&agroup);
    761:   MPI_Group_free(&bgroup);
    762:   if (bcomm != MPI_COMM_NULL) {
    763:     PetscInt   m,N;
    764:     Mat_MPIAdj *b;
    765:     MatGetLocalSize(A,&m,NULL);
    766:     MatGetSize(A,NULL,&N);
    767:     MatCreateMPIAdj(bcomm,m,N,a->i,a->j,a->values,B);
    768:     b          = (Mat_MPIAdj*)(*B)->data;
    769:     b->freeaij = PETSC_FALSE;
    770:     MPI_Comm_free(&bcomm);
    771:   }
    772:   return(0);
    773: }

    777: /*@
    778:    MatMPIAdjCreateNonemptySubcommMat - create the same MPIAdj matrix on a subcommunicator containing only processes owning a positive number of rows

    780:    Collective

    782:    Input Arguments:
    783: .  A - original MPIAdj matrix

    785:    Output Arguments:
    786: .  B - matrix on subcommunicator, NULL on ranks that owned zero rows of A

    788:    Level: developer

    790:    Note:
    791:    This function is mostly useful for internal use by mesh partitioning packages that require that every process owns at least one row.

    793:    The matrix B should be destroyed with MatDestroy(). The arrays are not copied, so B should be destroyed before A is destroyed.

    795: .seealso: MatCreateMPIAdj()
    796: @*/
    797: PetscErrorCode MatMPIAdjCreateNonemptySubcommMat(Mat A,Mat *B)
    798: {

    803:   PetscUseMethod(A,"MatMPIAdjCreateNonemptySubcommMat_C",(Mat,Mat*),(A,B));
    804:   return(0);
    805: }

    807: /*MC
    808:    MATMPIADJ - MATMPIADJ = "mpiadj" - A matrix type to be used for distributed adjacency matrices,
    809:    intended for use constructing orderings and partitionings.

    811:   Level: beginner

    813: .seealso: MatCreateMPIAdj
    814: M*/

    818: PETSC_EXTERN PetscErrorCode MatCreate_MPIAdj(Mat B)
    819: {
    820:   Mat_MPIAdj     *b;

    824:   PetscNewLog(B,&b);
    825:   B->data      = (void*)b;
    826:   PetscMemcpy(B->ops,&MatOps_Values,sizeof(struct _MatOps));
    827:   B->assembled = PETSC_FALSE;

    829:   PetscObjectComposeFunction((PetscObject)B,"MatMPIAdjSetPreallocation_C",MatMPIAdjSetPreallocation_MPIAdj);
    830:   PetscObjectComposeFunction((PetscObject)B,"MatMPIAdjCreateNonemptySubcommMat_C",MatMPIAdjCreateNonemptySubcommMat_MPIAdj);
    831:   PetscObjectChangeTypeName((PetscObject)B,MATMPIADJ);
    832:   return(0);
    833: }

    837: /*@C
    838:    MatMPIAdjSetPreallocation - Sets the array used for storing the matrix elements

    840:    Logically Collective on MPI_Comm

    842:    Input Parameters:
    843: +  A - the matrix
    844: .  i - the indices into j for the start of each row
    845: .  j - the column indices for each row (sorted for each row).
    846:        The indices in i and j start with zero (NOT with one).
    847: -  values - [optional] edge weights

    849:    Level: intermediate

    851: .seealso: MatCreate(), MatCreateMPIAdj(), MatSetValues()
    852: @*/
    853: PetscErrorCode  MatMPIAdjSetPreallocation(Mat B,PetscInt *i,PetscInt *j,PetscInt *values)
    854: {

    858:   PetscTryMethod(B,"MatMPIAdjSetPreallocation_C",(Mat,PetscInt*,PetscInt*,PetscInt*),(B,i,j,values));
    859:   return(0);
    860: }

    864: /*@C
    865:    MatCreateMPIAdj - Creates a sparse matrix representing an adjacency list.
    866:    The matrix does not have numerical values associated with it, but is
    867:    intended for ordering (to reduce bandwidth etc) and partitioning.

    869:    Collective on MPI_Comm

    871:    Input Parameters:
    872: +  comm - MPI communicator
    873: .  m - number of local rows
    874: .  N - number of global columns
    875: .  i - the indices into j for the start of each row
    876: .  j - the column indices for each row (sorted for each row).
    877:        The indices in i and j start with zero (NOT with one).
    878: -  values -[optional] edge weights

    880:    Output Parameter:
    881: .  A - the matrix

    883:    Level: intermediate

    885:    Notes: This matrix object does not support most matrix operations, include
    886:    MatSetValues().
    887:    You must NOT free the ii, values and jj arrays yourself. PETSc will free them
    888:    when the matrix is destroyed; you must allocate them with PetscMalloc(). If you
    889:     call from Fortran you need not create the arrays with PetscMalloc().
    890:    Should not include the matrix diagonals.

    892:    If you already have a matrix, you can create its adjacency matrix by a call
    893:    to MatConvert, specifying a type of MATMPIADJ.

    895:    Possible values for MatSetOption() - MAT_STRUCTURALLY_SYMMETRIC

    897: .seealso: MatCreate(), MatConvert(), MatGetOrdering()
    898: @*/
    899: PetscErrorCode  MatCreateMPIAdj(MPI_Comm comm,PetscInt m,PetscInt N,PetscInt *i,PetscInt *j,PetscInt *values,Mat *A)
    900: {

    904:   MatCreate(comm,A);
    905:   MatSetSizes(*A,m,PETSC_DETERMINE,PETSC_DETERMINE,N);
    906:   MatSetType(*A,MATMPIADJ);
    907:   MatMPIAdjSetPreallocation(*A,i,j,values);
    908:   return(0);
    909: }