Actual source code: jostle.c

      1: #define PETSCMAT_DLL

     3:  #include src/mat/impls/adj/mpi/mpiadj.h

      5: #ifdef PETSC_HAVE_UNISTD_H
      6: #include <unistd.h>
      7: #endif

      9: #ifdef PETSC_HAVE_STDLIB_H
     10: #include <stdlib.h>
     11: #endif

     13: #include "petscfix.h"

     17: #include "jostle.h"
     18: /* this function is not declared in 'jostle.h' */

     23: typedef struct {
     24:     int output;
     25:     int coarse_seq;
     26:     int nbvtxcoarsed;           /* number of vertices for the coarse graph */
     27:     char *mesg_log;
     28: } MatPartitioning_Jostle;

     30: #define SIZE_LOG 10000          /* size of buffer for msg_log */

     34: static PetscErrorCode MatPartitioningApply_Jostle(MatPartitioning part, IS * partitioning)
     35: {
     37:     int  size, rank, i;
     38:     Mat mat = part->adj, matMPI;
     39:     Mat_MPIAdj *adj = (Mat_MPIAdj *) mat->data;
     40:     MatPartitioning_Jostle *jostle_struct =
     41:         (MatPartitioning_Jostle *) part->data;
     42:     PetscTruth flg;
     43: #ifdef PETSC_HAVE_UNISTD_H
     44:     int fd_stdout, fd_pipe[2], count;
     45: #endif

     49:     /* check that the number of partitions is equal to the number of processors */
     50:     MPI_Comm_rank(mat->comm, &rank);
     51:     MPI_Comm_size(mat->comm, &size);
     52:     if (part->n != size) {
     53:         SETERRQ(PETSC_ERR_SUP, "Supports exactly one domain per processor");
     54:     }

     56:     /* convert adjacency matrix to MPIAdj if needed*/
     57:     PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);
     58:     if (!flg) {
     59:         MatConvert(mat, MATMPIADJ, MAT_INITIAL_MATRIX, &matMPI);
     60:     } else
     61:         matMPI = mat;

     63:     adj = (Mat_MPIAdj *) matMPI->data;  /* adj contains adjacency graph */
     64:     {
     65:         /* definition of Jostle library arguments */
     66:         int nnodes = matMPI->M; /* number of vertices in full graph */
     67:         int offset = 0;         /* 0 for C array indexing */
     68:         int core = matMPI->m;
     69:         int halo = 0;           /* obsolete with contiguous format */
     70:         int *index_jostle;      /* contribution of each processor */
     71:         int nparts = part->n;
     72:         int *part_wt = NULL;

     74:         int *partition;         /* set number of each vtx (length n) */
     75:         int *degree;            /* degree for each core nodes */
     76:         int *edges = adj->j;
     77:         int *node_wt = NULL;    /* nodes weights */
     78:         int *edge_wt = NULL;    /* edges weights */
     79:         double *coords = NULL;  /* not used (cf jostle documentation) */

     81:         int local_nedges = adj->nz;
     82:         int dimension = 0;      /* not used */
     83:         int output_level = jostle_struct->output;
     84:         char env_str[256];

     86:         /* allocate index_jostle */
     87:         PetscMalloc(nparts * sizeof(int), &index_jostle);

     89:         /* compute number of core nodes for each one */
     90:         for (i = 0; i < nparts - 1; i++)
     91:             index_jostle[i] = adj->rowners[i + 1] - adj->rowners[i];
     92:         index_jostle[nparts - 1] = nnodes - adj->rowners[nparts - 1];

     94:         /* allocate the partition vector */
     95:         PetscMalloc(core * sizeof(int), &partition);

     97:         /* build the degree vector and the local_nedges value */
     98:         PetscMalloc(core * sizeof(int), &degree);
     99:         for (i = 0; i < core; i++)
    100:             degree[i] = adj->i[i + 1] - adj->i[i];

    102:         /* library call */
    103:         pjostle_init(&size, &rank);
    104:         pjostle_comm(&matMPI->comm);
    105:         jostle_env("format = contiguous");
    106:         jostle_env("timer = off");

    108:         sprintf(env_str, "threshold = %d", jostle_struct->nbvtxcoarsed);
    109:         jostle_env(env_str);

    111:         if (jostle_struct->coarse_seq)
    112:           jostle_env("matching = local");

    114:         /* redirect output */
    115: #ifdef PETSC_HAVE_UNISTD_H
    116:         fd_stdout = dup(1);
    117:         pipe(fd_pipe);
    118:         close(1);
    119:         dup2(fd_pipe[1], 1);
    120: #endif

    122:         pjostle(&nnodes, &offset, &core, &halo, index_jostle, degree, node_wt,
    123:             partition, &local_nedges, edges, edge_wt, &nparts,
    124:             part_wt, &output_level, &dimension, coords);

    126:         printf("Jostle Partitioner statistics\ncut : %d, balance : %f, runtime : %f, mem used : %d\n",
    127:             jostle_cut(), jostle_bal(), jostle_tim(), jostle_mem());

    129: #ifdef PETSC_HAVE_UNISTD_H
    130:         PetscMalloc(SIZE_LOG * sizeof(char), &(jostle_struct->mesg_log));
    131:         fflush(stdout);
    132:         count = read(fd_pipe[0], jostle_struct->mesg_log, (SIZE_LOG - 1) * sizeof(char));
    133:         if (count < 0)
    134:             count = 0;
    135:         jostle_struct->mesg_log[count] = 0;
    136:         close(1);
    137:         dup2(fd_stdout, 1);
    138:         close(fd_stdout);
    139:         close(fd_pipe[0]);
    140:         close(fd_pipe[1]);
    141: #endif

    143:         /* We free the memory used by jostle */
    144:         PetscFree(index_jostle);
    145:         PetscFree(degree);

    147:         /* Creation of the index set */
    148:         ISCreateGeneral(part->comm, mat->m, partition, partitioning);

    150:         if (matMPI != mat) {
    151:             MatDestroy(matMPI);
    152:         }

    154:         PetscFree(partition);
    155:     }

    157:     return(0);
    158: }

    163: PetscErrorCode MatPartitioningView_Jostle(MatPartitioning part, PetscViewer viewer)
    164: {
    165:   MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;
    166:   PetscErrorCode         ierr;
    167:   PetscTruth             iascii;

    170:   PetscTypeCompare((PetscObject) viewer, PETSC_VIEWER_ASCII, &iascii);
    171:   if (iascii) {
    172:     if (jostle_struct->mesg_log) {
    173:       PetscViewerASCIIPrintf(viewer, "%s\n", jostle_struct->mesg_log);
    174:     }
    175:   } else {
    176:     SETERRQ1(PETSC_ERR_SUP, "Viewer type %s not supported for this Jostle partitioner",((PetscObject) viewer)->type_name);
    177:   }
    178:   return(0);
    179: }

    183: /*@
    184:     MatPartitioningJostleSetCoarseLevel - Set the coarse level
    185:
    186:   Input Parameter:
    187: .  part - the partitioning context
    188: .  level - the coarse level in range [0.0,1.0]

    190:    Level: advanced

    192: @*/
    193: PetscErrorCode PETSCMAT_DLLEXPORT MatPartitioningJostleSetCoarseLevel(MatPartitioning part, PetscReal level)
    194: {
    195:     MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;

    199:     if (level < 0.0 || level > 1.0) {
    200:         SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,
    201:             "Jostle: level of coarsening out of range [0.0-1.0]");
    202:     } else
    203:         jostle_struct->nbvtxcoarsed = (int)(part->adj->N * level);

    205:     if (jostle_struct->nbvtxcoarsed < 20)
    206:         jostle_struct->nbvtxcoarsed = 20;

    208:     return(0);
    209: }

    213: /*@
    214:      MatPartitioningJostleSetCoarseSequential - Use the sequential code to
    215:          do the partitioning of the coarse grid.

    217:   Input Parameter:
    218: .  part - the partitioning context

    220:    Level: advanced

    222: @*/
    223: PetscErrorCode PETSCMAT_DLLEXPORT MatPartitioningJostleSetCoarseSequential(MatPartitioning part)
    224: {
    225:     MatPartitioning_Jostle *jostle_struct =
    226:         (MatPartitioning_Jostle *) part->data;
    228:     jostle_struct->coarse_seq = 1;
    229:     return(0);
    230: }

    234: PetscErrorCode MatPartitioningSetFromOptions_Jostle(MatPartitioning part)
    235: {
    237:     PetscTruth flag;
    238:     PetscReal level;

    241:     PetscOptionsHead("Set Jostle partitioning options");

    243:     PetscOptionsReal("-mat_partitioning_jostle_coarse_level",
    244:         "Coarse level", "MatPartitioningJostleSetCoarseLevel", 0,
    245:         &level, &flag);
    246:     if (flag)
    247:         MatPartitioningJostleSetCoarseLevel(part, level);

    249:     PetscOptionsName("-mat_partitioning_jostle_coarse_sequential",
    250:         "Use sequential coarse partitioner",
    251:         "MatPartitioningJostleSetCoarseSequential", &flag);
    252:     if (flag) {
    253:         MatPartitioningJostleSetCoarseSequential(part);
    254:     }

    256:     PetscOptionsTail();
    257:     return(0);
    258: }

    263: PetscErrorCode MatPartitioningDestroy_Jostle(MatPartitioning part)
    264: {
    265:     MatPartitioning_Jostle *jostle_struct =
    266:         (MatPartitioning_Jostle *) part->data;

    271:     if (jostle_struct->mesg_log) {
    272:         PetscFree(jostle_struct->mesg_log);
    273:     }
    274:     PetscFree(jostle_struct);

    276:     return(0);
    277: }

    282: /*@C
    283:    MAT_PARTITIONING_JOSTLE - Creates a partitioning context via the external package Jostle.

    285:    Collective on MPI_Comm

    287:    Input Parameter:
    288: .  part - the partitioning context

    290:    Options Database Keys:
    291: +  -mat_partitioning_jostle_coarse_level <0>: Coarse level (MatPartitioningJostleSetCoarseLevel)
    292: -  -mat_partitioning_jostle_coarse_sequential: Use sequential coarse partitioner (MatPartitioningJostleSetCoarseSequential)

    294:    Level: beginner

    296:    Notes: See http://www.gre.ac.uk/~c.walshaw/jostle/

    298: .keywords: Partitioning, create, context

    300: .seealso: MatPartitioningSetType(), MatPartitioningType

    302: @*/
    303: PetscErrorCode PETSCMAT_DLLEXPORT MatPartitioningCreate_Jostle(MatPartitioning part)
    304: {
    306:     MatPartitioning_Jostle *jostle_struct;

    309:     PetscNew(MatPartitioning_Jostle, &jostle_struct);

    311:     jostle_struct->nbvtxcoarsed = 20;
    312:     jostle_struct->output = 0;
    313:     jostle_struct->coarse_seq = 0;
    314:     jostle_struct->mesg_log = NULL;

    316:     part->ops->apply = MatPartitioningApply_Jostle;
    317:     part->ops->view = MatPartitioningView_Jostle;
    318:     part->ops->destroy = MatPartitioningDestroy_Jostle;
    319:     part->ops->setfromoptions = MatPartitioningSetFromOptions_Jostle;
    320:     part->data = (void*) jostle_struct;

    322:     return(0);
    323: }