Actual source code: ex158.c

petsc-3.3-p6 2013-02-11
  1: static char help[] = "Illustrate how to use mpi FFTW and PETSc-FFTW interface \n\n";

  3: /*
  4:  Usage:
  5:    mpiexec -n <np> ./ex158 -use_FFTW_interface NO
  6:    mpiexec -n <np> ./ex158 -use_FFTW_interface YES
  7: */

  9: #include <petscmat.h>
 10: #include <fftw3-mpi.h>

 14: PetscInt main(PetscInt argc,char **args)
 15: {
 16:   PetscErrorCode  ierr;
 17:   PetscMPIInt     rank,size;
 18:   PetscInt        N0=50,N1=20,N=N0*N1;
 19:   PetscRandom     rdm;
 20:   PetscScalar     a;
 21:   PetscReal       enorm;
 22:   Vec             x,y,z;
 23:   PetscBool       view=PETSC_FALSE,use_interface=PETSC_TRUE;

 25:   PetscInitialize(&argc,&args,(char *)0,help);
 26: #if defined(PETSC_USE_COMPLEX)
 27:   SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_SUP, "This example requires real numbers. Your current scalar type is complex");
 28: #endif

 30:   PetscOptionsBegin(PETSC_COMM_WORLD, PETSC_NULL, "FFTW Options", "ex158");
 31:     PetscOptionsBool("-use_FFTW_interface", "Use PETSc-FFTW interface", "ex158",use_interface, &use_interface, PETSC_NULL);
 32:   PetscOptionsEnd();

 34:   MPI_Comm_size(PETSC_COMM_WORLD, &size);
 35:   MPI_Comm_rank(PETSC_COMM_WORLD, &rank);

 37:   PetscRandomCreate(PETSC_COMM_WORLD, &rdm);
 38:   PetscRandomSetFromOptions(rdm);

 40:   if (!use_interface){
 41:     /* Use mpi FFTW without PETSc-FFTW interface, 2D case only */
 42:     /*---------------------------------------------------------*/
 43:     fftw_plan       fplan,bplan;
 44:     fftw_complex    *data_in,*data_out,*data_out2;
 45:     ptrdiff_t       alloc_local,local_n0,local_0_start;

 47:     if (!rank) printf("Use FFTW without PETSc-FFTW interface\n");
 48:     fftw_mpi_init();
 49:     N = N0*N1;
 50:     alloc_local = fftw_mpi_local_size_2d(N0,N1,PETSC_COMM_WORLD,&local_n0,&local_0_start);

 52:     data_in   = (fftw_complex*)fftw_malloc(sizeof(fftw_complex)*alloc_local);
 53:     data_out  = (fftw_complex*)fftw_malloc(sizeof(fftw_complex)*alloc_local);
 54:     data_out2 = (fftw_complex*)fftw_malloc(sizeof(fftw_complex)*alloc_local);
 55:     VecCreateMPIWithArray(PETSC_COMM_WORLD,1,(PetscInt)local_n0*N1,(PetscInt)N,(const PetscScalar*)data_in,&x);
 56:     PetscObjectSetName((PetscObject) x, "Real Space vector");
 57:     VecCreateMPIWithArray(PETSC_COMM_WORLD,1,(PetscInt)local_n0*N1,(PetscInt)N,(const PetscScalar*)data_out,&y);
 58:     PetscObjectSetName((PetscObject) y, "Frequency space vector");
 59:     VecCreateMPIWithArray(PETSC_COMM_WORLD,1,(PetscInt)local_n0*N1,(PetscInt)N,(const PetscScalar*)data_out2,&z);
 60:     PetscObjectSetName((PetscObject) z, "Reconstructed vector");

 62:     fplan = fftw_mpi_plan_dft_2d(N0,N1,data_in,data_out,PETSC_COMM_WORLD,FFTW_FORWARD,FFTW_ESTIMATE);
 63:     bplan = fftw_mpi_plan_dft_2d(N0,N1,data_out,data_out2,PETSC_COMM_WORLD,FFTW_BACKWARD,FFTW_ESTIMATE);

 65:     VecSetRandom(x, rdm);
 66:     if (view){VecView(x,PETSC_VIEWER_STDOUT_WORLD);}

 68:     fftw_execute(fplan);
 69:     if (view){VecView(y,PETSC_VIEWER_STDOUT_WORLD);}

 71:     fftw_execute(bplan);

 73:     /* Compare x and z. FFTW computes an unnormalized DFT, thus z = N*x */
 74:     a = 1.0/(PetscReal)N;
 75:     VecScale(z,a);
 76:     if (view){VecView(z, PETSC_VIEWER_STDOUT_WORLD);}
 77:     VecAXPY(z,-1.0,x);
 78:     VecNorm(z,NORM_1,&enorm);
 79:     if (enorm > 1.e-11){
 80:       PetscPrintf(PETSC_COMM_SELF,"  Error norm of |x - z| %G\n",enorm);
 81:     }

 83:     /* Free spaces */
 84:     fftw_destroy_plan(fplan);
 85:     fftw_destroy_plan(bplan);
 86:     fftw_free(data_in);  VecDestroy(&x);
 87:     fftw_free(data_out); VecDestroy(&y);
 88:     fftw_free(data_out2);VecDestroy(&z);

 90:   } else {
 91:     /* Use PETSc-FFTW interface                  */
 92:     /*-------------------------------------------*/
 93:     PetscInt i,*dim,k,DIM;
 94:     Mat      A;
 95:     Vec      input,output;

 97:     N=30;
 98:     for (i=2; i<5; i++){
 99:       DIM = i;
100:       PetscMalloc(i*sizeof(PetscInt),&dim);
101:       for(k=0;k<i;k++){
102:         dim[k]=30;
103:       }
104:       N *= dim[i-1];
105: 
106:       /* Create FFTW object */
107:       if (!rank) printf("Use PETSc-FFTW interface...%d-DIM:%d \n",DIM,N);
108:       MatCreateFFT(PETSC_COMM_WORLD,DIM,dim,MATFFTW,&A);

110:       /* Create FFTW vectors that are compatible with parallel layout of A */
111:       MatGetVecsFFTW(A,&x,&y,&z);
112:       PetscObjectSetName((PetscObject) x, "Real space vector");
113:       PetscObjectSetName((PetscObject) y, "Frequency space vector");
114:       PetscObjectSetName((PetscObject) z, "Reconstructed vector");

116:       /* Create and set PETSc vector */
117:       VecCreate(PETSC_COMM_WORLD,&input);
118:       VecSetSizes(input,PETSC_DECIDE,N);
119:       VecSetFromOptions(input);
120:       VecSetRandom(input,rdm);
121:       VecDuplicate(input,&output);
122:       if (view){VecView(input,PETSC_VIEWER_STDOUT_WORLD);}

124:       /* Vector input is copied to another vector x using VecScatterPetscToFFTW. This is because the user data 
125:          can have any parallel layout. But FFTW requires special parallel layout of the data. Hence the original 
126:          data which is in the vector "input" here, needs to be copied to a vector x, which has the correct parallel
127:          layout for FFTW. Also, during parallel real transform, this pads extra zeros automatically
128:          at the end of last  dimension. This padding is required by FFTW to perform parallel real D.F.T.  */
129:       VecScatterPetscToFFTW(A,input,x);
130: 
131:       /* Apply FFTW_FORWARD and FFTW_BACKWARD */
132:       MatMult(A,x,y);
133:       if (view){VecView(y,PETSC_VIEWER_STDOUT_WORLD);}
134:       MatMultTranspose(A,y,z);

136:       /* Output from Backward DFT needs to be modified to obtain user readable data the routine VecScatterFFTWToPetsc 
137:          performs the job. In some sense this is the reverse operation of VecScatterPetscToFFTW. This routine gets rid of 
138:          the extra spaces that were artificially padded to perform real parallel transform.    */
139:       VecScatterFFTWToPetsc(A,z,output);

141:       /* Compare x and z. FFTW computes an unnormalized DFT, thus z = N*x */
142:       a = 1.0/(PetscReal)N;
143:       VecScale(output,a);
144:       if (view){VecView(output,PETSC_VIEWER_STDOUT_WORLD);}
145:       VecAXPY(output,-1.0,input);
146:       VecNorm(output,NORM_1,&enorm);
147:       if (enorm > 1.e-09 && !rank){
148:         PetscPrintf(PETSC_COMM_SELF,"  Error norm of |x - z| %e\n",enorm);
149:       }

151:       /* Free spaces */
152:       PetscFree(dim);
153:       VecDestroy(&input);
154:       VecDestroy(&output);
155:       VecDestroy(&x);
156:       VecDestroy(&y);
157:       VecDestroy(&z);
158:       MatDestroy(&A);
159:     }
160:   }
161:   PetscRandomDestroy(&rdm);
162:   PetscFinalize();
163:   return 0;
164: }