00001 /* 00002 TODO: 00003 1) Eliminate mb_ and grid_mb_ DONE 00004 00005 1) Allow user level control over whether to use a 1D or 2D data 00006 distribution. 00007 2) Figure out how to redistribute the vectors. (Allow no more than 00008 nb vectors at a time for now). Done - I think 00009 3) Move the code from software:MyExamples/TwodMap/TwoDMap.cpp in to 00010 this routine (for the 2D case) Done - I think 00011 4) Create the ScaLAPACK 2D dense matrix. Done - I think 00012 */ 00013 00014 // @HEADER 00015 // *********************************************************************** 00016 // 00017 // Amesos: Direct Sparse Solver Package 00018 // Copyright (2004) Sandia Corporation 00019 // 00020 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive 00021 // license for use of this work by or on behalf of the U.S. Government. 00022 // 00023 // This library is free software; you can redistribute it and/or modify 00024 // it under the terms of the GNU Lesser General Public License as 00025 // published by the Free Software Foundation; either version 2.1 of the 00026 // License, or (at your option) any later version. 00027 // 00028 // This library is distributed in the hope that it will be useful, but 00029 // WITHOUT ANY WARRANTY; without even the implied warranty of 00030 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 00031 // Lesser General Public License for more details. 00032 // 00033 // You should have received a copy of the GNU Lesser General Public 00034 // License along with this library; if not, write to the Free Software 00035 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 00036 // USA 00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 00038 // 00039 // *********************************************************************** 00040 // @HEADER 00041 00042 #ifndef _AMESOS_SCALAPACK_H_ 00043 #define _AMESOS_SCALAPACK_H_ 00044 00045 #include "Amesos_ConfigDefs.h" 00046 #include "Amesos_BaseSolver.h" 00047 #include "Amesos_NoCopiable.h" 00048 #include "Amesos_Utils.h" 00049 #include "Amesos_Time.h" 00050 #include "Amesos_Status.h" 00051 #include "Amesos_Control.h" 00052 #include "Epetra_LinearProblem.h" 00053 #include "Epetra_Time.h" 00054 #ifdef EPETRA_MPI 00055 #include "Epetra_MpiComm.h" 00056 #else 00057 #include "Epetra_Comm.h" 00058 #endif 00059 #include "Epetra_CrsGraph.h" 00060 00062 00146 class Amesos_Scalapack: public Amesos_BaseSolver, 00147 private Amesos_Time, 00148 private Amesos_NoCopiable, 00149 private Amesos_Utils, 00150 private Amesos_Control, 00151 private Amesos_Status 00152 { 00153 00154 public: 00155 00157 00158 00165 Amesos_Scalapack( const Epetra_LinearProblem& LinearProblem ); 00166 00168 00170 ~Amesos_Scalapack(void); 00172 00174 00176 00183 int SymbolicFactorization() ; 00184 00186 00211 int NumericFactorization() ; 00212 00214 00234 int Solve(); 00235 00237 00239 00240 #if 0 00242 char * Label() const {return(Epetra_Object::Label());}; 00243 #endif 00244 00246 const Epetra_LinearProblem *GetProblem() const { return(Problem_); }; 00247 00249 00252 bool MatrixShapeOK() const ; 00253 00255 00267 int SetUseTranspose(bool UseTranspose) {UseTranspose_ = UseTranspose; return(0);}; 00268 00270 bool UseTranspose() const {return(UseTranspose_);}; 00271 00273 const Epetra_Comm & Comm() const {return(GetProblem()->GetOperator()->Comm());}; 00274 00276 00292 int SetParameters( Teuchos::ParameterList &ParameterList ) ; 00293 00295 int NumSymbolicFact() const { return( Amesos_Status::NumSymbolicFact_ ); } 00296 00298 int NumNumericFact() const { return( Amesos_Status::NumNumericFact_ ); } 00299 00301 int NumSolve() const { return( Amesos_Status::NumSolve_ ); } 00302 00304 void PrintTiming() const; 00305 00307 void PrintStatus() const; 00308 00310 void GetTiming( Teuchos::ParameterList &TimingParameterList ) const { Amesos_Time::GetTiming(TimingParameterList); } 00311 00313 00314 private: 00315 /* 00316 RedistributeA - Convert matrix to a dense ScaLAPACK matrix 00317 Preconditions: 00318 Problem_ must be set 00319 SetParameters() 00320 ScaLAPACK1DMap and ScaLAPACK1DMatrix must either be 0 or be pointers to 00321 appropriatly allocate objects. If they are non-zero, those objects 00322 will be deleted and recreated. 00323 00324 Postconditions: 00325 nprow_, npcol_, DescA_ 00326 RowMatrixA_ 00327 ScaLAPACK1DMap_ 00328 ScaLAPACK1DMatrix_ 00329 ImportToScaLAPACK1D_ 00330 ImportBackToOriginal_ 00331 00332 */ 00333 int RedistributeA(); 00334 00335 /* 00336 ConvertToScalapack - Convert matirx to form expected by Scalapack: Ai, Ap, Aval 00337 Preconditions: 00338 Problem_ 00339 Postconditions: 00340 nprow_, npcol_, 00341 */ 00342 int ConvertToScalapack(); 00343 00344 /* 00345 PerformNumericFactorization - Call Scalapack to perform numeric factorization 00346 Preconditions: 00347 00348 Postconditions: 00349 DenseA_, DescA_ 00350 */ 00351 int PerformNumericFactorization(); 00352 00353 protected: 00354 00355 int iam_; // Process number (i.e. Comm().MyPID() 00356 00357 int NumGlobalElements_; // Number of rows and columns in the Problem_->GetOperator() 00358 int NumGlobalNonzeros_; 00359 00360 // 00361 // The following variables are required for the ScaLAPACK interface: 00362 // 00363 int nprow_ ; // number of process rows: 1 for now 00364 int npcol_ ; // number of process columns 00365 int ictxt_ ; // BLACS context 00366 int m_per_p_; // Number of columns per process 00367 int DescA_[10]; // ScaLAPACK array descriptor 00368 00369 Epetra_Map *ScaLAPACK1DMap_ ; // Points to a 1D Map which matches a ScaLAPACK 1D 00370 // blocked (not block cyclic) distribution 00371 Epetra_CrsMatrix *ScaLAPACK1DMatrix_ ; // Points to a ScaLAPACK 1D 00372 // blocked (not block cyclic) distribution 00373 Epetra_Map *VectorMap_ ; // Points to a Map for vectors X and B 00374 std::vector<double> DenseA_; // The data in a ScaLAPACK 1D blocked format 00375 std::vector<int> Ipiv_ ; // ScaLAPACK pivot information 00376 int NumOurRows_ ; 00377 int NumOurColumns_ ; 00378 00379 00380 bool UseTranspose_; 00381 const Epetra_LinearProblem * Problem_; 00382 00383 00384 // some timing internal to ScaLAPACK 00385 double ConTime_; // time to convert to ScaLAPACKformat 00386 double SymTime_; // time for symbolic factorization 00387 double NumTime_; // time for numeric factorization 00388 double SolTime_; // time for solution 00389 double VecTime_; // time to redistribute vectors 00390 double MatTime_; // time to redistribute matrix 00391 00392 // 00393 // Control of the data distribution 00394 // 00395 bool TwoD_distribution_; // True if 2D data distribution is used 00396 int grid_nb_; // Row and Column blocking factor (only used in 2D distribution) 00397 int mypcol_; // Process column in the ScaLAPACK2D grid 00398 int myprow_; // Process row in the ScaLAPACK2D grid 00399 Epetra_CrsMatrix* FatOut_;// 00400 00401 // 00402 // Blocking factors (For both 1D and 2D data distributions) 00403 // 00404 int nb_; 00405 int lda_; 00406 00407 Epetra_Time * Time_; 00408 00409 }; // End of class Amesos_Scalapack 00410 #endif /* _AMESOS_SCALAPACK_H_ */