AFEPack
|
00001 00051 #ifndef __MPI_LoadBalance_h__ 00052 #define __MPI_LoadBalance_h__ 00053 00054 #ifdef __MPI_ULoadBalance_h__ 00055 #error "MPI_LoadBalance.h 和 MPI_ULoadBalance.h 是冲突的,不能一起用。" 00056 #endif 00057 00058 #include <sys/types.h> 00059 #include <sys/stat.h> 00060 #include <unistd.h> 00061 #include <cstdio> 00062 00063 #include <boost/archive/binary_iarchive.hpp> 00064 #include <boost/archive/binary_oarchive.hpp> 00065 #include <boost/archive/text_iarchive.hpp> 00066 #include <boost/archive/text_oarchive.hpp> 00067 00068 #include <AFEPack/Serialization.h> 00069 #include "MPI.h" 00070 #include "MPI_HGeometry_archive.h" 00071 #include "MPI_Migration.h" 00072 00073 namespace MPI { 00074 00075 template <class FOREST> 00076 class HLoadBalance { 00077 public: 00078 typedef FOREST forest_t; 00079 enum {dim = FOREST::dim, dow = FOREST::dow}; 00080 typedef typename forest_t::matcher_t matcher_t; 00081 private: 00082 // 在计算每个几何体上的负载是使用的性质 00083 property_id_t<u_int> _pid_loading; 00084 00085 // 这个性质的 first 为新分区中的秩, second 为对其进行存储的秩 00086 property_id_t<std::map<int, int> > _pid_rank_map; 00087 00088 // 部分需要拆分和合并的几何体的全局标号 00089 property_id_t<unsigned long> _pid_global_idx; 00090 00091 // 元素为(全局标号,对象指针)的对 00092 std::map<unsigned long, void *> _global_pointer_to_merge; 00093 00094 // (进程,列表(全局标号,维数,对象指针)组)的映射 00095 typedef std::map<unsigned long, std::pair<int, void *> > tuple_map_t; 00096 std::map<int, tuple_map_t> _global_pointer_to_share; 00097 00098 typedef BirdView<forest_t> birdview_t; 00099 typedef BirdViewSet<forest_t> birdview_set_t; 00100 typedef HLoadBalance<forest_t> this_t; 00101 00102 forest_t * _forest; 00103 std::vector<u_int> _cut_point; 00104 std::vector<int> _new_rank; 00105 00106 public: 00107 HLoadBalance(forest_t& forest) : _forest(&forest) {} 00108 ~HLoadBalance() {} 00109 00110 private: 00111 template <class GEO> std::map<int,int> * 00112 new_rank_map(GEO& geo) const { 00113 return geo.new_property(_pid_rank_map); 00114 } 00115 template <class GEO> unsigned long * 00116 new_global_idx(const GEO& geo) const { 00117 return geo.new_property(_pid_global_idx); 00118 } 00119 00120 public: 00121 void clear() { 00122 free_property_id(_pid_loading); 00123 free_property_id(_pid_rank_map); 00124 free_property_id(_pid_global_idx); 00125 _global_pointer_to_merge.clear(); 00126 _global_pointer_to_share.clear(); 00127 _cut_point.clear(); 00128 _new_rank.clear(); 00129 } 00130 00131 template <class GEO> std::map<int,int> * 00132 get_rank_map(GEO& geo) const { 00133 return geo.get_property(_pid_rank_map); 00134 } 00135 template <class GEO> unsigned long * 00136 get_global_idx(const GEO& geo) const { 00137 return geo.get_property(_pid_global_idx); 00138 } 00139 00148 bool is_save_on_this_rank(std::map<int,int> * map, 00149 int new_rank) { 00150 typename std::map<int,int>::iterator 00151 the_pair = map->find(new_rank); 00153 assert (the_pair != map->end()); 00154 00156 return (the_pair->second == _forest->rank()); 00157 } 00158 00170 template <class GEO> 00171 void merge_global_pointer(int type, 00172 unsigned long global_idx, 00173 GEO *& p_geo) { 00174 std::map<unsigned long, void *>::iterator 00175 the_pair = _global_pointer_to_merge.find(global_idx); 00176 if (type == 2) { 00177 if (the_pair == _global_pointer_to_merge.end()) { 00178 _global_pointer_to_merge.insert(std::pair<unsigned long,void*>(global_idx, p_geo)); 00179 } else { 00180 assert (the_pair->second == p_geo); 00181 } 00182 } else { 00183 assert (type == 4 && the_pair != _global_pointer_to_merge.end()); 00184 p_geo = (GEO *)(the_pair->second); 00185 } 00186 } 00187 00196 template <class GEO> 00197 void share_global_pointer(int rank, 00198 unsigned long global_idx, 00199 GEO *& p_geo) { 00200 _global_pointer_to_share[rank][global_idx] = 00201 std::pair<int,void *>(p_geo->dimension, p_geo); 00202 } 00203 00204 private: 00208 void share_global_pointer() { 00209 std::cerr << "Exchanging shared global pointers ..." << std::endl; 00210 00211 typedef std::map<int, tuple_map_t> map_t; 00212 00213 int n = 0; 00214 std::list<int> target; 00215 std::list<BinaryBuffer<> > out_buf, in_buf; 00216 typename map_t::iterator 00217 the_pair = _global_pointer_to_share.begin(), 00218 end_pair = _global_pointer_to_share.end(); 00219 for (;the_pair != end_pair;++ the_pair, ++ n) { 00220 target.push_back(the_pair->first); 00221 00222 out_buf.push_back(BinaryBuffer<>()); 00223 AFEPack::ostream<> os(out_buf.back()); 00224 00225 in_buf.push_back(BinaryBuffer<>()); 00226 00227 u_int n_item = the_pair->second.size(); 00228 os << n_item; 00229 std::cerr << "sending " << the_pair->second.size() 00230 << " items from " << _forest->rank() 00231 << " to " << the_pair->first << std::endl; 00232 typename tuple_map_t::iterator 00233 the_tuple = the_pair->second.begin(), 00234 end_tuple = the_pair->second.end(); 00235 for (;the_tuple != end_tuple;++ the_tuple) { 00236 os << the_tuple->first 00237 << the_tuple->second.first 00238 << the_tuple->second.second; 00239 } 00240 } 00241 00242 MPI_Barrier(_forest->communicator()); 00243 sendrecv_data(_forest->communicator(), n, out_buf.begin(), 00244 in_buf.begin(), target.begin()); 00245 00246 typename std::list<int>::iterator 00247 the_rank = target.begin(), 00248 end_rank = target.end(); 00249 typename std::list<BinaryBuffer<> >::iterator 00250 the_buf = in_buf.begin(); 00251 for (;the_rank != end_rank;++ the_rank, ++ the_buf) { 00252 AFEPack::istream<> is(*the_buf); 00253 u_int n_item; 00254 is >> n_item; 00255 std::cerr << "received " << n_item 00256 << " items from " << *the_rank 00257 << " to " << _forest->rank() << std::endl; 00258 for (u_int i = 0;i < n_item;++ i) { 00259 unsigned long global_idx; 00260 int dimension; 00261 void * remote_obj; 00262 is >> global_idx >> dimension >> remote_obj; 00263 std::pair<int,void *>& the_pair = 00264 _global_pointer_to_share[*the_rank][global_idx]; 00265 assert (dimension == the_pair.first && 00266 dimension >= 0 && dimension <= 3); 00267 00268 void * local_obj = the_pair.second; 00269 if (dimension == 0) { 00270 #define GDIM 0 00271 #define GEO HGeometry<GDIM,dow> 00272 #define OBJ Shared_object<GEO> 00273 GEO * p_geo = (GEO *)local_obj; 00274 OBJ * p_info = _forest->get_shared_info(*p_geo); 00275 if (p_info == NULL) p_info = _forest->new_shared_info(*p_geo); 00276 p_info->add_clone(*the_rank, (GEO *)remote_obj); 00277 #undef OBJ 00278 #undef GEO 00279 #undef GDIM 00280 } else if (dimension == 1) { 00281 #define GDIM 1 00282 #define GEO HGeometry<GDIM,dow> 00283 #define OBJ Shared_object<GEO> 00284 GEO * p_geo = (GEO *)local_obj; 00285 OBJ * p_info = _forest->get_shared_info(*p_geo); 00286 if (p_info == NULL) p_info = _forest->new_shared_info(*p_geo); 00287 p_info->add_clone(*the_rank, (GEO *)remote_obj); 00288 #undef OBJ 00289 #undef GEO 00290 #undef GDIM 00291 } else if (dimension == 2) { 00292 #define GDIM 2 00293 #define GEO HGeometry<GDIM,dow> 00294 #define OBJ Shared_object<GEO> 00295 GEO * p_geo = (GEO *)local_obj; 00296 OBJ * p_info = _forest->get_shared_info(*p_geo); 00297 if (p_info == NULL) p_info = _forest->new_shared_info(*p_geo); 00298 p_info->add_clone(*the_rank, (GEO *)remote_obj); 00299 #undef OBJ 00300 #undef GEO 00301 #undef GDIM 00302 } else if (dimension == 3) { 00303 #define GDIM 3 00304 #define GEO HGeometry<GDIM,dow> 00305 #define OBJ Shared_object<GEO> 00306 GEO * p_geo = (GEO *)local_obj; 00307 OBJ * p_info = _forest->get_shared_info(*p_geo); 00308 if (p_info == NULL) p_info = _forest->new_shared_info(*p_geo); 00309 p_info->add_clone(*the_rank, (GEO *)remote_obj); 00310 #undef OBJ 00311 #undef GEO 00312 #undef GDIM 00313 } 00314 } 00315 } 00316 } 00317 00319 public: 00325 template <class LOADER> 00326 void config(birdview_t& ir_mesh, 00327 LOADER& loader, 00328 u_int (LOADER::*value)(GeometryBM&)) { 00329 00330 if (&(ir_mesh.getForest()) != _forest) { 00331 std::cerr << "The mesh should be on the same forest of the HLoadBalance." 00332 << std::endl; 00333 abort(); 00334 } 00335 00336 RegularMesh<dim,dow>& mesh = ir_mesh.regularMesh(); 00337 new_property_id(_pid_loading); 00338 u_int n_ele = mesh.n_geometry(dim); 00339 for (u_int i = 0;i < n_ele;++ i) { 00340 HGeometry<dim,dow> * p_geo = mesh.template h_geometry<dim>(i); 00341 u_int * p_loading = p_geo->get_property(_pid_loading); 00342 if (p_loading == NULL) { 00343 p_loading = p_geo->new_property(_pid_loading); 00344 } 00345 (*p_loading) = (loader.*value)(mesh.geometry(dim,i)); 00346 } 00347 } 00348 00349 void config(birdview_t& ir_mesh, 00350 u_int (*value)(GeometryBM&) = &_default_element_loading) { 00351 if (&(ir_mesh.getForest()) != _forest) { 00352 std::cerr << "The mesh should be on the same forest of the HLoadBalance." 00353 << std::endl; 00354 abort(); 00355 } 00356 00357 RegularMesh<dim,dow>& mesh = ir_mesh.regularMesh(); 00358 new_property_id(_pid_loading); 00359 u_int n_ele = mesh.n_geometry(dim); 00360 for (u_int i = 0;i < n_ele;++ i) { 00361 const HGeometry<dim,dow> * p_geo = mesh.template h_geometry<dim>(i); 00362 u_int * p_loading = p_geo->get_property(_pid_loading); 00363 if (p_loading == NULL) { 00364 p_loading = p_geo->new_property(_pid_loading); 00365 } 00366 (*p_loading) = (*value)(mesh.geometry(dim,i)); 00367 } 00368 } 00369 00370 static u_int _default_element_loading(GeometryBM&) { return 1; } 00372 00374 public: 00379 void partition(u_int n_new_rank = 0) { 00380 int rank = _forest->rank(); 00381 int n_rank = _forest->n_rank(); 00382 00383 u_int loading = this->lump_loading(); 00384 u_int total_loading, partial_loading; 00385 MPI_Comm comm = _forest->communicator(); 00386 MPI_Scan(&loading, &partial_loading, 1, MPI_UNSIGNED, MPI_SUM, comm); 00387 00388 if (rank == n_rank - 1) { 00389 total_loading = partial_loading; 00390 } 00391 if (n_new_rank == 0) n_new_rank = n_rank; 00392 00393 MPI_Bcast(&total_loading, 1, MPI_UNSIGNED, n_rank - 1, comm); 00394 double mean_loading = n_new_rank; 00395 mean_loading = total_loading/mean_loading; 00396 00397 _cut_point.clear(); 00398 _new_rank.clear(); 00399 00400 u_int idx = 0; 00401 _cut_point.push_back(idx); 00402 00403 loading = partial_loading - loading; 00404 u_int current_rank = (u_int)(loading/mean_loading); 00405 _new_rank.push_back(current_rank); 00406 00407 typename forest_t::RootIterator 00408 the_ele = _forest->beginRootElement(), 00409 end_ele = _forest->endRootElement(); 00410 for (;the_ele != end_ele;++ the_ele) { 00411 idx += 1; 00412 loading += this->get_loading(*the_ele); 00413 u_int the_rank = (u_int)(loading/mean_loading); 00414 the_rank = std::min(the_rank, n_new_rank - 1); 00415 if (the_rank > current_rank) { 00416 _cut_point.push_back(idx); 00417 _new_rank.push_back(++ current_rank); 00418 } 00419 } 00420 if (_cut_point.back() != idx) { 00421 _cut_point.push_back(idx); 00422 } else { 00423 _new_rank.pop_back(); 00424 } 00425 00426 free_property_id(_pid_loading); 00427 } 00428 00432 void reuse_partition() { 00433 _cut_point.clear(); 00434 _new_rank.clear(); 00435 00436 _cut_point.resize(2, 0); 00437 _cut_point[1] = _forest->n_rootElement(); 00438 _new_rank.resize(1, _forest->rank()); 00439 } 00440 00441 private: 00446 u_int lump_loading() const { 00447 u_int loading = 0; 00448 typename forest_t::RootIterator 00449 the_ele = _forest->beginRootElement(), 00450 end_ele = _forest->endRootElement(); 00451 for (;the_ele != end_ele;++ the_ele) { 00452 loading += this->get_loading(*the_ele); 00453 } 00454 return loading; 00455 } 00456 00461 template <class GEO> u_int 00462 get_loading(GEO& geo) const { 00463 u_int * p_loading = geo.get_property(_pid_loading); 00464 if (p_loading == NULL) { 00465 p_loading = geo.new_property(_pid_loading); 00466 for (u_int i = 0;i < geo.n_child;++ i) { 00467 (*p_loading) += get_loading(*geo.child[i]); 00468 } 00469 } 00470 return (*p_loading); 00471 } 00473 00475 private: 00481 void set_new_rank() { 00482 new_property_id(_pid_rank_map); 00483 00487 typename forest_t::RootIterator 00488 the_ele = _forest->beginRootElement(); 00489 u_int n_part = _new_rank.size(); 00490 for (u_int i = 0;i < n_part;++ i) { 00491 for (u_int j = _cut_point[i];j < _cut_point[i + 1];++ j, ++ the_ele) { 00492 geometry_set_new_rank(*the_ele, _new_rank[i]); 00493 } 00494 } 00495 00496 if (_forest->n_rank() <= 1) return; 00500 Shared_type_filter::only<0> type_filter; 00501 MPI_Comm comm = _forest->communicator(); 00502 #define SYNC_DATA(D) \ 00503 if (dim >= D) { \ 00504 sync_data(comm, _forest->template get_shared_list<D>(), *this, \ 00505 &this_t::template pack_set_new_rank<D>, \ 00506 &this_t::template unpack_set_new_rank<D>, \ 00507 type_filter); \ 00508 } 00509 SYNC_DATA(0); 00510 SYNC_DATA(1); 00511 SYNC_DATA(2); 00512 SYNC_DATA(3); 00513 #undef SYNC_DATA 00514 } 00515 00516 public: 00517 template <int GDIM> void 00518 pack_set_new_rank(HGeometry<GDIM,dow> * geo, 00519 int remote_rank, 00520 AFEPack::ostream<>& os) { 00521 std::map<int,int> * p_map = get_rank_map(*geo); 00522 assert (p_map != NULL); 00523 00524 os << p_map->size(); 00525 typename std::map<int,int>::iterator 00526 the_rank = p_map->begin(), 00527 end_rank = p_map->end(); 00528 for (;the_rank != end_rank;++ the_rank) { 00529 os << the_rank->first << the_rank->second; 00530 } 00531 } 00532 00533 template <int GDIM> void 00534 unpack_set_new_rank(HGeometry<GDIM,dow> * geo, 00535 int remote_rank, 00536 AFEPack::istream<>& is) { 00543 std::map<int,int> * p_map = get_rank_map(*geo); 00544 assert (p_map != NULL); 00545 00546 int new_rank, old_rank; 00547 std::size_t n; 00548 is >> n; 00549 for (std::size_t i = 0;i < n;++ i) { 00550 is >> new_rank >> old_rank; 00551 typename std::map<int,int>::iterator 00552 the_pair = p_map->find(new_rank); 00553 if (the_pair == p_map->end()) { 00554 p_map->insert(std::pair<int,int>(new_rank, old_rank)); 00555 } else { 00557 if (the_pair->second > old_rank) { 00558 the_pair->second = old_rank; 00559 } 00560 } 00561 } 00562 } 00563 00564 private: 00569 template <class GEO> void 00570 geometry_set_new_rank(GEO& geo, int new_rank) const { 00571 std::map<int,int> * p_map = get_rank_map(geo); 00572 if (p_map == NULL) p_map = new_rank_map(geo); 00573 p_map->insert(std::pair<int,int>(new_rank, _forest->rank())); 00574 00576 for (u_int i = 0;i < geo.n_vertex;++ i) { 00577 geometry_set_new_rank(*geo.vertex[i], new_rank); 00578 } 00579 for (u_int i = 0;i < geo.n_boundary;++ i) { 00580 geometry_set_new_rank(*geo.boundary[i], new_rank); 00581 } 00582 if (geo.isRefined()) { 00583 for (u_int i = 0;i < geo.n_child;++ i) { 00584 geometry_set_new_rank(*geo.child[i], new_rank); 00585 } 00586 } 00587 } 00589 00591 private: 00607 void global_index() { 00611 new_property_id(_pid_global_idx); 00612 unsigned long idx = 0; 00613 typename forest_t::RootIterator 00614 the_ele = _forest->beginRootElement(), 00615 end_ele = _forest->endRootElement(); 00616 for (;the_ele != end_ele;++ the_ele) { 00617 geometry_global_index(*the_ele, idx); 00618 } 00619 free_property_id(_pid_global_idx); 00620 00624 MPI_Comm comm = _forest->communicator(); 00625 unsigned long global_idx = idx; 00626 MPI_Scan(&idx, &global_idx, 1, MPI_UNSIGNED_LONG, MPI_SUM, comm); 00627 idx = global_idx - idx; 00628 00632 new_property_id(_pid_global_idx); 00633 the_ele = _forest->beginRootElement(); 00634 for (;the_ele != end_ele;++ the_ele) { 00635 geometry_global_index(*the_ele, idx); 00636 } 00637 00638 if (_forest->n_rank() <= 1) return; 00643 Shared_type_filter::only<0> type_filter; 00644 #define SYNC_DATA(D) \ 00645 if (dim >= D) { \ 00646 sync_data(comm, _forest->template get_shared_list<D>(), *this, \ 00647 &this_t::template pack_global_index<D>, \ 00648 &this_t::template unpack_global_index<D>, \ 00649 type_filter); \ 00650 } 00651 SYNC_DATA(0); 00652 SYNC_DATA(1); 00653 SYNC_DATA(2); 00654 SYNC_DATA(3); 00655 #undef SYNC_DATA 00656 } 00657 00658 public: 00659 template <int GDIM> void 00660 pack_global_index(HGeometry<GDIM,dow> * geo, 00661 int remote_rank, 00662 AFEPack::ostream<>& os) { 00663 unsigned long * p_idx = get_global_idx(*geo); 00664 assert (p_idx != NULL); 00665 00666 os << *p_idx; 00667 } 00668 00674 template <int GDIM> void 00675 unpack_global_index(HGeometry<GDIM,dow> * geo, 00676 int remote_rank, 00677 AFEPack::istream<>& is) { 00678 unsigned long * p_idx = get_global_idx(*geo); 00679 assert (p_idx != NULL); 00680 00681 unsigned long remote_idx; 00682 is >> remote_idx; 00683 if (*p_idx > remote_idx) { 00684 *p_idx = remote_idx; 00685 } 00686 } 00687 00688 private: 00692 template <class GEO> void 00693 geometry_global_index(GEO& geo, 00694 unsigned long& idx) const { 00695 unsigned long * p_idx = get_global_idx(geo); 00696 if (p_idx != NULL) return; 00697 00698 std::map<int,int> * p_map = get_rank_map(geo); 00699 if (_forest->get_shared_info(geo) != NULL || 00700 p_map->size() > 1) { 00701 p_idx = new_global_idx(geo); 00702 *p_idx = idx ++; 00703 } 00704 00706 for (u_int i = 0;i < geo.n_vertex;++ i) { 00707 geometry_global_index(*geo.vertex[i], idx); 00708 } 00709 for (u_int i = 0;i < geo.n_boundary;++ i) { 00710 geometry_global_index(*geo.boundary[i], idx); 00711 } 00712 if (geo.isRefined()) { 00713 for (u_int i = 0;i < geo.n_child;++ i) { 00714 geometry_global_index(*geo.child[i], idx); 00715 } 00716 } 00717 } 00719 00721 public: 00722 void write_config_file(const std::string& dirname) { 00723 char filename[1024]; 00724 sprintf(filename, "%s/.config", dirname.c_str()); 00725 std::ofstream os(filename); 00726 os << _forest->n_rank() << "\t# number of old rank" << std::endl; 00727 os.close(); 00728 Migration::save_config(dirname); 00729 } 00730 00731 private: 00732 void export_birdview(birdview_t& ir_mesh, 00733 Migration::data_id_t data_id, 00734 u_int idx) { 00735 typename birdview_t::ActiveIterator 00736 the_ele = ir_mesh.beginActiveElement(), 00737 end_ele = ir_mesh.endActiveElement(); 00738 for (;the_ele != end_ele;++ the_ele) { 00739 HGeometry<dim,dow> * p_geo = the_ele->h_element; 00740 AFEPack::ostream<> os(p_geo->buffer[data_id]); 00741 os << idx; 00742 } 00743 } 00744 00745 public: 00746 void save_data(const std::string& dirname, 00747 birdview_set_t& bvs) { 00748 set_new_rank(); 00749 global_index(); 00750 00751 char command[1024], *filename = command; 00752 00753 Migration::data_id_t id = Migration::name_to_id("__internal_birdviewflag__"); 00754 if (! Migration::is_valid(id)) { 00755 id = Migration::register_data_name("__internal_birdviewflag__"); 00756 } 00757 typename birdview_set_t::iterator 00758 the_bv = bvs.begin(), end_bv = bvs.end(); 00759 for (u_int idx = 0;the_bv != end_bv;++ the_bv) { 00760 export_birdview(*the_bv, id, idx ++); 00761 } 00762 00763 typedef boost::archive::HGeometry_oarchive<this_t> archive_t; 00764 typename forest_t::RootIterator 00765 the_ele = _forest->beginRootElement(); 00766 u_int n_part = _new_rank.size(); 00767 for (u_int i = 0;i < n_part;++ i) { 00768 sprintf(command, "mkdir -p %s && mkdir -p %s/%d", 00769 dirname.c_str(), dirname.c_str(), _new_rank[i]); 00770 system(command); 00771 sprintf(filename, "%s/%d/%d.dat", dirname.c_str(), 00772 _new_rank[i], _forest->rank()); 00773 std::ofstream os(filename, std::ios::binary); 00774 00775 { 00776 archive_t oa(*this, _new_rank[i], os); 00777 00778 u_int n_ele = _cut_point[i + 1] - _cut_point[i]; 00779 oa & boost::serialization::make_binary_object(&n_ele, sizeof(u_int)); 00780 std::cerr << "saving data in " << filename 00781 << ", # macro element = " << n_ele << std::endl; 00782 00783 for (u_int j = _cut_point[i];j < _cut_point[i + 1];++ j) { 00784 HGeometry<dim,dow> * p_ele = *(the_ele ++); 00785 oa & p_ele; 00786 } 00787 } 00788 00789 os.close(); 00790 } 00791 00792 if (_forest->rank() == 0) write_config_file(dirname); 00793 MPI_Barrier(_forest->communicator()); 00794 } 00796 00798 public: 00799 int load_config_file(const std::string& dirname) { 00800 Migration::load_config(dirname); 00801 00802 char filename[1024]; 00803 sprintf(filename, "%s/.config", dirname.c_str()); 00804 filtering_istream is; 00805 Migration::ensured_open_filtering_stream(filename, is); 00806 00807 int old_rank; 00808 is >> old_rank; 00809 return old_rank; 00810 } 00811 00812 private: 00813 void reconstruct_birdview(HElement<dim,dow>& ele, 00814 Migration::data_id_t data_id, 00815 u_int idx) const { 00816 HGeometry<dim,dow> * p_geo = ele.h_element; 00817 typename Migration::buffer_t::iterator it = p_geo->buffer.find(data_id); 00818 bool is_active = true; 00819 ele.value = 0; 00820 if (it == p_geo->buffer.end()) { 00821 is_active = false; 00822 } else { 00823 BinaryBuffer<>& buf = it->second; 00824 int n = buf.size()/sizeof(u_int), i; 00825 u_int idx1; 00826 AFEPack::istream<> is(buf); 00827 for (i = 0;i < n;++ i) { 00828 is >> idx1; 00829 if (idx1 == idx) break; 00830 } 00831 if (i == n) is_active = false; 00832 } 00833 if (! is_active) { 00834 ele.value = 1; 00835 ele.refine(); 00836 for (int i = 0;i < ele.n_child;++ i) { 00837 reconstruct_birdview(*ele.child[i], data_id, idx); 00838 } 00839 } 00840 } 00841 00842 void reconstruct_birdview(birdview_t& ir_mesh, 00843 Migration::data_id_t data_id, 00844 u_int idx) const { 00845 typename birdview_t::RootIterator 00846 the_ele = ir_mesh.beginRootElement(), 00847 end_ele = ir_mesh.endRootElement(); 00848 for (;the_ele != end_ele;++ the_ele) { 00849 reconstruct_birdview(*the_ele, data_id, idx); 00850 } 00851 } 00852 00858 template <class GEO> 00859 void set_shared_info_sent(GEO& geo) const { 00860 if ((_forest->get_shared_info(geo) != NULL) & 00861 (! _forest->is_shared_info_sent(geo))) { 00862 _forest->set_shared_info_sent(geo); 00863 } 00864 00865 for (u_int i = 0;i < geo.n_vertex;++ i) { 00866 if (! _forest->is_shared_info_sent(*geo.vertex[i])) { 00867 _forest->set_shared_info_sent(*geo.vertex[i]); 00868 } 00869 } 00870 for (u_int i = 0;i < geo.n_boundary;++ i) { 00871 this->set_shared_info_sent(*geo.boundary[i]); 00872 } 00873 if (geo.isRefined()) { 00874 for (u_int i = 0;i < geo.n_child;++ i) { 00875 this->set_shared_info_sent(*geo.child[i]); 00876 } 00877 } 00878 } 00879 00883 void set_shared_info_sent() const { 00884 typename forest_t::RootIterator 00885 the_ele = _forest->beginRootElement(), 00886 end_ele = _forest->endRootElement(); 00887 for (;the_ele != end_ele;++ the_ele) { 00888 this->set_shared_info_sent(*the_ele); 00889 } 00890 } 00891 00892 public: 00893 void load_data(const std::string& dirname, 00894 birdview_set_t& bvs) { 00895 clear(); 00896 00897 int n_old_rank = load_config_file(dirname); 00898 00900 MPI_Barrier(_forest->communicator()); 00901 00903 char filename[1024]; 00904 00905 typedef boost::archive::HGeometry_iarchive<this_t> archive_t; 00906 00907 int rank = 0; 00908 do { 00909 std::ifstream is; 00910 do { 00911 sprintf(filename, "%s/%d/%d.dat", dirname.c_str(), 00912 _forest->rank(), rank ++); 00913 if (rank > n_old_rank) { 00914 share_global_pointer(); 00915 set_shared_info_sent(); 00916 00918 Migration::data_id_t id = Migration::name_to_id("__internal_birdviewflag__"); 00919 typename birdview_set_t::iterator 00920 the_bv = bvs.begin(), end_bv = bvs.end(); 00921 for (u_int idx = 0;the_bv != end_bv;++ the_bv) { 00922 the_bv->reinit(*_forest); 00923 reconstruct_birdview(*the_bv, id, idx ++); 00924 } 00925 return; 00926 } 00927 is.open(filename, std::ios::binary); 00928 if (is.good()) break; 00929 } while (1); 00930 00931 { 00932 u_int n_ele; 00933 std::cerr << "loading data from " << filename; 00934 archive_t ia(*this, _forest->rank(), is); 00935 ia & boost::serialization::make_binary_object(&n_ele, sizeof(u_int)); 00936 std::cerr << ", # macro element = " << n_ele << std::endl; 00937 00938 for (u_int i = 0;i < n_ele;++ i) { 00939 HGeometry<dim,dow> * p_ele; 00940 ia & p_ele; 00941 _forest->rootElement().push_back(p_ele); 00942 } 00943 } 00944 00945 is.close(); 00946 } while (1); 00947 00948 } 00949 00950 }; 00951 00952 template <class FOREST> 00953 void load_forest(const std::string& dirname, 00954 FOREST& forest, 00955 bool is_has_orphans = false) { 00956 HLoadBalance<FOREST> hlb(forest); 00957 BirdViewSet<FOREST> bvs; 00958 hlb.load_data(dirname, bvs); 00959 } 00960 00961 template <class FOREST> 00962 void load_mesh(const std::string& dirname, 00963 FOREST& forest, 00964 BirdView<FOREST>& mesh, 00965 bool is_has_orphans = false) { 00966 HLoadBalance<FOREST> hlb(forest); 00967 BirdViewSet<FOREST> bvs; 00968 bvs.add(mesh); 00969 hlb.load_data(dirname, bvs); 00970 } 00971 00972 template <class FOREST> 00973 void load_mesh(const std::string& dirname, 00974 FOREST& forest, 00975 u_int n_mesh, ...) { 00976 HLoadBalance<FOREST> hlb(forest); 00977 typedef BirdView<FOREST> * mesh_ptr_t; 00978 00979 BirdViewSet<FOREST> bvs; 00980 va_list ap; 00981 va_start(ap, n_mesh); 00982 for (u_int i = 0;i < n_mesh;++ i) { 00983 mesh_ptr_t p_mesh = va_arg(ap, mesh_ptr_t); 00984 bvs.add(*p_mesh); 00985 } 00986 va_end(ap); 00987 00988 hlb.load_data(dirname, bvs); 00989 } 00990 00991 template <class FOREST> 00992 void load_mesh_set(const std::string& dirname, 00993 FOREST& forest, 00994 BirdViewSet<FOREST>& bvs, 00995 bool is_has_orphans = false) { 00996 HLoadBalance<FOREST> hlb(forest); 00997 hlb.load_data(dirname, bvs); 00998 } 00999 01007 void lb_collect_local_data_dir(MPI_Comm comm, 01008 const std::string& src_dir, 01009 const std::string& dst_dir); 01015 void lb_sync_local_data_dir(MPI_Comm comm, 01016 const std::string& src_dir, 01017 const std::string& dst_dir); 01018 } 01019 01020 #endif // __MPI_LoadBalance_h__ 01021