SHOGUN
v2.0.0
|
00001 /* 00002 * This program is free software; you can redistribute it and/or modify 00003 * it under the terms of the GNU General Public License as published by 00004 * the Free Software Foundation; either version 3 of the License, or 00005 * (at your option) any later version. 00006 * 00007 * Written (W) 2008-2009 Soeren Sonnenburg 00008 * Written (W) 2011-2012 Heiko Strathmann 00009 * Copyright (C) 2008-2009 Fraunhofer Institute FIRST and Max Planck Society 00010 */ 00011 00012 #include <shogun/lib/config.h> 00013 #include <shogun/base/SGObject.h> 00014 #include <shogun/io/SGIO.h> 00015 #include <shogun/base/Parallel.h> 00016 #include <shogun/base/init.h> 00017 #include <shogun/base/Version.h> 00018 #include <shogun/base/Parameter.h> 00019 #include <shogun/base/ParameterMap.h> 00020 #include <shogun/base/DynArray.h> 00021 #include <shogun/lib/Map.h> 00022 00023 00024 00025 #include <stdlib.h> 00026 #include <stdio.h> 00027 00028 #include <shogun/features/DenseFeatures.h> 00029 namespace shogun 00030 { 00031 class CMath; 00032 class Parallel; 00033 class IO; 00034 class Version; 00035 00036 extern CMath* sg_math; 00037 extern Parallel* sg_parallel; 00038 extern SGIO* sg_io; 00039 extern Version* sg_version; 00040 00041 template<> void CSGObject::set_generic<bool>() 00042 { 00043 m_generic = PT_BOOL; 00044 } 00045 00046 template<> void CSGObject::set_generic<char>() 00047 { 00048 m_generic = PT_CHAR; 00049 } 00050 00051 template<> void CSGObject::set_generic<int8_t>() 00052 { 00053 m_generic = PT_INT8; 00054 } 00055 00056 template<> void CSGObject::set_generic<uint8_t>() 00057 { 00058 m_generic = PT_UINT8; 00059 } 00060 00061 template<> void CSGObject::set_generic<int16_t>() 00062 { 00063 m_generic = PT_INT16; 00064 } 00065 00066 template<> void CSGObject::set_generic<uint16_t>() 00067 { 00068 m_generic = PT_UINT16; 00069 } 00070 00071 template<> void CSGObject::set_generic<int32_t>() 00072 { 00073 m_generic = PT_INT32; 00074 } 00075 00076 template<> void CSGObject::set_generic<uint32_t>() 00077 { 00078 m_generic = PT_UINT32; 00079 } 00080 00081 template<> void CSGObject::set_generic<int64_t>() 00082 { 00083 m_generic = PT_INT64; 00084 } 00085 00086 template<> void CSGObject::set_generic<uint64_t>() 00087 { 00088 m_generic = PT_UINT64; 00089 } 00090 00091 template<> void CSGObject::set_generic<float32_t>() 00092 { 00093 m_generic = PT_FLOAT32; 00094 } 00095 00096 template<> void CSGObject::set_generic<float64_t>() 00097 { 00098 m_generic = PT_FLOAT64; 00099 } 00100 00101 template<> void CSGObject::set_generic<floatmax_t>() 00102 { 00103 m_generic = PT_FLOATMAX; 00104 } 00105 00106 } /* namespace shogun */ 00107 00108 using namespace shogun; 00109 00110 CSGObject::CSGObject() 00111 { 00112 init(); 00113 set_global_objects(); 00114 00115 SG_GCDEBUG("SGObject created (%p)\n", this); 00116 } 00117 00118 CSGObject::CSGObject(const CSGObject& orig) 00119 :io(orig.io), parallel(orig.parallel), version(orig.version) 00120 { 00121 init(); 00122 set_global_objects(); 00123 } 00124 00125 CSGObject::~CSGObject() 00126 { 00127 SG_GCDEBUG("SGObject destroyed (%p)\n", this); 00128 00129 #ifdef HAVE_PTHREAD 00130 PTHREAD_LOCK_DESTROY(&m_ref_lock); 00131 #endif 00132 unset_global_objects(); 00133 delete m_parameters; 00134 delete m_model_selection_parameters; 00135 delete m_parameter_map; 00136 } 00137 00138 #ifdef USE_REFERENCE_COUNTING 00139 00140 int32_t CSGObject::ref() 00141 { 00142 #ifdef HAVE_PTHREAD 00143 PTHREAD_LOCK(&m_ref_lock); 00144 #endif //HAVE_PTHREAD 00145 ++m_refcount; 00146 int32_t count=m_refcount; 00147 #ifdef HAVE_PTHREAD 00148 PTHREAD_UNLOCK(&m_ref_lock); 00149 #endif //HAVE_PTHREAD 00150 SG_GCDEBUG("ref() refcount %ld obj %s (%p) increased\n", count, this->get_name(), this); 00151 return m_refcount; 00152 } 00153 00154 int32_t CSGObject::ref_count() 00155 { 00156 #ifdef HAVE_PTHREAD 00157 PTHREAD_LOCK(&m_ref_lock); 00158 #endif //HAVE_PTHREAD 00159 int32_t count=m_refcount; 00160 #ifdef HAVE_PTHREAD 00161 PTHREAD_UNLOCK(&m_ref_lock); 00162 #endif //HAVE_PTHREAD 00163 SG_GCDEBUG("ref_count(): refcount %d, obj %s (%p)\n", count, this->get_name(), this); 00164 return count; 00165 } 00166 00167 int32_t CSGObject::unref() 00168 { 00169 #ifdef HAVE_PTHREAD 00170 PTHREAD_LOCK(&m_ref_lock); 00171 #endif //HAVE_PTHREAD 00172 if (m_refcount==0 || --m_refcount==0) 00173 { 00174 SG_GCDEBUG("unref() refcount %ld, obj %s (%p) destroying\n", m_refcount, this->get_name(), this); 00175 #ifdef HAVE_PTHREAD 00176 PTHREAD_UNLOCK(&m_ref_lock); 00177 #endif //HAVE_PTHREAD 00178 delete this; 00179 return 0; 00180 } 00181 else 00182 { 00183 SG_GCDEBUG("unref() refcount %ld obj %s (%p) decreased\n", m_refcount, this->get_name(), this); 00184 #ifdef HAVE_PTHREAD 00185 PTHREAD_UNLOCK(&m_ref_lock); 00186 #endif //HAVE_PTHREAD 00187 return m_refcount; 00188 } 00189 } 00190 #endif //USE_REFERENCE_COUNTING 00191 00192 00193 void CSGObject::set_global_objects() 00194 { 00195 if (!sg_io || !sg_parallel || !sg_version) 00196 { 00197 fprintf(stderr, "call init_shogun() before using the library, dying.\n"); 00198 exit(1); 00199 } 00200 00201 SG_REF(sg_io); 00202 SG_REF(sg_parallel); 00203 SG_REF(sg_version); 00204 00205 io=sg_io; 00206 parallel=sg_parallel; 00207 version=sg_version; 00208 } 00209 00210 void CSGObject::unset_global_objects() 00211 { 00212 SG_UNREF(version); 00213 SG_UNREF(parallel); 00214 SG_UNREF(io); 00215 } 00216 00217 void CSGObject::set_global_io(SGIO* new_io) 00218 { 00219 SG_UNREF(sg_io); 00220 sg_io=new_io; 00221 SG_REF(sg_io); 00222 } 00223 00224 SGIO* CSGObject::get_global_io() 00225 { 00226 SG_REF(sg_io); 00227 return sg_io; 00228 } 00229 00230 void CSGObject::set_global_parallel(Parallel* new_parallel) 00231 { 00232 SG_UNREF(sg_parallel); 00233 sg_parallel=new_parallel; 00234 SG_REF(sg_parallel); 00235 } 00236 00237 bool CSGObject::update_parameter_hash() 00238 { 00239 uint32_t new_hash = 0; 00240 uint32_t carry = 0; 00241 uint32_t length = 0; 00242 00243 get_parameter_incremental_hash(m_parameters, new_hash, 00244 carry, length); 00245 00246 new_hash = CHash::FinalizeIncrementalMurmurHash3(new_hash, 00247 carry, length); 00248 00249 if(new_hash != m_hash) 00250 { 00251 m_hash = new_hash; 00252 return true; 00253 } 00254 00255 else 00256 return false; 00257 } 00258 00259 Parallel* CSGObject::get_global_parallel() 00260 { 00261 SG_REF(sg_parallel); 00262 return sg_parallel; 00263 } 00264 00265 void CSGObject::set_global_version(Version* new_version) 00266 { 00267 SG_UNREF(sg_version); 00268 sg_version=new_version; 00269 SG_REF(sg_version); 00270 } 00271 00272 Version* CSGObject::get_global_version() 00273 { 00274 SG_REF(sg_version); 00275 return sg_version; 00276 } 00277 00278 bool CSGObject::is_generic(EPrimitiveType* generic) const 00279 { 00280 *generic = m_generic; 00281 00282 return m_generic != PT_NOT_GENERIC; 00283 } 00284 00285 void CSGObject::unset_generic() 00286 { 00287 m_generic = PT_NOT_GENERIC; 00288 } 00289 00290 void CSGObject::print_serializable(const char* prefix) 00291 { 00292 SG_PRINT("\n%s\n================================================================================\n", get_name()); 00293 m_parameters->print(prefix); 00294 } 00295 00296 bool CSGObject::save_serializable(CSerializableFile* file, 00297 const char* prefix, int32_t param_version) 00298 { 00299 SG_DEBUG("START SAVING CSGObject '%s'\n", get_name()); 00300 try 00301 { 00302 save_serializable_pre(); 00303 } 00304 catch (ShogunException e) 00305 { 00306 SG_SWARNING("%s%s::save_serializable_pre(): ShogunException: " 00307 "%s\n", prefix, get_name(), 00308 e.get_exception_string()); 00309 return false; 00310 } 00311 if (!m_save_pre_called) 00312 { 00313 SG_SWARNING("%s%s::save_serializable_pre(): Implementation " 00314 "error: BASE_CLASS::LOAD_SERIALIZABLE_PRE() not " 00315 "called!\n", prefix, get_name()); 00316 return false; 00317 } 00318 00319 /* save parameter version */ 00320 if (!save_parameter_version(file, prefix, param_version)) 00321 return false; 00322 00323 if (!m_parameters->save(file, prefix)) 00324 return false; 00325 00326 try 00327 { 00328 save_serializable_post(); 00329 } 00330 catch (ShogunException e) 00331 { 00332 SG_SWARNING("%s%s::save_serializable_post(): ShogunException: " 00333 "%s\n", prefix, get_name(), 00334 e.get_exception_string()); 00335 return false; 00336 } 00337 00338 if (!m_save_post_called) 00339 { 00340 SG_SWARNING("%s%s::save_serializable_post(): Implementation " 00341 "error: BASE_CLASS::LOAD_SERIALIZABLE_POST() not " 00342 "called!\n", prefix, get_name()); 00343 return false; 00344 } 00345 00346 if (prefix == NULL || *prefix == '\0') 00347 file->close(); 00348 00349 SG_DEBUG("DONE SAVING CSGObject '%s' (%p)\n", get_name(), this); 00350 00351 return true;; 00352 } 00353 00354 bool CSGObject::load_serializable(CSerializableFile* file, 00355 const char* prefix, int32_t param_version) 00356 { 00357 SG_DEBUG("START LOADING CSGObject '%s'\n", get_name()); 00358 try 00359 { 00360 load_serializable_pre(); 00361 } 00362 catch (ShogunException e) 00363 { 00364 SG_SWARNING("%s%s::load_serializable_pre(): ShogunException: " 00365 "%s\n", prefix, get_name(), 00366 e.get_exception_string()); 00367 return false; 00368 } 00369 if (!m_load_pre_called) 00370 { 00371 SG_SWARNING("%s%s::load_serializable_pre(): Implementation " 00372 "error: BASE_CLASS::LOAD_SERIALIZABLE_PRE() not " 00373 "called!\n", prefix, get_name()); 00374 return false; 00375 } 00376 00377 /* try to load version of parameters */ 00378 int32_t file_version=load_parameter_version(file, prefix); 00379 SG_DEBUG("file_version=%d, current_version=%d\n", file_version, param_version); 00380 00381 if (file_version<0) 00382 { 00383 SG_WARNING("%s%s::load_serializable(): File contains no parameter " 00384 "version. Seems like your file is from the days before this " 00385 "was introduced. Ignore warning or serialize with this version " 00386 "of shogun to get rid of above and this warnings.\n", 00387 prefix, get_name()); 00388 } 00389 00390 if (file_version>param_version) 00391 { 00392 if (param_version==VERSION_PARAMETER) 00393 { 00394 SG_WARNING("%s%s::load_serializable(): parameter version of file " 00395 "larger than the one of shogun. Try with a more recent" 00396 "version of shogun.\n", prefix, get_name()); 00397 } 00398 else 00399 { 00400 SG_WARNING("%s%s::load_serializable(): parameter version of file " 00401 "larger than the current. This is probably an implementation" 00402 " error.\n", prefix, get_name()); 00403 } 00404 return false; 00405 } 00406 00407 if (file_version==param_version) 00408 { 00409 /* load normally if file has current version */ 00410 SG_DEBUG("loading normally\n"); 00411 00412 /* load all parameters, except new ones */ 00413 for (int32_t i=0; i<m_parameters->get_num_parameters(); i++) 00414 { 00415 TParameter* current=m_parameters->get_parameter(i); 00416 00417 /* skip new parameters */ 00418 if (is_param_new(SGParamInfo(current, param_version))) 00419 continue; 00420 00421 if (!current->load(file, prefix)) 00422 return false; 00423 } 00424 } 00425 else 00426 { 00427 /* load all parameters from file, mappings to current version */ 00428 DynArray<TParameter*>* param_base=load_all_file_parameters(file_version, 00429 param_version, file, prefix); 00430 00431 /* create an array of param infos from current parameters */ 00432 DynArray<const SGParamInfo*>* param_infos= 00433 new DynArray<const SGParamInfo*>(); 00434 for (index_t i=0; i<m_parameters->get_num_parameters(); ++i) 00435 { 00436 TParameter* current=m_parameters->get_parameter(i); 00437 00438 /* skip new parameters */ 00439 if (is_param_new(SGParamInfo(current, param_version))) 00440 continue; 00441 00442 param_infos->append_element( 00443 new SGParamInfo(current, param_version)); 00444 } 00445 00446 /* map all parameters, result may be empty if input is */ 00447 map_parameters(param_base, file_version, param_infos); 00448 SG_DEBUG("mapping is done!\n"); 00449 00450 /* this is assumed now, mapping worked or no parameters in base */ 00451 ASSERT(file_version==param_version || !param_base->get_num_elements()); 00452 00453 /* delete above created param infos */ 00454 for (index_t i=0; i<param_infos->get_num_elements(); ++i) 00455 delete param_infos->get_element(i); 00456 00457 delete param_infos; 00458 00459 /* replace parameters by loaded and mapped */ 00460 SG_DEBUG("replacing parameter data by loaded/mapped values\n"); 00461 for (index_t i=0; i<m_parameters->get_num_parameters(); ++i) 00462 { 00463 TParameter* current=m_parameters->get_parameter(i); 00464 char* s=SG_MALLOC(char, 200); 00465 current->m_datatype.to_string(s, 200); 00466 SG_DEBUG("processing \"%s\": %s\n", current->m_name, s); 00467 SG_FREE(s); 00468 00469 /* skip new parameters */ 00470 if (is_param_new(SGParamInfo(current, param_version))) 00471 { 00472 SG_DEBUG("%s is new, skipping\n", current->m_name); 00473 continue; 00474 } 00475 00476 /* search for current parameter in mapped ones */ 00477 index_t index=CMath::binary_search(param_base->get_array(), 00478 param_base->get_num_elements(), current); 00479 00480 TParameter* migrated=param_base->get_element(index); 00481 00482 /* now copy data from migrated TParameter instance 00483 * (this automatically deletes the old data allocations) */ 00484 SG_DEBUG("copying migrated data into parameter\n"); 00485 current->copy_data(migrated); 00486 } 00487 00488 /* delete the migrated parameter data base */ 00489 SG_DEBUG("deleting old parameter base\n"); 00490 for (index_t i=0; i<param_base->get_num_elements(); ++i) 00491 { 00492 TParameter* current=param_base->get_element(i); 00493 SG_DEBUG("deleting old \"%s\"\n", current->m_name); 00494 delete current; 00495 } 00496 delete param_base; 00497 } 00498 00499 try 00500 { 00501 load_serializable_post(); 00502 } 00503 catch (ShogunException e) 00504 { 00505 SG_SWARNING("%s%s::load_serializable_post(): ShogunException: " 00506 "%s\n", prefix, get_name(), 00507 e.get_exception_string()); 00508 return false; 00509 } 00510 00511 if (!m_load_post_called) 00512 { 00513 SG_SWARNING("%s%s::load_serializable_post(): Implementation " 00514 "error: BASE_CLASS::LOAD_SERIALIZABLE_POST() not " 00515 "called!\n", prefix, get_name()); 00516 return false; 00517 } 00518 SG_DEBUG("DONE LOADING CSGObject '%s' (%p)\n", get_name(), this); 00519 00520 return true; 00521 } 00522 00523 DynArray<TParameter*>* CSGObject::load_file_parameters( 00524 const SGParamInfo* param_info, int32_t file_version, 00525 CSerializableFile* file, const char* prefix) 00526 { 00527 /* ensure that recursion works */ 00528 SG_SDEBUG("entering %s::load_file_parameters\n", get_name()); 00529 if (file_version>param_info->m_param_version) 00530 { 00531 SG_SERROR("parameter version of \"%s\" in file (%d) is more recent than" 00532 " provided %d!\n", param_info->m_name, file_version, 00533 param_info->m_param_version); 00534 } 00535 00536 DynArray<TParameter*>* result_array=new DynArray<TParameter*>(); 00537 00538 /* do mapping */ 00539 char* s=param_info->to_string(); 00540 SG_SDEBUG("try to get mapping for: %s\n", s); 00541 SG_FREE(s); 00542 00543 /* mapping has only be deleted if was created here (no mapping was found) */ 00544 bool free_mapped=false; 00545 DynArray<const SGParamInfo*>* mapped=m_parameter_map->get(param_info); 00546 if (!mapped) 00547 { 00548 /* since a new mapped array will be created, set deletion flag */ 00549 free_mapped=true; 00550 mapped=new DynArray<const SGParamInfo*>(); 00551 00552 /* if no mapping was found, nothing has changed. Simply create new param 00553 * info with decreased version */ 00554 SG_SDEBUG("no mapping found\n"); 00555 if (file_version<param_info->m_param_version) 00556 { 00557 /* create new array and put param info with decreased version in */ 00558 mapped->append_element(new SGParamInfo(param_info->m_name, 00559 param_info->m_ctype, param_info->m_stype, 00560 param_info->m_ptype, param_info->m_param_version-1)); 00561 00562 SG_SDEBUG("using:\n"); 00563 for (index_t i=0; i<mapped->get_num_elements(); ++i) 00564 { 00565 s=mapped->get_element(i)->to_string(); 00566 SG_SDEBUG("\t%s\n", s); 00567 SG_FREE(s); 00568 } 00569 } 00570 else 00571 { 00572 /* create new array and put original param info in */ 00573 SG_SDEBUG("reached file version\n"); 00574 mapped->append_element(param_info->duplicate()); 00575 } 00576 } 00577 else 00578 { 00579 SG_SDEBUG("found:\n"); 00580 for (index_t i=0; i<mapped->get_num_elements(); ++i) 00581 { 00582 s=mapped->get_element(i)->to_string(); 00583 SG_SDEBUG("\t%s\n", s); 00584 SG_FREE(s); 00585 } 00586 } 00587 00588 00589 /* case file version same as provided version. 00590 * means that parameters have to be loaded from file, recursion stops */ 00591 if (file_version==param_info->m_param_version) 00592 { 00593 SG_SDEBUG("recursion stop, loading from file\n"); 00594 /* load all parameters in mapping from file */ 00595 for (index_t i=0; i<mapped->get_num_elements(); ++i) 00596 { 00597 const SGParamInfo* current=mapped->get_element(i); 00598 s=current->to_string(); 00599 SG_SDEBUG("loading %s\n", s); 00600 SG_FREE(s); 00601 00602 TParameter* loaded; 00603 /* allocate memory for length and matrix/vector 00604 * This has to be done because this stuff normally is in the class 00605 * variables which do not exist in this case. Deletion is handled 00606 * via the allocated_from_scratch flag of TParameter */ 00607 00608 /* create type and copy lengths, empty data for now */ 00609 TSGDataType type(current->m_ctype, current->m_stype, 00610 current->m_ptype); 00611 loaded=new TParameter(&type, NULL, current->m_name, ""); 00612 00613 /* allocate data/length variables for the TParameter, lengths are not 00614 * important now, so set to one */ 00615 loaded->allocate_data_from_scratch(1, 1); 00616 00617 /* tell instance to load data from file */ 00618 if (!loaded->load(file, prefix)) 00619 { 00620 s=param_info->to_string(); 00621 SG_ERROR("Could not load %s. The reason for this might be wrong " 00622 "parameter mappings\n", s); 00623 SG_FREE(s); 00624 } 00625 00626 SG_DEBUG("loaded lengths: y=%d, x=%d\n", 00627 loaded->m_datatype.m_length_y ? *loaded->m_datatype.m_length_y : -1, 00628 loaded->m_datatype.m_length_x ? *loaded->m_datatype.m_length_x : -1); 00629 00630 /* append new TParameter to result array */ 00631 result_array->append_element(loaded); 00632 } 00633 SG_SDEBUG("done loading from file\n"); 00634 } 00635 /* recursion with mapped type, a mapping exists in this case (ensured by 00636 * above assert) */ 00637 else 00638 { 00639 /* for all elements in mapping, do recursion */ 00640 for (index_t i=0; i<mapped->get_num_elements(); ++i) 00641 { 00642 const SGParamInfo* current=mapped->get_element(i); 00643 s=current->to_string(); 00644 SG_SDEBUG("starting recursion over %s\n", s); 00645 00646 /* recursively get all file parameters for this parameter */ 00647 DynArray<TParameter*>* recursion_array= 00648 load_file_parameters(current, file_version, file, prefix); 00649 00650 SG_SDEBUG("recursion over %s done\n", s); 00651 SG_FREE(s); 00652 00653 /* append all recursion data to current array */ 00654 SG_SDEBUG("appending all results to current result\n"); 00655 for (index_t j=0; j<recursion_array->get_num_elements(); ++j) 00656 result_array->append_element(recursion_array->get_element(j)); 00657 00658 /* clean up */ 00659 delete recursion_array; 00660 } 00661 } 00662 00663 SG_SDEBUG("cleaning up old mapping \n"); 00664 00665 00666 /* clean up mapping */ 00667 if (free_mapped) 00668 { 00669 for (index_t i=0; i<mapped->get_num_elements(); ++i) 00670 delete mapped->get_element(i); 00671 00672 delete mapped; 00673 } 00674 00675 SG_SDEBUG("leaving %s::load_file_parameters\n", get_name()); 00676 return result_array; 00677 } 00678 00679 DynArray<TParameter*>* CSGObject::load_all_file_parameters(int32_t file_version, 00680 int32_t current_version, CSerializableFile* file, const char* prefix) 00681 { 00682 DynArray<TParameter*>* result=new DynArray<TParameter*>(); 00683 00684 for (index_t i=0; i<m_parameters->get_num_parameters(); ++i) 00685 { 00686 TParameter* current=m_parameters->get_parameter(i); 00687 00688 /* extract current parameter info */ 00689 const SGParamInfo* info=new SGParamInfo(current, current_version); 00690 00691 /* skip new parameters */ 00692 if (is_param_new(*info)) 00693 { 00694 delete info; 00695 continue; 00696 } 00697 00698 /* in the other case, load parameters data from file */ 00699 DynArray<TParameter*>* temp=load_file_parameters(info, file_version, 00700 file, prefix); 00701 00702 /* and append them all to array */ 00703 for (index_t j=0; j<temp->get_num_elements(); ++j) 00704 result->append_element(temp->get_element(j)); 00705 00706 /* clean up */ 00707 delete temp; 00708 delete info; 00709 } 00710 00711 /* sort array before returning */ 00712 CMath::qsort(result->get_array(), result->get_num_elements()); 00713 00714 return result; 00715 } 00716 00717 void CSGObject::map_parameters(DynArray<TParameter*>* param_base, 00718 int32_t& base_version, DynArray<const SGParamInfo*>* target_param_infos) 00719 { 00720 SG_DEBUG("entering %s::map_parameters\n", get_name()); 00721 /* NOTE: currently the migration is done step by step over every version */ 00722 00723 if (!target_param_infos->get_num_elements()) 00724 { 00725 SG_DEBUG("no target parameter infos\n"); 00726 SG_DEBUG("leaving %s::map_parameters\n", get_name()); 00727 return; 00728 } 00729 00730 /* map all target parameter infos once */ 00731 DynArray<const SGParamInfo*>* mapped_infos= 00732 new DynArray<const SGParamInfo*>(); 00733 DynArray<SGParamInfo*>* to_delete=new DynArray<SGParamInfo*>(); 00734 for (index_t i=0; i<target_param_infos->get_num_elements(); ++i) 00735 { 00736 const SGParamInfo* current=target_param_infos->get_element(i); 00737 00738 char* s=current->to_string(); 00739 SG_DEBUG("trying to get parameter mapping for %s\n", s); 00740 SG_FREE(s); 00741 00742 DynArray<const SGParamInfo*>* mapped=m_parameter_map->get(current); 00743 00744 if (mapped) 00745 { 00746 mapped_infos->append_element(mapped->get_element(0)); 00747 for (index_t j=0; j<mapped->get_num_elements(); ++j) 00748 { 00749 s=mapped->get_element(j)->to_string(); 00750 SG_DEBUG("found mapping: %s\n", s); 00751 SG_FREE(s); 00752 } 00753 } 00754 else 00755 { 00756 /* these have to be deleted above */ 00757 SGParamInfo* no_change=new SGParamInfo(*current); 00758 no_change->m_param_version--; 00759 s=no_change->to_string(); 00760 SG_DEBUG("no mapping found, using %s\n", s); 00761 SG_FREE(s); 00762 mapped_infos->append_element(no_change); 00763 to_delete->append_element(no_change); 00764 } 00765 } 00766 00767 /* assert that at least one mapping exists */ 00768 ASSERT(mapped_infos->get_num_elements()); 00769 int32_t mapped_version=mapped_infos->get_element(0)->m_param_version; 00770 00771 /* assert that all param versions are equal for now (if not empty param) */ 00772 for (index_t i=1; i<mapped_infos->get_num_elements(); ++i) 00773 { 00774 ASSERT(mapped_infos->get_element(i)->m_param_version==mapped_version || 00775 *mapped_infos->get_element(i)==SGParamInfo()); 00776 } 00777 00778 /* recursion, after this call, base is at version of mapped infos */ 00779 if (mapped_version>base_version) 00780 map_parameters(param_base, base_version, mapped_infos); 00781 00782 /* delete mapped parameter infos array */ 00783 delete mapped_infos; 00784 00785 /* delete newly created parameter infos which have to name or type change */ 00786 for (index_t i=0; i<to_delete->get_num_elements(); ++i) 00787 delete to_delete->get_element(i); 00788 00789 delete to_delete; 00790 00791 ASSERT(base_version==mapped_version); 00792 00793 /* do migration of one version step, create new base */ 00794 DynArray<TParameter*>* new_base=new DynArray<TParameter*>(); 00795 for (index_t i=0; i<target_param_infos->get_num_elements(); ++i) 00796 { 00797 char* s=target_param_infos->get_element(i)->to_string(); 00798 SG_DEBUG("migrating one step to target: %s\n", s); 00799 SG_FREE(s); 00800 TParameter* p=migrate(param_base, target_param_infos->get_element(i)); 00801 new_base->append_element(p); 00802 } 00803 00804 /* replace base by new base, delete old base, if it was created in migrate */ 00805 SG_DEBUG("deleting parameters base version %d\n", base_version); 00806 for (index_t i=0; i<param_base->get_num_elements(); ++i) 00807 delete param_base->get_element(i); 00808 00809 SG_DEBUG("replacing old parameter base\n"); 00810 *param_base=*new_base; 00811 base_version=mapped_version+1; 00812 00813 SG_DEBUG("new parameter base of size %d:\n", param_base->get_num_elements()); 00814 for (index_t i=0; i<param_base->get_num_elements(); ++i) 00815 { 00816 TParameter* current=param_base->get_element(i); 00817 TSGDataType type=current->m_datatype; 00818 if (type.m_ptype==PT_SGOBJECT) 00819 { 00820 if (type.m_ctype==CT_SCALAR) 00821 { 00822 CSGObject* object=*(CSGObject**)current->m_parameter; 00823 SG_DEBUG("(%d:) \"%s\": sgobject \"%s\" at %p\n", i, 00824 current->m_name, object ? object->get_name() : "", 00825 object); 00826 } 00827 else 00828 { 00829 index_t len=1; 00830 len*=type.m_length_x ? *type.m_length_x : 1; 00831 len*=type.m_length_y ? *type.m_length_y : 1; 00832 CSGObject** array=*(CSGObject***)current->m_parameter; 00833 for (index_t j=0; j<len; ++j) 00834 { 00835 CSGObject* object=array[j]; 00836 SG_DEBUG("(%d:) \"%s\": sgobject \"%s\" at %p\n", i, 00837 current->m_name, object ? object->get_name() : "", 00838 object); 00839 } 00840 } 00841 } 00842 else 00843 { 00844 char* s=SG_MALLOC(char, 200); 00845 current->m_datatype.to_string(s, 200); 00846 SG_DEBUG("(%d:) \"%s\": type: %s at %p\n", i, current->m_name, s, 00847 current->m_parameter); 00848 SG_FREE(s); 00849 } 00850 } 00851 00852 /* because content was copied, new base may be deleted */ 00853 delete new_base; 00854 00855 /* sort the just created new base */ 00856 SG_DEBUG("sorting base\n"); 00857 CMath::qsort(param_base->get_array(), param_base->get_num_elements()); 00858 00859 /* at this point the param_base is at the same version as the version of 00860 * the provided parameter infos */ 00861 SG_DEBUG("leaving %s::map_parameters\n", get_name()); 00862 } 00863 00864 void CSGObject::one_to_one_migration_prepare(DynArray<TParameter*>* param_base, 00865 const SGParamInfo* target, TParameter*& replacement, 00866 TParameter*& to_migrate, char* old_name) 00867 { 00868 SG_DEBUG("CSGObject::entering CSGObject::one_to_one_migration_prepare() for " 00869 "\"%s\"\n", target->m_name); 00870 00871 /* generate type of target structure */ 00872 TSGDataType type(target->m_ctype, target->m_stype, target->m_ptype); 00873 00874 /* first find index of needed data. 00875 * in this case, element in base with same name or old name */ 00876 char* name=target->m_name; 00877 if (old_name) 00878 name=old_name; 00879 00880 /* dummy for searching, search and save result in to_migrate parameter */ 00881 TParameter* t=new TParameter(&type, NULL, name, ""); 00882 index_t i=CMath::binary_search(param_base->get_array(), 00883 param_base->get_num_elements(), t); 00884 delete t; 00885 00886 /* assert that something is found */ 00887 ASSERT(i>=0); 00888 to_migrate=param_base->get_element(i); 00889 00890 /* result structure, data NULL for now */ 00891 replacement=new TParameter(&type, NULL, target->m_name, 00892 to_migrate->m_description); 00893 00894 /* allocate content to write into, lengths are needed for this */ 00895 index_t len_x=1; 00896 if (to_migrate->m_datatype.m_length_x) 00897 len_x=*to_migrate->m_datatype.m_length_x; 00898 00899 index_t len_y=1; 00900 if (to_migrate->m_datatype.m_length_y) 00901 len_y=*to_migrate->m_datatype.m_length_y; 00902 00903 replacement->allocate_data_from_scratch(len_y, len_x); 00904 00905 /* in case of sgobject, copy pointer data and SG_REF */ 00906 if (to_migrate->m_datatype.m_ptype==PT_SGOBJECT) 00907 { 00908 /* note that the memory is already allocated before the migrate call */ 00909 CSGObject* object=*((CSGObject**)to_migrate->m_parameter); 00910 *((CSGObject**)replacement->m_parameter)=object; 00911 SG_REF(object); 00912 SG_DEBUG("copied and SG_REF sgobject pointer for \"%s\" at %p\n", 00913 object->get_name(), object); 00914 } 00915 00916 /* tell the old TParameter to delete its data on deletion */ 00917 to_migrate->m_delete_data=true; 00918 00919 SG_DEBUG("CSGObject::leaving CSGObject::one_to_one_migration_prepare() for " 00920 "\"%s\"\n", target->m_name); 00921 } 00922 00923 TParameter* CSGObject::migrate(DynArray<TParameter*>* param_base, 00924 const SGParamInfo* target) 00925 { 00926 SG_DEBUG("entering %s::migrate\n", get_name()); 00927 /* this is only executed, iff there was no migration method which handled 00928 * migration to the provided target. In this case, it is assumed that the 00929 * parameter simply has not changed. Verify this here and return copy of 00930 * data in case its true. 00931 * If not, throw an exception -- parameter migration HAS to be implemented 00932 * by hand everytime, a parameter changes type or name. */ 00933 00934 TParameter* result=NULL; 00935 00936 /* first find index of needed data. 00937 * in this case, element in base with same name */ 00938 /* type is also needed */ 00939 TSGDataType type(target->m_ctype, target->m_stype, 00940 target->m_ptype); 00941 00942 /* dummy for searching, search and save result */ 00943 TParameter* t=new TParameter(&type, NULL, target->m_name, ""); 00944 index_t i=CMath::binary_search(param_base->get_array(), 00945 param_base->get_num_elements(), t); 00946 delete t; 00947 00948 /* check if name change occurred while no migration method was specified */ 00949 if (i<0) 00950 { 00951 SG_ERROR("Name change for parameter that has to be mapped to \"%s\"," 00952 " and to no migration method available\n", target->m_name); 00953 } 00954 00955 TParameter* to_migrate=param_base->get_element(i); 00956 00957 /* check if element in base is equal to target one */ 00958 if (*target==SGParamInfo(to_migrate, target->m_param_version)) 00959 { 00960 char* s=SG_MALLOC(char, 200); 00961 to_migrate->m_datatype.to_string(s, 200); 00962 SG_DEBUG("nothing changed, using old data: %s\n", s); 00963 SG_FREE(s); 00964 result=new TParameter(&to_migrate->m_datatype, NULL, to_migrate->m_name, 00965 to_migrate->m_description); 00966 00967 int len_x=1; 00968 if (to_migrate->m_datatype.m_length_x) 00969 len_x=*to_migrate->m_datatype.m_length_x; 00970 00971 int len_y=1; 00972 if (to_migrate->m_datatype.m_length_y) 00973 len_y=*to_migrate->m_datatype.m_length_y; 00974 00975 /* allocate lengths and evtl scalar data but not non-scalar data (no 00976 * new_cont call */ 00977 result->allocate_data_from_scratch(len_y, len_x, false); 00978 00979 /* now use old data */ 00980 if (to_migrate->m_datatype.m_ctype==CT_SCALAR && 00981 to_migrate->m_datatype.m_ptype!=PT_SGOBJECT) 00982 { 00983 /* copy data */ 00984 SG_DEBUG("copying scalar data\n"); 00985 memcpy(result->m_parameter,to_migrate->m_parameter, 00986 to_migrate->m_datatype.get_size()); 00987 } 00988 else 00989 { 00990 /* copy content of pointer */ 00991 SG_DEBUG("copying content of poitner for non-scalar data\n"); 00992 *(void**)result->m_parameter=*(void**)(to_migrate->m_parameter); 00993 } 00994 } 00995 else 00996 { 00997 char* s=target->to_string(); 00998 SG_ERROR("No migration method available for %s!\n", s); 00999 SG_FREE(s); 01000 } 01001 01002 SG_DEBUG("leaving %s::migrate\n", get_name()); 01003 01004 return result; 01005 } 01006 01007 bool CSGObject::save_parameter_version(CSerializableFile* file, 01008 const char* prefix, int32_t param_version) 01009 { 01010 TSGDataType t(CT_SCALAR, ST_NONE, PT_INT32); 01011 TParameter p(&t, ¶m_version, "version_parameter", 01012 "Version of parameters of this object"); 01013 return p.save(file, prefix); 01014 } 01015 01016 int32_t CSGObject::load_parameter_version(CSerializableFile* file, 01017 const char* prefix) 01018 { 01019 TSGDataType t(CT_SCALAR, ST_NONE, PT_INT32); 01020 int32_t v; 01021 TParameter tp(&t, &v, "version_parameter", ""); 01022 if (tp.load(file, prefix)) 01023 return v; 01024 else 01025 return -1; 01026 } 01027 01028 void CSGObject::load_serializable_pre() throw (ShogunException) 01029 { 01030 m_load_pre_called = true; 01031 } 01032 01033 void CSGObject::load_serializable_post() throw (ShogunException) 01034 { 01035 m_load_post_called = true; 01036 } 01037 01038 void CSGObject::save_serializable_pre() throw (ShogunException) 01039 { 01040 m_save_pre_called = true; 01041 } 01042 01043 void CSGObject::save_serializable_post() throw (ShogunException) 01044 { 01045 m_save_post_called = true; 01046 } 01047 01048 #ifdef TRACE_MEMORY_ALLOCS 01049 #include <shogun/lib/Map.h> 01050 extern CMap<void*, shogun::MemoryBlock>* sg_mallocs; 01051 #endif 01052 01053 void CSGObject::init() 01054 { 01055 #ifdef HAVE_PTHREAD 01056 PTHREAD_LOCK_INIT(&m_ref_lock); 01057 #endif 01058 01059 #ifdef TRACE_MEMORY_ALLOCS 01060 if (sg_mallocs) 01061 { 01062 int32_t idx=sg_mallocs->index_of(this); 01063 if (idx>-1) 01064 { 01065 MemoryBlock* b=sg_mallocs->get_element_ptr(idx); 01066 b->set_sgobject(); 01067 } 01068 } 01069 #endif 01070 01071 m_refcount = 0; 01072 io = NULL; 01073 parallel = NULL; 01074 version = NULL; 01075 m_parameters = new Parameter(); 01076 m_model_selection_parameters = new Parameter(); 01077 m_parameter_map=new ParameterMap(); 01078 m_generic = PT_NOT_GENERIC; 01079 m_load_pre_called = false; 01080 m_load_post_called = false; 01081 m_hash = 0; 01082 } 01083 01084 void CSGObject::print_modsel_params() 01085 { 01086 SG_PRINT("parameters available for model selection for %s:\n", get_name()); 01087 01088 index_t num_param=m_model_selection_parameters->get_num_parameters(); 01089 01090 if (!num_param) 01091 SG_PRINT("\tnone\n"); 01092 01093 for (index_t i=0; i<num_param; i++) 01094 { 01095 TParameter* current=m_model_selection_parameters->get_parameter(i); 01096 index_t l=200; 01097 char* type=SG_MALLOC(char, l); 01098 if (type) 01099 { 01100 current->m_datatype.to_string(type, l); 01101 SG_PRINT("\t%s (%s): %s\n", current->m_name, current->m_description, 01102 type); 01103 SG_FREE(type); 01104 } 01105 } 01106 } 01107 01108 SGStringList<char> CSGObject::get_modelsel_names() 01109 { 01110 index_t num_param=m_model_selection_parameters->get_num_parameters(); 01111 01112 SGStringList<char> result(num_param, -1); 01113 01114 index_t max_string_length=-1; 01115 01116 for (index_t i=0; i<num_param; i++) 01117 { 01118 char* name=m_model_selection_parameters->get_parameter(i)->m_name; 01119 index_t len=strlen(name); 01120 // +1 to have a zero terminated string 01121 result.strings[i]=SGString<char>(name, len+1); 01122 01123 if (len>max_string_length) 01124 max_string_length=len; 01125 } 01126 01127 result.max_string_length=max_string_length; 01128 01129 return result; 01130 } 01131 01132 char* CSGObject::get_modsel_param_descr(const char* param_name) 01133 { 01134 index_t index=get_modsel_param_index(param_name); 01135 01136 if (index<0) 01137 { 01138 SG_ERROR("There is no model selection parameter called \"%s\" for %s", 01139 param_name, get_name()); 01140 } 01141 01142 return m_model_selection_parameters->get_parameter(index)->m_description; 01143 } 01144 01145 index_t CSGObject::get_modsel_param_index(const char* param_name) 01146 { 01147 /* use fact that names extracted from below method are in same order than 01148 * in m_model_selection_parameters variable */ 01149 SGStringList<char> names=get_modelsel_names(); 01150 01151 /* search for parameter with provided name */ 01152 index_t index=-1; 01153 for (index_t i=0; i<names.num_strings; i++) 01154 { 01155 TParameter* current=m_model_selection_parameters->get_parameter(i); 01156 if (!strcmp(param_name, current->m_name)) 01157 { 01158 index=i; 01159 break; 01160 } 01161 } 01162 01163 /* clean up */ 01164 names.destroy_list(); 01165 01166 return index; 01167 } 01168 01169 bool CSGObject::is_param_new(const SGParamInfo param_info) const 01170 { 01171 /* check if parameter is new in this version (has empty mapping) */ 01172 DynArray<const SGParamInfo*>* value=m_parameter_map->get(¶m_info); 01173 bool result=value && *value->get_element(0) == SGParamInfo(); 01174 01175 return result; 01176 } 01177 01178 void CSGObject::get_parameter_incremental_hash(Parameter* param, 01179 uint32_t& hash, uint32_t& carry, uint32_t& total_length) 01180 { 01181 if (param) 01182 { 01183 for (index_t i=0; i<param->get_num_parameters(); i++) 01184 { 01185 TParameter* p = param->get_parameter(i); 01186 01187 if (p->m_datatype.m_ptype == PT_SGOBJECT) 01188 { 01189 CSGObject* child = 01190 *((CSGObject**)(p->m_parameter)); 01191 01192 if (child) 01193 get_parameter_incremental_hash( 01194 child->m_parameters, hash, 01195 carry, total_length); 01196 } 01197 01198 else 01199 p->get_incremental_hash(hash, carry, total_length); 01200 } 01201 } 01202 } 01203 01204 void CSGObject::build_parameter_dictionary(CMap<TParameter*, CSGObject*>& dict) 01205 { 01206 if (m_parameters) 01207 { 01208 for (index_t i=0; i<m_parameters->get_num_parameters(); i++) 01209 { 01210 TParameter* p = m_parameters->get_parameter(i); 01211 01212 dict.add(p, this); 01213 01214 if (p->m_datatype.m_ptype == PT_SGOBJECT) 01215 { 01216 CSGObject* child = 01217 *((CSGObject**)(p->m_parameter)); 01218 if (child) 01219 child->build_parameter_dictionary(dict); 01220 } 01221 01222 } 01223 } 01224 } 01225