ompl/tools/benchmark/src/Benchmark.cpp
00001 /*********************************************************************
00002 * Software License Agreement (BSD License)
00003 *
00004 *  Copyright (c) 2010, Rice University
00005 *  All rights reserved.
00006 *
00007 *  Redistribution and use in source and binary forms, with or without
00008 *  modification, are permitted provided that the following conditions
00009 *  are met:
00010 *
00011 *   * Redistributions of source code must retain the above copyright
00012 *     notice, this list of conditions and the following disclaimer.
00013 *   * Redistributions in binary form must reproduce the above
00014 *     copyright notice, this list of conditions and the following
00015 *     disclaimer in the documentation and/or other materials provided
00016 *     with the distribution.
00017 *   * Neither the name of the Rice University nor the names of its
00018 *     contributors may be used to endorse or promote products derived
00019 *     from this software without specific prior written permission.
00020 *
00021 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
00022 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
00023 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
00024 *  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
00025 *  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
00026 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
00027 *  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
00028 *  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
00029 *  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
00030 *  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
00031 *  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
00032 *  POSSIBILITY OF SUCH DAMAGE.
00033 *********************************************************************/
00034 
00035 /* Author: Ioan Sucan, Luis G. Torres */
00036 
00037 #include "ompl/tools/benchmark/Benchmark.h"
00038 #include "ompl/tools/benchmark/MachineSpecs.h"
00039 #include "ompl/util/Time.h"
00040 #include "ompl/config.h"
00041 #include <boost/scoped_ptr.hpp>
00042 #include <boost/lexical_cast.hpp>
00043 #include <boost/progress.hpp>
00044 #include <boost/thread.hpp>
00045 #include <fstream>
00046 #include <sstream>
00047 
00049 namespace ompl
00050 {
00051     namespace tools
00052     {
00054         static std::string getResultsFilename(const Benchmark::CompleteExperiment &exp)
00055         {
00056             return "ompl_" + exp.host + "_" + boost::posix_time::to_iso_extended_string(exp.startTime) + ".log";
00057         }
00058 
00060         static std::string getConsoleFilename(const Benchmark::CompleteExperiment &exp)
00061         {
00062             return "ompl_" + exp.host + "_" + boost::posix_time::to_iso_extended_string(exp.startTime) + ".console";
00063         }
00064 
00065         static bool terminationCondition(const machine::MemUsage_t maxMem, const time::point &endTime)
00066         {
00067             if (time::now() < endTime && machine::getProcessMemoryUsage() < maxMem)
00068                 return false;
00069             return true;
00070         }
00071 
00072         class RunPlanner
00073         {
00074         public:
00075 
00076             RunPlanner(const Benchmark *benchmark, bool useThreads)
00077                 : benchmark_(benchmark), timeUsed_(0.0), memUsed_(0), useThreads_(useThreads)
00078             {
00079             }
00080 
00081             void run(const base::PlannerPtr &planner, const machine::MemUsage_t memStart, const machine::MemUsage_t maxMem, const double maxTime, const double timeBetweenUpdates)
00082             {
00083                 if (!useThreads_)
00084                 {
00085                     runThread(planner, memStart + maxMem, time::seconds(maxTime), time::seconds(timeBetweenUpdates));
00086                     return;
00087                 }
00088 
00089                 boost::thread t(boost::bind(&RunPlanner::runThread, this, planner, memStart + maxMem, time::seconds(maxTime), time::seconds(timeBetweenUpdates)));
00090 
00091                 // allow 25% more time than originally specified, in order to detect planner termination
00092 #if BOOST_VERSION < 105000
00093                 // For older versions of boost, we have to use this
00094                 // deprecated form of the timed join
00095                 if (!t.timed_join(time::seconds(maxTime * 1.25)))
00096 #else
00097                 if (!t.try_join_for(boost::chrono::duration<double>(maxTime * 1.25)))
00098 #endif
00099                 {
00100                     status_ = base::PlannerStatus::CRASH;
00101 
00102                     std::stringstream es;
00103                     es << "Planner " << benchmark_->getStatus().activePlanner << " did not complete run " << benchmark_->getStatus().activeRun
00104                        << " within the specified amount of time (possible crash). Attempting to force termination of planning thread ..." << std::endl;
00105                     std::cerr << es.str();
00106                     OMPL_ERROR(es.str().c_str());
00107 
00108                     t.interrupt();
00109                     t.join();
00110 
00111                     std::string m = "Planning thread cancelled";
00112                     std::cerr << m << std::endl;
00113                     OMPL_ERROR(m.c_str());
00114                 }
00115 
00116                 if (memStart < memUsed_)
00117                     memUsed_ -= memStart;
00118                 else
00119                     memUsed_ = 0;
00120             }
00121 
00122             double getTimeUsed() const
00123             {
00124                 return timeUsed_;
00125             }
00126 
00127             machine::MemUsage_t getMemUsed() const
00128             {
00129                 return memUsed_;
00130             }
00131 
00132             base::PlannerStatus getStatus() const
00133             {
00134                 return status_;
00135             }
00136 
00137             const Benchmark::RunProgressData& getRunProgressData() const
00138             {
00139                 return runProgressData_;
00140             }
00141 
00142         private:
00143 
00144             void runThread(const base::PlannerPtr &planner, const machine::MemUsage_t maxMem, const time::duration &maxDuration, const time::duration &timeBetweenUpdates)
00145             {
00146                 time::point timeStart = time::now();
00147 
00148                 try
00149                 {
00150                     base::PlannerTerminationConditionFn ptc = boost::bind(&terminationCondition, maxMem, time::now() + maxDuration);
00151                     solved_ = false;
00152                     // Only launch the planner progress property
00153                     // collector if there is any data for it to report
00154                     //
00155                     // \TODO issue here is that at least one sample
00156                     // always gets taken before planner even starts;
00157                     // might be worth adding a short wait time before
00158                     // collector begins sampling
00159                     boost::scoped_ptr<boost::thread> t;
00160                     if (planner->getPlannerProgressProperties().size() > 0)
00161                         t.reset(new boost::thread(boost::bind(&RunPlanner::collectProgressProperties,                                                               this,
00162                                                               planner->getPlannerProgressProperties(),
00163                                                               timeBetweenUpdates)));
00164                     status_ = planner->solve(ptc, 0.1);
00165                     solvedFlag_.lock();
00166                     solved_ = true;
00167                     solvedCondition_.notify_all();
00168                     solvedFlag_.unlock();
00169                     if (t)
00170                         t->join(); // maybe look into interrupting even if planner throws an exception
00171                 }
00172                 catch(std::runtime_error &e)
00173                 {
00174                     std::stringstream es;
00175                     es << "There was an error executing planner " << benchmark_->getStatus().activePlanner <<  ", run = " << benchmark_->getStatus().activeRun << std::endl;
00176                     es << "*** " << e.what() << std::endl;
00177                     std::cerr << es.str();
00178                     OMPL_ERROR(es.str().c_str());
00179                 }
00180 
00181                 timeUsed_ = time::seconds(time::now() - timeStart);
00182                 memUsed_ = machine::getProcessMemoryUsage();
00183             }
00184 
00185             void collectProgressProperties(const base::Planner::PlannerProgressProperties& properties,
00186                                            const time::duration &timePerUpdate)
00187             {
00188                 time::point timeStart = time::now();
00189 
00190                 boost::unique_lock<boost::mutex> ulock(solvedFlag_);
00191                 while (!solved_)
00192                 {
00193                     if (solvedCondition_.timed_wait(ulock, time::now() + timePerUpdate))
00194                         return;
00195                     else
00196                     {
00197                         double timeInSeconds = time::seconds(time::now() - timeStart);
00198                         std::string timeStamp =
00199                             boost::lexical_cast<std::string>(timeInSeconds);
00200                         std::map<std::string, std::string> data;
00201                         data["time REAL"] = timeStamp;
00202                         for (base::Planner::PlannerProgressProperties::const_iterator item = properties.begin();
00203                              item != properties.end();
00204                              ++item)
00205                         {
00206                             data[item->first] = item->second();
00207                         }
00208                         runProgressData_.push_back(data);
00209                     }
00210                 }
00211             }
00212 
00213             const Benchmark    *benchmark_;
00214             double              timeUsed_;
00215             machine::MemUsage_t memUsed_;
00216             base::PlannerStatus status_;
00217             bool                useThreads_;
00218             Benchmark::RunProgressData runProgressData_;
00219 
00220             // variables needed for progress property collection
00221             bool solved_;
00222             boost::mutex solvedFlag_;
00223             boost::condition_variable solvedCondition_;
00224         };
00225 
00226     }
00227 }
00229 
00230 bool ompl::tools::Benchmark::saveResultsToFile(const char *filename) const
00231 {
00232     bool result = false;
00233 
00234     std::ofstream fout(filename);
00235     if (fout.good())
00236     {
00237         result = saveResultsToStream(fout);
00238         OMPL_INFORM("Results saved to '%s'", filename);
00239     }
00240     else
00241     {
00242         // try to save to a different file, if we can
00243         if (getResultsFilename(exp_) != std::string(filename))
00244             result = saveResultsToFile();
00245 
00246         OMPL_ERROR("Unable to write results to '%s'", filename);
00247     }
00248     return result;
00249 }
00250 
00251 bool ompl::tools::Benchmark::saveResultsToFile() const
00252 {
00253     std::string filename = getResultsFilename(exp_);
00254     return saveResultsToFile(filename.c_str());
00255 }
00256 
00257 bool ompl::tools::Benchmark::saveResultsToStream(std::ostream &out) const
00258 {
00259     if (exp_.planners.empty())
00260     {
00261         OMPL_WARN("There is no experimental data to save");
00262         return false;
00263     }
00264 
00265     if (!out.good())
00266     {
00267         OMPL_ERROR("Unable to write to stream");
00268         return false;
00269     }
00270 
00271     out << "OMPL version " << OMPL_VERSION << std::endl;
00272     out << "Experiment " << (exp_.name.empty() ? "NO_NAME" : exp_.name) << std::endl;
00273     out << "Running on " << (exp_.host.empty() ? "UNKNOWN" : exp_.host) << std::endl;
00274     out << "Starting at " << boost::posix_time::to_iso_extended_string(exp_.startTime) << std::endl;
00275     out << "<<<|" << std::endl << exp_.setupInfo << "|>>>" << std::endl;
00276     out << "<<<|" << std::endl << exp_.cpuInfo << "|>>>" << std::endl;
00277 
00278     out << exp_.seed << " is the random seed" << std::endl;
00279     out << exp_.maxTime << " seconds per run" << std::endl;
00280     out << exp_.maxMem << " MB per run" << std::endl;
00281     out << exp_.runCount << " runs per planner" << std::endl;
00282     out << exp_.totalDuration << " seconds spent to collect the data" << std::endl;
00283 
00284     // change this if more enum types are added
00285     out << "1 enum type" << std::endl;
00286     out << "status";
00287     for (unsigned int i = 0 ; i < base::PlannerStatus::TYPE_COUNT ; ++i)
00288         out << '|' << base::PlannerStatus(static_cast<base::PlannerStatus::StatusType>(i)).asString();
00289     out << std::endl;
00290 
00291     out << exp_.planners.size() << " planners" << std::endl;
00292 
00293     for (unsigned int i = 0 ; i < exp_.planners.size() ; ++i)
00294     {
00295         out << exp_.planners[i].name << std::endl;
00296 
00297         // get names of common properties
00298         std::vector<std::string> properties;
00299         for (std::map<std::string, std::string>::const_iterator mit = exp_.planners[i].common.begin() ;
00300              mit != exp_.planners[i].common.end() ; ++mit)
00301             properties.push_back(mit->first);
00302         std::sort(properties.begin(), properties.end());
00303 
00304         // print names & values of common properties
00305         out << properties.size() << " common properties" << std::endl;
00306         for (unsigned int k = 0 ; k < properties.size() ; ++k)
00307         {
00308             std::map<std::string, std::string>::const_iterator it = exp_.planners[i].common.find(properties[k]);
00309             out << it->first << " = " << it->second << std::endl;
00310         }
00311 
00312         // construct the list of all possible properties for all runs
00313         std::map<std::string, bool> propSeen;
00314         for (unsigned int j = 0 ; j < exp_.planners[i].runs.size() ; ++j)
00315             for (std::map<std::string, std::string>::const_iterator mit = exp_.planners[i].runs[j].begin() ;
00316                  mit != exp_.planners[i].runs[j].end() ; ++mit)
00317                 propSeen[mit->first] = true;
00318 
00319         properties.clear();
00320 
00321         for (std::map<std::string, bool>::iterator it = propSeen.begin() ; it != propSeen.end() ; ++it)
00322             properties.push_back(it->first);
00323         std::sort(properties.begin(), properties.end());
00324 
00325         // print the property names
00326         out << properties.size() << " properties for each run" << std::endl;
00327         for (unsigned int j = 0 ; j < properties.size() ; ++j)
00328             out << properties[j] << std::endl;
00329 
00330         // print the data for each run
00331         out << exp_.planners[i].runs.size() << " runs" << std::endl;
00332         for (unsigned int j = 0 ; j < exp_.planners[i].runs.size() ; ++j)
00333         {
00334             for (unsigned int k = 0 ; k < properties.size() ; ++k)
00335             {
00336                 std::map<std::string, std::string>::const_iterator it = exp_.planners[i].runs[j].find(properties[k]);
00337                 if (it != exp_.planners[i].runs[j].end())
00338                     out << it->second;
00339                 out << "; ";
00340             }
00341             out << std::endl;
00342         }
00343 
00344         // print the run progress data if it was reported
00345         if (exp_.planners[i].runsProgressData.size() > 0)
00346         {
00347             // Print number of progress properties
00348             out << exp_.planners[i].progressPropertyNames.size() << " progress properties for each run" << std::endl;
00349             // Print progress property names
00350             for (std::vector<std::string>::const_iterator iter =
00351                      exp_.planners[i].progressPropertyNames.begin();
00352                  iter != exp_.planners[i].progressPropertyNames.end();
00353                  ++iter)
00354             {
00355                 out << *iter << std::endl;
00356             }
00357             // Print progress properties for each run
00358             out << exp_.planners[i].runsProgressData.size() << " runs" << std::endl;
00359             for (std::size_t r = 0; r < exp_.planners[i].runsProgressData.size(); ++r)
00360             {
00361                 // For each time point
00362                 for (std::size_t t = 0; t < exp_.planners[i].runsProgressData[r].size(); ++t)
00363                 {
00364                     // Print each of the properties at that time point
00365                     for (std::map<std::string, std::string>::const_iterator iter =
00366                              exp_.planners[i].runsProgressData[r][t].begin();
00367                          iter != exp_.planners[i].runsProgressData[r][t].end();
00368                          ++iter)
00369                     {
00370                         out << iter->second << ",";
00371                     }
00372 
00373                     // Separate time points by semicolons
00374                     out << ";";
00375                 }
00376 
00377                 // Separate runs by newlines
00378                 out << std::endl;
00379             }
00380         }
00381 
00382         out << '.' << std::endl;
00383     }
00384     return true;
00385 }
00386 
00387 void ompl::tools::Benchmark::benchmark(const Request &req)
00388 {
00389     // sanity checks
00390     if (gsetup_)
00391     {
00392         if (!gsetup_->getSpaceInformation()->isSetup())
00393             gsetup_->getSpaceInformation()->setup();
00394     }
00395     else
00396     {
00397         if (!csetup_->getSpaceInformation()->isSetup())
00398             csetup_->getSpaceInformation()->setup();
00399     }
00400 
00401     if (!(gsetup_ ? gsetup_->getGoal() : csetup_->getGoal()))
00402     {
00403         OMPL_ERROR("No goal defined");
00404         return;
00405     }
00406 
00407     if (planners_.empty())
00408     {
00409         OMPL_ERROR("There are no planners to benchmark");
00410         return;
00411     }
00412 
00413     status_.running = true;
00414     exp_.totalDuration = 0.0;
00415     exp_.maxTime = req.maxTime;
00416     exp_.maxMem = req.maxMem;
00417     exp_.runCount = req.runCount;
00418     exp_.host = machine::getHostname();
00419     exp_.cpuInfo = machine::getCPUInfo();
00420     exp_.seed = RNG::getSeed();
00421 
00422     exp_.startTime = time::now();
00423 
00424     OMPL_INFORM("Configuring planners ...");
00425 
00426     // clear previous experimental data
00427     exp_.planners.clear();
00428     exp_.planners.resize(planners_.size());
00429 
00430     const base::ProblemDefinitionPtr &pdef = gsetup_ ? gsetup_->getProblemDefinition() : csetup_->getProblemDefinition();
00431     // set up all the planners
00432     for (unsigned int i = 0 ; i < planners_.size() ; ++i)
00433     {
00434         // configure the planner
00435         planners_[i]->setProblemDefinition(pdef);
00436         if (!planners_[i]->isSetup())
00437             planners_[i]->setup();
00438         exp_.planners[i].name = (gsetup_ ? "geometric_" : "control_") + planners_[i]->getName();
00439         OMPL_INFORM("Configured %s", exp_.planners[i].name.c_str());
00440     }
00441 
00442     OMPL_INFORM("Done configuring planners.");
00443     OMPL_INFORM("Saving planner setup information ...");
00444 
00445     std::stringstream setupInfo;
00446     if (gsetup_)
00447         gsetup_->print(setupInfo);
00448     else
00449         csetup_->print(setupInfo);
00450     setupInfo << std::endl << "Properties of benchmarked planners:" << std::endl;
00451     for (unsigned int i = 0 ; i < planners_.size() ; ++i)
00452         planners_[i]->printProperties(setupInfo);
00453 
00454     exp_.setupInfo = setupInfo.str();
00455 
00456     OMPL_INFORM("Done saving information");
00457 
00458     OMPL_INFORM("Beginning benchmark");
00459     msg::OutputHandler *oh = msg::getOutputHandler();
00460     boost::scoped_ptr<msg::OutputHandlerFile> ohf;
00461     if (req.saveConsoleOutput)
00462     {
00463         ohf.reset(new msg::OutputHandlerFile(getConsoleFilename(exp_).c_str()));
00464         msg::useOutputHandler(ohf.get());
00465     }
00466     else
00467         msg::noOutputHandler();
00468     OMPL_INFORM("Beginning benchmark");
00469 
00470     boost::scoped_ptr<boost::progress_display> progress;
00471     if (req.displayProgress)
00472     {
00473         std::cout << "Running experiment " << exp_.name << "." << std::endl;
00474         std::cout << "Each planner will be executed " << req.runCount << " times for at most " << req.maxTime << " seconds. Memory is limited at "
00475                   << req.maxMem << "MB." << std::endl;
00476         progress.reset(new boost::progress_display(100, std::cout));
00477     }
00478 
00479     machine::MemUsage_t memStart = machine::getProcessMemoryUsage();
00480     machine::MemUsage_t maxMemBytes = (machine::MemUsage_t)(req.maxMem * 1024 * 1024);
00481 
00482     for (unsigned int i = 0 ; i < planners_.size() ; ++i)
00483     {
00484         status_.activePlanner = exp_.planners[i].name;
00485         // execute planner switch event, if set
00486         try
00487         {
00488             if (plannerSwitch_)
00489             {
00490                 OMPL_INFORM("Executing planner-switch event for planner %s ...", status_.activePlanner.c_str());
00491                 plannerSwitch_(planners_[i]);
00492                 OMPL_INFORM("Completed execution of planner-switch event");
00493             }
00494         }
00495         catch(std::runtime_error &e)
00496         {
00497             std::stringstream es;
00498             es << "There was an error executing the planner-switch event for planner " << status_.activePlanner << std::endl;
00499             es << "*** " << e.what() << std::endl;
00500             std::cerr << es.str();
00501             OMPL_ERROR(es.str().c_str());
00502         }
00503         if (gsetup_)
00504             gsetup_->setup();
00505         else
00506             csetup_->setup();
00507         planners_[i]->params().getParams(exp_.planners[i].common);
00508         planners_[i]->getSpaceInformation()->params().getParams(exp_.planners[i].common);
00509 
00510         // Add planner progress property names to struct
00511         exp_.planners[i].progressPropertyNames.push_back("time REAL");
00512         base::Planner::PlannerProgressProperties::const_iterator iter;
00513         for (iter = planners_[i]->getPlannerProgressProperties().begin();
00514              iter != planners_[i]->getPlannerProgressProperties().end();
00515              ++iter)
00516         {
00517             exp_.planners[i].progressPropertyNames.push_back(iter->first);
00518         }
00519         std::sort(exp_.planners[i].progressPropertyNames.begin(),
00520                   exp_.planners[i].progressPropertyNames.end());
00521 
00522         // run the planner
00523         for (unsigned int j = 0 ; j < req.runCount ; ++j)
00524         {
00525             status_.activeRun = j;
00526             status_.progressPercentage = (double)(100 * (req.runCount * i + j)) / (double)(planners_.size() * req.runCount);
00527 
00528             if (req.displayProgress)
00529                 while (status_.progressPercentage > progress->count())
00530                     ++(*progress);
00531 
00532             OMPL_INFORM("Preparing for run %d of %s", status_.activeRun, status_.activePlanner.c_str());
00533 
00534             // make sure all planning data structures are cleared
00535             try
00536             {
00537                 planners_[i]->clear();
00538                 if (gsetup_)
00539                 {
00540                     gsetup_->getProblemDefinition()->clearSolutionPaths();
00541                     gsetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
00542                 }
00543                 else
00544                 {
00545                     csetup_->getProblemDefinition()->clearSolutionPaths();
00546                     csetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
00547                 }
00548             }
00549             catch(std::runtime_error &e)
00550             {
00551                 std::stringstream es;
00552                 es << "There was an error while preparing for run " << status_.activeRun << " of planner " << status_.activePlanner << std::endl;
00553                 es << "*** " << e.what() << std::endl;
00554                 std::cerr << es.str();
00555                 OMPL_ERROR(es.str().c_str());
00556             }
00557 
00558             // execute pre-run event, if set
00559             try
00560             {
00561                 if (preRun_)
00562                 {
00563                     OMPL_INFORM("Executing pre-run event for run %d of planner %s ...", status_.activeRun, status_.activePlanner.c_str());
00564                     preRun_(planners_[i]);
00565                     OMPL_INFORM("Completed execution of pre-run event");
00566                 }
00567             }
00568             catch(std::runtime_error &e)
00569             {
00570                 std::stringstream es;
00571                 es << "There was an error executing the pre-run event for run " << status_.activeRun << " of planner " << status_.activePlanner << std::endl;
00572                 es << "*** " << e.what() << std::endl;
00573                 std::cerr << es.str();
00574                 OMPL_ERROR(es.str().c_str());
00575             }
00576 
00577             RunPlanner rp(this, req.useThreads);
00578             rp.run(planners_[i], memStart, maxMemBytes, req.maxTime, req.timeBetweenUpdates);
00579             bool solved = gsetup_ ? gsetup_->haveSolutionPath() : csetup_->haveSolutionPath();
00580 
00581             // store results
00582             try
00583             {
00584                 RunProperties run;
00585 
00586                 run["time REAL"] = boost::lexical_cast<std::string>(rp.getTimeUsed());
00587                 run["memory REAL"] = boost::lexical_cast<std::string>((double)rp.getMemUsed() / (1024.0 * 1024.0));
00588                 run["status ENUM"] = boost::lexical_cast<std::string>((int)static_cast<base::PlannerStatus::StatusType>(rp.getStatus()));
00589                 if (gsetup_)
00590                 {
00591                     run["solved BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->haveExactSolutionPath());
00592                     run["valid segment fraction REAL"] = boost::lexical_cast<std::string>(gsetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
00593                 }
00594                 else
00595                 {
00596                     run["solved BOOLEAN"] = boost::lexical_cast<std::string>(csetup_->haveExactSolutionPath());
00597                     run["valid segment fraction REAL"] = boost::lexical_cast<std::string>(csetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
00598                 }
00599 
00600                 if (solved)
00601                 {
00602                     if (gsetup_)
00603                     {
00604                         run["approximate solution BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->getProblemDefinition()->hasApproximateSolution());
00605                         run["solution difference REAL"] = boost::lexical_cast<std::string>(gsetup_->getProblemDefinition()->getSolutionDifference());
00606                         run["solution length REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().length());
00607                         run["solution smoothness REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().smoothness());
00608                         run["solution clearance REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().clearance());
00609                         run["solution segments INTEGER"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().getStateCount() - 1);
00610                         run["correct solution BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().check());
00611 
00612                         unsigned int factor = gsetup_->getStateSpace()->getValidSegmentCountFactor();
00613                         gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
00614                         run["correct solution strict BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().check());
00615                         gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
00616 
00617                         // simplify solution
00618                         time::point timeStart = time::now();
00619                         gsetup_->simplifySolution();
00620                         double timeUsed = time::seconds(time::now() - timeStart);
00621                         run["simplification time REAL"] = boost::lexical_cast<std::string>(timeUsed);
00622                         run["simplified solution length REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().length());
00623                         run["simplified solution smoothness REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().smoothness());
00624                         run["simplified solution clearance REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().clearance());
00625                         run["simplified solution segments INTEGER"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().getStateCount() - 1);
00626                         run["simplified correct solution BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().check());
00627                         gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
00628                         run["simplified correct solution strict BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().check());
00629                         gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
00630                     }
00631                     else
00632                     {
00633                         run["approximate solution BOOLEAN"] = boost::lexical_cast<std::string>(csetup_->getProblemDefinition()->hasApproximateSolution());
00634                         run["solution difference REAL"] = boost::lexical_cast<std::string>(csetup_->getProblemDefinition()->getSolutionDifference());
00635                         run["solution length REAL"] = boost::lexical_cast<std::string>(csetup_->getSolutionPath().length());
00636                         run["solution clearance REAL"] = boost::lexical_cast<std::string>(csetup_->getSolutionPath().asGeometric().clearance());
00637                         run["solution segments INTEGER"] = boost::lexical_cast<std::string>(csetup_->getSolutionPath().getControlCount());
00638                         run["correct solution BOOLEAN"] = boost::lexical_cast<std::string>(csetup_->getSolutionPath().check());
00639                     }
00640                 }
00641 
00642                 base::PlannerData pd (gsetup_ ? gsetup_->getSpaceInformation() : csetup_->getSpaceInformation());
00643                 planners_[i]->getPlannerData(pd);
00644                 run["graph states INTEGER"] = boost::lexical_cast<std::string>(pd.numVertices());
00645                 run["graph motions INTEGER"] = boost::lexical_cast<std::string>(pd.numEdges());
00646 
00647                 for (std::map<std::string, std::string>::const_iterator it = pd.properties.begin() ; it != pd.properties.end() ; ++it)
00648                     run[it->first] = it->second;
00649 
00650                 // execute post-run event, if set
00651                 try
00652                 {
00653                     if (postRun_)
00654                     {
00655                         OMPL_INFORM("Executing post-run event for run %d of planner %s ...", status_.activeRun, status_.activePlanner.c_str());
00656                         postRun_(planners_[i], run);
00657                         OMPL_INFORM("Completed execution of post-run event");
00658                     }
00659                 }
00660                 catch(std::runtime_error &e)
00661                 {
00662                     std::stringstream es;
00663                     es << "There was an error in the execution of the post-run event for run " << status_.activeRun << " of planner " << status_.activePlanner << std::endl;
00664                     es << "*** " << e.what() << std::endl;
00665                     std::cerr << es.str();
00666                     OMPL_ERROR(es.str().c_str());
00667                 }
00668 
00669                 exp_.planners[i].runs.push_back(run);
00670 
00671                 // Add planner progress data from the planner progress
00672                 // collector if there was anything to report
00673                 if (planners_[i]->getPlannerProgressProperties().size() > 0)
00674                 {
00675                     exp_.planners[i].runsProgressData.push_back(rp.getRunProgressData());
00676                 }
00677             }
00678             catch(std::runtime_error &e)
00679             {
00680                 std::stringstream es;
00681                 es << "There was an error in the extraction of planner results: planner = " << status_.activePlanner << ", run = " << status_.activePlanner << std::endl;
00682                 es << "*** " << e.what() << std::endl;
00683                 std::cerr << es.str();
00684                 OMPL_ERROR(es.str().c_str());
00685             }
00686         }
00687     }
00688 
00689     status_.running = false;
00690     status_.progressPercentage = 100.0;
00691     if (req.displayProgress)
00692     {
00693         while (status_.progressPercentage > progress->count())
00694             ++(*progress);
00695         std::cout << std::endl;
00696     }
00697 
00698     exp_.totalDuration = time::seconds(time::now() - exp_.startTime);
00699 
00700     OMPL_INFORM("Benchmark complete");
00701     msg::useOutputHandler(oh);
00702     OMPL_INFORM("Benchmark complete");
00703 }
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines