00001 // File: $Id$ 00002 // Author: John Wu <John.Wu at ACM.org> 00003 // Copyright 2000-2012 the Regents of the University of California 00004 #ifndef IBIS_PART_H 00005 #define IBIS_PART_H 00006 00007 00008 00009 00010 00011 00012 #include "column.h" 00013 #include "resource.h" 00014 #include "utilidor.h" 00015 00016 #include <string> 00017 #include <vector> 00018 00027 class FASTBIT_CXX_DLLSPEC ibis::part { 00028 public: 00029 enum TABLE_STATE { 00030 UNKNOWN_STATE=0, STABLE_STATE, RECEIVING_STATE, 00031 PRETRANSITION_STATE, TRANSITION_STATE, POSTTRANSITION_STATE 00032 }; 00033 struct info; // To hold some basic information about a table. 00034 class readLock; 00035 00036 /******************************************************************/ 00037 // public functions 00038 /******************************************************************/ 00039 00041 virtual ~part(); 00043 explicit part(const char* name=0, bool ro=false); 00045 part(const char* adir, const char* bdir, bool ro=false); 00047 part(const std::vector<const char*> &mtags, bool ro=false); 00049 part(const ibis::resource::vList &mtags, bool ro=false); 00050 00051 inline info* getInfo() const; 00052 TABLE_STATE getState() const; 00054 TABLE_STATE getStateNoLocking() const {return state;} 00055 00056 virtual int buildIndexes(const char* iopt=0, int nthr=1); 00057 void buildSorted(const char* colname) const; 00058 void loadIndexes(const char* iopt=0, int ropt=0) const; 00059 void unloadIndexes() const; 00060 void purgeIndexFiles() const; 00061 00063 const char* name() const {return (m_name?m_name:"?");} 00065 const char* description() const {return m_desc.c_str();} 00067 const char* indexSpec() const {return idxstr;} 00069 void indexSpec(const char*); 00071 time_t timestamp() const {return switchTime;} 00074 std::string metaTags() const; 00075 ibis::table::stringList columnNames() const; 00076 ibis::table::typeList columnTypes() const; 00077 column* getColumn(const char* name) const; 00078 column* getColumn(uint32_t ind) const; 00079 00081 const char* currentDataDir() const {return activeDir;} 00083 uint32_t nColumns() const {return columns.size();} 00085 uint32_t nRows() const {return nEvents;} 00086 00087 int updateData(); 00088 00090 void print(std::ostream &out) const; 00091 00093 bool matchNameValuePair(const char* name, const char* value) const; 00096 bool matchMetaTags(const std::vector<const char*> &mtags) const; 00098 bool matchMetaTags(const ibis::resource::vList &mtags) const; 00100 inline const char* getMetaTag(const char*) const; 00101 00103 virtual long selfTest(int nth=1, const char* pref=0) const; 00104 void computeMinMax(); 00105 00106 /******************************************************************/ 00107 // The following functions need to hold a read lock on the partition, 00108 // however in order for them to produce consistent results, they must 00109 // share the same read lock. These functions do not contain the lock 00110 // themselves, but rely on the caller to maintain a consistent lock. 00111 /******************************************************************/ 00113 uint32_t getRowNumber(const rid_t &rid) const; 00114 long evaluateRIDSet(const ibis::RIDSet&, ibis::bitvector&) const; 00115 array_t<rid_t>* getRIDs() const {return rids;} // all RIDs 00116 array_t<rid_t>* getRIDs(const ibis::bitvector &mask) const;// some RIDs 00117 bool hasRIDs() const {return (rids!=0) ? (rids->size()==nEvents) : false;} 00118 00120 virtual double estimateCost(const ibis::qContinuousRange &cmp) const; 00121 virtual double estimateCost(const ibis::qDiscreteRange &cmp) const; 00122 virtual double estimateCost(const ibis::qIntHod &cmp) const; 00123 virtual double estimateCost(const ibis::qUIntHod &cmp) const; 00124 virtual double estimateCost(const ibis::qString &cmp) const; 00125 virtual double estimateCost(const ibis::qMultiString &cmp) const; 00126 00128 virtual long estimateRange(const ibis::qContinuousRange &cmp) const; 00129 00131 virtual long estimateRange(const ibis::qDiscreteRange &cmp) const; 00133 virtual long estimateRange(const ibis::qIntHod &cmp) const; 00135 virtual long estimateRange(const ibis::qUIntHod &cmp) const; 00136 00138 virtual long evaluateRange(const ibis::qContinuousRange &cmp, 00139 const ibis::bitvector &mask, 00140 ibis::bitvector &res) const; 00142 virtual long evaluateRange(const ibis::qDiscreteRange &cmp, 00143 const ibis::bitvector &mask, 00144 ibis::bitvector &res) const; 00146 virtual long evaluateRange(const ibis::qIntHod &cmp, 00147 const ibis::bitvector &mask, 00148 ibis::bitvector &res) const; 00150 virtual long evaluateRange(const ibis::qUIntHod &cmp, 00151 const ibis::bitvector &mask, 00152 ibis::bitvector &res) const; 00153 00155 virtual long estimateRange(const ibis::qContinuousRange &cmp, 00156 ibis::bitvector &low, 00157 ibis::bitvector &high) const; 00158 00160 virtual float getUndecidable(const ibis::qContinuousRange &cmp, 00161 ibis::bitvector &iffy) const; 00162 00164 virtual long estimateRange(const ibis::qDiscreteRange &cmp, 00165 ibis::bitvector &low, 00166 ibis::bitvector &high) const; 00168 virtual float getUndecidable(const ibis::qDiscreteRange &cmp, 00169 ibis::bitvector &iffy) const; 00170 00172 virtual long estimateRange(const ibis::qIntHod &cmp, 00173 ibis::bitvector &low, 00174 ibis::bitvector &high) const; 00176 virtual float getUndecidable(const ibis::qIntHod &cmp, 00177 ibis::bitvector &iffy) const; 00178 00180 virtual long estimateRange(const ibis::qUIntHod &cmp, 00181 ibis::bitvector &low, 00182 ibis::bitvector &high) const; 00184 virtual float getUndecidable(const ibis::qUIntHod &cmp, 00185 ibis::bitvector &iffy) const; 00186 00187 virtual long doScan(const ibis::qRange &cmp, 00188 ibis::bitvector &hits) const; 00189 virtual long doScan(const ibis::qRange &cmp, 00190 const ibis::bitvector &mask, 00191 ibis::bitvector &hits) const; 00192 virtual long doScan(const ibis::qRange &cmp, 00193 const ibis::bitvector &mask, 00194 void *res) const; 00195 virtual long doScan(const ibis::qRange &cmp, 00196 const ibis::bitvector &mask, 00197 void *res, ibis::bitvector &hits) const; 00198 00199 virtual long negativeScan(const ibis::qRange &cmp, 00200 const ibis::bitvector &mask, 00201 ibis::bitvector &hits) const; 00202 00203 virtual long doScan(const ibis::math::term&, const ibis::bitvector&, 00204 ibis::bitvector&) const; 00205 virtual long doScan(const ibis::compRange &cmp, 00206 ibis::bitvector &hits) const; 00207 virtual long doScan(const ibis::compRange &cmp, 00208 const ibis::bitvector &mask, 00209 ibis::bitvector &hits) const; 00210 00211 template <typename E> static long 00212 doScan(const array_t<E> &varr, 00213 const ibis::qRange &cmp, 00214 const ibis::bitvector &mask, 00215 ibis::bitvector &hits); 00216 template <typename E> static long 00217 doScan(const array_t<E> &varr, 00218 const ibis::qContinuousRange &cmp, 00219 const ibis::bitvector &mask, 00220 ibis::bitvector &hits); 00221 template <typename E> static long 00222 doScan(const array_t<E> &varr, 00223 const ibis::qContinuousRange &cmp, 00224 const ibis::bitvector &mask, 00225 array_t<E> &res); 00226 template <typename E> static long 00227 doScan(const array_t<E> &varr, 00228 const ibis::qContinuousRange &cmp, 00229 const ibis::bitvector &mask, 00230 array_t<E> &res, ibis::bitvector &hits); 00231 00232 long countHits(const ibis::qRange &cmp) const; 00233 00234 virtual long estimateMatchAny(const ibis::qAnyAny &cmp, 00235 ibis::bitvector &low, 00236 ibis::bitvector &high) const; 00237 virtual long matchAny(const ibis::qAnyAny &cmp, 00238 ibis::bitvector &hits) const; 00239 virtual long matchAny(const ibis::qAnyAny &cmp, 00240 const ibis::bitvector &mask, 00241 ibis::bitvector &hits) const; 00242 00243 long patternSearch(const ibis::qLike &cmp, 00244 ibis::bitvector &low) const; 00245 long lookforString(const ibis::qString &cmp, 00246 ibis::bitvector &low) const; 00247 long lookforString(const ibis::qMultiString &cmp, 00248 ibis::bitvector &low) const; 00249 long patternSearch(const ibis::qLike &cmp) const; 00250 long lookforString(const ibis::qString &cmp) const; 00251 long lookforString(const ibis::qMultiString &cmp) const; 00252 00257 int64_t evaluateJoin(const ibis::deprecatedJoin &cmp, 00258 const ibis::bitvector &mask, 00259 ibis::bitvector64 &pairs) const; 00262 int64_t evaluateJoin(const ibis::deprecatedJoin &cmp, 00263 const ibis::bitvector &mask, 00264 const char* pairfile) const; 00266 int64_t evaluateJoin(const ibis::deprecatedJoin &cmp, 00267 const ibis::bitvector &mask) const; 00270 int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp, 00271 const ibis::bitvector &mask, 00272 ibis::bitvector64 &pairs) const; 00273 int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp, 00274 const ibis::bitvector &mask) const; 00279 int64_t evaluateJoin(const ibis::deprecatedJoin &cmp, 00280 const ibis::bitvector64 &trial, 00281 ibis::bitvector64 &result) const; 00282 int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp, 00283 const ibis::bitvector64 &trial, 00284 ibis::bitvector64 &result) const; 00285 /******************************************************************/ 00286 00288 array_t<signed char>* 00289 selectBytes(const char* name, const ibis::bitvector &mask) const; 00291 array_t<unsigned char>* 00292 selectUBytes(const char* name, const ibis::bitvector &mask) const; 00294 array_t<int16_t>* 00295 selectShorts(const char* name, const ibis::bitvector &mask) const; 00297 array_t<uint16_t>* 00298 selectUShorts(const char* name, const ibis::bitvector &mask) const; 00300 array_t<int32_t>* 00301 selectInts(const char* name, const ibis::bitvector &mask) const; 00303 array_t<uint32_t>* 00304 selectUInts(const char* name, const ibis::bitvector &mask) const; 00306 array_t<int64_t>* 00307 selectLongs(const char* name, const ibis::bitvector &mask) const; 00309 array_t<uint64_t>* 00310 selectULongs(const char* name, const ibis::bitvector &mask) const; 00312 array_t<float>* 00313 selectFloats(const char* name, const ibis::bitvector &mask) const; 00315 array_t<double>* 00316 selectDoubles(const char* name, const ibis::bitvector &mask) const; 00318 std::vector<std::string>* 00319 selectStrings(const char* name, const ibis::bitvector &mask) const; 00320 00321 long selectValues(const char* cname, const ibis::bitvector &mask, 00322 void* vals) const; 00323 long selectValues(const ibis::qContinuousRange& cond, void* vals) const; 00324 long calculate(const ibis::math::term&, const ibis::bitvector&, 00325 array_t<double>&) const; 00326 00327 /******************************************************************/ 00328 // A group of functions added August, 2005 to provide summary 00329 // information about the data. Logically, they are an extension of 00330 // getInfo. 00331 00333 double getActualMin(const char *name) const; 00335 double getActualMax(const char *name) const; 00337 double getColumnSum(const char *name) const; 00338 00342 long get1DDistribution(const char *constraints, const char *cname, 00343 double begin, double end, double stride, 00344 std::vector<uint32_t> &counts) const; 00346 long get2DDistribution(const char *constraints, const char *cname1, 00347 double begin1, double end1, double stride1, 00348 const char *cname2, 00349 double begin2, double end2, double stride2, 00350 std::vector<uint32_t> &counts) const; 00352 long get3DDistribution(const char *constraints, const char *cname1, 00353 double begin1, double end1, double stride1, 00354 const char *cname2, 00355 double begin2, double end2, double stride2, 00356 const char *cname3, 00357 double begin3, double end3, double stride3, 00358 std::vector<uint32_t> &counts) const; 00360 long get1DDistribution(const char *constraints, const char *cname, 00361 double begin, double end, double stride, 00362 const char *wtname, 00363 std::vector<double> &weights) const; 00365 long get2DDistribution(const char *constraints, const char *cname1, 00366 double begin1, double end1, double stride1, 00367 const char *cname2, 00368 double begin2, double end2, double stride2, 00369 const char *wtname, 00370 std::vector<double> &weights) const; 00372 long get3DDistribution(const char *constraints, const char *cname1, 00373 double begin1, double end1, double stride1, 00374 const char *cname2, 00375 double begin2, double end2, double stride2, 00376 const char *cname3, 00377 double begin3, double end3, double stride3, 00378 const char *wtname, 00379 std::vector<double> &weights) const; 00381 long get1DDistribution(const char *cname, uint32_t nbin, 00382 std::vector<double> &bounds, 00383 std::vector<uint32_t> &counts) const; 00385 long get1DDistribution(const char *constraints, 00386 const char *cname, uint32_t nbin, 00387 std::vector<double> &bounds, 00388 std::vector<uint32_t> &counts) const; 00390 long get2DDistribution(const char *cname1, const char *cname2, 00391 uint32_t nb1, uint32_t nb2, 00392 std::vector<double> &bounds1, 00393 std::vector<double> &bounds2, 00394 std::vector<uint32_t> &counts, 00395 const char* const option=0) const; 00397 long get2DDistribution(const char *constraints, 00398 const char *name1, const char *name2, 00399 uint32_t nb1, uint32_t nb2, 00400 std::vector<double> &bounds1, 00401 std::vector<double> &bounds2, 00402 std::vector<uint32_t> &counts) const; 00404 long get3DDistribution(const char *cname1, const char *cname2, 00405 const char *cname3, 00406 uint32_t nb1, uint32_t nb2, uint32_t nb3, 00407 std::vector<double> &bounds1, 00408 std::vector<double> &bounds2, 00409 std::vector<double> &bounds3, 00410 std::vector<uint32_t> &counts, 00411 const char* const option=0) const; 00413 long get3DDistribution(const char *constraints, 00414 const char *cname1, const char *cname2, 00415 const char *cname3, 00416 uint32_t nb1, uint32_t nb2, uint32_t nb3, 00417 std::vector<double> &bounds1, 00418 std::vector<double> &bounds2, 00419 std::vector<double> &bounds3, 00420 std::vector<uint32_t> &counts) const; 00422 long get1DBins(const char *constraints, const char *cname, 00423 double begin, double end, double stride, 00424 std::vector<ibis::bitvector> &bins) const; 00426 long get1DBins(const char *constraints, const char *cname, 00427 double begin, double end, double stride, 00428 std::vector<ibis::bitvector*> &bins) const; 00430 long get1DBins(const char *constraints, const char *cname, 00431 double begin, double end, double stride, 00432 const char *wtname, 00433 std::vector<double> &weights, 00434 std::vector<ibis::bitvector*> &bins) const; 00436 long get2DBins(const char *constraints, const char *cname1, 00437 double begin1, double end1, double stride1, 00438 const char *cname2, 00439 double begin2, double end2, double stride2, 00440 std::vector<ibis::bitvector> &bins) const; 00442 long get2DBins(const char *constraints, const char *cname1, 00443 double begin1, double end1, double stride1, 00444 const char *cname2, 00445 double begin2, double end2, double stride2, 00446 std::vector<ibis::bitvector*> &bins) const; 00448 long get2DBins(const char *constraints, const char *cname1, 00449 double begin1, double end1, double stride1, 00450 const char *cname2, 00451 double begin2, double end2, double stride2, 00452 const char *wtname, 00453 std::vector<double> &weights, 00454 std::vector<ibis::bitvector*> &bins) const; 00456 long get3DBins(const char *constraints, const char *cname1, 00457 double begin1, double end1, double stride1, 00458 const char *cname2, 00459 double begin2, double end2, double stride2, 00460 const char *cname3, 00461 double begin3, double end3, double stride3, 00462 std::vector<ibis::bitvector> &bins) const; 00464 long get3DBins(const char *constraints, const char *cname1, 00465 double begin1, double end1, double stride1, 00466 const char *cname2, 00467 double begin2, double end2, double stride2, 00468 const char *cname3, 00469 double begin3, double end3, double stride3, 00470 std::vector<ibis::bitvector*> &bins) const; 00472 long get3DBins(const char *constraints, const char *cname1, 00473 double begin1, double end1, double stride1, 00474 const char *cname2, 00475 double begin2, double end2, double stride2, 00476 const char *cname3, 00477 double begin3, double end3, double stride3, 00478 const char *wtname, 00479 std::vector<double> &weights, 00480 std::vector<ibis::bitvector*> &bins) const; 00483 long get1DBins(const char *constraints, const char *cname1, uint32_t nb1, 00484 std::vector<double> &bounds1, 00485 std::vector<ibis::bitvector> &bins) const; 00487 long get2DBins(const char *constraints, 00488 const char *cname1, const char *cname2, 00489 uint32_t nb1, uint32_t nb2, 00490 std::vector<double> &bounds1, 00491 std::vector<double> &bounds2, 00492 std::vector<ibis::bitvector> &bins) const; 00494 long get3DBins(const char *constraints, 00495 const char *cname1, const char *cname2, const char *cname3, 00496 uint32_t nb1, uint32_t nb2, uint32_t nb3, 00497 std::vector<double> &bounds1, 00498 std::vector<double> &bounds2, 00499 std::vector<double> &bounds3, 00500 std::vector<ibis::bitvector> &bins) const; 00502 00507 long getDistribution(const char *name, 00508 std::vector<double> &bounds, 00509 std::vector<uint32_t> &counts) const; 00511 long getDistribution(const char *constraints, 00512 const char *name, 00513 std::vector<double> &bounds, 00514 std::vector<uint32_t> &counts) const; 00517 long getDistribution(const char *name, uint32_t nbc, 00518 double *bounds, uint32_t *counts) const; 00521 long getDistribution(const char *name, const char *constraints, 00522 uint32_t nbc, double *bounds, 00523 uint32_t *counts) const; 00524 00526 long getJointDistribution(const char *constraints, 00527 const char *name1, const char *name2, 00528 std::vector<double> &bounds1, 00529 std::vector<double> &bounds2, 00530 std::vector<uint32_t> &counts) const; 00531 00533 long getCumulativeDistribution(const char *name, 00534 std::vector<double> &bounds, 00535 std::vector<uint32_t> &counts) const; 00538 long getCumulativeDistribution(const char *constraints, 00539 const char *name, 00540 std::vector<double> &bounds, 00541 std::vector<uint32_t> &counts) const; 00544 long getCumulativeDistribution(const char *name, uint32_t nbc, 00545 double *bounds, uint32_t *counts) const; 00548 long getCumulativeDistribution(const char *constraints, const char *name, 00549 uint32_t nbc, double *bounds, 00550 uint32_t *counts) const; 00552 00553 /******************************************************************/ 00564 const std::vector<uint32_t> &getMeshShape() const {return shapeSize;} 00567 const std::vector<std::string> &getMeshDimensions() const { 00568 return shapeName;} 00569 void setMeshShape(const char *shape); 00570 void updateMetaData() const; 00572 void combineNames(ibis::table::namesTypes &metalist) const; 00573 00574 /******************************************************************/ 00576 long append(const char* dir); 00581 long commit(const char* dir); 00584 long rollback(); 00585 long addColumn(const char* aexpr, const char* cname, 00586 ibis::TYPE_T ctype=ibis::DOUBLE); 00587 long addColumn(const ibis::math::term* xpr, ibis::bitvector& mask, 00588 const char* cname, ibis::TYPE_T ctype=ibis::DOUBLE); 00589 00590 /******************************************************************/ 00591 virtual long reorder(); 00592 virtual long reorder(const ibis::table::stringList &names); 00593 virtual long reorder(const ibis::table::stringList &names, 00594 const std::vector<bool> &directions); 00595 00596 long deactivate(const std::vector<uint32_t> &rows); 00597 long deactivate(const char* conds); 00598 long reactivate(const std::vector<uint32_t> &rows); 00599 long reactivate(const char* conds); 00600 long purgeInactive(); 00601 void emptyCache() const; 00603 const ibis::bitvector &getNullMask() const {return amask;} 00604 00605 static const char* skipPrefix(const char*); 00607 static char* readMetaTags(const char* const dir); 00609 static void genName(const std::vector<const char*> &mtags, 00610 std::string &name); 00612 static void genName(const ibis::resource::vList &mtags, 00613 std::string &name); 00614 void rename(const ibis::partAssoc& known); 00615 void rename(const char*); 00616 00618 static uint32_t countPages(const ibis::bitvector &mask, 00619 unsigned elemsize=4); 00621 ibis::fileManager::ACCESS_PREFERENCE 00622 accessHint(const ibis::bitvector &mask, unsigned elemsize=4) const; 00623 00625 struct thrArg { 00626 const part* et; 00627 const char* pref; 00628 long* nerrors; 00629 ibis::util::counter cnt; 00630 std::vector<std::string> conds; 00631 std::vector<unsigned> super; 00632 std::vector<unsigned> hits; 00633 }; 00635 struct indexBuilderPool { 00636 ibis::util::counter cnt; 00637 const char* opt; 00638 const part &tbl; 00639 indexBuilderPool(const part &t, const char* spec) 00640 : cnt(), opt(spec), tbl(t) {} 00641 }; 00642 00644 void queryTest(const char* pref, long* nerrors) const; 00646 void quickTest(const char* pref, long* nerrors) const; 00649 void testRangeOperators(const ibis::column* col, 00650 long* nerrors) const; 00651 00652 // These functions are used in classes barrel and vault 00653 void logWarning(const char* event, const char* fmt, ...) const; 00654 void logMessage(const char* event, const char* fmt, ...) const; 00655 00656 void doBackup(); 00657 00658 class barrel; 00659 class vault; 00661 typedef std::map< const char*, column*, lessi > columnList; 00662 00663 template <typename T> static int 00664 writeColumn(int fdes, ibis::bitvector::word_t nold, 00665 ibis::bitvector::word_t nnew, 00666 const array_t<T>& vals, const T& fill, 00667 ibis::bitvector& totmask, 00668 const ibis::bitvector& newmask); 00669 static int writeString(int fdes, ibis::bitvector::word_t nold, 00670 ibis::bitvector::word_t nnew, 00671 const std::vector<std::string>& vals, 00672 ibis::bitvector& totmask, 00673 const ibis::bitvector& newmask); 00674 static int writeRaw(int bdes, int sdes, ibis::bitvector::word_t nold, 00675 ibis::bitvector::word_t nnew, 00676 const ibis::array_t<unsigned char>& bytes, 00677 const ibis::array_t<int64_t>& starts, 00678 ibis::bitvector& totmask, 00679 const ibis::bitvector& newmask); 00680 00681 protected: 00682 class cleaner; 00683 class mutexLock; 00684 class writeLock; 00685 class softWriteLock; 00686 00687 friend struct info; 00688 friend class cleaner; 00689 friend class readLock; 00690 friend class writeLock; 00691 friend class softWriteLock; 00692 friend class mutexLock; 00693 00694 /******************************************************************/ 00695 // protected member variables 00696 // 00697 char* m_name; 00698 std::string m_desc; 00699 ibis::resource::vList metaList; 00700 mutable array_t<rid_t>* rids; 00701 columnList columns; 00702 uint32_t nEvents; 00703 char* activeDir; 00704 char* backupDir; 00705 time_t switchTime; 00706 TABLE_STATE state; 00707 char* idxstr; 00708 00709 ibis::bitvector amask; 00710 std::vector<const column*> colorder; 00711 std::vector<std::string> shapeName; 00712 std::vector<uint32_t> shapeSize; 00713 00714 ibis::part::cleaner* myCleaner; 00715 bool readonly; 00716 00717 00718 /******************************************************************/ 00719 // protected member functions 00720 // 00722 int readMetaData(uint32_t &nrows, columnList &plist, const char* dir); 00724 void writeMetaData(const uint32_t nrows, const columnList &plist, 00725 const char* dir) const; 00726 void readRIDs() const; 00727 void freeRIDs() const; 00728 00729 // functions to deal with meta tags -- those attributes that are 00730 // associated with a whole partition but are expected to be processed 00731 // like a categorical attribute in queries 00732 void extendMetaTags(); 00733 void setMetaTags(const ibis::resource::vList &mts); 00734 void setMetaTags(const std::vector<const char*> &mts); 00735 00737 void readMeshShape(const char* const dir); 00739 void digestMeshShape(const char* shape); 00740 00742 void logError(const char* event, const char* fmt, ...) const; 00743 00744 void makeBackupCopy(); // copy the content of activeDir to backupDir 00745 long verifyBackupDir(); // minimal consistency check 00746 void deriveBackupDirName(); 00747 long appendToBackup(const char* dir); // append to the backup directory 00748 00750 template <typename T> 00751 long writeValues(const char *fname, const array_t<uint32_t> &ind); 00753 template <typename T> 00754 long reorderValues(const char *fname, 00755 array_t<uint32_t> &starts, 00756 array_t<uint32_t> &indout, 00757 const array_t<uint32_t> &indin, 00758 bool ascending); 00759 long append1(const char* dir); 00760 long append2(const char* dir); 00761 00763 long deactivate(const ibis::bitvector &rows); 00765 long reactivate(const ibis::bitvector &rows); 00767 void numbersToBitvector(const std::vector<uint32_t>&, 00768 ibis::bitvector&) const; 00769 void stringToBitvector(const char*, ibis::bitvector&) const; 00770 00771 template <typename T> static long 00772 doCompare(const array_t<T> &array, 00773 const ibis::qRange &cmp, 00774 const ibis::bitvector &mask, 00775 ibis::array_t<T> &res, ibis::bitvector &hits); 00776 template <typename T> static long 00777 doCompare(const char *file, 00778 const ibis::qRange &cmp, 00779 const ibis::bitvector &mask, 00780 ibis::array_t<T> &res, ibis::bitvector &hits); 00781 00782 template <typename T> static long 00783 doCompare(const array_t<T> &array, 00784 const ibis::qRange &cmp, 00785 const ibis::bitvector &mask, 00786 ibis::array_t<T> &res); 00787 template <typename T> static long 00788 doCompare(const char *file, 00789 const ibis::qRange &cmp, 00790 const ibis::bitvector &mask, 00791 ibis::array_t<T> &res); 00792 00793 template <typename T> static long 00794 doCompare(const array_t<T> &array, 00795 const ibis::qRange &cmp, 00796 const ibis::bitvector &mask, 00797 ibis::bitvector &hits); 00798 template <typename T> static long 00799 doCompare(const char *file, 00800 const ibis::qRange &cmp, 00801 const ibis::bitvector &mask, 00802 ibis::bitvector &hits); 00803 00804 template <typename T> static long 00805 negativeCompare(const array_t<T> &array, 00806 const ibis::qRange &cmp, 00807 const ibis::bitvector &mask, 00808 ibis::bitvector &hits); 00809 template <typename T> static long 00810 negativeCompare(const char *file, 00811 const ibis::qRange &cmp, 00812 const ibis::bitvector &mask, 00813 ibis::bitvector &hits); 00814 00815 template <typename T> static long 00816 doCompare(const array_t<T> &array, 00817 const ibis::qIntHod &cmp, 00818 const ibis::bitvector &mask, 00819 ibis::bitvector &hits); 00820 template <typename T> static long 00821 doCompare(const char *file, 00822 const ibis::qIntHod &cmp, 00823 const ibis::bitvector &mask, 00824 ibis::bitvector &hits); 00825 template <typename T> static long 00826 negativeCompare(const array_t<T> &array, 00827 const ibis::qIntHod &cmp, 00828 const ibis::bitvector &mask, 00829 ibis::bitvector &hits); 00830 template <typename T> static long 00831 negativeCompare(const char *file, 00832 const ibis::qIntHod &cmp, 00833 const ibis::bitvector &mask, 00834 ibis::bitvector &hits); 00835 00836 template <typename T> static long 00837 doCompare(const array_t<T> &array, 00838 const ibis::qUIntHod &cmp, 00839 const ibis::bitvector &mask, 00840 ibis::bitvector &hits); 00841 template <typename T> static long 00842 doCompare(const char *file, 00843 const ibis::qUIntHod &cmp, 00844 const ibis::bitvector &mask, 00845 ibis::bitvector &hits); 00846 template <typename T> static long 00847 negativeCompare(const array_t<T> &array, 00848 const ibis::qUIntHod &cmp, 00849 const ibis::bitvector &mask, 00850 ibis::bitvector &hits); 00851 template <typename T> static long 00852 negativeCompare(const char *file, 00853 const ibis::qUIntHod &cmp, 00854 const ibis::bitvector &mask, 00855 ibis::bitvector &hits); 00856 00857 template <typename T, typename F> static long 00858 doComp(const array_t<T> &vals, F cmp, 00859 const ibis::bitvector &mask, 00860 array_t<T> &res, ibis::bitvector &hits); 00861 00862 template <typename T, typename F> static long 00863 doComp(const array_t<T> &vals, F cmp, 00864 const ibis::bitvector &mask, 00865 array_t<T> &res); 00866 00867 template <typename T, typename F> static long 00868 doComp(const array_t<T> &vals, F cmp, 00869 const ibis::bitvector &mask, 00870 ibis::bitvector &hits); 00871 00872 template <typename T, typename F1, typename F2> static long 00873 doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2, 00874 const ibis::bitvector &mask, 00875 array_t<T> &res, ibis::bitvector &hits); 00876 00877 template <typename T, typename F1, typename F2> static long 00878 doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2, 00879 const ibis::bitvector &mask, 00880 array_t<T> &res); 00881 00882 template <typename T, typename F1, typename F2> static long 00883 doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2, 00884 const ibis::bitvector &mask, 00885 ibis::bitvector &hits); 00886 00887 template <typename T, typename F> static long 00888 doComp0(const array_t<T> &vals, F cmp, 00889 const ibis::bitvector &mask, 00890 ibis::bitvector &hits); 00891 00892 template <typename T, typename F1, typename F2> static long 00893 doComp0(const array_t<T> &vals, F1 cmp1, F2 cmp2, 00894 const ibis::bitvector &mask, 00895 ibis::bitvector &hits); 00896 00897 template <typename T> 00898 long doCount(const ibis::qRange &cmp) const; 00899 00902 template <typename T> 00903 long doCount(const array_t<T> &vals, const ibis::qIntHod &cmp, 00904 const ibis::bitvector &mask) const; 00905 00908 template <typename T> 00909 long doCount(const array_t<T> &vals, const ibis::qUIntHod &cmp, 00910 const ibis::bitvector &mask) const; 00911 00914 template <typename T> 00915 long doCount(const array_t<T> &vals, const ibis::qRange &cmp, 00916 const ibis::bitvector &mask) const; 00917 00919 template <typename T, typename F> 00920 long doCount(const array_t<T> &vals, 00921 const ibis::bitvector &mask, F cmp) const; 00922 00924 template <typename T, typename F1, typename F2> 00925 long doCount(const array_t<T> &vals, 00926 const ibis::bitvector &mask, F1 cmp1, F2 cmp2) const; 00927 00930 long packCumulativeDistribution(const std::vector<double> &bounds, 00931 const std::vector<uint32_t> &counts, 00932 uint32_t nbc, 00933 double *bptr, uint32_t *cptr) const; 00935 long packDistribution(const std::vector<double> &bounds, 00936 const std::vector<uint32_t> &counts, 00937 uint32_t nbc, double *bptr, uint32_t *cptr) const; 00938 00940 template <typename T1, typename T2> 00941 long count2DBins(array_t<T1> &vals1, 00942 const double &begin1, const double &end1, 00943 const double &stride1, 00944 array_t<T2> &vals2, 00945 const double &begin2, const double &end2, 00946 const double &stride2, 00947 std::vector<uint32_t> &counts) const; 00949 template <typename T1, typename T2, typename T3> 00950 long count3DBins(const array_t<T1> &vals1, 00951 const double &begin1, const double &end1, 00952 const double &stride1, 00953 const array_t<T2> &vals2, 00954 const double &begin2, const double &end2, 00955 const double &stride2, 00956 const array_t<T3> &vals3, 00957 const double &begin3, const double &end3, 00958 const double &stride3, 00959 std::vector<uint32_t> &counts) const; 00961 template <typename T1, typename T2> 00962 long count2DWeights(array_t<T1> &vals1, 00963 const double &begin1, const double &end1, 00964 const double &stride1, 00965 array_t<T2> &vals2, 00966 const double &begin2, const double &end2, 00967 const double &stride2, 00968 array_t<double> &wts, 00969 std::vector<double> &weights) const; 00971 template <typename T1, typename T2, typename T3> 00972 long count3DWeights(const array_t<T1> &vals1, 00973 const double &begin1, const double &end1, 00974 const double &stride1, 00975 const array_t<T2> &vals2, 00976 const double &begin2, const double &end2, 00977 const double &stride2, 00978 const array_t<T3> &vals3, 00979 const double &begin3, const double &end3, 00980 const double &stride3, 00981 const array_t<double> &wts, 00982 std::vector<double> &weights) const; 00983 00985 template <typename T1> 00986 long fill1DBins(const ibis::bitvector &mask, const array_t<T1> &vals1, 00987 const double &begin1, const double &end1, 00988 const double &stride1, 00989 std::vector<ibis::bitvector> &bins) const; 00991 template <typename T1> 00992 long fill1DBins(const ibis::bitvector &mask, const array_t<T1> &vals1, 00993 const double &begin1, const double &end1, 00994 const double &stride1, 00995 std::vector<ibis::bitvector*> &bins) const; 00997 template <typename T1> 00998 long fill1DBinsWeighted(const ibis::bitvector &mask, 00999 const array_t<T1> &vals1, 01000 const double &begin1, const double &end1, 01001 const double &stride1, 01002 const array_t<double> &wts, 01003 std::vector<double> &weights, 01004 std::vector<ibis::bitvector*> &bins) const; 01006 template <typename T1, typename T2> 01007 long fill2DBins(const ibis::bitvector &mask, const array_t<T1> &vals1, 01008 const double &begin1, const double &end1, 01009 const double &stride1, 01010 const array_t<T2> &vals2, 01011 const double &begin2, const double &end2, 01012 const double &stride2, 01013 std::vector<ibis::bitvector> &bins) const; 01014 template <typename T1> 01015 long fill2DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1, 01016 const double &begin1, const double &end1, 01017 const double &stride1, 01018 const ibis::column &col2, 01019 const double &begin2, const double &end2, 01020 const double &stride2, 01021 std::vector<ibis::bitvector> &bins) const; 01023 template <typename T1, typename T2> 01024 long fill2DBins(const ibis::bitvector &mask, const array_t<T1> &vals1, 01025 const double &begin1, const double &end1, 01026 const double &stride1, 01027 const array_t<T2> &vals2, 01028 const double &begin2, const double &end2, 01029 const double &stride2, 01030 std::vector<ibis::bitvector*> &bins) const; 01031 template <typename T1> 01032 long fill2DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1, 01033 const double &begin1, const double &end1, 01034 const double &stride1, 01035 const ibis::column &col2, 01036 const double &begin2, const double &end2, 01037 const double &stride2, 01038 std::vector<ibis::bitvector*> &bins) const; 01040 template <typename T1, typename T2> 01041 long fill2DBinsWeighted(const ibis::bitvector &mask, 01042 const array_t<T1> &vals1, 01043 const double &begin1, const double &end1, 01044 const double &stride1, 01045 const array_t<T2> &vals2, 01046 const double &begin2, const double &end2, 01047 const double &stride2, 01048 const array_t<double> &wts, 01049 std::vector<double> &weights, 01050 std::vector<ibis::bitvector*> &bins) const; 01051 template <typename T1> 01052 long fill2DBinsWeighted2(const ibis::bitvector &mask, 01053 const array_t<T1> &vals1, 01054 const double &begin1, const double &end1, 01055 const double &stride1, 01056 const ibis::column &col2, 01057 const double &begin2, const double &end2, 01058 const double &stride2, 01059 const array_t<double> &wts, 01060 std::vector<double> &weights, 01061 std::vector<ibis::bitvector*> &bins) const; 01063 template <typename T1, typename T2, typename T3> 01064 long fill3DBins(const ibis::bitvector &mask, const array_t<T1> &vals1, 01065 const double &begin1, const double &end1, 01066 const double &stride1, 01067 const array_t<T2> &vals2, 01068 const double &begin2, const double &end2, 01069 const double &stride2, 01070 const array_t<T3> &vals3, 01071 const double &begin3, const double &end3, 01072 const double &stride3, 01073 std::vector<ibis::bitvector> &bins) const; 01074 template <typename T1> 01075 long fill3DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1, 01076 const double &begin1, const double &end1, 01077 const double &stride1, 01078 const ibis::column &col2, 01079 const double &begin2, const double &end2, 01080 const double &stride2, 01081 const ibis::column &col3, 01082 const double &begin3, const double &end3, 01083 const double &stride3, 01084 std::vector<bitvector> &bins) const; 01085 template <typename T1, typename T2> 01086 long fill3DBins3(const ibis::bitvector &mask, const array_t<T1> &vals1, 01087 const double &begin1, const double &end1, 01088 const double &stride1, 01089 const array_t<T2> &vals2, 01090 const double &begin2, const double &end2, 01091 const double &stride2, 01092 const ibis::column &col3, 01093 const double &begin3, const double &end3, 01094 const double &stride3, 01095 std::vector<bitvector> &bins) const; 01096 template <typename T1, typename T2, typename T3> 01097 long fill3DBins(const ibis::bitvector &mask, const array_t<T1> &vals1, 01098 const double &begin1, const double &end1, 01099 const double &stride1, 01100 const array_t<T2> &vals2, 01101 const double &begin2, const double &end2, 01102 const double &stride2, 01103 const array_t<T3> &vals3, 01104 const double &begin3, const double &end3, 01105 const double &stride3, 01106 std::vector<bitvector*> &bins) const; 01107 template <typename T1> 01108 long fill3DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1, 01109 const double &begin1, const double &end1, 01110 const double &stride1, 01111 const ibis::column &col2, 01112 const double &begin2, const double &end2, 01113 const double &stride2, 01114 const ibis::column &col3, 01115 const double &begin3, const double &end3, 01116 const double &stride3, 01117 std::vector<ibis::bitvector*> &bins) const; 01118 template <typename T1, typename T2> 01119 long fill3DBins3(const ibis::bitvector &mask, const array_t<T1> &vals1, 01120 const double &begin1, const double &end1, 01121 const double &stride1, 01122 const array_t<T2> &vals2, 01123 const double &begin2, const double &end2, 01124 const double &stride2, 01125 const ibis::column &col3, 01126 const double &begin3, const double &end3, 01127 const double &stride3, 01128 std::vector<ibis::bitvector*> &bins) const; 01129 template <typename T1, typename T2, typename T3> 01130 long fill3DBinsWeighted(const ibis::bitvector &mask, 01131 const array_t<T1> &vals1, 01132 const double &begin1, const double &end1, 01133 const double &stride1, 01134 const array_t<T2> &vals2, 01135 const double &begin2, const double &end2, 01136 const double &stride2, 01137 const array_t<T3> &vals3, 01138 const double &begin3, const double &end3, 01139 const double &stride3, 01140 const array_t<double> &wts, 01141 std::vector<double> &weights, 01142 std::vector<bitvector*> &bins) const; 01143 template <typename T1> 01144 long fill3DBinsWeighted2(const ibis::bitvector &mask, 01145 const array_t<T1> &vals1, 01146 const double &begin1, const double &end1, 01147 const double &stride1, 01148 const ibis::column &col2, 01149 const double &begin2, const double &end2, 01150 const double &stride2, 01151 const ibis::column &col3, 01152 const double &begin3, const double &end3, 01153 const double &stride3, 01154 const array_t<double> &wts, 01155 std::vector<double> &weights, 01156 std::vector<ibis::bitvector*> &bins) const; 01157 template <typename T1, typename T2> 01158 long fill3DBinsWeighted3(const ibis::bitvector &mask, 01159 const array_t<T1> &vals1, 01160 const double &begin1, const double &end1, 01161 const double &stride1, 01162 const array_t<T2> &vals2, 01163 const double &begin2, const double &end2, 01164 const double &stride2, 01165 const ibis::column &col3, 01166 const double &begin3, const double &end3, 01167 const double &stride3, 01168 const array_t<double> &wts, 01169 std::vector<double> &weights, 01170 std::vector<ibis::bitvector*> &bins) const; 01171 01173 long get1DBins_(const ibis::bitvector &mask, const ibis::column &col, 01174 uint32_t nbin, std::vector<double> &bounds, 01175 std::vector<ibis::bitvector> &bins, const char *mesg) const; 01176 01178 long get1DDistribution(const ibis::column &col, uint32_t nbin, 01179 std::vector<double> &bounds, 01180 std::vector<uint32_t> &counts) const; 01182 long get2DDistributionU(const ibis::column &col1, 01183 const ibis::column &col2, 01184 uint32_t nb1, uint32_t nb2, 01185 std::vector<double> &bounds1, 01186 std::vector<double> &bounds2, 01187 std::vector<uint32_t> &counts) const; 01189 long get2DDistributionA(const ibis::column &col1, 01190 const ibis::column &col2, 01191 uint32_t nb1, uint32_t nb2, 01192 std::vector<double> &bounds1, 01193 std::vector<double> &bounds2, 01194 std::vector<uint32_t> &counts) const; 01196 long get2DDistributionI(const ibis::column &col1, 01197 const ibis::column &col2, 01198 uint32_t nb1, uint32_t nb2, 01199 std::vector<double> &bounds1, 01200 std::vector<double> &bounds2, 01201 std::vector<uint32_t> &counts) const; 01202 long old2DDistribution(const char *constraints, 01203 const char *name1, const char *name2, 01204 uint32_t nb1, uint32_t nb2, 01205 std::vector<double> &bounds1, 01206 std::vector<double> &bounds2, 01207 std::vector<uint32_t> &counts) const; 01209 int coarsenBins(const ibis::column &col, uint32_t nbin, 01210 std::vector<double> &bnds, 01211 std::vector<ibis::bitvector*> &btmp) const; 01213 long get3DDistributionA(const ibis::bitvector &mask, 01214 const ibis::column &col1, 01215 const ibis::column &col2, 01216 const ibis::column &col3, 01217 uint32_t nb1, uint32_t nb2, uint32_t nb3, 01218 std::vector<double> &bounds1, 01219 std::vector<double> &bounds2, 01220 std::vector<double> &bounds3, 01221 std::vector<uint32_t> &counts) const; 01222 template <typename E1> 01223 long get3DDistributionA1(const ibis::bitvector &mask, 01224 const array_t<E1> &vals1, 01225 const ibis::column &col2, 01226 const ibis::column &col3, 01227 uint32_t nb1, uint32_t nb2, uint32_t nb3, 01228 std::vector<double> &bounds1, 01229 std::vector<double> &bounds2, 01230 std::vector<double> &bounds3, 01231 std::vector<uint32_t> &counts) const; 01232 template <typename E1, typename E2> 01233 long get3DDistributionA2(const ibis::bitvector &mask, 01234 const array_t<E1> &vals1, 01235 const array_t<E2> &vals2, 01236 const ibis::column &col3, 01237 uint32_t nb1, uint32_t nb2, uint32_t nb3, 01238 std::vector<double> &bounds1, 01239 std::vector<double> &bounds2, 01240 std::vector<double> &bounds3, 01241 std::vector<uint32_t> &counts) const; 01242 01243 template <typename E1, typename E2> 01244 static void mapValues(array_t<E1> &val1, array_t<E2> &val2, 01245 uint32_t nb1, uint32_t nb2, 01246 array_t<E1> &bnd1, array_t<E2> &bnd2, 01247 std::vector<uint32_t> &cnts); 01248 01249 template <typename T> 01250 static void mapValues(const array_t<T> &vals, 01251 std::map<T, uint32_t> &hist); 01252 01253 template <typename T> 01254 static void equalWeightBins(const array_t<T> &vals, 01255 uint32_t nbins, array_t<T> &bounds); 01256 01257 template <typename T> 01258 static long adaptiveInts(const array_t<T> &vals, const T vmin, 01259 const T vmax, uint32_t nbins, 01260 std::vector<double> &bounds, 01261 std::vector<uint32_t> &counts); 01262 01263 template <typename T> 01264 static long adaptiveFloats(const array_t<T> &vals, const T vmin, 01265 const T vmax, uint32_t nbins, 01266 std::vector<double> &bounds, 01267 std::vector<uint32_t> &counts); 01268 01269 template <typename T1, typename T2> 01270 static long adaptive2DBins(const array_t<T1> &vals1, 01271 const array_t<T2> &vals2, 01272 uint32_t nb1, uint32_t nb2, 01273 std::vector<double> &bounds1, 01274 std::vector<double> &bounds2, 01275 std::vector<uint32_t> &counts); 01276 01277 template <typename T1, typename T2, typename T3> 01278 static long adaptive3DBins(const array_t<T1> &vals1, 01279 const array_t<T2> &vals2, 01280 const array_t<T3> &vals3, 01281 uint32_t nb1, uint32_t nb2, uint32_t nb3, 01282 std::vector<double> &bounds1, 01283 std::vector<double> &bounds2, 01284 std::vector<double> &bounds3, 01285 std::vector<uint32_t> &counts); 01286 01287 template <typename T> static long 01288 adaptiveIntsDetailed(const ibis::bitvector &mask, 01289 const array_t<T> &vals, 01290 const T vmin, const T vmax, uint32_t nbins, 01291 std::vector<double> &bounds, 01292 std::vector<ibis::bitvector> &detail); 01293 01294 template <typename T> static long 01295 adaptiveFloatsDetailed(const ibis::bitvector &mask, 01296 const array_t<T> &vals, 01297 const T vmin, const T vmax, uint32_t nbins, 01298 std::vector<double> &bounds, 01299 std::vector<ibis::bitvector> &detail); 01300 01301 void gatherSortKeys(ibis::table::stringList& names); 01302 void composeQueryString(std::string &str, 01303 const ibis::column* col1, const ibis::column* col2, 01304 const double &lower1, const double &upper1, 01305 const double &lower2, const double &upper2) const; 01306 void buildQueryList(ibis::part::thrArg &lst, 01307 unsigned nc, unsigned nq) const; 01308 void checkQueryList(const ibis::part::thrArg &lst) const; 01309 uint32_t recursiveQuery(const char* pref, const column* att, 01310 double low, double high, long* nerr) const; 01311 01312 private: 01313 01314 /******************************************************************/ 01315 // private member variables 01316 mutable pthread_mutex_t mutex; 01317 mutable pthread_rwlock_t rwlock; 01318 01319 /******************************************************************/ 01320 // private funcations 01321 01322 void init(const char* prefix); 01323 01324 void gainReadAccess(const char* mesg) const; 01325 void releaseAccess(const char* mesg) const; 01326 void gainWriteAccess(const char* mesg) const; 01327 01328 void fillRIDs(const char* fn) const; 01329 void sortRIDs() const; 01330 uint32_t searchSortedRIDs(const ibis::rid_t &rid) const; 01331 uint32_t searchRIDs(const ibis::rid_t &rid) const; 01332 void searchSortedRIDs(const ibis::RIDSet&, ibis::bitvector&) const; 01333 void searchRIDs(const ibis::RIDSet&, ibis::bitvector&) const; 01334 01335 // functions to perform join operations. 01336 int64_t equiJoin(const ibis::deprecatedJoin &cmp, 01337 const ibis::bitvector64 &trial, 01338 ibis::bitvector64 &result) const; 01339 int64_t deprecatedJoin(const ibis::deprecatedJoin &cmp, 01340 const ibis::bitvector64 &trial, 01341 ibis::bitvector64 &result) const; 01342 int64_t compJoin(const ibis::deprecatedJoin &cmp, 01343 const ibis::bitvector64 &trial, 01344 ibis::bitvector64 &result) const; 01345 01346 int64_t loopJoin(const std::vector<const ibis::deprecatedJoin*> &cmp, 01347 const ibis::bitvector &mask, 01348 ibis::bitvector64 &pairs) const; 01349 int64_t loopJoin(const std::vector<const ibis::deprecatedJoin*> &cmp, 01350 const ibis::bitvector &mask) const; 01351 int64_t loopJoin(const ibis::deprecatedJoin &cmp, 01352 const ibis::bitvector &mask, 01353 ibis::bitvector64 &pairs) const; 01354 int64_t loopJoin(const ibis::deprecatedJoin &cmp, 01355 const ibis::bitvector &mask) const; 01356 int64_t equiJoinLoop1(const ibis::deprecatedJoin &cmp, 01357 const ibis::bitvector &mask, 01358 ibis::bitvector64 &pairs) const; 01359 int64_t equiJoinLoop1(const ibis::deprecatedJoin &cmp, 01360 const ibis::bitvector &mask) const; 01361 int64_t equiJoinLoop2(const ibis::deprecatedJoin &cmp, 01362 const ibis::bitvector &mask, 01363 ibis::bitvector64 &pairs) const; 01364 int64_t equiJoinLoop2(const ibis::deprecatedJoin &cmp, 01365 const ibis::bitvector &mask) const; 01366 template <class type1, class type2> 01367 void deprecatedJoinLoop(const array_t<type1> &arr1, 01368 const ibis::bitvector &msk1, 01369 const array_t<type2> &arr2, 01370 const ibis::bitvector &msk2, 01371 const double delta, 01372 ibis::bitvector64 &pairs) const; 01373 template <class type1, class type2> 01374 int64_t deprecatedJoinLoop(const array_t<type1> &arr1, 01375 const ibis::bitvector &msk1, 01376 const array_t<type2> &arr2, 01377 const ibis::bitvector &msk2, 01378 const double delta) const; 01379 int64_t deprecatedJoinLoop(const ibis::deprecatedJoin &cmp, 01380 const ibis::bitvector &mask, 01381 ibis::bitvector64 &pairs) const; 01382 int64_t deprecatedJoinLoop(const ibis::deprecatedJoin &cmp, 01383 const ibis::bitvector &mask) const; 01384 int64_t compJoinLoop(const ibis::deprecatedJoin &cmp, 01385 const ibis::bitvector &mask, 01386 ibis::bitvector64 &pairs) const; 01387 int64_t compJoinLoop(const ibis::deprecatedJoin &cmp, 01388 const ibis::bitvector &mask) const; 01389 01390 part(const part&); 01391 const part &operator=(const part&); 01392 }; // class ibis::part 01393 01394 namespace ibis { 01395 // Extends the name space ibis::util to contain three more functions to 01396 // deal with the reconstruction of partitions. 01397 namespace util { 01399 unsigned int FASTBIT_CXX_DLLSPEC 01400 gatherParts(ibis::partList &parts, 01401 const char *adir, const char *bdir, bool ro=false); 01403 unsigned int FASTBIT_CXX_DLLSPEC 01404 gatherParts(ibis::partList &parts, const char *adir, bool ro=false); 01407 unsigned int FASTBIT_CXX_DLLSPEC 01408 gatherParts(ibis::partList &parts, const ibis::resource &res, 01409 bool ro=false); 01410 } // namespace util 01411 } // namespace ibis 01412 01416 struct FASTBIT_CXX_DLLSPEC ibis::part::info { 01417 const char* name; 01418 const char* description; 01419 const char* metaTags; 01420 const uint64_t nrows; 01421 01422 std::vector<ibis::column::info*> cols; 01423 01424 info(const char* na, const char* de, const uint64_t &nr, 01425 const ibis::part::columnList &co); 01426 info(const ibis::part &tbl); 01427 ~info(); 01428 01429 private: 01430 info(); // private default constructor, not implemented! 01431 info& operator=(const info&); 01432 }; // ibis::part::info 01433 01435 class ibis::part::cleaner : public ibis::fileManager::cleaner { 01436 public: 01437 virtual void operator()() const; 01438 cleaner(const part* tbl) : thePart(tbl) {} 01439 virtual ~cleaner() {} 01440 01441 private: 01442 const part* thePart; 01443 }; // ibis::part::cleaner 01444 01449 class ibis::part::readLock { 01450 public: 01451 readLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) { 01452 tbl->gainReadAccess(m); 01453 } 01454 ~readLock() {thePart->releaseAccess(mesg);} 01455 01456 private: 01457 const part* thePart; 01458 const char* mesg; 01459 01460 readLock() {}; // no default constructor 01461 readLock(const readLock&) {}; // can not copy 01462 const readLock &operator=(const readLock&); 01463 }; // ibis::part::readLock 01464 01466 class ibis::part::writeLock { 01467 public: 01468 writeLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) { 01469 tbl->gainWriteAccess(m); 01470 } 01471 ~writeLock() {thePart->releaseAccess(mesg);} 01472 01473 private: 01474 const part* thePart; 01475 const char* mesg; 01476 01477 writeLock() {}; // no default constructor 01478 writeLock(const writeLock&) {}; // can not copy 01479 const writeLock &operator=(const writeLock&); 01480 }; // ibis::part::writeLock 01481 01485 class ibis::part::softWriteLock { 01486 public: 01488 softWriteLock(const part* tbl, const char* m); 01490 ~softWriteLock() {if (lckd==0) thePart->releaseAccess(mesg);} 01493 bool isLocked() const {return (lckd==0);} 01494 01495 private: 01496 const part* thePart; 01497 const char* mesg; 01498 const int lckd; 01499 01500 softWriteLock() : thePart(0), mesg(0), lckd(0) {}; 01501 softWriteLock(const softWriteLock &rhs) 01502 : thePart(rhs.thePart), mesg(rhs.mesg), lckd(0) {}; 01503 const softWriteLock &operator=(const softWriteLock&); 01504 }; // ibis::part::softWriteLock 01505 01508 class ibis::part::mutexLock { 01509 public: 01510 mutexLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) { 01511 if (ibis::gVerbose > 9) 01512 tbl->logMessage("gainExclusiveAccess", 01513 "pthread_mutex_lock for %s", m); 01514 int ierr = pthread_mutex_lock(&(tbl->mutex)); 01515 if (0 != ierr) 01516 tbl->logWarning("gainExclusiveAccess", "pthread_mutex_lock for %s " 01517 "returned %d (%s)", m, ierr, strerror(ierr)); 01518 } 01519 ~mutexLock() { 01520 if (ibis::gVerbose > 9) 01521 thePart->logMessage("releaseExclusiveAccess", 01522 "pthread_mutex_unlock for %s", mesg); 01523 int ierr = pthread_mutex_unlock(&(thePart->mutex)); 01524 if (0 != ierr) 01525 thePart->logWarning("releaseExclusiveAccess", 01526 "pthread_mutex_unlock for %s returned %d (%s)", 01527 mesg, ierr, strerror(ierr)); 01528 } 01529 01530 private: 01531 const part* thePart; 01532 const char* mesg; 01533 01534 mutexLock() {}; // no default constructor 01535 mutexLock(const mutexLock&) {}; // can not copy 01536 const mutexLock &operator=(const mutexLock&); 01537 }; // ibis::part::mutexLock 01538 01542 class ibis::part::barrel : public ibis::math::barrel { 01543 public: 01544 barrel(const ibis::part *t=0) : _tbl(t), position(0) {}; 01545 virtual ~barrel() {close();} 01546 01547 virtual long open(const ibis::part *t=0); 01548 virtual long close(); 01549 virtual long read(); 01550 01551 01552 virtual long seek(uint32_t pos); 01553 uint32_t tell() const {return position;} 01554 01555 void getNullMask(ibis::bitvector &mask) const; 01556 const ibis::column* getColumn(uint32_t i) const {return cols[i];} 01557 01558 protected: 01559 const ibis::part *_tbl; 01560 uint32_t position; 01561 std::vector<const ibis::column*> cols; 01562 std::vector<ibis::fileManager::storage*> stores; 01563 std::vector<int> fdes; 01564 }; // ibis::part::barrel 01565 01568 class ibis::part::vault : public ibis::part::barrel { 01569 public: 01570 vault(const ibis::roster &r); 01571 virtual ~vault() {close();} 01572 01573 virtual long open(const ibis::part *t=0); 01574 virtual long read(); 01575 virtual long seek(uint32_t pos); 01576 01577 long seek(double val); 01580 uint32_t tellReal() const; 01581 01582 private: 01583 const ibis::roster &_roster; 01584 01585 template <class T> 01586 uint32_t seekValue(int fd, const T &val) const; 01587 template <class T> 01588 uint32_t seekValue(const array_t<T>&arr, const T &val) const; 01589 01590 vault(); 01591 vault& operator=(const vault&); 01592 }; // ibis::part::vault 01593 /* 01594 barrel 01595 01596 Vault \Vault\ (v[add]lt; see Note, below), n. [OE. voute, OF. voute, volte, 01597 F. vo[^u]te, LL. volta, for voluta, volutio, fr. L. volvere, volutum, to 01598 roll, to turn about. See Voluble, and cf. Vault a leap, Volt a turn, 01599 Volute.] 1. (Arch.) An arched structure of masonry, forming a ceiling or 01600 canopy. 01601 01602 The long-drawn aisle and fretted vault. --Gray. 01603 01604 2. An arched apartment; especially, a subterranean room, use for storing 01605 articles, for a prison, for interment, or the like; a cell; a 01606 cellar. ``Charnel vaults.'' --Milton. 01607 01608 The silent vaults of death. --Sandys. 01609 01610 To banish rats that haunt our vault. --Swift. 01611 01612 3. The canopy of heaven; the sky. 01613 01614 That heaven's vault should crack. --Shak. 01615 01616 4. [F. volte, It. volta, originally, a turn, and the same word as volta an 01617 arch. See the Etymology above.] A leap or bound. Specifically: (a) 01618 (Man.) The bound or leap of a horse; a curvet. (b) A leap by aid of the 01619 hands, or of a pole, springboard, or the like. 01620 01621 Note: The l in this word was formerly often suppressed in pronunciation. 01622 01623 Barrel, Cradle, Cylindrical, or Wagon, vault (Arch.), a kind of vault 01624 having two parallel abutments, and the same section or profile at all 01625 points. It may be rampant, as over a staircase (see Rampant vault, under 01626 Rampant), or curved in plan, as around the apse of a church. 01627 01628 Coved vault. (Arch.) See under 1st Cove, v. t. 01629 01630 Groined vault (Arch.), a vault having groins, that is, one in which 01631 different cylindrical surfaces intersect one another, as distinguished from 01632 a barrel, or wagon, vault. 01633 01634 Rampant vault. (Arch.) See under Rampant. 01635 01636 Ribbed vault (Arch.), a vault differing from others in having solid ribs 01637 which bear the weight of the vaulted surface. True Gothic vaults are of 01638 this character. 01639 01640 Vault light, a partly glazed plate inserted in a pavement or ceiling to 01641 admit light to a vault below. 01642 01643 Source: Webster's Revised Unabridged Dictionary, (c) 1996, 1998 MICRA, Inc. 01644 */ 01645 01646 namespace ibis { 01647 // Explicit template specialization for member function 01648 // ibis::part::equalWeightBins 01649 template <> void 01650 part::equalWeightBins(const array_t<float> &vals, 01651 uint32_t nbins, array_t<float> &bounds); 01652 template <> void 01653 part::equalWeightBins(const array_t<double> &vals, 01654 uint32_t nbins, array_t<double> &bounds); 01655 01656 template <> long 01657 part::doScan(const array_t<float> &, 01658 const ibis::qContinuousRange &, 01659 const ibis::bitvector &, ibis::bitvector &); 01660 template <> long 01661 part::doScan(const array_t<double> &, 01662 const ibis::qContinuousRange &, 01663 const ibis::bitvector &, ibis::bitvector &); 01664 01665 template <> long 01666 part::doScan(const array_t<float> &, 01667 const ibis::qContinuousRange &, 01668 const ibis::bitvector &, ibis::array_t<float> &); 01669 template <> long 01670 part::doScan(const array_t<double> &, 01671 const ibis::qContinuousRange &, 01672 const ibis::bitvector &, ibis::array_t<double> &); 01673 01674 template <> long 01675 part::doScan(const array_t<float> &, 01676 const ibis::qContinuousRange &, 01677 const ibis::bitvector &, ibis::array_t<float> &, 01678 ibis::bitvector &); 01679 template <> long 01680 part::doScan(const array_t<double> &, 01681 const ibis::qContinuousRange &, 01682 const ibis::bitvector &, ibis::array_t<double> &, 01683 ibis::bitvector &); 01684 01685 template <> long 01686 part::doCount<float>(const ibis::qRange&) const; 01687 template <> long 01688 part::doCount<double>(const ibis::qRange&) const; 01689 } 01690 01692 inline ibis::part::info* ibis::part::getInfo() const { 01693 return new info(*this); 01694 } 01695 01700 inline ibis::column* ibis::part::getColumn(uint32_t ind) const { 01701 if (ind < columns.size()) { 01702 if (colorder.size() == columns.size()) { 01703 return const_cast<ibis::column*>(colorder[ind]); 01704 } 01705 else { 01706 ibis::part::columnList::const_iterator it = columns.begin(); 01707 for (; ind > 0; -- ind) ++ it; 01708 return (*it).second; 01709 } 01710 } 01711 else { 01712 return 0; 01713 } 01714 } // ibis::part::getColumn 01715 01716 inline const char* ibis::part::getMetaTag(const char* name) const { 01717 ibis::resource::vList::const_iterator it = metaList.find(name); 01718 if (it != metaList.end()) 01719 return (*it).second; 01720 else 01721 return static_cast<const char*>(0); 01722 } // ibis::part::getMetaTag 01723 01724 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &cmp, 01725 const ibis::bitvector &mask, 01726 ibis::bitvector64 &pairs) const { 01727 return loopJoin(cmp, mask, pairs); 01728 } // ibis::part::evaluateJoin 01729 01730 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &, 01731 const ibis::bitvector &, 01732 const char *) const { 01733 logWarning("evaluate", "not implemented yet"); 01734 return -1; 01735 } // ibis::part::evaluateJoin 01736 01737 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &cmp, 01738 const ibis::bitvector &mask) const { 01739 return loopJoin(cmp, mask); 01740 } // ibis::part::evaluateJoin 01741 01742 inline int64_t ibis::part::evaluateJoin 01743 (const std::vector<const ibis::deprecatedJoin*> &cmp, 01744 const ibis::bitvector &mask, ibis::bitvector64 &pairs) const { 01745 return loopJoin(cmp, mask, pairs); 01746 } // ibis::part::evaluateJoin 01747 01749 inline void ibis::part::cleaner::operator()() const { 01750 const uint32_t sz = ibis::fileManager::bytesInUse(); 01751 thePart->unloadIndexes(); 01752 if (sz == ibis::fileManager::bytesInUse() && 01753 thePart->getStateNoLocking() == ibis::part::STABLE_STATE) { 01754 thePart->freeRIDs(); 01755 } 01756 } // ibis::part::cleaner::operator 01757 #endif // IBIS_PART_H
![]() |