part.h
Go to the documentation of this file.
00001 // File: $Id$
00002 // Author: John Wu <John.Wu at ACM.org>
00003 // Copyright 2000-2011 the Regents of the University of California
00004 #ifndef IBIS_PART_H
00005 #define IBIS_PART_H
00006 
00007 
00008 
00009 
00010 
00011 
00012 #include "column.h"
00013 #include "resource.h"
00014 #include "utilidor.h"
00015 
00016 #include <string>
00017 #include <vector>
00018 
00027 class FASTBIT_CXX_DLLSPEC ibis::part {
00028 public:
00029     enum TABLE_STATE {
00030         UNKNOWN_STATE=0, STABLE_STATE, RECEIVING_STATE,
00031         PRETRANSITION_STATE, TRANSITION_STATE, POSTTRANSITION_STATE
00032     };
00033     struct info; // To hold some basic information about a table.
00034     class readLock;
00035 
00036     /******************************************************************/
00037     // public functions
00038     /******************************************************************/
00039 
00041     virtual ~part();
00043     explicit part(const char* name=0, bool ro=false);
00045     part(const char* adir, const char* bdir, bool ro=false);
00047     part(const std::vector<const char*> &mtags, bool ro=false);
00049     part(const ibis::resource::vList &mtags, bool ro=false);
00050 
00051     inline info* getInfo() const;
00052     TABLE_STATE getState() const;
00054     TABLE_STATE getStateNoLocking() const {return state;}
00055 
00056     virtual int buildIndexes(const char* iopt=0, int nthr=1);
00057     void buildSorted(const char* colname) const;
00058     void loadIndexes(const char* iopt=0, int ropt=0) const;
00059     void unloadIndexes() const;
00060     void purgeIndexFiles() const;
00061 
00063     const char* name()          const {return m_name;}
00065     const char* description()   const {return m_desc.c_str();}
00067     const char* indexSpec()     const {return idxstr;}
00069     void indexSpec(const char*);
00071     time_t timestamp()          const {return switchTime;}
00074     std::string metaTags()      const;
00075     ibis::table::stringList columnNames() const;
00076     ibis::table::typeList   columnTypes() const;
00077     column* getColumn(const char* name) const;
00078     column* getColumn(uint32_t ind) const;
00079 
00081     const char* currentDataDir() const {return activeDir;}
00083     uint32_t nColumns() const {return columns.size();}
00085     uint32_t nRows() const {return nEvents;}
00086 
00087     int updateData();
00088 
00090     void print(std::ostream &out) const;
00091 
00093     bool matchNameValuePair(const char* name, const char* value) const;
00096     bool matchMetaTags(const std::vector<const char*> &mtags) const;
00098     bool matchMetaTags(const ibis::resource::vList &mtags) const;
00100     inline const char* getMetaTag(const char*) const;
00101 
00103     virtual long selfTest(int nth=1, const char* pref=0) const;
00106     void computeMinMax();
00107 
00108     /******************************************************************/
00109     // The following functions need to hold a read lock on the partition,
00110     // however in order for them to produce consistent results, they must
00111     // share the same read lock.  These functions do not contain the lock
00112     // themselves, but rely on the caller to maintain a consistent lock.
00113     /******************************************************************/
00115     uint32_t getRowNumber(const rid_t &rid) const;
00116     long evaluateRIDSet(const ibis::RIDSet&, ibis::bitvector&) const; 
00117     array_t<rid_t>* getRIDs() const {return rids;} // all RIDs
00118     array_t<rid_t>* getRIDs(const ibis::bitvector &mask) const;// some RIDs
00119     bool hasRIDs() const {return (rids!=0) ? (rids->size()==nEvents) : false;}
00120 
00122     virtual double estimateCost(const ibis::qContinuousRange &cmp) const;
00123     virtual double estimateCost(const ibis::qDiscreteRange &cmp) const;
00124     virtual double estimateCost(const ibis::qIntHod &cmp) const;
00125     virtual double estimateCost(const ibis::qUIntHod &cmp) const;
00126     virtual double estimateCost(const ibis::qString &cmp) const;
00127     virtual double estimateCost(const ibis::qMultiString &cmp) const;
00128 
00130     virtual long estimateRange(const ibis::qContinuousRange &cmp) const;
00131 
00133     virtual long estimateRange(const ibis::qDiscreteRange &cmp) const;
00135     virtual long estimateRange(const ibis::qIntHod &cmp) const;
00137     virtual long estimateRange(const ibis::qUIntHod &cmp) const;
00138 
00140     virtual long evaluateRange(const ibis::qContinuousRange &cmp,
00141                                const ibis::bitvector &mask,
00142                                ibis::bitvector &res) const;
00144     virtual long evaluateRange(const ibis::qDiscreteRange &cmp,
00145                                const ibis::bitvector &mask,
00146                                ibis::bitvector &res) const;
00148     virtual long evaluateRange(const ibis::qIntHod &cmp,
00149                                const ibis::bitvector &mask,
00150                                ibis::bitvector &res) const;
00152     virtual long evaluateRange(const ibis::qUIntHod &cmp,
00153                                const ibis::bitvector &mask,
00154                                ibis::bitvector &res) const;
00155 
00157     virtual long estimateRange(const ibis::qContinuousRange &cmp,
00158                                ibis::bitvector &low,
00159                                ibis::bitvector &high) const;
00160 
00162     virtual float getUndecidable(const ibis::qContinuousRange &cmp,
00163                                  ibis::bitvector &iffy) const;
00164 
00166     virtual long estimateRange(const ibis::qDiscreteRange &cmp,
00167                                ibis::bitvector &low,
00168                                ibis::bitvector &high) const;
00170     virtual float getUndecidable(const ibis::qDiscreteRange &cmp,
00171                                  ibis::bitvector &iffy) const;
00172 
00174     virtual long estimateRange(const ibis::qIntHod &cmp,
00175                                ibis::bitvector &low,
00176                                ibis::bitvector &high) const;
00178     virtual float getUndecidable(const ibis::qIntHod &cmp,
00179                                  ibis::bitvector &iffy) const;
00180 
00182     virtual long estimateRange(const ibis::qUIntHod &cmp,
00183                                ibis::bitvector &low,
00184                                ibis::bitvector &high) const;
00186     virtual float getUndecidable(const ibis::qUIntHod &cmp,
00187                                  ibis::bitvector &iffy) const;
00188 
00191     virtual long doScan(const ibis::qRange &cmp,
00192                         ibis::bitvector &hits) const;
00195     virtual long doScan(const ibis::qRange &cmp,
00196                         const ibis::bitvector &mask,
00197                         ibis::bitvector &hits) const;
00200     virtual long negativeScan(const ibis::qRange &cmp,
00201                               const ibis::bitvector &mask,
00202                               ibis::bitvector &hits) const;
00203 
00204     long doScan(const ibis::math::term&, const ibis::bitvector&,
00205                 ibis::bitvector&) const;
00207     virtual long doScan(const ibis::compRange &cmp,
00208                         ibis::bitvector &hits) const;
00211     virtual long doScan(const ibis::compRange &cmp,
00212                         const ibis::bitvector &mask,
00213                         ibis::bitvector &hits) const;
00214 
00216     template <typename E>
00217         static long doScan(const array_t<E> &varr,
00218                            const ibis::qRange &cmp,
00219                            const ibis::bitvector &mask,
00220                            ibis::bitvector &hits);
00222     template <typename E>
00223         static long doScan(const array_t<E> &varr,
00224                            const ibis::qContinuousRange &cmp,
00225                            const ibis::bitvector &mask,
00226                            ibis::bitvector &hits);
00227 
00229     long countHits(const ibis::qRange &cmp) const;
00230 
00235     virtual long estimateMatchAny(const ibis::qAnyAny &cmp,
00236                                   ibis::bitvector &low,
00237                                   ibis::bitvector &high) const;
00238     virtual long matchAny(const ibis::qAnyAny &cmp,
00239                           ibis::bitvector &hits) const;
00240     virtual long matchAny(const ibis::qAnyAny &cmp,
00241                           const ibis::bitvector &mask,
00242                           ibis::bitvector &hits) const;
00243 
00244     long patternSearch(const ibis::qLike &cmp,
00245                     ibis::bitvector &low) const;
00246     long lookforString(const ibis::qString &cmp,
00247                        ibis::bitvector &low) const;
00248     long lookforString(const ibis::qMultiString &cmp,
00249                        ibis::bitvector &low) const;
00250     long patternSearch(const ibis::qLike &cmp) const;
00251     long lookforString(const ibis::qString &cmp) const;
00252     long lookforString(const ibis::qMultiString &cmp) const;
00253 
00258     int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
00259                          const ibis::bitvector &mask,
00260                          ibis::bitvector64 &pairs) const;
00263     int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
00264                          const ibis::bitvector &mask,
00265                          const char* pairfile) const;
00267     int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
00268                          const ibis::bitvector &mask) const;
00271     int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
00272                          const ibis::bitvector &mask,
00273                          ibis::bitvector64 &pairs) const;
00274     int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
00275                          const ibis::bitvector &mask) const;
00280     int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
00281                          const ibis::bitvector64 &trial,
00282                          ibis::bitvector64 &result) const;
00283     int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
00284                          const ibis::bitvector64 &trial,
00285                          ibis::bitvector64 &result) const;
00286     /******************************************************************/
00287 
00289     array_t<signed char>*
00290         selectBytes(const char* name, const ibis::bitvector &mask) const;
00292     array_t<unsigned char>*
00293         selectUBytes(const char* name, const ibis::bitvector &mask) const;
00295     array_t<int16_t>*
00296         selectShorts(const char* name, const ibis::bitvector &mask) const;
00298     array_t<uint16_t>*
00299         selectUShorts(const char* name, const ibis::bitvector &mask) const;
00301     array_t<int32_t>*
00302         selectInts(const char* name, const ibis::bitvector &mask) const;
00304     array_t<uint32_t>*
00305         selectUInts(const char* name, const ibis::bitvector &mask) const;
00307     array_t<int64_t>*
00308         selectLongs(const char* name, const ibis::bitvector &mask) const;
00310     array_t<uint64_t>*
00311         selectULongs(const char* name, const ibis::bitvector &mask) const;
00313     array_t<float>*
00314         selectFloats(const char* name, const ibis::bitvector &mask) const;
00316     array_t<double>*
00317         selectDoubles(const char* name, const ibis::bitvector &mask) const;
00319     std::vector<std::string>*
00320         selectStrings(const char* name, const ibis::bitvector &mask) const;
00322     long calculate(const ibis::math::term&, const ibis::bitvector&,
00323                    array_t<double>&) const;
00324 
00325     /******************************************************************/
00326     // A group of functions added August, 2005 to provide summary
00327     // information about the data.  Logically, they are an extension of
00328     // getInfo.
00329 
00331     double getActualMin(const char *name) const;
00333     double getActualMax(const char *name) const;
00335     double getColumnSum(const char *name) const;
00336 
00340     long get1DDistribution(const char *constraints, const char *cname,
00341                            double begin, double end, double stride,
00342                            std::vector<uint32_t> &counts) const;
00344     long get2DDistribution(const char *constraints, const char *cname1,
00345                            double begin1, double end1, double stride1,
00346                            const char *cname2,
00347                            double begin2, double end2, double stride2,
00348                            std::vector<uint32_t> &counts) const;
00350     long get3DDistribution(const char *constraints, const char *cname1,
00351                            double begin1, double end1, double stride1,
00352                            const char *cname2,
00353                            double begin2, double end2, double stride2,
00354                            const char *cname3,
00355                            double begin3, double end3, double stride3,
00356                            std::vector<uint32_t> &counts) const;
00358     long get1DDistribution(const char *constraints, const char *cname,
00359                            double begin, double end, double stride,
00360                            const char *wtname,
00361                            std::vector<double> &weights) const;
00363     long get2DDistribution(const char *constraints, const char *cname1,
00364                            double begin1, double end1, double stride1,
00365                            const char *cname2,
00366                            double begin2, double end2, double stride2,
00367                            const char *wtname,
00368                            std::vector<double> &weights) const;
00370     long get3DDistribution(const char *constraints, const char *cname1,
00371                            double begin1, double end1, double stride1,
00372                            const char *cname2,
00373                            double begin2, double end2, double stride2,
00374                            const char *cname3,
00375                            double begin3, double end3, double stride3,
00376                            const char *wtname,
00377                            std::vector<double> &weights) const;
00379     long get1DDistribution(const char *cname, uint32_t nbin,
00380                            std::vector<double> &bounds,
00381                            std::vector<uint32_t> &counts) const;
00383     long get1DDistribution(const char *constraints,
00384                            const char *cname, uint32_t nbin,
00385                            std::vector<double> &bounds,
00386                            std::vector<uint32_t> &counts) const;
00388     long get2DDistribution(const char *cname1, const char *cname2,
00389                            uint32_t nb1, uint32_t nb2,
00390                            std::vector<double> &bounds1,
00391                            std::vector<double> &bounds2,
00392                            std::vector<uint32_t> &counts,
00393                            const char* const option=0) const;
00395     long get2DDistribution(const char *constraints,
00396                            const char *name1, const char *name2,
00397                            uint32_t nb1, uint32_t nb2,
00398                            std::vector<double> &bounds1,
00399                            std::vector<double> &bounds2,
00400                            std::vector<uint32_t> &counts) const;
00402     long get3DDistribution(const char *cname1, const char *cname2,
00403                            const char *cname3,
00404                            uint32_t nb1, uint32_t nb2, uint32_t nb3,
00405                            std::vector<double> &bounds1,
00406                            std::vector<double> &bounds2,
00407                            std::vector<double> &bounds3,
00408                            std::vector<uint32_t> &counts,
00409                            const char* const option=0) const;
00411     long get3DDistribution(const char *constraints,
00412                            const char *cname1, const char *cname2,
00413                            const char *cname3,
00414                            uint32_t nb1, uint32_t nb2, uint32_t nb3,
00415                            std::vector<double> &bounds1,
00416                            std::vector<double> &bounds2,
00417                            std::vector<double> &bounds3,
00418                            std::vector<uint32_t> &counts) const;
00420     long get1DBins(const char *constraints, const char *cname,
00421                    double begin, double end, double stride,
00422                    std::vector<ibis::bitvector> &bins) const;
00424     long get1DBins(const char *constraints, const char *cname,
00425                    double begin, double end, double stride,
00426                    std::vector<ibis::bitvector*> &bins) const;
00428     long get1DBins(const char *constraints, const char *cname,
00429                    double begin, double end, double stride,
00430                    const char *wtname,
00431                    std::vector<double> &weights,
00432                    std::vector<ibis::bitvector*> &bins) const;
00434     long get2DBins(const char *constraints, const char *cname1,
00435                    double begin1, double end1, double stride1,
00436                    const char *cname2,
00437                    double begin2, double end2, double stride2,
00438                    std::vector<ibis::bitvector> &bins) const;
00440     long get2DBins(const char *constraints, const char *cname1,
00441                    double begin1, double end1, double stride1,
00442                    const char *cname2,
00443                    double begin2, double end2, double stride2,
00444                    std::vector<ibis::bitvector*> &bins) const;
00446     long get2DBins(const char *constraints, const char *cname1,
00447                    double begin1, double end1, double stride1,
00448                    const char *cname2,
00449                    double begin2, double end2, double stride2,
00450                    const char *wtname,
00451                    std::vector<double> &weights,
00452                    std::vector<ibis::bitvector*> &bins) const;
00454     long get3DBins(const char *constraints, const char *cname1,
00455                    double begin1, double end1, double stride1,
00456                    const char *cname2,
00457                    double begin2, double end2, double stride2,
00458                    const char *cname3,
00459                    double begin3, double end3, double stride3,
00460                    std::vector<ibis::bitvector> &bins) const;
00462     long get3DBins(const char *constraints, const char *cname1,
00463                    double begin1, double end1, double stride1,
00464                    const char *cname2,
00465                    double begin2, double end2, double stride2,
00466                    const char *cname3,
00467                    double begin3, double end3, double stride3,
00468                    std::vector<ibis::bitvector*> &bins) const;
00470     long get3DBins(const char *constraints, const char *cname1,
00471                    double begin1, double end1, double stride1,
00472                    const char *cname2,
00473                    double begin2, double end2, double stride2,
00474                    const char *cname3,
00475                    double begin3, double end3, double stride3,
00476                    const char *wtname,
00477                    std::vector<double> &weights,
00478                    std::vector<ibis::bitvector*> &bins) const;
00481     long get1DBins(const char *constraints, const char *cname1, uint32_t nb1,
00482                    std::vector<double> &bounds1,
00483                    std::vector<ibis::bitvector> &bins) const;
00485     long get2DBins(const char *constraints,
00486                    const char *cname1, const char *cname2,
00487                    uint32_t nb1, uint32_t nb2,
00488                    std::vector<double> &bounds1,
00489                    std::vector<double> &bounds2,
00490                    std::vector<ibis::bitvector> &bins) const;
00492     long get3DBins(const char *constraints,
00493                    const char *cname1, const char *cname2, const char *cname3,
00494                    uint32_t nb1, uint32_t nb2, uint32_t nb3,
00495                    std::vector<double> &bounds1,
00496                    std::vector<double> &bounds2,
00497                    std::vector<double> &bounds3,
00498                    std::vector<ibis::bitvector> &bins) const;
00500 
00505     long getDistribution(const char *name,
00506                          std::vector<double> &bounds,
00507                          std::vector<uint32_t> &counts) const;
00509     long getDistribution(const char *constraints,
00510                          const char *name,
00511                          std::vector<double> &bounds,
00512                          std::vector<uint32_t> &counts) const;
00515     long getDistribution(const char *name, uint32_t nbc,
00516                          double *bounds, uint32_t *counts) const;
00519     long getDistribution(const char *name, const char *constraints,
00520                          uint32_t nbc, double *bounds,
00521                          uint32_t *counts) const;
00522 
00524     long getJointDistribution(const char *constraints,
00525                               const char *name1, const char *name2,
00526                               std::vector<double> &bounds1,
00527                               std::vector<double> &bounds2,
00528                               std::vector<uint32_t> &counts) const;
00529 
00531     long getCumulativeDistribution(const char *name,
00532                                    std::vector<double> &bounds,
00533                                    std::vector<uint32_t> &counts) const;
00536     long getCumulativeDistribution(const char *constraints,
00537                                    const char *name,
00538                                    std::vector<double> &bounds,
00539                                    std::vector<uint32_t> &counts) const;
00542     long getCumulativeDistribution(const char *name, uint32_t nbc,
00543                                    double *bounds, uint32_t *counts) const;
00546     long getCumulativeDistribution(const char *constraints, const char *name,
00547                                    uint32_t nbc, double *bounds,
00548                                    uint32_t *counts) const;
00550 
00551     /******************************************************************/
00562     const std::vector<uint32_t> &getMeshShape() const {return shapeSize;}
00565     const std::vector<std::string> &getMeshDimensions() const {
00566         return shapeName;}
00570     void setMeshShape(const char *shape) {
00571         digestMeshShape(shape);
00572         writeMetaData(nEvents, columns, activeDir);
00573     }
00575     void updateMetaData() const {writeMetaData(nEvents, columns, activeDir);}
00577     void combineNames(ibis::table::namesTypes &metalist) const;
00578 
00579     /******************************************************************/
00581     long append(const char* dir);
00586     long commit(const char* dir);
00589     long rollback();
00590     long addColumn(const char* aexpr, const char* cname,
00591                    ibis::TYPE_T ctype=ibis::DOUBLE);
00592     long addColumn(const ibis::math::term* xpr, ibis::bitvector& mask,
00593                    const char* cname, ibis::TYPE_T ctype=ibis::DOUBLE);
00594 
00595     /******************************************************************/
00596     virtual long reorder();
00597     virtual long reorder(const ibis::table::stringList &names);
00598     virtual long reorder(const ibis::table::stringList &names,
00599                          const std::vector<bool> &directions);
00600 
00601     long deactivate(const std::vector<uint32_t> &rows);
00602     long deactivate(const char* conds);
00603     long reactivate(const std::vector<uint32_t> &rows);
00604     long reactivate(const char* conds);
00605     long purgeInactive();
00606     void emptyCache() const;
00608     const ibis::bitvector &getNullMask() const {return amask;}
00609 
00610     static const char* skipPrefix(const char*);
00612     static char* readMetaTags(const char* const dir);
00614     static void  genName(const std::vector<const char*> &mtags,
00615                          std::string &name);
00617     static void  genName(const ibis::resource::vList &mtags,
00618                          std::string &name);
00621     void rename(const ibis::partAssoc& known);
00623     static uint32_t countPages(const ibis::bitvector &mask,
00624                                unsigned elemsize=4);
00626     ibis::fileManager::ACCESS_PREFERENCE
00627     accessHint(const ibis::bitvector &mask, unsigned elemsize=4) const;
00628 
00630     struct thrArg {
00631         const part* et;
00632         const char* pref;
00633         long* nerrors;  
00634         ibis::util::counter cnt;
00635         std::vector<std::string> conds; 
00636         std::vector<unsigned> super;
00637         std::vector<unsigned> hits; 
00638     };
00640     struct indexBuilderPool {
00641         ibis::util::counter cnt;
00642         const char* opt;
00643         const part &tbl;
00644         indexBuilderPool(const part &t, const char* spec)
00645             : cnt(), opt(spec), tbl(t) {}
00646     };
00647 
00649     void queryTest(const char* pref, long* nerrors) const;
00651     void quickTest(const char* pref, long* nerrors) const;
00654     void testRangeOperators(const ibis::column* col,
00655                             long* nerrors) const;
00656 
00657     // These functions are used in classes barrel and vault
00658     void logWarning(const char* event, const char* fmt, ...) const;
00659     void logMessage(const char* event, const char* fmt, ...) const;
00660 
00661     void doBackup(); 
00662 
00663     class barrel;
00664     class vault;
00666     typedef std::map< const char*, column*, lessi > columnList;
00667 
00668     template <typename T> static int
00669         writeColumn(int fdes, ibis::bitvector::word_t nold,
00670                     ibis::bitvector::word_t nnew,
00671                     const array_t<T>& vals, const T& fill,
00672                     ibis::bitvector& totmask,
00673                     const ibis::bitvector& newmask);
00674     static int writeString(int fdes, ibis::bitvector::word_t nold,
00675                            ibis::bitvector::word_t nnew,
00676                            const std::vector<std::string>& vals,
00677                            ibis::bitvector& totmask,
00678                            const ibis::bitvector& newmask);
00679     static int writeRaw(int bdes, int sdes, ibis::bitvector::word_t nold,
00680                         ibis::bitvector::word_t nnew,
00681                         const ibis::array_t<unsigned char>& bytes,
00682                         const ibis::array_t<int64_t>& starts,
00683                         ibis::bitvector& totmask,
00684                         const ibis::bitvector& newmask);
00685 
00686 protected:
00687     class cleaner;      
00688     class writeLock;    
00689     class softWriteLock; 
00690     class mutexLock;    
00691 
00692     friend struct info;
00693     friend class cleaner;
00694     friend class readLock;
00695     friend class writeLock;
00696     friend class softWriteLock;
00697     friend class mutexLock;
00698 
00699     /******************************************************************/
00700     // protected member variables
00701     //
00702     char* m_name;               
00703     std::string m_desc;         
00704     ibis::resource::vList metaList;     
00705     mutable array_t<rid_t>* rids;       
00706     columnList columns;         
00707     uint32_t nEvents;           
00708     char* activeDir;            
00709     char* backupDir;            
00710     time_t switchTime;          
00711     TABLE_STATE state;
00712     char* idxstr;               
00713 
00714     ibis::bitvector amask;      
00715     std::vector<const column*> colorder;
00716     std::vector<std::string> shapeName; 
00717     std::vector<uint32_t> shapeSize;    
00718 
00719     ibis::part::cleaner* myCleaner;     
00720     bool readonly;                      
00721 
00722 
00723     /******************************************************************/
00724     // protected member functions
00725     //
00727     int  readMetaData(uint32_t &nrows, columnList &plist, const char* dir);
00729     void writeMetaData(const uint32_t nrows, const columnList &plist,
00730                        const char* dir) const;
00731     void readRIDs() const; 
00732     void freeRIDs() const; 
00733 
00734     // functions to deal with meta tags -- those attributes that are
00735     // associated with a whole partition but are expected to be processed
00736     // like a categorical attribute in queries
00737     void extendMetaTags();
00738     void setMetaTags(const ibis::resource::vList &mts);
00739     void setMetaTags(const std::vector<const char*> &mts);
00740 
00742     void readMeshShape(const char* const dir);
00744     void digestMeshShape(const char* shape);
00745 
00747     void logError(const char* event, const char* fmt, ...) const;
00748 
00749     void makeBackupCopy(); // copy the content of activeDir to backupDir
00750     long verifyBackupDir(); // minimal consistency check
00751     void deriveBackupDirName();
00752     long appendToBackup(const char* dir); // append to the backup directory
00753 
00755     template <typename T>
00756     long writeValues(const char *fname, const array_t<uint32_t> &ind);
00758     template <typename T>
00759     long reorderValues(const char *fname,
00760                        array_t<uint32_t> &starts,
00761                        array_t<uint32_t> &indout,
00762                        const array_t<uint32_t> &indin,
00763                        bool ascending);
00764     long append1(const char* dir);
00765     long append2(const char* dir);
00766 
00768     long deactivate(const ibis::bitvector &rows);
00770     long reactivate(const ibis::bitvector &rows);
00772     void numbersToBitvector(const std::vector<uint32_t>&,
00773                             ibis::bitvector&) const;
00774     void stringToBitvector(const char*, ibis::bitvector&) const;
00775 
00776     template <typename T>
00777     long doCompare(const array_t<T> &array,
00778                    const ibis::bitvector &mask,
00779                    ibis::bitvector &hits,
00780                    const ibis::qRange &cmp) const;
00781     template <typename T>
00782     long doCompare(const char *file,
00783                    const ibis::bitvector &mask,
00784                    ibis::bitvector &hits,
00785                    const ibis::qRange &cmp) const;
00786     template <typename T>
00787     long negativeCompare(const array_t<T> &array,
00788                          const ibis::bitvector &mask,
00789                          ibis::bitvector &hits,
00790                          const ibis::qRange &cmp) const;
00791     template <typename T>
00792     long negativeCompare(const char *file,
00793                          const ibis::bitvector &mask,
00794                          ibis::bitvector &hits,
00795                          const ibis::qRange &cmp) const;
00796 
00797     template <typename T>
00798     long doCompare(const array_t<T> &array,
00799                    const ibis::bitvector &mask,
00800                    ibis::bitvector &hits,
00801                    const ibis::qIntHod &cmp) const;
00802     template <typename T>
00803     long doCompare(const char *file,
00804                    const ibis::bitvector &mask,
00805                    ibis::bitvector &hits,
00806                    const ibis::qIntHod &cmp) const;
00807     template <typename T>
00808     long negativeCompare(const array_t<T> &array,
00809                          const ibis::bitvector &mask,
00810                          ibis::bitvector &hits,
00811                          const ibis::qIntHod &cmp) const;
00812     template <typename T>
00813     long negativeCompare(const char *file,
00814                          const ibis::bitvector &mask,
00815                          ibis::bitvector &hits,
00816                          const ibis::qIntHod &cmp) const;
00817 
00818     template <typename T>
00819     long doCompare(const array_t<T> &array,
00820                    const ibis::bitvector &mask,
00821                    ibis::bitvector &hits,
00822                    const ibis::qUIntHod &cmp) const;
00823     template <typename T>
00824     long doCompare(const char *file,
00825                    const ibis::bitvector &mask,
00826                    ibis::bitvector &hits,
00827                    const ibis::qUIntHod &cmp) const;
00828     template <typename T>
00829     long negativeCompare(const array_t<T> &array,
00830                          const ibis::bitvector &mask,
00831                          ibis::bitvector &hits,
00832                          const ibis::qUIntHod &cmp) const;
00833     template <typename T>
00834     long negativeCompare(const char *file,
00835                          const ibis::bitvector &mask,
00836                          ibis::bitvector &hits,
00837                          const ibis::qUIntHod &cmp) const;
00838 
00839     template <typename T, typename F>
00840         static long doCompare(const array_t<T> &vals, F cmp,
00841                               const ibis::bitvector &mask,
00842                               ibis::bitvector &hits);
00843 
00844     template <typename T, typename F>
00845         static long doCompare0(const array_t<T> &vals, F cmp,
00846                                const ibis::bitvector &mask,
00847                                ibis::bitvector &hits);
00848 
00849     template <typename T, typename F1, typename F2>
00850         static long doCompare(const array_t<T> &vals, F1 cmp1, F2 cmp2,
00851                               const ibis::bitvector &mask,
00852                               ibis::bitvector &hits);
00853 
00854     template <typename T, typename F1, typename F2>
00855         static long doCompare0(const array_t<T> &vals, F1 cmp1, F2 cmp2,
00856                                const ibis::bitvector &mask,
00857                                ibis::bitvector &hits);
00858 
00859     template <typename T>
00860         long doCount(const ibis::qRange &cmp) const;
00861 
00864     template <typename T>
00865         long doCount(const array_t<T> &vals, const ibis::qIntHod &cmp,
00866                      const ibis::bitvector &mask) const;
00867 
00870     template <typename T>
00871         long doCount(const array_t<T> &vals, const ibis::qUIntHod &cmp,
00872                      const ibis::bitvector &mask) const;
00873 
00876     template <typename T>
00877         long doCount(const array_t<T> &vals, const ibis::qRange &cmp,
00878                      const ibis::bitvector &mask) const;
00879 
00881     template <typename T, typename F>
00882         long doCount(const array_t<T> &vals,
00883                      const ibis::bitvector &mask, F cmp) const;
00884 
00886     template <typename T, typename F1, typename F2>
00887         long doCount(const array_t<T> &vals,
00888                      const ibis::bitvector &mask, F1 cmp1, F2 cmp2) const;
00889 
00892     long packCumulativeDistribution(const std::vector<double> &bounds,
00893                                     const std::vector<uint32_t> &counts,
00894                                     uint32_t nbc,
00895                                     double *bptr, uint32_t *cptr) const;
00897     long packDistribution(const std::vector<double> &bounds,
00898                           const std::vector<uint32_t> &counts,
00899                           uint32_t nbc, double *bptr, uint32_t *cptr) const;
00900 
00902     template <typename T1, typename T2>
00903     long count2DBins(array_t<T1> &vals1,
00904                      const double &begin1, const double &end1,
00905                      const double &stride1,
00906                      array_t<T2> &vals2,
00907                      const double &begin2, const double &end2,
00908                      const double &stride2,
00909                      std::vector<uint32_t> &counts) const;
00911     template <typename T1, typename T2, typename T3>
00912     long count3DBins(const array_t<T1> &vals1,
00913                      const double &begin1, const double &end1,
00914                      const double &stride1,
00915                      const array_t<T2> &vals2,
00916                      const double &begin2, const double &end2,
00917                      const double &stride2,
00918                      const array_t<T3> &vals3,
00919                      const double &begin3, const double &end3,
00920                      const double &stride3,
00921                      std::vector<uint32_t> &counts) const;
00923     template <typename T1, typename T2>
00924     long count2DWeights(array_t<T1> &vals1,
00925                         const double &begin1, const double &end1,
00926                         const double &stride1,
00927                         array_t<T2> &vals2,
00928                         const double &begin2, const double &end2,
00929                         const double &stride2,
00930                         array_t<double> &wts,
00931                         std::vector<double> &weights) const;
00933     template <typename T1, typename T2, typename T3>
00934     long count3DWeights(const array_t<T1> &vals1,
00935                         const double &begin1, const double &end1,
00936                         const double &stride1,
00937                         const array_t<T2> &vals2,
00938                         const double &begin2, const double &end2,
00939                         const double &stride2,
00940                         const array_t<T3> &vals3,
00941                         const double &begin3, const double &end3,
00942                         const double &stride3,
00943                         const array_t<double> &wts,
00944                         std::vector<double> &weights) const;
00945 
00947     template <typename T1>
00948     long fill1DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
00949                     const double &begin1, const double &end1,
00950                     const double &stride1,
00951                     std::vector<ibis::bitvector> &bins) const;
00953     template <typename T1>
00954     long fill1DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
00955                     const double &begin1, const double &end1,
00956                     const double &stride1,
00957                     std::vector<ibis::bitvector*> &bins) const;
00959     template <typename T1>
00960     long fill1DBinsWeighted(const ibis::bitvector &mask,
00961                             const array_t<T1> &vals1,
00962                             const double &begin1, const double &end1,
00963                             const double &stride1,
00964                             const array_t<double> &wts,
00965                             std::vector<double> &weights,
00966                             std::vector<ibis::bitvector*> &bins) const;
00968     template <typename T1, typename T2>
00969     long fill2DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
00970                     const double &begin1, const double &end1,
00971                     const double &stride1,
00972                     const array_t<T2> &vals2,
00973                     const double &begin2, const double &end2,
00974                     const double &stride2,
00975                     std::vector<ibis::bitvector> &bins) const;
00976     template <typename T1>
00977     long fill2DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
00978                      const double &begin1, const double &end1,
00979                      const double &stride1,
00980                      const ibis::column &col2,
00981                      const double &begin2, const double &end2,
00982                      const double &stride2,
00983                      std::vector<ibis::bitvector> &bins) const;
00985     template <typename T1, typename T2>
00986     long fill2DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
00987                     const double &begin1, const double &end1,
00988                     const double &stride1,
00989                     const array_t<T2> &vals2,
00990                     const double &begin2, const double &end2,
00991                     const double &stride2,
00992                     std::vector<ibis::bitvector*> &bins) const;
00993     template <typename T1>
00994     long fill2DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
00995                      const double &begin1, const double &end1,
00996                      const double &stride1,
00997                      const ibis::column &col2,
00998                      const double &begin2, const double &end2,
00999                      const double &stride2,
01000                      std::vector<ibis::bitvector*> &bins) const;
01002     template <typename T1, typename T2>
01003     long fill2DBinsWeighted(const ibis::bitvector &mask,
01004                             const array_t<T1> &vals1,
01005                             const double &begin1, const double &end1,
01006                             const double &stride1,
01007                             const array_t<T2> &vals2,
01008                             const double &begin2, const double &end2,
01009                             const double &stride2,
01010                             const array_t<double> &wts,
01011                             std::vector<double> &weights,
01012                             std::vector<ibis::bitvector*> &bins) const;
01013     template <typename T1>
01014     long fill2DBinsWeighted2(const ibis::bitvector &mask,
01015                              const array_t<T1> &vals1,
01016                              const double &begin1, const double &end1,
01017                              const double &stride1,
01018                              const ibis::column &col2,
01019                              const double &begin2, const double &end2,
01020                              const double &stride2,
01021                              const array_t<double> &wts,
01022                              std::vector<double> &weights,
01023                              std::vector<ibis::bitvector*> &bins) const;
01025     template <typename T1, typename T2, typename T3>
01026     long fill3DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
01027                     const double &begin1, const double &end1,
01028                     const double &stride1,
01029                     const array_t<T2> &vals2,
01030                     const double &begin2, const double &end2,
01031                     const double &stride2,
01032                     const array_t<T3> &vals3,
01033                     const double &begin3, const double &end3,
01034                     const double &stride3,
01035                     std::vector<ibis::bitvector> &bins) const;
01036     template <typename T1>
01037     long fill3DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
01038                      const double &begin1, const double &end1,
01039                      const double &stride1,
01040                      const ibis::column &col2,
01041                      const double &begin2, const double &end2,
01042                      const double &stride2,
01043                      const ibis::column &col3,
01044                      const double &begin3, const double &end3,
01045                      const double &stride3,
01046                      std::vector<bitvector> &bins) const;
01047     template <typename T1, typename T2>
01048     long fill3DBins3(const ibis::bitvector &mask, const array_t<T1> &vals1,
01049                      const double &begin1, const double &end1,
01050                      const double &stride1,
01051                      const array_t<T2> &vals2,
01052                      const double &begin2, const double &end2,
01053                      const double &stride2,
01054                      const ibis::column &col3,
01055                      const double &begin3, const double &end3,
01056                      const double &stride3,
01057                      std::vector<bitvector> &bins) const;
01058     template <typename T1, typename T2, typename T3>
01059     long fill3DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
01060                     const double &begin1, const double &end1,
01061                     const double &stride1,
01062                     const array_t<T2> &vals2,
01063                     const double &begin2, const double &end2,
01064                     const double &stride2,
01065                     const array_t<T3> &vals3,
01066                     const double &begin3, const double &end3,
01067                     const double &stride3,
01068                     std::vector<bitvector*> &bins) const;
01069     template <typename T1>
01070     long fill3DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
01071                      const double &begin1, const double &end1,
01072                      const double &stride1,
01073                      const ibis::column &col2,
01074                      const double &begin2, const double &end2,
01075                      const double &stride2,
01076                      const ibis::column &col3,
01077                      const double &begin3, const double &end3,
01078                      const double &stride3,
01079                      std::vector<ibis::bitvector*> &bins) const;
01080     template <typename T1, typename T2>
01081     long fill3DBins3(const ibis::bitvector &mask, const array_t<T1> &vals1,
01082                      const double &begin1, const double &end1,
01083                      const double &stride1,
01084                      const array_t<T2> &vals2,
01085                      const double &begin2, const double &end2,
01086                      const double &stride2,
01087                      const ibis::column &col3,
01088                      const double &begin3, const double &end3,
01089                      const double &stride3,
01090                      std::vector<ibis::bitvector*> &bins) const;
01091     template <typename T1, typename T2, typename T3>
01092     long fill3DBinsWeighted(const ibis::bitvector &mask,
01093                             const array_t<T1> &vals1,
01094                             const double &begin1, const double &end1,
01095                             const double &stride1,
01096                             const array_t<T2> &vals2,
01097                             const double &begin2, const double &end2,
01098                             const double &stride2,
01099                             const array_t<T3> &vals3,
01100                             const double &begin3, const double &end3,
01101                             const double &stride3,
01102                             const array_t<double> &wts,
01103                             std::vector<double> &weights,
01104                             std::vector<bitvector*> &bins) const;
01105     template <typename T1>
01106     long fill3DBinsWeighted2(const ibis::bitvector &mask,
01107                              const array_t<T1> &vals1,
01108                              const double &begin1, const double &end1,
01109                              const double &stride1,
01110                              const ibis::column &col2,
01111                              const double &begin2, const double &end2,
01112                              const double &stride2,
01113                              const ibis::column &col3,
01114                              const double &begin3, const double &end3,
01115                              const double &stride3,
01116                              const array_t<double> &wts,
01117                              std::vector<double> &weights,
01118                              std::vector<ibis::bitvector*> &bins) const;
01119     template <typename T1, typename T2>
01120     long fill3DBinsWeighted3(const ibis::bitvector &mask,
01121                              const array_t<T1> &vals1,
01122                              const double &begin1, const double &end1,
01123                              const double &stride1,
01124                              const array_t<T2> &vals2,
01125                              const double &begin2, const double &end2,
01126                              const double &stride2,
01127                              const ibis::column &col3,
01128                              const double &begin3, const double &end3,
01129                              const double &stride3,
01130                              const array_t<double> &wts,
01131                              std::vector<double> &weights,
01132                              std::vector<ibis::bitvector*> &bins) const;
01133 
01135     long get1DBins_(const ibis::bitvector &mask, const ibis::column &col,
01136                     uint32_t nbin, std::vector<double> &bounds,
01137                     std::vector<ibis::bitvector> &bins, const char *mesg) const;
01138 
01140     long get1DDistribution(const ibis::column &col, uint32_t nbin,
01141                            std::vector<double> &bounds,
01142                            std::vector<uint32_t> &counts) const;
01144     long get2DDistributionU(const ibis::column &col1,
01145                             const ibis::column &col2,
01146                             uint32_t nb1, uint32_t nb2,
01147                             std::vector<double> &bounds1,
01148                             std::vector<double> &bounds2,
01149                             std::vector<uint32_t> &counts) const;
01151     long get2DDistributionA(const ibis::column &col1,
01152                             const ibis::column &col2,
01153                             uint32_t nb1, uint32_t nb2,
01154                             std::vector<double> &bounds1,
01155                             std::vector<double> &bounds2,
01156                             std::vector<uint32_t> &counts) const;
01158     long get2DDistributionI(const ibis::column &col1,
01159                             const ibis::column &col2,
01160                             uint32_t nb1, uint32_t nb2,
01161                             std::vector<double> &bounds1,
01162                             std::vector<double> &bounds2,
01163                             std::vector<uint32_t> &counts) const;
01164     long old2DDistribution(const char *constraints,
01165                            const char *name1, const char *name2,
01166                            uint32_t nb1, uint32_t nb2,
01167                            std::vector<double> &bounds1,
01168                            std::vector<double> &bounds2,
01169                            std::vector<uint32_t> &counts) const;
01171     int coarsenBins(const ibis::column &col, uint32_t nbin,
01172                     std::vector<double> &bnds,
01173                     std::vector<ibis::bitvector*> &btmp) const;
01175     long get3DDistributionA(const ibis::bitvector &mask,
01176                             const ibis::column &col1,
01177                             const ibis::column &col2,
01178                             const ibis::column &col3,
01179                             uint32_t nb1, uint32_t nb2, uint32_t nb3,
01180                             std::vector<double> &bounds1,
01181                             std::vector<double> &bounds2,
01182                             std::vector<double> &bounds3,
01183                             std::vector<uint32_t> &counts) const;
01184     template <typename E1>
01185     long get3DDistributionA1(const ibis::bitvector &mask,
01186                              const array_t<E1> &vals1,
01187                              const ibis::column &col2,
01188                              const ibis::column &col3,
01189                              uint32_t nb1, uint32_t nb2, uint32_t nb3,
01190                              std::vector<double> &bounds1,
01191                              std::vector<double> &bounds2,
01192                              std::vector<double> &bounds3,
01193                              std::vector<uint32_t> &counts) const;
01194     template <typename E1, typename E2>
01195     long get3DDistributionA2(const ibis::bitvector &mask,
01196                              const array_t<E1> &vals1,
01197                              const array_t<E2> &vals2,
01198                              const ibis::column &col3,
01199                              uint32_t nb1, uint32_t nb2, uint32_t nb3,
01200                              std::vector<double> &bounds1,
01201                              std::vector<double> &bounds2,
01202                              std::vector<double> &bounds3,
01203                              std::vector<uint32_t> &counts) const;
01204 
01205     template <typename E1, typename E2>
01206         static void mapValues(array_t<E1> &val1, array_t<E2> &val2,
01207                               uint32_t nb1, uint32_t nb2,
01208                               array_t<E1> &bnd1, array_t<E2> &bnd2,
01209                               std::vector<uint32_t> &cnts);
01210 
01211     template <typename T>
01212         static void mapValues(const array_t<T> &vals,
01213                               std::map<T, uint32_t> &hist);
01214 
01215     template <typename T>
01216         static void equalWeightBins(const array_t<T> &vals,
01217                                     uint32_t nbins, array_t<T> &bounds);
01218 
01219     template <typename T>
01220         static long adaptiveInts(const array_t<T> &vals, const T vmin,
01221                                  const T vmax, uint32_t nbins,
01222                                  std::vector<double> &bounds,
01223                                  std::vector<uint32_t> &counts);
01224 
01225     template <typename T>
01226         static long adaptiveFloats(const array_t<T> &vals, const T vmin,
01227                                    const T vmax, uint32_t nbins,
01228                                    std::vector<double> &bounds,
01229                                    std::vector<uint32_t> &counts);
01230 
01231     template <typename T1, typename T2>
01232         static long adaptive2DBins(const array_t<T1> &vals1,
01233                                    const array_t<T2> &vals2,
01234                                    uint32_t nb1, uint32_t nb2,
01235                                    std::vector<double> &bounds1,
01236                                    std::vector<double> &bounds2,
01237                                    std::vector<uint32_t> &counts);
01238 
01239     template <typename T1, typename T2, typename T3>
01240         static long adaptive3DBins(const array_t<T1> &vals1,
01241                                    const array_t<T2> &vals2,
01242                                    const array_t<T3> &vals3,
01243                                    uint32_t nb1, uint32_t nb2, uint32_t nb3,
01244                                    std::vector<double> &bounds1,
01245                                    std::vector<double> &bounds2,
01246                                    std::vector<double> &bounds3,
01247                                    std::vector<uint32_t> &counts);
01248 
01249     template <typename T> static long
01250         adaptiveIntsDetailed(const ibis::bitvector &mask,
01251                              const array_t<T> &vals,
01252                              const T vmin, const T vmax, uint32_t nbins,
01253                              std::vector<double> &bounds,
01254                              std::vector<ibis::bitvector> &detail);
01255 
01256     template <typename T> static long
01257         adaptiveFloatsDetailed(const ibis::bitvector &mask,
01258                                const array_t<T> &vals,
01259                                const T vmin, const T vmax, uint32_t nbins,
01260                                std::vector<double> &bounds,
01261                                std::vector<ibis::bitvector> &detail);
01262 
01263     void gatherSortKeys(ibis::table::stringList& names);
01264     void composeQueryString(std::string &str,
01265                             const ibis::column* col1, const ibis::column* col2,
01266                             const double &lower1, const double &upper1,
01267                             const double &lower2, const double &upper2) const;
01268     void buildQueryList(ibis::part::thrArg &lst,
01269                         unsigned nc, unsigned nq) const;
01270     void checkQueryList(const ibis::part::thrArg &lst) const;
01271     uint32_t recursiveQuery(const char* pref, const column* att,
01272                             double low, double high, long* nerr) const;
01273 
01274 private:
01275 
01276     /******************************************************************/
01277     // private member variables
01278     mutable pthread_mutex_t mutex;      
01279     mutable pthread_rwlock_t rwlock;    
01280 
01281     /******************************************************************/
01282     // private funcations
01283 
01284     void init(const char* prefix); 
01285 
01286     void gainReadAccess(const char* mesg) const;
01287     void releaseAccess(const char* mesg) const;
01288     void gainWriteAccess(const char* mesg) const;
01289 
01290     void   fillRIDs(const char* fn) const; 
01291     void   sortRIDs() const; 
01292     uint32_t searchSortedRIDs(const ibis::rid_t &rid) const;
01293     uint32_t searchRIDs(const ibis::rid_t &rid) const;
01294     void   searchSortedRIDs(const ibis::RIDSet&, ibis::bitvector&) const;
01295     void   searchRIDs(const ibis::RIDSet&, ibis::bitvector&) const;
01296 
01297     // functions to perform join operations.
01298     int64_t equiJoin(const ibis::deprecatedJoin &cmp,
01299                      const ibis::bitvector64 &trial,
01300                      ibis::bitvector64 &result) const;
01301     int64_t deprecatedJoin(const ibis::deprecatedJoin &cmp,
01302                       const ibis::bitvector64 &trial,
01303                       ibis::bitvector64 &result) const;
01304     int64_t compJoin(const ibis::deprecatedJoin &cmp,
01305                      const ibis::bitvector64 &trial,
01306                      ibis::bitvector64 &result) const;
01307 
01308     int64_t loopJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
01309                      const ibis::bitvector &mask,
01310                      ibis::bitvector64 &pairs) const;
01311     int64_t loopJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
01312                      const ibis::bitvector &mask) const;
01313     int64_t loopJoin(const ibis::deprecatedJoin &cmp,
01314                      const ibis::bitvector &mask,
01315                      ibis::bitvector64 &pairs) const;
01316     int64_t loopJoin(const ibis::deprecatedJoin &cmp,
01317                      const ibis::bitvector &mask) const;
01318     int64_t equiJoinLoop1(const ibis::deprecatedJoin &cmp,
01319                           const ibis::bitvector &mask,
01320                           ibis::bitvector64 &pairs) const;
01321     int64_t equiJoinLoop1(const ibis::deprecatedJoin &cmp,
01322                           const ibis::bitvector &mask) const;
01323     int64_t equiJoinLoop2(const ibis::deprecatedJoin &cmp,
01324                           const ibis::bitvector &mask,
01325                           ibis::bitvector64 &pairs) const;
01326     int64_t equiJoinLoop2(const ibis::deprecatedJoin &cmp,
01327                           const ibis::bitvector &mask) const;
01328     template <class type1, class type2>
01329     void deprecatedJoinLoop(const array_t<type1> &arr1,
01330                        const ibis::bitvector &msk1,
01331                        const array_t<type2> &arr2,
01332                        const ibis::bitvector &msk2,
01333                        const double delta,
01334                        ibis::bitvector64 &pairs) const;
01335     template <class type1, class type2>
01336     int64_t deprecatedJoinLoop(const array_t<type1> &arr1,
01337                           const ibis::bitvector &msk1,
01338                           const array_t<type2> &arr2,
01339                           const ibis::bitvector &msk2,
01340                           const double delta) const;
01341     int64_t deprecatedJoinLoop(const ibis::deprecatedJoin &cmp,
01342                           const ibis::bitvector &mask,
01343                           ibis::bitvector64 &pairs) const;
01344     int64_t deprecatedJoinLoop(const ibis::deprecatedJoin &cmp,
01345                           const ibis::bitvector &mask) const;
01346     int64_t compJoinLoop(const ibis::deprecatedJoin &cmp,
01347                          const ibis::bitvector &mask,
01348                          ibis::bitvector64 &pairs) const;
01349     int64_t compJoinLoop(const ibis::deprecatedJoin &cmp,
01350                          const ibis::bitvector &mask) const;
01351 
01352     part(const part&);
01353     const part &operator=(const part&);
01354 }; // class ibis::part
01355 
01356 namespace ibis {
01357     // Extends the name space ibis::util to contain three more functions to
01358     // deal with the reconstruction of partitions.
01359     namespace util {
01361         unsigned int FASTBIT_CXX_DLLSPEC
01362         gatherParts(ibis::partList &parts,
01363                     const char *adir, const char *bdir, bool ro=false);
01365         unsigned int FASTBIT_CXX_DLLSPEC
01366         gatherParts(ibis::partList &parts, const char *adir, bool ro=false);
01369         unsigned int FASTBIT_CXX_DLLSPEC
01370         gatherParts(ibis::partList &parts, const ibis::resource &res,
01371                     bool ro=false);
01372     } // namespace util
01373 } // namespace ibis
01374 
01378 struct FASTBIT_CXX_DLLSPEC ibis::part::info {
01379     const char* name;           
01380     const char* description;    
01381     const char* metaTags;       
01382     const uint64_t nrows;       
01383 
01384     std::vector<ibis::column::info*> cols;
01385 
01386     info(const char* na, const char* de, const uint64_t &nr,
01387          const ibis::part::columnList &co);
01388     info(const ibis::part &tbl);
01389     ~info();
01390 
01391 private:
01392     info();     // private default constructor, not implemented!
01393     info& operator=(const info&);
01394 }; // ibis::part::info
01395 
01397 class ibis::part::cleaner : public ibis::fileManager::cleaner {
01398 public:
01399     virtual void operator()() const;
01400     cleaner(const part* tbl) : thePart(tbl) {}
01401     virtual ~cleaner() {}
01402 
01403 private:
01404     const part* thePart;
01405 }; // ibis::part::cleaner
01406 
01411 class ibis::part::readLock {
01412 public:
01413     readLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
01414         tbl->gainReadAccess(m);
01415     }
01416     ~readLock() {thePart->releaseAccess(mesg);}
01417 
01418 private:
01419     const part* thePart;
01420     const char* mesg;
01421 
01422     readLock() {}; // no default constructor
01423     readLock(const readLock&) {}; // can not copy
01424     const readLock &operator=(const readLock&);
01425 }; // ibis::part::readLock
01426 
01428 class ibis::part::writeLock {
01429 public:
01430     writeLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
01431         tbl->gainWriteAccess(m);
01432     }
01433     ~writeLock() {thePart->releaseAccess(mesg);}
01434 
01435 private:
01436     const part* thePart;
01437     const char* mesg;
01438 
01439     writeLock() {}; // no default constructor
01440     writeLock(const writeLock&) {}; // can not copy
01441     const writeLock &operator=(const writeLock&);
01442 }; // ibis::part::writeLock
01443 
01447 class ibis::part::softWriteLock {
01448 public:
01450     softWriteLock(const part* tbl, const char* m);
01452     ~softWriteLock() {if (lckd==0) thePart->releaseAccess(mesg);}
01455     bool isLocked() const {return (lckd==0);}
01456 
01457 private:
01458     const part* thePart;
01459     const char* mesg;
01460     const int lckd;
01461 
01462     softWriteLock() : thePart(0), mesg(0), lckd(0) {};
01463     softWriteLock(const softWriteLock &rhs)
01464         : thePart(rhs.thePart), mesg(rhs.mesg), lckd(0) {};
01465     const softWriteLock &operator=(const softWriteLock&);
01466 }; // ibis::part::softWriteLock
01467 
01470 class ibis::part::mutexLock {
01471 public:
01472     mutexLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
01473         if (ibis::gVerbose > 9)
01474             tbl->logMessage("gainExclusiveAccess",
01475                             "pthread_mutex_lock for %s", m);
01476         int ierr = pthread_mutex_lock(&(tbl->mutex));
01477         if (0 != ierr)
01478             tbl->logWarning("gainExclusiveAccess", "pthread_mutex_lock for %s "
01479                             "returned %d (%s)", m, ierr, strerror(ierr));
01480     }
01481     ~mutexLock() {
01482         if (ibis::gVerbose > 9)
01483             thePart->logMessage("releaseExclusiveAccess",
01484                                 "pthread_mutex_unlock for %s", mesg);
01485         int ierr = pthread_mutex_unlock(&(thePart->mutex));
01486         if (0 != ierr)
01487             thePart->logWarning("releaseExclusiveAccess",
01488                                 "pthread_mutex_unlock for %s returned %d (%s)",
01489                                 mesg, ierr, strerror(ierr));
01490     }
01491 
01492 private:
01493     const part* thePart;
01494     const char* mesg;
01495 
01496     mutexLock() {}; // no default constructor    
01497     mutexLock(const mutexLock&) {}; // can not copy      
01498     const mutexLock &operator=(const mutexLock&);        
01499 }; // ibis::part::mutexLock
01500 
01504 class ibis::part::barrel : public ibis::math::barrel {
01505 public:
01506     barrel(const ibis::part *t=0) : _tbl(t), position(0) {};
01507     virtual ~barrel() {close();} 
01508 
01509     virtual long open(const ibis::part *t=0); 
01510     virtual long close(); 
01511     virtual long read(); 
01512 
01513 
01514     virtual long seek(uint32_t pos);
01515     uint32_t tell() const {return position;}
01516 
01517     void getNullMask(ibis::bitvector &mask) const;
01518     const ibis::column* getColumn(uint32_t i) const {return cols[i];}
01519 
01520 protected:
01521     const ibis::part *_tbl;
01522     uint32_t position;
01523     std::vector<const ibis::column*> cols;
01524     std::vector<ibis::fileManager::storage*> stores;
01525     std::vector<int> fdes;
01526 }; // ibis::part::barrel
01527 
01530 class ibis::part::vault : public ibis::part::barrel {
01531 public:
01532     vault(const ibis::roster &r);
01533     virtual ~vault() {close();}
01534 
01535     virtual long open(const ibis::part *t=0); 
01536     virtual long read(); 
01537     virtual long seek(uint32_t pos); 
01538 
01539     long seek(double val);
01542     uint32_t tellReal() const;
01543 
01544 private:
01545     const ibis::roster &_roster;
01546 
01547     template <class T>
01548     uint32_t seekValue(int fd, const T &val) const;
01549     template <class T>
01550     uint32_t seekValue(const array_t<T>&arr, const T &val) const;
01551 
01552     vault();
01553     vault& operator=(const vault&);
01554 }; // ibis::part::vault
01555 /*
01556 barrel
01557 
01558 Vault \Vault\ (v[add]lt; see Note, below), n. [OE. voute, OF. voute, volte,
01559 F. vo[^u]te, LL. volta, for voluta, volutio, fr. L. volvere, volutum, to
01560 roll, to turn about. See Voluble, and cf. Vault a leap, Volt a turn,
01561 Volute.] 1. (Arch.) An arched structure of masonry, forming a ceiling or
01562 canopy.
01563 
01564 The long-drawn aisle and fretted vault. --Gray.
01565 
01566 2. An arched apartment; especially, a subterranean room, use for storing
01567    articles, for a prison, for interment, or the like; a cell; a
01568    cellar. ``Charnel vaults.'' --Milton.
01569 
01570 The silent vaults of death. --Sandys.
01571 
01572 To banish rats that haunt our vault. --Swift.
01573 
01574 3. The canopy of heaven; the sky.
01575 
01576 That heaven's vault should crack. --Shak.
01577 
01578 4. [F. volte, It. volta, originally, a turn, and the same word as volta an
01579    arch. See the Etymology above.] A leap or bound. Specifically: (a)
01580    (Man.) The bound or leap of a horse; a curvet. (b) A leap by aid of the
01581    hands, or of a pole, springboard, or the like.
01582 
01583 Note: The l in this word was formerly often suppressed in pronunciation.
01584 
01585 Barrel, Cradle, Cylindrical, or Wagon, vault (Arch.), a kind of vault
01586 having two parallel abutments, and the same section or profile at all
01587 points. It may be rampant, as over a staircase (see Rampant vault, under
01588 Rampant), or curved in plan, as around the apse of a church.
01589 
01590 Coved vault. (Arch.) See under 1st Cove, v. t.
01591 
01592 Groined vault (Arch.), a vault having groins, that is, one in which
01593 different cylindrical surfaces intersect one another, as distinguished from
01594 a barrel, or wagon, vault.
01595 
01596 Rampant vault. (Arch.) See under Rampant.
01597 
01598 Ribbed vault (Arch.), a vault differing from others in having solid ribs
01599 which bear the weight of the vaulted surface. True Gothic vaults are of
01600 this character.
01601 
01602 Vault light, a partly glazed plate inserted in a pavement or ceiling to
01603 admit light to a vault below.
01604 
01605 Source: Webster's Revised Unabridged Dictionary, (c) 1996, 1998 MICRA, Inc.
01606  */
01607 
01608 namespace ibis {
01609     // Explicit template specialization for member function
01610     // ibis::part::equalWeightBins
01611     template <> void
01612     part::equalWeightBins(const array_t<float> &vals,
01613                           uint32_t nbins, array_t<float> &bounds);
01614     template <> void
01615     part::equalWeightBins(const array_t<double> &vals,
01616                           uint32_t nbins, array_t<double> &bounds);
01617 
01618     template <> long
01619     part::doScan(const array_t<float> &,
01620                  const ibis::qContinuousRange &,
01621                  const ibis::bitvector &, ibis::bitvector &);
01622     template <> long
01623     part::doScan(const array_t<double> &,
01624                  const ibis::qContinuousRange &,
01625                  const ibis::bitvector &, ibis::bitvector &);
01626 
01627     template <> long
01628     part::doCount<float>(const ibis::qRange&) const;
01629     template <> long
01630     part::doCount<double>(const ibis::qRange&) const;
01631 }
01632 
01634 inline ibis::part::info* ibis::part::getInfo() const {
01635     return new info(*this);
01636 }
01637 
01641 inline ibis::column* ibis::part::getColumn(uint32_t ind) const {
01642     if (ind < columns.size()) {
01643         if (colorder.size() == columns.size()) {
01644             return const_cast<ibis::column*>(colorder[ind]);
01645         }
01646         else {
01647             ibis::part::columnList::const_iterator it = columns.begin();
01648             for (; ind > 0; -- ind) ++ it;
01649             return (*it).second;
01650         }
01651     }
01652     else {
01653         return 0;
01654     }
01655 } // ibis::part::getColumn
01656 
01657 inline const char* ibis::part::getMetaTag(const char* name) const {
01658     ibis::resource::vList::const_iterator it = metaList.find(name);
01659     if (it != metaList.end())
01660         return (*it).second;
01661     else
01662         return static_cast<const char*>(0);
01663 } // ibis::part::getMetaTag
01664 
01665 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &cmp,
01666                                         const ibis::bitvector &mask,
01667                                         ibis::bitvector64 &pairs) const {
01668     return loopJoin(cmp, mask, pairs);
01669 } // ibis::part::evaluateJoin
01670 
01671 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &,
01672                                         const ibis::bitvector &,
01673                                         const char *) const {
01674     logWarning("evaluate", "not implemented yet");
01675     return -1;
01676 } // ibis::part::evaluateJoin
01677 
01678 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &cmp,
01679                                         const ibis::bitvector &mask) const {
01680     return loopJoin(cmp, mask);
01681 } // ibis::part::evaluateJoin
01682 
01683 inline int64_t ibis::part::evaluateJoin
01684 (const std::vector<const ibis::deprecatedJoin*> &cmp,
01685  const ibis::bitvector &mask, ibis::bitvector64 &pairs) const {
01686     return loopJoin(cmp, mask, pairs);
01687 } // ibis::part::evaluateJoin
01688 
01690 inline void ibis::part::cleaner::operator()() const {
01691     const uint32_t sz = ibis::fileManager::bytesInUse();
01692     thePart->unloadIndexes();
01693     if (sz == ibis::fileManager::bytesInUse() &&
01694         thePart->getStateNoLocking() == ibis::part::STABLE_STATE) {
01695         thePart->freeRIDs();
01696     }
01697 } // ibis::part::cleaner::operator
01698 #endif // IBIS_PART_H

Make It A Bit Faster
Contact us
Disclaimers
FastBit source code
FastBit mailing list archive