part.h
Go to the documentation of this file.
00001 // File: $Id$
00002 // Author: John Wu <John.Wu at ACM.org>
00003 // Copyright 2000-2012 the Regents of the University of California
00004 #ifndef IBIS_PART_H
00005 #define IBIS_PART_H
00006 
00007 
00008 
00009 
00010 
00011 
00012 #include "column.h"
00013 #include "resource.h"
00014 #include "utilidor.h"
00015 
00016 #include <string>
00017 #include <vector>
00018 
00027 class FASTBIT_CXX_DLLSPEC ibis::part {
00028 public:
00029     enum TABLE_STATE {
00030         UNKNOWN_STATE=0, STABLE_STATE, RECEIVING_STATE,
00031         PRETRANSITION_STATE, TRANSITION_STATE, POSTTRANSITION_STATE
00032     };
00033     struct info; // To hold some basic information about a table.
00034     class readLock;
00035 
00036     /******************************************************************/
00037     // public functions
00038     /******************************************************************/
00039 
00041     virtual ~part();
00043     explicit part(const char* name=0, bool ro=false);
00045     part(const char* adir, const char* bdir, bool ro=false);
00047     part(const std::vector<const char*> &mtags, bool ro=false);
00049     part(const ibis::resource::vList &mtags, bool ro=false);
00050 
00051     inline info* getInfo() const;
00052     TABLE_STATE getState() const;
00054     TABLE_STATE getStateNoLocking() const {return state;}
00055 
00056     virtual int buildIndexes(const char* iopt=0, int nthr=1);
00057     void buildSorted(const char* colname) const;
00058     void loadIndexes(const char* iopt=0, int ropt=0) const;
00059     void unloadIndexes() const;
00060     void purgeIndexFiles() const;
00061 
00063     const char* name()          const {return (m_name?m_name:"?");}
00065     const char* description()   const {return m_desc.c_str();}
00067     const char* indexSpec()     const {return idxstr;}
00069     void indexSpec(const char*);
00071     time_t timestamp()          const {return switchTime;}
00074     std::string metaTags()      const;
00075     ibis::table::stringList columnNames() const;
00076     ibis::table::typeList   columnTypes() const;
00077     column* getColumn(const char* name) const;
00078     column* getColumn(uint32_t ind) const;
00079 
00081     const char* currentDataDir() const {return activeDir;}
00083     uint32_t nColumns() const {return columns.size();}
00085     uint32_t nRows() const {return nEvents;}
00086 
00087     int updateData();
00088 
00090     void print(std::ostream &out) const;
00091 
00093     bool matchNameValuePair(const char* name, const char* value) const;
00096     bool matchMetaTags(const std::vector<const char*> &mtags) const;
00098     bool matchMetaTags(const ibis::resource::vList &mtags) const;
00100     inline const char* getMetaTag(const char*) const;
00101 
00103     virtual long selfTest(int nth=1, const char* pref=0) const;
00104     void computeMinMax();
00105 
00106     /******************************************************************/
00107     // The following functions need to hold a read lock on the partition,
00108     // however in order for them to produce consistent results, they must
00109     // share the same read lock.  These functions do not contain the lock
00110     // themselves, but rely on the caller to maintain a consistent lock.
00111     /******************************************************************/
00113     uint32_t getRowNumber(const rid_t &rid) const;
00114     long evaluateRIDSet(const ibis::RIDSet&, ibis::bitvector&) const; 
00115     array_t<rid_t>* getRIDs() const {return rids;} // all RIDs
00116     array_t<rid_t>* getRIDs(const ibis::bitvector &mask) const;// some RIDs
00117     bool hasRIDs() const {return (rids!=0) ? (rids->size()==nEvents) : false;}
00118 
00120     virtual double estimateCost(const ibis::qContinuousRange &cmp) const;
00121     virtual double estimateCost(const ibis::qDiscreteRange &cmp) const;
00122     virtual double estimateCost(const ibis::qIntHod &cmp) const;
00123     virtual double estimateCost(const ibis::qUIntHod &cmp) const;
00124     virtual double estimateCost(const ibis::qString &cmp) const;
00125     virtual double estimateCost(const ibis::qMultiString &cmp) const;
00126 
00128     virtual long estimateRange(const ibis::qContinuousRange &cmp) const;
00129 
00131     virtual long estimateRange(const ibis::qDiscreteRange &cmp) const;
00133     virtual long estimateRange(const ibis::qIntHod &cmp) const;
00135     virtual long estimateRange(const ibis::qUIntHod &cmp) const;
00136 
00138     virtual long evaluateRange(const ibis::qContinuousRange &cmp,
00139                                const ibis::bitvector &mask,
00140                                ibis::bitvector &res) const;
00142     virtual long evaluateRange(const ibis::qDiscreteRange &cmp,
00143                                const ibis::bitvector &mask,
00144                                ibis::bitvector &res) const;
00146     virtual long evaluateRange(const ibis::qIntHod &cmp,
00147                                const ibis::bitvector &mask,
00148                                ibis::bitvector &res) const;
00150     virtual long evaluateRange(const ibis::qUIntHod &cmp,
00151                                const ibis::bitvector &mask,
00152                                ibis::bitvector &res) const;
00153 
00155     virtual long estimateRange(const ibis::qContinuousRange &cmp,
00156                                ibis::bitvector &low,
00157                                ibis::bitvector &high) const;
00158 
00160     virtual float getUndecidable(const ibis::qContinuousRange &cmp,
00161                                  ibis::bitvector &iffy) const;
00162 
00164     virtual long estimateRange(const ibis::qDiscreteRange &cmp,
00165                                ibis::bitvector &low,
00166                                ibis::bitvector &high) const;
00168     virtual float getUndecidable(const ibis::qDiscreteRange &cmp,
00169                                  ibis::bitvector &iffy) const;
00170 
00172     virtual long estimateRange(const ibis::qIntHod &cmp,
00173                                ibis::bitvector &low,
00174                                ibis::bitvector &high) const;
00176     virtual float getUndecidable(const ibis::qIntHod &cmp,
00177                                  ibis::bitvector &iffy) const;
00178 
00180     virtual long estimateRange(const ibis::qUIntHod &cmp,
00181                                ibis::bitvector &low,
00182                                ibis::bitvector &high) const;
00184     virtual float getUndecidable(const ibis::qUIntHod &cmp,
00185                                  ibis::bitvector &iffy) const;
00186 
00187     virtual long doScan(const ibis::qRange &cmp,
00188                         ibis::bitvector &hits) const;
00189     virtual long doScan(const ibis::qRange &cmp,
00190                         const ibis::bitvector &mask,
00191                         ibis::bitvector &hits) const;
00192     virtual long doScan(const ibis::qRange &cmp,
00193                         const ibis::bitvector &mask,
00194                         void *res) const;
00195     virtual long doScan(const ibis::qRange &cmp,
00196                         const ibis::bitvector &mask,
00197                         void *res, ibis::bitvector &hits) const;
00198 
00199     virtual long negativeScan(const ibis::qRange &cmp,
00200                               const ibis::bitvector &mask,
00201                               ibis::bitvector &hits) const;
00202 
00203     virtual long doScan(const ibis::math::term&, const ibis::bitvector&,
00204                         ibis::bitvector&) const;
00205     virtual long doScan(const ibis::compRange &cmp,
00206                         ibis::bitvector &hits) const;
00207     virtual long doScan(const ibis::compRange &cmp,
00208                         const ibis::bitvector &mask,
00209                         ibis::bitvector &hits) const;
00210 
00211     template <typename E> static long
00212         doScan(const array_t<E> &varr,
00213                const ibis::qRange &cmp,
00214                const ibis::bitvector &mask,
00215                ibis::bitvector &hits);
00216     template <typename E> static long
00217         doScan(const array_t<E> &varr,
00218                const ibis::qContinuousRange &cmp,
00219                const ibis::bitvector &mask,
00220                ibis::bitvector &hits);
00221     template <typename E> static long
00222         doScan(const array_t<E> &varr,
00223                const ibis::qContinuousRange &cmp,
00224                const ibis::bitvector &mask,
00225                array_t<E> &res);
00226     template <typename E> static long
00227         doScan(const array_t<E> &varr,
00228                const ibis::qContinuousRange &cmp,
00229                const ibis::bitvector &mask,
00230                array_t<E> &res, ibis::bitvector &hits);
00231 
00232     long countHits(const ibis::qRange &cmp) const;
00233 
00234     virtual long estimateMatchAny(const ibis::qAnyAny &cmp,
00235                                   ibis::bitvector &low,
00236                                   ibis::bitvector &high) const;
00237     virtual long matchAny(const ibis::qAnyAny &cmp,
00238                           ibis::bitvector &hits) const;
00239     virtual long matchAny(const ibis::qAnyAny &cmp,
00240                           const ibis::bitvector &mask,
00241                           ibis::bitvector &hits) const;
00242 
00243     long patternSearch(const ibis::qLike &cmp,
00244                     ibis::bitvector &low) const;
00245     long lookforString(const ibis::qString &cmp,
00246                        ibis::bitvector &low) const;
00247     long lookforString(const ibis::qMultiString &cmp,
00248                        ibis::bitvector &low) const;
00249     long patternSearch(const ibis::qLike &cmp) const;
00250     long lookforString(const ibis::qString &cmp) const;
00251     long lookforString(const ibis::qMultiString &cmp) const;
00252 
00257     int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
00258                          const ibis::bitvector &mask,
00259                          ibis::bitvector64 &pairs) const;
00262     int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
00263                          const ibis::bitvector &mask,
00264                          const char* pairfile) const;
00266     int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
00267                          const ibis::bitvector &mask) const;
00270     int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
00271                          const ibis::bitvector &mask,
00272                          ibis::bitvector64 &pairs) const;
00273     int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
00274                          const ibis::bitvector &mask) const;
00279     int64_t evaluateJoin(const ibis::deprecatedJoin &cmp,
00280                          const ibis::bitvector64 &trial,
00281                          ibis::bitvector64 &result) const;
00282     int64_t evaluateJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
00283                          const ibis::bitvector64 &trial,
00284                          ibis::bitvector64 &result) const;
00285     /******************************************************************/
00286 
00288     array_t<signed char>*
00289         selectBytes(const char* name, const ibis::bitvector &mask) const;
00291     array_t<unsigned char>*
00292         selectUBytes(const char* name, const ibis::bitvector &mask) const;
00294     array_t<int16_t>*
00295         selectShorts(const char* name, const ibis::bitvector &mask) const;
00297     array_t<uint16_t>*
00298         selectUShorts(const char* name, const ibis::bitvector &mask) const;
00300     array_t<int32_t>*
00301         selectInts(const char* name, const ibis::bitvector &mask) const;
00303     array_t<uint32_t>*
00304         selectUInts(const char* name, const ibis::bitvector &mask) const;
00306     array_t<int64_t>*
00307         selectLongs(const char* name, const ibis::bitvector &mask) const;
00309     array_t<uint64_t>*
00310         selectULongs(const char* name, const ibis::bitvector &mask) const;
00312     array_t<float>*
00313         selectFloats(const char* name, const ibis::bitvector &mask) const;
00315     array_t<double>*
00316         selectDoubles(const char* name, const ibis::bitvector &mask) const;
00318     std::vector<std::string>*
00319         selectStrings(const char* name, const ibis::bitvector &mask) const;
00320 
00321     long selectValues(const char* cname, const ibis::bitvector &mask,
00322                       void* vals) const;
00323     long selectValues(const ibis::qContinuousRange& cond, void* vals) const;
00324     long calculate(const ibis::math::term&, const ibis::bitvector&,
00325                    array_t<double>&) const;
00326 
00327     /******************************************************************/
00328     // A group of functions added August, 2005 to provide summary
00329     // information about the data.  Logically, they are an extension of
00330     // getInfo.
00331 
00333     double getActualMin(const char *name) const;
00335     double getActualMax(const char *name) const;
00337     double getColumnSum(const char *name) const;
00338 
00342     long get1DDistribution(const char *constraints, const char *cname,
00343                            double begin, double end, double stride,
00344                            std::vector<uint32_t> &counts) const;
00346     long get2DDistribution(const char *constraints, const char *cname1,
00347                            double begin1, double end1, double stride1,
00348                            const char *cname2,
00349                            double begin2, double end2, double stride2,
00350                            std::vector<uint32_t> &counts) const;
00352     long get3DDistribution(const char *constraints, const char *cname1,
00353                            double begin1, double end1, double stride1,
00354                            const char *cname2,
00355                            double begin2, double end2, double stride2,
00356                            const char *cname3,
00357                            double begin3, double end3, double stride3,
00358                            std::vector<uint32_t> &counts) const;
00360     long get1DDistribution(const char *constraints, const char *cname,
00361                            double begin, double end, double stride,
00362                            const char *wtname,
00363                            std::vector<double> &weights) const;
00365     long get2DDistribution(const char *constraints, const char *cname1,
00366                            double begin1, double end1, double stride1,
00367                            const char *cname2,
00368                            double begin2, double end2, double stride2,
00369                            const char *wtname,
00370                            std::vector<double> &weights) const;
00372     long get3DDistribution(const char *constraints, const char *cname1,
00373                            double begin1, double end1, double stride1,
00374                            const char *cname2,
00375                            double begin2, double end2, double stride2,
00376                            const char *cname3,
00377                            double begin3, double end3, double stride3,
00378                            const char *wtname,
00379                            std::vector<double> &weights) const;
00381     long get1DDistribution(const char *cname, uint32_t nbin,
00382                            std::vector<double> &bounds,
00383                            std::vector<uint32_t> &counts) const;
00385     long get1DDistribution(const char *constraints,
00386                            const char *cname, uint32_t nbin,
00387                            std::vector<double> &bounds,
00388                            std::vector<uint32_t> &counts) const;
00390     long get2DDistribution(const char *cname1, const char *cname2,
00391                            uint32_t nb1, uint32_t nb2,
00392                            std::vector<double> &bounds1,
00393                            std::vector<double> &bounds2,
00394                            std::vector<uint32_t> &counts,
00395                            const char* const option=0) const;
00397     long get2DDistribution(const char *constraints,
00398                            const char *name1, const char *name2,
00399                            uint32_t nb1, uint32_t nb2,
00400                            std::vector<double> &bounds1,
00401                            std::vector<double> &bounds2,
00402                            std::vector<uint32_t> &counts) const;
00404     long get3DDistribution(const char *cname1, const char *cname2,
00405                            const char *cname3,
00406                            uint32_t nb1, uint32_t nb2, uint32_t nb3,
00407                            std::vector<double> &bounds1,
00408                            std::vector<double> &bounds2,
00409                            std::vector<double> &bounds3,
00410                            std::vector<uint32_t> &counts,
00411                            const char* const option=0) const;
00413     long get3DDistribution(const char *constraints,
00414                            const char *cname1, const char *cname2,
00415                            const char *cname3,
00416                            uint32_t nb1, uint32_t nb2, uint32_t nb3,
00417                            std::vector<double> &bounds1,
00418                            std::vector<double> &bounds2,
00419                            std::vector<double> &bounds3,
00420                            std::vector<uint32_t> &counts) const;
00422     long get1DBins(const char *constraints, const char *cname,
00423                    double begin, double end, double stride,
00424                    std::vector<ibis::bitvector> &bins) const;
00426     long get1DBins(const char *constraints, const char *cname,
00427                    double begin, double end, double stride,
00428                    std::vector<ibis::bitvector*> &bins) const;
00430     long get1DBins(const char *constraints, const char *cname,
00431                    double begin, double end, double stride,
00432                    const char *wtname,
00433                    std::vector<double> &weights,
00434                    std::vector<ibis::bitvector*> &bins) const;
00436     long get2DBins(const char *constraints, const char *cname1,
00437                    double begin1, double end1, double stride1,
00438                    const char *cname2,
00439                    double begin2, double end2, double stride2,
00440                    std::vector<ibis::bitvector> &bins) const;
00442     long get2DBins(const char *constraints, const char *cname1,
00443                    double begin1, double end1, double stride1,
00444                    const char *cname2,
00445                    double begin2, double end2, double stride2,
00446                    std::vector<ibis::bitvector*> &bins) const;
00448     long get2DBins(const char *constraints, const char *cname1,
00449                    double begin1, double end1, double stride1,
00450                    const char *cname2,
00451                    double begin2, double end2, double stride2,
00452                    const char *wtname,
00453                    std::vector<double> &weights,
00454                    std::vector<ibis::bitvector*> &bins) const;
00456     long get3DBins(const char *constraints, const char *cname1,
00457                    double begin1, double end1, double stride1,
00458                    const char *cname2,
00459                    double begin2, double end2, double stride2,
00460                    const char *cname3,
00461                    double begin3, double end3, double stride3,
00462                    std::vector<ibis::bitvector> &bins) const;
00464     long get3DBins(const char *constraints, const char *cname1,
00465                    double begin1, double end1, double stride1,
00466                    const char *cname2,
00467                    double begin2, double end2, double stride2,
00468                    const char *cname3,
00469                    double begin3, double end3, double stride3,
00470                    std::vector<ibis::bitvector*> &bins) const;
00472     long get3DBins(const char *constraints, const char *cname1,
00473                    double begin1, double end1, double stride1,
00474                    const char *cname2,
00475                    double begin2, double end2, double stride2,
00476                    const char *cname3,
00477                    double begin3, double end3, double stride3,
00478                    const char *wtname,
00479                    std::vector<double> &weights,
00480                    std::vector<ibis::bitvector*> &bins) const;
00483     long get1DBins(const char *constraints, const char *cname1, uint32_t nb1,
00484                    std::vector<double> &bounds1,
00485                    std::vector<ibis::bitvector> &bins) const;
00487     long get2DBins(const char *constraints,
00488                    const char *cname1, const char *cname2,
00489                    uint32_t nb1, uint32_t nb2,
00490                    std::vector<double> &bounds1,
00491                    std::vector<double> &bounds2,
00492                    std::vector<ibis::bitvector> &bins) const;
00494     long get3DBins(const char *constraints,
00495                    const char *cname1, const char *cname2, const char *cname3,
00496                    uint32_t nb1, uint32_t nb2, uint32_t nb3,
00497                    std::vector<double> &bounds1,
00498                    std::vector<double> &bounds2,
00499                    std::vector<double> &bounds3,
00500                    std::vector<ibis::bitvector> &bins) const;
00502 
00507     long getDistribution(const char *name,
00508                          std::vector<double> &bounds,
00509                          std::vector<uint32_t> &counts) const;
00511     long getDistribution(const char *constraints,
00512                          const char *name,
00513                          std::vector<double> &bounds,
00514                          std::vector<uint32_t> &counts) const;
00517     long getDistribution(const char *name, uint32_t nbc,
00518                          double *bounds, uint32_t *counts) const;
00521     long getDistribution(const char *name, const char *constraints,
00522                          uint32_t nbc, double *bounds,
00523                          uint32_t *counts) const;
00524 
00526     long getJointDistribution(const char *constraints,
00527                               const char *name1, const char *name2,
00528                               std::vector<double> &bounds1,
00529                               std::vector<double> &bounds2,
00530                               std::vector<uint32_t> &counts) const;
00531 
00533     long getCumulativeDistribution(const char *name,
00534                                    std::vector<double> &bounds,
00535                                    std::vector<uint32_t> &counts) const;
00538     long getCumulativeDistribution(const char *constraints,
00539                                    const char *name,
00540                                    std::vector<double> &bounds,
00541                                    std::vector<uint32_t> &counts) const;
00544     long getCumulativeDistribution(const char *name, uint32_t nbc,
00545                                    double *bounds, uint32_t *counts) const;
00548     long getCumulativeDistribution(const char *constraints, const char *name,
00549                                    uint32_t nbc, double *bounds,
00550                                    uint32_t *counts) const;
00552 
00553     /******************************************************************/
00564     const std::vector<uint32_t> &getMeshShape() const {return shapeSize;}
00567     const std::vector<std::string> &getMeshDimensions() const {
00568         return shapeName;}
00569     void setMeshShape(const char *shape);
00570     void updateMetaData() const;
00572     void combineNames(ibis::table::namesTypes &metalist) const;
00573 
00574     /******************************************************************/
00576     long append(const char* dir);
00581     long commit(const char* dir);
00584     long rollback();
00585     long addColumn(const char* aexpr, const char* cname,
00586                    ibis::TYPE_T ctype=ibis::DOUBLE);
00587     long addColumn(const ibis::math::term* xpr, ibis::bitvector& mask,
00588                    const char* cname, ibis::TYPE_T ctype=ibis::DOUBLE);
00589 
00590     /******************************************************************/
00591     virtual long reorder();
00592     virtual long reorder(const ibis::table::stringList &names);
00593     virtual long reorder(const ibis::table::stringList &names,
00594                          const std::vector<bool> &directions);
00595 
00596     long deactivate(const std::vector<uint32_t> &rows);
00597     long deactivate(const char* conds);
00598     long reactivate(const std::vector<uint32_t> &rows);
00599     long reactivate(const char* conds);
00600     long purgeInactive();
00601     void emptyCache() const;
00603     const ibis::bitvector &getNullMask() const {return amask;}
00604 
00605     static const char* skipPrefix(const char*);
00607     static char* readMetaTags(const char* const dir);
00609     static void  genName(const std::vector<const char*> &mtags,
00610                          std::string &name);
00612     static void  genName(const ibis::resource::vList &mtags,
00613                          std::string &name);
00614     void rename(const ibis::partAssoc& known);
00615     void rename(const char*);
00616 
00618     static uint32_t countPages(const ibis::bitvector &mask,
00619                                unsigned elemsize=4);
00621     ibis::fileManager::ACCESS_PREFERENCE
00622     accessHint(const ibis::bitvector &mask, unsigned elemsize=4) const;
00623 
00625     struct thrArg {
00626         const part* et;
00627         const char* pref;
00628         long* nerrors;  
00629         ibis::util::counter cnt;
00630         std::vector<std::string> conds; 
00631         std::vector<unsigned> super;
00632         std::vector<unsigned> hits; 
00633     };
00635     struct indexBuilderPool {
00636         ibis::util::counter cnt;
00637         const char* opt;
00638         const part &tbl;
00639         indexBuilderPool(const part &t, const char* spec)
00640             : cnt(), opt(spec), tbl(t) {}
00641     };
00642 
00644     void queryTest(const char* pref, long* nerrors) const;
00646     void quickTest(const char* pref, long* nerrors) const;
00649     void testRangeOperators(const ibis::column* col,
00650                             long* nerrors) const;
00651 
00652     // These functions are used in classes barrel and vault
00653     void logWarning(const char* event, const char* fmt, ...) const;
00654     void logMessage(const char* event, const char* fmt, ...) const;
00655 
00656     void doBackup(); 
00657 
00658     class barrel;
00659     class vault;
00661     typedef std::map< const char*, column*, lessi > columnList;
00662 
00663     template <typename T> static int
00664         writeColumn(int fdes, ibis::bitvector::word_t nold,
00665                     ibis::bitvector::word_t nnew,
00666                     const array_t<T>& vals, const T& fill,
00667                     ibis::bitvector& totmask,
00668                     const ibis::bitvector& newmask);
00669     static int writeString(int fdes, ibis::bitvector::word_t nold,
00670                            ibis::bitvector::word_t nnew,
00671                            const std::vector<std::string>& vals,
00672                            ibis::bitvector& totmask,
00673                            const ibis::bitvector& newmask);
00674     static int writeRaw(int bdes, int sdes, ibis::bitvector::word_t nold,
00675                         ibis::bitvector::word_t nnew,
00676                         const ibis::array_t<unsigned char>& bytes,
00677                         const ibis::array_t<int64_t>& starts,
00678                         ibis::bitvector& totmask,
00679                         const ibis::bitvector& newmask);
00680 
00681 protected:
00682     class cleaner;
00683     class mutexLock;
00684     class writeLock;
00685     class softWriteLock;
00686 
00687     friend struct info;
00688     friend class cleaner;
00689     friend class readLock;
00690     friend class writeLock;
00691     friend class softWriteLock;
00692     friend class mutexLock;
00693 
00694     /******************************************************************/
00695     // protected member variables
00696     //
00697     char* m_name;               
00698     std::string m_desc;         
00699     ibis::resource::vList metaList;     
00700     mutable array_t<rid_t>* rids;       
00701     columnList columns;         
00702     uint32_t nEvents;           
00703     char* activeDir;            
00704     char* backupDir;            
00705     time_t switchTime;          
00706     TABLE_STATE state;
00707     char* idxstr;               
00708 
00709     ibis::bitvector amask;      
00710     std::vector<const column*> colorder;
00711     std::vector<std::string> shapeName; 
00712     std::vector<uint32_t> shapeSize;    
00713 
00714     ibis::part::cleaner* myCleaner;     
00715     bool readonly;                      
00716 
00717 
00718     /******************************************************************/
00719     // protected member functions
00720     //
00722     int  readMetaData(uint32_t &nrows, columnList &plist, const char* dir);
00724     void writeMetaData(const uint32_t nrows, const columnList &plist,
00725                        const char* dir) const;
00726     void readRIDs() const; 
00727     void freeRIDs() const; 
00728 
00729     // functions to deal with meta tags -- those attributes that are
00730     // associated with a whole partition but are expected to be processed
00731     // like a categorical attribute in queries
00732     void extendMetaTags();
00733     void setMetaTags(const ibis::resource::vList &mts);
00734     void setMetaTags(const std::vector<const char*> &mts);
00735 
00737     void readMeshShape(const char* const dir);
00739     void digestMeshShape(const char* shape);
00740 
00742     void logError(const char* event, const char* fmt, ...) const;
00743 
00744     void makeBackupCopy(); // copy the content of activeDir to backupDir
00745     long verifyBackupDir(); // minimal consistency check
00746     void deriveBackupDirName();
00747     long appendToBackup(const char* dir); // append to the backup directory
00748 
00750     template <typename T>
00751     long writeValues(const char *fname, const array_t<uint32_t> &ind);
00753     template <typename T>
00754     long reorderValues(const char *fname,
00755                        array_t<uint32_t> &starts,
00756                        array_t<uint32_t> &indout,
00757                        const array_t<uint32_t> &indin,
00758                        bool ascending);
00759     long append1(const char* dir);
00760     long append2(const char* dir);
00761 
00763     long deactivate(const ibis::bitvector &rows);
00765     long reactivate(const ibis::bitvector &rows);
00767     void numbersToBitvector(const std::vector<uint32_t>&,
00768                             ibis::bitvector&) const;
00769     void stringToBitvector(const char*, ibis::bitvector&) const;
00770 
00771     template <typename T> static long
00772         doCompare(const array_t<T> &array,
00773                   const ibis::qRange &cmp,
00774                   const ibis::bitvector &mask,
00775                   ibis::array_t<T> &res, ibis::bitvector &hits);
00776     template <typename T> static long
00777         doCompare(const char *file,
00778                   const ibis::qRange &cmp,
00779                   const ibis::bitvector &mask,
00780                   ibis::array_t<T> &res, ibis::bitvector &hits);
00781 
00782     template <typename T> static long
00783         doCompare(const array_t<T> &array,
00784                   const ibis::qRange &cmp,
00785                   const ibis::bitvector &mask,
00786                   ibis::array_t<T> &res);
00787     template <typename T> static long
00788         doCompare(const char *file,
00789                   const ibis::qRange &cmp,
00790                   const ibis::bitvector &mask,
00791                   ibis::array_t<T> &res);
00792 
00793     template <typename T> static long
00794         doCompare(const array_t<T> &array,
00795                   const ibis::qRange &cmp,
00796                   const ibis::bitvector &mask,
00797                   ibis::bitvector &hits);
00798     template <typename T> static long
00799         doCompare(const char *file,
00800                   const ibis::qRange &cmp,
00801                   const ibis::bitvector &mask,
00802                   ibis::bitvector &hits);
00803 
00804     template <typename T> static long
00805         negativeCompare(const array_t<T> &array,
00806                         const ibis::qRange &cmp,
00807                         const ibis::bitvector &mask,
00808                         ibis::bitvector &hits);
00809     template <typename T> static long
00810         negativeCompare(const char *file,
00811                         const ibis::qRange &cmp,
00812                         const ibis::bitvector &mask,
00813                         ibis::bitvector &hits);
00814 
00815     template <typename T> static long
00816         doCompare(const array_t<T> &array,
00817                   const ibis::qIntHod &cmp,
00818                   const ibis::bitvector &mask,
00819                   ibis::bitvector &hits);
00820     template <typename T> static long
00821         doCompare(const char *file,
00822                   const ibis::qIntHod &cmp,
00823                   const ibis::bitvector &mask,
00824                   ibis::bitvector &hits);
00825     template <typename T> static long
00826         negativeCompare(const array_t<T> &array,
00827                         const ibis::qIntHod &cmp,
00828                         const ibis::bitvector &mask,
00829                         ibis::bitvector &hits);
00830     template <typename T> static long
00831         negativeCompare(const char *file,
00832                         const ibis::qIntHod &cmp,
00833                         const ibis::bitvector &mask,
00834                         ibis::bitvector &hits);
00835 
00836     template <typename T> static long
00837         doCompare(const array_t<T> &array,
00838                   const ibis::qUIntHod &cmp,
00839                   const ibis::bitvector &mask,
00840                   ibis::bitvector &hits);
00841     template <typename T> static long
00842         doCompare(const char *file,
00843                   const ibis::qUIntHod &cmp,
00844                   const ibis::bitvector &mask,
00845                   ibis::bitvector &hits);
00846     template <typename T> static long
00847         negativeCompare(const array_t<T> &array,
00848                         const ibis::qUIntHod &cmp,
00849                         const ibis::bitvector &mask,
00850                         ibis::bitvector &hits);
00851     template <typename T> static long
00852         negativeCompare(const char *file,
00853                         const ibis::qUIntHod &cmp,
00854                         const ibis::bitvector &mask,
00855                         ibis::bitvector &hits);
00856 
00857     template <typename T, typename F> static long
00858         doComp(const array_t<T> &vals, F cmp,
00859                const ibis::bitvector &mask,
00860                array_t<T> &res, ibis::bitvector &hits);
00861 
00862     template <typename T, typename F> static long
00863         doComp(const array_t<T> &vals, F cmp,
00864                const ibis::bitvector &mask,
00865                array_t<T> &res);
00866 
00867     template <typename T, typename F> static long
00868         doComp(const array_t<T> &vals, F cmp,
00869                const ibis::bitvector &mask,
00870                ibis::bitvector &hits);
00871 
00872     template <typename T, typename F1, typename F2> static long
00873         doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2,
00874                const ibis::bitvector &mask,
00875                array_t<T> &res, ibis::bitvector &hits);
00876 
00877     template <typename T, typename F1, typename F2> static long
00878         doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2,
00879                const ibis::bitvector &mask,
00880                array_t<T> &res);
00881 
00882     template <typename T, typename F1, typename F2> static long
00883         doComp(const array_t<T> &vals, F1 cmp1, F2 cmp2,
00884                const ibis::bitvector &mask,
00885                ibis::bitvector &hits);
00886 
00887     template <typename T, typename F> static long
00888         doComp0(const array_t<T> &vals, F cmp,
00889                 const ibis::bitvector &mask,
00890                 ibis::bitvector &hits);
00891 
00892     template <typename T, typename F1, typename F2> static long
00893         doComp0(const array_t<T> &vals, F1 cmp1, F2 cmp2,
00894                 const ibis::bitvector &mask,
00895                 ibis::bitvector &hits);
00896 
00897     template <typename T>
00898         long doCount(const ibis::qRange &cmp) const;
00899 
00902     template <typename T>
00903         long doCount(const array_t<T> &vals, const ibis::qIntHod &cmp,
00904                      const ibis::bitvector &mask) const;
00905 
00908     template <typename T>
00909         long doCount(const array_t<T> &vals, const ibis::qUIntHod &cmp,
00910                      const ibis::bitvector &mask) const;
00911 
00914     template <typename T>
00915         long doCount(const array_t<T> &vals, const ibis::qRange &cmp,
00916                      const ibis::bitvector &mask) const;
00917 
00919     template <typename T, typename F>
00920         long doCount(const array_t<T> &vals,
00921                      const ibis::bitvector &mask, F cmp) const;
00922 
00924     template <typename T, typename F1, typename F2>
00925         long doCount(const array_t<T> &vals,
00926                      const ibis::bitvector &mask, F1 cmp1, F2 cmp2) const;
00927 
00930     long packCumulativeDistribution(const std::vector<double> &bounds,
00931                                     const std::vector<uint32_t> &counts,
00932                                     uint32_t nbc,
00933                                     double *bptr, uint32_t *cptr) const;
00935     long packDistribution(const std::vector<double> &bounds,
00936                           const std::vector<uint32_t> &counts,
00937                           uint32_t nbc, double *bptr, uint32_t *cptr) const;
00938 
00940     template <typename T1, typename T2>
00941     long count2DBins(array_t<T1> &vals1,
00942                      const double &begin1, const double &end1,
00943                      const double &stride1,
00944                      array_t<T2> &vals2,
00945                      const double &begin2, const double &end2,
00946                      const double &stride2,
00947                      std::vector<uint32_t> &counts) const;
00949     template <typename T1, typename T2, typename T3>
00950     long count3DBins(const array_t<T1> &vals1,
00951                      const double &begin1, const double &end1,
00952                      const double &stride1,
00953                      const array_t<T2> &vals2,
00954                      const double &begin2, const double &end2,
00955                      const double &stride2,
00956                      const array_t<T3> &vals3,
00957                      const double &begin3, const double &end3,
00958                      const double &stride3,
00959                      std::vector<uint32_t> &counts) const;
00961     template <typename T1, typename T2>
00962     long count2DWeights(array_t<T1> &vals1,
00963                         const double &begin1, const double &end1,
00964                         const double &stride1,
00965                         array_t<T2> &vals2,
00966                         const double &begin2, const double &end2,
00967                         const double &stride2,
00968                         array_t<double> &wts,
00969                         std::vector<double> &weights) const;
00971     template <typename T1, typename T2, typename T3>
00972     long count3DWeights(const array_t<T1> &vals1,
00973                         const double &begin1, const double &end1,
00974                         const double &stride1,
00975                         const array_t<T2> &vals2,
00976                         const double &begin2, const double &end2,
00977                         const double &stride2,
00978                         const array_t<T3> &vals3,
00979                         const double &begin3, const double &end3,
00980                         const double &stride3,
00981                         const array_t<double> &wts,
00982                         std::vector<double> &weights) const;
00983 
00985     template <typename T1>
00986     long fill1DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
00987                     const double &begin1, const double &end1,
00988                     const double &stride1,
00989                     std::vector<ibis::bitvector> &bins) const;
00991     template <typename T1>
00992     long fill1DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
00993                     const double &begin1, const double &end1,
00994                     const double &stride1,
00995                     std::vector<ibis::bitvector*> &bins) const;
00997     template <typename T1>
00998     long fill1DBinsWeighted(const ibis::bitvector &mask,
00999                             const array_t<T1> &vals1,
01000                             const double &begin1, const double &end1,
01001                             const double &stride1,
01002                             const array_t<double> &wts,
01003                             std::vector<double> &weights,
01004                             std::vector<ibis::bitvector*> &bins) const;
01006     template <typename T1, typename T2>
01007     long fill2DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
01008                     const double &begin1, const double &end1,
01009                     const double &stride1,
01010                     const array_t<T2> &vals2,
01011                     const double &begin2, const double &end2,
01012                     const double &stride2,
01013                     std::vector<ibis::bitvector> &bins) const;
01014     template <typename T1>
01015     long fill2DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
01016                      const double &begin1, const double &end1,
01017                      const double &stride1,
01018                      const ibis::column &col2,
01019                      const double &begin2, const double &end2,
01020                      const double &stride2,
01021                      std::vector<ibis::bitvector> &bins) const;
01023     template <typename T1, typename T2>
01024     long fill2DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
01025                     const double &begin1, const double &end1,
01026                     const double &stride1,
01027                     const array_t<T2> &vals2,
01028                     const double &begin2, const double &end2,
01029                     const double &stride2,
01030                     std::vector<ibis::bitvector*> &bins) const;
01031     template <typename T1>
01032     long fill2DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
01033                      const double &begin1, const double &end1,
01034                      const double &stride1,
01035                      const ibis::column &col2,
01036                      const double &begin2, const double &end2,
01037                      const double &stride2,
01038                      std::vector<ibis::bitvector*> &bins) const;
01040     template <typename T1, typename T2>
01041     long fill2DBinsWeighted(const ibis::bitvector &mask,
01042                             const array_t<T1> &vals1,
01043                             const double &begin1, const double &end1,
01044                             const double &stride1,
01045                             const array_t<T2> &vals2,
01046                             const double &begin2, const double &end2,
01047                             const double &stride2,
01048                             const array_t<double> &wts,
01049                             std::vector<double> &weights,
01050                             std::vector<ibis::bitvector*> &bins) const;
01051     template <typename T1>
01052     long fill2DBinsWeighted2(const ibis::bitvector &mask,
01053                              const array_t<T1> &vals1,
01054                              const double &begin1, const double &end1,
01055                              const double &stride1,
01056                              const ibis::column &col2,
01057                              const double &begin2, const double &end2,
01058                              const double &stride2,
01059                              const array_t<double> &wts,
01060                              std::vector<double> &weights,
01061                              std::vector<ibis::bitvector*> &bins) const;
01063     template <typename T1, typename T2, typename T3>
01064     long fill3DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
01065                     const double &begin1, const double &end1,
01066                     const double &stride1,
01067                     const array_t<T2> &vals2,
01068                     const double &begin2, const double &end2,
01069                     const double &stride2,
01070                     const array_t<T3> &vals3,
01071                     const double &begin3, const double &end3,
01072                     const double &stride3,
01073                     std::vector<ibis::bitvector> &bins) const;
01074     template <typename T1>
01075     long fill3DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
01076                      const double &begin1, const double &end1,
01077                      const double &stride1,
01078                      const ibis::column &col2,
01079                      const double &begin2, const double &end2,
01080                      const double &stride2,
01081                      const ibis::column &col3,
01082                      const double &begin3, const double &end3,
01083                      const double &stride3,
01084                      std::vector<bitvector> &bins) const;
01085     template <typename T1, typename T2>
01086     long fill3DBins3(const ibis::bitvector &mask, const array_t<T1> &vals1,
01087                      const double &begin1, const double &end1,
01088                      const double &stride1,
01089                      const array_t<T2> &vals2,
01090                      const double &begin2, const double &end2,
01091                      const double &stride2,
01092                      const ibis::column &col3,
01093                      const double &begin3, const double &end3,
01094                      const double &stride3,
01095                      std::vector<bitvector> &bins) const;
01096     template <typename T1, typename T2, typename T3>
01097     long fill3DBins(const ibis::bitvector &mask, const array_t<T1> &vals1,
01098                     const double &begin1, const double &end1,
01099                     const double &stride1,
01100                     const array_t<T2> &vals2,
01101                     const double &begin2, const double &end2,
01102                     const double &stride2,
01103                     const array_t<T3> &vals3,
01104                     const double &begin3, const double &end3,
01105                     const double &stride3,
01106                     std::vector<bitvector*> &bins) const;
01107     template <typename T1>
01108     long fill3DBins2(const ibis::bitvector &mask, const array_t<T1> &vals1,
01109                      const double &begin1, const double &end1,
01110                      const double &stride1,
01111                      const ibis::column &col2,
01112                      const double &begin2, const double &end2,
01113                      const double &stride2,
01114                      const ibis::column &col3,
01115                      const double &begin3, const double &end3,
01116                      const double &stride3,
01117                      std::vector<ibis::bitvector*> &bins) const;
01118     template <typename T1, typename T2>
01119     long fill3DBins3(const ibis::bitvector &mask, const array_t<T1> &vals1,
01120                      const double &begin1, const double &end1,
01121                      const double &stride1,
01122                      const array_t<T2> &vals2,
01123                      const double &begin2, const double &end2,
01124                      const double &stride2,
01125                      const ibis::column &col3,
01126                      const double &begin3, const double &end3,
01127                      const double &stride3,
01128                      std::vector<ibis::bitvector*> &bins) const;
01129     template <typename T1, typename T2, typename T3>
01130     long fill3DBinsWeighted(const ibis::bitvector &mask,
01131                             const array_t<T1> &vals1,
01132                             const double &begin1, const double &end1,
01133                             const double &stride1,
01134                             const array_t<T2> &vals2,
01135                             const double &begin2, const double &end2,
01136                             const double &stride2,
01137                             const array_t<T3> &vals3,
01138                             const double &begin3, const double &end3,
01139                             const double &stride3,
01140                             const array_t<double> &wts,
01141                             std::vector<double> &weights,
01142                             std::vector<bitvector*> &bins) const;
01143     template <typename T1>
01144     long fill3DBinsWeighted2(const ibis::bitvector &mask,
01145                              const array_t<T1> &vals1,
01146                              const double &begin1, const double &end1,
01147                              const double &stride1,
01148                              const ibis::column &col2,
01149                              const double &begin2, const double &end2,
01150                              const double &stride2,
01151                              const ibis::column &col3,
01152                              const double &begin3, const double &end3,
01153                              const double &stride3,
01154                              const array_t<double> &wts,
01155                              std::vector<double> &weights,
01156                              std::vector<ibis::bitvector*> &bins) const;
01157     template <typename T1, typename T2>
01158     long fill3DBinsWeighted3(const ibis::bitvector &mask,
01159                              const array_t<T1> &vals1,
01160                              const double &begin1, const double &end1,
01161                              const double &stride1,
01162                              const array_t<T2> &vals2,
01163                              const double &begin2, const double &end2,
01164                              const double &stride2,
01165                              const ibis::column &col3,
01166                              const double &begin3, const double &end3,
01167                              const double &stride3,
01168                              const array_t<double> &wts,
01169                              std::vector<double> &weights,
01170                              std::vector<ibis::bitvector*> &bins) const;
01171 
01173     long get1DBins_(const ibis::bitvector &mask, const ibis::column &col,
01174                     uint32_t nbin, std::vector<double> &bounds,
01175                     std::vector<ibis::bitvector> &bins, const char *mesg) const;
01176 
01178     long get1DDistribution(const ibis::column &col, uint32_t nbin,
01179                            std::vector<double> &bounds,
01180                            std::vector<uint32_t> &counts) const;
01182     long get2DDistributionU(const ibis::column &col1,
01183                             const ibis::column &col2,
01184                             uint32_t nb1, uint32_t nb2,
01185                             std::vector<double> &bounds1,
01186                             std::vector<double> &bounds2,
01187                             std::vector<uint32_t> &counts) const;
01189     long get2DDistributionA(const ibis::column &col1,
01190                             const ibis::column &col2,
01191                             uint32_t nb1, uint32_t nb2,
01192                             std::vector<double> &bounds1,
01193                             std::vector<double> &bounds2,
01194                             std::vector<uint32_t> &counts) const;
01196     long get2DDistributionI(const ibis::column &col1,
01197                             const ibis::column &col2,
01198                             uint32_t nb1, uint32_t nb2,
01199                             std::vector<double> &bounds1,
01200                             std::vector<double> &bounds2,
01201                             std::vector<uint32_t> &counts) const;
01202     long old2DDistribution(const char *constraints,
01203                            const char *name1, const char *name2,
01204                            uint32_t nb1, uint32_t nb2,
01205                            std::vector<double> &bounds1,
01206                            std::vector<double> &bounds2,
01207                            std::vector<uint32_t> &counts) const;
01209     int coarsenBins(const ibis::column &col, uint32_t nbin,
01210                     std::vector<double> &bnds,
01211                     std::vector<ibis::bitvector*> &btmp) const;
01213     long get3DDistributionA(const ibis::bitvector &mask,
01214                             const ibis::column &col1,
01215                             const ibis::column &col2,
01216                             const ibis::column &col3,
01217                             uint32_t nb1, uint32_t nb2, uint32_t nb3,
01218                             std::vector<double> &bounds1,
01219                             std::vector<double> &bounds2,
01220                             std::vector<double> &bounds3,
01221                             std::vector<uint32_t> &counts) const;
01222     template <typename E1>
01223     long get3DDistributionA1(const ibis::bitvector &mask,
01224                              const array_t<E1> &vals1,
01225                              const ibis::column &col2,
01226                              const ibis::column &col3,
01227                              uint32_t nb1, uint32_t nb2, uint32_t nb3,
01228                              std::vector<double> &bounds1,
01229                              std::vector<double> &bounds2,
01230                              std::vector<double> &bounds3,
01231                              std::vector<uint32_t> &counts) const;
01232     template <typename E1, typename E2>
01233     long get3DDistributionA2(const ibis::bitvector &mask,
01234                              const array_t<E1> &vals1,
01235                              const array_t<E2> &vals2,
01236                              const ibis::column &col3,
01237                              uint32_t nb1, uint32_t nb2, uint32_t nb3,
01238                              std::vector<double> &bounds1,
01239                              std::vector<double> &bounds2,
01240                              std::vector<double> &bounds3,
01241                              std::vector<uint32_t> &counts) const;
01242 
01243     template <typename E1, typename E2>
01244         static void mapValues(array_t<E1> &val1, array_t<E2> &val2,
01245                               uint32_t nb1, uint32_t nb2,
01246                               array_t<E1> &bnd1, array_t<E2> &bnd2,
01247                               std::vector<uint32_t> &cnts);
01248 
01249     template <typename T>
01250         static void mapValues(const array_t<T> &vals,
01251                               std::map<T, uint32_t> &hist);
01252 
01253     template <typename T>
01254         static void equalWeightBins(const array_t<T> &vals,
01255                                     uint32_t nbins, array_t<T> &bounds);
01256 
01257     template <typename T>
01258         static long adaptiveInts(const array_t<T> &vals, const T vmin,
01259                                  const T vmax, uint32_t nbins,
01260                                  std::vector<double> &bounds,
01261                                  std::vector<uint32_t> &counts);
01262 
01263     template <typename T>
01264         static long adaptiveFloats(const array_t<T> &vals, const T vmin,
01265                                    const T vmax, uint32_t nbins,
01266                                    std::vector<double> &bounds,
01267                                    std::vector<uint32_t> &counts);
01268 
01269     template <typename T1, typename T2>
01270         static long adaptive2DBins(const array_t<T1> &vals1,
01271                                    const array_t<T2> &vals2,
01272                                    uint32_t nb1, uint32_t nb2,
01273                                    std::vector<double> &bounds1,
01274                                    std::vector<double> &bounds2,
01275                                    std::vector<uint32_t> &counts);
01276 
01277     template <typename T1, typename T2, typename T3>
01278         static long adaptive3DBins(const array_t<T1> &vals1,
01279                                    const array_t<T2> &vals2,
01280                                    const array_t<T3> &vals3,
01281                                    uint32_t nb1, uint32_t nb2, uint32_t nb3,
01282                                    std::vector<double> &bounds1,
01283                                    std::vector<double> &bounds2,
01284                                    std::vector<double> &bounds3,
01285                                    std::vector<uint32_t> &counts);
01286 
01287     template <typename T> static long
01288         adaptiveIntsDetailed(const ibis::bitvector &mask,
01289                              const array_t<T> &vals,
01290                              const T vmin, const T vmax, uint32_t nbins,
01291                              std::vector<double> &bounds,
01292                              std::vector<ibis::bitvector> &detail);
01293 
01294     template <typename T> static long
01295         adaptiveFloatsDetailed(const ibis::bitvector &mask,
01296                                const array_t<T> &vals,
01297                                const T vmin, const T vmax, uint32_t nbins,
01298                                std::vector<double> &bounds,
01299                                std::vector<ibis::bitvector> &detail);
01300 
01301     void gatherSortKeys(ibis::table::stringList& names);
01302     void composeQueryString(std::string &str,
01303                             const ibis::column* col1, const ibis::column* col2,
01304                             const double &lower1, const double &upper1,
01305                             const double &lower2, const double &upper2) const;
01306     void buildQueryList(ibis::part::thrArg &lst,
01307                         unsigned nc, unsigned nq) const;
01308     void checkQueryList(const ibis::part::thrArg &lst) const;
01309     uint32_t recursiveQuery(const char* pref, const column* att,
01310                             double low, double high, long* nerr) const;
01311 
01312 private:
01313 
01314     /******************************************************************/
01315     // private member variables
01316     mutable pthread_mutex_t mutex;      
01317     mutable pthread_rwlock_t rwlock;    
01318 
01319     /******************************************************************/
01320     // private funcations
01321 
01322     void init(const char* prefix); 
01323 
01324     void gainReadAccess(const char* mesg) const;
01325     void releaseAccess(const char* mesg) const;
01326     void gainWriteAccess(const char* mesg) const;
01327 
01328     void   fillRIDs(const char* fn) const; 
01329     void   sortRIDs() const; 
01330     uint32_t searchSortedRIDs(const ibis::rid_t &rid) const;
01331     uint32_t searchRIDs(const ibis::rid_t &rid) const;
01332     void   searchSortedRIDs(const ibis::RIDSet&, ibis::bitvector&) const;
01333     void   searchRIDs(const ibis::RIDSet&, ibis::bitvector&) const;
01334 
01335     // functions to perform join operations.
01336     int64_t equiJoin(const ibis::deprecatedJoin &cmp,
01337                      const ibis::bitvector64 &trial,
01338                      ibis::bitvector64 &result) const;
01339     int64_t deprecatedJoin(const ibis::deprecatedJoin &cmp,
01340                       const ibis::bitvector64 &trial,
01341                       ibis::bitvector64 &result) const;
01342     int64_t compJoin(const ibis::deprecatedJoin &cmp,
01343                      const ibis::bitvector64 &trial,
01344                      ibis::bitvector64 &result) const;
01345 
01346     int64_t loopJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
01347                      const ibis::bitvector &mask,
01348                      ibis::bitvector64 &pairs) const;
01349     int64_t loopJoin(const std::vector<const ibis::deprecatedJoin*> &cmp,
01350                      const ibis::bitvector &mask) const;
01351     int64_t loopJoin(const ibis::deprecatedJoin &cmp,
01352                      const ibis::bitvector &mask,
01353                      ibis::bitvector64 &pairs) const;
01354     int64_t loopJoin(const ibis::deprecatedJoin &cmp,
01355                      const ibis::bitvector &mask) const;
01356     int64_t equiJoinLoop1(const ibis::deprecatedJoin &cmp,
01357                           const ibis::bitvector &mask,
01358                           ibis::bitvector64 &pairs) const;
01359     int64_t equiJoinLoop1(const ibis::deprecatedJoin &cmp,
01360                           const ibis::bitvector &mask) const;
01361     int64_t equiJoinLoop2(const ibis::deprecatedJoin &cmp,
01362                           const ibis::bitvector &mask,
01363                           ibis::bitvector64 &pairs) const;
01364     int64_t equiJoinLoop2(const ibis::deprecatedJoin &cmp,
01365                           const ibis::bitvector &mask) const;
01366     template <class type1, class type2>
01367     void deprecatedJoinLoop(const array_t<type1> &arr1,
01368                        const ibis::bitvector &msk1,
01369                        const array_t<type2> &arr2,
01370                        const ibis::bitvector &msk2,
01371                        const double delta,
01372                        ibis::bitvector64 &pairs) const;
01373     template <class type1, class type2>
01374     int64_t deprecatedJoinLoop(const array_t<type1> &arr1,
01375                           const ibis::bitvector &msk1,
01376                           const array_t<type2> &arr2,
01377                           const ibis::bitvector &msk2,
01378                           const double delta) const;
01379     int64_t deprecatedJoinLoop(const ibis::deprecatedJoin &cmp,
01380                           const ibis::bitvector &mask,
01381                           ibis::bitvector64 &pairs) const;
01382     int64_t deprecatedJoinLoop(const ibis::deprecatedJoin &cmp,
01383                           const ibis::bitvector &mask) const;
01384     int64_t compJoinLoop(const ibis::deprecatedJoin &cmp,
01385                          const ibis::bitvector &mask,
01386                          ibis::bitvector64 &pairs) const;
01387     int64_t compJoinLoop(const ibis::deprecatedJoin &cmp,
01388                          const ibis::bitvector &mask) const;
01389 
01390     part(const part&);
01391     const part &operator=(const part&);
01392 }; // class ibis::part
01393 
01394 namespace ibis {
01395     // Extends the name space ibis::util to contain three more functions to
01396     // deal with the reconstruction of partitions.
01397     namespace util {
01399         unsigned int FASTBIT_CXX_DLLSPEC
01400         gatherParts(ibis::partList &parts,
01401                     const char *adir, const char *bdir, bool ro=false);
01403         unsigned int FASTBIT_CXX_DLLSPEC
01404         gatherParts(ibis::partList &parts, const char *adir, bool ro=false);
01407         unsigned int FASTBIT_CXX_DLLSPEC
01408         gatherParts(ibis::partList &parts, const ibis::resource &res,
01409                     bool ro=false);
01410     } // namespace util
01411 } // namespace ibis
01412 
01416 struct FASTBIT_CXX_DLLSPEC ibis::part::info {
01417     const char* name;           
01418     const char* description;    
01419     const char* metaTags;       
01420     const uint64_t nrows;       
01421 
01422     std::vector<ibis::column::info*> cols;
01423 
01424     info(const char* na, const char* de, const uint64_t &nr,
01425          const ibis::part::columnList &co);
01426     info(const ibis::part &tbl);
01427     ~info();
01428 
01429 private:
01430     info();     // private default constructor, not implemented!
01431     info& operator=(const info&);
01432 }; // ibis::part::info
01433 
01435 class ibis::part::cleaner : public ibis::fileManager::cleaner {
01436 public:
01437     virtual void operator()() const;
01438     cleaner(const part* tbl) : thePart(tbl) {}
01439     virtual ~cleaner() {}
01440 
01441 private:
01442     const part* thePart;
01443 }; // ibis::part::cleaner
01444 
01449 class ibis::part::readLock {
01450 public:
01451     readLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
01452         tbl->gainReadAccess(m);
01453     }
01454     ~readLock() {thePart->releaseAccess(mesg);}
01455 
01456 private:
01457     const part* thePart;
01458     const char* mesg;
01459 
01460     readLock() {}; // no default constructor
01461     readLock(const readLock&) {}; // can not copy
01462     const readLock &operator=(const readLock&);
01463 }; // ibis::part::readLock
01464 
01466 class ibis::part::writeLock {
01467 public:
01468     writeLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
01469         tbl->gainWriteAccess(m);
01470     }
01471     ~writeLock() {thePart->releaseAccess(mesg);}
01472 
01473 private:
01474     const part* thePart;
01475     const char* mesg;
01476 
01477     writeLock() {}; // no default constructor
01478     writeLock(const writeLock&) {}; // can not copy
01479     const writeLock &operator=(const writeLock&);
01480 }; // ibis::part::writeLock
01481 
01485 class ibis::part::softWriteLock {
01486 public:
01488     softWriteLock(const part* tbl, const char* m);
01490     ~softWriteLock() {if (lckd==0) thePart->releaseAccess(mesg);}
01493     bool isLocked() const {return (lckd==0);}
01494 
01495 private:
01496     const part* thePart;
01497     const char* mesg;
01498     const int lckd;
01499 
01500     softWriteLock() : thePart(0), mesg(0), lckd(0) {};
01501     softWriteLock(const softWriteLock &rhs)
01502         : thePart(rhs.thePart), mesg(rhs.mesg), lckd(0) {};
01503     const softWriteLock &operator=(const softWriteLock&);
01504 }; // ibis::part::softWriteLock
01505 
01508 class ibis::part::mutexLock {
01509 public:
01510     mutexLock(const part* tbl, const char* m) : thePart(tbl), mesg(m) {
01511         if (ibis::gVerbose > 9)
01512             tbl->logMessage("gainExclusiveAccess",
01513                             "pthread_mutex_lock for %s", m);
01514         int ierr = pthread_mutex_lock(&(tbl->mutex));
01515         if (0 != ierr)
01516             tbl->logWarning("gainExclusiveAccess", "pthread_mutex_lock for %s "
01517                             "returned %d (%s)", m, ierr, strerror(ierr));
01518     }
01519     ~mutexLock() {
01520         if (ibis::gVerbose > 9)
01521             thePart->logMessage("releaseExclusiveAccess",
01522                                 "pthread_mutex_unlock for %s", mesg);
01523         int ierr = pthread_mutex_unlock(&(thePart->mutex));
01524         if (0 != ierr)
01525             thePart->logWarning("releaseExclusiveAccess",
01526                                 "pthread_mutex_unlock for %s returned %d (%s)",
01527                                 mesg, ierr, strerror(ierr));
01528     }
01529 
01530 private:
01531     const part* thePart;
01532     const char* mesg;
01533 
01534     mutexLock() {}; // no default constructor    
01535     mutexLock(const mutexLock&) {}; // can not copy      
01536     const mutexLock &operator=(const mutexLock&);        
01537 }; // ibis::part::mutexLock
01538 
01542 class ibis::part::barrel : public ibis::math::barrel {
01543 public:
01544     barrel(const ibis::part *t=0) : _tbl(t), position(0) {};
01545     virtual ~barrel() {close();} 
01546 
01547     virtual long open(const ibis::part *t=0); 
01548     virtual long close(); 
01549     virtual long read(); 
01550 
01551 
01552     virtual long seek(uint32_t pos);
01553     uint32_t tell() const {return position;}
01554 
01555     void getNullMask(ibis::bitvector &mask) const;
01556     const ibis::column* getColumn(uint32_t i) const {return cols[i];}
01557 
01558 protected:
01559     const ibis::part *_tbl;
01560     uint32_t position;
01561     std::vector<const ibis::column*> cols;
01562     std::vector<ibis::fileManager::storage*> stores;
01563     std::vector<int> fdes;
01564 }; // ibis::part::barrel
01565 
01568 class ibis::part::vault : public ibis::part::barrel {
01569 public:
01570     vault(const ibis::roster &r);
01571     virtual ~vault() {close();}
01572 
01573     virtual long open(const ibis::part *t=0); 
01574     virtual long read(); 
01575     virtual long seek(uint32_t pos); 
01576 
01577     long seek(double val);
01580     uint32_t tellReal() const;
01581 
01582 private:
01583     const ibis::roster &_roster;
01584 
01585     template <class T>
01586     uint32_t seekValue(int fd, const T &val) const;
01587     template <class T>
01588     uint32_t seekValue(const array_t<T>&arr, const T &val) const;
01589 
01590     vault();
01591     vault& operator=(const vault&);
01592 }; // ibis::part::vault
01593 /*
01594 barrel
01595 
01596 Vault \Vault\ (v[add]lt; see Note, below), n. [OE. voute, OF. voute, volte,
01597 F. vo[^u]te, LL. volta, for voluta, volutio, fr. L. volvere, volutum, to
01598 roll, to turn about. See Voluble, and cf. Vault a leap, Volt a turn,
01599 Volute.] 1. (Arch.) An arched structure of masonry, forming a ceiling or
01600 canopy.
01601 
01602 The long-drawn aisle and fretted vault. --Gray.
01603 
01604 2. An arched apartment; especially, a subterranean room, use for storing
01605    articles, for a prison, for interment, or the like; a cell; a
01606    cellar. ``Charnel vaults.'' --Milton.
01607 
01608 The silent vaults of death. --Sandys.
01609 
01610 To banish rats that haunt our vault. --Swift.
01611 
01612 3. The canopy of heaven; the sky.
01613 
01614 That heaven's vault should crack. --Shak.
01615 
01616 4. [F. volte, It. volta, originally, a turn, and the same word as volta an
01617    arch. See the Etymology above.] A leap or bound. Specifically: (a)
01618    (Man.) The bound or leap of a horse; a curvet. (b) A leap by aid of the
01619    hands, or of a pole, springboard, or the like.
01620 
01621 Note: The l in this word was formerly often suppressed in pronunciation.
01622 
01623 Barrel, Cradle, Cylindrical, or Wagon, vault (Arch.), a kind of vault
01624 having two parallel abutments, and the same section or profile at all
01625 points. It may be rampant, as over a staircase (see Rampant vault, under
01626 Rampant), or curved in plan, as around the apse of a church.
01627 
01628 Coved vault. (Arch.) See under 1st Cove, v. t.
01629 
01630 Groined vault (Arch.), a vault having groins, that is, one in which
01631 different cylindrical surfaces intersect one another, as distinguished from
01632 a barrel, or wagon, vault.
01633 
01634 Rampant vault. (Arch.) See under Rampant.
01635 
01636 Ribbed vault (Arch.), a vault differing from others in having solid ribs
01637 which bear the weight of the vaulted surface. True Gothic vaults are of
01638 this character.
01639 
01640 Vault light, a partly glazed plate inserted in a pavement or ceiling to
01641 admit light to a vault below.
01642 
01643 Source: Webster's Revised Unabridged Dictionary, (c) 1996, 1998 MICRA, Inc.
01644  */
01645 
01646 namespace ibis {
01647     // Explicit template specialization for member function
01648     // ibis::part::equalWeightBins
01649     template <> void
01650     part::equalWeightBins(const array_t<float> &vals,
01651                           uint32_t nbins, array_t<float> &bounds);
01652     template <> void
01653     part::equalWeightBins(const array_t<double> &vals,
01654                           uint32_t nbins, array_t<double> &bounds);
01655 
01656     template <> long
01657     part::doScan(const array_t<float> &,
01658                  const ibis::qContinuousRange &,
01659                  const ibis::bitvector &, ibis::bitvector &);
01660     template <> long
01661     part::doScan(const array_t<double> &,
01662                  const ibis::qContinuousRange &,
01663                  const ibis::bitvector &, ibis::bitvector &);
01664 
01665     template <> long
01666     part::doScan(const array_t<float> &,
01667                  const ibis::qContinuousRange &,
01668                  const ibis::bitvector &, ibis::array_t<float> &);
01669     template <> long
01670     part::doScan(const array_t<double> &,
01671                  const ibis::qContinuousRange &,
01672                  const ibis::bitvector &, ibis::array_t<double> &);
01673 
01674     template <> long
01675     part::doScan(const array_t<float> &,
01676                  const ibis::qContinuousRange &,
01677                  const ibis::bitvector &, ibis::array_t<float> &,
01678                  ibis::bitvector &);
01679     template <> long
01680     part::doScan(const array_t<double> &,
01681                  const ibis::qContinuousRange &,
01682                  const ibis::bitvector &, ibis::array_t<double> &,
01683                  ibis::bitvector &);
01684 
01685     template <> long
01686     part::doCount<float>(const ibis::qRange&) const;
01687     template <> long
01688     part::doCount<double>(const ibis::qRange&) const;
01689 }
01690 
01692 inline ibis::part::info* ibis::part::getInfo() const {
01693     return new info(*this);
01694 }
01695 
01700 inline ibis::column* ibis::part::getColumn(uint32_t ind) const {
01701     if (ind < columns.size()) {
01702         if (colorder.size() == columns.size()) {
01703             return const_cast<ibis::column*>(colorder[ind]);
01704         }
01705         else {
01706             ibis::part::columnList::const_iterator it = columns.begin();
01707             for (; ind > 0; -- ind) ++ it;
01708             return (*it).second;
01709         }
01710     }
01711     else {
01712         return 0;
01713     }
01714 } // ibis::part::getColumn
01715 
01716 inline const char* ibis::part::getMetaTag(const char* name) const {
01717     ibis::resource::vList::const_iterator it = metaList.find(name);
01718     if (it != metaList.end())
01719         return (*it).second;
01720     else
01721         return static_cast<const char*>(0);
01722 } // ibis::part::getMetaTag
01723 
01724 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &cmp,
01725                                         const ibis::bitvector &mask,
01726                                         ibis::bitvector64 &pairs) const {
01727     return loopJoin(cmp, mask, pairs);
01728 } // ibis::part::evaluateJoin
01729 
01730 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &,
01731                                         const ibis::bitvector &,
01732                                         const char *) const {
01733     logWarning("evaluate", "not implemented yet");
01734     return -1;
01735 } // ibis::part::evaluateJoin
01736 
01737 inline int64_t ibis::part::evaluateJoin(const ibis::deprecatedJoin &cmp,
01738                                         const ibis::bitvector &mask) const {
01739     return loopJoin(cmp, mask);
01740 } // ibis::part::evaluateJoin
01741 
01742 inline int64_t ibis::part::evaluateJoin
01743 (const std::vector<const ibis::deprecatedJoin*> &cmp,
01744  const ibis::bitvector &mask, ibis::bitvector64 &pairs) const {
01745     return loopJoin(cmp, mask, pairs);
01746 } // ibis::part::evaluateJoin
01747 
01749 inline void ibis::part::cleaner::operator()() const {
01750     const uint32_t sz = ibis::fileManager::bytesInUse();
01751     thePart->unloadIndexes();
01752     if (sz == ibis::fileManager::bytesInUse() &&
01753         thePart->getStateNoLocking() == ibis::part::STABLE_STATE) {
01754         thePart->freeRIDs();
01755     }
01756 } // ibis::part::cleaner::operator
01757 #endif // IBIS_PART_H

Make It A Bit Faster
Contact us
Disclaimers
FastBit source code
FastBit mailing list archive