00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029 #include "index.h"
00030
00031 #include "kmkernel.h"
00032 #include "kmfoldermgr.h"
00033 #include "kmmsgdict.h"
00034 #include "kmfolder.h"
00035 #include "kmsearchpattern.h"
00036 #include "kmfoldersearch.h"
00037
00038 #include <kdebug.h>
00039 #include <kapplication.h>
00040 #include <qfile.h>
00041 #include <qtimer.h>
00042 #include <qvaluestack.h>
00043 #include <qptrlist.h>
00044 #include <qfileinfo.h>
00045 #ifdef HAVE_INDEXLIB
00046 #include <indexlib/create.h>
00047 #endif
00048
00049 #include <sys/types.h>
00050 #include <sys/stat.h>
00051
00052 #include <iostream>
00053 #include <algorithm>
00054 #include <cstdlib>
00055
00056 namespace {
00057 const unsigned int MaintenanceLimit = 1000;
00058 const char* const folderIndexDisabledKey = "fulltextIndexDisabled";
00059 }
00060
00061 static
00062 QValueList<int> vectorToQValueList( const std::vector<Q_UINT32>& input ) {
00063 QValueList<int> res;
00064 std::copy( input.begin(), input.end(), std::back_inserter( res ) );
00065 return res;
00066 }
00067
00068
00069 static
00070 std::vector<Q_UINT32> QValueListToVector( const QValueList<int>& input ) {
00071 std::vector<Q_UINT32> res;
00072
00073 for ( QValueList<int>::const_iterator first = input.begin(), past = input.end(); first != past; ++first ) {
00074 res.push_back( *first );
00075 }
00076 return res;
00077 }
00078
00079 KMMsgIndex::KMMsgIndex( QObject* parent ):
00080 QObject( parent, "index" ),
00081 mState( s_idle ),
00082 #ifdef HAVE_INDEXLIB
00083 mLockFile( std::string( static_cast<const char*>( QFile::encodeName( defaultPath() ) + "/lock" ) ) ),
00084 mIndex( 0 ),
00085 #endif
00086 mIndexPath( QFile::encodeName( defaultPath() ) ),
00087 mTimer( new QTimer( this ) ),
00088
00089
00090 mSlowDown( false ) {
00091 kdDebug( 5006 ) << "KMMsgIndex::KMMsgIndex()" << endl;
00092
00093 connect( kmkernel->folderMgr(), SIGNAL( msgRemoved( KMFolder*, Q_UINT32 ) ), SLOT( slotRemoveMessage( KMFolder*, Q_UINT32 ) ) );
00094 connect( kmkernel->folderMgr(), SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ), SLOT( slotAddMessage( KMFolder*, Q_UINT32 ) ) );
00095 connect( kmkernel->dimapFolderMgr(), SIGNAL( msgRemoved( KMFolder*, Q_UINT32 ) ), SLOT( slotRemoveMessage( KMFolder*, Q_UINT32 ) ) );
00096 connect( kmkernel->dimapFolderMgr(), SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ), SLOT( slotAddMessage( KMFolder*, Q_UINT32 ) ) );
00097
00098 connect( mTimer, SIGNAL( timeout() ), SLOT( act() ) );
00099
00100
00101 #ifdef HAVE_INDEXLIB
00102 KConfigGroup cfg( KMKernel::config(), "text-index" );
00103 if ( !cfg.readBoolEntry( "enabled", false ) ) {
00104 indexlib::remove( mIndexPath );
00105 mLockFile.force_unlock();
00106 mState = s_disabled;
00107 return;
00108 }
00109 if ( !mLockFile.trylock() ) {
00110 indexlib::remove( mIndexPath );
00111
00112 mLockFile.force_unlock();
00113 mLockFile.trylock();
00114 } else {
00115 mIndex = indexlib::open( mIndexPath, indexlib::open_flags::fail_if_nonexistant ).release();
00116 }
00117 if ( !mIndex ) {
00118 QTimer::singleShot( 8000, this, SLOT( create() ) );
00119 mState = s_willcreate;
00120 } else {
00121 if ( cfg.readBoolEntry( "creating" ) ) {
00122 QTimer::singleShot( 8000, this, SLOT( continueCreation() ) );
00123 mState = s_creating;
00124 } else {
00125 mPendingMsgs = QValueListToVector( cfg.readIntListEntry( "pending" ) );
00126 mRemovedMsgs = QValueListToVector( cfg.readIntListEntry( "removed" ) );
00127 }
00128 }
00129 mIndex = 0;
00130 #else
00131 mState = s_error;
00132 #endif
00133
00134 }
00135
00136
00137 KMMsgIndex::~KMMsgIndex() {
00138 kdDebug( 5006 ) << "KMMsgIndex::~KMMsgIndex()" << endl;
00139 #ifdef HAVE_INDEXLIB
00140 KConfigGroup cfg( KMKernel::config(), "text-index" );
00141 cfg.writeEntry( "creating", mState == s_creating );
00142 QValueList<int> pendingMsg;
00143 if ( mState == s_processing ) {
00144 Q_ASSERT( mAddedMsgs.empty() );
00145 pendingMsg = vectorToQValueList( mPendingMsgs );
00146 }
00147 cfg.writeEntry( "pending", pendingMsg );
00148 cfg.writeEntry( "removed", vectorToQValueList( mRemovedMsgs ) );
00149 delete mIndex;
00150 #endif
00151 }
00152
00153 bool KMMsgIndex::isIndexable( KMFolder* folder ) const {
00154 if ( !folder || !folder->parent() ) return false;
00155 const KMFolderMgr* manager = folder->parent()->manager();
00156 return manager == kmkernel->folderMgr() || manager == kmkernel->dimapFolderMgr();
00157 }
00158
00159 bool KMMsgIndex::isIndexed( KMFolder* folder ) const {
00160 if ( !isIndexable( folder ) ) return false;
00161 KConfig* config = KMKernel::config();
00162 KConfigGroupSaver saver( config, "Folder-" + folder->idString() );
00163 return !config->readBoolEntry( folderIndexDisabledKey, false );
00164 }
00165
00166 void KMMsgIndex::setEnabled( bool e ) {
00167 kdDebug( 5006 ) << "KMMsgIndex::setEnabled( " << e << " )" << endl;
00168 KConfig* config = KMKernel::config();
00169 KConfigGroupSaver saver( config, "text-index" );
00170 if ( config->readBoolEntry( "enabled", !e ) == e ) return;
00171 config->writeEntry( "enabled", e );
00172 if ( e ) {
00173 switch ( mState ) {
00174 case s_idle:
00175 case s_willcreate:
00176 case s_creating:
00177 case s_processing:
00178
00179 return;
00180 case s_error:
00181
00182 return;
00183 case s_disabled:
00184 QTimer::singleShot( 8000, this, SLOT( create() ) );
00185 mState = s_willcreate;
00186 }
00187 } else {
00188 clear();
00189 }
00190 }
00191
00192 void KMMsgIndex::setIndexingEnabled( KMFolder* folder, bool e ) {
00193 KConfig* config = KMKernel::config();
00194 KConfigGroupSaver saver( config, "Folder-" + folder->idString() );
00195 if ( config->readBoolEntry( folderIndexDisabledKey, e ) == e ) return;
00196 config->writeEntry( folderIndexDisabledKey, e );
00197
00198 if ( e ) {
00199 switch ( mState ) {
00200 case s_idle:
00201 case s_creating:
00202 case s_processing:
00203 mPendingFolders.push_back( folder );
00204 scheduleAction();
00205 break;
00206 case s_willcreate:
00207
00208 break;
00209 case s_error:
00210 case s_disabled:
00211
00212 break;
00213 }
00214
00215 } else {
00216 switch ( mState ) {
00217 case s_willcreate:
00218
00219 break;
00220 case s_creating:
00221 if ( std::find( mPendingFolders.begin(), mPendingFolders.end(), folder ) != mPendingFolders.end() ) {
00222
00223 mPendingFolders.erase( std::find( mPendingFolders.begin(), mPendingFolders.end(), folder ) );
00224 break;
00225 }
00226
00227 case s_idle:
00228 case s_processing:
00229
00230 case s_error:
00231 case s_disabled:
00232
00233 break;
00234 }
00235 }
00236 }
00237
00238 void KMMsgIndex::clear() {
00239 kdDebug( 5006 ) << "KMMsgIndex::clear()" << endl;
00240 #ifdef HAVE_INDEXLIB
00241 delete mIndex;
00242 mLockFile.force_unlock();
00243 mIndex = 0;
00244 indexlib::remove( mIndexPath );
00245 mPendingMsgs.clear();
00246 mPendingFolders.clear();
00247 mMaintenanceCount = 0;
00248 mAddedMsgs.clear();
00249 mRemovedMsgs.clear();
00250 mExisting.clear();
00251 mState = s_disabled;
00252 for ( std::set<KMFolder*>::const_iterator first = mOpenedFolders.begin(), past = mOpenedFolders.end(); first != past; ++first ) {
00253 ( *first )->close();
00254 }
00255 mOpenedFolders.clear();
00256 for ( std::vector<Search*>::const_iterator first = mSearches.begin(), past = mSearches.end(); first != past; ++first ) {
00257 delete *first;
00258 }
00259 mSearches.clear();
00260 mTimer->stop();
00261 #endif
00262 }
00263
00264 void KMMsgIndex::maintenance() {
00265 #ifdef HAVE_INDEXLIB
00266 if ( mState != s_idle || kapp->hasPendingEvents() ) {
00267 QTimer::singleShot( 8000, this, SLOT( maintenance() ) );
00268 return;
00269 }
00270 mIndex->maintenance();
00271 #endif
00272 }
00273
00274 int KMMsgIndex::addMessage( Q_UINT32 serNum ) {
00275 kdDebug( 5006 ) << "KMMsgIndex::addMessage( " << serNum << " )" << endl;
00276 if ( mState == s_error ) return 0;
00277 #ifdef HAVE_INDEXLIB
00278 assert( mIndex );
00279 if ( !mExisting.empty() && std::binary_search( mExisting.begin(), mExisting.end(), serNum ) ) return 0;
00280
00281 int idx = -1;
00282 KMFolder* folder = 0;
00283 KMMsgDict::instance()->getLocation( serNum, &folder, &idx );
00284 if ( !folder || idx == -1 ) return -1;
00285 if ( !mOpenedFolders.count( folder ) ) {
00286 mOpenedFolders.insert( folder );
00287 folder->open();
00288 }
00289 KMMessage* msg = folder->getMsg( idx );
00290
00291
00292
00293
00294 QString body = msg->asPlainText( false, false );
00295 if ( !body.isEmpty() && static_cast<const char*>( body.latin1() ) ) {
00296 mIndex->add( body.latin1(), QString::number( serNum ).latin1() );
00297 } else {
00298 kdDebug( 5006 ) << "Funny, no body" << endl;
00299 }
00300 folder->unGetMsg( idx );
00301 #endif
00302 return 0;
00303 }
00304
00305 void KMMsgIndex::act() {
00306 kdDebug( 5006 ) << "KMMsgIndex::act()" << endl;
00307 if ( kapp->hasPendingEvents() ) {
00308
00309 mTimer->start( 500 );
00310 mSlowDown = true;
00311 return;
00312 }
00313 if ( mSlowDown ) {
00314 mSlowDown = false;
00315 mTimer->start( 0 );
00316 }
00317 if ( !mPendingMsgs.empty() ) {
00318 addMessage( mPendingMsgs.back() );
00319 mPendingMsgs.pop_back();
00320 return;
00321 }
00322 if ( !mPendingFolders.empty() ) {
00323 KMFolder *f = mPendingFolders.back();
00324 mPendingFolders.pop_back();
00325 if ( !mOpenedFolders.count( f ) ) {
00326 mOpenedFolders.insert( f );
00327 f->open();
00328 }
00329 const KMMsgDict* dict = KMMsgDict::instance();
00330 KConfig* config = KMKernel::config();
00331 KConfigGroupSaver saver( config, "Folder-" + f->idString() );
00332 if ( config->readBoolEntry( folderIndexDisabledKey, true ) ) {
00333 for ( int i = 0; i < f->count(); ++i ) {
00334 mPendingMsgs.push_back( dict->getMsgSerNum( f, i ) );
00335 }
00336 }
00337 return;
00338 }
00339 if ( !mAddedMsgs.empty() ) {
00340 std::swap( mAddedMsgs, mPendingMsgs );
00341 mState = s_processing;
00342 return;
00343 }
00344 for ( std::set<KMFolder*>::const_iterator first = mOpenedFolders.begin(), past = mOpenedFolders.end();
00345 first != past;
00346 ++first ) {
00347 ( *first )->close();
00348 }
00349 mOpenedFolders.clear();
00350 mState = s_idle;
00351 mTimer->stop();
00352 }
00353
00354 void KMMsgIndex::continueCreation() {
00355 kdDebug( 5006 ) << "KMMsgIndex::continueCreation()" << endl;
00356 #ifdef HAVE_INDEXLIB
00357 create();
00358 unsigned count = mIndex->ndocs();
00359 mExisting.clear();
00360 mExisting.reserve( count );
00361 for ( unsigned i = 0; i != count; ++i ) {
00362 mExisting.push_back( std::atoi( mIndex->lookup_docname( i ).c_str() ) );
00363 }
00364 std::sort( mExisting.begin(), mExisting.end() );
00365 #endif
00366 }
00367
00368 void KMMsgIndex::create() {
00369 kdDebug( 5006 ) << "KMMsgIndex::create()" << endl;
00370
00371 #ifdef HAVE_INDEXLIB
00372 if ( !QFileInfo( mIndexPath ).exists() ) {
00373 ::mkdir( mIndexPath, S_IRWXU );
00374 }
00375 mState = s_creating;
00376 if ( !mIndex ) mIndex = indexlib::create( mIndexPath ).release();
00377 if ( !mIndex ) {
00378 kdDebug( 5006 ) << "Error creating index" << endl;
00379 mState = s_error;
00380 return;
00381 }
00382 QValueStack<KMFolderDir*> folders;
00383 folders.push(&(kmkernel->folderMgr()->dir()));
00384 folders.push(&(kmkernel->dimapFolderMgr()->dir()));
00385 while ( !folders.empty() ) {
00386 KMFolderDir *dir = folders.pop();
00387 for(KMFolderNode *child = dir->first(); child; child = dir->next()) {
00388 if ( child->isDir() )
00389 folders.push((KMFolderDir*)child);
00390 else
00391 mPendingFolders.push_back( (KMFolder*)child );
00392 }
00393 }
00394 mTimer->start( 4000 );
00395 mSlowDown = true;
00396 #endif
00397 }
00398
00399 bool KMMsgIndex::startQuery( KMSearch* s ) {
00400 kdDebug( 5006 ) << "KMMsgIndex::startQuery( . )" << endl;
00401 if ( mState != s_idle ) return false;
00402 if ( !isIndexed( s->root() ) || !canHandleQuery( s->searchPattern() ) ) return false;
00403
00404 kdDebug( 5006 ) << "KMMsgIndex::startQuery( . ) starting query" << endl;
00405 Search* search = new Search( s );
00406 connect( search, SIGNAL( finished( bool ) ), s, SIGNAL( finished( bool ) ) );
00407 connect( search, SIGNAL( finished( bool ) ), s, SLOT( indexFinished() ) );
00408 connect( search, SIGNAL( destroyed( QObject* ) ), SLOT( removeSearch( QObject* ) ) );
00409 connect( search, SIGNAL( found( Q_UINT32 ) ), s, SIGNAL( found( Q_UINT32 ) ) );
00410 mSearches.push_back( search );
00411 return true;
00412 }
00413
00414
00415
00416
00417
00418
00419
00420
00421
00422
00423
00424
00425
00426
00427
00428
00429
00430
00431
00432
00433
00434 void KMMsgIndex::removeSearch( QObject* destroyed ) {
00435 mSearches.erase( std::find( mSearches.begin(), mSearches.end(), destroyed ) );
00436 }
00437
00438
00439 bool KMMsgIndex::stopQuery( KMSearch* s ) {
00440 kdDebug( 5006 ) << "KMMsgIndex::stopQuery( . )" << endl;
00441 for ( std::vector<Search*>::iterator iter = mSearches.begin(), past = mSearches.end(); iter != past; ++iter ) {
00442 if ( ( *iter )->search() == s ) {
00443 delete *iter;
00444 mSearches.erase( iter );
00445 return true;
00446 }
00447 }
00448 return false;
00449 }
00450
00451 std::vector<Q_UINT32> KMMsgIndex::simpleSearch( QString s, bool* ok ) const {
00452 kdDebug( 5006 ) << "KMMsgIndex::simpleSearch( -" << s.latin1() << "- )" << endl;
00453 if ( mState == s_error || mState == s_disabled ) {
00454 if ( ok ) *ok = false;
00455 return std::vector<Q_UINT32>();
00456 }
00457 std::vector<Q_UINT32> res;
00458 #ifdef HAVE_INDEXLIB
00459 assert( mIndex );
00460 std::vector<unsigned> residx = mIndex->search( s.latin1() )->list();
00461 res.reserve( residx.size() );
00462 for ( std::vector<unsigned>::const_iterator first = residx.begin(), past = residx.end();first != past; ++first ) {
00463 res.push_back( std::atoi( mIndex->lookup_docname( *first ).c_str() ) );
00464 }
00465 if ( ok ) *ok = true;
00466 #endif
00467 return res;
00468 }
00469
00470 bool KMMsgIndex::canHandleQuery( const KMSearchPattern* pat ) const {
00471 kdDebug( 5006 ) << "KMMsgIndex::canHandleQuery( . )" << endl;
00472 if ( !pat ) return false;
00473 QPtrListIterator<KMSearchRule> it( *pat );
00474 KMSearchRule* rule;
00475 while ( (rule = it.current()) != 0 ) {
00476 ++it;
00477 if ( !rule->field().isEmpty() && !rule->contents().isEmpty() &&
00478 rule->function() == KMSearchRule::FuncContains &&
00479 rule->field() == "<body>" ) return true;
00480 }
00481 return false;
00482 }
00483
00484 void KMMsgIndex::slotAddMessage( KMFolder* folder, Q_UINT32 serNum ) {
00485 kdDebug( 5006 ) << "KMMsgIndex::slotAddMessage( . , " << serNum << " )" << endl;
00486 if ( mState == s_error || mState == s_disabled ) return;
00487
00488 if ( mState == s_creating ) mAddedMsgs.push_back( serNum );
00489 else mPendingMsgs.push_back( serNum );
00490
00491 if ( mState == s_idle ) mState = s_processing;
00492 scheduleAction();
00493 }
00494
00495 void KMMsgIndex::slotRemoveMessage( KMFolder* folder, Q_UINT32 serNum ) {
00496 kdDebug( 5006 ) << "KMMsgIndex::slotRemoveMessage( . , " << serNum << " )" << endl;
00497 if ( mState == s_error || mState == s_disabled ) return;
00498
00499 if ( mState == s_idle ) mState = s_processing;
00500 mRemovedMsgs.push_back( serNum );
00501 scheduleAction();
00502 }
00503
00504 void KMMsgIndex::scheduleAction() {
00505 #ifdef HAVE_INDEXLIB
00506 if ( mState == s_willcreate || !mIndex ) return;
00507 if ( !mSlowDown ) mTimer->start( 0 );
00508 #endif
00509 }
00510
00511 void KMMsgIndex::removeMessage( Q_UINT32 serNum ) {
00512 kdDebug( 5006 ) << "KMMsgIndex::removeMessage( " << serNum << " )" << endl;
00513 if ( mState == s_error || mState == s_disabled ) return;
00514
00515 #ifdef HAVE_INDEXLIB
00516 mIndex->remove_doc( QString::number( serNum ).latin1() );
00517 ++mMaintenanceCount;
00518 if ( mMaintenanceCount > MaintenanceLimit && mRemovedMsgs.empty() ) {
00519 QTimer::singleShot( 100, this, SLOT( maintenance() ) );
00520 }
00521 #endif
00522 }
00523
00524 QString KMMsgIndex::defaultPath() {
00525 return KMKernel::localDataPath() + "text-index";
00526 }
00527
00528 bool KMMsgIndex::creating() const {
00529 return !mPendingMsgs.empty() || !mPendingFolders.empty();
00530 }
00531
00532 KMMsgIndex::Search::Search( KMSearch* s ):
00533 mSearch( s ),
00534 mTimer( new QTimer( this ) ),
00535 mResidual( new KMSearchPattern ),
00536 mState( s_starting ) {
00537 connect( mTimer, SIGNAL( timeout() ), SLOT( act() ) );
00538 mTimer->start( 0 );
00539 }
00540
00541 KMMsgIndex::Search::~Search() {
00542 delete mTimer;
00543 }
00544
00545 void KMMsgIndex::Search::act() {
00546 switch ( mState ) {
00547 case s_starting: {
00548 KMSearchPattern* pat = mSearch->searchPattern();
00549 QString terms;
00550 for ( KMSearchRule* rule = pat->first(); rule; rule = pat->next() ) {
00551 Q_ASSERT( rule->function() == KMSearchRule::FuncContains );
00552 terms += QString::fromLatin1( " %1 " ).arg( rule->contents() );
00553 }
00554
00555 mValues = kmkernel->msgIndex()->simpleSearch( terms, 0 );
00556 break;
00557 }
00558 case s_emitstopped:
00559 mTimer->start( 0 );
00560 mState = s_emitting;
00561
00562 case s_emitting:
00563 if ( kapp->hasPendingEvents() ) {
00564
00565 mTimer->start( 250 );
00566 mState = s_emitstopped;
00567 return;
00568 }
00569 for ( int i = 0; i != 16 && !mValues.empty(); ++i ) {
00570 KMFolder* folder;
00571 int index;
00572 KMMsgDict::instance()->getLocation( mValues.back(), &folder, &index );
00573 if ( folder &&
00574 mSearch->inScope( folder ) &&
00575 ( !mResidual || mResidual->matches( mValues.back() ) ) ) {
00576
00577 emit found( mValues.back() );
00578 }
00579 mValues.pop_back();
00580 }
00581 if ( mValues.empty() ) {
00582 emit finished( true );
00583 mState = s_done;
00584 mTimer->stop();
00585 delete this;
00586 }
00587 break;
00588 default:
00589 Q_ASSERT( 0 );
00590 }
00591 }
00592 #include "index.moc"
00593