KDevelop API Documentation

htdigindex.cpp

Go to the documentation of this file.
00001 /***************************************************************************
00002  *   Copyright (C) 1999-2001 by Matthias Hoelzer-Kluepfel                  *
00003  *   hoelzer@kde.org                                                       *
00004  *   Copyright (C) 2001 by Bernd Gehrmann                                  *
00005  *   bernd@kdevelop.org                                                    *
00006  *                                                                         *
00007  *   This program is free software; you can redistribute it and/or modify  *
00008  *   it under the terms of the GNU General Public License as published by  *
00009  *   the Free Software Foundation; either version 2 of the License, or     *
00010  *   (at your option) any later version.                                   *
00011  *                                                                         *
00012  ***************************************************************************/
00013 
00014 #include "htdigindex.h"
00015 
00016 #include <qapplication.h>
00017 #include <qdir.h>
00018 #include <qfile.h>
00019 #include <qlayout.h>
00020 #include <qtextstream.h>
00021 #include <qtimer.h>
00022 #include <kaboutdata.h>
00023 #include <kapplication.h>
00024 #include <kcmdlineargs.h>
00025 #include <kconfig.h>
00026 #include <kdebug.h>
00027 #include <kglobal.h>
00028 #include <klocale.h>
00029 #include <kmessagebox.h>
00030 #include <kstandarddirs.h>
00031 #include <kprocess.h>
00032 #include <kdeversion.h>
00033 
00034 #define INDEXER
00035 #include "misc.cpp"
00036 
00037 
00038 ProgressDialog::ProgressDialog(bool index, QWidget *parent, const char *name)
00039     : KDialogBase(KDialogBase::Plain, i18n("Generating Search Index"), Cancel | Ok, Close,
00040                   parent, name, false)
00041 {
00042     proc = 0;
00043 
00044     indexdir = kapp->dirs()->saveLocation("data", "kdevdoctreeview/helpindex");
00045     QDir d; d.mkdir(indexdir);
00046 
00047     KConfig config("kdevdoctreeviewrc", true);
00048     config.setGroup("htdig");
00049     databaseDir = config.readPathEntry( "databaseDir", indexdir );
00050 
00051     if( !index )
00052         return;
00053 
00054     d.mkdir( databaseDir );
00055 
00056     showButtonOK( false );
00057     QGridLayout *grid = new QGridLayout(plainPage(), 5,3, spacingHint());
00058 
00059     QLabel *l = new QLabel(i18n("Scanning for files"), plainPage());
00060     grid->addMultiCellWidget(l, 0, 0, 1, 2);
00061 
00062     filesLabel = new QLabel(plainPage());
00063     grid->addWidget(filesLabel, 1, 2);
00064     setFilesScanned(0);
00065 
00066     check1 = new QLabel(plainPage());
00067     grid->addWidget(check1, 0, 0);
00068 
00069     l = new QLabel(i18n("Extracting search terms"), plainPage());
00070     grid->addMultiCellWidget(l, 2,2, 1,2);
00071 
00072     bar = new KProgress(plainPage());
00073     grid->addWidget(bar, 3,2);
00074 
00075     check2 = new QLabel(plainPage());
00076     grid->addWidget(check2, 2,0);
00077 
00078     l = new QLabel(i18n("Generating index..."), plainPage());
00079     grid->addMultiCellWidget(l, 4,4, 1,2);
00080 
00081     check3 = new QLabel(plainPage());
00082     grid->addWidget(check3, 4,0);
00083 
00084     setState(0);
00085 
00086     setMinimumWidth(300);
00087     connect(this, SIGNAL(cancelClicked()), this, SLOT(cancelClicked()));
00088     connect(this, SIGNAL(okClicked()), this, SLOT(okClicked()));
00089     QTimer::singleShot(0, this, SLOT(slotDelayedStart()));
00090 }
00091 
00092 
00093 ProgressDialog::~ProgressDialog()
00094 {}
00095 
00096 void ProgressDialog::slotDelayedStart()
00097 {
00098     procdone = false;
00099     scanDirectories();
00100     if (!createConfig())
00101     {
00102       done(1);
00103       return;
00104     }
00105     generateIndex();
00106 }
00107 
00108 
00109 void ProgressDialog::done(int r)
00110 {
00111     if (!r)
00112     {
00113         showButtonCancel( false );
00114         showButtonOK( true );
00115     }
00116     else
00117         KDialogBase::done(r);
00118 }
00119 
00120 
00121 void ProgressDialog::setFilesScanned(int n)
00122 {
00123     filesLabel->setText(i18n("Files processed: %1").arg(n));
00124 }
00125 
00126 
00127 void ProgressDialog::setFilesToDig(int n)
00128 {
00129     bar->setRange(0, n);
00130 }
00131 
00132 
00133 void ProgressDialog::setFilesDigged(int n)
00134 {
00135     bar->setValue(n);
00136 }
00137 
00138 
00139 void ProgressDialog::setState(int n)
00140 {
00141     QPixmap unchecked = QPixmap(locate("data", "kdevdoctreeview/pics/unchecked.xpm"));
00142     QPixmap checked = QPixmap(locate("data", "kdevdoctreeview/pics/checked.xpm"));
00143 
00144     check1->setPixmap( n > 0 ? checked : unchecked);
00145     check2->setPixmap( n > 1 ? checked : unchecked);
00146     check3->setPixmap( n > 2 ? checked : unchecked);
00147 }
00148 
00149 
00150 void ProgressDialog::addDir(const QString &dir)
00151 {
00152     kdDebug(9002) << "Add dir : " << dir << endl;
00153     QDir d(dir, "*.html", QDir::Name|QDir::IgnoreCase, QDir::Files | QDir::Readable);
00154     QStringList list = d.entryList();
00155 
00156     QStringList::ConstIterator it;
00157     for ( it=list.begin(); it!=list.end(); ++it ) {
00158     if( (*it).right( 12 ).lower( ) == "-source.html" )
00159         continue;
00160 
00161         files.append(dir + "/" + *it);
00162         setFilesScanned(++filesScanned);
00163     }
00164 
00165     QDir d2(dir, QString::null, QDir::Name|QDir::IgnoreCase, QDir::Dirs);
00166     QStringList dlist = d2.entryList();
00167 
00168     for ( it=dlist.begin(); it != dlist.end(); ++it ) {
00169         if (*it != "." && *it != "..") {
00170             addDir(dir + "/" + *it);
00171             kapp->processEvents();
00172         }
00173         if (procdone)
00174         {
00175           return;
00176         }
00177     }
00178     kapp->processEvents();
00179 }
00180 
00181 
00182 void ProgressDialog::addKdocDir(FILE *f)
00183 {
00184     char buf[1024];
00185     while (fgets(buf, sizeof buf, f)) {
00186         QString s = buf;
00187         if (s.left(11) == "<BASE URL=\"") {
00188             int pos2 = s.find("\">", 11);
00189             if (pos2 != -1) {
00190                 addDir(s.mid(11, pos2-11));
00191                 return;
00192             }
00193         }
00194     }
00195 }
00196 
00197 
00198 void ProgressDialog::addTocFile(QDomDocument &doc)
00199 {
00200     QStringList candidates;
00201     QString base;
00202     QDomElement childEl = doc.documentElement().firstChild().toElement();
00203     while (!childEl.isNull()) {
00204         if (childEl.tagName() == "tocsect1") {
00205             QString url = childEl.attribute("url");
00206             if (!url.isEmpty()) {
00207                 url.prepend(base);
00208                 kdDebug(9002) << "candidate: " << url << endl;
00209                 candidates.append(url);
00210             }
00212             QDomElement grandchildEl = childEl.firstChild().toElement();
00213             while (!grandchildEl.isNull()) {
00214                 if (grandchildEl.tagName() == "tocsect2") {
00215                     QString url = grandchildEl.attribute("url");
00216                     if (!url.isEmpty()) {
00217                         url.prepend(base);
00218                         kdDebug(9002) << "candidate: " << url << endl;
00219                         candidates.append(url);
00220                     }
00221                 }
00222                 grandchildEl = grandchildEl.nextSibling().toElement();
00223             }
00224         } else if (childEl.tagName() == "base") {
00225             base = childEl.attribute("href");
00226             if (!base.isEmpty())
00227                 base += "/";
00228         }
00229         childEl = childEl.nextSibling().toElement();
00230     }
00231 
00232     QStringList::ConstIterator it;
00233     for (it = candidates.begin(); it != candidates.end(); ++it) {
00234         QString url = *it;
00235         int pos = url.findRev('#');
00236         if (pos != -1)
00237             url.truncate(pos);
00238         if ((url.startsWith("/") || url.startsWith("file://"))
00239             && !files.contains(url)) {
00240             files.append(url);
00241             kdDebug(9002) << "tocurl: " << url << endl;
00242             setFilesScanned(++filesScanned);
00243         }
00244     }
00245 }
00246 
00247 
00248 void ProgressDialog::scanDirectories()
00249 {
00250     KConfig config("kdevdoctreeviewrc", true);
00251     config.setGroup("Index");
00252     bool indexKDevelop = config.readBoolEntry("IndexKDevelop");
00253     bool indexQt = config.readBoolEntry("IndexQt");
00254     bool indexKdelibs = config.readBoolEntry("IndexKdelibs");
00255     bool indexBooks = config.readBoolEntry("IndexBooks");
00256     bool indexBookmarks = config.readBoolEntry("IndexBookmarks");
00257 
00258     bool indexShownLibs = true;
00259     bool indexHiddenLibs = true;
00260 
00261     filesScanned = 0;
00262 
00263     QStringList itemNames, fileNames, hiddenNames;
00264     DocTreeViewTool::getAllLibraries(&itemNames, &fileNames);
00265     DocTreeViewTool::getHiddenLibraries(&hiddenNames);
00266 
00267     QStringList::ConstIterator it1, it2;
00268     for (it1 = itemNames.begin(), it2 = fileNames.begin();
00269          it1 != itemNames.end() && it2 != fileNames.end();
00270          ++it1, ++it2) {
00271         bool ishidden = hiddenNames.contains(*it2);
00272         if ( (indexHiddenLibs && ishidden) || (indexShownLibs && !ishidden) ) {
00273             FILE *f;
00274             if ((*it2).right(3) != QString::fromLatin1(".gz")) {
00275                 if ( (f = fopen(QFile::encodeName( *it2 ).data(), "r")) != 0) {
00276                     addKdocDir(f);
00277                     fclose(f);
00278                 }
00279             } else {
00280                 QString cmd = "gzip -c -d ";
00281 #if (KDE_VERSION > 305)
00282                 cmd += KProcess::quote(*it2);
00283 #else
00284                 cmd += KShellProcess::quote(*it2);
00285 #endif
00286                 cmd += " 2>/dev/null";
00287                 if ( (f = popen(QFile::encodeName(cmd), "r")) != 0) {
00288                     addKdocDir(f);
00289                     pclose(f);
00290                 }
00291             }
00292         }
00293     }
00294 
00295     if (indexKDevelop) {
00297     }
00298 
00299     if (indexQt) {
00300       QString oldqtdocdir;
00301       config.setGroup("General Qt");
00302       QMap<QString, QString> emap = config.entryMap("General Qt");
00303       QMap<QString, QString>::Iterator it;
00304       for (it = emap.begin(); it != emap.end(); ++it)
00305       {
00306           QString qtdocdir = config.readPathEntry(it.key());
00307           if (!qtdocdir.isEmpty())
00308     {
00309       qtdocdir = qtdocdir.left(qtdocdir.findRev("/") + 1);
00310       if (qtdocdir != oldqtdocdir)
00311       {
00312                   addDir(qtdocdir);
00313         oldqtdocdir = qtdocdir;
00314       }
00315     }
00316       }
00317     }
00318 
00319     if (indexKdelibs) {
00320       config.setGroup("General Doxygen");
00321       QMap<QString, QString> xmap = config.entryMap("General Doxygen");
00322       QMap<QString, QString>::Iterator itx;
00323       for (itx = xmap.begin(); itx != xmap.end(); ++itx)
00324       {
00325           QString kdelibsdocdir =  config.readPathEntry(itx.key());
00326           if (!kdelibsdocdir.isEmpty())
00327                 addDir(kdelibsdocdir);
00328       }
00329     }
00330 
00331     if (indexBooks) {
00332         KStandardDirs *dirs = KGlobal::dirs();
00333         QStringList tocs = dirs->findAllResources("doctocs", QString::null, false, true);
00334 
00335         QStringList::ConstIterator it4;
00336         for (it4 = tocs.begin(); it4 != tocs.end(); ++it4) {
00337             QFile f(*it4);
00338             if (!f.open(IO_ReadOnly)) {
00339                 kdDebug(9002) << "Could not read doc toc: " << (*it4) << endl;
00340                 continue;
00341             }
00342             QDomDocument doc;
00343             if (!doc.setContent(&f) || doc.doctype().name() != "kdeveloptoc") {
00344                 kdDebug(9002) << "Not a valid kdeveloptoc file: " << (*it4) << endl;
00345                 continue;
00346             }
00347             f.close();
00348             addTocFile(doc);
00349         }
00350     }
00351 
00352     if (indexBookmarks) {
00353         QStringList bookmarksTitle, bookmarksURL;
00354         DocTreeViewTool::getBookmarks(&bookmarksTitle, &bookmarksURL);
00355         QStringList::ConstIterator it3;
00356         for (it3 = bookmarksURL.begin(); it3 != bookmarksURL.end(); ++it3) {
00358             // lives in
00359             files.append(*it3);
00360             setFilesScanned(++filesScanned);
00361         }
00362     }
00363 }
00364 
00365 
00366 bool ProgressDialog::createConfig()
00367 {
00368     // locate the common dir
00369     QString language = KGlobal::locale()->language();
00370     if (language == "C")
00371         language = "en";
00372 
00373     QString wrapper = locate("data", QString("kdevdoctreeview/%1/wrapper.html").arg(language));
00374     if (wrapper.isEmpty())
00375         wrapper = locate("data", QString("kdevdoctreeview/en/wrapper.html"));
00376     if (wrapper.isEmpty())
00377         return false;
00378     wrapper = wrapper.left(wrapper.length()-12);
00379 
00380     // locate the image dir
00381     QString images = locate("data", "kdevdoctreeview/pics/star.png");
00382     if (images.isEmpty())
00383         return false;
00384     images = images.left(images.length()-8);
00385 
00386     QFile f(indexdir + "/htdig.conf");
00387     if (f.open(IO_WriteOnly)) {
00388         QTextStream ts(&f);
00389 
00390         ts << "database_dir:\t\t" << databaseDir << endl;
00391         ts << "start_url:\t\t`" << indexdir << "/files`" << endl;
00392         ts << "local_urls:\t\thttp://localhost/=/" << endl;
00393 //        ts << "local_urls:\t\tfile://=" << endl;
00394         ts << "local_urls_only:\ttrue" << endl;
00395   ts << "limit_urls_to:\t\tfile://" << endl;
00396         ts << "maximum_pages:\t\t1" << endl;
00397         ts << "image_url_prefix:\t" << images << endl;
00398         ts << "star_image:\t\t" << images << "star.png" << endl;
00399         ts << "star_blank:\t\t" << images << "star_blank.png" << endl;
00400         ts << "compression_level:\t6" << endl;
00401         ts << "max_hop_count:\t\t0" << endl;
00402 
00403         ts << "search_results_wrapper:\t" << wrapper << "wrapper.html" << endl;
00404         ts << "nothing_found_file:\t" << wrapper << "nomatch.html" << endl;
00405         ts << "syntax_error_file:\t" << wrapper << "syntax.html" << endl;
00406         ts << "bad_word_list:\t\t" << wrapper << "bad_words" << endl;
00407 
00408         f.close();
00409         return true;
00410     }
00411 
00412     return false;
00413 }
00414 
00415 
00416 #define CHUNK_SIZE 100
00417 
00418 void ProgressDialog::startHtdigProcess(bool initial)
00419 {
00420   kdDebug(9002) << "htdig started" << endl;
00421   delete proc;
00422   proc = new KProcess();
00423   *proc << exe << "-c" << (indexdir + "/htdig.conf");
00424   if (initial) {
00425       *proc << "-i";
00426   }
00427   connect(proc, SIGNAL(processExited(KProcess *)),
00428           this, SLOT(htdigExited(KProcess *)));
00429 
00430   htdigRunning = true;
00431 
00432   // write out file
00433   QFile f(indexdir+"/files");
00434   if (!f.open(IO_WriteOnly)) {
00435       kdDebug(9002) << "Could not open `files` for writing" << endl;
00436       done(1);
00437       return;
00438   }
00439   QTextStream ts(&f);
00440   for (int i=0; i<CHUNK_SIZE; ++i, ++count) {
00441       if (count >= filesToDig) {
00442           procdone = true;
00443           break;
00444       }
00445 //    ts << "file://localhost/" + files[count] << endl;
00446       ts << "http://localhost/" + files[count] << endl;
00447   }
00448   f.close();
00449 
00450   // execute htdig
00451   proc->start(KProcess::NotifyOnExit, KProcess::Stdout);
00452 
00453 }
00454 
00455 bool ProgressDialog::generateIndex()
00456 {
00457     setState(1);
00458     procdone = false;
00459     // run htdig
00460     KConfig config("kdevdoctreeviewrc", true);
00461     config.setGroup("htdig");
00462     exe = config.readPathEntry("htdigbin", kapp->dirs()->findExe("htdig"));
00463     if (exe.isEmpty())
00464     {
00465         done(1);
00466         return true;
00467     }
00468     filesToDig = files.count();
00469     count = 0;
00470     setFilesToDig(filesToDig);
00471     filesDigged = 0;
00472 
00473     //    QDir d; d.mkdir(indexdir);
00474     startHtdigProcess(true);
00475     return true;
00476 }
00477 
00478 
00479 
00480 void ProgressDialog::htdigStdout(KProcess *, char *buffer, int len)
00481 {
00482     QString line = QString(buffer).left(len);
00483 
00484     int cnt=0, index=-1;
00485     while ( (index = line.find("http://", index+1)) > 0)
00486         cnt++;
00487     filesDigged += cnt;
00488 
00489     cnt=0, index=-1;
00490     while ( (index = line.find("not changed", index+1)) > 0)
00491         cnt++;
00492     filesDigged -= cnt;
00493 
00494     setFilesDigged(filesDigged);
00495 }
00496 
00497 
00498 void ProgressDialog::htdigExited(KProcess *proc)
00499 {
00500     kdDebug(9002) << "htdig terminated" << endl;
00501     if (!proc->normalExit())
00502     {
00503       delete proc;
00504       proc = 0L;
00505       done(1);
00506       return;
00507     }
00508     if (proc && proc->exitStatus() != 0) {
00509                 KMessageBox::sorry(0, i18n("Running htdig failed"));
00510                 delete proc;
00511                 proc = 0L;
00512                 done(1);
00513                 return;
00514             }
00515     htdigRunning = false;
00516     filesDigged += CHUNK_SIZE;
00517     setFilesDigged(filesDigged);
00518     if (!procdone)
00519     {
00520         startHtdigProcess(false);
00521     } else
00522     {
00523       setFilesDigged(filesToDig);
00524       setState(2);
00525 
00526       KConfig config("kdevdoctreeviewrc", true);
00527       config.setGroup("htdig");
00528       // run htmerge -----------------------------------------------------
00529       exe = config.readPathEntry("htmergebin", kapp->dirs()->findExe("htmerge"));
00530       if (exe.isEmpty())
00531       {
00532           done(1);
00533           return;
00534       }
00535       startHtmergeProcess();
00536     }
00537 }
00538 
00539 void ProgressDialog::startHtmergeProcess()
00540 {
00541     kdDebug(9002) << "htmerge started" << endl;
00542     delete proc;
00543     proc = new KProcess();
00544     *proc << exe << "-c" << (indexdir + "/htdig.conf");
00545 
00546     kdDebug(9002) << "Running htmerge" << endl;
00547 
00548     connect(proc, SIGNAL(processExited(KProcess *)),
00549             this, SLOT(htmergeExited(KProcess *)));
00550 
00551     htmergeRunning = true;
00552 
00553     proc->start(KProcess::NotifyOnExit, KProcess::Stdout);
00554 }
00555 
00556 void ProgressDialog::htmergeExited(KProcess *proc)
00557 {
00558     kdDebug(9002) << "htmerge terminated" << endl;
00559     htmergeRunning = false;
00560     if (!proc->normalExit())
00561     {
00562       delete proc;
00563       proc = 0L;
00564       done(1);
00565       return;
00566     }
00567     if (proc && proc->exitStatus() != 0) {
00568         KMessageBox::sorry(0, i18n("Running htmerge failed"));
00569         delete proc;
00570         proc = 0L;
00571         done(1);
00572         return;
00573     }
00574     setState(3);
00575     done(0);
00576 }
00577 
00578 void ProgressDialog::cancelClicked()
00579 {
00580   if ((htdigRunning || htmergeRunning) && proc && proc->isRunning())
00581   {
00582     kdDebug(9002) << "Killing " << (htdigRunning ? "htdig" : "htmerge") << "daemon with Sig. 9" << endl;
00583     proc->kill(9);
00584     htdigRunning = htmergeRunning = false;
00585   } else
00586   {
00587     procdone = true;
00588     done(2);
00589   }
00590 }
00591 
00592 void ProgressDialog::okClicked( )
00593 {
00594     if ( proc )
00595         proc->kill( );
00596 
00597     KDialogBase::done( 0 );
00598 }
00599 
00600 int main(int argc, char *argv[])
00601 {
00602 #if 0
00603     static KCmdLineOptions options[] = {
00604         { "+dirs", I18N_NOOP("The directories to index"), 0 },
00605         KCmdLineLastOption
00606     };
00607 #endif
00608 
00609     static const KCmdLineOptions options[] =
00610     {
00611         { "c", I18N_NOOP( "Update user's htdig configuration file only" ), 0 },
00612         { "i",  I18N_NOOP( "-c and generate index" ), 0 },
00613         KCmdLineLastOption
00614     };
00615 
00616     KAboutData aboutData("kdevdoctreeview", I18N_NOOP("KDevelop"),
00617                          "0.1", I18N_NOOP("KDE Index generator for help files."));
00618 
00619     KCmdLineArgs::init(argc, argv, &aboutData);
00620     KCmdLineArgs::addCmdLineOptions(options);
00621 
00622     KApplication app;
00623 
00624     KGlobal::dirs()->addResourceType("doctocs", KStandardDirs::kde_default("data") + "kdevdoctreeview/tocs/");
00625     KGlobal::locale()->setMainCatalogue("kdevelop");
00626 
00627     KCmdLineArgs   *args   = KCmdLineArgs::parsedArgs( );
00628 
00629     if( args->isSet( "c" ))
00630     {
00631         ProgressDialog *search = new ProgressDialog( false, 0, "progress dialog");
00632 
00633     if( search->createConfig( ))
00634         KMessageBox::information( 0, "Configuration file updated" );
00635     else
00636         KMessageBox::error( 0, "Configuration file update faild!" );
00637     }
00638     else
00639         if( args->isSet( "i" ))
00640         {
00641             ProgressDialog *search = new ProgressDialog( true, 0, "progress dialog");
00642             app.setMainWidget(search);
00643             search->show();
00644             app.exec();
00645         }
00646     else
00647     {
00648         fprintf( stderr, "Internal error generating index - unknown argument\n" );
00649         return 1;
00650     }
00651 
00652     return 0;
00653 }
00654 
00655 #include "htdigindex.moc"
KDE Logo
This file is part of the documentation for KDevelop Version 3.1.2.
Documentation copyright © 1996-2004 the KDE developers.
Generated on Wed Mar 23 00:03:55 2005 by doxygen 1.3.9.1 written by Dimitri van Heesch, © 1997-2003