KDevelop API Documentation

htdigindex.cpp

Go to the documentation of this file.
00001 /***************************************************************************
00002  *   Copyright (C) 1999-2001 by Matthias Hoelzer-Kluepfel                  *
00003  *   hoelzer@kde.org                                                       *
00004  *   Copyright (C) 2001 by Bernd Gehrmann                                  *
00005  *   bernd@kdevelop.org                                                    *
00006  *   Copyright (C) 2004 by Alexander Dymo                                  *
00007  *   cloudtemple@mksat.net                                                 *
00008  *                                                                         *
00009  *   This program is free software; you can redistribute it and/or modify  *
00010  *   it under the terms of the GNU General Public License as published by  *
00011  *   the Free Software Foundation; either version 2 of the License, or     *
00012  *   (at your option) any later version.                                   *
00013  *                                                                         *
00014  ***************************************************************************/
00015 #include "htdigindex.h"
00016 
00017 #include <iostream>
00018 
00019 #include <qapplication.h>
00020 #include <qdir.h>
00021 #include <qfile.h>
00022 #include <qlayout.h>
00023 #include <qtextstream.h>
00024 #include <qtimer.h>
00025 #include <qlabel.h>
00026 
00027 #include <kaboutdata.h>
00028 #include <kapplication.h>
00029 #include <kcmdlineargs.h>
00030 #include <kconfig.h>
00031 #include <kdebug.h>
00032 #include <kglobal.h>
00033 #include <klocale.h>
00034 #include <kmessagebox.h>
00035 #include <kstandarddirs.h>
00036 #include <kprocess.h>
00037 #include <kdeversion.h>
00038 #include <kprogress.h>
00039 
00040 #define INDEXER
00041 
00042 ProgressDialog::ProgressDialog(bool index, QWidget *parent, const char *name)
00043     :KDialogBase(KDialogBase::Plain, i18n("Generating Search Index"), Cancel | Ok, Close,
00044     parent, name, false)
00045 {
00046     proc = 0;
00047 
00048     indexdir = kapp->dirs()->saveLocation("data", "kdevdocumentation/search");
00049     QDir d; d.mkdir(indexdir);
00050 
00051     KConfig config("kdevdocumentation", true);
00052     config.setGroup("htdig");
00053     databaseDir = config.readPathEntry("databaseDir", indexdir);
00054 
00055     if (!index)
00056         return;
00057 
00058     d.mkdir( databaseDir );
00059 
00060     showButtonOK( false );
00061     QGridLayout *grid = new QGridLayout(plainPage(), 5,3, spacingHint());
00062 
00063     QLabel *l = new QLabel(i18n("Scanning for files"), plainPage());
00064     grid->addMultiCellWidget(l, 0, 0, 1, 2);
00065 
00066     filesLabel = new QLabel(plainPage());
00067     grid->addWidget(filesLabel, 1, 2);
00068     setFilesScanned(0);
00069 
00070     check1 = new QLabel(plainPage());
00071     grid->addWidget(check1, 0, 0);
00072 
00073     l = new QLabel(i18n("Extracting search terms"), plainPage());
00074     grid->addMultiCellWidget(l, 2,2, 1,2);
00075 
00076     bar = new KProgress(plainPage());
00077     grid->addWidget(bar, 3,2);
00078 
00079     check2 = new QLabel(plainPage());
00080     grid->addWidget(check2, 2,0);
00081 
00082     l = new QLabel(i18n("Generating index..."), plainPage());
00083     grid->addMultiCellWidget(l, 4,4, 1,2);
00084 
00085     check3 = new QLabel(plainPage());
00086     grid->addWidget(check3, 4,0);
00087 
00088     setState(0);
00089 
00090     setMinimumWidth(300);
00091     connect(this, SIGNAL(cancelClicked()), this, SLOT(cancelClicked()));
00092     connect(this, SIGNAL(okClicked()), this, SLOT(okClicked()));
00093     QTimer::singleShot(0, this, SLOT(slotDelayedStart()));
00094 }
00095 
00096 ProgressDialog::~ProgressDialog()
00097 {
00098 }
00099 
00100 void ProgressDialog::slotDelayedStart()
00101 {
00102     procdone = false;
00103     scanDirectories();
00104     if (!createConfig())
00105     {
00106       done(1);
00107       return;
00108     }
00109     generateIndex();
00110 }
00111 
00112 void ProgressDialog::done(int r)
00113 {
00114     if (!r)
00115     {
00116         showButtonCancel( false );
00117         showButtonOK( true );
00118     }
00119     else
00120         KDialogBase::done(r);
00121 }
00122 
00123 void ProgressDialog::setFilesScanned(int n)
00124 {
00125     filesLabel->setText(i18n("Files processed: %1").arg(n));
00126 }
00127 
00128 void ProgressDialog::setFilesToDig(int n)
00129 {
00130     bar->setTotalSteps(n);
00131 }
00132 
00133 void ProgressDialog::setFilesDigged(int n)
00134 {
00135     bar->setValue(n);
00136 }
00137 
00138 void ProgressDialog::setState(int n)
00139 {
00140     QPixmap unchecked = QPixmap(locate("data", "kdevdocumentation/pics/unchecked.xpm"));
00141     QPixmap checked = QPixmap(locate("data", "kdevdocumentation/pics/checked.xpm"));
00142 
00143     check1->setPixmap( n > 0 ? checked : unchecked);
00144     check2->setPixmap( n > 1 ? checked : unchecked);
00145     check3->setPixmap( n > 2 ? checked : unchecked);
00146 }
00147 
00148 
00149 void ProgressDialog::addDir(const QString &dir)
00150 {
00151     kdDebug(9002) << "Add dir : " << dir << endl;
00152     QDir d(dir, "*.html", QDir::Name|QDir::IgnoreCase, QDir::Files | QDir::Readable);
00153     QStringList list = d.entryList();
00154 
00155     QStringList::ConstIterator it;
00156     for ( it=list.begin(); it!=list.end(); ++it )
00157     {
00158         if( (*it).right( 12 ).lower( ) == "-source.html" )
00159             continue;
00160 
00161         files.append(dir + "/" + *it);
00162         setFilesScanned(++filesScanned);
00163     }
00164 
00165     QDir d2(dir, QString::null, QDir::Name|QDir::IgnoreCase, QDir::Dirs);
00166     QStringList dlist = d2.entryList();
00167 
00168     for ( it=dlist.begin(); it != dlist.end(); ++it ) 
00169     {
00170         if (*it != "." && *it != "..") 
00171         {
00172             addDir(dir + "/" + *it);
00173             kapp->processEvents();
00174         }
00175         if (procdone)
00176         {
00177           return;
00178         }
00179     }
00180     kapp->processEvents();
00181 }
00182 
00183 void ProgressDialog::scanDirectories()
00184 {
00185     QString ftsLocationsFile = locateLocal("data", "kdevdocumentation/search/locations.txt");
00186 
00187     QFile f(ftsLocationsFile);
00188     if (!f.open(IO_ReadOnly))
00189         return;
00190     QTextStream str(&f);
00191     
00192     filesScanned = 0;
00193     
00194     while (!str.eof())
00195     {
00196         QString loc = str.readLine();
00197         if (loc.isEmpty())
00198             continue;
00199         QFileInfo fi(loc);
00200         if (fi.isDir())
00201             addDir(loc);
00202         else if (fi.isFile())
00203         {
00204             files.append(loc);
00205             setFilesScanned(++filesScanned);
00206         }
00207     }
00208 }
00209 
00210 bool ProgressDialog::createConfig()
00211 {
00212     // locate the common dir
00213     QString language = KGlobal::locale()->language();
00214     if (language == "C")
00215         language = "en";
00216 
00217     QString wrapper = locate("data", QString("kdevdocumentation/%1/wrapper.html").arg(language));
00218     if (wrapper.isEmpty())
00219         wrapper = locate("data", QString("kdevdocumentation/en/wrapper.html"));
00220     if (wrapper.isEmpty())
00221         return false;
00222     wrapper = wrapper.left(wrapper.length()-12);
00223 
00224     // locate the image dir
00225     QString images = locate("data", "kdevdocumentation/pics/star.png");
00226     if (images.isEmpty())
00227         return false;
00228     images = images.left(images.length()-8);
00229 
00230     QFile f(indexdir + "/htdig.conf");
00231     if (f.open(IO_WriteOnly)) 
00232     {
00233         QTextStream ts(&f);
00234 
00235         ts << "database_dir:\t\t" << databaseDir << endl;
00236         ts << "start_url:\t\t`" << indexdir << "/files`" << endl;
00237         ts << "local_urls:\t\thttp://localhost/=/" << endl;
00238 //        ts << "local_urls:\t\tfile://=" << endl;
00239         ts << "local_urls_only:\ttrue" << endl;
00240         ts << "limit_urls_to:\t\tfile:// http://localhost/" << endl;
00241         ts << "maximum_pages:\t\t1" << endl;
00242         ts << "image_url_prefix:\t" << images << endl;
00243         ts << "star_image:\t\t" << images << "star.png" << endl;
00244         ts << "star_blank:\t\t" << images << "star_blank.png" << endl;
00245         ts << "compression_level:\t6" << endl;
00246         ts << "max_hop_count:\t\t0" << endl;
00247 
00248         ts << "search_results_wrapper:\t" << wrapper << "wrapper.html" << endl;
00249         ts << "nothing_found_file:\t" << wrapper << "nomatch.html" << endl;
00250         ts << "syntax_error_file:\t" << wrapper << "syntax.html" << endl;
00251         ts << "bad_word_list:\t\t" << wrapper << "bad_words" << endl;
00252 
00253         f.close();
00254         return true;
00255     }
00256 
00257     return false;
00258 }
00259 
00260 #define CHUNK_SIZE 100
00261 
00262 void ProgressDialog::startHtdigProcess(bool initial)
00263 {
00264     kdDebug(9002) << "htdig started" << endl;
00265     delete proc;
00266     proc = new KProcess();
00267     *proc << exe << "-c" << (indexdir + "/htdig.conf");
00268     if (initial) {
00269         *proc << "-i";
00270     }
00271     connect(proc, SIGNAL(processExited(KProcess *)),
00272             this, SLOT(htdigExited(KProcess *)));
00273     
00274     htdigRunning = true;
00275     
00276     // write out file
00277     QFile f(indexdir+"/files");
00278     if (!f.open(IO_WriteOnly)) {
00279         kdDebug(9002) << "Could not open `files` for writing" << endl;
00280         done(1);
00281         return;
00282     }
00283     QTextStream ts(&f);
00284     for (int i=0; i<CHUNK_SIZE; ++i, ++count) {
00285         if (count >= filesToDig) {
00286             procdone = true;
00287             break;
00288         }
00289     //    ts << "file://localhost/" + files[count] << endl;
00290         ts << "http://localhost/" + files[count] << endl;
00291     }
00292     f.close();
00293     
00294     // execute htdig
00295     proc->start(KProcess::NotifyOnExit, KProcess::Stdout);   
00296 }
00297 
00298 bool ProgressDialog::generateIndex()
00299 {
00300     setState(1);
00301     procdone = false;
00302     // run htdig
00303     KConfig config("kdevdocumentation", true);
00304     config.setGroup("htdig");
00305     exe = config.readPathEntry("htdigbin", kapp->dirs()->findExe("htdig"));
00306     if (exe.isEmpty())
00307     {
00308         done(1);
00309         return true;
00310     }
00311     filesToDig = files.count();
00312     count = 0;
00313     setFilesToDig(filesToDig);
00314     filesDigged = 0;
00315 
00316     //    QDir d; d.mkdir(indexdir);
00317     startHtdigProcess(true);
00318     return true;
00319 }
00320 
00321 void ProgressDialog::htdigStdout(KProcess *, char *buffer, int len)
00322 {
00323     QString line = QString(buffer).left(len);
00324 
00325     int cnt=0, index=-1;
00326     while ( (index = line.find("http://", index+1)) > 0)
00327         cnt++;
00328     filesDigged += cnt;
00329 
00330     cnt=0, index=-1;
00331     while ( (index = line.find("not changed", index+1)) > 0)
00332         cnt++;
00333     filesDigged -= cnt;
00334 
00335     setFilesDigged(filesDigged);
00336 }
00337 
00338 void ProgressDialog::htdigExited(KProcess *proc)
00339 {
00340     kdDebug(9002) << "htdig terminated" << endl;
00341     if (!proc->normalExit())
00342     {
00343         delete proc;
00344         proc = 0L;
00345         done(1);
00346         return;
00347     }
00348     if (proc && proc->exitStatus() != 0) 
00349     {
00350         KMessageBox::sorry(0, i18n("Running htdig failed"));
00351         delete proc;
00352         proc = 0L;
00353         done(1);
00354         return;
00355     }
00356     htdigRunning = false;
00357     filesDigged += CHUNK_SIZE;
00358     setFilesDigged(filesDigged);
00359     if (!procdone)
00360     {
00361         startHtdigProcess(false);
00362     }
00363     else
00364     {
00365         setFilesDigged(filesToDig);
00366         setState(2);
00367         
00368         KConfig config("kdevdocumentation", true);
00369         config.setGroup("htdig");
00370         // run htmerge -----------------------------------------------------
00371         exe = config.readPathEntry("htmergebin", kapp->dirs()->findExe("htmerge"));
00372         if (exe.isEmpty())
00373         {
00374             done(1);
00375             return;
00376         }
00377         startHtmergeProcess();
00378     }
00379 }
00380 
00381 void ProgressDialog::startHtmergeProcess()
00382 {
00383     kdDebug(9002) << "htmerge started" << endl;
00384     delete proc;
00385     proc = new KProcess();
00386     *proc << exe << "-c" << (indexdir + "/htdig.conf");
00387 
00388     kdDebug(9002) << "Running htmerge" << endl;
00389 
00390     connect(proc, SIGNAL(processExited(KProcess *)),
00391             this, SLOT(htmergeExited(KProcess *)));
00392 
00393     htmergeRunning = true;
00394 
00395     proc->start(KProcess::NotifyOnExit, KProcess::Stdout);
00396 }
00397 
00398 void ProgressDialog::htmergeExited(KProcess *proc)
00399 {
00400     kdDebug(9002) << "htmerge terminated" << endl;
00401     htmergeRunning = false;
00402     if (!proc->normalExit())
00403     {
00404         delete proc;
00405         proc = 0L;
00406         done(1);
00407         return;
00408     }
00409     if (proc && proc->exitStatus() != 0) 
00410     {
00411         KMessageBox::sorry(0, i18n("Running htmerge failed"));
00412         delete proc;
00413         proc = 0L;
00414         done(1);
00415         return;
00416     }
00417     setState(3);
00418     done(0);
00419 }
00420 
00421 void ProgressDialog::cancelClicked()
00422 {
00423     if ((htdigRunning || htmergeRunning) && proc && proc->isRunning())
00424     {
00425         kdDebug(9002) << "Killing " << (htdigRunning ? "htdig" : "htmerge") << "daemon with Sig. 9" << endl;
00426         proc->kill(9);
00427         htdigRunning = htmergeRunning = false;
00428     }
00429     else
00430     {
00431         procdone = true;
00432         done(2);
00433     }
00434 }
00435 
00436 void ProgressDialog::okClicked()
00437 {
00438     if (proc)
00439         proc->kill();
00440 
00441     KDialogBase::done(0);
00442 }
00443 
00444 int main(int argc, char *argv[])
00445 {
00446     static const KCmdLineOptions options[] =
00447     {
00448         { "c", I18N_NOOP( "Update user's htdig configuration file only" ), 0 },
00449         { "i",  I18N_NOOP( "-c and generate index" ), 0 },
00450         KCmdLineLastOption
00451     };
00452 
00453     KAboutData aboutData("kdevelop-htdig", I18N_NOOP("KDevelop ht://Dig Indexer"),
00454                          "0.2", I18N_NOOP("KDE Index generator for documentation files."));
00455 
00456     KCmdLineArgs::init(argc, argv, &aboutData);
00457     KCmdLineArgs::addCmdLineOptions(options);
00458 
00459     KApplication app;
00460 
00461     KGlobal::locale()->setMainCatalogue("kdevelop");
00462 
00463     KCmdLineArgs *args = KCmdLineArgs::parsedArgs();
00464 
00465     if (args->isSet("c"))
00466     {
00467         ProgressDialog *search = new ProgressDialog( false, 0, "progress dialog");
00468 
00469         if (search->createConfig())
00470             KMessageBox::information(0, i18n("Configuration file updated"));
00471         else
00472             KMessageBox::error(0, i18n("Configuration file update failed."));
00473     }
00474     else
00475         if (args->isSet("i"))
00476         {
00477             ProgressDialog *search = new ProgressDialog(true, 0, "progress dialog");
00478             app.setMainWidget(search);
00479             search->show();
00480             app.exec();
00481         }
00482         else
00483         {
00484             std::cerr << "Internal error generating index - unknown argument\n" << std::endl;
00485             return 1;
00486         }
00487 
00488     return 0;
00489 }
00490 
00491 #include "htdigindex.moc"
KDE Logo
This file is part of the documentation for KDevelop Version 3.1.2.
Documentation copyright © 1996-2004 the KDE developers.
Generated on Wed Mar 23 00:03:55 2005 by doxygen 1.3.9.1 written by Dimitri van Heesch, © 1997-2003