00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014 #include "htdigindex.h"
00015
00016 #include <qapplication.h>
00017 #include <qdir.h>
00018 #include <qfile.h>
00019 #include <qlayout.h>
00020 #include <qtextstream.h>
00021 #include <qtimer.h>
00022 #include <kaboutdata.h>
00023 #include <kapplication.h>
00024 #include <kcmdlineargs.h>
00025 #include <kconfig.h>
00026 #include <kdebug.h>
00027 #include <kglobal.h>
00028 #include <klocale.h>
00029 #include <kmessagebox.h>
00030 #include <kstandarddirs.h>
00031 #include <kprocess.h>
00032 #include <kdeversion.h>
00033
00034 #define INDEXER
00035 #include "misc.cpp"
00036
00037
00038 ProgressDialog::ProgressDialog(bool index, QWidget *parent, const char *name)
00039 : KDialogBase(KDialogBase::Plain, i18n("Generating Search Index"), Cancel | Ok, Close,
00040 parent, name, false)
00041 {
00042 proc = 0;
00043
00044 indexdir = kapp->dirs()->saveLocation("data", "kdevdoctreeview/helpindex");
00045 QDir d; d.mkdir(indexdir);
00046
00047 KConfig config("kdevdoctreeviewrc", true);
00048 config.setGroup("htdig");
00049 databaseDir = config.readPathEntry( "databaseDir", indexdir );
00050
00051 if( !index )
00052 return;
00053
00054 d.mkdir( databaseDir );
00055
00056 showButtonOK( false );
00057 QGridLayout *grid = new QGridLayout(plainPage(), 5,3, spacingHint());
00058
00059 QLabel *l = new QLabel(i18n("Scanning for files"), plainPage());
00060 grid->addMultiCellWidget(l, 0, 0, 1, 2);
00061
00062 filesLabel = new QLabel(plainPage());
00063 grid->addWidget(filesLabel, 1, 2);
00064 setFilesScanned(0);
00065
00066 check1 = new QLabel(plainPage());
00067 grid->addWidget(check1, 0, 0);
00068
00069 l = new QLabel(i18n("Extracting search terms"), plainPage());
00070 grid->addMultiCellWidget(l, 2,2, 1,2);
00071
00072 bar = new KProgress(plainPage());
00073 grid->addWidget(bar, 3,2);
00074
00075 check2 = new QLabel(plainPage());
00076 grid->addWidget(check2, 2,0);
00077
00078 l = new QLabel(i18n("Generating index..."), plainPage());
00079 grid->addMultiCellWidget(l, 4,4, 1,2);
00080
00081 check3 = new QLabel(plainPage());
00082 grid->addWidget(check3, 4,0);
00083
00084 setState(0);
00085
00086 setMinimumWidth(300);
00087 connect(this, SIGNAL(cancelClicked()), this, SLOT(cancelClicked()));
00088 connect(this, SIGNAL(okClicked()), this, SLOT(okClicked()));
00089 QTimer::singleShot(0, this, SLOT(slotDelayedStart()));
00090 }
00091
00092
00093 ProgressDialog::~ProgressDialog()
00094 {}
00095
00096 void ProgressDialog::slotDelayedStart()
00097 {
00098 procdone = false;
00099 scanDirectories();
00100 if (!createConfig())
00101 {
00102 done(1);
00103 return;
00104 }
00105 generateIndex();
00106 }
00107
00108
00109 void ProgressDialog::done(int r)
00110 {
00111 if (!r)
00112 {
00113 showButtonCancel( false );
00114 showButtonOK( true );
00115 }
00116 else
00117 KDialogBase::done(r);
00118 }
00119
00120
00121 void ProgressDialog::setFilesScanned(int n)
00122 {
00123 filesLabel->setText(i18n("Files processed: %1").arg(n));
00124 }
00125
00126
00127 void ProgressDialog::setFilesToDig(int n)
00128 {
00129 bar->setRange(0, n);
00130 }
00131
00132
00133 void ProgressDialog::setFilesDigged(int n)
00134 {
00135 bar->setValue(n);
00136 }
00137
00138
00139 void ProgressDialog::setState(int n)
00140 {
00141 QPixmap unchecked = QPixmap(locate("data", "kdevdoctreeview/pics/unchecked.xpm"));
00142 QPixmap checked = QPixmap(locate("data", "kdevdoctreeview/pics/checked.xpm"));
00143
00144 check1->setPixmap( n > 0 ? checked : unchecked);
00145 check2->setPixmap( n > 1 ? checked : unchecked);
00146 check3->setPixmap( n > 2 ? checked : unchecked);
00147 }
00148
00149
00150 void ProgressDialog::addDir(const QString &dir)
00151 {
00152 kdDebug(9002) << "Add dir : " << dir << endl;
00153 QDir d(dir, "*.html", QDir::Name|QDir::IgnoreCase, QDir::Files | QDir::Readable);
00154 QStringList list = d.entryList();
00155
00156 QStringList::ConstIterator it;
00157 for ( it=list.begin(); it!=list.end(); ++it ) {
00158 if( (*it).right( 12 ).lower( ) == "-source.html" )
00159 continue;
00160
00161 files.append(dir + "/" + *it);
00162 setFilesScanned(++filesScanned);
00163 }
00164
00165 QDir d2(dir, QString::null, QDir::Name|QDir::IgnoreCase, QDir::Dirs);
00166 QStringList dlist = d2.entryList();
00167
00168 for ( it=dlist.begin(); it != dlist.end(); ++it ) {
00169 if (*it != "." && *it != "..") {
00170 addDir(dir + "/" + *it);
00171 kapp->processEvents();
00172 }
00173 if (procdone)
00174 {
00175 return;
00176 }
00177 }
00178 kapp->processEvents();
00179 }
00180
00181
00182 void ProgressDialog::addKdocDir(FILE *f)
00183 {
00184 char buf[1024];
00185 while (fgets(buf, sizeof buf, f)) {
00186 QString s = buf;
00187 if (s.left(11) == "<BASE URL=\"") {
00188 int pos2 = s.find("\">", 11);
00189 if (pos2 != -1) {
00190 addDir(s.mid(11, pos2-11));
00191 return;
00192 }
00193 }
00194 }
00195 }
00196
00197
00198 void ProgressDialog::addTocFile(QDomDocument &doc)
00199 {
00200 QStringList candidates;
00201 QString base;
00202 QDomElement childEl = doc.documentElement().firstChild().toElement();
00203 while (!childEl.isNull()) {
00204 if (childEl.tagName() == "tocsect1") {
00205 QString url = childEl.attribute("url");
00206 if (!url.isEmpty()) {
00207 url.prepend(base);
00208 kdDebug(9002) << "candidate: " << url << endl;
00209 candidates.append(url);
00210 }
00212 QDomElement grandchildEl = childEl.firstChild().toElement();
00213 while (!grandchildEl.isNull()) {
00214 if (grandchildEl.tagName() == "tocsect2") {
00215 QString url = grandchildEl.attribute("url");
00216 if (!url.isEmpty()) {
00217 url.prepend(base);
00218 kdDebug(9002) << "candidate: " << url << endl;
00219 candidates.append(url);
00220 }
00221 }
00222 grandchildEl = grandchildEl.nextSibling().toElement();
00223 }
00224 } else if (childEl.tagName() == "base") {
00225 base = childEl.attribute("href");
00226 if (!base.isEmpty())
00227 base += "/";
00228 }
00229 childEl = childEl.nextSibling().toElement();
00230 }
00231
00232 QStringList::ConstIterator it;
00233 for (it = candidates.begin(); it != candidates.end(); ++it) {
00234 QString url = *it;
00235 int pos = url.findRev('#');
00236 if (pos != -1)
00237 url.truncate(pos);
00238 if ((url.startsWith("/") || url.startsWith("file://"))
00239 && !files.contains(url)) {
00240 files.append(url);
00241 kdDebug(9002) << "tocurl: " << url << endl;
00242 setFilesScanned(++filesScanned);
00243 }
00244 }
00245 }
00246
00247
00248 void ProgressDialog::scanDirectories()
00249 {
00250 KConfig config("kdevdoctreeviewrc", true);
00251 config.setGroup("Index");
00252 bool indexKDevelop = config.readBoolEntry("IndexKDevelop");
00253 bool indexQt = config.readBoolEntry("IndexQt");
00254 bool indexKdelibs = config.readBoolEntry("IndexKdelibs");
00255 bool indexBooks = config.readBoolEntry("IndexBooks");
00256 bool indexBookmarks = config.readBoolEntry("IndexBookmarks");
00257
00258 bool indexShownLibs = true;
00259 bool indexHiddenLibs = true;
00260
00261 filesScanned = 0;
00262
00263 QStringList itemNames, fileNames, hiddenNames;
00264 DocTreeViewTool::getAllLibraries(&itemNames, &fileNames);
00265 DocTreeViewTool::getHiddenLibraries(&hiddenNames);
00266
00267 QStringList::ConstIterator it1, it2;
00268 for (it1 = itemNames.begin(), it2 = fileNames.begin();
00269 it1 != itemNames.end() && it2 != fileNames.end();
00270 ++it1, ++it2) {
00271 bool ishidden = hiddenNames.contains(*it2);
00272 if ( (indexHiddenLibs && ishidden) || (indexShownLibs && !ishidden) ) {
00273 FILE *f;
00274 if ((*it2).right(3) != QString::fromLatin1(".gz")) {
00275 if ( (f = fopen(QFile::encodeName( *it2 ).data(), "r")) != 0) {
00276 addKdocDir(f);
00277 fclose(f);
00278 }
00279 } else {
00280 QString cmd = "gzip -c -d ";
00281 #if (KDE_VERSION > 305)
00282 cmd += KProcess::quote(*it2);
00283 #else
00284 cmd += KShellProcess::quote(*it2);
00285 #endif
00286 cmd += " 2>/dev/null";
00287 if ( (f = popen(QFile::encodeName(cmd), "r")) != 0) {
00288 addKdocDir(f);
00289 pclose(f);
00290 }
00291 }
00292 }
00293 }
00294
00295 if (indexKDevelop) {
00297 }
00298
00299 if (indexQt) {
00300 QString oldqtdocdir;
00301 config.setGroup("General Qt");
00302 QMap<QString, QString> emap = config.entryMap("General Qt");
00303 QMap<QString, QString>::Iterator it;
00304 for (it = emap.begin(); it != emap.end(); ++it)
00305 {
00306 QString qtdocdir = config.readPathEntry(it.key());
00307 if (!qtdocdir.isEmpty())
00308 {
00309 qtdocdir = qtdocdir.left(qtdocdir.findRev("/") + 1);
00310 if (qtdocdir != oldqtdocdir)
00311 {
00312 addDir(qtdocdir);
00313 oldqtdocdir = qtdocdir;
00314 }
00315 }
00316 }
00317 }
00318
00319 if (indexKdelibs) {
00320 config.setGroup("General Doxygen");
00321 QMap<QString, QString> xmap = config.entryMap("General Doxygen");
00322 QMap<QString, QString>::Iterator itx;
00323 for (itx = xmap.begin(); itx != xmap.end(); ++itx)
00324 {
00325 QString kdelibsdocdir = config.readPathEntry(itx.key());
00326 if (!kdelibsdocdir.isEmpty())
00327 addDir(kdelibsdocdir);
00328 }
00329 }
00330
00331 if (indexBooks) {
00332 KStandardDirs *dirs = KGlobal::dirs();
00333 QStringList tocs = dirs->findAllResources("doctocs", QString::null, false, true);
00334
00335 QStringList::ConstIterator it4;
00336 for (it4 = tocs.begin(); it4 != tocs.end(); ++it4) {
00337 QFile f(*it4);
00338 if (!f.open(IO_ReadOnly)) {
00339 kdDebug(9002) << "Could not read doc toc: " << (*it4) << endl;
00340 continue;
00341 }
00342 QDomDocument doc;
00343 if (!doc.setContent(&f) || doc.doctype().name() != "kdeveloptoc") {
00344 kdDebug(9002) << "Not a valid kdeveloptoc file: " << (*it4) << endl;
00345 continue;
00346 }
00347 f.close();
00348 addTocFile(doc);
00349 }
00350 }
00351
00352 if (indexBookmarks) {
00353 QStringList bookmarksTitle, bookmarksURL;
00354 DocTreeViewTool::getBookmarks(&bookmarksTitle, &bookmarksURL);
00355 QStringList::ConstIterator it3;
00356 for (it3 = bookmarksURL.begin(); it3 != bookmarksURL.end(); ++it3) {
00358
00359 files.append(*it3);
00360 setFilesScanned(++filesScanned);
00361 }
00362 }
00363 }
00364
00365
00366 bool ProgressDialog::createConfig()
00367 {
00368
00369 QString language = KGlobal::locale()->language();
00370 if (language == "C")
00371 language = "en";
00372
00373 QString wrapper = locate("data", QString("kdevdoctreeview/%1/wrapper.html").arg(language));
00374 if (wrapper.isEmpty())
00375 wrapper = locate("data", QString("kdevdoctreeview/en/wrapper.html"));
00376 if (wrapper.isEmpty())
00377 return false;
00378 wrapper = wrapper.left(wrapper.length()-12);
00379
00380
00381 QString images = locate("data", "kdevdoctreeview/pics/star.png");
00382 if (images.isEmpty())
00383 return false;
00384 images = images.left(images.length()-8);
00385
00386 QFile f(indexdir + "/htdig.conf");
00387 if (f.open(IO_WriteOnly)) {
00388 QTextStream ts(&f);
00389
00390 ts << "database_dir:\t\t" << databaseDir << endl;
00391 ts << "start_url:\t\t`" << indexdir << "/files`" << endl;
00392 ts << "local_urls:\t\thttp://localhost/=/" << endl;
00393
00394 ts << "local_urls_only:\ttrue" << endl;
00395 ts << "limit_urls_to:\t\tfile://" << endl;
00396 ts << "maximum_pages:\t\t1" << endl;
00397 ts << "image_url_prefix:\t" << images << endl;
00398 ts << "star_image:\t\t" << images << "star.png" << endl;
00399 ts << "star_blank:\t\t" << images << "star_blank.png" << endl;
00400 ts << "compression_level:\t6" << endl;
00401 ts << "max_hop_count:\t\t0" << endl;
00402
00403 ts << "search_results_wrapper:\t" << wrapper << "wrapper.html" << endl;
00404 ts << "nothing_found_file:\t" << wrapper << "nomatch.html" << endl;
00405 ts << "syntax_error_file:\t" << wrapper << "syntax.html" << endl;
00406 ts << "bad_word_list:\t\t" << wrapper << "bad_words" << endl;
00407
00408 f.close();
00409 return true;
00410 }
00411
00412 return false;
00413 }
00414
00415
00416 #define CHUNK_SIZE 100
00417
00418 void ProgressDialog::startHtdigProcess(bool initial)
00419 {
00420 kdDebug(9002) << "htdig started" << endl;
00421 delete proc;
00422 proc = new KProcess();
00423 *proc << exe << "-c" << (indexdir + "/htdig.conf");
00424 if (initial) {
00425 *proc << "-i";
00426 }
00427 connect(proc, SIGNAL(processExited(KProcess *)),
00428 this, SLOT(htdigExited(KProcess *)));
00429
00430 htdigRunning = true;
00431
00432
00433 QFile f(indexdir+"/files");
00434 if (!f.open(IO_WriteOnly)) {
00435 kdDebug(9002) << "Could not open `files` for writing" << endl;
00436 done(1);
00437 return;
00438 }
00439 QTextStream ts(&f);
00440 for (int i=0; i<CHUNK_SIZE; ++i, ++count) {
00441 if (count >= filesToDig) {
00442 procdone = true;
00443 break;
00444 }
00445
00446 ts << "http://localhost/" + files[count] << endl;
00447 }
00448 f.close();
00449
00450
00451 proc->start(KProcess::NotifyOnExit, KProcess::Stdout);
00452
00453 }
00454
00455 bool ProgressDialog::generateIndex()
00456 {
00457 setState(1);
00458 procdone = false;
00459
00460 KConfig config("kdevdoctreeviewrc", true);
00461 config.setGroup("htdig");
00462 exe = config.readPathEntry("htdigbin", kapp->dirs()->findExe("htdig"));
00463 if (exe.isEmpty())
00464 {
00465 done(1);
00466 return true;
00467 }
00468 filesToDig = files.count();
00469 count = 0;
00470 setFilesToDig(filesToDig);
00471 filesDigged = 0;
00472
00473
00474 startHtdigProcess(true);
00475 return true;
00476 }
00477
00478
00479
00480 void ProgressDialog::htdigStdout(KProcess *, char *buffer, int len)
00481 {
00482 QString line = QString(buffer).left(len);
00483
00484 int cnt=0, index=-1;
00485 while ( (index = line.find("http://", index+1)) > 0)
00486 cnt++;
00487 filesDigged += cnt;
00488
00489 cnt=0, index=-1;
00490 while ( (index = line.find("not changed", index+1)) > 0)
00491 cnt++;
00492 filesDigged -= cnt;
00493
00494 setFilesDigged(filesDigged);
00495 }
00496
00497
00498 void ProgressDialog::htdigExited(KProcess *proc)
00499 {
00500 kdDebug(9002) << "htdig terminated" << endl;
00501 if (!proc->normalExit())
00502 {
00503 delete proc;
00504 proc = 0L;
00505 done(1);
00506 return;
00507 }
00508 if (proc && proc->exitStatus() != 0) {
00509 KMessageBox::sorry(0, i18n("Running htdig failed"));
00510 delete proc;
00511 proc = 0L;
00512 done(1);
00513 return;
00514 }
00515 htdigRunning = false;
00516 filesDigged += CHUNK_SIZE;
00517 setFilesDigged(filesDigged);
00518 if (!procdone)
00519 {
00520 startHtdigProcess(false);
00521 } else
00522 {
00523 setFilesDigged(filesToDig);
00524 setState(2);
00525
00526 KConfig config("kdevdoctreeviewrc", true);
00527 config.setGroup("htdig");
00528
00529 exe = config.readPathEntry("htmergebin", kapp->dirs()->findExe("htmerge"));
00530 if (exe.isEmpty())
00531 {
00532 done(1);
00533 return;
00534 }
00535 startHtmergeProcess();
00536 }
00537 }
00538
00539 void ProgressDialog::startHtmergeProcess()
00540 {
00541 kdDebug(9002) << "htmerge started" << endl;
00542 delete proc;
00543 proc = new KProcess();
00544 *proc << exe << "-c" << (indexdir + "/htdig.conf");
00545
00546 kdDebug(9002) << "Running htmerge" << endl;
00547
00548 connect(proc, SIGNAL(processExited(KProcess *)),
00549 this, SLOT(htmergeExited(KProcess *)));
00550
00551 htmergeRunning = true;
00552
00553 proc->start(KProcess::NotifyOnExit, KProcess::Stdout);
00554 }
00555
00556 void ProgressDialog::htmergeExited(KProcess *proc)
00557 {
00558 kdDebug(9002) << "htmerge terminated" << endl;
00559 htmergeRunning = false;
00560 if (!proc->normalExit())
00561 {
00562 delete proc;
00563 proc = 0L;
00564 done(1);
00565 return;
00566 }
00567 if (proc && proc->exitStatus() != 0) {
00568 KMessageBox::sorry(0, i18n("Running htmerge failed"));
00569 delete proc;
00570 proc = 0L;
00571 done(1);
00572 return;
00573 }
00574 setState(3);
00575 done(0);
00576 }
00577
00578 void ProgressDialog::cancelClicked()
00579 {
00580 if ((htdigRunning || htmergeRunning) && proc && proc->isRunning())
00581 {
00582 kdDebug(9002) << "Killing " << (htdigRunning ? "htdig" : "htmerge") << "daemon with Sig. 9" << endl;
00583 proc->kill(9);
00584 htdigRunning = htmergeRunning = false;
00585 } else
00586 {
00587 procdone = true;
00588 done(2);
00589 }
00590 }
00591
00592 void ProgressDialog::okClicked( )
00593 {
00594 if ( proc )
00595 proc->kill( );
00596
00597 KDialogBase::done( 0 );
00598 }
00599
00600 int main(int argc, char *argv[])
00601 {
00602 #if 0
00603 static KCmdLineOptions options[] = {
00604 { "+dirs", I18N_NOOP("The directories to index"), 0 },
00605 KCmdLineLastOption
00606 };
00607 #endif
00608
00609 static const KCmdLineOptions options[] =
00610 {
00611 { "c", I18N_NOOP( "Update user's htdig configuration file only" ), 0 },
00612 { "i", I18N_NOOP( "-c and generate index" ), 0 },
00613 KCmdLineLastOption
00614 };
00615
00616 KAboutData aboutData("kdevdoctreeview", I18N_NOOP("KDevelop"),
00617 "0.1", I18N_NOOP("KDE Index generator for help files."));
00618
00619 KCmdLineArgs::init(argc, argv, &aboutData);
00620 KCmdLineArgs::addCmdLineOptions(options);
00621
00622 KApplication app;
00623
00624 KGlobal::dirs()->addResourceType("doctocs", KStandardDirs::kde_default("data") + "kdevdoctreeview/tocs/");
00625 KGlobal::locale()->setMainCatalogue("kdevelop");
00626
00627 KCmdLineArgs *args = KCmdLineArgs::parsedArgs( );
00628
00629 if( args->isSet( "c" ))
00630 {
00631 ProgressDialog *search = new ProgressDialog( false, 0, "progress dialog");
00632
00633 if( search->createConfig( ))
00634 KMessageBox::information( 0, "Configuration file updated" );
00635 else
00636 KMessageBox::error( 0, "Configuration file update faild!" );
00637 }
00638 else
00639 if( args->isSet( "i" ))
00640 {
00641 ProgressDialog *search = new ProgressDialog( true, 0, "progress dialog");
00642 app.setMainWidget(search);
00643 search->show();
00644 app.exec();
00645 }
00646 else
00647 {
00648 fprintf( stderr, "Internal error generating index - unknown argument\n" );
00649 return 1;
00650 }
00651
00652 return 0;
00653 }
00654
00655 #include "htdigindex.moc"