00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015 #include "htdigindex.h"
00016
00017 #include <iostream>
00018
00019 #include <qapplication.h>
00020 #include <qdir.h>
00021 #include <qfile.h>
00022 #include <qlayout.h>
00023 #include <qtextstream.h>
00024 #include <qtimer.h>
00025 #include <qlabel.h>
00026
00027 #include <kaboutdata.h>
00028 #include <kapplication.h>
00029 #include <kcmdlineargs.h>
00030 #include <kconfig.h>
00031 #include <kdebug.h>
00032 #include <kglobal.h>
00033 #include <klocale.h>
00034 #include <kmessagebox.h>
00035 #include <kstandarddirs.h>
00036 #include <kprocess.h>
00037 #include <kdeversion.h>
00038 #include <kprogress.h>
00039
00040 #define INDEXER
00041
00042 ProgressDialog::ProgressDialog(bool index, QWidget *parent, const char *name)
00043 :KDialogBase(KDialogBase::Plain, i18n("Generating Search Index"), Cancel | Ok, Close,
00044 parent, name, false)
00045 {
00046 proc = 0;
00047
00048 indexdir = kapp->dirs()->saveLocation("data", "kdevdocumentation/search");
00049 QDir d; d.mkdir(indexdir);
00050
00051 KConfig config("kdevdocumentation", true);
00052 config.setGroup("htdig");
00053 databaseDir = config.readPathEntry("databaseDir", indexdir);
00054
00055 if (!index)
00056 return;
00057
00058 d.mkdir( databaseDir );
00059
00060 showButtonOK( false );
00061 QGridLayout *grid = new QGridLayout(plainPage(), 5,3, spacingHint());
00062
00063 QLabel *l = new QLabel(i18n("Scanning for files"), plainPage());
00064 grid->addMultiCellWidget(l, 0, 0, 1, 2);
00065
00066 filesLabel = new QLabel(plainPage());
00067 grid->addWidget(filesLabel, 1, 2);
00068 setFilesScanned(0);
00069
00070 check1 = new QLabel(plainPage());
00071 grid->addWidget(check1, 0, 0);
00072
00073 l = new QLabel(i18n("Extracting search terms"), plainPage());
00074 grid->addMultiCellWidget(l, 2,2, 1,2);
00075
00076 bar = new KProgress(plainPage());
00077 grid->addWidget(bar, 3,2);
00078
00079 check2 = new QLabel(plainPage());
00080 grid->addWidget(check2, 2,0);
00081
00082 l = new QLabel(i18n("Generating index..."), plainPage());
00083 grid->addMultiCellWidget(l, 4,4, 1,2);
00084
00085 check3 = new QLabel(plainPage());
00086 grid->addWidget(check3, 4,0);
00087
00088 setState(0);
00089
00090 setMinimumWidth(300);
00091 connect(this, SIGNAL(cancelClicked()), this, SLOT(cancelClicked()));
00092 connect(this, SIGNAL(okClicked()), this, SLOT(okClicked()));
00093 QTimer::singleShot(0, this, SLOT(slotDelayedStart()));
00094 }
00095
00096 ProgressDialog::~ProgressDialog()
00097 {
00098 }
00099
00100 void ProgressDialog::slotDelayedStart()
00101 {
00102 procdone = false;
00103 scanDirectories();
00104 if (!createConfig())
00105 {
00106 done(1);
00107 return;
00108 }
00109 generateIndex();
00110 }
00111
00112 void ProgressDialog::done(int r)
00113 {
00114 if (!r)
00115 {
00116 showButtonCancel( false );
00117 showButtonOK( true );
00118 }
00119 else
00120 KDialogBase::done(r);
00121 }
00122
00123 void ProgressDialog::setFilesScanned(int n)
00124 {
00125 filesLabel->setText(i18n("Files processed: %1").arg(n));
00126 }
00127
00128 void ProgressDialog::setFilesToDig(int n)
00129 {
00130 bar->setTotalSteps(n);
00131 }
00132
00133 void ProgressDialog::setFilesDigged(int n)
00134 {
00135 bar->setValue(n);
00136 }
00137
00138 void ProgressDialog::setState(int n)
00139 {
00140 QPixmap unchecked = QPixmap(locate("data", "kdevdocumentation/pics/unchecked.xpm"));
00141 QPixmap checked = QPixmap(locate("data", "kdevdocumentation/pics/checked.xpm"));
00142
00143 check1->setPixmap( n > 0 ? checked : unchecked);
00144 check2->setPixmap( n > 1 ? checked : unchecked);
00145 check3->setPixmap( n > 2 ? checked : unchecked);
00146 }
00147
00148
00149 void ProgressDialog::addDir(const QString &dir)
00150 {
00151 kdDebug(9002) << "Add dir : " << dir << endl;
00152 QDir d(dir, "*.html", QDir::Name|QDir::IgnoreCase, QDir::Files | QDir::Readable);
00153 QStringList list = d.entryList();
00154
00155 QStringList::ConstIterator it;
00156 for ( it=list.begin(); it!=list.end(); ++it )
00157 {
00158 if( (*it).right( 12 ).lower( ) == "-source.html" )
00159 continue;
00160
00161 files.append(dir + "/" + *it);
00162 setFilesScanned(++filesScanned);
00163 }
00164
00165 QDir d2(dir, QString::null, QDir::Name|QDir::IgnoreCase, QDir::Dirs);
00166 QStringList dlist = d2.entryList();
00167
00168 for ( it=dlist.begin(); it != dlist.end(); ++it )
00169 {
00170 if (*it != "." && *it != "..")
00171 {
00172 addDir(dir + "/" + *it);
00173 kapp->processEvents();
00174 }
00175 if (procdone)
00176 {
00177 return;
00178 }
00179 }
00180 kapp->processEvents();
00181 }
00182
00183 void ProgressDialog::scanDirectories()
00184 {
00185 QString ftsLocationsFile = locateLocal("data", "kdevdocumentation/search/locations.txt");
00186
00187 QFile f(ftsLocationsFile);
00188 if (!f.open(IO_ReadOnly))
00189 return;
00190 QTextStream str(&f);
00191
00192 filesScanned = 0;
00193
00194 while (!str.eof())
00195 {
00196 QString loc = str.readLine();
00197 if (loc.isEmpty())
00198 continue;
00199 QFileInfo fi(loc);
00200 if (fi.isDir())
00201 addDir(loc);
00202 else if (fi.isFile())
00203 {
00204 files.append(loc);
00205 setFilesScanned(++filesScanned);
00206 }
00207 }
00208 }
00209
00210 bool ProgressDialog::createConfig()
00211 {
00212
00213 QString language = KGlobal::locale()->language();
00214 if (language == "C")
00215 language = "en";
00216
00217 QString wrapper = locate("data", QString("kdevdocumentation/%1/wrapper.html").arg(language));
00218 if (wrapper.isEmpty())
00219 wrapper = locate("data", QString("kdevdocumentation/en/wrapper.html"));
00220 if (wrapper.isEmpty())
00221 return false;
00222 wrapper = wrapper.left(wrapper.length()-12);
00223
00224
00225 QString images = locate("data", "kdevdocumentation/pics/star.png");
00226 if (images.isEmpty())
00227 return false;
00228 images = images.left(images.length()-8);
00229
00230 QFile f(indexdir + "/htdig.conf");
00231 if (f.open(IO_WriteOnly))
00232 {
00233 QTextStream ts(&f);
00234
00235 ts << "database_dir:\t\t" << databaseDir << endl;
00236 ts << "start_url:\t\t`" << indexdir << "/files`" << endl;
00237 ts << "local_urls:\t\thttp://localhost/=/" << endl;
00238
00239 ts << "local_urls_only:\ttrue" << endl;
00240 ts << "limit_urls_to:\t\tfile:// http://localhost/" << endl;
00241 ts << "maximum_pages:\t\t1" << endl;
00242 ts << "image_url_prefix:\t" << images << endl;
00243 ts << "star_image:\t\t" << images << "star.png" << endl;
00244 ts << "star_blank:\t\t" << images << "star_blank.png" << endl;
00245 ts << "compression_level:\t6" << endl;
00246 ts << "max_hop_count:\t\t0" << endl;
00247
00248 ts << "search_results_wrapper:\t" << wrapper << "wrapper.html" << endl;
00249 ts << "nothing_found_file:\t" << wrapper << "nomatch.html" << endl;
00250 ts << "syntax_error_file:\t" << wrapper << "syntax.html" << endl;
00251 ts << "bad_word_list:\t\t" << wrapper << "bad_words" << endl;
00252
00253 f.close();
00254 return true;
00255 }
00256
00257 return false;
00258 }
00259
00260 #define CHUNK_SIZE 100
00261
00262 void ProgressDialog::startHtdigProcess(bool initial)
00263 {
00264 kdDebug(9002) << "htdig started" << endl;
00265 delete proc;
00266 proc = new KProcess();
00267 *proc << exe << "-c" << (indexdir + "/htdig.conf");
00268 if (initial) {
00269 *proc << "-i";
00270 }
00271 connect(proc, SIGNAL(processExited(KProcess *)),
00272 this, SLOT(htdigExited(KProcess *)));
00273
00274 htdigRunning = true;
00275
00276
00277 QFile f(indexdir+"/files");
00278 if (!f.open(IO_WriteOnly)) {
00279 kdDebug(9002) << "Could not open `files` for writing" << endl;
00280 done(1);
00281 return;
00282 }
00283 QTextStream ts(&f);
00284 for (int i=0; i<CHUNK_SIZE; ++i, ++count) {
00285 if (count >= filesToDig) {
00286 procdone = true;
00287 break;
00288 }
00289
00290 ts << "http://localhost/" + files[count] << endl;
00291 }
00292 f.close();
00293
00294
00295 proc->start(KProcess::NotifyOnExit, KProcess::Stdout);
00296 }
00297
00298 bool ProgressDialog::generateIndex()
00299 {
00300 setState(1);
00301 procdone = false;
00302
00303 KConfig config("kdevdocumentation", true);
00304 config.setGroup("htdig");
00305 exe = config.readPathEntry("htdigbin", kapp->dirs()->findExe("htdig"));
00306 if (exe.isEmpty())
00307 {
00308 done(1);
00309 return true;
00310 }
00311 filesToDig = files.count();
00312 count = 0;
00313 setFilesToDig(filesToDig);
00314 filesDigged = 0;
00315
00316
00317 startHtdigProcess(true);
00318 return true;
00319 }
00320
00321 void ProgressDialog::htdigStdout(KProcess *, char *buffer, int len)
00322 {
00323 QString line = QString(buffer).left(len);
00324
00325 int cnt=0, index=-1;
00326 while ( (index = line.find("http://", index+1)) > 0)
00327 cnt++;
00328 filesDigged += cnt;
00329
00330 cnt=0, index=-1;
00331 while ( (index = line.find("not changed", index+1)) > 0)
00332 cnt++;
00333 filesDigged -= cnt;
00334
00335 setFilesDigged(filesDigged);
00336 }
00337
00338 void ProgressDialog::htdigExited(KProcess *proc)
00339 {
00340 kdDebug(9002) << "htdig terminated" << endl;
00341 if (!proc->normalExit())
00342 {
00343 delete proc;
00344 proc = 0L;
00345 done(1);
00346 return;
00347 }
00348 if (proc && proc->exitStatus() != 0)
00349 {
00350 KMessageBox::sorry(0, i18n("Running htdig failed"));
00351 delete proc;
00352 proc = 0L;
00353 done(1);
00354 return;
00355 }
00356 htdigRunning = false;
00357 filesDigged += CHUNK_SIZE;
00358 setFilesDigged(filesDigged);
00359 if (!procdone)
00360 {
00361 startHtdigProcess(false);
00362 }
00363 else
00364 {
00365 setFilesDigged(filesToDig);
00366 setState(2);
00367
00368 KConfig config("kdevdocumentation", true);
00369 config.setGroup("htdig");
00370
00371 exe = config.readPathEntry("htmergebin", kapp->dirs()->findExe("htmerge"));
00372 if (exe.isEmpty())
00373 {
00374 done(1);
00375 return;
00376 }
00377 startHtmergeProcess();
00378 }
00379 }
00380
00381 void ProgressDialog::startHtmergeProcess()
00382 {
00383 kdDebug(9002) << "htmerge started" << endl;
00384 delete proc;
00385 proc = new KProcess();
00386 *proc << exe << "-c" << (indexdir + "/htdig.conf");
00387
00388 kdDebug(9002) << "Running htmerge" << endl;
00389
00390 connect(proc, SIGNAL(processExited(KProcess *)),
00391 this, SLOT(htmergeExited(KProcess *)));
00392
00393 htmergeRunning = true;
00394
00395 proc->start(KProcess::NotifyOnExit, KProcess::Stdout);
00396 }
00397
00398 void ProgressDialog::htmergeExited(KProcess *proc)
00399 {
00400 kdDebug(9002) << "htmerge terminated" << endl;
00401 htmergeRunning = false;
00402 if (!proc->normalExit())
00403 {
00404 delete proc;
00405 proc = 0L;
00406 done(1);
00407 return;
00408 }
00409 if (proc && proc->exitStatus() != 0)
00410 {
00411 KMessageBox::sorry(0, i18n("Running htmerge failed"));
00412 delete proc;
00413 proc = 0L;
00414 done(1);
00415 return;
00416 }
00417 setState(3);
00418 done(0);
00419 }
00420
00421 void ProgressDialog::cancelClicked()
00422 {
00423 if ((htdigRunning || htmergeRunning) && proc && proc->isRunning())
00424 {
00425 kdDebug(9002) << "Killing " << (htdigRunning ? "htdig" : "htmerge") << "daemon with Sig. 9" << endl;
00426 proc->kill(9);
00427 htdigRunning = htmergeRunning = false;
00428 }
00429 else
00430 {
00431 procdone = true;
00432 done(2);
00433 }
00434 }
00435
00436 void ProgressDialog::okClicked()
00437 {
00438 if (proc)
00439 proc->kill();
00440
00441 KDialogBase::done(0);
00442 }
00443
00444 int main(int argc, char *argv[])
00445 {
00446 static const KCmdLineOptions options[] =
00447 {
00448 { "c", I18N_NOOP( "Update user's htdig configuration file only" ), 0 },
00449 { "i", I18N_NOOP( "-c and generate index" ), 0 },
00450 KCmdLineLastOption
00451 };
00452
00453 KAboutData aboutData("kdevelop-htdig", I18N_NOOP("KDevelop ht://Dig Indexer"),
00454 "0.2", I18N_NOOP("KDE Index generator for documentation files."));
00455
00456 KCmdLineArgs::init(argc, argv, &aboutData);
00457 KCmdLineArgs::addCmdLineOptions(options);
00458
00459 KApplication app;
00460
00461 KGlobal::locale()->setMainCatalogue("kdevelop");
00462
00463 KCmdLineArgs *args = KCmdLineArgs::parsedArgs();
00464
00465 if (args->isSet("c"))
00466 {
00467 ProgressDialog *search = new ProgressDialog( false, 0, "progress dialog");
00468
00469 if (search->createConfig())
00470 KMessageBox::information(0, i18n("Configuration file updated"));
00471 else
00472 KMessageBox::error(0, i18n("Configuration file update failed."));
00473 }
00474 else
00475 if (args->isSet("i"))
00476 {
00477 ProgressDialog *search = new ProgressDialog(true, 0, "progress dialog");
00478 app.setMainWidget(search);
00479 search->show();
00480 app.exec();
00481 }
00482 else
00483 {
00484 std::cerr << "Internal error generating index - unknown argument\n" << std::endl;
00485 return 1;
00486 }
00487
00488 return 0;
00489 }
00490
00491 #include "htdigindex.moc"