00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
#include "htdigindex.h"
00015
00016
#include <qapplication.h>
00017
#include <qdir.h>
00018
#include <qfile.h>
00019
#include <qlayout.h>
00020
#include <qtextstream.h>
00021
#include <qtimer.h>
00022
#include <kaboutdata.h>
00023
#include <kapplication.h>
00024
#include <kcmdlineargs.h>
00025
#include <kconfig.h>
00026
#include <kdebug.h>
00027
#include <kglobal.h>
00028
#include <klocale.h>
00029
#include <kmessagebox.h>
00030
#include <kstandarddirs.h>
00031
#include <kprocess.h>
00032
#include <kdeversion.h>
00033
00034 #define INDEXER
00035
#include "misc.cpp"
00036
00037
00038 ProgressDialog::ProgressDialog(
QWidget *parent,
const char *name)
00039 :
KDialogBase(
KDialogBase::Plain, i18n("Generating Search Index"), Cancel, Cancel,
00040 parent, name, false)
00041 {
00042
proc = 0;
00043
00044
indexdir = kapp->dirs()->saveLocation(
"data",
"kdevdoctreeview/helpindex");
00045
QDir d; d.mkdir(
indexdir);
00046
00047
QGridLayout *grid =
new QGridLayout(
plainPage(), 5,3,
spacingHint());
00048
00049
QLabel *l =
new QLabel(i18n(
"Scanning for files"),
plainPage());
00050 grid->addMultiCellWidget(l, 0, 0, 1, 2);
00051
00052
filesLabel =
new QLabel(
plainPage());
00053 grid->addWidget(
filesLabel, 1, 2);
00054
setFilesScanned(0);
00055
00056
check1 =
new QLabel(
plainPage());
00057 grid->addWidget(
check1, 0, 0);
00058
00059 l =
new QLabel(i18n(
"Extracting search terms"),
plainPage());
00060 grid->addMultiCellWidget(l, 2,2, 1,2);
00061
00062
bar =
new KProgress(
plainPage());
00063 grid->addWidget(
bar, 3,2);
00064
00065
check2 =
new QLabel(
plainPage());
00066 grid->addWidget(
check2, 2,0);
00067
00068 l =
new QLabel(i18n(
"Generating index..."),
plainPage());
00069 grid->addMultiCellWidget(l, 4,4, 1,2);
00070
00071
check3 =
new QLabel(
plainPage());
00072 grid->addWidget(
check3, 4,0);
00073
00074
setState(0);
00075
00076 setMinimumWidth(300);
00077 connect(
this, SIGNAL(
cancelClicked()),
this, SLOT(
cancelClicked()));
00078 QTimer::singleShot(0,
this, SLOT(
slotDelayedStart()));
00079 }
00080
00081
00082 ProgressDialog::~ProgressDialog()
00083 {}
00084
00085 void ProgressDialog::slotDelayedStart()
00086 {
00087
procdone =
false;
00088
scanDirectories();
00089
if (!
createConfig())
00090 {
00091
done(1);
00092
return;
00093 }
00094
generateIndex();
00095 }
00096
00097
00098 void ProgressDialog::done(
int r)
00099 {
00100
if (!r &&
proc)
00101
proc->
kill();
00102 KDialogBase::done(r);
00103 }
00104
00105
00106 void ProgressDialog::setFilesScanned(
int n)
00107 {
00108
filesLabel->setText(i18n(
"Files processed: %1").arg(n));
00109 }
00110
00111
00112 void ProgressDialog::setFilesToDig(
int n)
00113 {
00114
bar->
setRange(0, n);
00115 }
00116
00117
00118 void ProgressDialog::setFilesDigged(
int n)
00119 {
00120
bar->
setValue(n);
00121 }
00122
00123
00124 void ProgressDialog::setState(
int n)
00125 {
00126
QPixmap unchecked =
QPixmap(
locate(
"data",
"kdevdoctreeview/pics/unchecked.xpm"));
00127 QPixmap checked = QPixmap(
locate(
"data",
"kdevdoctreeview/pics/checked.xpm"));
00128
00129
check1->setPixmap( n > 0 ? checked : unchecked);
00130
check2->setPixmap( n > 1 ? checked : unchecked);
00131
check3->setPixmap( n > 2 ? checked : unchecked);
00132 }
00133
00134
00135 void ProgressDialog::addDir(
const QString &dir)
00136 {
00137
kdDebug(9002) <<
"Add dir : " << dir <<
endl;
00138
QDir d(dir,
"*.html", QDir::Name|QDir::IgnoreCase, QDir::Files | QDir::Readable);
00139
QStringList list = d.entryList();
00140
00141 QStringList::ConstIterator it;
00142
for ( it=list.begin(); it!=list.end(); ++it ) {
00143
files.append(dir +
"/" + *it);
00144
setFilesScanned(++
filesScanned);
00145 }
00146
00147
QDir d2(dir, QString::null, QDir::Name|QDir::IgnoreCase, QDir::Dirs);
00148
QStringList dlist = d2.entryList();
00149
00150
for ( it=dlist.begin(); it != dlist.end(); ++it ) {
00151
if (*it !=
"." && *it !=
"..") {
00152
addDir(dir +
"/" + *it);
00153 kapp->processEvents();
00154 }
00155
if (
procdone)
00156 {
00157
return;
00158 }
00159 }
00160 kapp->processEvents();
00161 }
00162
00163
00164 void ProgressDialog::addKdocDir(FILE *f)
00165 {
00166
char buf[1024];
00167
while (fgets(buf,
sizeof buf, f)) {
00168
QString s = buf;
00169
if (s.left(11) ==
"<BASE URL=\"") {
00170
int pos2 = s.find(
"\">", 11);
00171
if (pos2 != -1) {
00172
addDir(s.mid(11, pos2-11));
00173
return;
00174 }
00175 }
00176 }
00177 }
00178
00179
00180 void ProgressDialog::addTocFile(
QDomDocument &doc)
00181 {
00182
QStringList candidates;
00183
QString base;
00184
QDomElement childEl = doc.documentElement().firstChild().toElement();
00185
while (!childEl.isNull()) {
00186
if (childEl.tagName() ==
"tocsect1") {
00187
QString url = childEl.attribute(
"url");
00188
if (!url.isEmpty()) {
00189 url.prepend(base);
00190
kdDebug(9002) <<
"candidate: " << url <<
endl;
00191 candidates.append(url);
00192 }
00194
QDomElement grandchildEl = childEl.firstChild().toElement();
00195
while (!grandchildEl.isNull()) {
00196
if (grandchildEl.tagName() ==
"tocsect2") {
00197
QString url = grandchildEl.attribute(
"url");
00198
if (!url.isEmpty()) {
00199 url.prepend(base);
00200
kdDebug(9002) <<
"candidate: " << url <<
endl;
00201 candidates.append(url);
00202 }
00203 }
00204 grandchildEl = grandchildEl.nextSibling().toElement();
00205 }
00206 }
else if (childEl.tagName() ==
"base") {
00207 base = childEl.attribute(
"href");
00208
if (!base.isEmpty())
00209 base +=
"/";
00210 }
00211 childEl = childEl.nextSibling().toElement();
00212 }
00213
00214 QStringList::ConstIterator it;
00215
for (it = candidates.begin(); it != candidates.end(); ++it) {
00216
QString url = *it;
00217
int pos = url.findRev(
'#');
00218
if (pos != -1)
00219 url.truncate(pos);
00220
if ((url.startsWith(
"/") || url.startsWith(
"file://"))
00221 && !
files.contains(url)) {
00222
files.append(url);
00223
kdDebug(9002) <<
"tocurl: " << url <<
endl;
00224
setFilesScanned(++
filesScanned);
00225 }
00226 }
00227 }
00228
00229
00230 void ProgressDialog::scanDirectories()
00231 {
00232
KConfig config(
"kdevdoctreeviewrc",
true);
00233 config.
setGroup(
"Index");
00234
bool indexKDevelop = config.
readBoolEntry(
"IndexKDevelop");
00235
bool indexQt = config.
readBoolEntry(
"IndexQt");
00236
bool indexKdelibs = config.
readBoolEntry(
"IndexKdelibs");
00237
bool indexBooks = config.
readBoolEntry(
"IndexBooks");
00238
bool indexBookmarks = config.
readBoolEntry(
"IndexBookmarks");
00239
00240
bool indexShownLibs =
true;
00241
bool indexHiddenLibs =
true;
00242
00243
filesScanned = 0;
00244
00245
QStringList itemNames, fileNames, hiddenNames;
00246 DocTreeViewTool::getAllLibraries(&itemNames, &fileNames);
00247 DocTreeViewTool::getHiddenLibraries(&hiddenNames);
00248
00249 QStringList::ConstIterator it1, it2;
00250
for (it1 = itemNames.begin(), it2 = fileNames.begin();
00251 it1 != itemNames.end() && it2 != fileNames.end();
00252 ++it1, ++it2) {
00253
bool ishidden = hiddenNames.contains(*it2);
00254
if ( (indexHiddenLibs && ishidden) || (indexShownLibs && !ishidden) ) {
00255 FILE *f;
00256
if ((*it2).right(3) != QString::fromLatin1(
".gz")) {
00257
if ( (f = fopen(QFile::encodeName( *it2 ).data(),
"r")) != 0) {
00258
addKdocDir(f);
00259 fclose(f);
00260 }
00261 }
else {
00262
QString cmd =
"gzip -c -d ";
00263
#if (KDE_VERSION > 305)
00264
cmd += KProcess::quote(*it2);
00265
#else
00266
cmd += KShellProcess::quote(*it2);
00267
#endif
00268
cmd +=
" 2>/dev/null";
00269
if ( (f = popen(QFile::encodeName(cmd),
"r")) != 0) {
00270
addKdocDir(f);
00271 pclose(f);
00272 }
00273 }
00274 }
00275 }
00276
00277
if (indexKDevelop) {
00279 }
00280
00281
if (indexQt) {
00282
QString oldqtdocdir;
00283 config.
setGroup(
"General Qt");
00284
QMap<QString, QString> emap = config.
entryMap(
"General Qt");
00285
QMap<QString, QString>::Iterator it;
00286
for (it = emap.begin(); it != emap.end(); ++it)
00287 {
00288
QString qtdocdir = config.
readPathEntry(it.key());
00289
if (!qtdocdir.isEmpty())
00290 {
00291 qtdocdir = qtdocdir.left(qtdocdir.findRev(
"/") + 1);
00292
if (qtdocdir != oldqtdocdir)
00293 {
00294
addDir(qtdocdir);
00295 oldqtdocdir = qtdocdir;
00296 }
00297 }
00298 }
00299 }
00300
00301
if (indexKdelibs) {
00302 config.
setGroup(
"General Doxygen");
00303
QMap<QString, QString> xmap = config.
entryMap(
"General Doxygen");
00304
QMap<QString, QString>::Iterator itx;
00305
for (itx = xmap.begin(); itx != xmap.end(); ++itx)
00306 {
00307
QString kdelibsdocdir = config.
readPathEntry(itx.key());
00308
if (!kdelibsdocdir.isEmpty())
00309
addDir(kdelibsdocdir);
00310 }
00311 }
00312
00313
if (indexBooks) {
00314
KStandardDirs *dirs =
KGlobal::dirs();
00315
QStringList tocs = dirs->
findAllResources(
"doctocs", QString::null,
false,
true);
00316
00317 QStringList::ConstIterator it4;
00318
for (it4 = tocs.begin(); it4 != tocs.end(); ++it4) {
00319
QFile f(*it4);
00320
if (!f.open(IO_ReadOnly)) {
00321
kdDebug(9002) <<
"Could not read doc toc: " << (*it4) <<
endl;
00322
continue;
00323 }
00324
QDomDocument doc;
00325
if (!doc.setContent(&f) || doc.doctype().name() !=
"kdeveloptoc") {
00326
kdDebug(9002) <<
"Not a valid kdeveloptoc file: " << (*it4) <<
endl;
00327
continue;
00328 }
00329 f.close();
00330
addTocFile(doc);
00331 }
00332 }
00333
00334
if (indexBookmarks) {
00335
QStringList bookmarksTitle, bookmarksURL;
00336 DocTreeViewTool::getBookmarks(&bookmarksTitle, &bookmarksURL);
00337 QStringList::ConstIterator it3;
00338
for (it3 = bookmarksURL.begin(); it3 != bookmarksURL.end(); ++it3) {
00340
00341
files.append(*it3);
00342
setFilesScanned(++
filesScanned);
00343 }
00344 }
00345 }
00346
00347
00348 bool ProgressDialog::createConfig()
00349 {
00350
00351
QString language =
KGlobal::locale()->
language();
00352
if (language ==
"C")
00353 language =
"en";
00354
00355
QString wrapper =
locate(
"data",
QString(
"kdevdoctreeview/%1/wrapper.html").arg(language));
00356
if (wrapper.isEmpty())
00357 wrapper =
locate(
"data",
QString(
"kdevdoctreeview/en/wrapper.html"));
00358
if (wrapper.isEmpty())
00359
return false;
00360 wrapper = wrapper.left(wrapper.length()-12);
00361
00362
00363
QString images =
locate(
"data",
"kdevdoctreeview/pics/star.png");
00364
if (images.isEmpty())
00365
return false;
00366 images = images.left(images.length()-8);
00367
00368
QFile f(
indexdir +
"/htdig.conf");
00369
if (f.open(IO_WriteOnly)) {
00370
QTextStream ts(&f);
00371
00372 ts <<
"database_dir:\t\t" <<
indexdir <<
endl;
00373 ts <<
"start_url:\t\t`" <<
indexdir <<
"/files`" <<
endl;
00374 ts <<
"local_urls:\t\thttp://localhost/=/" <<
endl;
00375
00376 ts <<
"local_urls_only:\ttrue" <<
endl;
00377 ts <<
"limit_urls_to:\t\tfile:// http://localhost/" <<
endl;
00378 ts <<
"maximum_pages:\t\t1" <<
endl;
00379 ts <<
"image_url_prefix:\t" << images <<
endl;
00380 ts <<
"star_image:\t\t" << images <<
"star.png" <<
endl;
00381 ts <<
"star_blank:\t\t" << images <<
"star_blank.png" <<
endl;
00382 ts <<
"compression_level:\t6" <<
endl;
00383 ts <<
"max_hop_count:\t\t0" <<
endl;
00384
00385 ts <<
"search_results_wrapper:\t" << wrapper <<
"wrapper.html" <<
endl;
00386 ts <<
"nothing_found_file:\t" << wrapper <<
"nomatch.html" <<
endl;
00387 ts <<
"syntax_error_file:\t" << wrapper <<
"syntax.html" <<
endl;
00388 ts <<
"bad_word_list:\t\t" << wrapper <<
"bad_words" <<
endl;
00389
00390 f.close();
00391
return true;
00392 }
00393
00394
return false;
00395 }
00396
00397
00398 #define CHUNK_SIZE 100
00399
00400 void ProgressDialog::startHtdigProcess(
bool initial)
00401 {
00402
kdDebug(9002) <<
"htdig started" <<
endl;
00403
delete proc;
00404
proc =
new KProcess();
00405 *
proc <<
exe <<
"-c" << (
indexdir +
"/htdig.conf");
00406
if (initial) {
00407 *
proc <<
"-i";
00408 }
00409 connect(
proc, SIGNAL(processExited(KProcess *)),
00410
this, SLOT(
htdigExited(KProcess *)));
00411
00412
htdigRunning =
true;
00413
00414
00415
QFile f(
indexdir+
"/files");
00416
if (!f.open(IO_WriteOnly)) {
00417
kdDebug(9002) <<
"Could not open `files` for writing" <<
endl;
00418
done(1);
00419
return;
00420 }
00421
QTextStream ts(&f);
00422
for (
int i=0; i<
CHUNK_SIZE; ++i, ++
count) {
00423
if (
count >=
filesToDig) {
00424
procdone =
true;
00425
break;
00426 }
00427
00428 ts <<
"http://localhost/" +
files[
count] <<
endl;
00429 }
00430 f.close();
00431
00432
00433
proc->
start(KProcess::NotifyOnExit, KProcess::Stdout);
00434
00435 }
00436
00437 bool ProgressDialog::generateIndex()
00438 {
00439
setState(1);
00440
procdone =
false;
00441
00442
KConfig config(
"kdevdoctreeviewrc",
true);
00443 config.
setGroup(
"htdig");
00444
exe = config.
readPathEntry(
"htdigbin", kapp->dirs()->findExe(
"htdig"));
00445
if (
exe.isEmpty())
00446 {
00447
done(1);
00448
return true;
00449 }
00450
filesToDig =
files.count();
00451
count = 0;
00452
setFilesToDig(
filesToDig);
00453
filesDigged = 0;
00454
00455
00456
startHtdigProcess(
true);
00457
return true;
00458 }
00459
00460
00461
00462 void ProgressDialog::htdigStdout(
KProcess *,
char *buffer,
int len)
00463 {
00464
QString line =
QString(buffer).left(
len);
00465
00466
int cnt=0, index=-1;
00467
while ( (index = line.find(
"http://", index+1)) > 0)
00468 cnt++;
00469
filesDigged += cnt;
00470
00471 cnt=0, index=-1;
00472
while ( (index = line.find(
"not changed", index+1)) > 0)
00473 cnt++;
00474
filesDigged -= cnt;
00475
00476
setFilesDigged(
filesDigged);
00477 }
00478
00479
00480 void ProgressDialog::htdigExited(
KProcess *proc)
00481 {
00482
kdDebug(9002) <<
"htdig terminated" <<
endl;
00483
if (!proc->
normalExit())
00484 {
00485
delete proc;
00486 proc = 0L;
00487
done(1);
00488
return;
00489 }
00490
if (proc && proc->
exitStatus() != 0) {
00491 KMessageBox::sorry(0, i18n(
"Running htdig failed"));
00492
delete proc;
00493 proc = 0L;
00494
done(1);
00495
return;
00496 }
00497
htdigRunning =
false;
00498
filesDigged +=
CHUNK_SIZE;
00499
setFilesDigged(
filesDigged);
00500
if (!
procdone)
00501 {
00502
startHtdigProcess(
false);
00503 }
else
00504 {
00505
setFilesDigged(
filesToDig);
00506
setState(2);
00507
00508
KConfig config(
"kdevdoctreeviewrc",
true);
00509 config.
setGroup(
"htdig");
00510
00511
exe = config.
readPathEntry(
"htmergebin", kapp->dirs()->findExe(
"htmerge"));
00512
if (
exe.isEmpty())
00513 {
00514
done(1);
00515
return;
00516 }
00517
startHtmergeProcess();
00518 }
00519 }
00520
00521 void ProgressDialog::startHtmergeProcess()
00522 {
00523
kdDebug(9002) <<
"htmerge started" <<
endl;
00524
delete proc;
00525
proc =
new KProcess();
00526 *
proc <<
exe <<
"-c" << (
indexdir +
"/htdig.conf");
00527
00528
kdDebug(9002) <<
"Running htmerge" <<
endl;
00529
00530 connect(
proc, SIGNAL(processExited(KProcess *)),
00531
this, SLOT(
htmergeExited(KProcess *)));
00532
00533
htmergeRunning =
true;
00534
00535
proc->
start(KProcess::NotifyOnExit, KProcess::Stdout);
00536 }
00537
00538 void ProgressDialog::htmergeExited(
KProcess *proc)
00539 {
00540
kdDebug(9002) <<
"htmerge terminated" <<
endl;
00541
htmergeRunning =
false;
00542
if (!proc->
normalExit())
00543 {
00544
delete proc;
00545 proc = 0L;
00546
done(1);
00547
return;
00548 }
00549
if (proc && proc->
exitStatus() != 0) {
00550 KMessageBox::sorry(0, i18n(
"Running htmerge failed"));
00551
delete proc;
00552 proc = 0L;
00553
done(1);
00554
return;
00555 }
00556
setState(3);
00557
done(0);
00558 }
00559
00560 void ProgressDialog::cancelClicked()
00561 {
00562
if ((
htdigRunning ||
htmergeRunning) &&
proc &&
proc->
isRunning())
00563 {
00564
kdDebug(9002) <<
"Killing " << (
htdigRunning ?
"htdig" :
"htmerge") <<
"daemon with Sig. 9" <<
endl;
00565
proc->
kill(9);
00566
htdigRunning =
htmergeRunning =
false;
00567 }
else
00568 {
00569
procdone =
true;
00570
done(2);
00571 }
00572 }
00573
00574 int main(
int argc,
char *argv[])
00575 {
00576
#if 0
00577
static KCmdLineOptions options[] = {
00578 {
"+dirs",
I18N_NOOP(
"The directories to index."), 0 },
00579 KCmdLineLastOption
00580 };
00581
#endif
00582
00583
KAboutData aboutData(
"kdevdoctreeview",
I18N_NOOP(
"KDevelop"),
00584
"0.1",
I18N_NOOP(
"KDE Index generator for help files."));
00585
00586 KCmdLineArgs::init(argc, argv, &aboutData);
00587
00588
00589
KApplication app;
00590
00591
KGlobal::dirs()->
addResourceType(
"doctocs", KStandardDirs::kde_default(
"data") +
"kdevdoctreeview/tocs/");
00592
KGlobal::locale()->
setMainCatalogue(
"kdevelop");
00593
00594
ProgressDialog *search =
new ProgressDialog(0,
"progress dialog");
00595 app.setMainWidget(search);
00596 search->show();
00597 app.exec();
00598
00599
return 0;
00600 }
00601
00602
#include "htdigindex.moc"