kdecore Library API Documentation

kcharsets.cpp

00001 /* This file is part of the KDE libraries
00002     Copyright (C) 1999 Lars Knoll (knoll@kde.org)
00003     $Id: kcharsets.cpp,v 1.134.2.3 2003/06/29 21:42:59 deller Exp $
00004 
00005     This library is free software; you can redistribute it and/or
00006     modify it under the terms of the GNU Library General Public
00007     License as published by the Free Software Foundation; either
00008     version 2 of the License, or (at your option) any later version.
00009 
00010     This library is distributed in the hope that it will be useful,
00011     but WITHOUT ANY WARRANTY; without even the implied warranty of
00012     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00013     Library General Public License for more details.
00014 
00015     You should have received a copy of the GNU Library General Public License
00016     along with this library; see the file COPYING.LIB.  If not, write to
00017     the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
00018     Boston, MA 02111-1307, USA.
00019 */
00020 #include "kcharsets.h"
00021 
00022 #include "kentities.c"
00023 
00024 #include <kapplication.h>
00025 #include <kglobal.h>
00026 #include <klocale.h>
00027 #include <kconfig.h>
00028 
00029 #include <qfontinfo.h>
00030 #include <qstrlist.h>
00031 #include <qfontdatabase.h>
00032 #include <kdebug.h>
00033 
00034 #include <qtextcodec.h>
00035 #include <qmap.h>
00036 #include <qcstring.h>
00037 
00038 #include <assert.h>
00039 
00040 #define CHARSETS_COUNT 33
00041 
00042 static const char * const language_names[] = {
00043     I18N_NOOP( "Other" ),
00044     I18N_NOOP( "Arabic" ),
00045     I18N_NOOP( "Baltic" ),
00046     I18N_NOOP( "Central European" ),
00047     I18N_NOOP( "Chinese Simplified" ),
00048     I18N_NOOP( "Chinese Traditional" ),
00049     I18N_NOOP( "Cyrillic" ),
00050     I18N_NOOP( "Greek" ),
00051     I18N_NOOP( "Hebrew" ),
00052     I18N_NOOP( "Japanese" ),
00053     I18N_NOOP( "Korean" ),
00054     I18N_NOOP( "Thai" ),
00055     I18N_NOOP( "Turkish" ),
00056     I18N_NOOP( "Western European" ),
00057     I18N_NOOP( "Tamil" ),
00058     I18N_NOOP( "Unicode" ),
00059     I18N_NOOP( "Northern Saami" )
00060 };
00061 
00062 // this list gives the charsets that can be used to display a file given in a certain encoding.
00063 // the list should be in order of preference
00064 // left side is the name returned by the codec used, right side the name of the charset as
00065 // used in kcharsets.cpp
00066 // 'unicode' will always be chosen as last resort, so it only needs to be added to the list,
00067 // if it should get a higher priority
00068 // every line must end with 0
00069 
00070 static const char* const charsets_for_encoding[] = {
00071     "koi8-r",                "koi8-r","cp 1251","koi8-u","iso-8859-5", 0,
00072     "koi8-u",                "koi8-u","cp 1251","iso-8859-5","koi8-r", 0,
00073     "iso 8859-1",            "iso8859-1","iso8859-15", 0,
00074     "iso 8859-2",            "iso8859-2","unicode","iso8859-1", 0,
00075     "iso 8859-3",            "iso8859-3","unicode","iso8859-1", 0,
00076     "iso 8859-4",            "iso8859-4","unicode","iso8859-13", "iso8859-1", 0,
00077     "iso 8859-5",            "iso8859-5","koi8-u","koi8-r", 0,
00078     "iso 8859-6",            "unicode","iso8859-6", 0,
00079     "iso 8859-7",            "iso8859-7", 0,
00080     "iso 8859-8",            "iso8859-8", 0,
00081     "iso 8859-8-i",          "iso8859-8", 0,
00082     "iso 8859-9",            "iso8859-9","unicode","iso8859-1", 0,
00083     "iso 8859-11",           "iso8859-11", 0,
00084     "iso 8859-13",           "iso8859-13","unicode","iso8859-4", "iso8859-1", 0,
00085     "iso 8859-15",           "iso8859-15","unicode","iso8859-1", 0,
00086     "utf8",                  "unicode","iso8859-1", 0,
00087     "utf16",                 "unicode","iso8859-1", 0,
00088     "iso-10646-ucs-2",       "unicode","iso8859-1", 0,
00089     "cp 1250",               "iso8859-2", 0,
00090     "cp 1251",               "cp 1251","koi8-u","koi8-r","iso8859-5", 0,
00091     "cp 1252",               "iso8859-1", 0,
00092     "cp 1253",               "iso8859-7", 0,
00093     "cp 1254",               "iso8859-9", 0,
00094     "cp 1255",               "iso8859-8", 0,
00095     "cp 1256",               "unicode","iso8859-6", 0,
00096     "cp 1257",               "iso8859-13", "iso8859-4", 0,
00097     "ibm852",                "unicode","iso-8859-2", 0,
00098     "ibm866",                "ibm866","koi8-r","cp 1251","koi8-u","iso-8859-5", 0,
00099     "tis620",                "iso8859-11", 0,
00100     "eucjp",                 "eucjp","unicode","iso8859-1", 0,
00101     "sjis",                  "eucjp","unicode","iso8859-1", 0,
00102     "jis7",                  "eucjp","unicode","iso8859-1", 0,
00103     "big5",                  "big5","unicode","iso8859-1", 0,
00104     "gbk",                   "gb2312.1980-0","gbk-0","unicode","iso8859-1", 0,
00105     "gb18030",               "gb18030.2000-1", "gb18030.2000-0", "unicode", "gbk-0", "gb2313.1980-0", "iso8859-1", 0,
00106     "gb2312",                "gb2312.1980-0","unicode","iso8859-1", 0,
00107     "euckr",                 "euckr","unicode","iso8859-1", 0,
00108     "tscii",                 "tscii", 0,
00109     "pt 154",                "pt 154","cp 1251","koi8-u","koi8-r","iso8859-5", 0,
00110     "winsami2",              "winsami2", "cp1252", "unicode", 0,
00111     0 }; // extra 0 for end
00112 
00113 // 0 other
00114 // 1 Arabic
00115 // 2 Baltic
00116 // 3 Central European
00117 // 4 Chinese Simplified
00118 // 5 Chinese Traditional
00119 // 6 Cyrillic
00120 // 7 Greek
00121 // 8 Hebrew
00122 // 9 Japanese
00123 // 10 Korean
00124 // 11 Thai
00125 // 12 Turkish
00126 // 13 Western European
00127 // 14 Tamil
00128 // 15 Unicode
00129 // 16 Northern Sami
00130 static struct LanguageForEncoding
00131     {
00132     const char* index;
00133     int data;
00134     } const language_for_encoding[] = {
00135     { "iso 8859-1", 13 },
00136     { "iso 8859-15", 13 },
00137     { "cp 1252", 13 },
00138     { "iso 8859-2", 3 },
00139     { "iso 8859-3", 3 },
00140     { "iso 8859-4", 2 },
00141     { "iso 8859-13", 2 },
00142     { "cp 1250", 3 },
00143     { "cp 1254", 12 },
00144     { "cp 1257", 2 },
00145     { "ibm852", 3 },
00146     { "ibm866", 6 },
00147     { "koi8-r", 6 },
00148     { "iso 8859-5", 6 },
00149     { "cp 1251", 6 },
00150     { "koi8-u", 6 },
00151     { "pt 154", 6 },
00152     { "big5", 5 },
00153     { "gb18030", 4 },
00154     { "gbk", 4 },
00155     { "gb2312", 4 },
00156     { "euckr", 10 },
00157     { "sjis", 9 },
00158     { "jis7", 9 },
00159     { "eucjp", 9 },
00160     { "iso 8859-7", 7 },
00161     { "cp 1253", 7 },
00162     { "iso 8859-6", 1 },
00163     { "cp 1256", 1 },
00164     { "iso 8859-8", 8 },
00165     { "iso 8859-8-i", 8 },
00166     { "cp 1255", 8 },
00167     { "iso 8859-9", 12 },
00168     { "tis620", 11 },
00169     { "iso 8859-11", 11 },
00170     { "utf8", 15 },
00171     { "utf16", 15 },
00172     { "utf7", 15 },
00173     { "ucs2", 15 },
00174     { "iso-10646-ucs-2", 15 },
00175     { "winsami2", 16},
00176     { 0, 0 } };
00177 
00178 // defines some different names for codecs that are built into Qt.
00179 static struct Builtin
00180     {
00181     const char* index;
00182     const char* data;
00183     } const builtin[] = {
00184     { "iso-ir-111", "koi8-r" },
00185     { "koi8-ru", "koi8-u" },
00186     { "koi8r", "koi8-r" },
00187     { "koi8u", "koi8-u" },
00188     { "koi8u", "koi8-u" },
00189     { "koi unified", "koi8-r" },
00190     { "us-ascii", "iso 8859-1" },
00191     { "usascii", "iso 8859-1" },
00192     { "x-utf-8", "utf-8" },
00193     { "x-utf-7", "utf-7" },
00194     { "unicode-1-1-utf-7", "utf-7" },
00195     { "ucs2", "iso-10646-ucs-2" },
00196     { "iso10646-1", "iso-10646-ucs-2" },
00197     { "big5-0", "big5" },
00198     { "gb18030.2000-1", "gb18030" },
00199     { "gb18030.2000-0", "gb18030" },
00200     { "gbk-0", "gbk" },
00201     { "gb2312", "gbk" },
00202     { "gb2312.1980-0", "gbk" },
00203     { "euc-kr", "euckr" },
00204     { "x-euc-kr", "euckr" },
00205     { "euc-jp", "eucjp" },
00206     { "x-euc-jp", "eucjp" },
00207     { "jisx0201.1976-0", "eucjp" },
00208     { "jisx0208.1983-0", "eucjp" },
00209     { "jisx0208.1990-0", "eucjp" },
00210     { "jisx0208.1997-0", "eucjp" },
00211     { "jisx0212.1990-0", "eucjp" },
00212     { "jisx0213.2000-1", "eucjp" },
00213     { "jisx0213.2000-2", "eucjp" },
00214     { "shift_jis", "sjis" },
00215     { "shift-jis", "sjis" },
00216     { "x-sjis", "sjis" },
00217     { "iso-2022-jp", "jis7" },
00218     { "windows1250", "cp 1250" },
00219     { "windows1251", "cp 1251" },
00220     { "windows1252", "cp 1252" },
00221     { "windows1253", "cp 1253" },
00222     { "windows1254", "cp 1254" },
00223     { "windows1255", "cp 1255" },
00224     { "windows1256", "cp 1256" },
00225     { "windows1257", "cp 1257" },
00226     { "windows-1250", "cp 1250" },
00227     { "windows-1251", "cp 1251" },
00228     { "windows-1252", "cp 1252" },
00229     { "windows-1253", "cp 1253" },
00230     { "windows-1254", "cp 1254" },
00231     { "windows-1255", "cp 1255" },
00232     { "windows-1256", "cp 1256" },
00233     { "windows-1257", "cp 1257" },
00234     { "x-windows-1250", "cp 1250" },
00235     { "x-windows-1251", "cp 1251" },
00236     { "x-windows-1252", "cp 1252" },
00237     { "x-windows-1253", "cp 1253" },
00238     { "x-windows-1254", "cp 1254" },
00239     { "x-windows-1255", "cp 1255" },
00240     { "x-windows-1256", "cp 1256" },
00241     { "x-windows-1257", "cp 1257" },
00242     { "cp-1250", "cp 1250" },
00243     { "cp-1251", "cp 1251" },
00244     { "cp-1252", "cp 1252" },
00245     { "cp-1253", "cp 1253" },
00246     { "cp-1254", "cp 1254" },
00247     { "cp-1255", "cp 1255" },
00248     { "cp-1256", "cp 1256" },
00249     { "cp-1257", "cp 1257" },
00250     { "x-cp-1250", "cp 1250" },
00251     { "x-cp-1251", "cp 1251" },
00252     { "x-cp-1252", "cp 1252" },
00253     { "x-cp-1253", "cp 1253" },
00254     { "x-cp-1254", "cp 1254" },
00255     { "x-cp-1255", "cp 1255" },
00256     { "x-cp-1256", "cp 1256" },
00257     { "x-cp-1257", "cp 1257" },
00258     { "tis620", "iso 8859-11" },
00259     { "tis-620", "iso 8859-11" },
00260     { "thai-tis620", "iso 8859-11" },
00261     { "windows-874", "iso 8859-11" },
00262     { "windows874", "iso 8859-11" },
00263     { "x-windows-874", "iso 8859-11" },
00264     { "cp874", "iso 8859-11" },
00265     { "cp-874", "iso 8859-11" },
00266     { "x-cp-874", "iso 8859-11" },
00267     { "ksc5601.1987-0", "euckr" },
00268     { "ks_c_5601-1987", "euckr" },
00269     { "iso-8859-1", "iso 8859-1" },
00270     { "iso-8859-2", "iso 8859-2" },
00271     { "iso-8859-3", "iso 8859-3" },
00272     { "iso-8859-4", "iso 8859-4" },
00273     { "iso-8859-5", "iso 8859-5" },
00274     { "iso-8859-6", "iso 8859-6" },
00275     { "iso-8859-7", "iso 8859-7" },
00276     { "iso-8859-8", "iso 8859-8" },
00277     { "iso-8859-9", "iso 8859-9" },
00278     { "iso-8859-10", "iso 8859-10" },
00279     { "iso-8859-11", "iso 8859-11" },
00280     { "iso-8859-12", "iso 8859-12" },
00281     { "iso-8859-13", "iso 8859-13" },
00282     { "iso-8859-14", "iso 8859-14" },
00283     { "iso-8859-15", "iso 8859-15" },
00284     { "tscii", "tscii" },
00285     { "paratype-154", "pt 154" },
00286     { "pt-154", "pt 154" },
00287     { "x-winsami2", "winsami2" },
00288     { 0, 0 }};
00289 
00290 // some different names for the encodings defined in the charmaps files.
00291 // even though the charmap file names are all uppercase, the names are all lowercase here.
00292 static struct Aliases
00293     {
00294     const char* index;
00295     const char* data;
00296     } const aliases[] = {
00297     { "cp852", "ibm852" },
00298     { "cp866", "ibm866" },
00299     { 0, 0 }};
00300 
00301 // some last resort hints in case the charmap file couldn't be found. This gives at least a partial conversion
00302 // and helps making things readable.
00303 // the name used as input here is already converted to the more canonical name as defined in the aliases array.
00304 static struct ConversionHints
00305     {
00306     const char* index;
00307     const char* data;
00308     } const conversion_hints[] = {
00309     { "cp1250", "iso-8859-2" },
00310     { "koi8-r", "iso-8859-5" },
00311     { "koi8-u", "koi8-r" },
00312     { "utf16", "iso-10646-ucs-2" },
00313     { 0, 0 }};
00314 
00315 
00316 // search an array of items index/data, index is const char*, data is T, find first matching index
00317 // and return data, or return 0
00318 template< typename T, typename Data >
00319 Data kcharsets_array_search( const T* start, const char* entry )
00320 {
00321     for( const T* pos = start;
00322          pos->index != 0;
00323          ++pos )
00324         if( qstrcmp( pos->index, entry ) == 0 )
00325             return pos->data;
00326     return 0;
00327 }
00328 
00329 
00330 class KCharsetsPrivate
00331 {
00332 public:
00333     KCharsetsPrivate(KCharsets* _kc)
00334         : codecForNameDict(43, false) // case insensitive
00335     {
00336         db = 0;
00337         kc = _kc;
00338     }
00339     ~KCharsetsPrivate()
00340     {
00341         delete db;
00342     }
00343     QFontDatabase *db;
00344     QAsciiDict<QTextCodec> codecForNameDict;
00345     KCharsets* kc;
00346 };
00347 
00348 // --------------------------------------------------------------------------
00349 
00350 KCharsets::KCharsets()
00351 {
00352     d = new KCharsetsPrivate(this);
00353 }
00354 
00355 KCharsets::~KCharsets()
00356 {
00357     delete d;
00358 }
00359 
00360 QChar KCharsets::fromEntity(const QString &str) 
00361 {
00362     QChar res = QChar::null;
00363 
00364     int pos = 0;
00365     if(str[pos] == '&') pos++;
00366 
00367     // Check for '&#000' or '&#x0000' sequence
00368     if (str[pos] == '#' && str.length()-pos > 1) {
00369         bool ok;
00370         pos++;
00371         if (str[pos] == 'x' || str[pos] == 'X') {
00372             pos++;
00373             // '&#x0000', hexadeciaml character reference
00374             QString tmp(str.unicode()+pos, str.length()-pos);
00375             res = tmp.toInt(&ok, 16);
00376         } else {
00377             //  '&#0000', deciaml character reference
00378             QString tmp(str.unicode()+pos, str.length()-pos);
00379             res = tmp.toInt(&ok, 10);
00380         }
00381         return res;
00382     }
00383 
00384     const entity *e = findEntity(str.ascii(), str.length());
00385 
00386     if(!e)
00387     {
00388         //kdDebug( 0 ) << "unknown entity " << str <<", len = " << str.length() << endl;
00389         return QChar::null;
00390     }
00391     //kdDebug() << "got entity " << str << " = " << e->code << endl;
00392 
00393     return QChar(e->code);
00394 }
00395 
00396 QChar KCharsets::fromEntity(const QString &str, int &len) 
00397 {
00398     // entities are never longer than 8 chars... we start from
00399     // that length and work backwards...
00400     len = 8;
00401     while(len > 0)
00402     {
00403         QString tmp = str.left(len);
00404         QChar res = fromEntity(tmp);
00405         if( res != QChar::null ) return res;
00406         len--;
00407     }
00408     return QChar::null;
00409 }
00410 
00411 
00412 QString KCharsets::toEntity(const QChar &ch) 
00413 {
00414     QString ent;
00415     ent.sprintf("&#0x%x;", ch.unicode());
00416     return ent;
00417 }
00418 
00419 QString KCharsets::resolveEntities( const QString &input )
00420 {
00421     QString text = input;
00422     const QChar *p = text.unicode();
00423     const QChar *end = p + text.length();
00424     const QChar *ampersand = 0;
00425     bool scanForSemicolon = false;
00426 
00427     for ( ; p < end; ++p ) {
00428         QChar ch = *p;
00429 
00430         if ( ch == '&' ) {
00431             ampersand = p;
00432             scanForSemicolon = true;
00433             continue;
00434         }
00435 
00436         if ( ch != ';' || scanForSemicolon == false )
00437             continue;
00438 
00439         assert( ampersand );
00440 
00441         scanForSemicolon = false;
00442 
00443         const QChar *entityBegin = ampersand + 1;
00444 
00445         uint entityLength = p - entityBegin;
00446         if ( entityLength == 0 )
00447             continue;
00448 
00449         QChar entityValue = KCharsets::fromEntity( QConstString( entityBegin, entityLength ).string() );
00450         if ( entityValue.isNull() )
00451             continue;
00452 
00453         uint ampersandPos = ampersand - text.unicode();
00454 
00455         text[ ampersandPos ] = entityValue;
00456         text.remove( ampersandPos + 1, entityLength + 1 );
00457         p = text.unicode() + ampersandPos;
00458         end = text.unicode() + text.length();
00459         ampersand = 0;
00460     }
00461 
00462     return text;
00463 }
00464 
00465 QStringList KCharsets::availableEncodingNames()
00466 {
00467     QStringList available;
00468 
00469     const char* const* pos = charsets_for_encoding;
00470     while( *pos != 0 ) {
00471         //kdDebug(0) << "key = " << *pos << endl;
00472 
00473 
00474         // iterate thorugh the list and find the first charset that is available
00475         for( const char* const* charsets = pos + 1;
00476              *charsets != 0;
00477              ++charsets ) {
00478             //kdDebug(0) << "checking for " << *charsets << endl;
00479 #ifdef __GNUC__
00480 #warning FIXME?
00481 #endif
00482             if( true ) {
00483                 //kdDebug(0) << *charsets << " available" << endl;
00484                 available.append( QString::fromLatin1( *pos ));
00485                 break;
00486             }
00487         }
00488         while( *pos != 0 ) // find end of line
00489             ++pos;
00490         ++pos; // move to the next line
00491     }
00492     return available;
00493 }
00494 
00495 QString KCharsets::languageForEncoding( const QString &encoding )
00496 {
00497     int lang = kcharsets_array_search< LanguageForEncoding, int >
00498         ( language_for_encoding, encoding.latin1());
00499     return i18n( language_names[lang] );
00500 }
00501 
00502 QString KCharsets::encodingForName( const QString &descriptiveName )
00503 {
00504     int left = descriptiveName.find( "( " );
00505     return descriptiveName.mid( left + 2, descriptiveName.find( " )" )
00506       - left - 2 );
00507 }
00508 
00509 QStringList KCharsets::descriptiveEncodingNames()
00510 {
00511   QStringList encodings = availableEncodingNames();
00512   QStringList::Iterator it;
00513   for( it = encodings.begin(); it != encodings.end(); ++it ) {
00514       QString lang = KGlobal::charsets()->languageForEncoding( *it );
00515       *it = lang + " ( " + *it + " )";
00516   }
00517   encodings.sort();
00518   return encodings;
00519 }
00520 
00521 QTextCodec *KCharsets::codecForName(const QString &n) const
00522 {
00523     bool b;
00524     return codecForName( n, b );
00525 }
00526 
00527 QTextCodec *KCharsets::codecForName(const QString &n, bool &ok) const
00528 {
00529     ok = true;
00530 
00531     QTextCodec* codec = 0;
00532     // dict lookup is case insensitive anyway
00533     if((codec = d->codecForNameDict[n.isEmpty() ? "->locale<-" : n.latin1()]))
00534         return codec; // cache hit, return
00535 
00536     if (n.isEmpty()) {
00537         codec = KGlobal::locale()->codecForEncoding();
00538         d->codecForNameDict.replace("->locale<-", codec);
00539         return codec;
00540     }
00541 
00542     QCString name = n.lower().latin1();
00543     QCString key = name;
00544     if (name.right(8) == "_charset")
00545        name.truncate(name.length()-8);
00546 
00547     if (name.isEmpty()) {
00548       ok = false;
00549       return QTextCodec::codecForName("iso8859-1");
00550     }
00551 
00552     codec = QTextCodec::codecForName(name);
00553 
00554     if(codec) {
00555         d->codecForNameDict.replace(key, codec);
00556         return codec;
00557     }
00558 
00559     // these codecs are built into Qt, but the name given for the codec is different,
00560     // so QTextCodec did not recognise it.
00561     QCString cname = kcharsets_array_search< Builtin, const char* >( builtin, name.data());
00562 
00563     if(!cname.isEmpty())
00564         codec = QTextCodec::codecForName(cname);
00565 
00566     if(codec)
00567     {
00568         d->codecForNameDict.replace(key, codec);
00569         return codec;
00570     }
00571 
00572     QString dir;
00573     {
00574     KConfigGroupSaver cfgsav( KGlobal::config(), "i18n" );
00575     dir = KGlobal::config()->readPathEntry("i18ndir", QString::fromLatin1("/usr/share/i18n/charmaps"));
00576     dir += "/";
00577     }
00578 
00579     // these are codecs not included in Qt. They can be build up if the corresponding charmap
00580     // is available in the charmap directory.
00581     cname = kcharsets_array_search< Aliases, const char* >( aliases, name.data());
00582 
00583     if(cname.isEmpty())
00584         cname = name;
00585     cname = cname.upper();
00586 
00587     codec = QTextCodec::loadCharmapFile((QString)(dir + cname.data()));
00588 
00589     if(codec) {
00590         d->codecForNameDict.replace(key, codec);
00591         return codec;
00592     }
00593 
00594     // this also failed, the last resort is now to take some compatibility charmap
00595 
00596     cname = cname.lower();
00597     cname = kcharsets_array_search< ConversionHints, const char* >( conversion_hints, (const char*)cname );
00598 
00599     if(!cname.isEmpty())
00600         codec = QTextCodec::codecForName(cname);
00601 
00602     if(codec) {
00603         d->codecForNameDict.replace(key, codec);
00604         return codec;
00605     }
00606 
00607     // could not assign a codec, let's return Latin1
00608     ok = false;
00609     return QTextCodec::codecForName("iso8859-1");
00610 }
KDE Logo
This file is part of the documentation for kdelibs Version 3.1.4.
Documentation copyright © 1996-2002 the KDE developers.
Generated on Wed Mar 16 18:43:27 2005 by doxygen 1.3.3 written by Dimitri van Heesch, © 1997-2001