00001
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023 #ifndef XAPIAN_INCLUDED_UNICODE_H
00024 #define XAPIAN_INCLUDED_UNICODE_H
00025
00026 #include <xapian/visibility.h>
00027
00028 #include <string>
00029
00030 namespace Xapian {
00031
00035 class XAPIAN_VISIBILITY_DEFAULT Utf8Iterator {
00036 const unsigned char *p;
00037 const unsigned char *end;
00038 mutable unsigned seqlen;
00039
00040 void calculate_sequence_length() const;
00041
00042 unsigned get_char() const;
00043
00044 Utf8Iterator(const unsigned char *p_, const unsigned char *end_, unsigned seqlen_)
00045 : p(p_), end(end_), seqlen(seqlen_) { }
00046
00047 public:
00049 const char * raw() const {
00050 return reinterpret_cast<const char *>(p ? p : end);
00051 }
00052
00054 size_t left() const { return p ? end - p : 0; }
00055
00067 void assign(const char *p_, size_t len) {
00068 if (len) {
00069 p = reinterpret_cast<const unsigned char*>(p_);
00070 end = p + len;
00071 seqlen = 0;
00072 } else {
00073 p = NULL;
00074 }
00075 }
00076
00087 void assign(const std::string &s) { assign(s.data(), s.size()); }
00088
00097 explicit Utf8Iterator(const char *p_);
00098
00109 Utf8Iterator(const char *p_, size_t len) { assign(p_, len); }
00110
00120 Utf8Iterator(const std::string &s) { assign(s.data(), s.size()); }
00121
00127 Utf8Iterator() : p(NULL), end(0), seqlen(0) { }
00128
00133 unsigned operator*() const;
00134
00139 Utf8Iterator operator++(int) {
00140
00141 if (seqlen == 0) calculate_sequence_length();
00142 const unsigned char *old_p = p;
00143 unsigned old_seqlen = seqlen;
00144 p += seqlen;
00145 if (p == end) p = NULL;
00146 seqlen = 0;
00147 return Utf8Iterator(old_p, end, old_seqlen);
00148 }
00149
00154 Utf8Iterator & operator++() {
00155 if (seqlen == 0) calculate_sequence_length();
00156 p += seqlen;
00157 if (p == end) p = NULL;
00158 seqlen = 0;
00159 return *this;
00160 }
00161
00166 bool operator==(const Utf8Iterator &other) const { return p == other.p; }
00167
00172 bool operator!=(const Utf8Iterator &other) const { return p != other.p; }
00173
00175
00176 typedef std::input_iterator_tag iterator_category;
00177 typedef unsigned value_type;
00178 typedef size_t difference_type;
00179 typedef const unsigned * pointer;
00180 typedef const unsigned & reference;
00182 };
00183
00184 namespace Unicode {
00185
00187 typedef enum {
00188 UNASSIGNED,
00189 UPPERCASE_LETTER,
00190 LOWERCASE_LETTER,
00191 TITLECASE_LETTER,
00192 MODIFIER_LETTER,
00193 OTHER_LETTER,
00194 NON_SPACING_MARK,
00195 ENCLOSING_MARK,
00196 COMBINING_SPACING_MARK,
00197 DECIMAL_DIGIT_NUMBER,
00198 LETTER_NUMBER,
00199 OTHER_NUMBER,
00200 SPACE_SEPARATOR,
00201 LINE_SEPARATOR,
00202 PARAGRAPH_SEPARATOR,
00203 CONTROL,
00204 FORMAT,
00205 PRIVATE_USE,
00206 SURROGATE,
00207 CONNECTOR_PUNCTUATION,
00208 DASH_PUNCTUATION,
00209 OPEN_PUNCTUATION,
00210 CLOSE_PUNCTUATION,
00211 INITIAL_QUOTE_PUNCTUATION,
00212 FINAL_QUOTE_PUNCTUATION,
00213 OTHER_PUNCTUATION,
00214 MATH_SYMBOL,
00215 CURRENCY_SYMBOL,
00216 MODIFIER_SYMBOL,
00217 OTHER_SYMBOL
00218 } category;
00219
00220 namespace Internal {
00221
00222
00223
00224
00225
00226 XAPIAN_VISIBILITY_DEFAULT
00227 int get_character_info(unsigned ch);
00228
00230 inline int get_case_type(int info) { return ((info & 0xe0) >> 5); }
00231
00233 inline category get_category(int info) { return static_cast<category>(info & 0x1f); }
00234
00236 inline int get_delta(int info) {
00237
00238
00239
00240
00241
00242 return (info >= 0) ? (info >> 15) : (~(~info >> 15));
00243 }
00244 }
00245
00255 XAPIAN_VISIBILITY_DEFAULT
00256 unsigned nonascii_to_utf8(unsigned ch, char * buf);
00257
00265 inline unsigned to_utf8(unsigned ch, char *buf) {
00266 if (ch < 128) {
00267 *buf = static_cast<unsigned char>(ch);
00268 return 1;
00269 }
00270 return Xapian::Unicode::nonascii_to_utf8(ch, buf);
00271 }
00272
00276 inline void append_utf8(std::string &s, unsigned ch) {
00277 char buf[4];
00278 s.append(buf, to_utf8(ch, buf));
00279 }
00280
00282 inline category get_category(unsigned ch) {
00283
00284 if (ch >= 0x110000) return Xapian::Unicode::UNASSIGNED;
00285 return Internal::get_category(Internal::get_character_info(ch));
00286 }
00287
00289 inline bool is_wordchar(unsigned ch) {
00290 const unsigned int WORDCHAR_MASK =
00291 (1 << Xapian::Unicode::UPPERCASE_LETTER) |
00292 (1 << Xapian::Unicode::LOWERCASE_LETTER) |
00293 (1 << Xapian::Unicode::TITLECASE_LETTER) |
00294 (1 << Xapian::Unicode::MODIFIER_LETTER) |
00295 (1 << Xapian::Unicode::OTHER_LETTER) |
00296 (1 << Xapian::Unicode::DECIMAL_DIGIT_NUMBER) |
00297 (1 << Xapian::Unicode::LETTER_NUMBER) |
00298 (1 << Xapian::Unicode::OTHER_NUMBER) |
00299 (1 << Xapian::Unicode::CONNECTOR_PUNCTUATION);
00300 return ((WORDCHAR_MASK >> get_category(ch)) & 1);
00301 }
00302
00304 inline bool is_whitespace(unsigned ch) {
00305 const unsigned int WHITESPACE_MASK =
00306 (1 << Xapian::Unicode::CONTROL) |
00307 (1 << Xapian::Unicode::SPACE_SEPARATOR) |
00308 (1 << Xapian::Unicode::LINE_SEPARATOR) |
00309 (1 << Xapian::Unicode::PARAGRAPH_SEPARATOR);
00310 return ((WHITESPACE_MASK >> get_category(ch)) & 1);
00311 }
00312
00314 inline bool is_currency(unsigned ch) {
00315 return (get_category(ch) == Xapian::Unicode::CURRENCY_SYMBOL);
00316 }
00317
00319 inline unsigned tolower(unsigned ch) {
00320 int info;
00321
00322 if (ch >= 0x110000 || !(Internal::get_case_type((info = Xapian::Unicode::Internal::get_character_info(ch))) & 2))
00323 return ch;
00324 return ch + Internal::get_delta(info);
00325 }
00326
00328 inline std::string
00329 tolower(const std::string &term)
00330 {
00331 std::string result;
00332 result.reserve(term.size());
00333 for (Utf8Iterator i(term); i != Utf8Iterator(); ++i) {
00334 append_utf8(result, tolower(*i));
00335 }
00336 return result;
00337 }
00338
00339 }
00340
00341 }
00342
00343 #endif // XAPIAN_INCLUDED_UNICODE_H