1 // tokeniser_helper.hpp
2 // Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 // file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 #ifndef BOOST_LEXER_RE_TOKENISER_HELPER_H
7 #define BOOST_LEXER_RE_TOKENISER_HELPER_H
9 #include "../../char_traits.hpp"
12 #include "../../size_t.hpp"
13 #include "re_tokeniser_state.hpp"
21 template<typename CharT, typename Traits = char_traits<CharT> >
22 class basic_re_tokeniser_helper
25 typedef basic_re_tokeniser_state<CharT> state;
26 typedef std::basic_string<CharT> string;
28 static const CharT *escape_sequence (state &state_, CharT &ch_,
29 std::size_t &str_len_)
31 bool eos_ = state_.eos ();
35 throw runtime_error ("Unexpected end of regex "
39 const CharT *str_ = charset_shortcut (*state_._curr, str_len_);
53 // This function can call itself.
54 static void charset (state &state_, string &chars_, bool &negated_)
57 bool eos_ = state_.next (ch_);
61 // Pointless returning index if at end of string
62 throw runtime_error ("Unexpected end of regex "
66 negated_ = ch_ == '^';
70 eos_ = state_.next (ch_);
74 // Pointless returning index if at end of string
75 throw runtime_error ("Unexpected end of regex "
87 std::size_t str_len_ = 0;
88 const CharT *str_ = escape_sequence (state_, prev_, str_len_);
94 state temp_state_ (str_ + 1, str_ + str_len_,
95 state_._flags, state_._locale);
97 bool temp_negated_ = false;
99 charset (temp_state_, temp_chars_, temp_negated_);
101 if (negated_ != temp_negated_)
103 std::ostringstream ss_;
105 ss_ << "Mismatch in charset negation preceding "
106 "index " << state_.index () << '.';
107 throw runtime_error (ss_.str ().c_str ());
110 chars_ += temp_chars_;
114 else if (ch_ == '[' && !state_.eos () && *state_._curr == ':')
116 // TODO: POSIX charsets
125 eos_ = state_.next (ch_);
127 // Covers preceding if, else if and else
130 // Pointless returning index if at end of string
131 throw runtime_error ("Unexpected end of regex "
137 charset_range (chset_, state_, eos_, ch_, prev_, chars_);
141 if ((state_._flags & icase) &&
142 (std::isupper (prev_, state_._locale) ||
143 std::islower (prev_, state_._locale)))
145 CharT upper_ = std::toupper (prev_, state_._locale);
146 CharT lower_ = std::tolower (prev_, state_._locale);
158 if (!negated_ && chars_.empty ())
160 throw runtime_error ("Empty charsets not allowed.");
164 static CharT chr (state &state_)
168 // eos_ has already been checked for.
169 switch (*state_._curr)
179 ch_ = decode_octal (state_);
190 ch_ = decode_control_char (state_);
193 ch_ = 27; // '\e' not recognised by compiler
217 ch_ = decode_hex (state_);
229 static const char *charset_shortcut (const char ch_,
230 std::size_t &str_len_)
232 const char *str_ = 0;
243 str_ = "[ \t\n\r\f\v]";
246 str_ = "[^ \t\n\r\f\v]";
249 str_ = "[_0-9A-Za-z]";
252 str_ = "[^_0-9A-Za-z]";
258 // Some systems have strlen in namespace std.
261 str_len_ = strlen (str_);
271 static const wchar_t *charset_shortcut (const wchar_t ch_,
272 std::size_t &str_len_)
274 const wchar_t *str_ = 0;
285 str_ = L"[ \t\n\r\f\v]";
288 str_ = L"[^ \t\n\r\f\v]";
291 str_ = L"[_0-9A-Za-z]";
294 str_ = L"[^_0-9A-Za-z]";
300 // Some systems have wcslen in namespace std.
303 str_len_ = wcslen (str_);
313 static CharT decode_octal (state &state_)
315 std::size_t accumulator_ = 0;
316 CharT ch_ = *state_._curr;
317 unsigned short count_ = 3;
323 accumulator_ += ch_ - '0';
326 eos_ = state_.eos ();
328 if (!count_ || eos_) break;
332 // Don't consume invalid chars!
333 if (ch_ < '0' || ch_ > '7')
339 return static_cast<CharT> (accumulator_);
342 static CharT decode_control_char (state &state_)
348 bool eos_ = state_.next (ch_);
352 // Pointless returning index if at end of string
353 throw runtime_error ("Unexpected end of regex following \\c.");
357 if (ch_ >= 'a' && ch_ <= 'z')
361 else if (ch_ >= 'A' && ch_ <= 'Z')
372 std::ostringstream ss_;
374 ss_ << "Invalid control char at index " <<
375 state_.index () - 1 << '.';
376 throw runtime_error (ss_.str ().c_str ());
383 static CharT decode_hex (state &state_)
389 bool eos_ = state_.next (ch_);
393 // Pointless returning index if at end of string
394 throw runtime_error ("Unexpected end of regex following \\x.");
397 if (!((ch_ >= '0' && ch_ <= '9') || (ch_ >= 'a' && ch_ <= 'f') ||
398 (ch_ >= 'A' && ch_ <= 'F')))
400 std::ostringstream ss_;
402 ss_ << "Illegal char following \\x at index " <<
403 state_.index () - 1 << '.';
404 throw runtime_error (ss_.str ().c_str ());
407 std::size_t hex_ = 0;
413 if (ch_ >= '0' && ch_ <= '9')
417 else if (ch_ >= 'a' && ch_ <= 'f')
419 hex_ += 10 + (ch_ - 'a');
423 hex_ += 10 + (ch_ - 'A');
426 eos_ = state_.eos ();
432 // Don't consume invalid chars!
433 if (((ch_ >= '0' && ch_ <= '9') ||
434 (ch_ >= 'a' && ch_ <= 'f') || (ch_ >= 'A' && ch_ <= 'F')))
445 return static_cast<CharT> (hex_);
448 static void charset_range (const bool chset_, state &state_, bool &eos_,
449 CharT &ch_, const CharT prev_, string &chars_)
453 std::ostringstream ss_;
455 ss_ << "Charset cannot form start of range preceding "
456 "index " << state_.index () - 1 << '.';
457 throw runtime_error (ss_.str ().c_str ());
460 eos_ = state_.next (ch_);
464 // Pointless returning index if at end of string
465 throw runtime_error ("Unexpected end of regex "
473 std::size_t str_len_ = 0;
475 if (escape_sequence (state_, curr_, str_len_))
477 std::ostringstream ss_;
479 ss_ << "Charset cannot form end of range preceding index "
480 << state_.index () << '.';
481 throw runtime_error (ss_.str ().c_str ());
485 else if (ch_ == '[' && !state_.eos () && *state_._curr == ':')
487 std::ostringstream ss_;
489 ss_ << "POSIX char class cannot form end of range at "
490 "index " << state_.index () - 1 << '.';
491 throw runtime_error (ss_.str ().c_str ());
499 eos_ = state_.next (ch_);
501 // Covers preceding if and else
504 // Pointless returning index if at end of string
505 throw runtime_error ("Unexpected end of regex "
509 std::size_t start_ = static_cast<typename Traits::index_type> (prev_);
510 std::size_t end_ = static_cast<typename Traits::index_type> (curr_);
515 std::ostringstream ss_;
517 ss_ << "Invalid range in charset preceding index " <<
518 state_.index () - 1 << '.';
519 throw runtime_error (ss_.str ().c_str ());
522 chars_.reserve (chars_.size () + (end_ + 1 - start_));
524 for (; start_ <= end_; ++start_)
526 CharT ch_ = static_cast<CharT> (start_);
528 if ((state_._flags & icase) &&
529 (std::isupper (ch_, state_._locale) ||
530 std::islower (ch_, state_._locale)))
532 CharT upper_ = std::toupper (ch_, state_._locale);
533 CharT lower_ = std::tolower (ch_, state_._locale);