2 // Copyright (c) 2007-2009 Ben Hanson (http://www.benhanson.net/)
4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 // file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 #ifndef BOOST_LEXER_RE_TOKENISER_HPP
7 #define BOOST_LEXER_RE_TOKENISER_HPP
12 #include "num_token.hpp"
13 #include "../../runtime_error.hpp"
14 #include "../../size_t.hpp"
16 #include "../../string_token.hpp"
17 #include "re_tokeniser_helper.hpp"
25 template<typename CharT>
26 class basic_re_tokeniser
29 typedef basic_num_token<CharT> num_token;
30 typedef basic_re_tokeniser_state<CharT> state;
31 typedef basic_string_token<CharT> string_token;
32 typedef typename string_token::string string;
33 typedef std::map<string_token, std::size_t> token_map;
34 typedef std::pair<string_token, std::size_t> token_pair;
36 static void next (state &state_, token_map &map_, num_token &token_)
39 bool eos_ = state_.next (ch_);
41 token_.min_max (0, false, 0);
43 while (!eos_ && ch_ == '"')
45 state_._in_string ^= 1;
46 eos_ = state_.next (ch_);
51 if (state_._in_string)
53 throw runtime_error ("Unexpected end of regex "
57 if (state_._paren_count)
59 throw runtime_error ("Unexpected end of regex "
63 token_.set (num_token::END, null_token);
69 // Even if we are in a string, respect escape sequences...
70 escape (state_, map_, token_);
72 else if (state_._in_string)
74 // All other meta characters lose their special meaning
76 create_charset_token (string (1, ch_), false, map_, token_);
80 // Not an escape sequence and not inside a string, so
81 // check for meta characters.
85 token_.set (num_token::OPENPAREN, null_token);
86 ++state_._paren_count;
87 read_options (state_);
90 --state_._paren_count;
92 if (state_._paren_count < 0)
94 std::ostringstream ss_;
96 ss_ << "Number of open parenthesis < 0 at index " <<
97 state_.index () - 1 << '.';
98 throw runtime_error (ss_.str ().c_str ());
101 token_.set (num_token::CLOSEPAREN, null_token);
103 if (!state_._flags_stack.empty ())
105 state_._flags = state_._flags_stack.top ();
106 state_._flags_stack.pop ();
110 if (!state_.eos () && *state_._curr == '?')
112 token_.set (num_token::AOPT, null_token);
117 token_.set (num_token::OPT, null_token);
122 if (!state_.eos () && *state_._curr == '?')
124 token_.set (num_token::AZEROORMORE, null_token);
129 token_.set (num_token::ZEROORMORE, null_token);
134 if (!state_.eos () && *state_._curr == '?')
136 token_.set (num_token::AONEORMORE, null_token);
141 token_.set (num_token::ONEORMORE, null_token);
146 open_curly (state_, token_);
149 token_.set (num_token::OR, null_token);
152 if (state_._curr - 1 == state_._start)
154 token_.set (num_token::CHARSET, bol_token);
155 state_._seen_BOL_assertion = true;
159 create_charset_token (string (1, ch_), false,
165 if (state_._curr == state_._end)
167 token_.set (num_token::CHARSET, eol_token);
168 state_._seen_EOL_assertion = true;
172 create_charset_token (string (1, ch_), false,
181 if (state_._flags & dot_not_newline)
186 create_charset_token (dot_, true, map_, token_);
191 charset (state_, map_, token_);
195 throw runtime_error("Lookahead ('/') is not supported yet.");
198 if ((state_._flags & icase) &&
199 (std::isupper (ch_, state_._locale) ||
200 std::islower (ch_, state_._locale)))
202 CharT upper_ = std::toupper (ch_, state_._locale);
203 CharT lower_ = std::tolower (ch_, state_._locale);
205 string str_ (1, upper_);
208 create_charset_token (str_, false, map_, token_);
212 create_charset_token (string (1, ch_), false,
223 typedef basic_re_tokeniser_helper<CharT> tokeniser_helper;
225 static void read_options (state &state_)
227 if (!state_.eos () && *state_._curr == '?')
231 bool negate_ = false;
234 eos_ = state_.next (ch_);
235 state_._flags_stack.push (state_._flags);
237 while (!eos_ && ch_ != ':')
247 state_._flags = static_cast<regex_flags>
248 (state_._flags & ~icase);
252 state_._flags = static_cast<regex_flags>
253 (state_._flags | icase);
261 state_._flags = static_cast<regex_flags>
262 (state_._flags | dot_not_newline);
266 state_._flags = static_cast<regex_flags>
267 (state_._flags & ~dot_not_newline);
274 std::ostringstream ss_;
276 ss_ << "Unknown option at index " <<
277 state_.index () - 1 << '.';
278 throw runtime_error (ss_.str ().c_str ());
282 eos_ = state_.next (ch_);
285 // End of string handler will handle early termination
287 else if (!state_._flags_stack.empty ())
289 state_._flags_stack.push (state_._flags);
293 static void escape (state &state_, token_map &map_, num_token &token_)
296 std::size_t str_len_ = 0;
297 const CharT *str_ = tokeniser_helper::escape_sequence (state_,
302 state state2_ (str_ + 1, str_ + str_len_, state_._flags,
305 charset (state2_, map_, token_);
309 create_charset_token (string (1, ch_), false, map_, token_);
313 static void charset (state &state_, token_map &map_, num_token &token_)
316 bool negated_ = false;
318 tokeniser_helper::charset (state_, chars_, negated_);
319 create_charset_token (chars_, negated_, map_, token_);
322 static void create_charset_token (const string &charset_,
323 const bool negated_, token_map &map_, num_token &token_)
325 std::size_t id_ = null_token;
326 string_token stok_ (negated_, charset_);
328 stok_.remove_duplicates ();
331 typename token_map::const_iterator iter_ = map_.find (stok_);
333 if (iter_ == map_.end ())
336 map_.insert (token_pair (stok_, id_));
343 token_.set (num_token::CHARSET, id_);
346 static void open_curly (state &state_, num_token &token_)
350 throw runtime_error ("Unexpected end of regex "
353 else if (*state_._curr >= '0' && *state_._curr <= '9')
355 repeat_n (state_, token_);
357 if (!state_.eos () && *state_._curr == '?')
359 token_._type = num_token::AREPEATN;
365 macro (state_, token_);
372 // {0} - INVALID (throw exception)
374 // {0,0} - INVALID (throw exception)
377 // {min,max} where min == max - {min}
378 // {min,max} where max < min - INVALID (throw exception)
379 static void repeat_n (state &state_, num_token &token_)
382 bool eos_ = state_.next (ch_);
384 while (!eos_ && ch_ >= '0' && ch_ <= '9')
387 token_._min += ch_ - '0';
388 eos_ = state_.next (ch_);
393 throw runtime_error ("Unexpected end of regex "
397 bool min_max_ = false;
398 bool repeatn_ = true;
400 token_._comma = ch_ == ',';
404 eos_ = state_.next (ch_);
408 throw runtime_error ("Unexpected end of regex "
414 // Small optimisation: Check for '*' equivalency.
415 if (token_._min == 0)
417 token_.set (num_token::ZEROORMORE, null_token);
420 // Small optimisation: Check for '+' equivalency.
421 else if (token_._min == 1)
423 token_.set (num_token::ONEORMORE, null_token);
429 if (ch_ < '0' || ch_ > '9')
431 std::ostringstream ss_;
433 ss_ << "Missing '}' at index " <<
434 state_.index () - 1 << '.';
435 throw runtime_error (ss_.str ().c_str ());
443 token_._max += ch_ - '0';
444 eos_ = state_.next (ch_);
445 } while (!eos_ && ch_ >= '0' && ch_ <= '9');
449 throw runtime_error ("Unexpected end of regex "
453 // Small optimisation: Check for '?' equivalency.
454 if (token_._min == 0 && token_._max == 1)
456 token_.set (num_token::OPT, null_token);
459 // Small optimisation: if min == max, then min.
460 else if (token_._min == token_._max)
462 token_._comma = false;
471 std::ostringstream ss_;
473 ss_ << "Missing '}' at index " << state_.index () - 1 << '.';
474 throw runtime_error (ss_.str ().c_str ());
479 // SEMANTIC VALIDATION follows:
480 // NOTE: {0,} has already become *
481 // therefore we don't check for a comma.
482 if (token_._min == 0 && token_._max == 0)
484 std::ostringstream ss_;
486 ss_ << "Cannot have exactly zero repeats preceding index " <<
487 state_.index () << '.';
488 throw runtime_error (ss_.str ().c_str ());
491 if (min_max_ && token_._max < token_._min)
493 std::ostringstream ss_;
495 ss_ << "Max less than min preceding index " <<
496 state_.index () << '.';
497 throw runtime_error (ss_.str ().c_str ());
500 token_.set (num_token::REPEATN, null_token);
504 static void macro (state &state_, num_token &token_)
508 const CharT *start_ = state_._curr;
512 if (ch_ != '_' && !(ch_ >= 'A' && ch_ <= 'Z') &&
513 !(ch_ >= 'a' && ch_ <= 'z'))
515 std::ostringstream ss_;
517 ss_ << "Invalid MACRO name at index " <<
518 state_.index () - 1 << '.';
519 throw runtime_error (ss_.str ().c_str ());
524 eos_ = state_.next (ch_);
528 throw runtime_error ("Unexpected end of regex "
531 } while (ch_ == '_' || ch_ == '-' || (ch_ >= 'A' && ch_ <= 'Z') ||
532 (ch_ >= 'a' && ch_ <= 'z') || (ch_ >= '0' && ch_ <= '9'));
536 std::ostringstream ss_;
538 ss_ << "Missing '}' at index " << state_.index () - 1 << '.';
539 throw runtime_error (ss_.str ().c_str ());
542 std::size_t len_ = state_._curr - 1 - start_;
544 if (len_ > max_macro_len)
546 std::basic_stringstream<CharT> ss_;
547 std::ostringstream os_;
549 os_ << "MACRO name '";
553 os_ << ss_.narrow (*start_++, ' ');
557 os_ << "' too long.";
558 throw runtime_error (os_.str ());
561 token_.set (num_token::MACRO, null_token);
563 // Some systems have memcpy in namespace std.
566 memcpy (token_._macro, start_, len_ * sizeof (CharT));
567 token_._macro[len_] = 0;