Swiftdiff options
Diffstat (limited to '3rdParty/Boost/src/boost/spirit/home/lex/tokenize_and_parse.hpp')
| -rw-r--r-- | 3rdParty/Boost/src/boost/spirit/home/lex/tokenize_and_parse.hpp | 325 |
1 files changed, 325 insertions, 0 deletions
diff --git a/3rdParty/Boost/src/boost/spirit/home/lex/tokenize_and_parse.hpp b/3rdParty/Boost/src/boost/spirit/home/lex/tokenize_and_parse.hpp new file mode 100644 index 0000000..db12935 --- /dev/null +++ b/3rdParty/Boost/src/boost/spirit/home/lex/tokenize_and_parse.hpp @@ -0,0 +1,325 @@ +// Copyright (c) 2001-2011 Hartmut Kaiser +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#if !defined(BOOST_SPIRIT_LEXER_PARSE_NOV_17_2007_0246PM) +#define BOOST_SPIRIT_LEXER_PARSE_NOV_17_2007_0246PM + +#if defined(_MSC_VER) +#pragma once +#endif + +#include <boost/spirit/home/qi/skip_over.hpp> +#include <boost/spirit/home/qi/parse.hpp> +#include <boost/spirit/home/qi/nonterminal/grammar.hpp> +#include <boost/spirit/home/support/unused.hpp> +#include <boost/spirit/home/lex/lexer.hpp> +#include <boost/mpl/assert.hpp> + +namespace boost { namespace spirit { namespace lex +{ + /////////////////////////////////////////////////////////////////////////// + // Import skip_flag enumerator type from Qi namespace + using qi::skip_flag; + + /////////////////////////////////////////////////////////////////////////// + // + // The tokenize_and_parse() function is one of the main Spirit API + // functions. It simplifies using a lexer as the underlying token source + // while parsing a given input sequence. + // + // The function takes a pair of iterators spanning the underlying input + // stream to parse, the lexer object (built from the token definitions) + // and a parser object (built from the parser grammar definition). + // + // The second version of this function additionally takes an attribute to + // be used as the top level data structure instance the parser should use + // to store the recognized input to. + // + // The function returns true if the parsing succeeded (the given input + // sequence has been successfully matched by the given grammar). + // + // first, last: The pair of iterators spanning the underlying input + // sequence to parse. These iterators must at least + // conform to the requirements of the std::intput_iterator + // category. + // On exit the iterator 'first' will be updated to the + // position right after the last successfully matched + // token. + // lex: The lexer object (encoding the token definitions) to be + // used to convert the input sequence into a sequence of + // tokens. This token sequence is passed to the parsing + // process. The LexerExpr type must conform to the + // lexer interface described in the corresponding section + // of the documentation. + // xpr: The grammar object (encoding the parser grammar) to be + // used to match the token sequence generated by the lex + // object instance. The ParserExpr type must conform to + // the grammar interface described in the corresponding + // section of the documentation. + // attr: The top level attribute passed to the parser. It will + // be populated during the parsing of the input sequence. + // On exit it will hold the 'parser result' corresponding + // to the matched input sequence. + // + /////////////////////////////////////////////////////////////////////////// + template <typename Iterator, typename Lexer, typename ParserExpr> + inline bool + tokenize_and_parse(Iterator& first, Iterator last, Lexer const& lex, + ParserExpr const& xpr) + { + // Report invalid expression error as early as possible. + // If you got an error_invalid_expression error message here, + // then the expression (expr) is not a valid spirit qi expression. + BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); + + typename Lexer::iterator_type iter = lex.begin(first, last); + return compile<qi::domain>(xpr).parse( + iter, lex.end(), unused, unused, unused); + } + + /////////////////////////////////////////////////////////////////////////// + template <typename Iterator, typename Lexer, typename ParserExpr + , typename Attribute> + inline bool + tokenize_and_parse(Iterator& first, Iterator last, Lexer const& lex + , ParserExpr const& xpr, Attribute& attr) + { + // Report invalid expression error as early as possible. + // If you got an error_invalid_expression error message here, + // then the expression (expr) is not a valid spirit qi expression. + BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); + + typename Lexer::iterator_type iter = lex.begin(first, last); + return compile<qi::domain>(xpr).parse( + iter, lex.end(), unused, unused, attr); + } + + /////////////////////////////////////////////////////////////////////////// + // + // The tokenize_and_phrase_parse() function is one of the main Spirit API + // functions. It simplifies using a lexer as the underlying token source + // while phrase parsing a given input sequence. + // + // The function takes a pair of iterators spanning the underlying input + // stream to parse, the lexer object (built from the token definitions) + // and a parser object (built from the parser grammar definition). The + // additional skipper parameter will be used as the skip parser during + // the parsing process. + // + // The second version of this function additionally takes an attribute to + // be used as the top level data structure instance the parser should use + // to store the recognized input to. + // + // The function returns true if the parsing succeeded (the given input + // sequence has been successfully matched by the given grammar). + // + // first, last: The pair of iterators spanning the underlying input + // sequence to parse. These iterators must at least + // conform to the requirements of the std::intput_iterator + // category. + // On exit the iterator 'first' will be updated to the + // position right after the last successfully matched + // token. + // lex: The lexer object (encoding the token definitions) to be + // used to convert the input sequence into a sequence of + // tokens. This token sequence is passed to the parsing + // process. The LexerExpr type must conform to the + // lexer interface described in the corresponding section + // of the documentation. + // xpr: The grammar object (encoding the parser grammar) to be + // used to match the token sequence generated by the lex + // object instance. The ParserExpr type must conform to + // the grammar interface described in the corresponding + // section of the documentation. + // skipper: The skip parser to be used while parsing the given + // input sequence. Note, the skip parser will have to + // act on the same token sequence as the main parser + // 'xpr'. + // post_skip: The post_skip flag controls whether the function will + // invoke an additional post skip after the main parser + // returned. + // attr: The top level attribute passed to the parser. It will + // be populated during the parsing of the input sequence. + // On exit it will hold the 'parser result' corresponding + // to the matched input sequence. + // + /////////////////////////////////////////////////////////////////////////// + template <typename Iterator, typename Lexer, typename ParserExpr + , typename Skipper> + inline bool + tokenize_and_phrase_parse(Iterator& first, Iterator last + , Lexer const& lex, ParserExpr const& xpr, Skipper const& skipper + , BOOST_SCOPED_ENUM(skip_flag) post_skip = skip_flag::postskip) + { + // Report invalid expression error as early as possible. + // If you got an error_invalid_expression error message here, + // then the expression (expr) is not a valid spirit qi expression. + BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); + BOOST_SPIRIT_ASSERT_MATCH(qi::domain, Skipper); + + typedef + typename spirit::result_of::compile<qi::domain, Skipper>::type + skipper_type; + skipper_type const skipper_ = compile<qi::domain>(skipper); + + typename Lexer::iterator_type iter = lex.begin(first, last); + typename Lexer::iterator_type end = lex.end(); + if (!compile<qi::domain>(xpr).parse( + iter, end, unused, skipper_, unused)) + return false; + + // do a final post-skip + if (post_skip == skip_flag::postskip) + qi::skip_over(iter, end, skipper_); + return true; + } + + template <typename Iterator, typename Lexer, typename ParserExpr + , typename Skipper, typename Attribute> + inline bool + tokenize_and_phrase_parse(Iterator& first, Iterator last + , Lexer const& lex, ParserExpr const& xpr, Skipper const& skipper + , BOOST_SCOPED_ENUM(skip_flag) post_skip, Attribute& attr) + { + // Report invalid expression error as early as possible. + // If you got an error_invalid_expression error message here, + // then the expression (expr) is not a valid spirit qi expression. + BOOST_SPIRIT_ASSERT_MATCH(qi::domain, ParserExpr); + BOOST_SPIRIT_ASSERT_MATCH(qi::domain, Skipper); + + typedef + typename spirit::result_of::compile<qi::domain, Skipper>::type + skipper_type; + skipper_type const skipper_ = compile<qi::domain>(skipper); + + typename Lexer::iterator_type iter = lex.begin(first, last); + typename Lexer::iterator_type end = lex.end(); + if (!compile<qi::domain>(xpr).parse( + iter, end, unused, skipper_, attr)) + return false; + + // do a final post-skip + if (post_skip == skip_flag::postskip) + qi::skip_over(iter, end, skipper_); + return true; + } + + /////////////////////////////////////////////////////////////////////////// + template <typename Iterator, typename Lexer, typename ParserExpr + , typename Skipper, typename Attribute> + inline bool + tokenize_and_phrase_parse(Iterator& first, Iterator last + , Lexer const& lex, ParserExpr const& xpr, Skipper const& skipper + , Attribute& attr) + { + return tokenize_and_phrase_parse(first, last, lex, xpr, skipper + , skip_flag::postskip, attr); + } + + /////////////////////////////////////////////////////////////////////////// + // + // The tokenize() function is one of the main Spirit API functions. It + // simplifies using a lexer to tokenize a given input sequence. It's main + // purpose is to use the lexer to tokenize all the input. + // + // The second version below discards all generated tokens afterwards. + // This is useful whenever all the needed functionality has been + // implemented directly inside the lexer semantic actions, which are being + // executed while the tokens are matched. + // + // The function takes a pair of iterators spanning the underlying input + // stream to scan, the lexer object (built from the token definitions), + // and a (optional) functor being called for each of the generated tokens. + // + // The function returns true if the scanning of the input succeeded (the + // given input sequence has been successfully matched by the given token + // definitions). + // + // first, last: The pair of iterators spanning the underlying input + // sequence to parse. These iterators must at least + // conform to the requirements of the std::intput_iterator + // category. + // On exit the iterator 'first' will be updated to the + // position right after the last successfully matched + // token. + // lex: The lexer object (encoding the token definitions) to be + // used to convert the input sequence into a sequence of + // tokens. The LexerExpr type must conform to the + // lexer interface described in the corresponding section + // of the documentation. + // f: A functor (callable object) taking a single argument of + // the token type and returning a bool, indicating whether + // the tokenization should be canceled. + // initial_state: The name of the state the lexer should start matching. + // The default value is zero, causing the lexer to start + // in its 'INITIAL' state. + // + /////////////////////////////////////////////////////////////////////////// + namespace detail + { + template <typename Token, typename F> + bool tokenize_callback(Token const& t, F f) + { + return f(t); + } + + template <typename Token, typename Eval> + bool tokenize_callback(Token const& t, phoenix::actor<Eval> const& f) + { + f(t); + return true; + } + + template <typename Token> + bool tokenize_callback(Token const& t, void (*f)(Token const&)) + { + f(t); + return true; + } + + template <typename Token> + bool tokenize_callback(Token const& t, bool (*f)(Token const&)) + { + return f(t); + } + } + + template <typename Iterator, typename Lexer, typename F> + inline bool + tokenize(Iterator& first, Iterator last, Lexer const& lex, F f + , typename Lexer::char_type const* initial_state = 0) + { + typedef typename Lexer::iterator_type iterator_type; + + iterator_type iter = lex.begin(first, last, initial_state); + iterator_type end = lex.end(); + for (/**/; iter != end && token_is_valid(*iter); ++iter) + { + if (!detail::tokenize_callback(*iter, f)) + return false; + } + return (iter == end) ? true : false; + } + + /////////////////////////////////////////////////////////////////////////// + template <typename Iterator, typename Lexer> + inline bool + tokenize(Iterator& first, Iterator last, Lexer const& lex + , typename Lexer::char_type const* initial_state = 0) + { + typedef typename Lexer::iterator_type iterator_type; + + iterator_type iter = lex.begin(first, last, initial_state); + iterator_type end = lex.end(); + + while (iter != end && token_is_valid(*iter)) + ++iter; + + return (iter == end) ? true : false; + } + +}}} + +#endif |