/* * Copyright (c) 2013-2016 Isode Limited. * All rights reserved. * See the COPYING file for more information. */ #include <Sluift/tokenize.h> #include <cctype> #include <boost/tokenizer.hpp> using namespace Swift; namespace { struct LuaTokenizeFunctor { void reset() { } template<typename InputIterator, typename Token> bool operator()(InputIterator& next, InputIterator& end, Token& result) { while (next != end && std::isspace(*next)) { ++next; } if (next == end) { return false; } std::vector<char> token; char c = *next++; token.push_back(c); // String literal if (c == '\'' || c == '"') { char quote = c; bool inEscape = false; for (; next != end; ++next) { c = *next; token.push_back(c); if (inEscape) { inEscape = false; } else if (c == '\\') { inEscape = true; } else if (c == quote) { break; } } if (next != end) { ++next; } } // Identifier else if (std::isalpha(c) || c == '_') { while (next != end && (std::isalpha(*next) || *next == '_' || std::isdigit(*next))) { token.push_back(*next); ++next; } } // Digit else if (std::isdigit(c)) { while (next != end && !std::isspace(*next)) { token.push_back(*next); ++next; } } // Dots else if (c == '.') { while (next != end && *next == '.') { token.push_back(*next); ++next; } } result = Token(&token[0], token.size()); return true; } }; } std::vector<std::string> Lua::tokenize(const std::string& input) { boost::tokenizer<LuaTokenizeFunctor> tokenizer(input); return std::vector<std::string>(tokenizer.begin(), tokenizer.end()); }