summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat (limited to 'Sluift/tokenize.cpp')
-rw-r--r--Sluift/tokenize.cpp131
1 files changed, 66 insertions, 65 deletions
diff --git a/Sluift/tokenize.cpp b/Sluift/tokenize.cpp
index 55ab744..ff162d6 100644
--- a/Sluift/tokenize.cpp
+++ b/Sluift/tokenize.cpp
@@ -1,86 +1,87 @@
/*
- * Copyright (c) 2013 Isode Limited.
+ * Copyright (c) 2013-2016 Isode Limited.
* All rights reserved.
* See the COPYING file for more information.
*/
#include <Sluift/tokenize.h>
-#include <boost/tokenizer.hpp>
#include <cctype>
+#include <boost/tokenizer.hpp>
+
using namespace Swift;
namespace {
- struct LuaTokenizeFunctor {
- void reset() {
- }
+ struct LuaTokenizeFunctor {
+ void reset() {
+ }
+
+ template<typename InputIterator, typename Token>
+ bool operator()(InputIterator& next, InputIterator& end, Token& result) {
+ while (next != end && std::isspace(*next)) {
+ ++next;
+ }
+ if (next == end) {
+ return false;
+ }
- template<typename InputIterator, typename Token>
- bool operator()(InputIterator& next, InputIterator& end, Token& result) {
- while (next != end && std::isspace(*next)) {
- ++next;
- }
- if (next == end) {
- return false;
- }
+ std::vector<char> token;
+ char c = *next++;
+ token.push_back(c);
- std::vector<char> token;
- char c = *next++;
- token.push_back(c);
+ // String literal
+ if (c == '\'' || c == '"') {
+ char quote = c;
+ bool inEscape = false;
+ for (; next != end; ++next) {
+ c = *next;
+ token.push_back(c);
+ if (inEscape) {
+ inEscape = false;
+ }
+ else if (c == '\\') {
+ inEscape = true;
+ }
+ else if (c == quote) {
+ break;
+ }
+ }
+ if (next != end) {
+ ++next;
+ }
+ }
+ // Identifier
+ else if (std::isalpha(c) || c == '_') {
+ while (next != end && (std::isalpha(*next) || *next == '_' || std::isdigit(*next))) {
+ token.push_back(*next);
+ ++next;
+ }
+ }
+ // Digit
+ else if (std::isdigit(c)) {
+ while (next != end && !std::isspace(*next)) {
+ token.push_back(*next);
+ ++next;
+ }
+ }
+ // Dots
+ else if (c == '.') {
+ while (next != end && *next == '.') {
+ token.push_back(*next);
+ ++next;
+ }
+ }
- // String literal
- if (c == '\'' || c == '"') {
- char quote = c;
- bool inEscape = false;
- for (; next != end; ++next) {
- c = *next;
- token.push_back(c);
- if (inEscape) {
- inEscape = false;
- }
- else if (c == '\\') {
- inEscape = true;
- }
- else if (c == quote) {
- break;
- }
- }
- if (next != end) {
- ++next;
- }
- }
- // Identifier
- else if (std::isalpha(c) || c == '_') {
- while (next != end && (std::isalpha(*next) || *next == '_' || std::isdigit(*next))) {
- token.push_back(*next);
- ++next;
- }
- }
- // Digit
- else if (std::isdigit(c)) {
- while (next != end && !std::isspace(*next)) {
- token.push_back(*next);
- ++next;
- }
- }
- // Dots
- else if (c == '.') {
- while (next != end && *next == '.') {
- token.push_back(*next);
- ++next;
- }
- }
-
- result = Token(&token[0], token.size());
- return true;
- }
- };
+ result = Token(&token[0], token.size());
+ return true;
+ }
+ };
}
std::vector<std::string> Lua::tokenize(const std::string& input) {
- boost::tokenizer<LuaTokenizeFunctor> tokenizer(input);
- return std::vector<std::string>(tokenizer.begin(), tokenizer.end());
+ boost::tokenizer<LuaTokenizeFunctor> tokenizer(input);
+ return std::vector<std::string>(tokenizer.begin(), tokenizer.end());
}