/*
 * Copyright (c) 2013 Isode Limited.
 * All rights reserved.
 * See the COPYING file for more information.
 */

#include <Sluift/tokenize.h>

#include <boost/tokenizer.hpp>
#include <cctype>

using namespace Swift;

namespace {
	struct LuaTokenizeFunctor {
		void reset() {
		}

		template<typename InputIterator, typename Token>
		bool operator()(InputIterator& next, InputIterator& end, Token& result) {
			while (next != end && std::isspace(*next)) {
				++next;
			}
			if (next == end) {
				return false;
			}

			std::vector<char> token;
			char c = *next++;
			token.push_back(c);

			// String literal
			if (c == '\'' || c == '"') {
				char quote = c;
				bool inEscape = false;
				for (; next != end; ++next) {
					c = *next;
					token.push_back(c);
					if (inEscape) {
						inEscape = false;
					}
					else if (c == '\\') {
						inEscape = true;
					}
					else if (c == quote) {
						break;
					}
				}
				if (next != end) {
					++next;
				}
			}
			// Identifier
			else if (std::isalpha(c) || c == '_') {
				while (next != end && (std::isalpha(*next) || *next == '_' || std::isdigit(*next))) {
					token.push_back(*next);
					++next;
				}
			}
			// Digit
			else if (std::isdigit(c)) {
				while (next != end && !std::isspace(*next)) {
					token.push_back(*next);
					++next;
				}
			}
			// Dots
			else if (c == '.') {
				while (next != end && *next == '.') {
					token.push_back(*next);
					++next;
				}
			}
			
			result = Token(&token[0], token.size());
			return true;
		}
	};
}


std::vector<std::string> Lua::tokenize(const std::string& input) {
	boost::tokenizer<LuaTokenizeFunctor> tokenizer(input);
	return std::vector<std::string>(tokenizer.begin(), tokenizer.end());
}