diff options
author | Remko Tronçon <git@el-tramo.be> | 2013-12-28 16:13:08 (GMT) |
---|---|---|
committer | Remko Tronçon <git@el-tramo.be> | 2014-01-03 11:09:06 (GMT) |
commit | 0b19dc7292b7672c9fbb711a411c392bc5b2bb34 (patch) | |
tree | d5af147193d52a3e63d348dd794ca92cdd934974 /Sluift/tokenize.cpp | |
parent | 5a89265623214164fa7ce36721de05183d53058d (diff) | |
download | swift-0b19dc7292b7672c9fbb711a411c392bc5b2bb34.zip swift-0b19dc7292b7672c9fbb711a411c392bc5b2bb34.tar.bz2 |
Sluift: Custom console/interpreter
- Prints results of each command (if it can be interpreted as an
expression)
- Stores results of last command in _1, _2, ...
- Supports tab completion
- Compatible with Lua 5.2
Other changes:
- Add support for specifying custom editline library
- Don't load sluift into global namespace. Tab completion should be
convenient enough.
Change-Id: I2a26346469d67c281d09d47cacaa0b267f5ea9f9
Diffstat (limited to 'Sluift/tokenize.cpp')
-rw-r--r-- | Sluift/tokenize.cpp | 86 |
1 files changed, 86 insertions, 0 deletions
diff --git a/Sluift/tokenize.cpp b/Sluift/tokenize.cpp new file mode 100644 index 0000000..b089cdb --- /dev/null +++ b/Sluift/tokenize.cpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2013 Remko Tronçon + * Licensed under the GNU General Public License. + * See the COPYING file for more information. + */ + +#include <Sluift/tokenize.h> + +#include <boost/tokenizer.hpp> +#include <cctype> + +using namespace Swift; + +namespace { + struct LuaTokenizeFunctor { + void reset() { + } + + template<typename InputIterator, typename Token> + bool operator()(InputIterator& next, InputIterator& end, Token& result) { + while (next != end && std::isspace(*next)) { + ++next; + } + if (next == end) { + return false; + } + + std::vector<char> token; + char c = *next++; + token.push_back(c); + + // String literal + if (c == '\'' || c == '"') { + char quote = c; + bool inEscape = false; + for (; next != end; ++next) { + c = *next; + token.push_back(c); + if (inEscape) { + inEscape = false; + } + else if (c == '\\') { + inEscape = true; + } + else if (c == quote) { + break; + } + } + if (next != end) { + ++next; + } + } + // Identifier + else if (std::isalpha(c) || c == '_') { + while (next != end && (std::isalpha(*next) || *next == '_' || std::isdigit(*next))) { + token.push_back(*next); + ++next; + } + } + // Digit + else if (std::isdigit(c)) { + while (next != end && !std::isspace(*next)) { + token.push_back(*next); + ++next; + } + } + // Dots + else if (c == '.') { + while (next != end && *next == '.') { + token.push_back(*next); + ++next; + } + } + + result = Token(&token[0], token.size()); + return true; + } + }; +} + + +std::vector<std::string> Lua::tokenize(const std::string& input) { + boost::tokenizer<LuaTokenizeFunctor> tokenizer(input); + return std::vector<std::string>(tokenizer.begin(), tokenizer.end()); +} + |