summaryrefslogtreecommitdiffstats
blob: ff162d641c8047212b6615012e963ac84d67571b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
/*
 * Copyright (c) 2013-2016 Isode Limited.
 * All rights reserved.
 * See the COPYING file for more information.
 */

#include <Sluift/tokenize.h>

#include <cctype>

#include <boost/tokenizer.hpp>

using namespace Swift;

namespace {
    struct LuaTokenizeFunctor {
        void reset() {
        }

        template<typename InputIterator, typename Token>
        bool operator()(InputIterator& next, InputIterator& end, Token& result) {
            while (next != end && std::isspace(*next)) {
                ++next;
            }
            if (next == end) {
                return false;
            }

            std::vector<char> token;
            char c = *next++;
            token.push_back(c);

            // String literal
            if (c == '\'' || c == '"') {
                char quote = c;
                bool inEscape = false;
                for (; next != end; ++next) {
                    c = *next;
                    token.push_back(c);
                    if (inEscape) {
                        inEscape = false;
                    }
                    else if (c == '\\') {
                        inEscape = true;
                    }
                    else if (c == quote) {
                        break;
                    }
                }
                if (next != end) {
                    ++next;
                }
            }
            // Identifier
            else if (std::isalpha(c) || c == '_') {
                while (next != end && (std::isalpha(*next) || *next == '_' || std::isdigit(*next))) {
                    token.push_back(*next);
                    ++next;
                }
            }
            // Digit
            else if (std::isdigit(c)) {
                while (next != end && !std::isspace(*next)) {
                    token.push_back(*next);
                    ++next;
                }
            }
            // Dots
            else if (c == '.') {
                while (next != end && *next == '.') {
                    token.push_back(*next);
                    ++next;
                }
            }

            result = Token(&token[0], token.size());
            return true;
        }
    };
}


std::vector<std::string> Lua::tokenize(const std::string& input) {
    boost::tokenizer<LuaTokenizeFunctor> tokenizer(input);
    return std::vector<std::string>(tokenizer.begin(), tokenizer.end());
}