A11 (C++ runtime)
Native C++ implementation of the A11 streaming action runtime
Loading...
Searching...
No Matches
token.h
Go to the documentation of this file.
1/*
2 * Copyright 2026 The A11 Authors
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef A11_FLOW_TOKEN_H_
18#define A11_FLOW_TOKEN_H_
19
20#include <cstddef>
21#include <string>
22#include <string_view>
23#include <vector>
24
25#include <absl/time/time.h>
26
27namespace a11::flow {
28
37enum class TokenKind {
44 kString,
46 kNumber,
50 kWord,
51
52 kDot,
56 kRange,
58 kSpread,
59 kArrow,
60 kCarry,
61 kEqual,
64 kLess,
68 kPlus,
69 kMinus,
70 kPipe,
71 kColon,
73 kScope,
74 kStar,
75 kSlash,
77 kComma,
84
87 kBad,
89 kEnd,
90};
91
94 size_t start = 0;
95 size_t end = 0;
96 int line = 1;
97 int column = 1;
98};
99
104struct Token {
107 std::string_view text;
108 size_t start = 0;
109 size_t end = 0;
111 int line = 1;
112 int column = 1;
113
115 std::string string_value;
117 std::vector<StringEscape> string_escapes;
120 double number = 0.0;
121 long long integer = 0;
122 bool is_integer = false;
124 absl::Duration duration;
125
126 [[nodiscard]] bool IsWord() const { return kind == TokenKind::kWord; }
127
129 [[nodiscard]] bool EndsStatement() const {
132 }
133};
134
139std::string_view KindName(TokenKind kind);
140
142TokenKind KindFromName(std::string_view name);
143
144} // namespace a11::flow
145
146#endif // A11_FLOW_TOKEN_H_
TokenKind kind
Definition format.cc:1372
std::string name
The declaration name and its colon.
Definition format.cc:50
Definition catalogue.cc:29
TokenKind KindFromName(std::string_view name)
The kind a name from [KindName] refers to, or kBad if it is not one.
@ kBad
Something the language has no meaning for.
std::string_view KindName(TokenKind kind)
The name of a kind, as a11.flow.lexer spells it.
@ kCarry
What a repeat carries.
TokenKind
What kind of thing a token is.
Definition token.h:37
@ kScope
Static namespace qualification, distinct from runtime projection.
@ kRange
.. – the range between two bounds, either of which may be left out.
@ kEnd
One past the last token, so lookahead never runs off the end.
@ kWord
A bare word. What it means is the grammar's business, not the lexer's.
@ kNewline
The end of a statement. One per run of blank lines, never leading.
@ kSpread
... or ... – everything the thing after it holds, spread in here.
The source range of an escape inside a string token.
Definition token.h:93
size_t start
Definition token.h:94
int line
Definition token.h:96
size_t end
Definition token.h:95
int column
Definition token.h:97
One token: what it is, where it is, and what it holds.
Definition token.h:104
size_t start
Definition token.h:108
int line
1-based, as a11.flow.lexer has always reported them.
Definition token.h:111
std::vector< StringEscape > string_escapes
Escape ranges before multiline indentation is removed.
Definition token.h:117
bool IsWord() const
Definition token.h:126
int column
Definition token.h:112
long long integer
Definition token.h:121
bool is_integer
Definition token.h:122
std::string_view text
The source slice, exactly as written.
Definition token.h:107
bool EndsStatement() const
Whether this token ends a statement.
Definition token.h:129
size_t end
Definition token.h:109
std::string string_value
A kString's value, with escapes resolved.
Definition token.h:115
double number
A kNumber's value.
Definition token.h:120
absl::Duration duration
A kDuration's value.
Definition token.h:124
TokenKind kind
Definition token.h:105