A11 (C++ runtime)
Native C++ implementation of the A11 streaming action runtime
Loading...
Searching...
No Matches
utf8.h
Go to the documentation of this file.
1/*
2 * Copyright 2026 The A11 Authors
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
25#ifndef A11_UTF8_H_
26#define A11_UTF8_H_
27
28#include <cstddef>
29#include <cstdint>
30#include <string_view>
31
32namespace a11::utf8 {
33
35inline bool IsContinuation(char byte) {
36 return (static_cast<unsigned char>(byte) & 0xC0U) == 0x80U;
37}
38
45inline size_t SequenceWidth(char lead) {
46 const auto byte = static_cast<unsigned char>(lead);
47 if (byte >= 0xF0U) {
48 return 4;
49 }
50 if (byte >= 0xE0U) {
51 return 3;
52 }
53 if (byte >= 0xC0U) {
54 return 2;
55 }
56 return 1;
57}
58
61inline size_t Utf16Units(char lead) {
62 return static_cast<unsigned char>(lead) >= 0xF0U ? 2 : 1;
63}
64
70inline bool IsValid(std::string_view text) {
71 // The smallest code point each width may encode; anything smaller is an
72 // overlong form of a shorter sequence.
73 static constexpr std::uint32_t kSmallest[] = {0, 0, 0x80, 0x800, 0x10000};
74 size_t at = 0;
75 while (at < text.size()) {
76 const auto lead = static_cast<unsigned char>(text[at]);
77 if (lead < 0x80U) {
78 ++at;
79 continue;
80 }
81 size_t width = 0;
82 std::uint32_t point = 0;
83 if ((lead & 0xE0U) == 0xC0U) {
84 width = 2;
85 point = lead & 0x1FU;
86 } else if ((lead & 0xF0U) == 0xE0U) {
87 width = 3;
88 point = lead & 0x0FU;
89 } else if ((lead & 0xF8U) == 0xF0U) {
90 width = 4;
91 point = lead & 0x07U;
92 } else {
93 // A continuation byte with no lead, or a 5-byte form UTF-8 has not had
94 // since 2003.
95 return false;
96 }
97 if (text.size() - at < width) {
98 return false;
99 }
100 for (size_t index = 1; index < width; ++index) {
101 if (!IsContinuation(text[at + index])) {
102 return false;
103 }
104 point = (point << 6U) |
105 (static_cast<unsigned char>(text[at + index]) & 0x3FU);
106 }
107 if (point < kSmallest[width] || point > 0x10FFFFU ||
108 (point >= 0xD800U && point <= 0xDFFFU)) {
109 return false;
110 }
111 at += width;
112 }
113 return true;
114}
115
116} // namespace a11::utf8
117
118#endif // A11_UTF8_H_
std::optional< std::string > text
The text, where the value is a string literal or a name bound to one.
Definition discover.cc:596
Definition utf8.h:32
bool IsContinuation(char byte)
Whether byte is a continuation byte – the middle of a character.
Definition utf8.h:35
size_t SequenceWidth(char lead)
Bytes in the sequence lead starts, from the lead byte alone.
Definition utf8.h:45
size_t Utf16Units(char lead)
UTF-16 code units the character at lead occupies: 2 above the BMP, else.
Definition utf8.h:61
bool IsValid(std::string_view text)
Whether text is well-formed UTF-8.
Definition utf8.h:70