A11 (C++ runtime)
Native C++ implementation of the A11 streaming action runtime
Loading...
Searching...
No Matches
json_codec.cc
Go to the documentation of this file.
1// Copyright 2026 The A11 Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#include "a11/json_codec.h"
16
17#include <bit>
18#include <cstddef>
19#include <cstdint>
20#include <exception>
21#include <limits>
22#include <string>
23#include <string_view>
24#include <vector>
25
26#include <absl/status/status.h>
27#include <absl/status/status_macros.h>
28#include <absl/status/statusor.h>
29#include <absl/strings/str_cat.h>
30#include <nlohmann/json.hpp>
31
32#include "a11/utf8.h"
33
34namespace a11 {
35namespace {
36
37using Json = nlohmann::json;
38
40constexpr int kMaxMsgpackNesting = 256;
41
42// Decodes MessagePack markers into the same JSON representations as
43// nlohmann::json::from_msgpack.
44class MsgpackDecoder {
45 public:
46 explicit MsgpackDecoder(std::string_view bytes) : bytes_(bytes) {}
47
49 absl::StatusOr<Json> Decode(int depth);
50
52 [[nodiscard]] size_t position() const { return position_; }
53
54 private:
55 absl::Status Need(size_t count) const {
56 if (position_ > bytes_.size() || count > bytes_.size() - position_) {
57 return absl::InvalidArgumentError("truncated");
58 }
59 return absl::OkStatus();
60 }
61
63 template <typename Unsigned>
64 absl::StatusOr<Unsigned> ReadBigEndian() {
65 ABSL_RETURN_IF_ERROR(Need(sizeof(Unsigned)));
66 Unsigned value = 0;
67 for (size_t index = 0; index < sizeof(Unsigned); ++index) {
68 value = static_cast<Unsigned>(
69 static_cast<Unsigned>(value << 8U) |
70 static_cast<unsigned char>(bytes_[position_ + index]));
71 }
72 position_ += sizeof(Unsigned);
73 return value;
74 }
75
77 absl::StatusOr<std::string_view> ReadPayload(std::uint64_t length) {
78 if (length > std::numeric_limits<size_t>::max()) {
79 return absl::ResourceExhaustedError("value is too large");
80 }
81 const auto count = static_cast<size_t>(length);
82 ABSL_RETURN_IF_ERROR(Need(count));
83 const std::string_view payload = bytes_.substr(position_, count);
84 position_ += count;
85 return payload;
86 }
87
89 absl::StatusOr<std::uint64_t> ReadStringLength(std::uint8_t marker);
90 absl::StatusOr<std::string> DecodeString(std::uint8_t marker);
91 absl::StatusOr<Json> DecodeBinary(std::uint8_t marker);
92 absl::StatusOr<Json> DecodeArray(std::uint64_t count, int depth);
93 absl::StatusOr<Json> DecodeObject(std::uint64_t count, int depth);
94
95 std::string_view bytes_;
96 size_t position_ = 0;
97};
98
99absl::StatusOr<std::uint64_t> MsgpackDecoder::ReadStringLength(
100 std::uint8_t marker) {
101 if ((marker & 0xe0U) == 0xa0U) { // fixstr
102 return marker & 0x1fU;
103 }
104 switch (marker) {
105 case 0xd9:
106 return ReadBigEndian<std::uint8_t>();
107 case 0xda:
108 return ReadBigEndian<std::uint16_t>();
109 case 0xdb:
110 return ReadBigEndian<std::uint32_t>();
111 default:
112 return absl::InvalidArgumentError(
113 absl::StrCat("expected a string, found marker 0x",
114 absl::Hex(marker, absl::kZeroPad2)));
115 }
116}
117
118absl::StatusOr<std::string> MsgpackDecoder::DecodeString(std::uint8_t marker) {
119 ABSL_ASSIGN_OR_RETURN(const std::uint64_t length, ReadStringLength(marker));
120 ABSL_ASSIGN_OR_RETURN(const std::string_view payload, ReadPayload(length));
121 return std::string(payload);
122}
123
124absl::StatusOr<Json> MsgpackDecoder::DecodeBinary(std::uint8_t marker) {
125 std::uint64_t length = 0;
126 bool has_subtype = false;
127 switch (marker) {
128 case 0xc4: {
129 ABSL_ASSIGN_OR_RETURN(length, ReadBigEndian<std::uint8_t>());
130 break;
131 }
132 case 0xc5: {
133 ABSL_ASSIGN_OR_RETURN(length, ReadBigEndian<std::uint16_t>());
134 break;
135 }
136 case 0xc6: {
137 ABSL_ASSIGN_OR_RETURN(length, ReadBigEndian<std::uint32_t>());
138 break;
139 }
140 // ext carries its length before the subtype byte; fixext has the subtype
141 // byte and a length the marker names.
142 case 0xc7: {
143 ABSL_ASSIGN_OR_RETURN(length, ReadBigEndian<std::uint8_t>());
144 has_subtype = true;
145 break;
146 }
147 case 0xc8: {
148 ABSL_ASSIGN_OR_RETURN(length, ReadBigEndian<std::uint16_t>());
149 has_subtype = true;
150 break;
151 }
152 case 0xc9: {
153 ABSL_ASSIGN_OR_RETURN(length, ReadBigEndian<std::uint32_t>());
154 has_subtype = true;
155 break;
156 }
157 case 0xd4:
158 length = 1;
159 has_subtype = true;
160 break;
161 case 0xd5:
162 length = 2;
163 has_subtype = true;
164 break;
165 case 0xd6:
166 length = 4;
167 has_subtype = true;
168 break;
169 case 0xd7:
170 length = 8;
171 has_subtype = true;
172 break;
173 case 0xd8:
174 length = 16;
175 has_subtype = true;
176 break;
177 default:
178 return absl::InternalError("Unreachable MessagePack binary marker");
179 }
180 std::uint8_t subtype = 0;
181 if (has_subtype) {
182 ABSL_ASSIGN_OR_RETURN(subtype, ReadBigEndian<std::uint8_t>());
183 }
184 ABSL_ASSIGN_OR_RETURN(const std::string_view payload, ReadPayload(length));
185 std::vector<std::uint8_t> bytes(payload.begin(), payload.end());
186 return has_subtype ? Json::binary(std::move(bytes), subtype)
187 : Json::binary(std::move(bytes));
188}
189
190absl::StatusOr<Json> MsgpackDecoder::DecodeArray(std::uint64_t count,
191 int depth) {
192 Json value = Json::array();
193 for (std::uint64_t index = 0; index < count; ++index) {
194 ABSL_ASSIGN_OR_RETURN(Json element, Decode(depth + 1));
195 value.emplace_back(std::move(element));
196 }
197 return value;
198}
199
200absl::StatusOr<Json> MsgpackDecoder::DecodeObject(std::uint64_t count,
201 int depth) {
202 Json value = Json::object();
203 for (std::uint64_t index = 0; index < count; ++index) {
204 ABSL_RETURN_IF_ERROR(Need(1));
205 const auto marker = static_cast<std::uint8_t>(bytes_[position_++]);
206 ABSL_ASSIGN_OR_RETURN(const std::string key, DecodeString(marker));
207 ABSL_ASSIGN_OR_RETURN(Json element, Decode(depth + 1));
208 // Assignment preserves from_msgpack's last-value rule for repeated keys.
209 value[key] = std::move(element);
210 }
211 return value;
212}
213
214absl::StatusOr<Json> MsgpackDecoder::Decode(int depth) {
215 if (depth > kMaxMsgpackNesting) {
216 return absl::ResourceExhaustedError("nesting is too deep");
217 }
218 ABSL_RETURN_IF_ERROR(Need(1));
219 const auto marker = static_cast<std::uint8_t>(bytes_[position_++]);
220
221 if (marker <= 0x7f) { // positive fixint
222 return Json(static_cast<std::uint64_t>(marker));
223 }
224 if (marker >= 0xe0) { // negative fixint
225 return Json(static_cast<std::int64_t>(static_cast<std::int8_t>(marker)));
226 }
227 if ((marker & 0xe0U) == 0xa0U) { // fixstr
228 ABSL_ASSIGN_OR_RETURN(std::string text, DecodeString(marker));
229 return Json(std::move(text));
230 }
231 if ((marker & 0xf0U) == 0x90U) { // fixarray
232 return DecodeArray(marker & 0x0fU, depth);
233 }
234 if ((marker & 0xf0U) == 0x80U) { // fixmap
235 return DecodeObject(marker & 0x0fU, depth);
236 }
237
238 switch (marker) {
239 case 0xc0:
240 return Json(nullptr);
241 case 0xc2:
242 return Json(false);
243 case 0xc3:
244 return Json(true);
245 case 0xc4:
246 case 0xc5:
247 case 0xc6:
248 case 0xc7:
249 case 0xc8:
250 case 0xc9:
251 case 0xd4:
252 case 0xd5:
253 case 0xd6:
254 case 0xd7:
255 case 0xd8:
256 return DecodeBinary(marker);
257 case 0xca: {
258 ABSL_ASSIGN_OR_RETURN(const std::uint32_t bits,
259 ReadBigEndian<std::uint32_t>());
260 return Json(static_cast<double>(std::bit_cast<float>(bits)));
261 }
262 case 0xcb: {
263 ABSL_ASSIGN_OR_RETURN(const std::uint64_t bits,
264 ReadBigEndian<std::uint64_t>());
265 return Json(std::bit_cast<double>(bits));
266 }
267 case 0xcc: {
268 ABSL_ASSIGN_OR_RETURN(const std::uint8_t number,
269 ReadBigEndian<std::uint8_t>());
270 return Json(static_cast<std::uint64_t>(number));
271 }
272 case 0xcd: {
273 ABSL_ASSIGN_OR_RETURN(const std::uint16_t number,
274 ReadBigEndian<std::uint16_t>());
275 return Json(static_cast<std::uint64_t>(number));
276 }
277 case 0xce: {
278 ABSL_ASSIGN_OR_RETURN(const std::uint32_t number,
279 ReadBigEndian<std::uint32_t>());
280 return Json(static_cast<std::uint64_t>(number));
281 }
282 case 0xcf: {
283 ABSL_ASSIGN_OR_RETURN(const std::uint64_t number,
284 ReadBigEndian<std::uint64_t>());
285 return Json(number);
286 }
287 case 0xd0: {
288 ABSL_ASSIGN_OR_RETURN(const std::uint8_t raw,
289 ReadBigEndian<std::uint8_t>());
290 return Json(static_cast<std::int64_t>(static_cast<std::int8_t>(raw)));
291 }
292 case 0xd1: {
293 ABSL_ASSIGN_OR_RETURN(const std::uint16_t raw,
294 ReadBigEndian<std::uint16_t>());
295 return Json(static_cast<std::int64_t>(static_cast<std::int16_t>(raw)));
296 }
297 case 0xd2: {
298 ABSL_ASSIGN_OR_RETURN(const std::uint32_t raw,
299 ReadBigEndian<std::uint32_t>());
300 return Json(static_cast<std::int64_t>(static_cast<std::int32_t>(raw)));
301 }
302 case 0xd3: {
303 ABSL_ASSIGN_OR_RETURN(const std::uint64_t raw,
304 ReadBigEndian<std::uint64_t>());
305 return Json(static_cast<std::int64_t>(raw));
306 }
307 case 0xd9:
308 case 0xda:
309 case 0xdb: {
310 ABSL_ASSIGN_OR_RETURN(std::string text, DecodeString(marker));
311 return Json(std::move(text));
312 }
313 case 0xdc: {
314 ABSL_ASSIGN_OR_RETURN(const std::uint16_t count,
315 ReadBigEndian<std::uint16_t>());
316 return DecodeArray(count, depth);
317 }
318 case 0xdd: {
319 ABSL_ASSIGN_OR_RETURN(const std::uint32_t count,
320 ReadBigEndian<std::uint32_t>());
321 return DecodeArray(count, depth);
322 }
323 case 0xde: {
324 ABSL_ASSIGN_OR_RETURN(const std::uint16_t count,
325 ReadBigEndian<std::uint16_t>());
326 return DecodeObject(count, depth);
327 }
328 case 0xdf: {
329 ABSL_ASSIGN_OR_RETURN(const std::uint32_t count,
330 ReadBigEndian<std::uint32_t>());
331 return DecodeObject(count, depth);
332 }
333 default:
334 return absl::InvalidArgumentError(absl::StrCat(
335 "unsupported marker 0x", absl::Hex(marker, absl::kZeroPad2)));
336 }
337}
338
339template <typename JsonValue>
340class StatusJsonParser final
341 : public nlohmann::detail::json_sax_dom_parser<
342 JsonValue, nlohmann::detail::iterator_input_adapter<const char*>> {
343 public:
344 explicit StatusJsonParser(JsonValue& value)
345 : nlohmann::detail::json_sax_dom_parser<
346 JsonValue, nlohmann::detail::iterator_input_adapter<const char*>>(
347 value, false) {}
348
349 template <typename Exception>
350 bool parse_error(size_t, const std::string&, const Exception& error) {
351 diagnostic = error.what();
352 return false;
353 }
354
355 std::string diagnostic;
356};
357
358template <typename JsonValue>
359absl::StatusOr<JsonValue> ParseJsonImpl(std::string_view encoded,
360 std::string_view what) {
361 JsonValue value;
362 StatusJsonParser<JsonValue> parser(value);
363 // The local SAX type keeps error handling independent of other translation
364 // units' JSON_NOEXCEPTION instantiations and retains nlohmann's diagnostics.
365 if (JsonValue::sax_parse(encoded.begin(), encoded.end(), &parser)) {
366 return value;
367 }
368 return absl::InvalidArgumentError(
369 absl::StrCat("Failed to parse ", what, ": ", parser.diagnostic));
370}
371} // namespace
372
373absl::StatusOr<Json> ParseJson(std::string_view encoded,
374 std::string_view what) {
375 return ParseJsonImpl<Json>(encoded, what);
376}
377
378absl::StatusOr<nlohmann::ordered_json> ParseOrderedJson(
379 std::string_view encoded, std::string_view what) {
380 return ParseJsonImpl<nlohmann::ordered_json>(encoded, what);
381}
382
383bool IsValidUtf8(std::string_view text) {
384 return utf8::IsValid(text);
385}
386
387namespace {
388template <typename JsonValue>
389const JsonValue* FindUnencodableStringImpl(const JsonValue& value) {
390 std::vector<const JsonValue*> pending{&value};
391 while (!pending.empty()) {
392 const JsonValue* one = pending.back();
393 pending.pop_back();
394 if (one->is_string()) {
395 if (!IsValidUtf8(one->template get_ref<const std::string&>())) {
396 return one;
397 }
398 continue;
399 }
400 if (one->is_object() || one->is_array()) {
401 for (const auto& element : *one) {
402 pending.push_back(&element);
403 }
404 }
405 }
406 return nullptr;
407}
408
409template <typename JsonValue>
410absl::StatusOr<std::string> DumpJsonImpl(const JsonValue& value,
411 std::string_view what) {
412 // Validate UTF-8 before a linked no-exceptions dump() can abort on it.
413 if (FindUnencodableStringImpl(value) != nullptr) {
414 return absl::InvalidArgumentError(absl::StrCat(
415 "Failed to serialize ", what,
416 ": it holds bytes that are not valid UTF-8, which JSON has no spelling "
417 "for. MessagePack carries them exactly, and base64 carries them "
418 "through JSON."));
419 }
420 // No non-throwing form of a strict dump exists: error_handler_t::strict is
421 // the request to be told, and being told means a throw. Kept for everything
422 // that is not a bad string, where a throw is still a throw here.
423 try {
424 return value.dump(-1, ' ', false, JsonValue::error_handler_t::strict);
425 } catch (const std::exception& error) {
426 return absl::InvalidArgumentError(
427 absl::StrCat("Failed to serialize ", what, ": ", error.what()));
428 } catch (...) {
429 return absl::InvalidArgumentError(absl::StrCat(
430 "Failed to serialize ", what, " with a non-standard exception"));
431 }
432}
433
434} // namespace
435
436const Json* FindUnencodableString(const Json& value) {
437 return FindUnencodableStringImpl(value);
438}
439
440absl::StatusOr<std::string> DumpJson(const Json& value, std::string_view what) {
441 return DumpJsonImpl(value, what);
442}
443
444absl::StatusOr<std::string> DumpOrderedJson(const nlohmann::ordered_json& value,
445 std::string_view what) {
446 return DumpJsonImpl(value, what);
447}
448
449std::string DumpJsonLossy(const Json& value) {
450 // Cannot fail: replace substitutes U+FFFD for anything that is not valid
451 // UTF-8, which is why this one needs no Status and no try.
452 return value.dump(-1, ' ', false, Json::error_handler_t::replace);
453}
454
455namespace {
456template <typename JsonValue>
457absl::StatusOr<std::string> PackMsgpackImpl(const JsonValue& value,
458 std::string_view what) {
459 // to_msgpack has no allow_exceptions overload, and it does raise: a string
460 // field holding invalid UTF-8 is type_error.316.
461 try {
462 // The string-backed adapter writes directly into the returned buffer.
463 std::string encoded;
464 JsonValue::to_msgpack(value,
465 nlohmann::detail::output_adapter<char>(encoded));
466 return encoded;
467 } catch (const std::exception& error) {
468 return absl::InvalidArgumentError(absl::StrCat(
469 "Failed to encode ", what, " as MessagePack: ", error.what()));
470 } catch (...) {
471 return absl::InvalidArgumentError(
472 absl::StrCat("Failed to encode ", what,
473 " as MessagePack with a non-standard exception"));
474 }
475}
476
477} // namespace
478
479absl::StatusOr<std::string> PackMsgpack(const Json& value,
480 std::string_view what) {
481 return PackMsgpackImpl(value, what);
482}
483
484absl::StatusOr<std::string> PackOrderedMsgpack(
485 const nlohmann::ordered_json& value, std::string_view what) {
486 return PackMsgpackImpl(value, what);
487}
488
489absl::StatusOr<Json> UnpackMsgpack(std::string_view encoded,
490 std::string_view what) {
491 MsgpackDecoder decoder(encoded);
492 absl::StatusOr<Json> value = decoder.Decode(0);
493 if (!value.ok()) {
494 return absl::Status(value.status().code(),
495 absl::StrCat("Invalid ", what, " MessagePack data: ",
496 value.status().message()));
497 }
498 // One record contains exactly one value.
499 if (decoder.position() != encoded.size()) {
500 return absl::InvalidArgumentError(
501 absl::StrCat("Invalid ", what, " MessagePack data: ",
502 encoded.size() - decoder.position(), " trailing bytes"));
503 }
504 return value;
505}
506
507} // namespace a11
std::string value
Definition discover.cc:100
std::string key
Definition discover.cc:764
std::optional< std::string > text
The text, where the value is a string literal or a name bound to one.
Definition discover.cc:596
Status-returning JSON and MessagePack codecs.
size_t depth
Definition module.cc:55
std::string Decode(std::string_view value, bool plus_is_space=false)
Percent-decodes value, leaving anything malformed as it stands.
Definition percent.h:63
bool IsValid(std::string_view text)
Whether text is well-formed UTF-8.
Definition utf8.h:70
Definition action.cc:66
absl::StatusOr< Json > ParseJson(std::string_view encoded, std::string_view what)
Parses JSON text, returning a status for invalid input.
Definition json_codec.cc:373
const Json * FindUnencodableString(const Json &value)
Definition json_codec.cc:436
absl::StatusOr< std::string > DumpOrderedJson(const nlohmann::ordered_json &value, std::string_view what)
Serialize host fields retaining their declared insertion order.
Definition json_codec.cc:444
std::string DumpJsonLossy(const Json &value)
Definition json_codec.cc:449
absl::StatusOr< Json > UnpackMsgpack(std::string_view encoded, std::string_view what)
Decodes MessagePack bytes, or explains why they are not.
Definition json_codec.cc:489
absl::StatusOr< std::string > PackOrderedMsgpack(const nlohmann::ordered_json &value, std::string_view what)
Encode host fields retaining their declared insertion order.
Definition json_codec.cc:484
absl::StatusOr< std::string > DumpJson(const Json &value, std::string_view what)
Definition json_codec.cc:440
bool IsValidUtf8(std::string_view text)
Whether text is valid strict UTF-8.
Definition json_codec.cc:383
absl::StatusOr< nlohmann::ordered_json > ParseOrderedJson(std::string_view encoded, std::string_view what)
Parse host fields retaining their declared insertion order.
Definition json_codec.cc:378
absl::StatusOr< std::string > PackMsgpack(const Json &value, std::string_view what)
Definition json_codec.cc:479
std::string_view detail
Definition navigate.cc:475
std::optional< std::uint64_t > length
Definition runtime_streams.cc:37
UTF-8 byte-level primitives, as inline functions over a string_view.
int bits
Definition values.cc:64