35#ifndef A11_NODES_ASYNC_NODE_H_
36#define A11_NODES_ASYNC_NODE_H_
47#include <absl/base/thread_annotations.h>
48#include <absl/status/status.h>
49#include <absl/status/status_macros.h>
50#include <absl/status/statusor.h>
51#include <absl/time/time.h>
87 std::optional<std::uint32_t>
seq = {};
114class AsyncNode :
public std::enable_shared_from_this<AsyncNode> {
129 static absl::StatusOr<std::shared_ptr<AsyncNode>>
Create(
130 std::shared_ptr<stores::ChunkStore>
store,
147 absl::StatusOr<std::string>
GetId()
const;
155 [[nodiscard]] std::shared_ptr<stores::ChunkStore>
GetChunkStore()
const;
161 [[nodiscard]] std::shared_ptr<data::SerializationRegistry>
170 std::shared_ptr<data::SerializationRegistry> registry);
176 absl::StatusOr<std::shared_ptr<stores::ChunkStoreReader>>
reader();
182 absl::StatusOr<std::shared_ptr<stores::ChunkStoreWriter>>
writer();
204 std::optional<stores::ChunkStoreReaderOptions>
options = std::nullopt);
263 data::Chunk chunk, std::optional<std::uint32_t>
seq = std::nullopt,
304 template <
typename T>
307 std::optional<std::uint32_t>
seq = std::nullopt,
bool final =
false) {
308 std::shared_ptr<data::SerializationRegistry> registry;
311 registry = serialization_registry_;
314 data::MakeChunkObject<T>(std::move(
value), data::SerialTypeTag<T>(),
319 template <
typename T>
321 const T&
value, std::optional<std::uint32_t>
seq = std::nullopt,
322 bool final =
false, std::string_view
mimetype = {}) {
323 std::shared_ptr<data::SerializationRegistry> registry;
326 registry = serialization_registry_;
328 absl::StatusOr<data::Chunk> chunk = registry->ToChunk<T>(
value,
mimetype);
330 return a11::FailedFuture<std::uint32_t>(chunk.status());
379 template <
typename T>
381 std::shared_ptr<data::SerializationRegistry> registry;
384 registry = serialization_registry_;
386 absl::StatusOr<data::Chunk> chunk =
405 size_t limit, absl::Duration
timeout = absl::InfiniteDuration());
414 absl::Duration
timeout = absl::InfiniteDuration());
423 absl::Duration
timeout = absl::InfiniteDuration());
425 absl::Duration
timeout = absl::InfiniteDuration());
428 absl::Duration
timeout = absl::InfiniteDuration());
439 absl::Duration
timeout = absl::InfiniteDuration());
450 template <
typename T>
452 absl::Duration
timeout = absl::InfiniteDuration(),
453 std::vector<std::string> mimetype_patterns = {}) {
454 std::shared_ptr<AsyncNode> self = shared_from_this();
455 return a11::Submit<std::optional<T>>(
456 [self = std::move(self),
timeout,
457 mimetype_patterns = std::move(
458 mimetype_patterns)]()
mutable -> absl::StatusOr<std::optional<T>> {
462 absl::StatusOr<std::optional<data::NodeFragment>> fragment =
463 self->NextLogicalFragment(
timeout).Await();
464 if (!fragment.ok()) {
465 return fragment.status();
467 if (!fragment->has_value()) {
470 absl::StatusOr<const data::Chunk*> chunk = (*fragment)->GetChunk();
472 return chunk.status();
477 if ((*chunk)->HasObject()) {
481 if constexpr (data::HasSerialTypeTag<T>) {
482 if (std::optional<T> taken =
483 data::TryTakeObject<T>(**chunk, data::SerialTypeTag<T>());
491 data::Chunk materialised = **chunk;
492 ABSL_RETURN_IF_ERROR(materialised.Materialize());
493 std::shared_ptr<data::SerializationRegistry> registry;
496 registry = self->serialization_registry_;
498 absl::StatusOr<T>
value =
499 registry->FromChunk<T>(materialised, mimetype_patterns);
501 return value.status();
503 return std::optional<T>(std::move(*
value));
505 if ((*chunk)->IsNull()) {
506 if ((*fragment)->continued) {
507 return absl::FailedPreconditionError(
508 "A null stream marker must be final");
512 std::shared_ptr<data::SerializationRegistry> registry;
515 registry = self->serialization_registry_;
517 absl::StatusOr<T>
value =
518 registry->FromChunk<T>(**chunk, mimetype_patterns);
520 return value.status();
522 return std::optional<T>(std::move(*
value));
572 absl::Status
AttachStream(std::shared_ptr<net::WireStream> stream);
574 std::shared_ptr<net::WireStream> stream,
582 absl::Status
DetachStream(
const std::shared_ptr<net::WireStream>& stream);
601 void SetOwner(
const std::shared_ptr<NodeMap>& owner);
602 [[nodiscard]] std::shared_ptr<NodeMap>
Owner()
const;
604 absl::StatusOr<data::NodeRef>
Ref(
606 std::optional<std::uint64_t>
length = std::nullopt,
607 std::shared_ptr<NodeMap> destination =
nullptr,
610 absl::StatusOr<std::shared_ptr<stores::ChunkStoreReader>>
OpenReader(
611 stores::ChunkStoreReaderOptions
options)
const;
614 absl::StatusOr<std::shared_ptr<stores::LogicalChunkStoreReader>>
618 std::shared_ptr<data::SerializationRegistry> registry,
619 stores::ChunkStoreReaderOptions reader_options,
620 stores::ChunkStoreWriterOptions writer_options)
621 : store_(std::move(
store)),
622 serialization_registry_(std::move(registry)),
623 reader_options_(reader_options),
624 writer_options_(writer_options) {}
626 const std::shared_ptr<stores::ChunkStore> store_;
628 std::shared_ptr<data::SerializationRegistry> serialization_registry_
629 ABSL_GUARDED_BY(mu_);
630 stores::ChunkStoreReaderOptions reader_options_ ABSL_GUARDED_BY(mu_);
631 stores::ChunkStoreWriterOptions writer_options_ ABSL_GUARDED_BY(mu_);
632 std::shared_ptr<stores::ChunkStoreReader> reader_ ABSL_GUARDED_BY(mu_);
633 std::shared_ptr<stores::LogicalChunkStoreReader> logical_reader_
634 ABSL_GUARDED_BY(mu_);
635 std::shared_ptr<stores::ChunkStoreWriter> writer_ ABSL_GUARDED_BY(mu_);
636 std::weak_ptr<NodeMap> owner_ ABSL_GUARDED_BY(mu_);
std::shared_ptr< stores::ChunkStore > store
Definition async_node.cc:55
A11's pluggable storage interface for streamed node data: an ordered, appendable log of fragments key...
A read cursor over a ChunkStore that pulls fragments out in order (or by arrival),...
A write cursor over a ChunkStore that admits chunks into the store in sequence, applying backpressure...
An asynchronous, ordered stream of chunks read from and written to A11.
Definition async_node.h:114
bool LogicalSawFinal() const
Whether the logical source has supplied a final raw fragment.
Definition async_node.cc:661
absl::Status SetSerializationRegistry(std::shared_ptr< data::SerializationRegistry > registry)
Replace the serialization registry.
Definition async_node.cc:293
a11::Future< std::vector< std::optional< data::NodeFragment > > > NextLogicalFragments(size_t limit, absl::Duration timeout=absl::InfiniteDuration())
Definition async_node.cc:652
a11::Future< std::uint32_t > PutObject(T value, std::string_view mimetype, std::optional< std::uint32_t > seq=std::nullopt, bool final=false)
Serialize and write a typed value to the stream.
Definition async_node.h:305
std::shared_ptr< stores::ChunkStore > GetChunkStore() const
Return the underlying chunk store backing this node.
Definition async_node.cc:283
a11::Task WaitForBufferToDrain()
Wait for the write buffer to drain.
Definition async_node.cc:717
a11::Task Finalize(FinalizeOptions options={})
End the stream with an explicit null terminator (no value).
Definition async_node.cc:548
a11::Future< std::uint32_t > PutChunk(data::Chunk chunk, std::optional< std::uint32_t > seq=std::nullopt, bool final=false)
Enqueue a raw chunk into the stream.
Definition async_node.cc:520
stores::ChunkStoreWriterOptions GetWriterOptions() const
Return a copy of the writer's current options.
Definition async_node.cc:430
a11::Task Finalize(const T &value, FinalizeOptions options={})
Serialize value as the stream's final data, then end the stream.
Definition async_node.h:380
a11::Future< std::optional< data::NodeFragment > > NextLogicalFragment(absl::Duration timeout=absl::InfiniteDuration())
Definition async_node.cc:642
a11::Task Close()
Flush all buffered chunks and close the writer, without finality.
Definition async_node.cc:726
stores::ChunkStoreReaderOptions GetReaderOptions() const
Return a copy of the reader's current options.
Definition async_node.cc:385
static absl::StatusOr< std::shared_ptr< AsyncNode > > Create(std::shared_ptr< stores::ChunkStore > store, std::shared_ptr< data::SerializationRegistry > serialization_registry=nullptr, stores::ChunkStoreReaderOptions reader_options={}, stores::ChunkStoreWriterOptions writer_options={})
Create a node over an existing chunk store.
Definition async_node.cc:204
a11::Task AbortWithStatus(absl::Status status)
Fail the stream with an error status.
Definition async_node.cc:732
absl::Status DetachStream(const std::shared_ptr< net::WireStream > &stream)
Stop mirroring chunks over a previously attached wire stream.
Definition async_node.cc:752
void SetOwner(const std::shared_ptr< NodeMap > &owner)
Link a node created by a map to its owning resolution context.
Definition async_node.cc:237
std::shared_ptr< NodeMap > Owner() const
Definition async_node.cc:242
AsyncNode(const AsyncNode &)=delete
a11::Future< std::optional< data::Chunk > > NextChunk(absl::Duration timeout=absl::InfiniteDuration())
Read the next raw chunk.
Definition async_node.cc:692
absl::Status SetReaderOptions(stores::ChunkStoreReaderOptions options)
Replace the reader options.
Definition async_node.cc:390
absl::Status SetWriterOptions(stores::ChunkStoreWriterOptions options)
Replace the writer options.
Definition async_node.cc:435
absl::StatusOr< data::NodeRef > Ref(std::uint32_t offset=0, std::optional< std::uint64_t > length=std::nullopt, std::shared_ptr< NodeMap > destination=nullptr, ReferenceStoragePolicy policy=ReferenceStoragePolicy::kAuto) const
Return a fragment reference using the destination's resolution context.
Definition async_node.cc:247
a11::Future< std::uint32_t > Put(const T &value, std::optional< std::uint32_t > seq=std::nullopt, bool final=false, std::string_view mimetype={})
Definition async_node.h:320
void CancelWriter()
Cancel the writer, unblocking pending Put/drain awaits.
Definition async_node.cc:777
absl::Status GetWriterStatus() const
Return the current status of the node's writer.
Definition async_node.cc:461
a11::Future< std::optional< data::NodeFragment > > NextFragment(absl::Duration timeout=absl::InfiniteDuration())
Read the next raw fragment.
Definition async_node.cc:670
absl::Status GetReaderStatus() const
Return the current status of the node's reader.
Definition async_node.cc:447
absl::Status ResetReader(std::optional< stores::ChunkStoreReaderOptions > options=std::nullopt)
Rewind/reconfigure the reader (e.g.
Definition async_node.cc:406
AsyncNode & operator=(const AsyncNode &)=delete
std::optional< absl::Status > GetWriterAbortStatus() const
Return the status the writer was aborted with.
Definition async_node.cc:488
absl::StatusOr< std::shared_ptr< stores::ChunkStoreWriter > > writer()
Return the node's chunk store writer (the producing half).
Definition async_node.cc:372
void CancelReader()
Cancel the reader, unblocking pending Next* awaits.
Definition async_node.cc:761
a11::Task ObserveCompletion()
Observe logical completion without consuming data or closing the node.
Definition async_node.cc:474
void Cancel()
Cancel both reader and writer, tearing down all pending streaming operations on the node at once.
Definition async_node.cc:788
absl::Status AttachStream(std::shared_ptr< net::WireStream > stream)
Tee this node's stored chunks onto a wire stream.
Definition async_node.cc:738
a11::Future< bool > IsWritable()
Report whether the node can currently accept writes.
Definition async_node.cc:497
std::shared_ptr< data::SerializationRegistry > serialization_registry() const
Return the registry used to (de)serialize typed values.
Definition async_node.cc:287
absl::StatusOr< std::shared_ptr< stores::ChunkStoreReader > > OpenReader(stores::ChunkStoreReaderOptions options) const
Open an independent cursor with this node's reference resolution context.
Definition async_node.cc:360
a11::Future< std::uint32_t > PutReference(data::NodeRef reference, std::optional< std::uint32_t > seq=std::nullopt, bool final=false)
Definition async_node.cc:539
a11::Future< std::optional< T > > NextObject(absl::Duration timeout=absl::InfiniteDuration(), std::vector< std::string > mimetype_patterns={})
Read and deserialize the next value.
Definition async_node.h:451
absl::StatusOr< std::shared_ptr< stores::ChunkStoreReader > > reader()
Return the node's chunk store reader (the consuming half).
Definition async_node.cc:303
a11::Future< std::optional< data::NodeFragment > > NextFragmentRaw(absl::Duration timeout=absl::InfiniteDuration())
Read the next fragment without producing bytes for it.
Definition async_node.cc:633
a11::Future< std::vector< std::optional< data::NodeFragment > > > NextFragments(size_t limit, absl::Duration timeout=absl::InfiniteDuration())
Read up to limit fragments in a single await.
Definition async_node.cc:624
absl::StatusOr< std::string > GetId() const
Return the node's stable identifier.
Definition async_node.cc:233
a11::Future< std::uint32_t > PutFragment(data::NodeFragment fragment)
Enqueue a fragment carrying its own sequence and final flag.
Definition async_node.cc:530
std::function< absl::StatusOr< data::NodeRef >(const data::NodeRef &)> ReferenceRewriter
Definition chunk_store_writer.h:105
Definition boost_primitives.h:28
Definition boost_primitives.h:21
Whether T publishes a serialisation tag.
Definition serializable.h:171
std::string value
Definition discover.cc:100
Completion values used by every asynchronous A11 operation.
std::optional< absl::Duration > timeout
Definition main.cc:146
ReferenceStoragePolicy
Definition async_node.h:70
Task FailedTask(absl::Status status)
Return an already-failed Task.
Definition future.h:440
stores::ChunkStoreReaderOptions options
Definition runtime_streams.cc:31
std::optional< std::uint64_t > length
Definition runtime_streams.cc:37
std::uint32_t offset
Definition runtime_streams.cc:36
ADL-based customization points that make a custom C++ type serializable through a11::data::Serializat...
Type-and-mimetype indexed serialization of values to/from Chunks.
std::uint32_t seq
Definition sqlite_chunk_store.cc:186
std::string mimetype
Definition sqlite_chunk_store.cc:191
A unit of data: bytes plus optional descriptive metadata.
Definition types.h:173
One piece of a node's stream: an inline chunk or a node reference.
Definition types.h:366
A reference to a (slice of a) logical node, in lieu of inline data.
Definition types.h:311
How an AsyncNode::Finalize() call ends the stream.
Definition async_node.h:78
std::string_view mimetype
MIME type selecting the encoding of a typed value.
Definition async_node.h:90
bool close
Whether to close the writer after the final chunk.
Definition async_node.h:84
std::optional< std::uint32_t > seq
Explicit sequence number for the final chunk; assigned in order when omitted.
Definition async_node.h:87
bool wait
Whether to resolve only once the store confirmed the write and, if close is set, closed.
Definition async_node.h:81
Tunables controlling how a ChunkStoreReader delivers and buffers fragments.
Definition chunk_store_reader.h:69
Tunables controlling how a ChunkStoreWriter batches and buffers chunks.
Definition chunk_store_writer.h:56
A11's core wire value types: chunks, node fragments and messages.