3#ifndef A11_STORES_REDIS_CHUNK_STORE_SCRIPT_H_
4#define A11_STORES_REDIS_CHUNK_STORE_SCRIPT_H_
14local meta_key = KEYS[1]
15local stream_key = KEYS[2]
16local seq_key = KEYS[3]
17local arrival_key = KEYS[4]
18local blobs_key = KEYS[5]
19local events_channel = KEYS[6]
21local operation = ARGV[1]
22local node_id = ARGV[2]
23local max_seq_value = 4294967295
24local max_count_value = 4294967296
25-- At most 2^32 successful puts, 2^32 first-time clears, and one close can
26-- publish a revision. Keeping every Lua counter below 2^53 also avoids the
27-- lossy integer conversions of Redis's Lua 5.1 number type.
28local max_revision_value = 8589934593
30if type(operation) ~= 'string' or type(node_id) ~= 'string' or node_id == '' then
31 return {'error', 'INVALID_ARGUMENT', 'Invalid chunk-store script envelope'}
34local function failure(code, message)
35 return {'error', code, message}
38local function key_type(key)
39 local reply = redis.call('TYPE', key)
40 if type(reply) == 'table' then
46local function check_type(key, expected)
47 local actual = key_type(key)
48 return actual == 'none' or actual == expected
51local function is_canonical_decimal(value, maximum)
52 if type(value) ~= 'string' or value == '' or
53 string.find(value, '[^0-9]') then
56 if #value > 1 and string.sub(value, 1, 1) == '0' then
59 if #value ~= #maximum then
60 return #value < #maximum
62 return value <= maximum
65local function canonical_uint(value, maximum, maximum_text)
66 if not is_canonical_decimal(value, maximum_text) then
69 local parsed = tonumber(value)
70 if not parsed or parsed < 0 or parsed > maximum or
71 math.floor(parsed) ~= parsed then
77if not check_type(meta_key, 'hash') then
78 return failure('DATA_LOSS', 'Chunk store metadata key is not a hash')
80if not check_type(stream_key, 'stream') then
81 return failure('DATA_LOSS', 'Chunk store data key is not a stream')
83if not check_type(seq_key, 'hash') then
84 return failure('DATA_LOSS', 'Chunk store sequence index is not a hash')
86if not check_type(arrival_key, 'hash') then
87 return failure('DATA_LOSS', 'Chunk store arrival index is not a hash')
89if not check_type(blobs_key, 'hash') then
90 return failure('DATA_LOSS', 'Chunk store blob key is not a hash')
93local meta_exists = redis.call('EXISTS', meta_key) == 1
94local metadata_closed = false
95local metadata_size = 0
96local metadata_put_count = 0
97local metadata_next_cursor = 0
98local metadata_revision = 0
99local metadata_final_seq = nil
100local metadata_max_seq = nil
101if not meta_exists then
102 if redis.call('EXISTS', stream_key, seq_key, arrival_key, blobs_key) ~= 0 then
103 return failure('DATA_LOSS', 'Chunk data exists without store metadata')
106 local stored_id = redis.call('HGET', meta_key, 'id')
107 local schema = redis.call('HGET', meta_key, 'schema')
108 local closed = redis.call('HGET', meta_key, 'closed')
109 local size = redis.call('HGET', meta_key, 'size')
110 local put_count = redis.call('HGET', meta_key, 'put_count')
111 local next_cursor = redis.call('HGET', meta_key, 'next_cursor')
112 local revision = redis.call('HGET', meta_key, 'revision')
113 local final_seq = redis.call('HGET', meta_key, 'final_seq')
114 local max_seq = redis.call('HGET', meta_key, 'max_seq')
115 metadata_size = canonical_uint(size, max_count_value, '4294967296')
116 metadata_put_count = canonical_uint(
117 put_count, max_count_value, '4294967296')
118 metadata_next_cursor = canonical_uint(
119 next_cursor, max_count_value, '4294967296')
120 metadata_revision = canonical_uint(
121 revision, max_revision_value, '8589934593')
123 metadata_final_seq = canonical_uint(
124 final_seq, max_seq_value, '4294967295')
127 metadata_max_seq = canonical_uint(max_seq, max_seq_value, '4294967295')
129 if not stored_id or schema ~= '1' or
130 (closed ~= '0' and closed ~= '1') or metadata_size == nil or
131 metadata_put_count == nil or metadata_next_cursor == nil or
132 metadata_revision == nil or
133 (final_seq and metadata_final_seq == nil) or
134 (max_seq and metadata_max_seq == nil) then
135 return failure('DATA_LOSS', 'Chunk store metadata is incomplete or corrupt')
137 if stored_id ~= node_id then
138 return failure('DATA_LOSS', 'Chunk store key belongs to a different node')
140 metadata_closed = closed == '1'
141 local stored_status = redis.call('HGET', meta_key, 'status')
142 if metadata_closed and not stored_status then
143 return failure('DATA_LOSS', 'Closed chunk store has no terminal status')
145 if not metadata_closed and stored_status then
146 return failure('DATA_LOSS', 'Open chunk store has a terminal status')
148 if metadata_size ~= metadata_put_count or
149 metadata_next_cursor > metadata_size then
150 return failure('DATA_LOSS', 'Chunk store counters are inconsistent')
152 if (metadata_size == 0 and metadata_max_seq ~= nil) or
153 (metadata_size > 0 and metadata_max_seq == nil) or
154 (metadata_final_seq ~= nil and
155 metadata_final_seq ~= metadata_max_seq) then
156 return failure('DATA_LOSS', 'Chunk store sequence metadata is inconsistent')
158 if redis.call('HLEN', seq_key) ~= metadata_size or
159 redis.call('HLEN', arrival_key) ~= metadata_put_count then
160 return failure('DATA_LOSS', 'Chunk store index cardinality is inconsistent')
162 if redis.call('HLEN', blobs_key) > metadata_size then
163 return failure('DATA_LOSS', 'Chunk store contains unindexed blobs')
165 local expected_stream_size = metadata_size + (metadata_closed and 1 or 0)
166 if redis.call('XLEN', stream_key) ~= expected_stream_size then
167 return failure('DATA_LOSS', 'Chunk store stream cardinality is inconsistent')
171local function ensure_meta()
172 if not meta_exists then
173 redis.call('HSET', meta_key,
185local function revision()
186 if not meta_exists then
189 return redis.call('HGET', meta_key, 'revision') or '0'
192local function publish_change()
193 -- Every caller checks this before its first mutation. HINCRBY therefore
194 -- cannot overflow after a batch has partially committed.
195 local changed = redis.call('HINCRBY', meta_key, 'revision', 1)
196 redis.call('PUBLISH', events_channel, tostring(changed))
200local function can_publish_change()
201 if metadata_revision >= max_revision_value then
202 return failure('RESOURCE_EXHAUSTED',
203 'Chunk store mutation revision exhausted')
208local function field_map(entry)
212 local fields = entry[1][2]
214 for index = 1, #fields, 2 do
215 result[fields[index]] = fields[index + 1]
220local function can_append_stream_entry()
221 if redis.call('EXISTS', stream_key) == 0 then
224 local info = redis.call('XINFO', 'STREAM', stream_key)
225 local stream_id = nil
226 for index = 1, #info, 2 do
227 if info[index] == 'last-generated-id' then
228 stream_id = info[index + 1]
232 if not stream_id then
235 local separator = string.find(stream_id, '-', 1, true)
236 if not separator then
239 -- XADD * may increment the sequence component, but no automatic ID can be
240 -- greater once the millisecond component itself has reached uint64_t max.
241 return string.sub(stream_id, 1, separator - 1) ~= '18446744073709551615'
244-- Returns state, payload-or-ref, stream id, arrival, and storage. Every
245-- payload is an encoded Chunk. Tombstones retain a separately prepared,
246-- data-free encoding so ClearData can preserve metadata atomically without
247-- teaching Lua A11's MessagePack schema.
248local function load_chunk(seq)
249 local stream_id = redis.call('HGET', seq_key, tostring(seq))
250 if not stream_id then
251 return 'missing', '', '', '', ''
253 local entry = redis.call('XRANGE', stream_key, stream_id, stream_id, 'COUNT', 1)
254 local fields = field_map(entry)
256 return 'data_loss', 'Sequence index references a missing stream entry',
259 if fields['v'] ~= '1' or fields['kind'] ~= 'chunk' or
260 fields['seq'] ~= tostring(seq) or not fields['arrival'] then
261 return 'data_loss', 'Stream entry metadata is corrupt', stream_id, '', ''
263 local arrival = canonical_uint(
264 fields['arrival'], max_seq_value, '4294967295')
266 redis.call('HGET', arrival_key, fields['arrival']) ~= tostring(seq) then
267 return 'data_loss', 'Stream entry arrival index is corrupt', stream_id,
268 fields['arrival'], ''
270 local storage = fields['storage']
271 if storage == 'inline' then
272 if fields['payload'] == nil then
273 return 'data_loss', 'Inline stream entry has no payload', stream_id,
274 fields['arrival'], storage
276 return 'item', fields['payload'], stream_id, fields['arrival'], storage
278 if storage == 'redis' then
279 if fields['ref'] ~= tostring(seq) then
280 return 'data_loss', 'Redis stream entry has no blob reference', stream_id,
281 fields['arrival'], storage
283 local payload = redis.call('HGET', blobs_key, fields['ref'])
284 if payload == false then
285 return 'data_loss', 'Redis stream entry references a missing blob',
286 stream_id, fields['arrival'], storage
288 return 'item', payload, stream_id, fields['arrival'], storage
290 if storage == 'tombstone' then
291 if fields['payload'] == nil then
292 return 'data_loss', 'Tombstone stream entry has no payload', stream_id,
293 fields['arrival'], storage
295 return 'item', fields['payload'], stream_id, fields['arrival'], storage
297 if storage == 's3' then
298 return 's3', fields['ref'] or '', stream_id, fields['arrival'], storage
300 return 'data_loss', 'Stream entry has an unknown storage kind', stream_id,
301 fields['arrival'], storage or ''
304local function final_seq()
305 if not meta_exists then
308 return redis.call('HGET', meta_key, 'final_seq') or ''
311local function missing_result(kind, value)
312 if meta_exists and redis.call('HGET', meta_key, 'closed') == '1' then
313 return {'closed', redis.call('HGET', meta_key, 'status')}
315 return {'wait', revision(), kind, tostring(value)}
318if operation == 'initialize' then
320 return failure('INVALID_ARGUMENT', 'Invalid initialize arguments')
326if operation == 'put' then
327 if metadata_closed then
328 return failure('FAILED_PRECONDITION', 'Chunk store is closed for writes')
330 local count = canonical_uint(ARGV[3], max_count_value, '4294967296')
332 return failure('INVALID_ARGUMENT', 'Invalid fragment count')
334 if #ARGV ~= 3 + count * 5 then
335 return failure('INVALID_ARGUMENT', 'Fragment count does not match arguments')
340 if metadata_put_count + count > max_count_value then
341 return failure('RESOURCE_EXHAUSTED',
342 'Maximum chunk-store cardinality exceeded')
344 local explicit = ARGV[4] ~= ''
345 local put_count = metadata_put_count
346 local candidate = put_count
349 local batch_final = nil
350 local saw_final = false
351 local pending_max = metadata_max_seq
353 for index = 1, count do
354 local base = 4 + (index - 1) * 5
355 local supplied_seq = ARGV[base]
356 local is_final = ARGV[base + 1]
357 local storage = ARGV[base + 2]
358 local payload = ARGV[base + 3]
359 local tombstone = ARGV[base + 4]
360 if (supplied_seq ~= '') ~= explicit then
361 return failure('INVALID_ARGUMENT',
362 'Sequence numbers must be set on every fragment or none')
366 seq = canonical_uint(supplied_seq, max_seq_value, '4294967295')
368 return failure('INVALID_ARGUMENT', 'Invalid explicit sequence number')
371 while candidate <= max_seq_value and
372 redis.call('HEXISTS', seq_key, tostring(candidate)) == 1 do
373 candidate = candidate + 1
375 if candidate > max_seq_value then
376 return failure('RESOURCE_EXHAUSTED',
377 'Maximum implicit sequence number exceeded')
380 candidate = candidate + 1
382 local seq_text = tostring(seq)
383 if seen[seq_text] then
384 return failure('INVALID_ARGUMENT',
385 'A sequence occurs more than once in the batch')
387 seen[seq_text] = true
388 if redis.call('HEXISTS', seq_key, seq_text) == 1 then
389 return failure('ALREADY_EXISTS',
390 'A fragment with seq ' .. seq_text .. ' already exists')
392 if redis.call('HEXISTS', blobs_key, seq_text) == 1 then
393 return failure('DATA_LOSS',
394 'An unindexed blob collides with seq ' .. seq_text)
396 local arrival = tostring(put_count + index - 1)
397 if redis.call('HEXISTS', arrival_key, arrival) == 1 then
398 return failure('DATA_LOSS',
399 'Arrival index ' .. arrival .. ' already exists')
401 if storage ~= 'inline' and storage ~= 'redis' then
402 return failure('INVALID_ARGUMENT', 'Invalid chunk storage kind')
404 if payload == nil then
405 return failure('INVALID_ARGUMENT', 'Chunk payload is missing')
407 if tombstone == nil then
408 return failure('INVALID_ARGUMENT', 'Chunk tombstone payload is missing')
410 if is_final ~= '0' and is_final ~= '1' then
411 return failure('INVALID_ARGUMENT', 'Invalid final-fragment marker')
413 if is_final == '1' then
415 return failure('INVALID_ARGUMENT',
416 'More than one fragment in the batch is marked final')
418 if not explicit and index ~= count then
419 return failure('INVALID_ARGUMENT',
420 'The final implicit fragment must be last')
425 assigned[index] = seq
426 if not pending_max or seq > pending_max then
431 local existing_final = metadata_final_seq
432 if batch_final and existing_final and batch_final ~= existing_final then
433 return failure('FAILED_PRECONDITION',
434 'The chunk store already has a different final sequence')
436 local pending_final = batch_final or existing_final
437 if pending_final then
438 local existing_max = metadata_max_seq
439 if existing_max and existing_max > pending_final then
440 return failure('INVALID_ARGUMENT',
441 'An existing fragment exceeds the proposed final sequence')
443 for index = 1, count do
444 if assigned[index] > pending_final then
445 return failure('INVALID_ARGUMENT',
446 'A fragment sequence exceeds the final sequence')
451 local revision_error = can_publish_change()
452 if revision_error then
453 return revision_error
455 if not can_append_stream_entry() then
456 return failure('RESOURCE_EXHAUSTED', 'Redis Stream ID space is exhausted')
459 local response = {'ok'}
460 for index = 1, count do
461 local base = 4 + (index - 1) * 5
462 local storage = ARGV[base + 2]
463 local payload = ARGV[base + 3]
464 local tombstone = ARGV[base + 4]
465 local seq_text = tostring(assigned[index])
466 local arrival = tostring(put_count + index - 1)
467 local stream_id = nil
468 if storage == 'redis' then
469 redis.call('HSET', blobs_key, seq_text, payload)
470 stream_id = redis.call('XADD', stream_key, '*',
471 'v', '1', 'kind', 'chunk', 'seq', seq_text, 'arrival', arrival,
472 'storage', 'redis', 'ref', seq_text, 'tombstone', tombstone)
474 stream_id = redis.call('XADD', stream_key, '*',
475 'v', '1', 'kind', 'chunk', 'seq', seq_text, 'arrival', arrival,
476 'storage', 'inline', 'payload', payload, 'tombstone', tombstone)
478 redis.call('HSET', seq_key, seq_text, stream_id)
479 redis.call('HSET', arrival_key, arrival, seq_text)
480 response[#response + 1] = seq_text
482 redis.call('HINCRBY', meta_key, 'size', count)
483 redis.call('HINCRBY', meta_key, 'put_count', count)
484 redis.call('HSET', meta_key, 'max_seq', tostring(pending_max))
486 redis.call('HSET', meta_key, 'final_seq', tostring(batch_final))
492if operation == 'lookup' then
494 return failure('INVALID_ARGUMENT', 'Invalid lookup arguments')
497 local value = ARGV[4]
498 local seq_text = value
499 if kind == 'arrival' then
500 if not is_canonical_decimal(value, '18446744073709551615') then
501 return failure('INVALID_ARGUMENT', 'Invalid arrival-order lookup')
503 seq_text = redis.call('HGET', arrival_key, value)
505 return missing_result(kind, value)
507 elseif kind ~= 'sequence' then
508 return failure('INVALID_ARGUMENT', 'Invalid Redis chunk lookup kind')
510 local seq = canonical_uint(seq_text, max_seq_value, '4294967295')
512 if kind == 'arrival' then
513 return failure('DATA_LOSS', 'Arrival index contains an invalid sequence')
515 return failure('INVALID_ARGUMENT', 'Invalid lookup sequence')
517 local state, value_or_error, _, _, storage = load_chunk(seq)
518 if state == 'missing' then
519 return missing_result(kind, value)
521 if state == 'data_loss' then
522 return failure('DATA_LOSS', value_or_error)
524 if state == 's3' then
525 return {'item', seq_text, 's3', value_or_error, final_seq()}
527 return {'item', seq_text, storage, value_or_error, final_seq()}
530if operation == 'next' then
532 return failure('INVALID_ARGUMENT', 'Invalid next arguments')
534 local limit = canonical_uint(ARGV[3], 1024, '1024')
535 if limit == nil or limit == 0 then
536 return failure('INVALID_ARGUMENT', 'limit must be positive')
538 if not meta_exists then
539 return {'next', 'wait', '', '0'}
541 local cursor = metadata_next_cursor
542 local final_text = final_seq()
543 local final = metadata_final_seq
546 local disposition = nil
550 -- LocalChunkStore treats exhausting the uint32_t sequence namespace as an
551 -- end sentinel even when no final fragment or close status was recorded.
552 if cursor > max_seq_value then
556 if final and cursor > final then
557 if redis.call('HGET', meta_key, 'closed') == '1' then
558 disposition = 'closed'
559 detail = redis.call('HGET', meta_key, 'status')
565 if item_count == limit then
566 disposition = 'ready'
569 local state, value_or_error, _, _, storage = load_chunk(cursor)
570 if state == 'missing' then
571 if redis.call('HGET', meta_key, 'closed') == '1' then
572 disposition = 'closed'
573 detail = redis.call('HGET', meta_key, 'status')
579 if state == 'data_loss' then
580 disposition = 'data_loss'
581 detail = value_or_error
584 if state == 's3' then
586 detail = value_or_error
589 item_count = item_count + 1
590 items[#items + 1] = tostring(cursor)
591 items[#items + 1] = storage
592 items[#items + 1] = value_or_error
593 items[#items + 1] = final_text
597 if item_count > 0 and disposition ~= 'data_loss' and disposition ~= 's3' then
598 redis.call('HSET', meta_key, 'next_cursor', tostring(cursor))
600 local response = {'next', disposition, detail, tostring(item_count)}
601 for index = 1, #items do
602 response[#response + 1] = items[index]
607if operation == 'clear' then
609 return failure('INVALID_ARGUMENT', 'Invalid clear arguments')
611 local seq_text = ARGV[3]
612 local seq = canonical_uint(seq_text, max_seq_value, '4294967295')
614 return failure('INVALID_ARGUMENT', 'Invalid sequence to clear')
616 local state, value_or_error, stream_id, arrival, storage = load_chunk(seq)
617 if state == 'missing' then
618 return failure('NOT_FOUND', 'No fragment with seq ' .. seq_text .. ' exists')
620 if state == 'data_loss' then
621 return failure('DATA_LOSS', value_or_error)
623 if state == 's3' then
624 return failure('UNIMPLEMENTED',
625 'Clearing S3-backed chunks is not implemented')
627 if storage == 'tombstone' then
628 return {'item', seq_text, 'tombstone', value_or_error, final_seq()}
630 local entry = redis.call('XRANGE', stream_key, stream_id, stream_id,
632 local fields = field_map(entry)
633 if not fields or fields['tombstone'] == nil then
634 return failure('DATA_LOSS',
635 'Stream entry has no prepared tombstone payload')
637 local revision_error = can_publish_change()
638 if revision_error then
639 return revision_error
641 if not can_append_stream_entry() then
642 return failure('RESOURCE_EXHAUSTED', 'Redis Stream ID space is exhausted')
644 local tombstone = fields['tombstone']
645 -- Append first: XADD is the only remaining command that can reject valid
646 -- arguments because of stream-ID state. No old payload has been removed if
648 local tombstone_id = redis.call('XADD', stream_key, '*',
649 'v', '1', 'kind', 'chunk', 'seq', seq_text, 'arrival', arrival,
650 'storage', 'tombstone', 'payload', tombstone,
651 'tombstone', tombstone)
652 if storage == 'redis' then
653 redis.call('HDEL', blobs_key, seq_text)
655 redis.call('XDEL', stream_key, stream_id)
656 redis.call('HSET', seq_key, seq_text, tombstone_id)
658 return {'item', seq_text, storage, value_or_error, final_seq()}
661if operation == 'arrival_seq' then
663 return failure('INVALID_ARGUMENT', 'Invalid arrival lookup arguments')
665 if not is_canonical_decimal(ARGV[3], '18446744073709551615') then
666 return failure('INVALID_ARGUMENT', 'Invalid arrival order')
668 local seq = redis.call('HGET', arrival_key, ARGV[3])
670 return failure('NOT_FOUND',
671 'No fragment has arrival order ' .. ARGV[3])
673 if canonical_uint(seq, max_seq_value, '4294967295') == nil or
674 redis.call('HEXISTS', seq_key, seq) ~= 1 then
675 return failure('DATA_LOSS',
676 'Arrival index references an invalid or missing sequence')
678 return {'value', seq}
681if operation == 'final' then
683 return failure('INVALID_ARGUMENT', 'Invalid final-sequence arguments')
685 return {'optional', final_seq()}
688if operation == 'size' then
690 return failure('INVALID_ARGUMENT', 'Invalid size arguments')
692 if not meta_exists then
693 return {'value', '0'}
695 return {'value', redis.call('HGET', meta_key, 'size')}
698if operation == 'close' then
700 return failure('INVALID_ARGUMENT', 'Invalid close arguments')
702 local requested_status = ARGV[3]
703 local return_existing = ARGV[4]
704 if requested_status == nil or
705 (return_existing ~= '0' and return_existing ~= '1') then
706 return failure('INVALID_ARGUMENT', 'Invalid chunk-store close arguments')
708 if metadata_closed then
709 if return_existing == '1' then
710 return {'status', redis.call('HGET', meta_key, 'status')}
712 return failure('FAILED_PRECONDITION',
713 'Chunk store is already closed for writes')
715 local revision_error = can_publish_change()
716 if revision_error then
717 return revision_error
719 if not can_append_stream_entry() then
720 return failure('RESOURCE_EXHAUSTED', 'Redis Stream ID space is exhausted')
722 local changed = metadata_revision + 1
723 -- As in ClearData, append the stream transition before making any other
724 -- mutation so an invalid stream-ID state cannot leave a half-closed store.
725 redis.call('XADD', stream_key, '*', 'v', '1', 'kind', 'control',
726 'event', 'close', 'revision', tostring(changed))
728 redis.call('HSET', meta_key, 'closed', '1', 'status', requested_status)
729 redis.call('HINCRBY', meta_key, 'revision', 1)
730 redis.call('PUBLISH', events_channel, tostring(changed))
731 return {'status', requested_status}
734if operation == 'metadata' then
736 return failure('INVALID_ARGUMENT', 'Invalid metadata arguments')
738 if not meta_exists then
739 return {'metadata', node_id, '0', '', '', '0', '0', '0', '', '0'}
741 local closed = redis.call('HGET', meta_key, 'closed')
743 redis.call('HGET', meta_key, 'id'),
745 closed == '1' and redis.call('HGET', meta_key, 'status') or '',
746 redis.call('HGET', meta_key, 'final_seq') or '',
747 redis.call('HGET', meta_key, 'size'),
748 redis.call('HGET', meta_key, 'put_count'),
749 redis.call('HGET', meta_key, 'next_cursor'),
750 redis.call('HGET', meta_key, 'max_seq') or '',
751 redis.call('HGET', meta_key, 'revision')}
754return failure('INVALID_ARGUMENT', 'Unknown chunk store operation')
Definition redis_chunk_store_script.h:8
constexpr std::string_view kRedisChunkStoreScript
Definition redis_chunk_store_script.h:13