DMS-Parser-XS
view release on metacpan or search on metacpan
vendor/dms-c/dms.c view on Meta::CPAN
uint64_t state = (uint64_t)(uintptr_t)t ^ 0xD1B54A32D192ED03ULL;
/* Fisher-Yates over items[]. */
for (size_t i = t->len - 1; i > 0; i--) {
uint64_t r = splitmix64_step(&state);
size_t j = (size_t)(r % (uint64_t)(i + 1));
if (j != i) {
dms_kv tmp = t->items[i];
t->items[i] = t->items[j];
t->items[j] = tmp;
}
}
/* The open-addressed hash slots indexed the pre-shuffle layout;
drop them. dms_table_has / table_get fall back to linear scan
when hash_cap == 0, which is fine for the read paths we still
care about (encoder + lite emit walk items[] directly). */
if (t->hash_slots) {
free(t->hash_slots);
t->hash_slots = NULL;
t->hash_cap = 0;
}
}
static void mark_unordered_walk(dms_value *v) {
if (!v) return;
switch (v->type) {
case DMS_TABLE:
v->u.t.unordered = true;
shuffle_table_items(&v->u.t);
for (size_t i = 0; i < v->u.t.len; i++) {
mark_unordered_walk(v->u.t.items[i].value);
}
return;
case DMS_LIST:
for (size_t i = 0; i < v->u.l.len; i++) {
mark_unordered_walk(v->u.l.items[i]);
}
return;
default:
return;
}
}
static dms_document *parse_document_with_mode_t1(const char *src, size_t srclen, dms_error *err, int lite, int unordered, int tier1_mode);
static dms_document *parse_document_with_mode(const char *src, size_t srclen, dms_error *err, int lite, int unordered) {
return parse_document_with_mode_t1(src, srclen, err, lite, unordered, 0);
}
static dms_document *parse_document_with_mode_t1(const char *src, size_t srclen, dms_error *err, int lite, int unordered, int tier1_mode) {
parser p;
memset(&p, 0, sizeof p);
p.src = src;
p.len = srclen;
p.line = 1;
p.record_forms = 1;
p.lite = lite;
p.unordered = unordered;
p.tier1_mode = tier1_mode;
/* SPEC §"UTF-8 only, NFC-normalized": DMS source is plain UTF-8 with
no byte-order mark. A leading U+FEFF is not silently consumed —
reject it explicitly so encoding mistakes surface loudly. (BOMs
*inside* string/heredoc bodies are fine; this only fires at offset 0.) */
if (srclen >= 3 && (unsigned char)src[0] == 0xEF && (unsigned char)src[1] == 0xBB && (unsigned char)src[2] == 0xBF) {
set_err_at(&p, 1, 0, 0,
"BOM (U+FEFF) at file start is not allowed; DMS source is plain UTF-8");
if (err) *err = p.err;
pending_free(&p.pending);
path_free(&p.path);
attached_free(&p.comments);
original_free(&p.original_forms);
return NULL;
}
/* U+0000 is not allowed anywhere in DMS source (see SPEC §Strings). */
for (size_t i = 0; i < srclen; i++) {
if (src[i] == '\0') {
size_t line = 1, line_start = 0;
for (size_t j = 0; j < i; j++) {
if (src[j] == '\n') { line++; line_start = j + 1; }
}
set_err_at(&p, line, line_start, line_start + (i - line_start),
"U+0000 (NUL) is not allowed in DMS source");
if (err) *err = p.err;
pending_free(&p.pending);
path_free(&p.path);
attached_free(&p.comments);
original_free(&p.original_forms);
return NULL;
}
}
/* SPEC §Unicode normalization: NFC the source before tokenization.
utf8proc allocates; we free at every exit path.
Fast path: if the source is pure ASCII (every byte < 0x80), NFC
is a no-op — the output is byte-identical. utf8proc_map still
walks every byte and allocates a fresh buffer (~420 µs on a
25 KB input vs ~9 µs for the ASCII scan). Skip it: borrow `src`
directly, leave `normalized` NULL, and let the existing
`free(normalized)` cleanup paths no-op (free(NULL) is a no-op).
Pure-ASCII is the common case for config / values files. */
utf8proc_uint8_t *normalized = NULL;
{
const char *scan_src = src + p.pos;
size_t scan_len = srclen - p.pos;
int all_ascii = 1;
for (size_t i = 0; i < scan_len; i++) {
if ((unsigned char)scan_src[i] >= 0x80) { all_ascii = 0; break; }
}
if (all_ascii) {
p.src = scan_src;
p.len = scan_len;
p.pos = 0;
p.line_start = 0;
} else {
utf8proc_ssize_t n = utf8proc_map(
(const utf8proc_uint8_t *)scan_src,
(utf8proc_ssize_t)scan_len,
&normalized,
UTF8PROC_STABLE | UTF8PROC_COMPOSE);
if (n < 0) {
set_err_at(&p, 1, 0, 0, "input is not valid UTF-8");
if (err) *err = p.err;
pending_free(&p.pending);
path_free(&p.path);
attached_free(&p.comments);
original_free(&p.original_forms);
vendor/dms-c/dms.c view on Meta::CPAN
}
/* Tier-1 variant: same as dms_decode_document but allows _dms_tier = 1
and _dms_imports in the front matter. Used by tier1.c only. */
dms_document *dms_decode_document_t1(const char *src, size_t srclen, dms_error *err) {
return parse_document_with_mode_t1(src, srclen, err, 0, 0, 1);
}
dms_document *dms_decode_document_lite(const char *src, size_t srclen, dms_error *err) {
return parse_document_with_mode(src, srclen, err, 1, 0);
}
dms_document *dms_decode_document_unordered(const char *src, size_t srclen, dms_error *err) {
return parse_document_with_mode(src, srclen, err, 0, 1);
}
dms_document *dms_decode_document_lite_unordered(const char *src, size_t srclen, dms_error *err) {
return parse_document_with_mode(src, srclen, err, 1, 1);
}
dms_value *dms_decode(const char *src, size_t srclen, dms_error *err) {
dms_document *d = dms_decode_document(src, srclen, err);
if (!d) return NULL;
dms_value *v = d->body;
free_table_inplace(d->meta);
for (size_t i = 0; i < d->num_comments; i++) {
free(d->comments[i].content);
for (size_t j = 0; j < d->comments[i].path_len; j++) {
if (!d->comments[i].path[j].is_index) free(d->comments[i].path[j].key);
}
free(d->comments[i].path);
}
free(d->comments);
for (size_t i = 0; i < d->num_original_forms; i++) {
original_entry_free(&d->original_forms[i]);
}
free(d->original_forms);
free(d);
return v;
}
void dms_table_free(dms_table *t) {
free_table_inplace(t);
}
/* Fast byte-level pre-scan that bounds the front-matter region without
touching body bytes. Mirrors the line-by-line shape of
`skip_trivia` + `parse_front_matter_tbl`'s closer scan, but operates
on raw (pre-NFC) bytes and only resolves comment delimiters, blank
lines, and `+++` literals — never tokenizes keys/values.
Returns:
1 → `*fm_end_out` is the byte offset (in raw `src`) one past the
closing `+++`'s EOL. The caller can NUL/NFC-validate just
`src[0..*fm_end_out]` and feed that slice to
`parse_front_matter_tbl` — body bytes are skipped entirely.
0 → "no FM present": leading trivia walks cleanly, but the next
non-trivia byte is not `+++`. `*fm_end_out` is unset.
-1 → bail to the slow path. The fast scanner can't bound the FM
region without doing real validation work — either the input
starts with a BOM, contains an unterminated/malformed comment
in leading trivia, or never closes the FM. The slow path
surfaces the matching diagnostic.
The closing `+++` is detected by the same trimmed-line test
`parse_front_matter_tbl` uses (lines 2202-2206), so the bounded
region byte-matches what the slow path would have consumed. */
static int find_fm_end_fast(const char *src, size_t srclen, size_t *fm_end_out) {
/* BOM at offset 0 — slow path so the dedicated diagnostic fires. */
if (srclen >= 3
&& (unsigned char)src[0] == 0xEF
&& (unsigned char)src[1] == 0xBB
&& (unsigned char)src[2] == 0xBF) {
return -1;
}
size_t pos = 0;
/* ----- leading trivia: blank lines + line/block comments ----- */
while (pos < srclen) {
/* skip inline ws */
size_t ls = pos;
while (pos < srclen && (src[pos] == ' ' || src[pos] == '\t')) pos++;
if (pos >= srclen) break;
char c = src[pos];
if (c == '\n') {
pos++;
continue;
}
if (c == '\r') {
/* bare CR is a parse error per skip_trivia — slow path. */
if (pos + 1 >= srclen || src[pos + 1] != '\n') return -1;
pos += 2;
continue;
}
if (c == '#') {
if (pos + 2 < srclen && src[pos + 1] == '#' && src[pos + 2] == '#') {
/* ### [LABEL] block comment. Replicate the byte shape
of skip_hash_block_comment: read a label, expect EOL,
then scan for a line trimmed to the terminator. */
size_t op = pos;
pos += 3;
size_t label_start = pos;
while (pos < srclen) {
char lc = src[pos];
if (!(lc == '_' || (lc >= '0' && lc <= '9') ||
(lc >= 'a' && lc <= 'z') || (lc >= 'A' && lc <= 'Z'))) break;
pos++;
}
size_t label_len = pos - label_start;
/* Empty label OK (matches `###`); first-char rule
("must start with a letter or underscore") is a
diagnostic the slow path owns — bail if violated. */
if (label_len > 0) {
char fc = src[label_start];
if (!(fc == '_' || (fc >= 'a' && fc <= 'z') || (fc >= 'A' && fc <= 'Z'))) {
return -1;
}
}
/* trailing ws + EOL on opener line */
while (pos < srclen && (src[pos] == ' ' || src[pos] == '\t')) pos++;
if (pos < srclen) {
if (src[pos] == '\n') pos++;
else if (pos + 1 < srclen && src[pos] == '\r' && src[pos + 1] == '\n') pos += 2;
else return -1; /* opener-must-be-on-its-own-line diag */
}
/* scan for terminator */
int closed = 0;
while (pos < srclen) {
size_t lb = pos;
while (pos < srclen && src[pos] != '\n' && src[pos] != '\r') pos++;
vendor/dms-c/dms.c view on Meta::CPAN
} else {
pos++;
}
}
continue;
}
/* non-trivia byte: rewind to line start (we may have consumed
leading inline ws that turns out to be indentation before a
non-trivia char — `skip_trivia` rewinds to `start` in that
case at line 1513). */
pos = ls;
break;
}
/* ----- opener: expect "+++" ----- */
if (pos + 3 > srclen || src[pos] != '+' || src[pos + 1] != '+' || src[pos + 2] != '+') {
/* No FM present. */
return 0;
}
pos += 3;
/* opener line must end in EOL (with optional trailing ws). Trailing
junk is a parse error owned by the slow path — bail in that case
so the right diagnostic surfaces. */
while (pos < srclen && (src[pos] == ' ' || src[pos] == '\t')) pos++;
if (pos < srclen) {
if (src[pos] == '\n') pos++;
else if (pos + 1 < srclen && src[pos] == '\r' && src[pos + 1] == '\n') pos += 2;
else return -1; /* trailing junk → "opener must be on its own line" */
}
/* ----- inner lines until closing "+++" (trimmed-line test) ----- */
while (pos < srclen) {
size_t lb = pos;
while (pos < srclen && src[pos] != '\n' && src[pos] != '\r') pos++;
size_t le = pos;
size_t after_eol = pos;
if (pos < srclen) {
if (src[pos] == '\n') after_eol = pos + 1;
else if (pos + 1 < srclen && src[pos] == '\r' && src[pos + 1] == '\n') after_eol = pos + 2;
else return -1; /* bare CR */
}
/* trim ws */
size_t ts = lb, te = le;
while (ts < te && (src[ts] == ' ' || src[ts] == '\t')) ts++;
while (te > ts && (src[te - 1] == ' ' || src[te - 1] == '\t')) te--;
if (te - ts == 3 && src[ts] == '+' && src[ts + 1] == '+' && src[ts + 2] == '+') {
*fm_end_out = after_eol;
return 1;
}
if (after_eol == pos) {
/* EOF without closer — slow path emits "unterminated FM". */
return -1;
}
pos = after_eol;
}
/* Hit EOF without finding closer. */
return -1;
}
/* SPEC §Front-matter-only decode (tier-0, required).
Pre-scans raw bytes to bound the FM region (`find_fm_end_fast`),
then BOM/NUL/NFC-validates only that prefix and feeds it to
`parse_front_matter_tbl`. Body bytes are never visited — call cost
scales with FM-region size, not source size. On any pre-scan
ambiguity (BOM, malformed leading comment, missing closer, etc.)
we fall through to the slow path which validates the entire source
and emits the canonical diagnostic.
`parse_front_matter_tbl`'s contract:
- returns NULL with `p.has_err == 0` when the document has no
opening `+++` (no FM present),
- returns a calloc'd `dms_table *` (possibly with `len == 0`
for an empty `+++\n+++`) on success,
- returns NULL with `p.has_err == 1` on any in-FM error.
We translate that into `(has_front_matter, return value, err)`. */
dms_table *dms_decode_front_matter(const char *src, size_t srclen,
bool *has_front_matter,
dms_error *err) {
if (has_front_matter) *has_front_matter = false;
parser p;
memset(&p, 0, sizeof p);
p.src = src;
p.len = srclen;
p.line = 1;
p.record_forms = 1;
/* SPEC §Front-matter-only decode: this entry point runs in lite
mode (no comment AST, no original_forms inside the FM). */
p.lite = 1;
p.unordered = 0;
/* Fast pre-scan: bound the FM region so the BOM/NUL/NFC pass below
only touches FM bytes. NFC of a stable-boundary prefix equals
the prefix of NFC(full) (the closing `+++` is ASCII and therefore
a stable boundary), so byte offsets and line numbers inside the
FM region are identical to the full-source path — diagnostics
remain byte-identical per SPEC. */
size_t scan_len = srclen;
{
size_t fm_end = 0;
int rc = find_fm_end_fast(src, srclen, &fm_end);
if (rc == 0) {
/* No FM present — fast bail without touching body bytes. */
if (has_front_matter) *has_front_matter = false;
return NULL;
} else if (rc == 1) {
/* Restrict validation/parse to the FM region. */
scan_len = fm_end;
}
/* rc == -1: fall through to the slow full-source path. */
}
/* SPEC §"UTF-8 only, NFC-normalized": reject a leading BOM so
encoding mistakes surface loudly. */
if (scan_len >= 3
&& (unsigned char)src[0] == 0xEF
&& (unsigned char)src[1] == 0xBB
&& (unsigned char)src[2] == 0xBF) {
set_err_at(&p, 1, 0, 0,
"BOM (U+FEFF) at file start is not allowed; DMS source is plain UTF-8");
if (err) *err = p.err;
pending_free(&p.pending);
path_free(&p.path);
attached_free(&p.comments);
original_free(&p.original_forms);
return NULL;
}
/* U+0000 is not allowed anywhere in DMS source. We scan the FM
region only — body NULs are not surfaced by this entry point
per SPEC (body bytes aren't tokenized here). */
for (size_t i = 0; i < scan_len; i++) {
if (src[i] == '\0') {
size_t line = 1, line_start = 0;
for (size_t j = 0; j < i; j++) {
if (src[j] == '\n') { line++; line_start = j + 1; }
}
set_err_at(&p, line, line_start, line_start + (i - line_start),
"U+0000 (NUL) is not allowed in DMS source");
if (err) *err = p.err;
pending_free(&p.pending);
path_free(&p.path);
attached_free(&p.comments);
original_free(&p.original_forms);
return NULL;
}
}
/* NFC-normalize the FM region. ASCII fast path matches the body
decoder so the in-FM byte offsets reported in error messages
are identical. */
utf8proc_uint8_t *normalized = NULL;
{
const char *scan_src = src + p.pos;
size_t inner_len = scan_len - p.pos;
int all_ascii = 1;
for (size_t i = 0; i < inner_len; i++) {
if ((unsigned char)scan_src[i] >= 0x80) { all_ascii = 0; break; }
}
if (all_ascii) {
p.src = scan_src;
p.len = inner_len;
p.pos = 0;
p.line_start = 0;
} else {
utf8proc_ssize_t n = utf8proc_map(
(const utf8proc_uint8_t *)scan_src,
(utf8proc_ssize_t)inner_len,
&normalized,
UTF8PROC_STABLE | UTF8PROC_COMPOSE);
if (n < 0) {
set_err_at(&p, 1, 0, 0, "input is not valid UTF-8");
if (err) *err = p.err;
pending_free(&p.pending);
path_free(&p.path);
attached_free(&p.comments);
original_free(&p.original_forms);
return NULL;
}
p.src = (const char *)normalized;
p.len = (size_t)n;
p.pos = 0;
vendor/dms-c/dms.c view on Meta::CPAN
dms_table *meta = parse_front_matter_tbl(&p);
bool got_err = p.has_err;
/* parse_front_matter_tbl signals "no FM present" by returning NULL
and rewinding — without setting has_err. Anything else where the
return is non-NULL means an FM block was actually parsed. */
bool fm_present = (meta != NULL) || got_err;
if (got_err) {
if (err) *err = p.err;
free(normalized);
free_table_inplace(meta);
pending_free(&p.pending);
path_free(&p.path);
attached_free(&p.comments);
original_free(&p.original_forms);
return NULL;
}
/* No error at this point. Free the parser's accumulated scratch —
lite mode still pushes the occasional thing into pending /
comments / originals during trivia-skip; drop it all. */
free(normalized);
pending_free(&p.pending);
path_free(&p.path);
attached_free(&p.comments);
original_free(&p.original_forms);
if (has_front_matter) *has_front_matter = fm_present;
return meta;
}
/* Tier-1 variant of dms_decode_front_matter: same as above but with
tier1_mode=1 set on the parser, allowing _dms_tier = 1 and
_dms_imports in the front matter. Used only by tier1.c. */
dms_table *dms_decode_front_matter_t1(const char *src, size_t srclen,
bool *has_front_matter,
dms_error *err) {
if (has_front_matter) *has_front_matter = false;
parser p;
memset(&p, 0, sizeof p);
p.src = src;
p.len = srclen;
p.line = 1;
p.record_forms = 1;
p.lite = 1;
p.unordered = 0;
p.tier1_mode = 1;
size_t scan_len = srclen;
{
size_t fm_end = 0;
int rc = find_fm_end_fast(src, srclen, &fm_end);
if (rc == 0) {
if (has_front_matter) *has_front_matter = false;
return NULL;
} else if (rc == 1) {
scan_len = fm_end;
}
}
if (scan_len >= 3
&& (unsigned char)src[0] == 0xEF
&& (unsigned char)src[1] == 0xBB
&& (unsigned char)src[2] == 0xBF) {
set_err_at(&p, 1, 0, 0,
"BOM (U+FEFF) at file start is not allowed; DMS source is plain UTF-8");
if (err) *err = p.err;
pending_free(&p.pending);
path_free(&p.path);
attached_free(&p.comments);
original_free(&p.original_forms);
return NULL;
}
for (size_t i = 0; i < scan_len; i++) {
if (src[i] == '\0') {
size_t line = 1, line_start = 0;
for (size_t j = 0; j < i; j++) {
if (src[j] == '\n') { line++; line_start = j + 1; }
}
set_err_at(&p, line, line_start, line_start + (i - line_start),
"U+0000 (NUL) is not allowed in DMS source");
if (err) *err = p.err;
pending_free(&p.pending);
path_free(&p.path);
attached_free(&p.comments);
original_free(&p.original_forms);
return NULL;
}
}
utf8proc_uint8_t *normalized = NULL;
{
const char *scan_src = src + p.pos;
size_t inner_len = scan_len - p.pos;
int all_ascii = 1;
for (size_t i = 0; i < inner_len; i++) {
if ((unsigned char)scan_src[i] >= 0x80) { all_ascii = 0; break; }
}
if (all_ascii) {
p.src = scan_src;
p.len = inner_len;
p.pos = 0;
p.line_start = 0;
} else {
utf8proc_ssize_t n = utf8proc_map(
(const utf8proc_uint8_t *)scan_src,
(utf8proc_ssize_t)inner_len,
&normalized,
UTF8PROC_STABLE | UTF8PROC_COMPOSE);
if (n < 0) {
set_err_at(&p, 1, 0, 0, "input is not valid UTF-8");
if (err) *err = p.err;
pending_free(&p.pending);
path_free(&p.path);
attached_free(&p.comments);
original_free(&p.original_forms);
return NULL;
}
p.src = (const char *)normalized;
p.len = (size_t)n;
p.pos = 0;
p.line_start = 0;
}
}
dms_table *meta = parse_front_matter_tbl(&p);
bool got_err = p.has_err;
bool fm_present = (meta != NULL) || got_err;
( run in 0.702 second using v1.01-cache-2.11-cpan-9581c071862 )