view release on metacpan or  search on metacpan

include/frx/frx_lex.h  view on Meta::CPAN

#ifndef FRX_LEX_H
#define FRX_LEX_H

/* Bytes in, tokens out, every refusal with an offset.
 *
 * This is the whole of the attacker-facing surface: everything after it
 * operates on validated names and decoded strings already in the arena. A
 * pull tokeniser rather than a lexer folded into the parser, so that every
 * refusal is testable through the _lex accessor without a tree, and so that
 * the every-offset test can hit the lexer alone.
 *
 * What it accepts: XML 1.0 with the five predefined entities and numeric
 * character references, in UTF-8. What it refuses, each with the offset it
 * was found at: any DOCTYPE (and any <! that is not a comment or CDATA -
 * one rule removes entity expansion, external entities, the billion laughs
 * and XXE as a class, with no option to allow them); UTF-16 and UTF-32 by
 * BOM or by first pair; a declared encoding other than utf-8; version 1.1;
 * a <?xml declaration anywhere but offset 0; a named reference other than
 * the five; a reference to a non-Char; ]]> in content; a literal < in an
 * attribute value; -- inside a comment; a PI whose target is xml in any
 * case; a Name that is not a QName (at most one colon, both parts
 * non-empty); anything not well-formed.
 *
 * Normalisation, in this order and with this word: line ends first (XML
 * 1.0 section 2.11, CRLF and CR become LF, in text and in attribute
 * values), then attribute values (section 3.3.3: with no DTD every
 * attribute is CDATA, so each LITERAL TAB, LF or CR becomes a space). A
 * character reference to one of those - &#x9; &#xA; &#xD; - stays the
 * character. Canonical XML escapes TAB, LF and CR in attribute values, and
 * the only way such a character reaches the serialiser is through a
 * reference, because the literal form was already a space. A lexer that
 * normalises after expanding references produces canonical bytes that
 * differ from every other implementation's.
 *
 * Every string a token carries is in the arena, NUL-terminated. CDATA is a
 * TEXT token with the cdata flag, so the parser can merge it with its
 * neighbours; there is no CDATA node kind. The attribute list of a START or
 * EMPTY token is borrowed from the lexer's scratch and valid until the next
 * frx_lex_next.
 *
 * THE FULL PROFILE reads a DOCTYPE and expands the internal subset's
 * entities. An entity is an input frame, not a string substitution: the
 * lexer's in/len/pos describe the frame being read, a reference saves them
 * and points them at the replacement text, and reaching the end of that
 * text restores them. Every token-level function reads through in/len/pos
 * unchanged; the frame boundary is the one place the rules of section 4.4
 * are enforced (an element, comment, PI, CDATA section or attribute value
 * that opened inside a frame must close inside it; an end tag inside a
 * frame must close an element opened inside it), and that is the whole of
 * the well-formedness of a parsed entity. Offsets: `base` is the document
 * offset the current frame's byte 0 corresponds to, 0 for the document
 * itself and the entity literal's first byte otherwise, so a refusal
 * inside an entity points into its declaration; FRX_LEX_FAIL takes a
 * frame-relative offset and adds base, FRX_LEX_FAIL_ABS takes a document
 * offset. Attribute types come from the DTD before an element is lexed
 * (the DTD is complete before the root starts), which is what lets the
 * 3.3.3 normalisation by type happen here, in the one place the literal
 * whitespace rule already lives. The declaration parser is frx_dtd.h and
 * the frame stack's push, pop and budgets are frx_entity.h; both need this
 * file's primitives, so they follow it and are declared below.
 *
 * Needs frx_err.h, frx_arena.h, frx_buf.h, frx_utf8.h, frx_enc.h,
 * frx_tree.h. */

typedef struct frx_eldecl  frx_eldecl;   /* frx_dtd.h; frx_dtd and frx_doctype are frx_tree.h's */
typedef struct frx_attdef  frx_attdef;   /* frx_dtd.h */
typedef struct frx_entity  frx_entity;   /* frx_dtd.h */

/* what an entity reference saves: the frame it interrupted */
typedef struct frx_frame_in {
    const unsigned char *in;        /* the interrupted frame's bytes */
    size_t      len, pos, base;
    frx_str     cur_base;           /* its base URI */
    int         flat;               /* its offsets were flat */
    int         ext_decl;           /* it allowed PE references inside declarations */
    int         in_dtd;             /* it was inside a parameter entity or the external subset */
    frx_entity *entity;             /* the entity whose text is now current; NULL for fetched text with none */

include/frx/frx_lex.h  view on Meta::CPAN

            i += 3;
            continue;
        }
        if (frx_lex_starts(l, i, "<?")) {
            i += 2;
            while (i < l->len && !frx_lex_starts(l, i, "?>")) i++;
            if (i >= l->len) return 0;
            i += 2;
            continue;
        }
        if (c == '[') depth++;
        else if (c == ']') { if (depth) depth--; }
        else if (c == '>' && depth == 0) return 1;
        i++;
    }
    return 0;
}

/* a pseudo-attribute of the XML declaration: name S? = S? quoted; the
 * value is returned as a slice of the input */
static int
frx_lex_pseudo(frx_lex *l, const char *name, const unsigned char **v, size_t *vlen)
{
    unsigned char q;
    size_t start;
    if (!frx_lex_starts(l, l->pos, name)) {
        if (!l->eof && frx_lex_short(l, l->pos, name)) l->need_more = 1;
        return 0;
    }
    l->pos += strlen(name);
    frx_lex_skip_s(l);
    if (l->pos >= l->len) { if (!l->eof) l->need_more = 1; return 0; }
    if (l->in[l->pos] != '=') return 0;
    l->pos++;
    frx_lex_skip_s(l);
    if (l->pos >= l->len) { if (!l->eof) l->need_more = 1; return 0; }
    q = l->in[l->pos];
    if (q != '"' && q != '\'') return 0;
    start = ++l->pos;
    while (l->pos < l->len && l->in[l->pos] != q) l->pos++;
    if (l->pos >= l->len) { if (!l->eof) l->need_more = 1; return 0; }
    *v    = l->in + start;
    *vlen = l->pos - start;
    l->pos++;
    return 1;
}

static int
frx_lex_ieq(const unsigned char *v, size_t n, const char *lit)
{
    size_t i;
    if (n != strlen(lit)) return 0;
    for (i = 0; i < n; i++) {
        unsigned char a = v[i], b = (unsigned char)lit[i];
        if (a >= 'A' && a <= 'Z') a = (unsigned char)(a - 'A' + 'a');
        if (a != b) return 0;
    }
    return 1;
}

/* the prolog: max_bytes, the BOMs, the first pair, the declaration. With
 * eof clear it asks for more bytes wherever the declaration may go on. */
#define FRX_PROLOG_NEED(l, what) \
    (((l)->need_more || ((l)->pos >= (l)->len && !(l)->eof)) ? ((l)->need_more = 1, 0) : FRX_LEX_FAIL((l), FRX_E_SYNTAX, at, (what)))

static int
frx_lex_prolog(frx_lex *l)
{
    const unsigned char *in = l->in;
    size_t len = l->len;
    const unsigned char *v;
    size_t vlen, had_s;

    if (l->prolog_max_bytes && len > l->prolog_max_bytes)
        return FRX_LEX_FAIL(l, FRX_E_TOO_LARGE, l->prolog_max_bytes, "input exceeds max_bytes");
    /* the marks below need four bytes, and a declaration is judged whole */
    if (!l->eof && (len < 4 || frx_lex_short(l, 0, "<?xml "))) { l->need_more = 1; return 0; }

    if (len >= 2 && ((in[0] == 0xFE && in[1] == 0xFF) || (in[0] == 0xFF && in[1] == 0xFE)))
        return FRX_LEX_FAIL(l, FRX_E_ENCODING, 0, "UTF-16 or UTF-32 byte order mark; only UTF-8 is accepted");
    if (len >= 4 && in[0] == 0 && in[1] == 0 && in[2] == 0xFE && in[3] == 0xFF)
        return FRX_LEX_FAIL(l, FRX_E_ENCODING, 0, "UTF-32 byte order mark; only UTF-8 is accepted");
    if (len >= 2 && ((in[0] == '<' && in[1] == 0) || (in[0] == 0 && in[1] == '<')))
        return FRX_LEX_FAIL(l, FRX_E_ENCODING, 0, "UTF-16 without a byte order mark; only UTF-8 is accepted");
    if (len >= 3 && in[0] == 0xEF && in[1] == 0xBB && in[2] == 0xBF)
        l->pos = 3;

    /* the declaration: <?xml S version S? = S? "1.0" (S encoding ...)? (S standalone ...)? S? ?> */
    if (frx_lex_starts(l, l->pos, "<?xml") && l->pos + 5 < len && frx_is_s(in[l->pos + 5])) {
        size_t at = l->pos;
        l->pos += 5;
        frx_lex_skip_s(l);
        if (!frx_lex_pseudo(l, "version", &v, &vlen))
            return FRX_PROLOG_NEED(l, "the XML declaration must start with version");
        if (vlen == 3 && memcmp(v, "1.1", 3) == 0 && l->profile == FRX_PROFILE_FULL) {
            l->version11 = 1;                              /* full only; strict keeps 0.01's refusal */
        } else if (vlen == 3 && memcmp(v, "1.0", 3) == 0) {
            /* the one version both profiles know */
        } else if (l->profile == FRX_PROFILE_FULL && vlen > 2 && v[0] == '1' && v[1] == '.') {
            /* XML 1.0 fifth edition section 2.8: a 1.x this processor does
             * not know is processed as 1.0; the declared string is kept */
            size_t k;
            for (k = 2; k < vlen; k++)
                if (v[k] < '0' || v[k] > '9')
                    return FRX_LEX_FAIL(l, FRX_E_SYNTAX, at, "only XML version 1.0 is accepted");
        } else {
            return FRX_LEX_FAIL(l, FRX_E_SYNTAX, at, "only XML version 1.0 is accepted");
        }
        if (!frx_lex_intern(l, v, vlen, &l->version)) return 0;
        had_s = frx_lex_skip_s(l);
        if (l->pos >= l->len)
            return FRX_PROLOG_NEED(l, "unterminated XML declaration");
        if (frx_lex_starts(l, l->pos, "encoding")) {
            if (!had_s)
                return FRX_LEX_FAIL(l, FRX_E_SYNTAX, l->pos, "expected whitespace before encoding in the XML declaration");
            if (!frx_lex_pseudo(l, "encoding", &v, &vlen))
                return FRX_PROLOG_NEED(l, "malformed encoding declaration");
            if (l->profile == FRX_PROFILE_FULL) {
                /* XML 1.0 4.3.3: the declaration must name what the
                 * document arrived in. What arrived was detected by
                 * frx_enc.h and the bytes here are already UTF-8. */