Apophis

 view release on metacpan or  search on metacpan

Apophis.xs  view on Meta::CPAN

/*
 * Apophis.xs - Content-addressable storage with deterministic UUID v5
 *
 * Named after Apophis, the Egyptian serpent of chaos — here tamed
 * to bring order to content through deterministic hashing.
 *
 * 100% XS: all logic in C, Perl layer is just XSLoader.
 * Uses Horus library for RFC 9562 UUID v5 (SHA-1 namespace) generation.
 */

#define PERL_NO_GET_CONTEXT
#include "EXTERN.h"
#include "perl.h"
#include "XSUB.h"

#include "ppport.h"
#include "xop_compat.h"

#include <sys/stat.h>
#include <errno.h>
#ifndef _WIN32
#  include <unistd.h>
#else
#  include <io.h>
#  include <direct.h>
#  include <windows.h>   /* MoveFileEx; perl.h has it already, this is a guard */
#endif

/*
 * Windows compatibility:
 * perl.h on threaded Windows redefines mkdir, stat, unlink etc. as macros
 * requiring the interpreter context (my_perl).  Our static C helper functions
 * don't have pTHX, so we undo those overrides and use the real libc calls.
 */
#ifdef WIN32
#  undef mkdir
#  undef stat
#  undef unlink
#  undef rename
#  undef open
#  undef close
#  undef read
#  undef write
   /* Windows mkdir takes only one arg */
#  define apophis_mkdir(p, m) _mkdir(p)
   /*
    * Stat_t is struct w32_stat, the layout perl's own stat() fills in - but
    * that stat() is the macro undef'd just above, so what is left is the CRT
    * one, which writes a differently shaped struct.  Pair the call and the
    * type instead: _stat64 keeps st_size 64 bit, which the CRT's struct stat
    * does not on Win64.
    */
#  define apophis_stat_t struct _stat64
#  define apophis_stat(p, b) _stat64((p), (b))
#else
#  define apophis_mkdir(p, m) mkdir(p, m)
#  define apophis_stat_t struct stat
#  define apophis_stat(p, b) stat((p), (b))
#endif

/*
 * The CRT rename() refuses a destination that already exists, so the second
 * store of the same content with metadata could not replace its sidecar.
 * MoveFileEx replaces, and stays atomic.
 */
static int
apophis_rename(const char *from, const char *to)
{
#ifdef WIN32
    if (MoveFileExA(from, to, MOVEFILE_REPLACE_EXISTING))
        return 0;
    errno = EACCES;
    return -1;
#else
    return rename(from, to);
#endif
}

/* Horus UUID library - pure C, no Perl deps */
#define HORUS_FATAL(msg) croak("%s", (msg))
#include "horus_core.h"

/* ------------------------------------------------------------------ */
/* Constants                                                           */
/* ------------------------------------------------------------------ */

#define APOPHIS_STREAM_BUF  65536   /* 64KB read chunks for streaming */
#define APOPHIS_PATH_MAX    4096

/* ------------------------------------------------------------------ */
/* Internal: namespace UUID generation                                 */
/* ------------------------------------------------------------------ */

/* Derive a namespace UUID from a human-readable string via v5(DNS, name) */
static void
apophis_derive_namespace(unsigned char *ns_out,
                         const char *name, STRLEN name_len)
{
    horus_uuid_v5(ns_out, HORUS_NS_DNS,
                  (const unsigned char *)name, (size_t)name_len);
}

/* Format 16-byte UUID binary to 36-char string SV */
static SV *
apophis_uuid_to_sv(pTHX_ const unsigned char *uuid)
{
    char buf[HORUS_FMT_STR_LEN + 1];
    horus_format_uuid(buf, uuid, HORUS_FMT_STR);
    return newSVpvn(buf, HORUS_FMT_STR_LEN);
}

/* ------------------------------------------------------------------ */
/* Internal: content identification                                    */
/* ------------------------------------------------------------------ */

/* Identify in-memory content: v5(namespace, content) */
static void
apophis_identify_content(unsigned char *uuid_out,

Apophis.xs  view on Meta::CPAN


    if (apophis_rename(tmp_path, path) != 0) {
        unlink(tmp_path);
        croak("Apophis: cannot rename '%s' -> '%s': %s",
              tmp_path, path, strerror(errno));
    }
}

/* ------------------------------------------------------------------ */
/* Internal: metadata sidecar (key=value\n format)                     */
/* ------------------------------------------------------------------ */

static void
apophis_meta_write(pTHX_ const char *meta_path, HV *meta)
{
    PerlIO *fh;
    HE *entry;
    char tmp_path[APOPHIS_PATH_MAX];

    snprintf(tmp_path, sizeof(tmp_path), "%s.tmp.%d",
             meta_path, (int)getpid());

    fh = PerlIO_open(tmp_path, "w");
    if (!fh)
        croak("Apophis: cannot write metadata '%s': %s",
              tmp_path, strerror(errno));

    hv_iterinit(meta);
    while ((entry = hv_iternext(meta))) {
        SV *val = hv_iterval(meta, entry);
        I32 klen;
        const char *key = hv_iterkey(entry, &klen);
        STRLEN vlen;
        const char *vstr = SvPV(val, vlen);
        PerlIO_write(fh, key, (SSize_t)klen);
        PerlIO_write(fh, "=", 1);
        PerlIO_write(fh, vstr, (SSize_t)vlen);
        PerlIO_write(fh, "\n", 1);
    }

    PerlIO_close(fh);

    if (apophis_rename(tmp_path, meta_path) != 0) {
        unlink(tmp_path);
        croak("Apophis: cannot rename metadata '%s': %s",
              tmp_path, strerror(errno));
    }
}

static HV *
apophis_meta_read(pTHX_ const char *meta_path)
{
    PerlIO *fh;
    HV *meta;
    apophis_stat_t st;
    char *buf, *p, *end;
    SSize_t nread;

    /* A directory opens, and on the BSDs, Cygwin and the Hurd reads
     * too, so it is refused by its mode rather than by the read. */
    if (apophis_stat(meta_path, &st) != 0 || S_ISDIR(st.st_mode)) return NULL;

    fh = PerlIO_open(meta_path, "r");
    if (!fh) return NULL;

    buf = (char *)malloc((size_t)st.st_size + 1);
    /*
     # uncoverable branch true
     # uncoverable condition left count:1 */
    if (!buf) { PerlIO_close(fh); return NULL; }

    nread = PerlIO_read(fh, buf, (Size_t)st.st_size);
    /* A read that fails after the open and the stat succeeded is
     * nothing a test can arrange on a file.
     # uncoverable branch true
     # uncoverable condition left count:1
     # uncoverable condition left count:2 */
    if (nread < 0 || PerlIO_error(fh)) {
        PerlIO_close(fh); /* # uncoverable statement */
        free(buf); /* # uncoverable statement */
        return NULL; /* # uncoverable statement */
    }
    PerlIO_close(fh);
    buf[nread] = '\0';

    meta = newHV();
    p = buf;
    end = buf + nread;

    while (p < end) {
        char *line_end = strchr(p, '\n');
        char *eq;
        if (!line_end) line_end = end;

        eq = (char *)memchr(p, '=', (size_t)(line_end - p));
        if (eq) {
            hv_store(meta, p, (I32)(eq - p),
                     newSVpvn(eq + 1, (STRLEN)(line_end - eq - 1)), 0);
        }

        p = line_end + 1;
    }

    free(buf);
    return meta;
}

/* ------------------------------------------------------------------ */
/* Internal: object field accessors                                    */
/* ------------------------------------------------------------------ */

/* Get the 16-byte namespace bytes from the object */
static const unsigned char *
apophis_get_ns(pTHX_ HV *self)
{
    SV **svp = hv_fetchs(self, "_ns_bytes", 0);
    if (!svp || !SvOK(*svp))
        croak("Apophis: object has no namespace (not properly constructed)");
    return (const unsigned char *)SvPV_nolen(*svp);
}

Apophis.xs  view on Meta::CPAN

    I32 i;

    if (items <= 2)
        return NULL;
    if ((items - 2) % 2 != 0)
        croak("%s: odd number of optional arguments", who);
    opts = (HV *)sv_2mortal((SV *)newHV());
    for (i = 2; i < items; i += 2) {
        STRLEN klen;
        const char *k = SvPV(args[i], klen);
        SV *v = args[i + 1];
        if (meta && strEQ(k, "meta") && SvROK(v) && SvTYPE(SvRV(v)) == SVt_PVHV)
            *meta = (HV *)SvRV(v);
        else
            (void)hv_store(opts, k, klen, SvREFCNT_inc(v), 0);
    }
    return opts;
}

static const char *
apophis_content_of(pTHX_ const char *who, SV *content_ref, STRLEN *len)
{
    if (!SvROK(content_ref))
        croak("%s: argument must be a scalar reference", who);
    return SvPV(SvRV(content_ref), *len);
}

static void
apophis_do_identify(pTHX_ const char *who, SV *self, SV *content_ref, unsigned char *uuid)
{
    HV *hv = apophis_self_hv(aTHX_ who, self);
    const unsigned char *ns = apophis_get_ns(aTHX_ hv);
    STRLEN content_len;
    const char *content = apophis_content_of(aTHX_ who, content_ref, &content_len);

    apophis_identify_content(uuid, ns, content, content_len);
}

/* Writes the id, HORUS_FMT_STR_LEN bytes and a NUL, into id_str. */
static void
apophis_do_store(pTHX_ const char *who, SV *self, SV *content_ref, SV **args, I32 items, char *id_str)
{
    HV *hv = apophis_self_hv(aTHX_ who, self);
    const unsigned char *ns = apophis_get_ns(aTHX_ hv);
    STRLEN content_len, store_dir_len;
    const char *content = apophis_content_of(aTHX_ who, content_ref, &content_len);
    HV *meta = NULL;
    HV *opts = apophis_opts(aTHX_ who, args, items, &meta);
    const char *store_dir;
    unsigned char uuid[16];
    char path[APOPHIS_PATH_MAX];
    int path_len;
    apophis_stat_t st;

    apophis_identify_content(uuid, ns, content, content_len);
    horus_format_uuid(id_str, uuid, HORUS_FMT_STR);

    store_dir = apophis_get_store_dir(aTHX_ hv, opts, &store_dir_len);
    path_len = apophis_path(path, sizeof(path), store_dir, store_dir_len, id_str, HORUS_FMT_STR_LEN);

    if (apophis_stat(path, &st) != 0) {
        apophis_ensure_parent_dir(path);
        apophis_atomic_write(aTHX_ path, content, content_len);
    }

    if (meta) {
        char meta_path[APOPHIS_PATH_MAX];
        apophis_build_meta_path(meta_path, sizeof(meta_path), path, path_len);
        apophis_meta_write(aTHX_ meta_path, meta);
    }
}

/* The sharded path of an id under the store, for the four methods that
 * take an id and options. Gives the id's bytes back for verify. */
static int
apophis_id_path(pTHX_ const char *who, HV *hv, SV *id, SV **args, I32 items,
                char *path, size_t path_size, const char **id_str, STRLEN *id_len)
{
    HV *opts = apophis_opts(aTHX_ who, args, items, NULL);
    STRLEN store_dir_len;
    const char *store_dir = apophis_get_store_dir(aTHX_ hv, opts, &store_dir_len);

    *id_str = SvPV(id, *id_len);
    return apophis_path(path, path_size, store_dir, store_dir_len, *id_str, *id_len);
}

static bool
apophis_do_exists(pTHX_ const char *who, SV *self, SV *id, SV **args, I32 items)
{
    const char *id_str;
    STRLEN id_len;
    char path[APOPHIS_PATH_MAX];
    apophis_stat_t st;

    apophis_id_path(aTHX_ who, apophis_self_hv(aTHX_ who, self), id, args, items, path, sizeof(path), &id_str, &id_len);
    return apophis_stat(path, &st) == 0;
}

/* A new reference to the content, or &PL_sv_undef when there is none. */
static SV *
apophis_do_fetch(pTHX_ const char *who, SV *self, SV *id, SV **args, I32 items)
{
    const char *id_str;
    STRLEN id_len;
    char path[APOPHIS_PATH_MAX];
    apophis_stat_t st;
    PerlIO *fh;
    SV *content;
    SSize_t nread;

    apophis_id_path(aTHX_ who, apophis_self_hv(aTHX_ who, self), id, args, items, path, sizeof(path), &id_str, &id_len);
    if (apophis_stat(path, &st) != 0)
        return &PL_sv_undef;
    if (S_ISDIR(st.st_mode))
        croak("%s: read error on '%s': %s", who, path, strerror(EISDIR));

    fh = PerlIO_open(path, "rb");
    if (!fh)
        croak("%s: cannot open '%s': %s", who, path, strerror(errno));

    content = newSV((STRLEN)st.st_size + 1);
    SvPOK_on(content);
    nread = PerlIO_read(fh, SvPVX(content), (Size_t)st.st_size);
    /* A read that fails after the open and the stat succeeded is
     * nothing a test can arrange on a file.
     # uncoverable branch true
     # uncoverable condition left count:1
     # uncoverable condition left count:2 */
    if (nread < 0 || PerlIO_error(fh)) {
        PerlIO_close(fh); /* # uncoverable statement */
        SvREFCNT_dec(content); /* # uncoverable statement */
        croak("%s: read error on '%s'", who, path); /* # uncoverable statement */
    }
    PerlIO_close(fh);
    SvCUR_set(content, (STRLEN)nread);
    *SvEND(content) = '\0';
    return newRV_noinc(content);
}

static bool
apophis_do_verify(pTHX_ const char *who, SV *self, SV *id, SV **args, I32 items)
{
    const char *id_str;
    STRLEN id_len;
    char path[APOPHIS_PATH_MAX];
    PerlIO *fh;
    unsigned char uuid[16];
    char recomputed[HORUS_FMT_STR_LEN + 1];
    HV *hv = apophis_self_hv(aTHX_ who, self);
    const unsigned char *ns = apophis_get_ns(aTHX_ hv);

    apophis_id_path(aTHX_ who, hv, id, args, items, path, sizeof(path), &id_str, &id_len);
    fh = PerlIO_open(path, "rb");
    if (!fh)
        return FALSE;

    apophis_identify_stream(aTHX_ uuid, ns, fh);
    PerlIO_close(fh);
    horus_format_uuid(recomputed, uuid, HORUS_FMT_STR);
    return id_len == HORUS_FMT_STR_LEN && memcmp(id_str, recomputed, HORUS_FMT_STR_LEN) == 0;
}

static bool
apophis_do_remove(pTHX_ const char *who, SV *self, SV *id, SV **args, I32 items)
{
    const char *id_str;
    STRLEN id_len;
    char path[APOPHIS_PATH_MAX];
    char meta_path[APOPHIS_PATH_MAX];
    int path_len, removed;

    path_len = apophis_id_path(aTHX_ who, apophis_self_hv(aTHX_ who, self), id, args, items, path, sizeof(path), &id_str, &id_len);

Apophis.xs  view on Meta::CPAN

        SV *refs
    PREINIT:
        HV *hv;
        HV *opts = NULL;
        const unsigned char *ns;
        const char *store_dir;
        STRLEN store_dir_len;
        AV *av;
        I32 len, i;
    PPCODE:
        if (!sv_isobject(self))
            croak("Apophis::store_many: not an object");
        hv = (HV *)SvRV(self);
        ns = apophis_get_ns(aTHX_ hv);

        if (!SvROK(refs) || SvTYPE(SvRV(refs)) != SVt_PVAV)
            croak("Apophis::store_many: first argument must be an array ref");
        av = (AV *)SvRV(refs);
        len = av_len(av) + 1;

        if (items > 2) {
            int j;
            if ((items - 2) % 2 != 0)
                croak("Apophis::store_many: odd number of optional arguments");
            opts = newHV();
            sv_2mortal((SV *)opts);
            for (j = 2; j < items; j += 2) {
                STRLEN klen;
                const char *k = SvPV(ST(j), klen);
                hv_store(opts, k, klen, SvREFCNT_inc(ST(j+1)), 0);
            }
        }

        store_dir = apophis_get_store_dir(aTHX_ hv, opts, &store_dir_len);

        EXTEND(SP, len);
        for (i = 0; i < len; i++) {
            SV **svp = av_fetch(av, i, 0);
            SV *content_sv;
            const char *content;
            STRLEN content_len;
            unsigned char uuid[16];
            char id_str[HORUS_FMT_STR_LEN + 1];
            char path[APOPHIS_PATH_MAX];
            apophis_stat_t st;

            if (!svp || !SvROK(*svp))
                croak("Apophis::store_many: element %d must be a scalar ref",
                      (int)i);

            content_sv = SvRV(*svp);
            content = SvPV(content_sv, content_len);

            apophis_identify_content(uuid, ns, content, content_len);
            horus_format_uuid(id_str, uuid, HORUS_FMT_STR);

            apophis_path(path, sizeof(path),
                               store_dir, store_dir_len,
                               id_str, HORUS_FMT_STR_LEN);

            if (apophis_stat(path, &st) != 0) {
                apophis_ensure_parent_dir(path);
                apophis_atomic_write(aTHX_ path, content, content_len);
            }

            PUSHs(sv_2mortal(newSVpvn(id_str, HORUS_FMT_STR_LEN)));
        }

# ------------------------------------------------------------------ #
# find_missing(\@ids, %opts) -> @missing_ids                           #
# ------------------------------------------------------------------ #

void
find_missing(self, ids, ...)
        SV *self
        SV *ids
    PREINIT:
        HV *hv;
        HV *opts = NULL;
        const char *store_dir;
        STRLEN store_dir_len;
        AV *av;
        I32 len, i;
    PPCODE:
        if (!sv_isobject(self))
            croak("Apophis::find_missing: not an object");
        hv = (HV *)SvRV(self);

        if (!SvROK(ids) || SvTYPE(SvRV(ids)) != SVt_PVAV)
            croak("Apophis::find_missing: first argument must be an array ref");
        av = (AV *)SvRV(ids);
        len = av_len(av) + 1;

        if (items > 2) {
            int j;
            if ((items - 2) % 2 != 0)
                croak("Apophis::find_missing: odd number of optional arguments");
            opts = newHV();
            sv_2mortal((SV *)opts);
            for (j = 2; j < items; j += 2) {
                STRLEN klen;
                const char *k = SvPV(ST(j), klen);
                hv_store(opts, k, klen, SvREFCNT_inc(ST(j+1)), 0);
            }
        }

        store_dir = apophis_get_store_dir(aTHX_ hv, opts, &store_dir_len);

        for (i = 0; i < len; i++) {
            SV **svp = av_fetch(av, i, 0);
            const char *id_str;
            STRLEN id_len;
            char path[APOPHIS_PATH_MAX];
            apophis_stat_t st;

            if (!svp || !SvOK(*svp)) continue;

            id_str = SvPV(*svp, id_len);
            apophis_path(path, sizeof(path),
                               store_dir, store_dir_len, id_str, id_len);

            if (apophis_stat(path, &st) != 0) {
                XPUSHs(sv_2mortal(newSVpvn(id_str, id_len)));
            }
        }

# ------------------------------------------------------------------ #
# meta($id, %opts) -> \%meta or undef                                 #
# ------------------------------------------------------------------ #

SV *
meta(self, id, ...)
        SV *self
        SV *id
    PREINIT:
        HV *hv;
        HV *opts = NULL;
        const char *store_dir;
        STRLEN store_dir_len;
        const char *id_str;
        STRLEN id_len;
        char path[APOPHIS_PATH_MAX];
        int path_len;
        char meta_path[APOPHIS_PATH_MAX];
        HV *meta;
    CODE:
        if (!sv_isobject(self))
            croak("Apophis::meta: not an object");
        hv = (HV *)SvRV(self);

        if (items > 2) {
            int i;
            if ((items - 2) % 2 != 0)
                croak("Apophis::meta: odd number of optional arguments");
            opts = newHV();
            sv_2mortal((SV *)opts);
            for (i = 2; i < items; i += 2) {
                STRLEN klen;
                const char *k = SvPV(ST(i), klen);
                hv_store(opts, k, klen, SvREFCNT_inc(ST(i+1)), 0);
            }
        }

        store_dir = apophis_get_store_dir(aTHX_ hv, opts, &store_dir_len);
        id_str = SvPV(id, id_len);
        path_len = apophis_path(path, sizeof(path),
                                       store_dir, store_dir_len,
                                       id_str, id_len);
        apophis_build_meta_path(meta_path, sizeof(meta_path),
                                path, path_len);

        meta = apophis_meta_read(aTHX_ meta_path);
        if (meta) {
            RETVAL = newRV_noinc((SV *)meta);
        } else {
            RETVAL = &PL_sv_undef;
        }
    OUTPUT:
        RETVAL

# ------------------------------------------------------------------ #
# The public C ABI (ap_abi.h)                                          #



( run in 1.402 second using v1.01-cache-2.11-cpan-500dd33cbaf )