Algorithm-ToNumberMunger
view release on metacpan or search on metacpan
lib/Algorithm/ToNumberMunger.pm view on Meta::CPAN
our $VERSION = '0.0.1';
# Feature hashing (the 'hash' munger) is a tight per-byte FNV-1a loop with a
# 32-bit modular multiply. That is exactly the kind of work XS is good at and
# pure Perl is bad at (both for speed and, on a 32-bit perl, for correctness of
# the wrap-around), so we compile it in C when we can. Everything else here is a
# hash lookup or a couple of flops -- crossing the XS boundary per row would
# only make those slower, so they stay pure Perl. If the XS did not build (no
# compiler at install time) we fall back to a pure-Perl FNV-1a, which is exact
# on a 64-bit perl.
our $HAVE_XS = 0;
eval {
require XSLoader;
XSLoader::load( __PACKAGE__, $VERSION );
$HAVE_XS = 1;
1;
};
=head1 SYNOPSIS
use Algorithm::ToNumberMunger;
# one munger from a spec hash
my $code = Algorithm::ToNumberMunger->build(
{ munger => 'enum', map => { GET => 0, POST => 1, PUT => 2 } },
);
my $n = $code->('POST'); # 1
# a whole table of them at once, from a 'field => spec' hash
my $by_tag = Algorithm::ToNumberMunger->build_all(
\%mungers,
);
my $row_value = $by_tag->{method}->($raw{method});
=head1 DESCRIPTION
Many numeric pipelines -- anomaly detectors, feature stores, CSV loaders --
want every column to be a number, but the values they are handed are not always
numbers to begin with: an HTTP method is a string, a timestamp is a formatted
date, a high-cardinality label wants bucketing. An B<input munger> turns such a
raw value into a single number. Munging happens on the input side, before a row
is stored.
Mungers are declared as a plain data spec -- a hash naming a built-in munger and
carrying that munger's parameters -- so a table of them can be read straight out
of JSON or a config file:
{
"method": { "munger": "enum", "map": { "GET": 0, "POST": 1 } },
"bytes": { "munger": "log", "offset": 1 },
"label": { "munger": "hash", "buckets": 1024 }
}
B<Any field without an entry is raw> and is passed through unchanged; this module
is only concerned with fields that name a munger.
This class does not read or write files. It B<compiles> a spec into a closure
that maps one raw value to one number, so a caller can build its mungers once
from configuration and then apply them per row with no re-parsing. All
configuration errors are caught at build time; the returned closure only croaks
on genuinely un-mungeable I<input>.
=head1 CLASS METHODS
=head2 build
my $code = ...->build( \%spec );
my $code = ...->build( \%spec, $tag_name ); # $tag_name only sharpens errors
Compile a single munger spec into a coderef. C<%spec> must contain a C<munger>
key naming one of the L</BUILT-IN MUNGERS>; the remaining keys are that munger's
parameters. Croaks on an unknown munger name or an invalid parameter set. The
optional second argument is only used to make error messages point at a tag.
=cut
# name => builder. Each builder validates its slice of the spec up front and
# returns the per-value closure. Keeping them in a table (rather than a big
# if/elsif) is what makes known_mungers() and has_munger() cheap and honest.
my %BUILDERS = (
enum => \&_build_enum,
frozen_freq_map => \&_build_frozen_freq_map,
bool => \&_build_bool,
length => \&_build_length,
entropy => \&_build_entropy,
ngram => \&_build_ngram,
char => \&_build_char,
run => \&_build_run,
count => \&_build_count,
match => \&_build_match,
bucket => \&_build_bucket,
quantile => \&_build_quantile,
scale => \&_build_scale,
zscore => \&_build_zscore,
log => \&_build_log,
clamp => \&_build_clamp,
num => \&_build_num,
bit => \&_build_bit,
ip_class => \&_build_ip_class,
cidr => \&_build_cidr,
datetime => \&_build_datetime,
hash => \&_build_hash,
chain => \&_build_chain,
eps => \&_build_eps,
mgcp_enum => \&_build_mgcp_enum,
);
# Status-class mungers (http_enum, smtp_enum, sip_enum, ...) are one transform
# -- collapse a numeric reply code to its leading digit, int(code/div), with a
# divisor of 100 (10 for gemini's two-digit codes) -- differing only in which
# range 'strict' accepts. Register them all from this table so a new protocol
# is a single line and they can never drift apart. mgcp_enum is deliberately
# NOT a row here: its strict range has a hole (8xx exists, 6xx/7xx do not),
# which a single [lo, hi] cannot express, so it has its own builder below.
my %STATUS_PROTO = (
http => [ 100, 599 ], # 1xx-5xx
smtp => [ 200, 599 ], # 2xx-5xx; SMTP never issues 1yz in practice
sip => [ 100, 699 ], # 1xx-6xx; SIP adds a 6xx global-failure class
ftp => [ 100, 599 ], # 1xx-5xx FTP reply codes
rtsp => [ 100, 599 ], # RTSP (RFC 2326) reuses HTTP's status scheme
nntp => [ 100, 599 ], # 1xx-5xx NNTP (RFC 3977), SMTP-convention codes
dict => [ 100, 599 ], # DICT (RFC 2229) uses SMTP-style codes
gemini => [ 10, 69, 10 ], # two-digit codes, 1x-6x; class = int(code/10)
);
for my $proto ( keys %STATUS_PROTO ) {
my ( $lo, $hi, $div ) = @{ $STATUS_PROTO{$proto} };
$div = 100 unless defined $div;
$BUILDERS{"${proto}_enum"}
= sub { _status_class_munger( $proto, $lo, $hi, $div, @_ ) };
}
# ratio and combine consume several source fields at once, so they are only
# buildable through compile()'s multi-input form ('from' as an arrayref) -- a
# scalar build can never hand them more than one value. Registering a stub
# keeps known_mungers() honest and turns "used it as a scalar munger" into a
# pointed error instead of an unknown-munger one.
for my $name (qw(ratio combine)) {
$BUILDERS{$name} = sub {
my ( $spec, $where ) = @_;
croak "$name munger$where combines several inputs; it is only usable "
. "via compile() with 'from' as an arrayref of source fields";
};
}
sub build {
my ( $class, $spec, $tag ) = @_;
my $where = defined $tag ? " for tag '$tag'" : '';
croak "munger spec$where must be a hashref"
unless ref $spec eq 'HASH';
my $name = $spec->{munger};
croak "munger spec$where has no 'munger' name"
unless defined $name && length $name;
my $builder = $BUILDERS{$name}
or croak "unknown munger '$name'$where (known: " . join( ', ', $class->known_mungers ) . ')';
return $builder->( $spec, $where );
} ## end sub build
=head2 build_all
my $by_tag = ...->build_all( $info->{mungers} );
Compile a whole C<mungers> hash (tag name => spec) into a hash of tag name =>
coderef. A false/absent argument yields an empty hashref (every tag is raw).
Croaks if any spec is invalid, naming the offending tag.
=cut
sub build_all {
my ( $class, $mungers ) = @_;
return {} unless $mungers;
croak "'mungers' must be a hashref"
unless ref $mungers eq 'HASH';
my %by_tag;
for my $tag ( keys %$mungers ) {
$by_tag{$tag} = $class->build( $mungers->{$tag}, $tag );
}
return \%by_tag;
} ## end sub build_all
=head2 compile
my $plan = ...->compile( tags => \@tags, mungers => $info->{mungers} );
my $row = $plan->apply_named( \%named_input ); # numbers, in tags order
Compile a set's C<tags> and (optional) C<mungers> into a B<plan> object that maps
one input record to a fully-numeric row in tag order. Unlike L</build_all> (which
just compiles each spec in isolation), C<compile> understands the whole set:
=over 4
=item * a scalar munger, keyed by its output tag, fills that one column; its
input is read from the tag's own name, or from C<< from => 'other' >> to alias a
source field;
=item * an B<expanding> munger, keyed by any label and carrying C<< into =>
[tag, ...] >>, reads one source (C<from>, defaulting to the label) and fills
several columns at once -- this is how a single timestamp becomes both a
C<sin>/C<cos> pair without the two ever drifting apart (see L</datetime>);
=item * a B<combining> munger, keyed by its output tag and carrying a C<from>
B<list> (C<< from => ['bytes_out', 'bytes_in'] >>), reads several source
fields and fills that one column -- this is how a ratio becomes a single
feature without precomputing it upstream (see L</ratio> and L</combine>). The
sources are raw input fields, not other (possibly munged) columns;
=item * every remaining tag is B<raw> and passed through unchanged.
=back
Coverage is validated up front: C<compile> croaks if two mungers write the same
column, if an C<into> names a column not in C<tags>, if a munger key is neither a
tag nor an expander, if an expander's output count does not match its C<into>,
or if a C<from> list is given to a munger that cannot combine inputs. The
returned plan has two methods, both returning an arrayref of numbers in C<tags>
order: C<apply_named(\%hash)> (keyed by field name, the only form that supports
expanders and combiners) and C<apply_positional(\@row)> (positional; croaks if
the set has any expanding or combining munger, since a shared or combined
source cannot be expressed by position).
=cut
# name => builder returning ($list_returning_code, $arity), for the mungers that
# can fan one input out into several columns via 'into'.
my %MULTI_BUILDERS = (
datetime => \&_build_datetime_multi,
eps => \&_build_eps_multi,
chain => \&_build_chain_multi,
);
sub _build_multi {
my ( $class, $spec, $where ) = @_;
my $name = $spec->{munger};
croak "munger spec$where has no 'munger' name"
unless defined $name && length $name;
my $builder = $MULTI_BUILDERS{$name}
or croak "munger '$name'$where does not support multiple outputs "
. "('into'); only these do: "
. join( ', ', sort keys %MULTI_BUILDERS );
return $builder->( $spec, $where );
} ## end sub _build_multi
# name => builder returning the N-input closure, for the mungers that combine
# several source fields ('from' as an arrayref) into one column. The builder is
# handed the source count so arity errors surface at compile time.
my %COMBINE_BUILDERS = (
ratio => \&_build_ratio,
combine => \&_build_combine_op,
);
sub _build_combine {
my ( $class, $spec, $where, $nsrc ) = @_;
my $name = $spec->{munger};
croak "munger spec$where has no 'munger' name"
unless defined $name && length $name;
my $builder = $COMBINE_BUILDERS{$name}
or croak "munger '$name'$where does not support multiple inputs "
. "(a 'from' list); only these do: "
. join( ', ', sort keys %COMBINE_BUILDERS );
return $builder->( $spec, $where, $nsrc );
} ## end sub _build_combine
sub compile {
my ( $class, %args ) = @_;
my $tags = $args{tags};
croak "compile requires a non-empty 'tags' arrayref"
unless ref $tags eq 'ARRAY' && @$tags;
my $mungers = $args{mungers} || {};
croak "compile: 'mungers' must be a hashref"
unless ref $mungers eq 'HASH';
my %pos;
for my $i ( 0 .. $#$tags ) {
croak "compile: duplicate tag '$tags->[$i]'"
if exists $pos{ $tags->[$i] };
$pos{ $tags->[$i] } = $i;
}
my ( @scalar, @expand, @combine, %claimed );
my $claim = sub {
my ( $tag, $by ) = @_;
croak "munger '$by' targets unknown column '$tag'"
unless exists $pos{$tag};
croak "two mungers write column '$tag'"
if $claimed{$tag}++;
};
for my $key ( sort keys %$mungers ) {
my $spec = $mungers->{$key};
croak "munger '$key' spec must be a hashref"
unless ref $spec eq 'HASH';
my $from = defined $spec->{from} ? $spec->{from} : $key;
if ( ref $from eq 'ARRAY' ) {
croak "munger '$key': a 'from' list needs at least 2 source fields"
unless @$from >= 2;
croak "munger '$key': 'into' cannot be combined with a 'from' list"
if defined $spec->{into};
croak "munger '$key' is not a declared tag and has no 'into'"
unless exists $pos{$key};
my $code = $class->_build_combine( $spec, " for '$key'", scalar @$from );
$claim->( $key, $key );
push @combine, { tag => $key, from => [@$from], code => $code };
} elsif ( defined $spec->{into} ) {
lib/Algorithm/ToNumberMunger.pm view on Meta::CPAN
croak "enum munger$where: map value for '$k' ('"
. ( defined $map->{$k} ? $map->{$k} : 'undef' )
. "') is not numeric"
unless looks_like_number( $map->{$k} );
}
my $has_default = exists $spec->{default};
my $default = $spec->{default};
croak "enum munger$where: 'default' must be numeric"
if $has_default && !looks_like_number($default);
# Copy so a later edit of the caller's spec cannot mutate a live munger.
my %m = %$map;
return sub {
my ($v) = @_;
return $m{$v} if defined $v && exists $m{$v};
return $default if $has_default;
croak "enum munger$where: no mapping for '" . ( defined $v ? $v : 'undef' ) . "'";
};
} ## end sub _build_enum
=head2 frozen_freq_map
{ munger => 'frozen_freq_map', counts => { jpg => 40213, exe => 12, scr => 3 },
total => 67560 }
# defaults: mode => 'neg_log_prob', smoothing => 1, unseen => 'rare'
Frequency-encoding from a B<precomputed, frozen> count table: the rarer a value
was when the table was built, the more anomalous it scores. This is C<enum>'s
cousin -- a value-to-number map -- except the numbers are derived from observed
C<counts> rather than hand-authored, with the smoothing and unseen-value policy
that "rare = interesting" needs. It stays a stateless munger: the table is
computed offline and shipped in C<info.json>; this class only I<applies> it.
C<counts> maps each value to how many times it was seen. C<total> is the overall
observation count; it defaults to the sum of C<counts>, but may be given
explicitly and larger so you can B<prune the long tail> out of C<counts> while
still computing correct probabilities. The emitted number depends on C<mode>:
=over 4
=item * C<neg_log_prob> (default) - self-information C<-ln(prob)>: rare values
score high, common ones low. This is the axis "rare = interesting" describes and
what an Isolation Forest splits on most naturally.
=item * C<freq> - the probability itself, C<(count + smoothing) / denom>.
=item * C<log_count> - C<ln(1 + count)>, the count with its heavy tail tamed.
=item * C<count> - the raw count.
=back
Probabilities use add-one style C<smoothing> (default C<1>), treating "unseen" as
one aggregate bucket: C<prob(v) = (count + smoothing) / (total + smoothing*(V+1))>
where C<V> is the number of listed values. C<unseen> controls what a value absent
from the table maps to -- C<'rare'> (default) emits that value under the current
mode as if it had been seen zero times (for C<neg_log_prob>/C<freq> this is the
smoothed unseen bucket, for C<count>/C<log_count> it is C<0>), or a number to
force a fixed default. Because an unseen value is usually the very thing you are
hunting, mapping it to "maximally rare" rather than erroring is the point.
C<frozen_freq_map> only suits B<bounded, moderate-cardinality> columns (extensions,
vendor classes, named pipes, keyboard layouts, link addresses): the table lives
in C<info.json>, so a huge one bloats every read -- building one past
C<$Algorithm::ToNumberMunger::FROZEN_FREQ_MAP_WARN_KEYS>
values (default 10000) warns. For unbounded cardinality (JA3, full user-agents)
use L</hash> instead, which needs no table but keeps only decorrelation, not
commonness.
=cut
# name => 1 for the recognized frozen_freq_map output modes.
my %FREQ_MODE = map { $_ => 1 } qw(neg_log_prob freq log_count count);
# Building a table larger than this warns: info.json ships the whole map, so a
# high-cardinality column belongs in the 'hash' munger instead.
our $FROZEN_FREQ_MAP_WARN_KEYS = 10_000;
sub _build_frozen_freq_map {
my ( $spec, $where ) = @_;
my $counts = $spec->{counts};
croak "frozen_freq_map munger$where requires a non-empty 'counts' hashref"
unless ref $counts eq 'HASH' && %$counts;
my $sum = 0;
for my $k ( keys %$counts ) {
my $c = $counts->{$k};
croak "frozen_freq_map munger$where: count for '$k' ('"
. ( defined $c ? $c : 'undef' )
. "') is not a non-negative number"
unless looks_like_number($c) && $c >= 0;
$sum += $c;
}
my $V = keys %$counts;
carp "frozen_freq_map munger$where: 'counts' has $V keys; a table this large bloats "
. "info.json -- consider the 'hash' munger for unbounded cardinality"
if $V > $FROZEN_FREQ_MAP_WARN_KEYS;
my $total = defined $spec->{total} ? $spec->{total} : $sum;
croak "frozen_freq_map munger$where: 'total' must be numeric"
unless looks_like_number($total);
croak "frozen_freq_map munger$where: 'total' ($total) must be >= sum of counts ($sum)"
if $total < $sum;
my $mode = defined $spec->{mode} ? $spec->{mode} : 'neg_log_prob';
croak "frozen_freq_map munger$where: unknown mode '$mode' (known: " . join( ', ', sort keys %FREQ_MODE ) . ')'
unless $FREQ_MODE{$mode};
my $s = defined $spec->{smoothing} ? $spec->{smoothing} : 1;
croak "frozen_freq_map munger$where: 'smoothing' must be a non-negative number"
unless looks_like_number($s) && $s >= 0;
my $unseen = defined $spec->{unseen} ? $spec->{unseen} : 'rare';
croak "frozen_freq_map munger$where: 'unseen' must be 'rare' or a number"
unless $unseen eq 'rare' || looks_like_number($unseen);
# An unseen value under neg_log_prob has probability s/denom; with no
# smoothing that is 0 and -ln(0) is infinite, which would poison the column.
lib/Algorithm/ToNumberMunger.pm view on Meta::CPAN
# raw count -> emitted number under the chosen mode.
my $emit_for = sub {
my ($c) = @_;
return $c if $mode eq 'count';
return log( 1 + $c ) if $mode eq 'log_count';
my $p = ( $c + $s ) / $denom;
return $p if $mode eq 'freq';
return -log($p); # neg_log_prob
};
my %emit = map { $_ => $emit_for->( $counts->{$_} ) } keys %$counts;
my $unseen_value = $unseen eq 'rare' ? $emit_for->(0) : $unseen;
return sub {
my ($v) = @_;
return defined $v && exists $emit{$v} ? $emit{$v} : $unseen_value;
};
} ## end sub _build_frozen_freq_map
=head2 http_enum
{ munger => 'http_enum' }
{ munger => 'http_enum', strict => 1 }
Collapse an HTTP status code to its class: C<1xx> to C<1>, C<2xx> to C<2>, C<3xx>
to C<3>, and so on (i.e. C<int(code / 100)>). This is the usual bucketing for an
HTTP status column -- the forest cares far more about "was this a 4xx vs a 2xx"
than about C<403> vs C<404>, and it keeps the feature low-cardinality without
having to spell out every code in an C<enum> C<map>. The input must be numeric.
By default any numeric input is bucketed, so a bogus C<700> would quietly become
C<7>. With a true C<strict>, inputs outside the valid HTTP status range
(C<100>-C<599>) croak instead, so a malformed code is caught at write time rather
than smuggled into the model as a spurious class.
=head2 smtp_enum
{ munger => 'smtp_enum' }
{ munger => 'smtp_enum', strict => 1 }
The SMTP counterpart of L</http_enum>: collapse an SMTP reply code to its leading
digit (C<int(code / 100)>), since that digit I<is> the reply's meaning -- C<2yz>
completion, C<3yz> intermediate, C<4yz> transient failure, C<5yz> permanent
failure. As with C<http_enum> this keeps the column low-cardinality and lets the
forest weigh "a 5xx where a 2xx was expected" without enumerating every code.
With a true C<strict>, inputs outside the valid SMTP reply range (C<200>-C<599>)
croak. SMTP never issues C<1yz> replies in practice (no command permits a
positive-preliminary reply), so the strict floor is C<200> rather than
C<http_enum>'s C<100>.
=head2 sip_enum
{ munger => 'sip_enum' }
{ munger => 'sip_enum', strict => 1 }
The SIP counterpart of L</http_enum>: collapse a SIP status code to its leading
digit (C<int(code / 100)>). SIP reuses HTTP's class scheme but adds a sixth
class -- C<1xx> provisional, C<2xx> success, C<3xx> redirection, C<4xx> client
error, C<5xx> server error, C<6xx> global failure.
With a true C<strict>, inputs outside the valid SIP status range (C<100>-C<699>)
croak. The ceiling is C<699> rather than C<http_enum>'s C<599> precisely because
of that C<6xx> global-failure class.
=head2 ftp_enum
{ munger => 'ftp_enum' }
{ munger => 'ftp_enum', strict => 1 }
The FTP counterpart of L</http_enum>, for FTP reply codes: C<int(code / 100)>,
bucketing into C<1yz>-C<5yz>. With a true C<strict>, inputs outside C<100>-C<599>
croak.
=head2 rtsp_enum
{ munger => 'rtsp_enum' }
{ munger => 'rtsp_enum', strict => 1 }
The RTSP counterpart of L</http_enum>. RTSP (RFC 2326) deliberately reuses
HTTP's status scheme, so codes collapse to their leading digit the same way.
With a true C<strict>, inputs outside C<100>-C<599> croak.
=head2 nntp_enum
{ munger => 'nntp_enum' }
{ munger => 'nntp_enum', strict => 1 }
The NNTP counterpart of L</http_enum>, for NNTP (RFC 3977) reply codes, which
follow the SMTP convention -- C<1xx> informational, C<2xx> success, C<3xx>
send-more-input, C<4xx> transient failure, C<5xx> permanent failure. Unlike
SMTP, NNTP does issue C<1xx> replies (help text, capability lists), so the
strict floor is C<100> rather than C<smtp_enum>'s C<200>. With a true
C<strict>, inputs outside C<100>-C<599> croak.
=head2 dict_enum
{ munger => 'dict_enum' }
{ munger => 'dict_enum', strict => 1 }
The DICT counterpart of L</http_enum>, for DICT protocol (RFC 2229) status
codes, which use the SMTP-style code classes. With a true C<strict>, inputs
outside C<100>-C<599> croak.
=head2 gemini_enum
{ munger => 'gemini_enum' }
{ munger => 'gemini_enum', strict => 1 }
Like L</http_enum> but for the Gemini protocol, whose status codes are B<two>
digits -- C<1x> input expected, C<2x> success, C<3x> redirect, C<4x> temporary
failure, C<5x> permanent failure, C<6x> client certificate required -- so the
class is C<int(code / 10)>. With a true C<strict>, inputs outside C<10>-C<69>
croak.
=cut
# Shared closure for the status-class mungers registered from %STATUS_PROTO.
sub _status_class_munger {
my ( $proto, $lo, $hi, $div, $spec, $where ) = @_;
my $strict = $spec->{strict} ? 1 : 0;
return sub {
my ($v) = @_;
croak "${proto}_enum munger$where: '" . ( defined $v ? $v : 'undef' ) . "' is not a numeric status code"
unless looks_like_number($v);
croak "${proto}_enum munger$where: status code '$v' is out of range " . "($lo-$hi)"
if $strict && ( $v < $lo || $v > $hi );
return int( $v / $div );
};
} ## end sub _status_class_munger
=head2 mgcp_enum
{ munger => 'mgcp_enum' }
{ munger => 'mgcp_enum', strict => 1 }
The MGCP counterpart of L</http_enum>, for MGCP (RFC 3435) response codes:
C<int(code / 100)>. MGCP's classes are C<1xx> provisional, C<2xx> success,
C<4xx> transient error, C<5xx> permanent error, and C<8xx> package-specific --
there are no C<6xx> or C<7xx> codes, so the valid set has a B<hole> in it.
With a true C<strict>, inputs outside C<100>-C<599> B<and> outside
C<800>-C<899> croak. (That hole is why this is a hand-written builder rather
than another row of the shared status-class table, which can only express one
contiguous range.)
=cut
# MGCP's strict range is [100,599] union [800,899] -- 8xx package-specific
# codes are real, 6xx/7xx are not -- which %STATUS_PROTO's single [lo, hi]
# cannot express, hence this dedicated builder.
sub _build_mgcp_enum {
my ( $spec, $where ) = @_;
my $strict = $spec->{strict} ? 1 : 0;
return sub {
my ($v) = @_;
croak "mgcp_enum munger$where: '" . ( defined $v ? $v : 'undef' ) . "' is not a numeric status code"
unless looks_like_number($v);
croak "mgcp_enum munger$where: status code '$v' is out of range " . "(100-599 or 800-899)"
if $strict && !( ( $v >= 100 && $v <= 599 ) || ( $v >= 800 && $v <= 899 ) );
return int( $v / 100 );
};
} ## end sub _build_mgcp_enum
=head2 dns_rcode_enum
{ munger => 'dns_rcode_enum' }
{ munger => 'dns_rcode_enum', default => -1 }
The first of the B<named-map enums>: like L</enum>, except the C<map> is baked
in from a well-known registry instead of hand-authored (and inevitably
typo'd). All named-map enums share the same lookup rules: names are matched
B<case-insensitively>; where the emitted numbers are the protocol's own wire
encoding (as here -- rcode C<3> I<is> C<NXDOMAIN>), a numeric input is passed
through unchanged, so mixed feeds (one tool logs C<NXDOMAIN>, another logs
C<3>) land in one consistent column; and an unmapped value croaks unless the
spec supplies a numeric C<default>. As with C<enum>, an unrecognized value is
often exactly the anomaly worth keeping, so C<< default => -1 >> is the usual
escape hatch.
This one maps DNS RCODE names to their IANA values: C<NOERROR> 0, C<FORMERR>
1, C<SERVFAIL> 2, C<NXDOMAIN> 3, C<NOTIMP> 4 (alias C<NOTIMPL>), C<REFUSED> 5,
C<YXDOMAIN> 6, C<YXRRSET> 7, C<NXRRSET> 8, C<NOTAUTH> 9, C<NOTZONE> 10,
C<DSOTYPENI> 11, and the extended rcodes C<BADVERS>/C<BADSIG> 16, C<BADKEY>
17, C<BADTIME> 18, C<BADMODE> 19, C<BADNAME> 20, C<BADALG> 21, C<BADTRUNC> 22,
C<BADCOOKIE> 23.
=head2 dns_qtype_enum
{ munger => 'dns_qtype_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>; numeric inputs pass
through) mapping DNS RR type names to their IANA numbers: C<A> 1, C<NS> 2,
C<CNAME> 5, C<SOA> 6, C<NULL> 10, C<PTR> 12, C<MX> 15, C<TXT> 16, C<AAAA> 28,
C<SRV> 33, C<NAPTR> 35, C<DS> 43, C<RRSIG> 46, C<DNSKEY> 48, C<TLSA> 52,
C<SVCB> 64, C<HTTPS> 65, C<AXFR> 252, C<ANY> (or C<*>) 255, C<URI> 256,
C<CAA> 257, and the rest of the commonly-observed registry. The query-type mix
is a classic DNS-tunneling feature -- C<TXT>/C<NULL>-heavy traffic where
C<A>/C<AAAA> is normal.
=head2 syslog_severity_enum
{ munger => 'syslog_severity_enum' }
Named-map enum (lookup rules as L</dns_rcode_enum>; numeric inputs pass
through) mapping syslog severity names to their RFC 5424 codes: C<emerg> 0
(alias C<panic>), C<alert> 1, C<crit> 2, C<err> 3 (alias C<error>),
C<warning> 4 (alias C<warn>), C<notice> 5, C<info> 6 (alias
C<informational>), C<debug> 7. Genuinely ordinal -- lower is more severe --
so a threshold split on it is meaningful.
=head2 syslog_facility_enum
{ munger => 'syslog_facility_enum' }
Named-map enum (lookup rules as L</dns_rcode_enum>; numeric inputs pass
through) mapping syslog facility names to their RFC 5424 codes: C<kern> 0,
C<user> 1, C<mail> 2, C<daemon> 3, C<auth> 4 (alias C<security>), C<syslog> 5,
C<lpr> 6, C<news> 7, C<uucp> 8, C<cron> 9, C<authpriv> 10, C<ftp> 11, C<ntp>
12, C<audit> 13, C<alert> 14, C<clock> 15, and C<local0>-C<local7> 16-23.
=head2 ip_proto_enum
{ munger => 'ip_proto_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>; numeric inputs pass
through) mapping IP protocol names to their IANA protocol numbers: C<icmp> 1,
C<igmp> 2, C<ipip> 4 (alias C<ipencap>), C<tcp> 6, C<egp> 8, C<udp> 17,
C<dccp> 33, C<ipv6> 41, C<rsvp> 46, C<gre> 47, C<esp> 50, C<ah> 51,
C<icmpv6> 58 (alias C<ipv6-icmp>), C<ospf> 89, C<pim> 103, C<sctp> 132,
C<udplite> 136. The map is frozen here rather than delegated to
C<getprotobyname> so a value munges to the same number on every host.
=head2 tls_version_enum
{ munger => 'tls_version_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) mapping a TLS/SSL protocol
version name to an B<ordinal>: C<SSLv2> 0, C<SSLv3> 1, C<TLSv1> 2, C<TLSv1.1>
3, C<TLSv1.2> 4, C<TLSv1.3> 5, with the common spelling variants (C<ssl3>,
C<tls1.2>, ...) accepted. Ordinal so "older than expected" is a monotone
feature a threshold split can express. Because these ordinals are this
module's invention rather than a wire encoding, numeric inputs are B<not>
passed through -- a C<1.2> would land on the wrong scale -- and croak like
any other unmapped value (or take the C<default>).
=head2 http_method_enum
{ munger => 'http_method_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the registered HTTP
request methods: C<GET> 0, C<HEAD> 1, C<POST> 2, C<PUT> 3, C<DELETE> 4,
C<CONNECT> 5, C<OPTIONS> 6, C<TRACE> 7, C<PATCH> 8. HTTP has no numeric
method encoding, so these are unordered ordinals of this module's invention
(a canonical map beats every set inventing its own numbering) and numeric
inputs are not passed through. An unlisted -- possibly probing -- method
croaks unless a C<default> is given, and that unlisted-method signal is often
the interesting one.
=head2 sip_method_enum
{ munger => 'sip_method_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the SIP request
methods: C<INVITE> 0, C<ACK> 1, C<BYE> 2, C<CANCEL> 3, C<REGISTER> 4,
C<OPTIONS> 5, C<PRACK> 6, C<SUBSCRIBE> 7, C<NOTIFY> 8, C<PUBLISH> 9,
C<INFO> 10, C<REFER> 11, C<MESSAGE> 12, C<UPDATE> 13. Like
lib/Algorithm/ToNumberMunger.pm view on Meta::CPAN
=head2 tcp_state_enum
{ munger => 'tcp_state_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) mapping the TCP state
machine, as Suricata logs it under C<flow.tcp.state>, to an B<ordinal> along
the connection lifecycle: C<none> 0, C<syn_sent> 1, C<syn_recv> 2,
C<established> 3, C<fin_wait1> 4, C<fin_wait2> 5, C<closing> 6, C<time_wait>
7, C<close_wait> 8, C<last_ack> 9, C<closed> 10. Ordinal so "further along
teardown than expected" is a monotone feature a threshold split can express.
Being ordinals of our own invention, numeric inputs are not passed through.
=head2 flow_state_enum
{ munger => 'flow_state_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for Suricata's
C<flow.state>: C<new> 0, C<established> 1, C<closed> 2, C<bypassed> 3,
C<local_bypass> 4 -- roughly ordinal along the flow lifecycle. Numeric inputs
are not passed through.
=head2 flow_reason_enum
{ munger => 'flow_reason_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for Suricata's
C<flow.reason>, why a flow was logged out: C<timeout> 0, C<forced> 1,
C<shutdown> 2, C<unknown> 3. Numeric inputs are not passed through.
=head2 suricata_action_enum
{ munger => 'suricata_action_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for Suricata's
C<alert.action> and the related rule/drop actions: C<allowed> 0, C<blocked>
1, C<pass> 2, C<drop> 3, C<reject> 4, C<alert> 5. In IDS mode the field is
C<allowed>/C<blocked>; the rule-action names are accepted too for IPS feeds
and C<drop> events. Numeric inputs are not passed through.
=head2 postfix_status_enum
{ munger => 'postfix_status_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for Postfix's delivery
C<status=> disposition, numbered in a rough sent-to-failed severity order so
a threshold split is meaningful: C<sent> 0, C<deferred> 1, C<bounced> 2,
C<expired> 3, C<deliverable> 4, C<undeliverable> 5, C<hold> 6, C<discard> 7,
C<filtered> 8, C<reject> 9, C<softbounce> 10. Stock delivery agents emit only
C<sent>/C<deferred>/C<bounced>/C<expired>; C<deliverable>/C<undeliverable> come
from address verification (C<verify>), and the remainder cover HOLD/DISCARD
actions and values common log normalizers emit. Being labels of this module's
numbering, numeric inputs are not passed through.
=head2 spf_result_enum
{ munger => 'spf_result_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for an SPF check result
(RFC 7208), as logged by policyd-spf or carried in an C<Authentication-Results>
header, numbered pass-to-fail: C<pass> 0, C<neutral> 1, C<none> 2, C<softfail>
3, C<fail> 4, C<temperror> 5, C<permerror> 6. The older spellings C<error>
(for C<temperror>) and C<unknown> (for C<permerror>) are accepted as aliases.
Numeric inputs are not passed through.
=head2 dkim_result_enum
{ munger => 'dkim_result_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for a DKIM verification
result (RFC 8601), as logged by opendkim or carried in an
C<Authentication-Results> header, numbered pass-to-fail: C<pass> 0, C<neutral>
1, C<none> 2, C<policy> 3, C<fail> 4, C<temperror> 5, C<permerror> 6. The older
spellings C<error> (for C<temperror>) and C<unknown> (for C<permerror>) are
accepted as aliases. Numeric inputs are not passed through.
=head2 dmarc_result_enum
{ munger => 'dmarc_result_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for a DMARC evaluation
result (RFC 7489 / RFC 8601), as logged by opendmarc or carried in an
C<Authentication-Results> header: C<pass> 0, C<none> 1, C<fail> 2, C<temperror>
3, C<permerror> 4, and opendmarc's C<bestguesspass> 5. This is the DMARC
I<result> (did the message pass alignment), not the policy I<disposition>
(C<none>/C<quarantine>/C<reject>) -- for that, use a plain L</enum>. Numeric
inputs are not passed through.
=head2 sasl_mech_enum
{ munger => 'sasl_mech_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the SASL
authentication mechanism -- as Dovecot logs C<mech=>, Postfix logs
C<sasl_method=>, and submission/IMAP/POP3 auth report -- numbered in a rough
B<weakest-to-strongest> order so the ordinal carries a little signal on its
own: the cleartext and C<anonymous> mechanisms sort low, the legacy
challenge-response ones (C<cram-md5>, C<digest-md5>, C<ntlm>, ...) in the
middle, then C<srp>/C<scram-*>, the OAuth/federated tokens, and finally the
Kerberos/GSS and certificate (C<external>) mechanisms. About two dozen
mechanisms are baked in, covering the IANA registry plus the ubiquitous
non-registered C<login>, C<xoauth2>, and C<apop>. Being ordinals of this
module's numbering, numeric inputs are not passed through; an unlisted
mechanism croaks unless a numeric C<default> is given.
If you would rather the number carry B<no> implied gradient, see
L</sasl_mech_iana_enum>, which numbers the same set alphabetically.
=head2 sasl_mech_iana_enum
{ munger => 'sasl_mech_iana_enum', default => -1 }
The nominal counterpart of L</sasl_mech_enum>: the B<same> set of SASL
mechanisms (the two share one list, so they can never cover different
mechanisms), but numbered B<alphabetically> rather than by strength -- a
purely categorical encoding for when a strength gradient would be a
misleading feature. Lookup rules and the no-numeric-passthrough behaviour are
as L</sasl_mech_enum>.
=head2 http_version_enum
{ munger => 'http_version_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) mapping the HTTP protocol
version to an B<ordinal>: C<HTTP/0.9> 0, C<HTTP/1.0> 1, C<HTTP/1.1> 2,
C<HTTP/2.0> 3, C<HTTP/3.0> 4. The access-log spelling (C<HTTP/1.1>), the bare
number (C<1.1>), and the ALPN/shorthand forms (C<h2>, C<h2c>, C<h3>) are all
accepted, so an Apache/nginx C<%H> field and a Suricata C<http.protocol> land
in one column. Ordinal so "older than expected" (a C<0.9>/C<1.0> request from
a scanner) is a monotone feature a threshold split can express. Because these
are ordinals of our own numbering -- and a logged C<2> denotes version C<2.0>,
not the integer two -- numeric inputs are B<not> passed through.
=head2 spamassassin_autolearn_enum
{ munger => 'spamassassin_autolearn_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for SpamAssassin's
C<autolearn=> field: C<no> 0, C<ham> 1, C<spam> 2, C<disabled> 3, C<failed> 4,
C<unavailable> 5, C<unknown> 6. The numbering is essentially nominal (the
Bayes auto-learn outcome is a category, not a scale), arranged only so the
"nothing was learned" states cluster away from the ham/spam ones. The spam
I<score> is already a number for L</num>, and the spam/ham verdict a L</bool>;
this covers the one autolearn field neither derives. Numeric inputs are not
lib/Algorithm/ToNumberMunger.pm view on Meta::CPAN
Named-map enum (lookup rules as L</dns_rcode_enum>) for a ClamAV per-target
verdict as logged by C<clamd> / clamav-milter: C<OK> 0, C<FOUND> 1, C<ERROR>
2. Small but exactly the signal worth flagging (a C<FOUND> line). Numeric
inputs are not passed through.
=head2 kerberos_etype_enum
{ munger => 'kerberos_etype_enum', default => -1 }
Named-map enum for the Kerberos ticket encryption type -- Windows events
4768/4769 (logged as hex, C<0x17>) and any other AD/Kerberos source. The
values are the RFC 3961 etype numbers, which B<are> the wire encoding, so
(unlike most of the enums here) a decimal input passes through unchanged: the
map exists only to resolve the hex spellings (C<0x17> => 23, C<0x12> => 18,
...) and the RFC/MIT names (C<rc4-hmac>/C<rc4>/C<arcfour-hmac> => 23,
C<aes256-cts-hmac-sha1-96>/C<aes256> => 18, C<aes128> => 17, C<des3> => 16,
C<des-cbc-md5> => 3, C<des-cbc-crc> => 1) onto that same number. The classic
use is flagging C<rc4-hmac> (0x17) as a downgrade/roasting signal against an
C<aes256> baseline. Lookup is case-insensitive; an unlisted etype croaks
unless a numeric C<default> is given.
=head2 windows_integrity_level_enum
{ munger => 'windows_integrity_level_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the Windows / Sysmon
process integrity level, as an B<ordinal>: C<untrusted> 0, C<low> 1, C<medium>
2, C<high> 3, C<system> 4 (with C<mediumplus> folded into C<medium>). The
C<S-1-16-*> mandatory-label SIDs Windows sometimes logs in place of the word
(C<S-1-16-12288> => 3, ...) are accepted as aliases. Ordinal so "higher
privilege than expected" is a monotone feature. Numeric inputs are not passed
through.
=head2 windows_logon_status_enum
{ munger => 'windows_logon_status_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the NTSTATUS
sub-status on a failed Windows logon (events 4625/4776), collapsed to a
compact B<reason category> rather than the raw 32-bit code: C<0xC0000064>
(no such user) 0, C<0xC000006A> (bad password) 1, C<0xC000006D> (generic bad
user/pass) 2, C<0xC000006F> (outside hours) 3, C<0xC0000070> (workstation
restriction) 4, C<0xC0000071> (password expired) 5, C<0xC0000072> (disabled)
6, C<0xC0000193> (account expired) 7, C<0xC0000133> (clock skew) 8,
C<0xC0000224> (must change password) 9, C<0xC0000234> (locked out) 10,
C<0xC000015B> (logon type not granted) 11. Keys are the hex codes exactly as
logged (matched case-insensitively); only the common logon subset is baked in,
so an unlisted code croaks unless a numeric C<default> is given. Numeric
inputs are not passed through.
=head2 windows_impersonation_level_enum
{ munger => 'windows_impersonation_level_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the event 4624
impersonation level, ordered by reach: C<anonymous> 0, C<identification> 1,
C<impersonation> 2, C<delegation> 3. The C<%%1832> / C<%%1833> message tokens
Windows often emits in place of the words (Identification / Impersonation) are
accepted as aliases. Numeric inputs are not passed through.
=head2 aad_signin_error_enum
{ munger => 'aad_signin_error_enum', default => -1 }
Named-map enum for the Azure AD / Entra sign-in C<ResultType> error code,
collapsed to a compact B<reason category> rather than the raw code: C<0>
(success) 0, invalid-password (C<50126>, C<50056>) 1, no-such-user (C<50034>)
2, disabled (C<50057>) 3, locked / smart-lockout (C<50053>) 4, password-expired
(C<50055>, C<50144>) 5, MFA-required (C<50074>, C<50076>, C<50079>) 6,
MFA-failed (C<500121>, C<50158>) 7, blocked-by-conditional-access (C<53003>,
C<53000>, C<53001>, C<530032>) 8, session-expired (C<50173>) 9. Although
C<ResultType> is already numeric, the code space is huge and sparse and its
magnitude carries no signal -- this maps the common codes onto a handful of
meaningful buckets (and, unlike L</hash>, keeps related codes together). Keys
are the codes as logged; only the common subset is baked in, so an unlisted
code croaks unless a numeric C<default> is given. Because the output is a
category of our own numbering, numeric inputs are B<not> passed through.
=head2 risk_level_enum
{ munger => 'risk_level_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the Entra Identity
Protection C<riskLevel> as an B<ordinal>: C<none> 0, C<low> 1, C<medium> 2,
C<high> 3. C<hidden> and C<unknownFutureValue> are left to the C<default>
(they are not points on the scale). Numeric inputs are not passed through.
=head2 aws_principal_type_enum
{ munger => 'aws_principal_type_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the CloudTrail
C<userIdentity.type>: C<Root> 0, C<IAMUser> 1, C<AssumedRole> 2,
C<FederatedUser> 3, C<SAMLUser> 4, C<WebIdentityUser> 5, C<Directory> 6,
C<IdentityCenterUser> 7, C<AWSAccount> 8, C<AWSService> 9, C<Unknown> 10. The
numbering is nominal (distinct stable numbers, not a scale); C<Root> is the
value you actually alert on. Numeric inputs are not passed through.
=head2 aad_client_app_enum
{ munger => 'aad_client_app_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the Azure AD
C<ClientAppUsed>, numbered so the modern clients sort low (C<Browser> 0,
C<Mobile Apps and Desktop clients> 1) and the B<legacy-auth> protocols -- which
cannot satisfy MFA -- sort high (C<Exchange ActiveSync>, C<IMAP4>, C<POP3>,
C<Authenticated SMTP>, C<MAPI Over HTTP>, C<Exchange Web Services>, C<Exchange
Online PowerShell>, C<AutoDiscover>, C<Offline Address Book>, C<Other
clients>, from 2 up). A "C<< >= 2 >> means legacy auth" threshold is the
intended feature. C<imap>/C<pop>/C<mapi> short forms are accepted as aliases.
Numeric inputs are not passed through.
=head2 risk_state_enum
{ munger => 'risk_state_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the Entra Identity
Protection C<riskState>: C<none> 0, C<confirmedSafe> 1, C<remediated> 2,
C<dismissed> 3, C<atRisk> 4, C<confirmedCompromised> 5. Numeric inputs are not
passed through.
=head2 vpc_flow_log_status_enum
{ munger => 'vpc_flow_log_status_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the AWS VPC Flow Logs
C<log-status>: C<OK> 0, C<NODATA> 1, C<SKIPDATA> 2. (The per-flow
C<ACCEPT>/C<REJECT> action is a plain L</bool>; this is the capture-health
field.) Numeric inputs are not passed through.
=head2 aws_event_type_enum
{ munger => 'aws_event_type_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the CloudTrail
C<eventType>: C<AwsApiCall> 0, C<AwsServiceEvent> 1, C<AwsConsoleAction> 2,
C<AwsConsoleSignIn> 3, C<AwsCloudTrailInsight> 4. C<AwsConsoleSignIn> is the
one worth flagging. Numeric inputs are not passed through.
=head2 conditional_access_result_enum
{ munger => 'conditional_access_result_enum', default => -1 }
Named-map enum (lookup rules as L</dns_rcode_enum>) for the Azure AD sign-in
C<conditionalAccessStatus>: C<success> 0, C<notApplied> 1, C<notEnabled> 2,
C<reportOnly> 3, C<failure> 4. Numeric inputs are not passed through.
=cut
# Named-map enums: baked-in value->number maps for well-known registries,
# registered as "<name>_enum". Keys are stored lowercase; lookup lowercases
# the input, giving the case-insensitive matching the POD promises. 'numeric'
# says whether a numeric input is passed through as-is -- set only where the
# numbers are the protocol's own encoding (a qtype 28 IS AAAA on the wire);
# where they are ordinals of our own invention (tls_version, the method
# enums), passthrough would mix scales, so a number croaks like any other
# unmapped value.
# SASL mechanism names, ordered weakest-to-strongest, for sasl_mech_enum. The
# same set is numbered alphabetically for sasl_mech_iana_enum, so the two
# mungers can never end up covering different mechanisms. Includes the IANA
# registry plus the ubiquitous non-registered login/xoauth2/apop.
my @SASL_MECHS_BY_STRENGTH = qw(
anonymous plain login
apop cram-md5 digest-md5 ntlm skey otp securid rpa kerberos_v4
srp scram-sha-1 scram-sha-1-plus scram-sha-256 scram-sha-256-plus
xoauth2 oauthbearer openid20 saml20
gssapi gs2-krb5 gss-spnego external
);
my %NAMED_ENUM = (
dns_rcode => {
numeric => 1,
map => {
noerror => 0,
formerr => 1,
servfail => 2,
nxdomain => 3,
notimp => 4,
notimpl => 4,
refused => 5,
yxdomain => 6,
yxrrset => 7,
nxrrset => 8,
notauth => 9,
notzone => 10,
dsotypeni => 11,
badvers => 16,
badsig => 16,
badkey => 17,
badtime => 18,
badmode => 19,
badname => 20,
badalg => 21,
badtrunc => 22,
badcookie => 23,
},
},
dns_qtype => {
numeric => 1,
map => {
a => 1,
ns => 2,
cname => 5,
soa => 6,
mb => 7,
mg => 8,
mr => 9,
'null' => 10,
wks => 11,
ptr => 12,
hinfo => 13,
minfo => 14,
mx => 15,
txt => 16,
rp => 17,
afsdb => 18,
sig => 24,
key => 25,
aaaa => 28,
loc => 29,
srv => 33,
naptr => 35,
kx => 36,
cert => 37,
dname => 39,
opt => 41,
ds => 43,
sshfp => 44,
ipseckey => 45,
rrsig => 46,
nsec => 47,
dnskey => 48,
dhcid => 49,
nsec3 => 50,
nsec3param => 51,
tlsa => 52,
smimea => 53,
hip => 55,
cds => 59,
cdnskey => 60,
openpgpkey => 61,
csync => 62,
zonemd => 63,
svcb => 64,
https => 65,
eui48 => 108,
eui64 => 109,
tkey => 249,
tsig => 250,
ixfr => 251,
axfr => 252,
any => 255,
'*' => 255,
uri => 256,
caa => 257,
},
},
syslog_severity => {
numeric => 1,
map => {
emerg => 0,
panic => 0,
alert => 1,
crit => 2,
err => 3,
error => 3,
warning => 4,
warn => 4,
notice => 5,
info => 6,
informational => 6,
debug => 7,
},
},
syslog_facility => {
numeric => 1,
map => {
kern => 0,
user => 1,
mail => 2,
daemon => 3,
auth => 4,
security => 4,
syslog => 5,
lpr => 6,
news => 7,
uucp => 8,
cron => 9,
authpriv => 10,
ftp => 11,
ntp => 12,
audit => 13,
alert => 14,
clock => 15,
( map { ( "local$_" => 16 + $_ ) } 0 .. 7 ),
},
},
ip_proto => {
numeric => 1,
map => {
icmp => 1,
igmp => 2,
ipip => 4,
ipencap => 4,
tcp => 6,
egp => 8,
udp => 17,
dccp => 33,
ipv6 => 41,
rsvp => 46,
gre => 47,
esp => 50,
ah => 51,
icmpv6 => 58,
'ipv6-icmp' => 58,
ospf => 89,
pim => 103,
sctp => 132,
udplite => 136,
},
},
tls_version => {
numeric => 0,
map => {
sslv2 => 0,
ssl2 => 0,
lib/Algorithm/ToNumberMunger.pm view on Meta::CPAN
close_wait last_ack closed
);
map { $order[$_] => $_ } 0 .. $#order;
},
},
},
flow_state => {
numeric => 0,
map => {
new => 0,
established => 1,
closed => 2,
bypassed => 3,
local_bypass => 4,
},
},
flow_reason => {
numeric => 0,
map => {
timeout => 0,
forced => 1,
shutdown => 2,
unknown => 3,
},
},
suricata_action => {
numeric => 0,
map => {
allowed => 0,
blocked => 1,
pass => 2,
drop => 3,
reject => 4,
alert => 5,
},
},
postfix_status => {
numeric => 0,
map => {
sent => 0,
deferred => 1,
bounced => 2,
expired => 3,
deliverable => 4,
undeliverable => 5,
hold => 6,
discard => 7,
filtered => 8,
reject => 9,
softbounce => 10,
},
},
spf_result => {
numeric => 0,
map => {
pass => 0,
neutral => 1,
none => 2,
softfail => 3,
fail => 4,
temperror => 5,
error => 5, # older spelling of temperror
permerror => 6,
unknown => 6, # older spelling of permerror
},
},
dkim_result => {
numeric => 0,
map => {
pass => 0,
neutral => 1,
none => 2,
policy => 3,
fail => 4,
temperror => 5,
error => 5, # older spelling of temperror
permerror => 6,
unknown => 6, # older spelling of permerror
},
},
dmarc_result => {
numeric => 0,
map => {
pass => 0,
none => 1,
fail => 2,
temperror => 3,
permerror => 4,
bestguesspass => 5,
},
},
sasl_mech => {
numeric => 0,
map => {
do {
my @o = @SASL_MECHS_BY_STRENGTH;
map { $o[$_] => $_ } 0 .. $#o;
},
},
},
sasl_mech_iana => {
numeric => 0,
map => {
do {
my @o = sort @SASL_MECHS_BY_STRENGTH;
map { $o[$_] => $_ } 0 .. $#o;
},
},
},
http_version => {
numeric => 0, # a logged "2" is version 2.0, not the integer two
map => {
# Ordinal so "older than expected" is a monotone feature. Accepts
# the access-log spelling (HTTP/1.1), the bare number (1.1), and
# the ALPN/h2 shorthands.
do {
my %v = (
'http/0.9' => 0,
'http/1.0' => 1,
'http/1.1' => 2,
'http/2.0' => 3,
'http/3.0' => 4,
);
$v{'0.9'} = 0;
$v{'1.0'} = 1;
$v{'1.1'} = 2;
$v{'2.0'} = $v{'2'} = $v{'http/2'} = $v{'h2'} = $v{'h2c'} = 3;
$v{'3.0'} = $v{'3'} = $v{'http/3'} = $v{'h3'} = 4;
%v;
},
},
},
spamassassin_autolearn => {
numeric => 0,
map => {
# SpamAssassin's autolearn= field. Nominal, but ordered so
# "no learning happened" sorts below the ham/spam outcomes.
no => 0,
ham => 1,
spam => 2,
disabled => 3,
failed => 4,
unavailable => 5,
unknown => 6,
},
},
rspamd_action => {
numeric => 0,
lib/Algorithm/ToNumberMunger.pm view on Meta::CPAN
},
ssh_auth_method => {
numeric => 0,
map => {
# OpenSSH authentication method, ordered by credential strength
# (weakest first) so "weaker than expected" is monotone. The
# ordering is a judgement call, not an IANA registry.
do {
my @o = qw(
none password keyboard-interactive
hostbased publickey
gssapi-with-mic gssapi-keyex
);
my %m = map { $o[$_] => $_ } 0 .. $#o;
$m{'gssapi'} = $m{'gssapi-with-mic'};
%m;
},
},
},
amavis_category => {
numeric => 0,
map => {
# amavisd-new content category, ordered clean -> worst.
do {
my @o = qw(
clean oversized unchecked spammy spam
badheader banned infected mtablocked
);
my %m = map { $o[$_] => $_ } 0 .. $#o;
$m{'bad-header'} = $m{'badheader'};
$m{'virus'} = $m{'infected'};
$m{'mta-blocked'} = $m{'mtablocked'};
%m;
},
},
},
systemd_result => {
numeric => 0,
map => {
# systemd unit "result" (e.g. "Failed with result 'timeout'").
do {
my @o = qw(
success protocol timeout exit-code signal
core-dump watchdog start-limit-hit oom-kill resources
);
my %m = map { $o[$_] => $_ } 0 .. $#o;
$m{'exit_code'} = $m{'exit-code'};
$m{'core_dump'} = $m{'core-dump'};
$m{'start_limit_hit'} = $m{'start-limit-hit'};
$m{'oom_kill'} = $m{'oom-kill'};
%m;
},
},
},
clamav_result => {
numeric => 0,
map => {
# clamd / clamav-milter per-target verdict.
ok => 0,
found => 1,
error => 2,
},
},
kerberos_etype => {
numeric => 1, # RFC 3961 etype numbers are the wire value; a logged 23 IS rc4
map => {
# Windows logs the ticket encryption type as hex ("0x17"); the
# RFC/MIT names appear in other Kerberos logs. Everything resolves
# to the decimal etype number, so a decimal input passes through.
do {
my %e = (
'des-cbc-crc' => 1,
'des-cbc-md5' => 3,
'des3-cbc-sha1' => 16,
'aes128-cts-hmac-sha1-96' => 17,
'aes256-cts-hmac-sha1-96' => 18,
'rc4-hmac' => 23,
'rc4-hmac-exp' => 24,
);
$e{'des3'} = 16;
$e{'aes128'} = 17;
$e{'aes256'} = 18;
$e{'arcfour-hmac'} = $e{'rc4'} = 23;
$e{'0x1'} = 1;
$e{'0x3'} = 3;
$e{'0x10'} = 16;
$e{'0x11'} = 17;
$e{'0x12'} = 18;
$e{'0x17'} = 23;
$e{'0x18'} = 24;
%e;
},
},
},
windows_integrity_level => {
numeric => 0,
map => {
# Sysmon IntegrityLevel; ordinal so "higher privilege than
# expected" is monotone. Text labels plus the S-1-16-* mandatory
# label SIDs Windows sometimes logs in their place.
do {
my %m = (
untrusted => 0,
low => 1,
medium => 2,
high => 3,
system => 4,
);
$m{'mediumplus'} = 2;
$m{'s-1-16-0'} = 0;
$m{'s-1-16-4096'} = 1;
$m{'s-1-16-8192'} = 2;
$m{'s-1-16-12288'} = 3;
$m{'s-1-16-16384'} = 4;
%m;
},
},
},
windows_logon_status => {
numeric => 0,
map => {
# Common NTSTATUS sub-status codes on failed logons (4625/4776),
# mapped to a compact reason category -- the raw 32-bit value is
# not itself a useful feature. Keys are the hex codes as logged.
'0xc0000064' => 0, # user name does not exist
'0xc000006a' => 1, # bad password
'0xc000006d' => 2, # bad user name or password (generic)
'0xc000006f' => 3, # outside authorized hours
'0xc0000070' => 4, # workstation restriction
'0xc0000071' => 5, # password expired
'0xc0000072' => 6, # account disabled
'0xc0000193' => 7, # account expired
'0xc0000133' => 8, # clock skew between client and server
'0xc0000224' => 9, # must change password at next logon
'0xc0000234' => 10, # account locked out
'0xc000015b' => 11, # logon type not granted
},
},
windows_impersonation_level => {
numeric => 0,
map => {
# 4624 ImpersonationLevel; ordinal by reach. Text labels plus the
# two "%%18xx" message tokens Windows most often emits in place.
anonymous => 0,
identification => 1,
impersonation => 2,
delegation => 3,
'%%1832' => 1,
'%%1833' => 2,
},
},
aad_signin_error => {
numeric => 0,
map => {
# Azure AD / Entra sign-in ResultType codes collapsed to a compact
# reason category -- the raw code is a huge sparse space whose
# magnitude carries no signal. Keys are the numeric codes as
# logged; only the common subset is baked in, the rest take the
# default.
'0' => 0, # success
'50126' => 1, # invalid username or password
'50056' => 1, # invalid or null password
'50034' => 2, # user does not exist in directory
'50057' => 3, # account disabled
'50053' => 4, # account locked / smart lockout
'50055' => 5, # password expired
'50144' => 5, # AD password expired
'50074' => 6, # strong auth (MFA) required
'50076' => 6, # MFA required by conditional access
'50079' => 6, # user must enroll for MFA
'500121' => 7, # MFA denied / authentication failed
'50158' => 7, # external security challenge not satisfied
'53003' => 8, # blocked by conditional access
'53000' => 8, # device not compliant (CA)
'53001' => 8, # device not domain joined (CA)
'530032' => 8, # blocked by security policy (CA)
'50173' => 9, # fresh auth token required (session expired)
},
},
risk_level => {
numeric => 0,
map => {
# Entra Identity Protection riskLevel, ordinal. hidden /
# unknownFutureValue are left to the default.
none => 0,
low => 1,
medium => 2,
high => 3,
},
},
aws_principal_type => {
numeric => 0,
map => {
# CloudTrail userIdentity.type. Nominal (distinct stable numbers);
# 'root' is the value you actually alert on.
do {
my @o = qw(
root iamuser assumedrole federateduser samluser
webidentityuser directory identitycenteruser
awsaccount awsservice unknown
);
map { $o[$_] => $_ } 0 .. $#o;
},
},
},
aad_client_app => {
numeric => 0,
map => {
# Azure AD ClientAppUsed. Numbered so the modern clients sort low
# and the legacy-auth protocols (which cannot do MFA) sort high --
# a ">= 2 means legacy auth" threshold is the feature you want.
do {
lib/Algorithm/ToNumberMunger.pm view on Meta::CPAN
}; ## end sub
} ## end sub _build_num
=head2 ratio
# 'io_ratio' is a tag; bytes_out and bytes_in are input fields
"io_ratio": { "munger": "ratio", "from": ["bytes_out", "bytes_in"] }
{ munger => 'ratio', from => [qw(bytes_out bytes_in)], zero => -1 }
First source divided by the second: with C<< from => [a, b] >> the column gets
C<a / b>. Asymmetry between two counters is a classic feature the counters
alone cannot express -- bytes out over bytes in flags exfiltration, requests
over responses flags scanning -- and the division has to happen at munge time
because a forest split only ever sees one column. A zero denominator yields
C<zero> (default C<0>) instead of dying, since "nothing came back" is a
legitimate row, not bad input; pick a C<zero> outside the ratio's normal range
if you want those rows to stand out. Both inputs must be numeric.
This is a B<multi-input> munger: it only makes sense with several sources, so
it is only usable through L</compile> with C<from> as an arrayref of exactly
two field names (and thus C<apply_named> / C<write_named>). The sources are
raw input fields, not other columns.
=head2 combine
{ munger => 'combine', op => 'sum', from => [qw(bytes_in bytes_out)] }
{ munger => 'combine', op => 'max', from => [qw(req_time resp_time)] }
Fold two or more numeric source fields into one column with C<op>: C<sum>,
C<diff> (first minus second; exactly two sources), C<product>, C<min>, C<max>,
or C<mean>. The general-purpose sibling of L</ratio> for when the interesting
feature is a total, a gap, or an extreme across fields rather than any one
field. Every input must be numeric.
Like C<ratio>, this is a B<multi-input> munger: only usable through
L</compile> with C<from> as an arrayref of source field names.
=cut
sub _build_ratio {
my ( $spec, $where, $nsrc ) = @_;
croak "ratio munger$where takes exactly 2 source fields (numerator, denominator), not $nsrc"
unless $nsrc == 2;
my $zero = defined $spec->{zero} ? $spec->{zero} : 0;
croak "ratio munger$where: 'zero' must be numeric"
unless looks_like_number($zero);
return sub {
for my $v (@_) {
croak "ratio munger$where: '" . ( defined $v ? $v : 'undef' ) . "' is not numeric"
unless looks_like_number($v);
}
return $zero if $_[1] == 0;
return $_[0] / $_[1];
};
} ## end sub _build_ratio
# op => fold over the already numeric-checked source values. A table so the
# error message can enumerate them and a new op is one line.
my %COMBINE_OPS = (
sum => sub { my $t = 0; $t += $_ for @_; return $t },
diff => sub { return $_[0] - $_[1] },
product => sub { my $t = 1; $t *= $_ for @_; return $t },
min => sub {
my $t = shift;
for (@_) { $t = $_ if $_ < $t }
return $t;
},
max => sub {
my $t = shift;
for (@_) { $t = $_ if $_ > $t }
return $t;
},
mean => sub { my $t = 0; $t += $_ for @_; return $t / @_ },
);
sub _build_combine_op {
my ( $spec, $where, $nsrc ) = @_;
my $op = $spec->{op};
croak "combine munger$where requires an 'op' (one of: " . join( ', ', sort keys %COMBINE_OPS ) . ')'
unless defined $op && length $op;
my $fold = $COMBINE_OPS{$op}
or croak "combine munger$where: unknown op '$op' (known: " . join( ', ', sort keys %COMBINE_OPS ) . ')';
croak "combine munger$where: op 'diff' takes exactly 2 source fields, not $nsrc"
if $op eq 'diff' && $nsrc != 2;
return sub {
for my $v (@_) {
croak "combine munger$where: '" . ( defined $v ? $v : 'undef' ) . "' is not numeric"
unless looks_like_number($v);
}
return $fold->(@_);
};
} ## end sub _build_combine_op
=head2 bit
{ munger => 'bit', mask => '0x12' } # SYN or ACK set?
{ munger => 'bit', mask => '0x02', mode => 'all' } # the SYN bit itself
{ munger => 'bit', mode => 'popcount' } # how many flags at all
{ munger => 'bit', mask => '0x0f', mode => 'value' } # low nibble, 0-15
{ munger => 'bit', mask => '0x02', base => 16 } # Suricata tcp_flags "1b"
Bit-level features from an integer flags word (TCP flags, DNS header flags,
protocol option words): the raw word is meaningless to a threshold split, but
individual bits and bit I<counts> are real signals. The input must be a
non-negative integer, in decimal or C<0x> hex (so a logged C<0x12> works
as-is); C<mask> may be written either way too.
Set C<< base => 16 >> to read the B<input> as bare hexadecimal with no C<0x>
prefix -- Suricata logs C<tcp.tcp_flags> (and C<tcp_flags_ts>/C<tcp_flags_tc>)
as e.g. C<"1b">, which is otherwise ambiguous with decimal. A C<0x> prefix on
the input is still accepted under C<< base => 16 >>. C<mask> is always written
in decimal or C<0x> hex regardless of C<base>. Modes:
=over 4
=item * C<any> (default) - C<1> if any bit of C<mask> is set in the value.
lib/Algorithm/ToNumberMunger.pm view on Meta::CPAN
} ## end if ($fam)
return $default if $has_default;
croak "cidr munger$where: '" . ( defined $v ? $v : 'undef' ) . "' is not a parseable IP address";
}; ## end sub
} ## end sub _build_cidr
=head2 datetime
{ munger => 'datetime', format => '%Y-%m-%dT%H:%M:%S', part => 'epoch' }
{ munger => 'datetime', format => '%Y-%m-%d %H:%M:%S', part => 'hour' }
Parse a formatted timestamp with L<Time::Piece> (C<strptime>, so C<format> is a
standard strptime pattern) and extract one numeric C<part>:
=over 4
=item * C<epoch> (default) - seconds since the epoch.
=item * C<year>, C<mon> (1-12), C<mday> (1-31), C<hour>, C<min>, C<sec>.
=item * C<wday> - day of week, C<0>=Sunday .. C<6>=Saturday.
=item * C<yday> - day of year, C<0>-based.
=item * C<frac_day> - time of day as a fraction in C<[0, 1)>, i.e.
C<(hour*3600 + min*60 + sec) / 86400>. Handy as a cyclic-ish time-of-day feature.
=item * C<frac_week> - position within the week as a fraction in C<[0, 1)>, the
week starting Sunday to match C<wday>: C<(wday*86400 + hour*3600 + min*60 + sec)
/ 604800>. Like C<frac_day> but cycling over a week, so a weekly rhythm (weekend
vs. weekday, or a Monday-morning batch) shows up as a feature.
=item * C<sin_day> / C<cos_day>, C<sin_week> / C<cos_week> - the C<frac_*> value
mapped onto a circle, C<sin(2*pi*frac)> and C<cos(2*pi*frac)>. Prefer these over
the raw C<frac_*> when feeding the forest: a plain fraction has a false seam at
the wrap (23:59 and 00:00 sit at opposite ends, 1 vs 0, though they are a minute
apart), whereas the sin/cos pair is continuous across midnight/Sunday. Store
I<both> of a pair in two columns so the position is unambiguous.
=back
Time features often carry the anomaly (a job that normally runs at 03:00
suddenly firing at noon, or a weekday task firing on a Sunday), which is why this
is a first-class munger.
B<Multi-output form.> A cyclic pair belongs together -- C<sin> alone collides
(C<sin> is symmetric about its peak, so two different times map to one value) and
the forest then treats distinct times as identical. To emit a pair atomically,
give C<parts> (plural) and route them to two columns with C<into> (see
L</compile>):
"time_of_week": {
"munger": "datetime", "from": "timestamp",
"format": "%Y-%m-%dT%H:%M:%S",
"parts": [ "sin_week", "cos_week" ],
"into": [ "time_sin", "time_cos" ]
}
The timestamp is parsed once and both columns are filled together, so they can
never drift apart or be half-configured. C<parts> and C<into> must be the same
length. (Using C<parts> without C<into>, or C<part> with C<into>, is an error.)
B<Performance.> Two transparent accelerations, both value-identical to the plain
path: a one-slot memo returns the previous result when the same stamp string
repeats (the common case in bursty event streams); and when the format is built
from only the six numeric codes C<%Y %m %d %H %M %S> (once each, e.g.
C<%Y-%m-%dT%H:%M:%S>), parsing skips C<strptime> for a compiled regex plus
integer date math, falling back to C<strptime> for any value the regex does not
match B<or whose fields are out of range> (a month C<13>, an hour C<24>, a
C<Feb 30>) -- so an invalid stamp croaks or normalizes exactly as C<strptime>
would, never silently feeding nonsense to the date math. Like C<strptime>
without a zone code, stamps are treated as UTC.
=cut
# Fraction (in [0,1)) of the way through the day / week, shared by the frac_*
# parts and their sin/cos cyclic encodings.
sub _frac_day {
my $t = shift;
return ( $t->hour * 3600 + $t->min * 60 + $t->sec ) / 86400;
}
sub _frac_week {
my $t = shift;
return ( $t->day_of_week * 86400 + $t->hour * 3600 + $t->min * 60 + $t->sec ) / 604800;
}
my $TWO_PI = 2 * atan2( 0, -1 ); # atan2(0,-1) == pi, core-only, no POSIX
# part name => how to pull it off a Time::Piece object.
my %DATETIME_PART = (
epoch => sub { $_[0]->epoch },
year => sub { $_[0]->year },
mon => sub { $_[0]->mon },
mday => sub { $_[0]->mday },
hour => sub { $_[0]->hour },
min => sub { $_[0]->min },
sec => sub { $_[0]->sec },
wday => sub { $_[0]->day_of_week },
yday => sub { $_[0]->yday },
frac_day => \&_frac_day,
frac_week => \&_frac_week,
sin_day => sub { sin( $TWO_PI * _frac_day( $_[0] ) ) },
cos_day => sub { cos( $TWO_PI * _frac_day( $_[0] ) ) },
sin_week => sub { sin( $TWO_PI * _frac_week( $_[0] ) ) },
cos_week => sub { cos( $TWO_PI * _frac_week( $_[0] ) ) },
);
# ---- fast fixed-format engine ----------------------------------------------
#
# Time::Piece->strptime costs microseconds per call. When the format is built
# from only the six all-numeric codes below (once each, e.g. the ubiquitous
# '%Y-%m-%dT%H:%M:%S'), we can compile it to a capture regex and derive every
# part with integer math instead -- several times faster, and bit-identical:
# both paths treat the stamp as UTC (strptime with no zone does the same).
# Anything fancier (%b, %z, %j, ...) stays on strptime.
# strptime code => [ field name, capture pattern ].
my %FAST_CODE = (
Y => [ 'year', '[0-9]{4}' ],
m => [ 'mon', '[0-9]{2}' ],
lib/Algorithm/ToNumberMunger.pm view on Meta::CPAN
my $code = sub {
my $s = defined $_[0] ? "$_[0]" : '';
$s = $_->($s) for @$ops;
return $term->($s);
};
return ( $code, $arity );
} ## end sub _build_chain_multi
=head2 eps
{ munger => 'eps', prefix => 'http-req:', from => 'src_ip' }
{ munger => 'eps', prefix => 'dns-nxd:', from => 'src_ip',
read => 'rate', mark => 0 }
# multi-output: one daemon round trip fills several columns
{ munger => 'eps', prefix => 'http-req:', from => 'src_ip',
parts => [ 'rate', 'count' ], into => [ 'req_rate', 'req_count' ] }
Per-entity sliding-window event rates via the C<iqbi-damiq> daemon shipped with
L<Algorithm::EventsPerSecond> (see
L<Algorithm::EventsPerSecond::Sukkal>). The input value becomes a meter B<key>
(after C<prefix> is prepended); by default the munger B<marks> one event against
that key and returns the key's current events-per-second, using the daemon's
C<MARKRATE> command -- mark and query in a single command with a single reply.
This is the munger behind rate columns like a per-source request rate: every
event marks its source's meter and stores the rate the meter now reads.
Unlike every other munger this one consults external state -- but the state
lives in the daemon, not here, so the munger itself remains a stateless client
and rows stay reproducible I<given> the daemon. Because the daemon is shared,
multiple writer processes marking the same keys see one B<global> rate, which an
in-process meter could never give.
Spec keys:
=over 4
=item * C<socket> - unix socket path of the daemon. Defaults to
C<$Algorithm::ToNumberMunger::EPS_SOCKET>
(C</var/run/iqbi-damiq.sock>).
=item * C<prefix> - string prepended to the input to form the key, namespacing
this column's meters (two columns keyed on the same field need different
prefixes or they share meters). No whitespace/control characters. Default C<''>.
=item * C<mark> - whether to mark the key (default C<1>). Marking rides
C<MARKRATE>: with C<< read => 'rate' >> that one command is the whole exchange;
with C<count>/C<total> the read is pipelined after it (the C<MARKRATE> rate
reply is discarded), so a marking failure still comes back as an ordinary
first reply. With C<< mark => 0 >> the munger only reads, for columns whose
marking is done elsewhere -- e.g. an NXDOMAIN rate is I<marked> by the pipeline
only on NXDOMAIN responses but I<read> on every row.
=item * C<read> - what to read: C<rate> (events/sec over the daemon's window,
default), C<count> (events inside the window), or C<total> (lifetime).
=item * C<parts> + C<into> - multi-output form (see L</compile>): read several
of C<rate>/C<count>/C<total> for the one key in a single round trip, filling one
column each. When marking, the C<MARKRATE> reply itself serves the first C<rate>
part, so C<< parts => ['rate', 'count'] >> costs exactly two commands.
=item * C<on_error> - C<'die'> (default) croaks the write when the daemon is
unreachable or replies C<ERR>; a number is returned instead as a quiet fallback.
Note C<0> is indistinguishable from a genuinely idle key, so quiet fallback
biases the column -- loud is the default on purpose.
=item * C<timeout> - per-operation socket timeout in seconds (default 5,
best-effort via C<SO_RCVTIMEO>/C<SO_SNDTIMEO>), so a wedged daemon cannot hang a
writer forever.
=back
Semantics worth knowing: a marked read B<includes the event just marked>; keys
have whitespace/control bytes replaced with C<_> to satisfy the daemon's key
rules; connections are made lazily on first use and kept open (reconnecting
transparently after a fork or an error), so compiling a plan -- including the
eager validation in C<write_info> -- needs no running daemon. Each eps column
costs one unix-socket round trip per row; the multi-output form exists so
rate+count of the same key costs one round trip, not two.
=cut
# Default socket path of the iqbi-damiq daemon.
our $EPS_SOCKET = '/var/run/iqbi-damiq.sock';
# Persistent daemon connections, keyed by socket path, shared by every eps
# munger in the process. Entries record the pid that opened them so a forked
# writer transparently reopens instead of sharing a socket with its parent.
# Connections are made lazily on first use -- never at munger build time, so a
# plan can compile (eager validation) with no daemon running.
my %EPS_CONN;
sub _eps_conn {
my ( $path, $timeout ) = @_;
my $c = $EPS_CONN{$path};
return $c->{fh} if $c && $c->{pid} == $$;
require Socket;
require IO::Socket::UNIX;
my $fh = IO::Socket::UNIX->new(
Type => Socket::SOCK_STREAM(),
Peer => $path,
) or die "cannot connect to iqbi-damiq at $path: $!\n";
# Best-effort read/write timeouts so a wedged daemon cannot hang a writer.
eval {
my $tv = pack( 'l!l!', $timeout, 0 );
setsockopt( $fh, Socket::SOL_SOCKET(), Socket::SO_RCVTIMEO(), $tv );
setsockopt( $fh, Socket::SOL_SOCKET(), Socket::SO_SNDTIMEO(), $tv );
};
$EPS_CONN{$path} = { fh => $fh, pid => $$ };
return $fh;
} ## end sub _eps_conn
# One pipelined transaction: send $cmd (possibly several lines) and read
# $nreplies "OK n" lines, one per command sent. The munger only ever sends
# commands that reply exactly once -- MARKRATE (which marks AND returns the
# rate in one go), RATE, COUNT, TOTAL; never a bare MARK, whose reply-only-on-
# error behavior would let a failure desynchronize the reply stream. Dies on
# ERR, EOF, or timeout; the caller still drops the cached connection on error
# as belt and braces.
sub _eps_txn {
my ( $path, $timeout, $cmd, $nreplies ) = @_;
my $fh = _eps_conn( $path, $timeout );
print {$fh} $cmd or die "write to iqbi-damiq failed: $!\n";
my @out;
for ( 1 .. $nreplies ) {
my $reply = <$fh>;
die "iqbi-damiq closed the connection (or timed out)\n"
unless defined $reply;
$reply =~ /\AOK (\S+)/
or die "iqbi-damiq replied: $reply";
push @out, $1 + 0;
}
return @out;
} ## end sub _eps_txn
# Validate the spec keys shared by the scalar and multi-output eps builders.
sub _eps_spec {
my ( $spec, $where ) = @_;
my $socket = defined $spec->{socket} ? $spec->{socket} : $EPS_SOCKET;
croak "eps munger$where: 'socket' must be a non-empty path"
unless length $socket;
my $prefix = defined $spec->{prefix} ? $spec->{prefix} : '';
croak "eps munger$where: 'prefix' may not contain whitespace or control " . 'characters'
if $prefix =~ /[\s[:cntrl:]]/;
my $mark = exists $spec->{mark} ? ( $spec->{mark} ? 1 : 0 ) : 1;
my $timeout = defined $spec->{timeout} ? $spec->{timeout} : 5;
croak "eps munger$where: 'timeout' must be a positive number"
unless looks_like_number($timeout) && $timeout > 0;
my $on_error = defined $spec->{on_error} ? $spec->{on_error} : 'die';
croak "eps munger$where: 'on_error' must be 'die' or a number"
unless $on_error eq 'die' || looks_like_number($on_error);
return ( $socket, $prefix, $mark, $timeout, $on_error );
} ## end sub _eps_spec
my %EPS_READ = map { $_ => 1 } qw(rate count total);
sub _build_eps {
my ( $spec, $where ) = @_;
croak "eps munger$where: 'parts' is for the multi-output form (needs " . "'into'); use 'read' for a single column"
if defined $spec->{parts};
my ( $socket, $prefix, $mark, $timeout, $on_error ) = _eps_spec( $spec, $where );
my $read = defined $spec->{read} ? $spec->{read} : 'rate';
croak "eps munger$where: unknown read '$read' (known: " . join( ', ', sort keys %EPS_READ ) . ')'
unless $EPS_READ{$read};
# Command plan, fixed at build time. The common case -- mark and read the
# rate -- is the daemon's single MARKRATE command. mark+count/total rides
# MARKRATE too (its rate reply is discarded) so marking failures come back
# as an ordinary first reply instead of a bare MARK's error-only surprise.
my @cmds
= !$mark ? ( uc $read )
: $read eq 'rate' ? ('MARKRATE')
: ( 'MARKRATE', uc $read );
return sub {
my ($v) = @_;
my $key = $prefix . ( defined $v ? "$v" : '' );
$key =~ s/[\s[:cntrl:]]/_/g;
my @replies = eval {
die "empty key\n" unless length $key;
_eps_txn( $socket, $timeout, join( '', map { "$_ $key\n" } @cmds ), scalar @cmds );
};
if ($@) {
my $err = $@;
delete $EPS_CONN{$socket}; # reconnect fresh next call
croak "eps munger$where: $err" if $on_error eq 'die';
return $on_error + 0;
}
return $replies[-1]; # the requested read is always the last reply
}; ## end sub
} ## end sub _build_eps
# Multi-output eps: one key, several reads (rate/count/total), one round trip.
# Returns ($list_returning_code, $arity) for compile()'s 'into' check.
sub _build_eps_multi {
my ( $spec, $where ) = @_;
my $parts = $spec->{parts};
croak "eps munger$where: 'parts' must be a non-empty arrayref"
unless ref $parts eq 'ARRAY' && @$parts;
for my $p (@$parts) {
croak "eps munger$where: unknown part '"
. ( defined $p ? $p : 'undef' )
. "' (known: "
. join( ', ', sort keys %EPS_READ ) . ')'
unless defined $p && $EPS_READ{$p};
}
my ( $socket, $prefix, $mark, $timeout, $on_error ) = _eps_spec( $spec, $where );
# Command plan, fixed at build time. When marking, the mark is a MARKRATE
# whose own reply serves the first 'rate' part for free; the remaining
# parts become one read command each. @take maps each part to the reply
# index that answers it, so the output stays in 'parts' order.
my ( @cmds, @take );
my $rate_served = 0;
push @cmds, 'MARKRATE' if $mark;
for my $i ( 0 .. $#$parts ) {
if ( $mark && !$rate_served && $parts->[$i] eq 'rate' ) {
$take[$i] = 0; # MARKRATE's reply is the rate
$rate_served = 1;
next;
}
push @cmds, uc $parts->[$i];
$take[$i] = $#cmds;
}
my $n = @$parts;
my $nreplies = @cmds;
my $code = sub {
my ($v) = @_;
my $key = $prefix . ( defined $v ? "$v" : '' );
$key =~ s/[\s[:cntrl:]]/_/g;
my @replies = eval {
die "empty key\n" unless length $key;
_eps_txn( $socket, $timeout, join( '', map { "$_ $key\n" } @cmds ), $nreplies );
};
if ($@) {
my $err = $@;
delete $EPS_CONN{$socket};
croak "eps munger$where: $err" if $on_error eq 'die';
return ( $on_error + 0 ) x $n;
}
return @replies[@take];
}; ## end $code = sub
return ( $code, $n );
} ## end sub _build_eps_multi
# A compiled munging plan for one set, produced by Mungers->compile. It turns an
# input record into a fully-numeric row in tags order; the Writer then only has
# to validate and append. Kept in its own package so the assembly logic is
# testable without a Writer or the filesystem.
package Algorithm::ToNumberMunger::Plan;
use strict;
use warnings;
use Carp qw(croak);
sub tags { return $_[0]->{tags} }
# Assemble a row from a name-keyed record. Scalar/raw columns read their own tag
# (or the munger's 'from'); expanding mungers read one source and fill several
# columns; combining mungers read several sources and fill one. This is the only
# form that supports expanders and combiners.
sub apply_named {
my ( $self, $hash ) = @_;
croak 'apply_named requires a hashref' unless ref $hash eq 'HASH';
my @row;
for my $s ( @{ $self->{scalar} } ) {
croak "missing value for '$s->{from}'"
unless exists $hash->{ $s->{from} };
my $v = $hash->{ $s->{from} };
$row[ $self->{pos}{ $s->{tag} } ] = $s->{code} ? $s->{code}->($v) : $v;
}
for my $e ( @{ $self->{expand} } ) {
croak "missing value for '$e->{from}'"
unless exists $hash->{ $e->{from} };
my @vals = $e->{code}->( $hash->{ $e->{from} } );
croak "expanding munger for [@{ $e->{into} }] returned "
. scalar(@vals)
. ' value(s), expected '
. scalar( @{ $e->{into} } )
unless @vals == @{ $e->{into} };
for my $i ( 0 .. $#{ $e->{into} } ) {
$row[ $self->{pos}{ $e->{into}[$i] } ] = $vals[$i];
}
} ## end for my $e ( @{ $self->{expand} } )
for my $c ( @{ $self->{combine} } ) {
my @vals;
for my $f ( @{ $c->{from} } ) {
croak "missing value for '$f'"
unless exists $hash->{$f};
push @vals, $hash->{$f};
}
$row[ $self->{pos}{ $c->{tag} } ] = $c->{code}->(@vals);
}
return \@row;
} ## end sub apply_named
# Assemble a row from an already-ordered positional row, applying scalar mungers
# in place. Expanding and combining mungers cannot be expressed positionally
# (there is no named source), so a set that has any is a hard error here -- use
# apply_named.
sub apply_positional {
my ( $self, $row ) = @_;
croak 'apply_positional requires an arrayref row' unless ref $row eq 'ARRAY';
croak 'positional write is unsupported for a set with expanding mungers; ' . 'use write_named'
if @{ $self->{expand} };
croak 'positional write is unsupported for a set with multi-input mungers; ' . 'use write_named'
if @{ $self->{combine} };
croak 'row has ' . scalar(@$row) . ' fields but info.json declares ' . scalar( @{ $self->{tags} } )
unless @$row == @{ $self->{tags} };
my @out = @$row;
for my $s ( @{ $self->{scalar} } ) {
next unless $s->{code};
my $i = $self->{pos}{ $s->{tag} };
$out[$i] = $s->{code}->( $out[$i] );
}
return \@out;
} ## end sub apply_positional
=head1 AUTHOR
Zane C. Bowers-Hadley, C<< <vvelox at vvelox.net> >>
=head1 LICENSE AND COPYRIGHT
This software is Copyright (c) 2026 by Zane C. Bowers-Hadley.
This is free software, licensed under:
The GNU Lesser General Public License, Version 2.1, February 1999
=cut
1; # End of Algorithm::ToNumberMunger
( run in 0.757 second using v1.01-cache-2.11-cpan-0b5f733616e )