view release on metacpan or  search on metacpan

lib/BATsh/MB.pm  view on Meta::CPAN

# Character-oriented helpers, operating on GUARDED strings.
# While the guard is inactive they fall back to byte semantics, so
# existing ASCII behaviour is preserved bit-for-bit.
#
#   mb_length($guarded)             -> number of characters
#   mb_substr($guarded, $off, $len) -> guarded substring by characters
#       $off may be negative (counts back from the end, like substr);
#       $len may be omitted (rest of string).
# ----------------------------------------------------------------
sub mb_length {
    my ($s) = @_;
    return 0 unless defined $s;
    return length($s) unless $ACTIVE;
    my $chars = _mb_chars(dec($s));
    return scalar @{$chars};
}

sub mb_substr {
    my ($s, $off, $len) = @_;
    return '' unless defined $s;
    unless ($ACTIVE) {
        return defined $len ? substr($s, $off, $len) : substr($s, $off);
    }
    my $chars = _mb_chars(dec($s));
    my $n = scalar @{$chars};
    $off = $n + $off if $off < 0;
    $off = 0 if $off < 0;
    return '' if $off >= $n;
    my $end = defined $len ? $off + $len : $n;
    $end = $n if $end > $n;
    return '' if $end <= $off;
    return enc(join('', @{$chars}[$off .. $end - 1]));
}

# Split RAW (decoded) DBCS bytes into characters, forward-scanning:
# a lead byte followed by a valid trail byte forms one two-byte
# character; anything else is a single byte.
sub _mb_chars {
    my ($s) = @_;
    my @c = ();
    my $i = 0;
    my $n = length($s);
    while ($i < $n) {
        my $b = substr($s, $i, 1);
        if ($b =~ /[$LEAD]/ && $i + 1 < $n
            && substr($s, $i + 1, 1) =~ /[$TRAIL]/) {
            push @c, substr($s, $i, 2);
            $i += 2;
        }
        else {
            push @c, $b;
            $i++;
        }
    }
    return [ @c ];
}

# ----------------------------------------------------------------
# strip_bom: remove a UTF-8 byte-order mark from the head of the
# first script line (harmless everywhere, required for editors that
# save UTF-8-with-BOM).
# ----------------------------------------------------------------
sub strip_bom {
    my $s = defined $_[0] && $_[0] !~ /\ABATsh::MB\z/ ? $_[0] : $_[1];
    return $s unless defined $s;
    $s =~ s/\A\xef\xbb\xbf//;
    return $s;
}

1;

__END__

=head1 NAME

BATsh::MB - Multibyte (CP932/DBCS) script guard for BATsh

=head1 VERSION

Version 0.11

=head1 SYNOPSIS

  use BATsh;

  # Explicit encoding
  BATsh->run('nihongo.batsh', encoding => 'cp932');

  # Or rely on auto-detection (the default):
  # a non-UTF-8 script containing bytes >= 0x80 is treated as CP932
  BATsh->run('nihongo.batsh');

  # Environment-variable override
  #   set BATSH_ENCODING=cp932

=head1 DESCRIPTION

BATsh::MB makes BATsh safe for scripts written in CP932 (Shift_JIS as
used on Japanese Windows) and other double-byte character sets whose
trail bytes overlap the ASCII range 0x40-0x7E.

Without protection, byte-oriented shell parsing tears such characters
apart: the trail byte 0x5C is mistaken for a backslash escape or path
separator, 0x7C for a pipeline separator, 0x60 for a backtick, 0x5E
for the cmd.exe caret escape, and uc()/lc() corrupt trail bytes in the
a-z range.  This is the well-known "dame-moji" problem affecting very
common characters such as SO (0x835C), HYOU (0x955C), NOH (0x945C),
PO (0x837C), CHI (0x8360), and DA (0x835E).

Rather than teaching every scanner about lead/trail bytes, BATsh::MB
applies a reversible guard transform on input: each two-byte character
whose trail byte falls in the dangerous ASCII range is rewritten to a
three-byte form C<\x01 LEAD (TRAIL+0x80)> containing no ASCII bytes.
The inverse transform is applied at output boundaries (print, external
commands, filesystem calls, %ENV export).  Between the two, all of
BATsh's byte-oriented parsing is automatically DBCS-safe.

=head1 FUNCTIONS

=over 4