File-BOM
view release on metacpan or search on metacpan
#!/usr/bin/perl
=head1 NAME
Build.PL - Build script generator for File::BOM
=head1 SYNOPSIS
perl Build.PL
./Build test
./Build install
or
perl Makefile.PL
=cut
use lib 'lib';
use strict;
use warnings;
use Module::Build;
my $build = Module::Build->new (
module_name => 'File::BOM',
dist_version_from => 'lib/File/BOM.pm',
create_readme => 1,
create_makefile_pl => 'passthrough',
license => 'perl',
requires => {
'perl' => '5.8.3',
'Readonly' => '0.06',
'Encode' => '1.99', # This shipped with perl 5.8.3
},
build_requires => {
'Test::More' => '0.10',
'Test::Exception' => '0.20',
'Module::Build' => '0.20',
},
meta_merge => {
meta_spec => {
version => 2,
},
resources => {
repository => 'https://github.com/mattlaw/File-BOM.git',
web => 'https://github.com/mattlaw/File-BOM',
type => 'git',
},
},
);
$build->create_build_script;
__END__
=head1 SEE ALSO
File::BOM changes document
0.18 - Fri May 2020 1
- Update manifest to include testrules.yml
0.17 - Fri May 1 2020
- Fix tests to be runnable in parallel. Thanks to Tom Hukins.
- Fix documentation typos. Thanks again to Tom Hukins.
0.16 - Wed Feb 6 2019
- Fix tests failing under Encode 2.99. Thanks to Petr Pisar.
0.12 - Tue Jul 11 2006
- Added a description section to docs
- Added information about test failures on cygwin
- Added overridable test skipping for tests that cause hangs on cygwin
0.11
- Added Test::Pod and Test::Pod::Coverage tests at the behest of CPANTS
0.10
- Changed open_bom to work more like open()
- Added defuse function to process BOMs on open handles
- Updated tests to cope with PerlIO::via's limitations
- Removed most test files, they will now be generated by the test framework,
so that they have native line endings
0.09
- Added tell() support to work with seek() properly
- Hopefully fixed compatibility with older versions of Encode
- PerlIO::via should now work with the latest perl
0.08
Build.PL
Changes
lib/File/BOM.pm
Makefile.PL
MANIFEST
META.json
META.yml Module meta-data (added by MakeMaker)
README
README-cygwin
t/00..setup.t
t/01..bom.t
t/02..perlio-via.t
t/03..exceptions.t
],
"dynamic_config" : 1,
"generated_by" : "Module::Build version 0.4229",
"license" : [
"perl_5"
],
"meta-spec" : {
"url" : "http://search.cpan.org/perldoc?CPAN::Meta::Spec",
"version" : 2
},
"name" : "File-BOM",
"prereqs" : {
"build" : {
"requires" : {
"Module::Build" : "0.20",
"Test::Exception" : "0.20",
"Test::More" : "0.10"
}
},
"configure" : {
"requires" : {
},
"runtime" : {
"requires" : {
"Encode" : "1.99",
"Readonly" : "0.06",
"perl" : "v5.8.3"
}
}
},
"provides" : {
"File::BOM" : {
"file" : "lib/File/BOM.pm",
"version" : "0.18"
}
},
"release_status" : "stable",
"resources" : {
"license" : [
"http://dev.perl.org/licenses/"
],
"repository" : {
"url" : "https://github.com/mattlaw/File-BOM.git"
},
"x_type" : "git",
"x_web" : "https://github.com/mattlaw/File-BOM"
},
"version" : "0.18",
"x_meta_spec" : {
"version" : 2
},
"x_serialization_backend" : "JSON::PP version 2.97001"
}
Test::Exception: '0.20'
Test::More: '0.10'
configure_requires:
Module::Build: '0.42'
dynamic_config: 1
generated_by: 'Module::Build version 0.4229, CPAN::Meta::Converter version 2.150010'
license: perl
meta-spec:
url: http://module-build.sourceforge.net/META-spec-v1.4.html
version: '1.4'
name: File-BOM
provides:
File::BOM:
file: lib/File/BOM.pm
version: '0.18'
requires:
Encode: '1.99'
Readonly: '0.06'
perl: v5.8.3
resources:
Type: git
Web: https://github.com/mattlaw/File-BOM
license: http://dev.perl.org/licenses/
repository: https://github.com/mattlaw/File-BOM.git
version: '0.18'
x_meta_spec:
version: 2
x_serialization_backend: 'CPAN::Meta::YAML version 0.018'
NAME
File::BOM - Utilities for handling Byte Order Marks
SYNOPSIS
use File::BOM qw( :all )
high-level functions
# read a file with encoding from the BOM:
open_bom(FH, $file)
open_bom(FH, $file, ':utf8') # the same but with a default encoding
# get encoding too
$encoding = open_bom(FH, $file, ':utf8');
# open a potentially unseekable file:
($encoding, $spillage) = open_bom(FH, $file, ':utf8');
# change encoding of an open handle according to BOM
$encoding = defuse(*HANDLE);
($encoding, $spillage) = defuse(*HANDLE);
# Decode a string according to leading BOM:
$unicode = decode_from_bom($string_with_bom);
# Decode a string and get the encoding:
($unicode, $encoding) = decode_from_bom($string_with_bom)
PerlIO::via interface
# Read the Right Thing from a unicode file with BOM:
open(HANDLE, '<:via(File::BOM)', $filename)
# Writing little-endian UTF-16 file with BOM:
open(HANDLE, '>:encoding(UTF-16LE):via(File::BOM)', $filename)
lower-level functions
# read BOM encoding from a filehandle:
$encoding = get_encoding_from_filehandle(FH)
# Get encoding even if FH is unseekable:
($encoding, $spillage) = get_encoding_from_filehandle(FH);
# Get encoding from a known unseekable handle:
($encdoing, $spillage) = get_encoding_from_stream(FH);
# get encoding and BOM length from BOM at start of string:
($encoding, $offset) = get_encoding_from_bom($string);
variables
# print a BOM for a known encoding
print FH $enc2bom{$encoding};
# get an encoding from a known BOM
$enc = $bom2enc{$bom}
DESCRIPTION
This module provides functions for handling unicode byte order marks,
which are to be found at the beginning of some files and streams.
For details about what a byte order mark is, see
<http://www.unicode.org/unicode/faq/utf_bom.html#BOM>
The intention of File::BOM is for files with BOMs to be readable as
seamlessly as possible, regardless of the encoding used. To that end,
several different interfaces are available, as shown in the synopsis
above.
EXPORTS
Nothing by default.
symbols
* open_bom()
subroutines only
* :vars
just %bom2enc and %enc2bom
VARIABLES
%bom2enc
Maps Byte Order marks to their encodings.
The keys of this hash are strings which represent the BOMs, the values
are their encodings, in a format which is understood by Encode
The encodings represented in this hash are: UTF-8, UTF-16BE, UTF-16LE,
UTF-32BE and UTF-32LE
%enc2bom
A reverse-lookup hash for bom2enc, with a few aliases used in Encode,
namely utf8, iso-10646-1 and UCS-2.
Note that UTF-16, UTF-32 and UCS-4 are not included in this hash. Mainly
because Encode::encode automatically puts BOMs on output. See
Encode::Unicode
FUNCTIONS
open_bom
$encoding = open_bom(HANDLE, $filename, $default_mode)
($encoding, $spill) = open_bom(HANDLE, $filename, $default_mode)
opens HANDLE for reading on $filename, setting the mode to the
appropriate encoding for the BOM stored in the file.
On failure, a fatal error is raised, see the DIAGNOSTICS section for
details on how to catch these. This is in order to allow the return
value(s) to be used for other purposes.
If the file doesn't contain a BOM, $default_mode is used instead. Hence:
open_bom(FH, 'my_file.txt', ':utf8')
Opens my_file.txt for reading in an appropriate encoding found from the
BOM in that file, or as a UTF-8 file if none is found.
In the absence of a $default_mode argument, the following 2 calls should
be equivalent:
open_bom(FH, 'no_bom.txt');
open(FH, '<', 'no_bom.txt');
If an undefined value is passed as the handle, a symbol will be
generated for it like open() does:
# create filehandle on the fly
$enc = open_bom(my $fh, $filename, ':utf8');
$line = <$fh>;
The filehandle will be cued up to read after the BOM. Unseekable files
(e.g. fifos) will cause croaking, unless called in list context to catch
spillage from the handle. Any spillage will be automatically decoded
from the encoding, if found.
e.g.
# croak if my_socket is unseekable
open_bom(FH, 'my_socket');
# keep spillage if my_socket is unseekable
# discard any spillage from open_bom
($encoding) = open_bom(FH, 'my_socket');
defuse
$enc = defuse(FH);
($enc, $spill) = defuse(FH);
FH should be a filehandle opened for reading, it will have the relevant
encoding layer pushed onto it be binmode if a BOM is found. Spillage
should be Unicode, not bytes.
Any uncaptured spillage will be silently lost. If the handle is
unseekable, use list context to avoid data loss.
If no BOM is found, the mode will be unaffected.
decode_from_bom
$unicode_string = decode_from_bom($string, $default, $check)
($unicode_string, $encoding) = decode_from_bom($string, $default, $check)
Reads a BOM from the beginning of $string, decodes $string (minus the
BOM) and returns it to you as a perl unicode string.
if $string doesn't have a BOM, $default is used instead.
$check, if supplied, is passed to Encode::decode as the third argument.
If there's no BOM and no default, the original string is returned and
encoding is ''.
See Encode
get_encoding_from_filehandle
$encoding = get_encoding_from_filehandle(HANDLE)
($encoding, $spillage) = get_encoding_from_filehandle(HANDLE)
Returns the encoding found in the given filehandle.
The handle should be opened in a non-unicode way (e.g. mode '<:bytes')
so that the BOM can be read in its natural state.
After calling, the handle will be set to read at a point after the BOM
(or at the beginning of the file if no BOM was found)
If called in scalar context, unseekable handles cause a croak().
If called in list context, unseekable handles will be read byte-by-byte
and any spillage will be returned. See get_encoding_from_stream()
get_encoding_from_stream
($encoding, $spillage) = get_encoding_from_stream(*FH);
Read a BOM from an unrewindable source. This means reading the stream
one byte at a time until either a BOM is found or every possible BOM is
ruled out. Any non-BOM bytes read from the handle will be returned in
$spillage.
If a BOM is found and the spillage contains a partial character (judging
by the expected character width for the encoding) more bytes will be
read from the handle to ensure that a complete character is returned.
Spillage is always in bytes, not characters.
This function is less efficient than get_encoding_from_filehandle, but
should work just as well on a seekable handle as on an unseekable one.
get_encoding_from_bom
($encoding, $offset) = get_encoding_from_bom($string)
Returns the encoding and length in bytes of the BOM in $string.
If there is no BOM, an empty string is returned and $offset is zero.
To get the data from the string, the following should work:
use Encode;
my($encoding, $offset) = get_encoding_from_bom($string);
if ($encoding) {
$string = decode($encoding, substr($string, $offset))
}
PerlIO::via interface
File::BOM can be used as a PerlIO::via interface.
open(HANDLE, '<:via(File::BOM)', 'my_file.txt');
open(HANDLE, '>:encoding(UTF-16LE):via(File::BOM)', 'out_file.txt');
print "foo\n"; # BOM is written to file here
This method is less prone to errors on non-seekable files as spillage is
incorporated into an internal buffer, but it doesn't give you any
information about the encoding being used, or indeed whether or not a
BOM was present.
There are a few known problems with this interface, especially
surrounding seek() and tell(), please see the BUGS section for more
details about this.
Reading
The via(File::BOM) layer must be added before the handle is read from,
otherwise any BOM will be missed. If there is no BOM, no decoding will
be done.
Because of a limitation in PerlIO::via, read() always works on bytes,
not characters. BOM decoding will still be done but output will be bytes
of UTF-8.
open(BOM, '<:via(File::BOM)', $file);
$bytes_read = read(BOM, $buffer, $length);
$unicode = decode('UTF-8', $buffer, Encode::FB_QUIET);
# Now $unicode is valid unicode and $buffer contains any left-over bytes
Writing
Add the via(File::BOM) layer on top of a unicode encoding layer to print
a BOM at the start of the output file. This needs to be done before any
data is written. The BOM is written as part of the first print command
on the handle, so if you don't print anything to the handle, you won't
get a BOM.
There is a "Wide character in print" warning generated when the
via(File::BOM) layer doesn't receive utf8 on writing. This glitch was
resolved in perl version 5.8.7, but if your perl version is older than
that, you'll need to make sure that the via(File::BOM) layer receives
utf8 like this:
# This works OK
open(FH, '>:encoding(UTF-16LE):via(File::BOM):utf8', $filename)
# This generates warnings with older perls
open(FH, '>:encoding(UTF-16LE):via(File::BOM)', $filename)
Seeking
Seeking with SEEK_SET results in an offset equal to the length of any
detected BOM being applied to the position parameter. Thus:
# Seek to end of BOM (not start of file!)
seek(FILE_BOM_HANDLE, 0, SEEK_SET)
Telling
In order to work correctly with seek(), tell() also returns a postion
adjusted by the length of the BOM.
SEE ALSO
* Encode
* Encode::Unicode
* <http://www.unicode.org/unicode/faq/utf_bom.html#BOM>
DIAGNOSTICS
The following exceptions are raised via croak()
* Couldn't read '<filename>': $!
open_bom() couldn't open the given file for reading
* Couldn't set binmode of handle opened on '<filename>' to '<mode>':
$!
get_encoding_from_filehandle() or open_bom() called on an unseekable
file or handle in scalar context.
* Couldn't read from handle: $!
_get_encoding_seekable() couldn't read the handle. This function is
called from get_encoding_from_filehandle(), defuse() and open_bom()
* Couldn't reset read position: $!
_get_encoding_seekable couldn't seek to the position after the BOM.
* Couldn't read byte: $!
get_encoding_from_stream couldn't read from the handle. This
function is called from get_encoding_from_filehandle() and
open_bom() when the handle or file is unseekable.
BUGS
Older versions of PerlIO::via have a few problems with writing, see
above.
README-cygwin view on Meta::CPAN
e.g.
TEST_FIFO=1 ./Build test
I have been informed that the the cause of this problem has been remedied in
Cygwin dll 1.5.20
It is also worth noting that some exception tests are known to fail under
cygwin. This is because the read() call doesn't return undef on a failure. This
is a problem with perl under cygwin, not with File::BOM.
Matt Lawrence <mattlaw@cpan.org>
TODO list for File::BOM
Iron out bugs in the PerlIO::via interface.
o Investigate bugs in PerlIO::via - patch?
- patched XS code and tested locally.
o get patch submitted
- done! perl 5.8.7 doesn't have this bug
o There are still problems with PerlIO::via. seek and tell don't work terribly
well. This really needs to get raised via perlbug, I think. Other
lib/File/BOM.pm view on Meta::CPAN
package File::BOM;
=head1 NAME
File::BOM - Utilities for handling Byte Order Marks
=head1 SYNOPSIS
use File::BOM qw( :all )
=head2 high-level functions
# read a file with encoding from the BOM:
open_bom(FH, $file)
open_bom(FH, $file, ':utf8') # the same but with a default encoding
# get encoding too
$encoding = open_bom(FH, $file, ':utf8');
# open a potentially unseekable file:
($encoding, $spillage) = open_bom(FH, $file, ':utf8');
# change encoding of an open handle according to BOM
$encoding = defuse(*HANDLE);
($encoding, $spillage) = defuse(*HANDLE);
# Decode a string according to leading BOM:
$unicode = decode_from_bom($string_with_bom);
# Decode a string and get the encoding:
($unicode, $encoding) = decode_from_bom($string_with_bom)
=head2 PerlIO::via interface
# Read the Right Thing from a unicode file with BOM:
open(HANDLE, '<:via(File::BOM)', $filename)
# Writing little-endian UTF-16 file with BOM:
open(HANDLE, '>:encoding(UTF-16LE):via(File::BOM)', $filename)
=head2 lower-level functions
# read BOM encoding from a filehandle:
$encoding = get_encoding_from_filehandle(FH)
# Get encoding even if FH is unseekable:
($encoding, $spillage) = get_encoding_from_filehandle(FH);
# Get encoding from a known unseekable handle:
($encdoing, $spillage) = get_encoding_from_stream(FH);
# get encoding and BOM length from BOM at start of string:
($encoding, $offset) = get_encoding_from_bom($string);
=head2 variables
# print a BOM for a known encoding
print FH $enc2bom{$encoding};
# get an encoding from a known BOM
$enc = $bom2enc{$bom}
=head1 DESCRIPTION
This module provides functions for handling unicode byte order marks, which are
to be found at the beginning of some files and streams.
For details about what a byte order mark is, see
L<http://www.unicode.org/unicode/faq/utf_bom.html#BOM>
The intention of File::BOM is for files with BOMs to be readable as seamlessly
as possible, regardless of the encoding used. To that end, several different
interfaces are available, as shown in the synopsis above.
=cut
use strict;
use warnings;
# We don't want any character semantics at all
use bytes;
lib/File/BOM.pm view on Meta::CPAN
=back
=cut
=head1 VARIABLES
=head2 %bom2enc
Maps Byte Order marks to their encodings.
The keys of this hash are strings which represent the BOMs, the values are their
encodings, in a format which is understood by L<Encode>
The encodings represented in this hash are: UTF-8, UTF-16BE, UTF-16LE,
UTF-32BE and UTF-32LE
=head2 %enc2bom
A reverse-lookup hash for bom2enc, with a few aliases used in L<Encode>, namely utf8, iso-10646-1 and UCS-2.
Note that UTF-16, UTF-32 and UCS-4 are not included in this hash. Mainly
because Encode::encode automatically puts BOMs on output. See L<Encode::Unicode>
=cut
our(%bom2enc, %enc2bom, $MAX_BOM_LENGTH, $bom_re);
# length in bytes of the longest BOM
$MAX_BOM_LENGTH = 4;
Readonly %bom2enc => (
map { encode($_, "\x{feff}") => $_ } qw(
UTF-8
UTF-16BE
UTF-16LE
UTF-32BE
UTF-32LE
)
);
lib/File/BOM.pm view on Meta::CPAN
iso-10646-1
utf8
)
);
{
local $" = '|';
my @bombs = sort { length $b <=> length $a } keys %bom2enc;
Readonly $MAX_BOM_LENGTH => length $bombs[0];
Readonly $bom_re => qr/^(@bombs)/o;
}
=head1 FUNCTIONS
=head2 open_bom
$encoding = open_bom(HANDLE, $filename, $default_mode)
($encoding, $spill) = open_bom(HANDLE, $filename, $default_mode)
opens HANDLE for reading on $filename, setting the mode to the appropriate
encoding for the BOM stored in the file.
On failure, a fatal error is raised, see the DIAGNOSTICS section for details on
how to catch these. This is in order to allow the return value(s) to be used for
other purposes.
If the file doesn't contain a BOM, $default_mode is used instead. Hence:
open_bom(FH, 'my_file.txt', ':utf8')
Opens my_file.txt for reading in an appropriate encoding found from the BOM in
that file, or as a UTF-8 file if none is found.
In the absence of a $default_mode argument, the following 2 calls should be equivalent:
open_bom(FH, 'no_bom.txt');
open(FH, '<', 'no_bom.txt');
If an undefined value is passed as the handle, a symbol will be generated for it
like open() does:
# create filehandle on the fly
$enc = open_bom(my $fh, $filename, ':utf8');
$line = <$fh>;
The filehandle will be cued up to read after the BOM. Unseekable files (e.g.
fifos) will cause croaking, unless called in list context to catch spillage
from the handle. Any spillage will be automatically decoded from the encoding,
if found.
e.g.
# croak if my_socket is unseekable
open_bom(FH, 'my_socket');
# keep spillage if my_socket is unseekable
lib/File/BOM.pm view on Meta::CPAN
return wantarray ? ($enc, $spill) : $enc;
}
=head2 defuse
$enc = defuse(FH);
($enc, $spill) = defuse(FH);
FH should be a filehandle opened for reading, it will have the relevant encoding
layer pushed onto it be binmode if a BOM is found. Spillage should be Unicode,
not bytes.
Any uncaptured spillage will be silently lost. If the handle is unseekable, use
list context to avoid data loss.
If no BOM is found, the mode will be unaffected.
=cut
sub defuse (*) {
my $fh = qualify_to_ref(shift, caller);
my($enc, $spill) = get_encoding_from_filehandle($fh);
if ($enc) {
binmode($fh, ":encoding($enc)");
lib/File/BOM.pm view on Meta::CPAN
return wantarray ? ($enc, $spill) : $enc;
}
=head2 decode_from_bom
$unicode_string = decode_from_bom($string, $default, $check)
($unicode_string, $encoding) = decode_from_bom($string, $default, $check)
Reads a BOM from the beginning of $string, decodes $string (minus the BOM) and
returns it to you as a perl unicode string.
if $string doesn't have a BOM, $default is used instead.
$check, if supplied, is passed to Encode::decode as the third argument.
If there's no BOM and no default, the original string is returned and encoding
is ''.
See L<Encode>
=cut
sub decode_from_bom ($;$$) {
my($string, $default, $check) = @_;
croak "No string" unless defined $string;
lib/File/BOM.pm view on Meta::CPAN
=head2 get_encoding_from_filehandle
$encoding = get_encoding_from_filehandle(HANDLE)
($encoding, $spillage) = get_encoding_from_filehandle(HANDLE)
Returns the encoding found in the given filehandle.
The handle should be opened in a non-unicode way (e.g. mode '<:bytes') so that
the BOM can be read in its natural state.
After calling, the handle will be set to read at a point after the BOM (or at
the beginning of the file if no BOM was found)
If called in scalar context, unseekable handles cause a croak().
If called in list context, unseekable handles will be read byte-by-byte and any
spillage will be returned. See get_encoding_from_stream()
=cut
sub get_encoding_from_filehandle (*) {
my $fh = qualify_to_ref(shift, caller);
lib/File/BOM.pm view on Meta::CPAN
croak "Unseekable handle: $!";
}
return wantarray ? ($enc, $spill) : $enc;
}
=head2 get_encoding_from_stream
($encoding, $spillage) = get_encoding_from_stream(*FH);
Read a BOM from an unrewindable source. This means reading the stream one byte
at a time until either a BOM is found or every possible BOM is ruled out. Any
non-BOM bytes read from the handle will be returned in $spillage.
If a BOM is found and the spillage contains a partial character (judging by the
expected character width for the encoding) more bytes will be read from the
handle to ensure that a complete character is returned.
Spillage is always in bytes, not characters.
This function is less efficient than get_encoding_from_filehandle, but should
work just as well on a seekable handle as on an unseekable one.
=cut
sub get_encoding_from_stream (*) {
my $fh = qualify_to_ref(shift, caller);
_get_encoding_unseekable($fh);
}
# internal:
#
# Return encoding and seek to position after BOM
sub _get_encoding_seekable (*) {
my $fh = shift;
# This doesn't work on all platforms:
# defined(read($fh, my $bom, $MAX_BOM_LENGTH))
# or croak "Couldn't read from handle: $!";
my $bom = eval { _safe_read($fh, $MAX_BOM_LENGTH) };
croak "Couldn't read from handle: $@" if $@;
my($enc, $off) = get_encoding_from_bom($bom);
seek($fh, $off, SEEK_SET) or croak "Couldn't reset read position: $!";
return $enc;
}
# internal:
#
# Return encoding and non-BOM overspill
sub _get_encoding_unseekable (*) {
my $fh = shift;
my $so_far = '';
for my $c (1 .. $MAX_BOM_LENGTH) {
# defined(read($fh, my $byte, 1)) or croak "Couldn't read byte: $!";
my $byte = eval { _safe_read($fh, 1) };
croak "Couldn't read byte: $@" if $@;
$so_far .= $byte;
# find matching BOMs
my @possible = grep { $so_far eq substr($_, 0, $c) } keys %bom2enc;
if (@possible == 1 and my $enc = $bom2enc{$so_far}) {
# There's only one match, this must be it
return ($enc, '');
}
elsif (@possible == 0) {
# might need to backtrack one byte
my $spill = chop $so_far;
lib/File/BOM.pm view on Meta::CPAN
my $extra = eval {
_safe_read($fh, $char_length - length $spill);
};
croak "Coudln't read byte: $@" if $@;
$spill .= $extra;
return ($enc, $spill);
}
else {
# no BOM
return ('', $so_far . $spill);
}
}
}
}
sub _safe_read {
my ($fh, $count) = @_;
# read is supposed to return undef on error, but on some platforms it
lib/File/BOM.pm view on Meta::CPAN
die $! if !$status && $!;
return $out;
}
=head2 get_encoding_from_bom
($encoding, $offset) = get_encoding_from_bom($string)
Returns the encoding and length in bytes of the BOM in $string.
If there is no BOM, an empty string is returned and $offset is zero.
To get the data from the string, the following should work:
use Encode;
my($encoding, $offset) = get_encoding_from_bom($string);
if ($encoding) {
$string = decode($encoding, substr($string, $offset))
}
lib/File/BOM.pm view on Meta::CPAN
elsif ($enc =~ /^UTF-(16|32)/) {
return $1 / 8;
}
else {
return;
}
}
=head1 PerlIO::via interface
File::BOM can be used as a PerlIO::via interface.
open(HANDLE, '<:via(File::BOM)', 'my_file.txt');
open(HANDLE, '>:encoding(UTF-16LE):via(File::BOM)', 'out_file.txt');
print "foo\n"; # BOM is written to file here
This method is less prone to errors on non-seekable files as spillage is
incorporated into an internal buffer, but it doesn't give you any information
about the encoding being used, or indeed whether or not a BOM
was present.
There are a few known problems with this interface, especially surrounding
seek() and tell(), please see the BUGS section for more details about this.
=head2 Reading
The via(File::BOM) layer must be added before the handle is read from, otherwise
any BOM will be missed. If there is no BOM, no decoding will be done.
Because of a limitation in PerlIO::via, read() always works on bytes, not characters. BOM decoding will still be done but output will be bytes of UTF-8.
open(BOM, '<:via(File::BOM)', $file);
$bytes_read = read(BOM, $buffer, $length);
$unicode = decode('UTF-8', $buffer, Encode::FB_QUIET);
# Now $unicode is valid unicode and $buffer contains any left-over bytes
=head2 Writing
Add the via(File::BOM) layer on top of a unicode encoding layer to print a BOM
at the start of the output file. This needs to be done before any data is
written. The BOM is written as part of the first print command on the handle, so
if you don't print anything to the handle, you won't get a BOM.
There is a "Wide character in print" warning generated when the via(File::BOM)
layer doesn't receive utf8 on writing. This glitch was resolved in perl version
5.8.7, but if your perl version is older than that, you'll need to make sure
that the via(File::BOM) layer receives utf8 like this:
# This works OK
open(FH, '>:encoding(UTF-16LE):via(File::BOM):utf8', $filename)
# This generates warnings with older perls
open(FH, '>:encoding(UTF-16LE):via(File::BOM)', $filename)
=head2 Seeking
Seeking with SEEK_SET results in an offset equal to the length of any detected
BOM being applied to the position parameter. Thus:
# Seek to end of BOM (not start of file!)
seek(FILE_BOM_HANDLE, 0, SEEK_SET)
=head2 Telling
In order to work correctly with seek(), tell() also returns a postion adjusted
by the length of the BOM.
=cut
sub PUSHED { bless({offset => 0}, $_[0]) || -1 }
sub UTF8 {
# There is a bug with this method previous to 5.8.7
if ($] >= 5.008007) {
return 1;
lib/File/BOM.pm view on Meta::CPAN
__END__
=head1 SEE ALSO
=over 4
=item * L<Encode>
=item * L<Encode::Unicode>
=item * L<http://www.unicode.org/unicode/faq/utf_bom.html#BOM>
=back
=head1 DIAGNOSTICS
The following exceptions are raised via croak()
=over 4
=item * Couldn't read '<filename>': $!
lib/File/BOM.pm view on Meta::CPAN
get_encoding_from_filehandle() or open_bom() called on an unseekable file or handle in scalar context.
=item * Couldn't read from handle: $!
_get_encoding_seekable() couldn't read the handle. This function is called from
get_encoding_from_filehandle(), defuse() and open_bom()
=item * Couldn't reset read position: $!
_get_encoding_seekable couldn't seek to the position after the BOM.
=item * Couldn't read byte: $!
get_encoding_from_stream couldn't read from the handle. This function is called
from get_encoding_from_filehandle() and open_bom() when the handle or file is
unseekable.
=back
=head1 BUGS
t/01..bom.t view on Meta::CPAN
use Encode qw( encode decode :fallback_all );
use Fcntl qw( :seek );
our @encodings;
BEGIN {
# encodings to use in unseekable test
@encodings = qw( UTF-8 UTF-16LE UTF-16BE UTF-32LE UTF-32BE );
plan tests => 11 + (@test_files * 14) + (@encodings * 4);
use_ok("File::BOM", ':all');
}
# Ignore known harmless warning
local $SIG{__WARN__} = sub {
my $warning = "@_";
if ($warning !~ /^UTF-(?:16|32)LE:Partial character/) {
warn $warning;
}
};
t/01..bom.t view on Meta::CPAN
my $line = <FH>;
chomp $line;
is($line, $expect, "$file: test content returned OK");
close FH;
{
# test defuse
open BOMB, '<', $file2path{$file}
or die "Couldn't read '$file2path{$file}': $!";
my $enc = defuse BOMB;
is($enc, $file_enc, "$file: defuse returns correct encoding ($enc)");
$line = <BOMB>;
chomp $line;
is($line, $expect, "$file: defused version content OK");
close BOMB;
}
open FH, '<', $file2path{$file};
my $first_line;
{
local $/ = $fileeol{$file};
$first_line = <FH>;
chomp $first_line;
}
t/01..bom.t view on Meta::CPAN
. encode($encoding, $test, FB_CROAK);
($pid, $fifo) = write_fifo($bytes);
($enc, $spill) = open_bom(my $fh, $fifo);
$result = $spill . <$fh>;
close $fh;
waitpid($pid, 0);
unlink $fifo;
is($enc, $encoding, "Read BOM correctly in unseekable $encoding file");
is($result, $expected, "Read $encoding data from unseekable source");
# Now test defuse too
($pid, $fifo) = write_fifo($bytes);
open($fh, '<:utf8', $fifo) or die "Couldn't read '$fifo': $!";
($enc, $spill) = defuse $fh;
$result = $spill . <$fh>;
close $fh;
waitpid($pid, 0);
t/01..bom.t view on Meta::CPAN
is($result, $expected, "read defused fifo OK ($encoding)")
or diag(
"Hex dump:\n".
"Got: ". hexdump($result) ."\n".
"Expected: ". hexdump($expected) ."\n".
"Spillage: ". hexdump($spill)
);
}
}
# Test broken BOM
{
my $broken_content = "\xff\xffThis file has a broken BOM";
my $broken_file = 't/data/broken_bom.txt';
my($enc, $spill) = open_bom(my $fh, $broken_file);
is($enc, '', "open_bom on file with broken BOM has no encoding");
{
my $line = <$fh>;
chomp $line;
is($line, $broken_content, "handle with broken BOM returns as expected");
}
SKIP: {
skip "mkfifo not supported on this platform", 3
unless $fifo_supported;
skip "mkfifo tests skipped on cygwin, set TEST_FIFO to enable them", 3
if $^O eq 'cygwin' && !$ENV{'TEST_FIFO'};
my($pid, $fifo) = write_fifo($broken_content);
t/01..bom.t view on Meta::CPAN
is($spill . <$fh>, $broken_content, "spillage + content as expected");
close $fh;
waitpid($pid, 0);
unlink $fifo;
}
}
# Test internals
is(File::BOM::_get_char_length('UTF-8', 0xe5), 3, '_get_char_length() on UTF-8 start byte (3)');
is(File::BOM::_get_char_length('UTF-8', 0xd5), 2, '_get_char_length() on UTF-8 start byte (2)');
is(File::BOM::_get_char_length('UTF-8', 0x7f), 1, '_get_char_langth() on UTF-8 single byte char');
is(File::BOM::_get_char_length('', ''), undef, '_get_char_length() on undef');
is(File::BOM::_get_char_length('UTF-32BE', ''), 4, '_get_char_length() on UTF-32');
__END__
vim: ft=perl
t/02..perlio-via.t view on Meta::CPAN
use strict;
use warnings;
use lib qw( t/lib );
use Test::More;
use Test::Framework;
use Fcntl qw( :seek );
use File::BOM qw( %enc2bom );
# Expected data for "moose" tests (below)
our %should_be = (
'UTF-8' => "\x{ef}\x{bb}\x{bf}m\x{c3}\x{b8}\x{c3}\x{b8}se\x{e2}\x{80}\x{a6}",
'UTF-16BE' => "\x{fe}\x{ff}\x{0}m\x{0}\x{f8}\x{0}\x{f8}\x{0}s\x{0}e &",
'UTF-16LE' => "\x{ff}\x{fe}m\x{0}\x{f8}\x{0}\x{f8}\x{0}s\x{0}e\x{0}& ",
'UTF-32BE' => "\x{0}\x{0}\x{fe}\x{ff}\x{0}\x{0}\x{0}m\x{0}\x{0}\x{0}\x{f8}\x{0}\x{0}\x{0}\x{f8}\x{0}\x{0}\x{0}s\x{0}\x{0}\x{0}e\x{0}\x{0} &",
'UTF-32LE' => "\x{ff}\x{fe}\x{0}\x{0}m\x{0}\x{0}\x{0}\x{f8}\x{0}\x{0}\x{0}\x{f8}\x{0}\x{0}\x{0}s\x{0}\x{0}\x{0}e\x{0}\x{0}\x{0}& \x{0}\x{0}",
);
t/02..perlio-via.t view on Meta::CPAN
# Ignore known harmless warning
local $SIG{__WARN__} = sub {
my $warning = "@_";
if ($warning !~ /^UTF-(?:16|32)LE:Partial character/) {
warn $warning;
}
};
for my $test_file (@test_files) {
ok(
open(FH, "<:via(File::BOM)$compat", $file2path{$test_file}),
"$test_file: opened through layer"
) or diag "$test_file: $!";
my $line = <FH>; chomp $line;
is($line, $filecontent{$test_file}, "$test_file: read OK through layer")
or diag("HEX: ".hexdump($line));
close FH;
}
for my $enc (sort keys %enc2bom) {
my $file = "test_file-$enc.txt";
ok(
open(BOM_OUT, ">:encoding($enc):via(File::BOM)$compat", $file),
"Opened file for writing $enc via layer"
) or diag "$file: $!";
my $line_one = "Unicode text\x{2026}";
my $test = print(BOM_OUT "$line_one\n");
ok($test, 'print() through layer')
or diag("print() returned ". (defined($test)?$test:'undef'));
my $line_two = "\x{62cd}\x{8ce3}";
$test = print(BOM_OUT "$line_two\n");
ok($test, 'print() through layer again')
or diag("print() returned ". (defined($test)?$test:'undef'));
close BOM_OUT;
# check BOM
if (open my $fh, '<:bytes', $file) {
read $fh, my $sample, $File::BOM::MAX_BOM_LENGTH;
like($sample, qr/^\Q$enc2bom{$enc}/, "BOM written correctly");
close $fh;
}
else {
diag "Couldn't open $file: $!";
fail(1);
}
# now re-read
my $line;
open(BOM_IN, "<:via(File::BOM)$compat", $file);
$line = <BOM_IN>; chomp $line;
is($line, $line_one, 'BOM was written successfully via layer');
$line = <BOM_IN>; chomp $line;
is($line, $line_two, 'BOM not written in second print call');
close BOM_IN;
unlink $file or diag "Couldn't remove $file: $!";
}
# Mark Fowler's "moose" test:
{
# This is 'moose...' (with slashes in the 'o's them, and the '...'
# as one char). As the '...' can't be represented in latin-1 then
# perl will store the thing internally as a utf8 string with the
# utf8 flag enabled.
my $moose = "m\x{f8}\x{f8}se\x{2026}";
for my $enc (keys %should_be) {
my $file = "moose-$enc.txt";
open(FH, ">:encoding($enc):via(File::BOM)$compat", $file) or die "Can't write to $file: $!\n";
print FH $moose;
close FH;
open(FH, '<', $file) or die "Can't read $file: $!\n";
local $/ = undef;
my $value = <FH>;
close FH;
is(
reasciify($value),
t/02..perlio-via.t view on Meta::CPAN
{
use utf8;
my $file = 't/data/utf8_data.csv';
open my $fh, '>:utf8', $file or die "Can't write $file: $!";
print $fh <<"END_DATA";
\x{feff}id,street,town,pc,country,english,french,chinese,arabic
'10,"écoles",zoom,12,france,auctions,"Enchères","æ‹è³£","مزاد"
END_DATA
open $fh, '<:via(File::BOM)', $file
or die "Can't read $file: $!\n";
my $first_line = <$fh>;
my $pos = tell($fh); # position of second line
my $rest = join('', <$fh>);
seek($fh, 0, SEEK_SET) or die "Couldn't seek: $!";
my $new_first_line = <$fh>;
seek($fh, $pos, SEEK_SET) or die "Couldn't seek: $!";
t/03..exceptions.t view on Meta::CPAN
#!/usr/bin/perl
use strict;
use warnings;
use lib 't/lib';
use Test::Framework;
use File::BOM qw(
open_bom
decode_from_bom
get_encoding_from_filehandle
);
use File::Temp qw( tmpnam );
use Test::Exception ( tests => 10 );
use Test::More;
t/03..exceptions.t view on Meta::CPAN
{
# The following tests are known to produce warnings
local $SIG{__WARN__} = sub {};
my $tmpfile = tmpnam();
open WRITER, '>', $tmpfile or die "Couldn't write to '$tmpfile': $!";
# _get_encoding_* functions don't qualify refs as they are not public
# Therefore _get_encoding_seekable(WRITER) will not work
throws_ok { File::BOM::_get_encoding_seekable(\*WRITER) }
qr/^Couldn't read from handle/,
"_get_encoding_seekable on unreadable handle fails";
throws_ok { File::BOM::_get_encoding_unseekable(\*WRITER) }
qr/^Couldn't read byte/,
"_get_encoding_unseekable() on unreadable handle fails";
close WRITER;
unlink $tmpfile;
SKIP:
{
skip "mkfifo not supported on this platform", 3
unless $fifo_supported;
skip "mkfifo tests skipped on cygwin, set TEST_FIFO to enable them", 3
if $^O eq 'cygwin' && !$ENV{'TEST_FIFO'};
my($pid, $fifo);
($pid, $fifo) = write_fifo('');
open(STREAM, '<:bytes', $fifo) or die "Couldn't read fifo '$fifo': $!";
throws_ok { File::BOM::_get_encoding_seekable(\*STREAM) }
qr/^Couldn't reset read position/,
"_get_encoding_seekable on unseekable handle fails";
throws_ok { get_encoding_from_filehandle(STREAM) }
qr/^Unseekable handle/,
"get_encoding_from_filehandle on unseekable handle fails";
close STREAM; waitpid($pid, 0); unlink $fifo;
($pid, $fifo) = write_fifo('');
t/04..pod.t view on Meta::CPAN
BEGIN {
our @modules = qw(
File::BOM
);
}
use File::Spec::Functions qw( catfile );
use Test::More tests => our @modules * 2;
SKIP: {
eval 'use Test::Pod';
t/data/broken_bom.txt view on Meta::CPAN
ÿÿThis file has a broken BOM
( run in 1.047 second using v1.01-cache-2.11-cpan-e9daa2b36ef )