escape results from the CPAN

escape

App-MtAws

view release on metacpan or search on metacpan

  if dir started with "..". This due to bug https://rt.perl.org/Public/Bug/Display.html?id=111510 in File::Spec module.
  Currently upload-file behaviour changed (see above in ChangeLog) so  mtglacier not affected. In previous versions this
  would result in wrong relative filenames in journal and Amazon glacier metadata
  (precisely, those filenames are without path prefix, as if they would be in current directory, otherwise filename
  part is correct).

  * Workaround: Digest::SHA perl module prior to version 5.62 calculates SHA256 wrong on 32bit machines, when data
  size is more than 2^29 bytes. Now mtglacier throws an error if --partsize >= 512Mb and machine is 32bit and digest-sha
  version is below 5.62. Commands which don't use --partsize are unaffected.

  * Fixed: Amazon CSV format parsing: Amazon escapes doublequote with backslash but.. does not escape backslash itself.
  https://forums.aws.amazon.com/thread.jspa?threadID=141807
  This format is undocumented and broken by design. Fixing parser now to parse this.
  this bug was not affecting any real use of mtglacier as mtglacier does not use backslashes in metadata and ignores
  foreign metadata.

  * Cosmetic changes to process manager code

### 2013-12-14 v1.103

  * Fixed: issue #48 download-inventory was crashing if there was a request for inventory retrieval in CSV format

MANIFEST view on Meta::CPAN

t/unit/queue_job/multipart_finish.t
t/unit/queue_job/multipart_part.t
t/unit/queue_job/retrieve.t
t/unit/queue_job/retrieve_inventory.t
t/unit/queue_job/upload.t
t/unit/queue_job/upload_multipart.t
t/unit/queue_job/verify.t
t/unit/queue_job/verify_and_upload.t
t/unit/queue_job_result.t
t/unit/shahash.t
t/unit/string_escape.t
t/unit/sysread_syswrite_unit.t
t/unit/test_test.t
t/unit/u_treehash.t
test.t

lib/App/MtAws/Glacier/Inventory/CSV.pm view on Meta::CPAN

	my $self = { rawdata => \$_[0] };
	bless $self, $class;
	$self;
}

sub _parse
{
	my ($self) = @_;

	# Text::CSV with below options does not seem to work for our case
	# ( { binary => 1 , allow_whitespace => 1, quote_char => '"', allow_loose_quotes => 1, escape_char => "\\", auto_diag=>1} )
	# because Amazon CSV is buggy https://forums.aws.amazon.com/thread.jspa?threadID=141807&tstart=0

	my $re = undef;
	my @fields;
	my @records;
	while (${$self->{rawdata}} =~ /^(.*?)\r?$/gsm) {
		my $line = $1;
		if(!defined $re) {
			@fields = split /,/, $line;
			for (@fields) {

lib/App/MtAws/GlacierRequest.pm view on Meta::CPAN

	my @all_headers = sort { $a->{name} cmp $b->{name} } (@{$self->{headers}}, @{$self->{req_headers}});


	my $canonical_headers = join ("\n", map { lc($_->{name}).":".trim($_->{value}) } @all_headers);
	my $signed_headers = join (';', map { lc($_->{name}) } @all_headers);

	my $bodyhash = $self->{data_sha256} ?
		$self->{data_sha256} :
		( $self->{dataref} ? large_sha256_hex(${$self->{dataref}}) : sha256_hex('') );

	$self->{params_s} = $self->{params} ? join ('&', map { "$_=".uri_escape($self->{params}->{$_}) } sort keys %{$self->{params}}) : "";
	my $canonical_query_string = $self->{params_s};

	my $canonical_url = join("\n", $self->{method}, $self->{url}, $canonical_query_string, $canonical_headers, "", $signed_headers, $bodyhash);
	my $canonical_url_hash = sha256_hex($canonical_url);


	# /getting canonical URL

	my $credentials = "$datestr/$self->{region}/$self->{service}/aws4_request";

lib/App/MtAws/LineProtocol.pm view on Meta::CPAN

use warnings;
use utf8;
use Carp;

use JSON::XS;
use App::MtAws::Utils;

use Exporter 'import';

our @EXPORT = qw/ get_data send_data/;
our @EXPORT_OK = qw/escape unescape encode_data decode_data/;

# yes, a module, so we can unit-test it (JSON and YAML have different serialization implementeation)
my $json_coder = JSON::XS->new->ascii(1)->allow_nonref;

sub decode_data
{
	my ($data_e) = @_;
	return $json_coder->decode($data_e);
}

lib/App/MtAws/MetaData.pm view on Meta::CPAN

our @EXPORT = qw/meta_decode meta_job_decode meta_encode meta_job_encode META_JOB_TYPE_FULL/;
=head1 MT-AWS-GLACIER metadata format ('x-amz-archive-description' field).

Function definitions:
=====================
base64url() input - byte sequence, output - byte sequence
	Is Base64 URL algorithm: http://en.wikipedia.org/wiki/Base64#URL_applications
	basically it's base64 but with '=' padding removed, characters '+', '/' replaced with '-', '_' resp. and no new lines.

json_utf8() - input - Hash, output - byte sequence
	JSON string in UTF-8 representation. Can contain not-escaped UTF-8 characters. Will not contain linefeed. Hash objects are unordered.

latin1_to_utf8() - input - byte sequence, output - byte sequence
	Treats input data as Latin1 (ISO 8859-1) encoded sequence and converts it to UTF-8 sequence

isoO8601() - input - time, output - character string
	ISOO8601 time in the following format YYYYMMDDTHHMMSSZ. Only UTC timezone. No leap seconds supported.
	Supported year range is from 1000 to 9999
	When encoding isoO8601() mt-aws-glacier will not store leap seconds. When decoding from isoO8601 leap seconds will be dropped.

{'filename': FILENAME, 'mtime': iso8601(MTIME)}

t/integration/metadata.t view on Meta::CPAN

		['Ð´Ð¸Ñ€ÐµÐºÑ‚Ð¾Ñ€Ð¸Ñ/Ñ„Ð°Ð¹Ð»',1352124178],
		['Ð´Ð¸Ñ€ÐµÐºÑ‚Ð¾Ñ€Ð¸Ñ/Ñ„Ð°Ð¹Ð»',0],
		['Ð´Ð¸Ñ€ÐµÐºÑ‚Ð¾Ñ€Ð¸Ñ/Ñ„Ð°Ð¹Ð»','0'],
	) {
		my $result = App::MtAws::MetaData::_encode_json(App::MtAws::MetaData::_encode_filename_and_mtime($_->[0], $_->[1]));
		my $recoded = JSON::XS->new->utf8->allow_nonref->decode($result);
		ok ($result !~ /[\r\n]/m, 'no linefeed');
##		ok( $result =~ /\:\s*$_->[1]/, "result should contain mtime as numeric");
		is_deeply($recoded, { mtime => to_iso8601($_->[1]), filename => $_->[0]}, "jsone string should be json with correct filename and mtime");
		my $result_decoded =decode("UTF-8", $result, Encode::DIE_ON_ERR|Encode::LEAVE_SRC);
		ok ($result_decoded =~ /\Q$_->[0]\E/m, "json string should contain UTF without escapes");

		my ($filename, $mtime) = App::MtAws::MetaData::_decode_filename_and_mtime(App::MtAws::MetaData::_decode_json($result));
		ok ($filename eq $_->[0], 'filename match');
		ok ($mtime == $_->[1], 'mtime match');
	}
}

# test meta_encode/meta_decode with fixtures
{
	for (

t/integration/utf8_line_protocol.t view on Meta::CPAN

		'somekey1' => { 'keyA' => 'data1', 'keyB' => 'data2', 'keyC' => '99999' },
		'somekey2' => { 'XkeyA' => '2734234', 'XkeyB' => "data2", 'XkeyC' => '76324' },
	},
	{
	'somekey1' => { 'keyA' => [ 'data1', 'data2', '837'], 'keyB' => 'data2', 'keyC' => {'99999'=>undef}, 'z' => {undef => undef} },
	'somekey2' => { 'XkeyA' => '2734234', 'XkeyB' => { '123' => "data1", "g1" => "x5", "c4" => "x4"}, 'XkeyC' => [undef] },
},
[ { a=> 123, b=> 456 }, { c => 789 }, { e => 909}]
];

my $escape_samples = [
"a\nb",
"ab\n",
"ab\n",
"\nab\n",
"\n\n",
"\n",
"\\",
"\\\\",
"\\n",
"\\\n",

t/integration/utf8_line_protocol.t view on Meta::CPAN

"\\\r",
"\\\r\\",
"\r\\",
"a\r\\c",
"\r\t",
"\\\\\\\\",
"\n\r\@\~",
"x" x 8192,
];

for my $src (@$samples, map { { x => $_} } @$escape_samples) {
	my $enc = encode_data($src);
	my $dst = decode_data($enc);
	cmp_deeply $dst, $src;
	ok $enc =~ qr/\A[^\n\r]+\z/s;
}


sub sending
{
	local $file;

t/unit/filter.t view on Meta::CPAN

}

for (' ', 'a/ ', '/a/ ', 'a/b/ ', '/a/b/ ', '*', '/*', '/a/*', 'a*', 'a/b/* *', 'a/b** *', 'a/b**c') {
	my $F = App::MtAws::Filter->new();
	my ($re) = $F->_patterns_to_regexp({pattern => $_});
	ok !$re->{match_subdirs}, "does not match subdirs [$_]";
}


#
# _patterns_to_regexp correctness of escapes
#


check 'z/ex.mple',
	ismatch => ['z/ex.mple'],
	nomatch => ['z/exNmple'];

check 'z/ex\\dmple',
	ismatch => ['z/ex\\dmple'],
	nomatch => ['z/ex1mple'];

( run in 1.071 second using v1.01-cache-2.11-cpan-c21f80fb71c )