view release on metacpan or  search on metacpan

lib/App/DubiousHTTP/Tests/Compressed.pm  view on Meta::CPAN

    [ 'INVALID: invalid content-encodings should not be ignored' ],
    [ INVALID, 'ce:gzip_x' => 'content-encoding "gzip x", but not encoded' ],
    [ INVALID, 'ce:deflate;ce:gzip_x;deflate' => 'content-encoding deflate + "gzip x", but only deflated' ],
    [ INVALID, 'ce:gzip_x;ce:deflate;deflate' => 'content-encoding  "gzip x" + deflate, but only deflated' ],
    [ INVALID, 'ce:foo', '"content-encoding:foo" and no encoding' ],
    [ INVALID, 'ce:rfc2047-deflate', '"content-encoding:rfc2047(deflate)" and no encoding' ],
    [ INVALID, 'ce:rfc2047-deflate;deflate', '"content-encoding:rfc2047(deflate)" with encoding' ],

    [ 'VALID: transfer-encoding should be ignored for compression' ],
    [ UNCOMMON_VALID,'te:gzip' => 'transfer-encoding gzip but not compressed'],

    [ 'INVALID: "Hiding the Content-encoding header"' ],
    [ INVALID, 'ce-space-colon-deflate;deflate' => '"Content-Encoding<space>: deflate", served with deflate' ],
    [ UNCOMMON_INVALID, 'ce-space-colon-deflate' => '"Content-Encoding<space>: deflate", served not with deflate' ],
    [ INVALID, 'ce-space-colon-gzip;gzip' => '"Content-Encoding<space>: gzip", served with gzip' ],
    [ UNCOMMON_INVALID, 'ce-space-colon-gzip' => '"Content-Encoding<space>: gzip", served not with gzip' ],

    [ INVALID, 'ce-colon-colon-deflate;deflate' => '"Content-Encoding:: deflate", served with deflate' ],
    [ UNCOMMON_INVALID, 'ce-colon-colon-deflate' => '"Content-Encoding:: deflate", served not with deflate' ],
    [ INVALID, 'ce-colon-colon-gzip;gzip' => '"Content-Encoding:: gzip", served with gzip' ],
    [ UNCOMMON_INVALID, 'ce-colon-colon-gzip' => '"Content-Encoding:: gzip", served not with gzip' ],

    [ INVALID, 'cronly-deflate;deflate' => 'Content-Encoding with only <CR> as line delimiter before, served deflate' ],
    [ INVALID, 'crxonly-deflate;deflate' => 'Only <CR> as line delimiter followed by "xContent-Encoding", served deflate' ],
    [ UNCOMMON_INVALID, 'cronly-deflate' => 'Content-Encoding with only <CR> as line delimiter before, not served deflate' ],
    [ INVALID, 'cronly-gzip;gzip' => 'Content-Encoding with only <CR> as line delimiter before, served gzip' ],
    [ INVALID, 'crxonly-gzip;gzip' => 'Only <CR> as line delimiter followed by "xContent-Encoding", served gzip' ],
    [ UNCOMMON_INVALID, 'cronly-gzip' => 'Content-Encoding with only <CR> as line delimiter before, not served gzip' ],

    [ UNCOMMON_INVALID, 'lfonly-deflate;deflate' => 'Content-Encoding with only <LF> as line delimiter before, served deflate' ],
    [ INVALID, 'lfonly-deflate' => 'Content-Encoding with only <LF> as line delimiter before, not served deflate' ],
    [ UNCOMMON_INVALID, 'lfonly-gzip;gzip' => 'Content-Encoding with only <LF> as line delimiter before, served gzip' ],
    [ INVALID, 'lfonly-gzip' => 'Content-Encoding with only <LF> as line delimiter before, not served gzip' ],

    [ INVALID, 'ce:crdeflate;deflate' => 'Content-Encoding:<CR>deflate, served with deflate' ],
    [ INVALID, 'ce:crdeflate' => 'Content-Encoding:<CR>deflate, not served with deflate' ],
    [ INVALID, 'ce:cr-deflate;deflate' => 'Content-Encoding:<CR><space>deflate, served with deflate' ],
    [ INVALID, 'ce:cr-deflate' => 'Content-Encoding:<CR><space>deflate, not served with deflate' ],
    [ INVALID, 'ce:crgzip;gzip' => 'Content-Encoding:<CR>gzip, served with gzip' ],
    [ INVALID, 'ce:crgzip' => 'Content-Encoding:<CR>gzip, not served with gzip' ],
    [ INVALID, 'ce:cr-gzip;gzip' => 'Content-Encoding:<CR><space>gzip, served with gzip' ],
    [ INVALID, 'ce:cr-gzip' => 'Content-Encoding:<CR><space>gzip, not served with gzip' ],

    [ 'INVALID: slightly invalid gzip encodings' ],
    [ INVALID,'ce:gzip;gzip;replace:0,2=1f8c', 'wrong gzip magic header'],
    [ INVALID,'ce:gzip;gzip;replace:2,1=88', 'wrong compression method 88 instead of 08'],
    [ UNCOMMON_VALID,'ce:gzip;gzip;replace:3,1|01', 'set flag FTEXT'],
    [ INVALID,'ce:gzip;gzip;replace:3,1|02', 'set flag FHCRC without having CRC'],
    [ INVALID,'ce:gzip;gzip;replace:3,1|02;replace:10,0=0000', 'set flag FHCRC and add CRC with 0'],
    [ INVALID,'ce:gzip;gzip;replace:3,1|02;replace:10,0=0001', 'set flag FHCRC and add CRC with 1'],
    [ UNCOMMON_VALID,'ce:gzip;gzip;replace:3,1|04;replace:10,0=0000', 'set flag FEXTRA and extra part with XLEN 0'],
    [ UNCOMMON_VALID,'ce:gzip;gzip;replace:3,1|04;replace:10,0=05004170010000', 'set flag FEXTRA and extra part with XLEN 5'],
    [ INVALID,'ce:gzip;gzip;replace:3,1|04;replace:10,0=0500', 'set flag FEXTRA and XLEN 5 but no extra part'],
    [ INVALID,'ce:gzip;gzip-payload-as-extra', 'gzip, but hide the real (deflate) payload inside the EXTRA part'],
    [ UNCOMMON_VALID,'ce:gzip;gzip;replace:3,1|08;replace:10,0=2000', 'set flag FNAME and add short file name'],
    [ UNCOMMON_VALID,'ce:gzip;gzip;replace:3,1|10;replace:10,0=2000', 'set flag FCOMMENT and add short comment'],
    [ INVALID,'ce:gzip;gzip;replace:3,1|20', 'set flag reserved bit 5'],
    [ INVALID,'ce:gzip;gzip;replace:3,1|40', 'set flag reserved bit 6'],
    [ INVALID,'ce:gzip;gzip;replace:3,1|80', 'set flag reserved bit 7'],
    [ INVALID,'ce:gzip;gzip;replace:-8,4^ffffffff', 'invalidate final checksum'],
    [ INVALID,'ce:gzip;gzip;replace:-4,1^ff', 'invalidate length'],
    [ INVALID,'ce:gzip;gzip;replace:-4,4=', 'remove length'],
    [ INVALID,'ce:gzip;gzip;replace:-8,8=', 'remove checksum and length'],
    [ INVALID,'ce:gzip;gzip;replace:-4,4=;clen+4', 'remove length but set content-length header to original size'],
    [ INVALID,'ce:gzip;gzip;replace:-8,8=;clen+8', 'remove checksum and length but set content-length header to original size'],
    [ INVALID,'ce:gzip;gzip;replace:-4,4=;noclen', 'remove length and close with eof without sending length'],
    [ INVALID,'ce:gzip;gzip;replace:-8,8=;noclen', 'remove checksum and and close with eof without sending length'],
    # and now hide the 'gzip' behind a \r so that some firewalls will use the
    # heuristics of the antivirus which might be different from the the proxy
    [ INVALID,'ce:cr-gzip;gzip;replace:3,1|01', 'set flag FTEXT (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:3,1|02;replace:10,0=0000', 'set flag FHCRC and add CRC with 0 (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:3,1|08;replace:10,0=2000', 'set flag FNAME and add short file name (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:3,1|10;replace:10,0=2000', 'set flag FCOMMENT and add short comment (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:3,1|04;replace:10,0=0000', 'set flag FEXTRA and extra part with XLEN 0 (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:3,1|04;replace:10,0=05004170010000', 'set flag FEXTRA and extra part with XLEN 5 (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:3,1|20', 'set flag reserved bit 5 (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:3,1|40', 'set flag reserved bit 6 (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:3,1|80', 'set flag reserved bit 7 (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:-8,4^ffffffff', 'invalidate final checksum (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:-4,1^ff', 'invalidate length (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:-4,4=', 'remove length (hide gzip with "content-encoding:\r gzip")'],
    [ INVALID,'ce:cr-gzip;gzip;replace:-8,8=', 'remove checksum and length (hide gzip with "content-encoding:\r gzip")'],
    # same game, but with Content-Encoding<space>: for other firewalls
    [ INVALID,'ce-space-colon-gzip;gzip;replace:3,1|01', 'set flag FTEXT (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:3,1|02;replace:10,0=0000', 'set flag FHCRC and add CRC with 0 (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:3,1|08;replace:10,0=2000', 'set flag FNAME and add short file name (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:3,1|10;replace:10,0=2000', 'set flag FCOMMENT and add short comment (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:3,1|04;replace:10,0=0000', 'set flag FEXTRA and extra part with XLEN 0 (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:3,1|04;replace:10,0=05004170010000', 'set flag FEXTRA and extra part with XLEN 5 (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:3,1|20', 'set flag reserved bit 5 (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:3,1|40', 'set flag reserved bit 6 (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:3,1|80', 'set flag reserved bit 7 (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:-8,4^ffffffff', 'invalidate final checksum (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:-4,1^ff', 'invalidate length (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:-4,4=', 'remove length (hide gzip with "content-encoding : gzip")'],
    [ INVALID,'ce-space-colon-gzip;gzip;replace:-8,8=', 'remove checksum and length (hide gzip with "content-encoding : gzip")'],
    # and then used with an additional chunked transfer encoding
    [ INVALID,'chunked;ce:gzip;gzip;replace:3,1|20', 'set flag reserved bit 5, chunked'],
    [ INVALID,'chunked;ce:gzip;gzip;replace:3,1|40', 'set flag reserved bit 6, chunked'],
    [ INVALID,'chunked;ce:gzip;gzip;replace:3,1|80', 'set flag reserved bit 7, chunked'],
    [ INVALID,'chunked;ce:gzip;gzip;replace:-8,4^ffffffff', 'invalidate final checksum, chunked'],
    [ INVALID,'chunked;ce:gzip;gzip;replace:-4,1^ff', 'invalidate length, chunked'],
    [ INVALID,'chunked;ce:gzip;gzip;replace:-4,4=', 'remove length, chunked'],
    [ INVALID,'chunked;ce:gzip;gzip;replace:-8,8=', 'remove checksum and length, chunked'],
    [ INVALID,'chunked;ce:gzip;gzip;replace:-4,4=;clen+4', 'remove length but set content-length header to original size, chunked'],
    [ INVALID,'chunked;ce:gzip;gzip;replace:-8,8=;clen+8', 'remove checksum and length but set content-length header to original size, chunked'],
    [ INVALID,'chunked;ce:gzip;gzip;replace:-4,4=;noclen', 'remove length and close with eof without sending length, chunked'],
    [ INVALID,'chunked;ce:gzip;gzip;replace:-8,8=;noclen', 'remove checksum and and close with eof without sending length, chunked'],

    # data before gzip
    [ INVALID,'ce:gzip;gzip;\012-before-body','new line at start of gzip body' ],
);

sub make_response {
    my ($self,$page,$spec) = @_;
    return make_index_page() if $page eq '';
    my ($hdr,$data) = content($page,$self->ID."-".$spec) or die "unknown page $page";
    my $version = '1.1';
    my $clen_extend;
    my $body_prefix = '';
    my $te = 'clen';
    my @data; # preferred against $data if given
    for (split(';',$spec)) {
	if ($_ eq 'ce:rfc2047-deflate') {
	    $hdr .= "Content-Encoding: =?UTF-8?B?ZGVmbGF0ZQo=?=\r\n";
	} elsif ( my ($field,$v) = m{^(ce|te):(.*)$}i ) {
	    my $changed;
	    $changed++ if $v =~s{(?<=cr|lf|nl)-}{ }g;
	    $changed++ if $v =~s{cr}{\r}g;
	    $changed++ if $v =~s{lf}{\n}g;
	    $changed++ if $v =~s{nl}{\r\n}g;
	    $changed++ if $v =~s{_}{ }g;
	    $changed++ if $v =~s{hugespace-}{ ' ' x 10000 }eg;
	    $v =~s{(?<!x)-}{}g;
	    $hdr .= "Connection: close\r\n" if $changed;
	    $hdr .= $field eq 'ce' ? 'Content-Encoding:':'Transfer-Encoding:';
	    $hdr .= "$v\r\n";
	} elsif ( m{^(pkt|chk):zlib\+deflate(\+deflate)?\z} ) {
	    # [zlib-header][deflate][more-deflate]....
	    # zlib will return with Z_DATA_ERROR when trying to process
	    # more-deflate because it actually expected the correct ADLER32
	    # checksum there. Browsers will then assume that this should be
	    # raw-deflate instead and retry with an edded zlib header.
	    # With some browsers this process can be repeated.

	    my ($enc,$nchunks) = ($1, $2? 3:2);
	    my $size = int(length($data)/$nchunks);
	    @data = ();
	    for(my $i=0;$i<$nchunks;$i++) {
		push @data, substr($data,0,$size,'');
	    }
	    $data[-1] .= $data; # in case something left
	    $_ = zlib_compress($_,'deflate') for(@data);
	    $data[0] = "\x78\x9c".$data[0];
	    $te = 'chunked' if $enc eq 'chk';
	} elsif ( m{^(?:(gzip)|deflate|(zlib))(?:(\d+)([ps]))?(?:,(sync|partial|block|full|finish))?$} ) {
	    my $zlib = Compress::Raw::Zlib::Deflate->new(
		-WindowBits => $1 ? WANT_GZIP : $2 ? +MAX_WBITS() : -MAX_WBITS(),
		-AppendOutput => 1,
	    );
	    my $size = int(length($data)/($3||1)) || 1;
	    my @chunks;