view release on metacpan or  search on metacpan

t/012-spider-server.t  view on Meta::CPAN

<urlset	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	    xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
	    xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">

	<url>
		<loc>$base/hello</loc>
		<lastmod>$now+00:00</lastmod>
		<changefreq>weekly</changefreq>
		<priority>0.6</priority>
	</url>
	<url>
		<loc>$base/nosuchlink</loc>
		<lastmod>$yesterday+00:00</lastmod>
		<changefreq>daily</changefreq>
		<priority>0.6</priority>
	</url>
	<url>
		<loc>http://elsewhere.foo/bar</loc>
		<lastmod>$yesterday+00:00</lastmod>
		<changefreq>hourly</changefreq>
		<priority>0.6</priority>
	</url>
</urlset>
XML

        }
    }

    use_ok('Dezi::Test::Indexer');

    my $port     = 5002;
    my $server   = MyServer->new($port);
    my $base_uri = $server->started_ok('start http server');
    if ( !$base_uri ) {
        die "server failed to start";
    }
    my $debug = $ENV{DEZI_DEBUG} || 0;

    ok( my $spider = Dezi::Aggregator::Spider->new(
            verbose => $debug,
            debug   => $debug,
            email   => 'noone@dezi.org',
            agent   => 'dezi-app-test',

            #max_depth => 2, # unlimited

            file_rules => [
                'filename contains bad-pattern',

                #'filename contains \?',    # anything with query string
                'filename contains \?.*query=pass',    # specific query param
            ],

            # hurry up and fail
            delay => 0,

            filter => sub {
                $debug and diag( "doc filter on " . $_[0]->url );
                $debug and diag( "body:" . $_[0]->content );
            },
            credentials    => 'foo:bar',
            same_hosts     => ["127.0.0.1"],
            modified_since => time2str( time() ),
            max_size       => 4096,
        ),
        "new spider"
    );

    diag( "spidering " . $base_uri );
    is( $spider->crawl($base_uri), 5, "crawl" );

}