view release on metacpan or  search on metacpan

lib/AWS/S3/FileIterator.pm  view on Meta::CPAN

    # Advance to page X before proceding:
    if ( ( !$s->{__fetched_first_page}++ ) && $s->page_number ) {

        # Advance to $s->page_number
        my $start_page = $s->page_number;
        my $to_discard = $start_page * $s->page_size;
        my $discarded  = 0;
        while ( 1 ) {
            my $item = $s->_next
              or last;
            $discarded++ if $item->{key} =~ $s->pattern;
            last if $discarded > $to_discard;
        }    # end while()
    }    # end if()

    my @chunk = ();
    while ( my $item = $s->_next() ) {
        next unless $item->{key} =~ $s->pattern;
        push @chunk, $item;
        last if @chunk == $s->page_size;
    }    # end while()

    my @out = map {
        my $owner = AWS::S3::Owner->new( %{ $_->{owner} } );
        delete $_->{owner};
        AWS::S3::File->new( %$_, owner => $owner );
    } @chunk;

    $s->{page_number}++;

    return unless @out;
    wantarray ? @out : \@out;
}    # end next_page()

sub _next {
    my $s = shift;

    if ( my $item = shift( @{ $s->{data} } ) ) {
        return $item;
    } else {
        if ( my @chunk = $s->_fetch() ) {
            push @{ $s->{data} }, @chunk;
            return shift( @{ $s->{data} } );
        } else {
            return;
        }    # end if()
    }    # end if()
}    # end _next()

sub _fetch {
    my ( $s ) = @_;

    my $path   = $s->{bucket}->name . '/';
    my %params = ();
    $params{marker} = $s->{marker} if $s->{marker};
    $params{prefix} = $s->{prefix} if $s->{prefix};
    $params{max_keys} = 1000;
    $params{delimiter} = $s->{delimiter} if $s->{delimiter};

    my $type     = 'ListBucket';
    my $request  = $s->{bucket}->s3->request( $type, %params, bucket => $s->{bucket}->name );
    my $response = $request->request();

    $s->{has_next} = ( $response->xpc->findvalue( '//s3:IsTruncated' ) || '' ) eq 'true' ? 1 : 0;

    my @files = ();
    foreach my $node ( $response->xpc->findnodes( '//s3:Contents' ) ) {
        my ( $owner_node ) = $response->xpc->findnodes( './/s3:Owner', $node );
        my $owner = {
            id           => $response->xpc->findvalue( './/s3:ID',          $owner_node ),
            display_name => $response->xpc->findvalue( './/s3:DisplayName', $owner_node )
        };
        my $etag = $response->xpc->findvalue( './/s3:ETag', $node );
        push @files,
          {
            bucket => $s->{bucket},
            key          => $response->xpc->findvalue( './/s3:Key',          $node ),
            lastmodified => $response->xpc->findvalue( './/s3:LastModified', $node ),
            etag         => $response->xpc->findvalue( './/s3:ETag',         $node ),
            size         => $response->xpc->findvalue( './/s3:Size',         $node ),
            owner        => $owner,
          };
    }    # end foreach()

    if ( @files ) {
        $s->{marker} = $files[-1]->{key};
    }    # end if()

    return unless defined wantarray;
    @files ? return @files : return;
}    # end _fetch()

1;

__END__

=pod

=head1 NAME

AWS::S3::FileIterator - Easily access and iterate through your S3 files.

=head1 SYNOPSIS

  # Iterate through all ".txt" files, 100 at a time:
  my $iter = $bucket->files(
    # Required params:
    page_size   => 100,
    page_number => 1,
    # Optional params:
    pattern     => qr(\.txt$),
    prefix      => 'notes',
  );
  
  while( my @files = $iter->next_page )
  {
    warn $iter->page_number, "\n";
    foreach my $file ( @files )
    {
      print "\t", $file->key, "\n";
    }# end foreach()