view release on metacpan or  search on metacpan

lib/AWS/S3/FileIterator.pm  view on Meta::CPAN

sub _init {
    my ( $s ) = @_;

    foreach ( qw( bucket page_size page_number ) ) {
        confess "Required argument '$_' was not provided"
          unless $s->{$_};
    }    # end foreach()

    $s->{page_number}--;
    $s->{marker}               = '' unless defined( $s->{marker} );
    $s->{__fetched_first_page} = 0;
    $s->{data}                 = [];
    $s->{pattern} ||= qr(.*);
}    # end _init()

sub marker    { shift->{marker} }
sub pattern   { shift->{pattern} }
sub bucket    { shift->{bucket} }
sub page_size { shift->{page_size} }
sub prefix    { shift->{prefix} }

sub has_prev {
    my $s = shift;

    return $s->page_number > 1;
}    # end has_prev()

sub has_next { shift->{has_next} }

sub next {
  my $s = shift;

  if( exists( $s->{data}->[ $s->{idx} ] ) ) {
    return $s->{data}->[ $s->{idx}++ ];
  } else {
    # End of the current resultset, see if we can get another page of records:
    if( my $page = $s->next_page ) {
      $s->{data} = $page;
      $s->{idx} = 0;
      return $s->{data}->[ $s->{idx}++ ];
    } else {
      # No more pages, no more data:
      return;
    }
  }
}

sub reset {
  my $s = shift;
  $s->{idx} = 0;
}


sub page_number {
    my $s = shift;
    @_ ? $s->{page_number} = $_[0] - 1 : $s->{page_number};
}    # end page_number()

# S3 returns files 100 at a time.  If we want more or less than 100, we can't
# just fetch the next page over and over - that would be inefficient and likely
# to cause errors.

# If the page size is 5 and page number is 2, then we:
#   - fetch 100 items
#   - store them
#   - iterate internally until we get to 'page 2'
#   - return the result.
# If the page size is 105 and page number is 1, then we:
#   - fetch 100 items
#   - fetch the next 100 items
#   - return the first 105 items, keeping the remaining 95 items
#   - on page '2', fetch the next 100 items and return 105 items, saving 90 items.
# If the page size is 105 and page number is 3, then we:
#   - fetch items until our internal 'start' marker is 316-420
#   - return items 316-420
sub next_page {
    my $s = shift;

    # Advance to page X before proceding:
    if ( ( !$s->{__fetched_first_page}++ ) && $s->page_number ) {

        # Advance to $s->page_number
        my $start_page = $s->page_number;
        my $to_discard = $start_page * $s->page_size;
        my $discarded  = 0;
        while ( 1 ) {
            my $item = $s->_next
              or last;
            $discarded++ if $item->{key} =~ $s->pattern;
            last if $discarded > $to_discard;
        }    # end while()
    }    # end if()

    my @chunk = ();
    while ( my $item = $s->_next() ) {
        next unless $item->{key} =~ $s->pattern;
        push @chunk, $item;
        last if @chunk == $s->page_size;
    }    # end while()

    my @out = map {
        my $owner = AWS::S3::Owner->new( %{ $_->{owner} } );
        delete $_->{owner};
        AWS::S3::File->new( %$_, owner => $owner );
    } @chunk;

    $s->{page_number}++;

    return unless @out;
    wantarray ? @out : \@out;
}    # end next_page()

sub _next {
    my $s = shift;

    if ( my $item = shift( @{ $s->{data} } ) ) {
        return $item;
    } else {
        if ( my @chunk = $s->_fetch() ) {
            push @{ $s->{data} }, @chunk;
            return shift( @{ $s->{data} } );