ETLp

 view release on metacpan or  search on metacpan

lib/ETLp/Audit/FileProcess.pm  view on Meta::CPAN

class ETLp::Audit::FileProcess with (ETLp::Role::Config, ETLp::Role::Schema,
                             ETLp::Role::Audit) {
    
    use ETLp::Types;
    use Try::Tiny;
    use ETLp::Exception;
    
    has 'item' =>
      (is => 'ro', isa => 'ETLp::Audit::Item', required => 1, weak_ref => 1);
    has 'filename' => (is => 'ro', isa => 'Str',     required => 1);
    has 'config'   => (is => 'ro', isa => 'HashRef', required => 1);
    
=head2 method_id

Return the primary key of the file process record

=cut
    method id {
        return $self->{id};
    }
    

lib/ETLp/Audit/Item.pm  view on Meta::CPAN

    use Try::Tiny;
    use ETLp::Types;
    use ETLp::Audit::FileProcess;
    use DateTime;
    
    has 'job' =>
      (is => 'ro', isa => 'ETLp::Audit::Job', required => 1, weak_ref => 1);
    has 'name'   => (is => 'ro', isa => 'Str',     required => 1);
    has 'phase'  => (is => 'ro', isa => 'Str',     required => 1);
    has 'type'   => (is => 'ro', isa => 'Str',     required => 1);
    has 'config' => (is => 'ro', isa => 'HashRef', required => 1);
    
    method id {
        return $self->{id};
    }
    
=head2 update_message

Sets the item message

=cut

lib/ETLp/Audit/Job.pm  view on Meta::CPAN

=head2 item

The currently executing item

=cut
    
    has 'session_id' => (is => 'rw', isa => 'Int', required => 0);   
    has 'id' => (is => 'rw', isa => 'Int', required => 0);
    has 'name' => (is => 'ro', isa => 'Str', required => 1);
    has 'section' => (is => 'ro', isa => 'Str', required => 1);
    has 'config'  => (is => 'ro', isa => 'HashRef', required => 1);
    
=head1 METHODS

=head2 create_item

Create an audit item

=head3 parameters (Hash)

    * Str name: Name of the item

lib/ETLp/Execute/Iteration.pm  view on Meta::CPAN

class ETLp::Execute::Iteration with ETLp::Role::Config {
    use Try::Tiny;
    use Data::Dumper;
    use File::LocalizeNewlines;
    use File::Basename;
    use File::Copy;
    use Carp;
    use ETLp::Exception;
    
    has 'pipeline' => (is => 'ro', isa => 'ArrayRef', required => 1);
    has 'config'   => (is => 'ro', isa => 'HashRef',  required => 1);
    
    $Data::Dumper::Deparse = 1;
    
    method run {
        my $warning = 0;
        
        try {
            FILE_LOOP: foreach my $filename ($self->_get_source_files) {
                ITEM_LOOP:
                foreach my $item (@{$self->pipeline}) {

lib/ETLp/Execute/Serial.pm  view on Meta::CPAN

    * void

=cut

class ETLp::Execute::Serial with ETLp::Role::Config {
    use Try::Tiny;
    use Carp;
    use ETLp::Exception;
    use Data::Dumper;
    
    has 'pipeline' => (is => 'ro', isa => 'ArrayRef', required => 1);
    has 'config'   => (is => 'ro', isa => 'HashRef', required => 1);
    
    method run {
        my $warning = 0;
        
        try {
            ITEM_LOOP: foreach my $item (@{$self->pipeline}) {
                $self->logger->debug("Item: " . Dumper($item));
    
                # Create an audit entry for the item
                my $audit_item = $self->audit->create_item(

lib/ETLp/File/Read/CSV.pm  view on Meta::CPAN

    * A hashref where each key is populated with the filed value from the
      file record
    
=cut

class ETLp::File::Read::CSV extends ETLp::File::Read {
    use Text::CSV;
    use Data::Dumper;
    use ETLp::Exception;
    
    has 'csv_options' => (is => 'ro', isa => 'HashRef', default => sub { {} });
    has 'fields' => (is => 'ro', isa => 'ArrayRef');
    has 'ignore_field_count' => (is => 'ro', isa => 'Bool', default => 0);

    method get_fields {
        my $line = $self->get_line;
        return unless $line;
        my $csv    = $self->{_csv};
        my $status = $csv->parse($line);

        if ($status) {
            my @columns = $csv->fields;

lib/ETLp/File/Validate.pm  view on Meta::CPAN

    use Time::Piece;
    use ETLp::Exception;
    use ETLp::File::Config;

    has 'data_directory'        => (is => 'ro', isa => 'Str');
    has 'file_config_directory' => (is => 'ro', isa => 'Str');
    has 'file_definition' => => (is => 'ro', isa => 'Str', required => 1);
    has 'type'     => (is => 'ro', isa => 'Str',  required => 1);
    has 'localize' => (is => 'ro', isa => 'Bool', default  => 0);
    has 'skip'     => (is => 'ro', isa => 'Int', default  => 0);
    has 'csv_options' => (is => 'ro', isa => 'HashRef', required => 0, default => sub{{}});

    our @errors;

=head2 validate

Validate the file

Parameters

    * The name of the file being validated,

lib/ETLp/ItemBuilder.pm  view on Meta::CPAN

=cut

class ETLp::ItemBuilder with ETLp::Role::Config {
    use ETLp::Exception;
    use Data::Dumper;
    use UNIVERSAL::require;
    use Try::Tiny;
    use Clone qw(clone);
    use File::Basename;

    has 'plugins'       => (is => 'ro', isa => 'HashRef', required => 1);
    has 'pipeline_type' => (is => 'ro', isa => 'Str',     required => 1);
    has 'allow_env_vars' =>
      (is => 'ro', isa => 'Bool', required => 0, default => 0);
    has 'env_conf' =>
      (is => 'ro', isa => 'HashRef', required => 0, default => sub { {} });
    has 'app_root' => (is => 'ro', isa => 'Maybe[Str]', required => 1);
    has 'config'   => (is => 'ro', isa => 'HashRef',    required => 1);

    # Recursively navigate the items, replacing the placeholders
    # with their values
    
    method _parse_item(HashRef $config, $item, Maybe [Str] $filename) {
        my $app_root = $self->app_root;
        my %master_config = (%{$self->env_conf}, %{$config->{config}});
        $self->logger->debug("Master config: " . Dumper(\%master_config));
        if (ref $item eq 'HASH') {
            foreach my $item_key (keys %$item) {

lib/ETLp/Loader/CSV.pm  view on Meta::CPAN

    use ETLp::File::Read::CSV;
    use Data::Dumper;
    use File::Basename;
    use Convert::NLS_DATE_FORMAT qw(posix2oracle);
    use DBI::Const::GetInfoType;
    use ETLp::Exception;
    use Try::Tiny;
    
    has 'directory' => (is => 'ro', isa => 'Str');
    has 'table'     => (is => 'ro', isa => 'Str', required => 1);
    has 'columns'   => (is => 'ro', isa => 'ArrayRef', required => 1);
    has 'rules'     => (is => 'ro', isa => 'HashRef', required => 0);
    has 'localize'  => (is => 'ro', isa => 'Int', default => 0);
    has 'error'     => (is => 'rw', isa => 'Str');
    has 'file_id'   => (is => 'ro', isa => 'Int', required => 1);
    has 'skip'      => (is => 'ro', isa => 'Int', required => 0, default => 0);
    has 'ignore_field_count' => (is => 'ro', isa => 'Bool', default => 0);
    has 'csv_options' => (is => 'ro', isa => 'HashRef', required => 0,
                          default => sub{{allow_whitespace => 1}});
    
=head1 METHODS

=head2 new

Create a loader, specifying the characteristics

Parameters

lib/ETLp/Loader/OracleSQLLoader.pm  view on Meta::CPAN

    has 'table'            => (is => 'ro', isa => 'Str',  required => 1);
    has 'filename'         => (is => 'ro', isa => 'Str',  required => 1);
    has 'specification'    => (is => 'ro', isa => 'Str',  required => 1);
    has 'logfile'          => (is => 'ro', isa => 'Maybe[Str]');
    has 'badfile'          => (is => 'ro', isa => 'Maybe[Str]');
    has 'discardfile'      => (is => 'ro', isa => 'Maybe[Str]');
    has 'keep_controlfile' => (is => 'ro', isa => 'Bool', default  => 0);
    has 'keep_logfile'     => (is => 'ro', isa => 'Bool', default  => 1);
    has 'keep_badfile'     => (is => 'ro', isa => 'Bool', default  => 1);
    has 'keep_discardfile' => (is => 'ro', isa => 'Bool', default  => 1);
    has 'parameters'       => (is => 'ro', isa => 'Maybe[HashRef]');
    has 'localize'         => (is => 'ro', isa => 'Bool', default  => 0);
    has 'mode' => (is => 'ro', isa => 'SQLLoaderMode', default => 'append');
    
    our @keywords = qw/
        discardmax
        skip
        load
        errors
        rows
        bindsize

lib/ETLp/Plugin.pm  view on Meta::CPAN

    method run(Str $filename) {
        .... <<code>> ....
    }

=cut

class ETLp::Plugin with ETLp::Role::Config {
    use ETLp::Exception;
    
    has 'config'        => (is => 'ro', isa => 'HashRef', required => 1);
    has 'item'          => (is => 'ro', isa => 'HashRef', required => 1);
    has 'original_item' => (is => 'ro', isa => 'HashRef', required => 1);
    has 'env_conf'      => (is => 'ro', isa => 'HashRef', required => 1);

    sub type {
        ETLpException->throw(error => "Abstract method 'type'");
    }

    method run {
        ETLpException->throw(error => "Abstract method 'run'");
    }
}