view release on metacpan or  search on metacpan

lib/Module/CPANTS/ProcessCPAN.pm  view on Meta::CPAN

    $me->prev_run($prev[0]);
     
    my $now=DateTime->now;
    my $run=$me->db->resultset('Run')->create({
        mcanalyse_version=>$Module::CPANTS::Analyse::VERSION,
        mcprocess_version=>$Module::CPANTS::ProcessCPAN::VERSION,
        available_kwalitee=>$mck->available_kwalitee,
        total_kwalitee=>$mck->total_kwalitee,
        date=>$now,
    });
    $me->run($run);
    print $run->id,"\n";

    return $me;
}

sub process_cpan {
    my $me=shift;
    
    my $p=Parse::CPAN::Packages->new($me->cpan_02packages);
    my $db=$me->db;
    my $lint=$me->lint;
    my %seen=('Core'=>1);
    
    # prefill in_db
    my %in_db;
    my $all_dists=$db->resultset('Dist')->search;
    if($all_dists) {
        while (my $d=$all_dists->next) {
            next unless $d->vname;
            next if $d->dist eq 'Core';
            $in_db{$d->vname}++;
        }
    }
    $db->resultset('Dist')->find_or_create({dist=>'Core',id=>0,is_core=>1});

    my %authors;
    foreach my $dist (sort {$a->dist cmp $b->dist} $p->latest_distributions) {
        my $vname=$dist->distvname;
        next if $vname=~/^perl[-_]/;
        next if $vname=~/^ponie-/;
        next if $vname=~/^Perl6-Pugs/;
        next if $vname=~/^parrot-/;
        next if $vname=~/^Bundle-/;
        $seen{$dist->dist}++;

        if ($in_db{$dist->distvname}) {
            if ($me->force) {
                print "forced reindex of $vname\n";
            }
            else {
                print "skipping $vname\n";
                next;
            }
        }
        else {
            print "new version of $vname\n";
        }
    
        print "analyse $vname\n";
        my $file=$me->cpan_path_to_dist($dist->prefix);
        
        # call cpants_lint.pl
        my $from_lint=`$^X $lint --yaml $file`;
        $me->process_yaml($from_lint);   
    
    }

    # dump old dists from DB
    my @distributions=$p->distributions;
    my %dists=map {$_->dist => 1} grep { $_->dist }   @distributions;
    $all_dists->reset;
    while (my $d=$all_dists->next) {
        unless ($seen{$d->dist}) {
            print $d->dist." not on CPAN anymore, deleted from DB\n";
            $d->delete;
        }
    }
}

sub process_yaml {
    my ($me,$yaml)=@_;
    
    my $db=$me->db;
    my $run=$me->run;
    my $data; 
    eval { $data=Load($yaml) };
    if ($@) {
        print "Cannot parse YAML: $@";
        next;
    }

    #use Data::Dumper;
    #print Dumper $data;

    # remove data that references other tables;
    my $kwalitee    = delete $data->{kwalitee};
    my $modules     = delete $data->{modules};
    my $uses        = delete $data->{uses};
    my $prereq      = delete $data->{prereq};
    my $author      = delete $data->{author};
    my $error       = delete $data->{error};
    my $versions    = delete $data->{versions};
    my $licenses    = delete $data->{licenses};
    my $test_files  = delete $data->{test_files};
    $data->{test_files_list} = join(';',@$test_files) if $test_files && ref($test_files) eq 'ARRAY';

    # TODO store licenses & versions
    foreach (qw(files_array ignored_files_array files_hash dirs_array meta_yml)) {
        delete $data->{$_};
    }
        
    my ($db_author,$db_dist,$db_error);
    eval {
        if ($db_dist=$db->resultset('Dist')->find({dist=>$data->{dist}})) {
            $me->make_dist_history($db_dist);
            $db_dist->run($run->id);
            $db_author=$db_dist->author;
        }
        else {    
            $db_author=$db->resultset('Author')->find_or_create({pauseid=>$author});

lib/Module/CPANTS/ProcessCPAN.pm  view on Meta::CPAN

            num_dists=>$author->num_dists || 0,
            rank=>$author->rank || 0,
        });
    
        # set conveniece fields in current author
        $author->prev_av_kw($author->average_kwalitee || 0);
        $author->prev_rank($author->rank|| 0);
        $author->update; 
    };
}

sub make_dist_history {
    my ($me,$dist)=@_;
    eval {
        my $old_kw=$dist->kwalitee ? $dist->kwalitee->kwalitee : 0;
        $me->db->resultset('HistoryDist')->find_or_create({
            dist=>$dist->id,
            run=>$me->run->id,
            distname=>$dist->dist,
            version=>$dist->version,
            kwalitee=>$old_kw,
        });
    };
    return;
}

sub db {
    my $me=shift;
    return $me->_db if $me->_db;
   
    return $me->_db(Module::CPANTS::Schema->connect(
        $me->dsn
    ));
}

=head3 dsn

Returns the DSN as a three element list (dbname, user, pwd)

=cut

sub dsn {
    my $me=shift;
    return (
        'dbi:Pg:dbname=cpants',
        Module::CPANTS::ProcessCPAN::ConfigData->config('db_user'),
        Module::CPANTS::ProcessCPAN::ConfigData->config('db_pwd')
    );
}

sub cpan_01mailrc {
    my $me=shift;
    return catfile($me->cpan,'authors','01mailrc.txt.gz');
}

sub cpan_02packages {
    my $me=shift;
    return catfile($me->cpan,'modules','02packages.details.txt.gz');
}

sub cpan_path_to_dist {
    my $me=shift;
    my $prefix=shift;
    return catfile($me->cpan,'authors','id',$prefix);
}

=head2 Accessors to various directories

=cut

sub home_dir {
    my $me=shift;
    return Module::CPANTS::ProcessCPAN::ConfigData->config('home');
}


1;

__END__


=pod

=head1 NAME

Module::CPANTS::ProcessCPAN - Generate Kwalitee ratings for the whole CPAN

=head1 SYNOPSIS
  
=head1 DESCRIPTION

Run CPANTS on the whole of CPAN. Includes a DBIx::Class based DB 
abstraction layer. More docs soon...

=head2 How to set up a local CPANTS processor

=head3 Prereqs

=over

=item * A PostgreSQL DB named C<cpants>

=item * A local CPAN mirror (eg one mirrored with CPAN::Mini)

=item * All the prereqs of Module::CPANTS::Analyse & 
Module::CPANTS::ProcessCPAN

=back

=head3 Set up the DB

You can find the current schema of the CPANTS DB in 
F<sql/cpants.schema>. Use this schema to set up a Postgres DB:

  psql cpants < sql/cpants.schema

You will also need to set up an account in the DB. If you don't know 
how to do that, read the postgres docs...

=head3 Install Module::CPANTS::ProcessCPAN