AC-MrGamoo

 view release on metacpan or  search on metacpan

META.yml  view on Meta::CPAN

author:
    - AdCopy <http://www.adcopy.com>
license:            perl
distribution_type:  module
configure_requires:
    ExtUtils::MakeMaker:  0
requires:
    AC::DC:               0
    Digest::SHA1:         0
    Google::ProtocolBuffers:  0
    JSON:                 0
    POSIX:                0
    Sys::Hostname:        0
    Time::HiRes:          0
no_index:
    directory:
        - t
        - inc
generated_by:       ExtUtils::MakeMaker version 6.48
meta-spec:
    url:      http://module-build.sourceforge.net/META-spec-v1.4.html

Makefile.PL  view on Meta::CPAN

use ExtUtils::MakeMaker;
WriteMakefile(
              NAME            => 'AC::MrGamoo',
              VERSION_FROM    => 'lib/AC/MrGamoo.pm',
              ABSTRACT_FROM   => 'lib/AC/MrGamoo.pm',
              AUTHOR          => 'AdCopy <http://www.adcopy.com>',
              LICENSE         => 'perl',
              PREREQ_PM       => {
                  'POSIX'                       => 0,
                  'Sys::Hostname'	        => 0,
                  'JSON'		        => 0,
                  'Digest::SHA1'	        => 0,
                  'Time::HiRes'			=> 0,
                  'Google::ProtocolBuffers'	=> 0,
		  'AC::DC'			=> 0,
              }
);

eg/filelist.pm  view on Meta::CPAN

# -*- perl -*-
# example filelist

# $Id: filelist.pm,v 1.1 2010/11/01 19:04:21 jaw Exp $

package Local::MrMagoo::FileList;
use AC::ISOTime;
use AC::Yenta::Direct;
use JSON;
use strict;

my $YDBFILE = "/data/files.ydb";

sub get_file_list {
    my $config = shift;

    # get files + metadata from yenta
    my $yenta = AC::Yenta::Direct->new( 'files', $YDBFILE );

eg/readinput.pm  view on Meta::CPAN

# -*- perl -*-
# example readinput

# $Id: readinput.pm,v 1.1 2010/11/01 19:04:22 jaw Exp $

package Local::MrMagoo::ReadInput;
use AC::MrMagoo::User;
use JSON;
use strict;

our $R;		# exported by AC::MrMagoo::User

sub readinput {
    my $fd = shift;	# file handle

    # our file is newline delimted json data

    # read next line

lib/AC/MrGamoo/AC/FileList.pm  view on Meta::CPAN

# Author: Jeff Weisberg
# Created: 2010-Jan-14 17:04 (EST)
# Function: get list of files to map
#
# $Id: FileList.pm,v 1.3 2010/11/10 16:24:38 jaw Exp $

package AC::MrGamoo::AC::FileList;
use AC::MrGamoo::Debug 'files';
use AC::ISOTime;
use AC::Yenta::Direct;
use JSON;
use strict;

my $YDBFILE = "/home/acdata/logfile.ydb";

# return an array of:
#   {
#     filename    => www/2010/01/17/23/5943_prod_5x2N5qyerdeddsNi
#     location    => [ scrib@a2be021bd31c, scrib@a2be021ad31c ]
#     size        => 10863
#     [anything else]

lib/AC/MrGamoo/Client.pm  view on Meta::CPAN

# $Id: Client.pm,v 1.5 2011/01/18 17:19:12 jaw Exp $

package AC::MrGamoo::Client;
use AC::MrGamoo::Submit::Compile;
use AC::MrGamoo::Submit::Request;
use AC::MrGamoo::Debug;
use AC::Daemon;
use AC::Conf;
use AC::Misc;
use AC::MrGamoo::Protocol;
use JSON;
use Sys::Hostname;
use Socket;

require 'AC/protobuf/mrgamoo.pl';
require 'AC/protobuf/mrgamoo_status.pl';
require 'AC/protobuf/std_reply.pl';
use strict;


sub new {

lib/AC/MrGamoo/Iter/File.pm  view on Meta::CPAN

# Copyright (c) 2010 AdCopy
# Author: Jeff Weisberg
# Created: 2010-Jan-14 12:46 (EST)
# Function: iterate over a file
#
# $Id: File.pm,v 1.1 2010/11/01 18:41:55 jaw Exp $

package AC::MrGamoo::Iter::File;
use AC::MrGamoo::Iter;
use JSON;
our @ISA = 'AC::MrGamoo::Iter';
use strict;

sub new {
    my $class = shift;
    my $fd    = shift;
    my $pf    = shift;

    return bless {
        fd	 => $fd,

lib/AC/MrGamoo/Job.pm  view on Meta::CPAN

use AC::MrGamoo::Job::Done;
use AC::MrGamoo::Job::Info;
use AC::MrGamoo::Job::Action;
use AC::MrGamoo::FileList;
use AC::MrGamoo::PeerList;
use AC::MrGamoo::MySelf;
use AC::MrGamoo::EUConsole;
use AC::MrGamoo::Stats;
use AC::DC::IO;
use AC::Misc;
use JSON;
use Time::HiRes 'time';
use strict;

# RSN - config? tune?
our $TASKTIMEOUT = 10;
our $XFERTIMEOUT = 10;
our $TASKSRVRMAX = 4;
our $XFERSRVRMAX = 4;
our $REQMAX      = 10;
our $MAXLOAD     = 0.5;

lib/AC/MrGamoo/Kibitz/Peers.pm  view on Meta::CPAN

# $Id: Peers.pm,v 1.1 2010/11/01 18:41:59 jaw Exp $

package AC::MrGamoo::Kibitz::Peers;
use AC::MrGamoo::Debug 'kibitz_peers';
use AC::MrGamoo::About;
use AC::MrGamoo::MySelf;
use AC::MrGamoo::Config;
use AC::DC::Sched;
use AC::Misc;
use AC::Import;
use JSON;
use strict;

our @EXPORT = qw(pick_best_addr_for_peer peer_list_all get_peer_by_id);

my $KEEPDOWN = 300;     # keep data about down servers for how long?
my $KEEPLOST = 600;     # keep data about servers we have not heard about for how long?

my %SCEPTICAL;
my %ALLPEER;
my %MAYBEDOWN;

lib/AC/MrGamoo/Task.pm  view on Meta::CPAN

# $Id: Task.pm,v 1.10 2011/01/14 20:58:26 jaw Exp $

package AC::MrGamoo::Task;
use AC::MrGamoo::Debug 'task';
use AC::MrGamoo::Submit::Compile;
use AC::MrGamoo::Submit::Request;
use AC::MrGamoo::Task::Running;
use AC::MrGamoo::PeerList;
use AC::MrGamoo::Config;
use AC::DC::IO::Forked;
use JSON;
use strict;

my $TSTART  = $^T;
my $TIMEOUT = 3600;
my $MAXREQ  = 2;
my $MAXRUNNING = 7;	# tune me!
my %REGISTRY;
my $msgid = $$;

lib/AC/MrGamoo/Task/Running.pm  view on Meta::CPAN

use AC::MrGamoo::EUConsole;
use AC::MrGamoo::ReadInput;
use AC::MrGamoo::OutFile;
use AC::MrGamoo::MySelf;
use AC::Daemon;
use Digest::SHA1 'sha1';
use Digest::MD5  'md5';
use File::Path;
use Sys::Syslog;
use Socket;
use JSON;
use strict;

my $STATUSTIME = 5;			# seconds
my $MAXRUN     = 3600;			# override with %attr maxrun
my $SORTPROG   = '/usr/bin/sort';	# override with %attr sortprog or config file
my $GZPROG     = '/usr/bin/gzcat';	# override with %attr gzprog or config file

# in child process
sub _start_task {
    my $me = shift;