view release on metacpan or  search on metacpan

lib/Net/FullAuto/ISets/Amazon/Hadoop_is.pm  view on Meta::CPAN

package Net::FullAuto::ISets::Amazon::Hadoop_is;

### OPEN SOURCE LICENSE - GNU AFFERO PUBLIC LICENSE Version 3.0 #######
#
#    Net::FullAuto - Distributed Workload Automation Software
#    Copyright © 2000-2026  Brian M. Kelly
#
#    This program is free software: you can redistribute it and/or
#    modify it under the terms of the GNU Affero General Public License
#    as published by the Free Software Foundation, either version 3 of
#    the License, or any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but **WITHOUT ANY WARRANTY**; without even the implied warranty
#    of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public
#    License along with this program.  If not, see:
#    <http://www.gnu.org/licenses/agpl.html>.
#
#######################################################################

our $VERSION='0.01';
our $DISPLAY='Apache™ Hadoop®';
our $CONNECT='ssh';
our $defaultInstanceType='t2.micro';

use 5.005;


use strict;
use warnings;

require Exporter;
our @ISA = qw(Exporter);
our @EXPORT = qw($select_hadoop_setup);

use Net::FullAuto::Cloud::fa_amazon;

my $configure_hadoop=sub {

   # https://letsdobigdata.wordpress.com/2014/01/13 \
   # /setting-up-hadoop-multi-node-cluster-on-amazon-ec2-part-1/

   # https://letsdobigdata.wordpress.com/2014/01/13 \
   # /setting-up-hadoop-1-2-1-multi-node-cluster-on-amazon-ec2-part-2/

   # http://hortonworks.com/hadoop-tutorial
   # /how-to-process-data-with-apache-hive/

   # http://www.reddit.com/r/hadoop/comments/xkf4e/ \
   # is_apache_pig_slower_than_streaming_perl/

   my $server_type=$_[0];
   my $cnt=$_[1];
   my $selection=$_[2]||'';
   my $server_host_block=$_[3]||'';
   my $handle=$main::aws->{$server_type}->[$cnt]->[1];
   my ($stdout,$stderr,$exitcode,$error)=('','','','');
   ($stdout,$stderr)=$handle->cmd(
      "sudo apt-get -o Dpkg::Progress=true update 2>&1",'__display__');
   ($stdout,$stderr)=$handle->cmd(
      "sudo add-apt-repository -y ppa:webupd8team/java",'__display__');
   ($stdout,$stderr)=$handle->cmd(
      "sudo apt-get -o Dpkg::Progress=true update 2>&1",'__display__');
   ($stdout,$stderr)=$handle->cmd(
      "echo debconf shared/accepted-oracle-license-v1-1 select true | ".
      "sudo debconf-set-selections");
   ($stdout,$stderr)=$handle->cmd(
      "echo debconf shared/accepted-oracle-license-v1-1 seen true | ".
      "sudo debconf-set-selections");
   ($stdout,$stderr)=$handle->cmd(
      "sudo apt-get -o Dpkg::Progress=true -y --force-yes install ".
      "oracle-java8-installer 2>&1",'__display__');
   my $gcnt=2;my $md5='';my $targz='';my $hadoop_mirror='';
   while (--$gcnt) {
      my $url='http://www.apache.org/dyn/closer.cgi/hadoop/common/';
      my $wcnt=4;
      while (--$wcnt) {
         ($stdout,$stderr)=$handle->cmd("wget -qO- $url");
         my $flag=0;
         foreach my $line (split "\n",$stdout) {
            if ($line=~/We\s+suggest/) {
               $flag=1;
            } elsif ($flag && $line=~/^.*[Hh][Rr][Ee][Ff]=["](.*?)["].*$/) {
               $hadoop_mirror=$1;last;
            }
         } last if $hadoop_mirror=~/^http/;
         sleep 1;
      }
      unless ($hadoop_mirror=~/^http/) {
         print "Can't get Apache™ Hadoop® mirror -> $hadoop_mirror\n\n",
               "after 5 attempts\n";
         &Net::FullAuto::FA_Core::cleanup;
      }
      ($stdout,$stderr)=$handle->cmd("wget -qO- ".
         "${hadoop_mirror}stable");
      my $flag=0;
      foreach my $line (split "\n",$stdout) {
         chomp($line);
         if ($line=~/(?<!src)[.]tar[.]gz["]/) {
            $targz=$line;
            $targz=~s/^.*[Hh][Rr][Ee][Ff]=["](.*?)["].*$/$1/;
            ($stdout,$stderr)=$handle->cmd(
               "sudo wget --random-wait --progress=dot $targz 2>&1");
         } elsif ($line=~/(?<!src)[.]tar[.]gz[.]mds/) {
            $md5=$line;
            $md5=~s/^.*[Hh][Rr][Ee][Ff]=["](.*?)["].*$/$1/;
            ($stdout,$stderr)=$handle->cmd(
               "sudo wget --random-wait --progress=dot $md5 2>&1");
            last unless $stderr;
         }