App-Fetchware
view release on metacpan or search on metacpan
lib/App/Fetchware.pm view on Meta::CPAN
package App::Fetchware;
our $VERSION = '1.016'; # VERSION: generated by DZP::OurPkgVersion
# ABSTRACT: App::Fetchware is Fetchware's API used to make extensions.
###BUGALERT### Uses die instead of croak. croak is the preferred way of throwing
#exceptions in modules. croak says that the caller was the one who caused the
#error not the specific code that actually threw the error.
use strict;
use warnings;
# CPAN modules making Fetchwarefile better.
use File::Spec::Functions qw(catfile splitpath splitdir file_name_is_absolute);
use Path::Class;
use Data::Dumper;
use File::Copy 'cp';
use HTML::TreeBuilder;
use Scalar::Util qw(blessed looks_like_number);
use Digest::SHA;
use Digest::MD5;
#use Crypt::OpenPGP::KeyRing;
#use Crypt::OpenPGP;
use Archive::Tar;
use Archive::Zip qw(:ERROR_CODES :CONSTANTS);
use Cwd 'cwd';
use Sub::Mage;
use URI::Split qw(uri_split uri_join);
use Text::ParseWords 'quotewords';
use File::Temp 'tempfile';
use Term::ReadLine;
use Term::UI;
use App::Fetchware::Util ':UTIL';
use App::Fetchware::Config ':CONFIG';
# Enable Perl 6 knockoffs, and use 5.10.1, because smartmatching and other
# things in 5.10 were changed in 5.10.1+.
use 5.010001;
# Set up Exporter to bring App::Fetchware's API to everyone who use's it
# including fetchware's ability to let you rip into its guts, and customize it
# as you need.
use Exporter qw( import );
# By default fetchware exports its configuration file like subroutines.
#
# These days popular dogma considers it bad to import stuff without being asked
# to do so, but App::Fetchware is meant to be a configuration file that is both
# human readable, and most importantly flexible enough to allow customization.
# This is done by making the configuration file a perl source code file called a
# Fetchwarefile that fetchware simply executes with eval.
our @EXPORT = qw(
program
filter
temp_dir
fetchware_db_path
user
prefix
configure_options
make_options
build_commands
install_commands
uninstall_commands
lookup_url
lookup_method
gpg_keys_url
gpg_sig_url
sha1_url
md5_url
user_agent
verify_method
no_install
verify_failure_ok
user_keyring
stay_root
mirror
config
new
lib/App/Fetchware.pm view on Meta::CPAN
# ones.
$year_or_time =~ s/://; # Make 12:00 1200 for numerical sort.
$timestamp = "9999$month{$month}$day$year_or_time";
# It's a year.
} elsif ($year_or_time =~ /\d\d\d\d/) {
# the $month{} hash access replaces text months with numerical
# ones.
$timestamp = "$year_or_time$month{$month}${day}0000";
}
push @filename_listing, [$filename, $timestamp];
}
return \@filename_listing;
}
sub http_parse_filelist {
my $http_listing = shift;
# Use HTML::TreeBuilder to parse the scalar of html into a tree of tags.
my $tree = HTML::TreeBuilder->new_from_content($http_listing);
my @filename_listing;
my @matching_links = $tree->look_down(
_tag => 'a',
sub {
my $h = shift;
#parse out archive name.
my $link = $h->as_text();
# NOTE: The weird alternations adding .asc, .md5, and .sha.?,
# and also a KEYS file are to allow fetchware new to also use
# this subroutine to parse http file listings to analyze the
# contents of the user's lookup_url. It does not make any sense
# to copy and paste this function or even add a callback argument
# allowing you to change the regex.
if ($link =~
/(\.(tar\.(gz|bz2|xz)|(tgz|tbz2|txz))|(asc|md5|sha.?))|KEYS$/) {
# Should I strip out dirs just to be safe?
my $filename = $link;
# Obtain the tag to the right of the archive link to find the
# timestamp.
if (my $rh = $h->right()) {
my $listing_line;
if (blessed($rh)) {
$listing_line = $rh->as_text();
} else {
$listing_line = $rh;
}
my @fields = split ' ', $listing_line;
###BUGALERT### Internationalization probably breaks this
#datetime parsing? Can a library do it?
# day-month-year time
# $fields[0] $fields[1]
# Normalize format for lookup algorithms .
my ($day, $month, $year) = split /-/, $fields[0];
# Ditch the ':' in the time.
$fields[1] =~ s/://;
# Some dirlistings use string months Aug, Jun, etc...
if (looks_like_number($month)) {
# Strip leading 0 if it exists by converting the
# string with the useless leading 0 into an integer.
# The %num_month hash lookup will add back a leading
# 0 if there was one. This stupid roundabout code is
# to ensure that there always is a leading 0 if the
# number is less than 10 to ensure that all of the
# numbers this hacky datetime parser outputs all
# have the same length so that the numbers can
# easily be compared with each other.
$month = sprintf("%u", $month);
push @filename_listing, [$filename,
"$year$num_month{$month}$day$fields[1]"];
# ...and some use numbers 8, 6, etc....
} else {
push @filename_listing, [$filename,
"$year$month{$month}$day$fields[1]"];
}
} else {
###BUGALERT### Add support for other http servers such as lighttpd, nginx,
#cherokee, starman?, AND use the Server: header to determine which algorithm to
#use.
die <<EOD;
App-Fetchware: run-time error. A hardcoded algorithm to parse HTML directory
listings has failed! Fetchware currently only supports parseing Apache HTML
directory listings. This is a huge limitation, but surprisingly pretty much
everyone who runs a mirror uses apache for http support. This is a bug so
please report it. Also, if you want to try a possible workaround, just use a ftp
mirror instead of a http one, because ftp directory listings are a easy to
parse. See perldoc App::Fetchware.
EOD
}
}
}
);
# Delete the $tree, so perl can garbage collect it.
$tree = $tree->delete;
return \@filename_listing;
}
} # end bare block for %month.
sub file_parse_filelist {
my $file_listing = shift;
for my $file (@$file_listing) {
my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, $atime,$mtime,$ctime,
$blksize,$blocks)
= stat($file) or die <<EOD;
App-Fetchware: Fetchware failed to stat() the file [$file] while trying to parse
your local [file://] lookup_url. The OS error was [$!]. This should not happen,
and is either a bug in fetchware or some sort of race condition.
EOD
lib/App/Fetchware.pm view on Meta::CPAN
=item B<upgrade>
A C<fetchware upgrade [installed program name]> while using a App::Fetchware
Fetchwarefile will simply run the same thing as install all over again, which
ill upgrade your program if a new version is available.
=item B<uninstall>
A C<fetchware uninstall [installed program name]> will cause fetchware to run
the command C<make uninstall>, or run the commands specified by the
C<uninstall_commands> configuration option. C<make uninstall> is only available
from some programs that use AutoTools such as ctags, but apache, for example,
also uses AutoTools, but does not provide a uninstall make target. Apache for
example, therefore, cannot be uninstalled by fetchware automatically.
=item B<upgrade-all>
A C<fetchware upgrade-all> will cause fetchware to run C<fetchware upgrade> for
all installed packages that fetchware is tracking in its internal fetchware
database. This command can be used to have fetchware upgrade all currently
installed programs that fetchware installed.
If you would like C<fetchware upgrade-all> to be run every night automatically
by cron, then just create a file say fetchware with the contents below in it,
and add it to /etc/cron.daily.
#!/bin/sh
# Update all already installed fetchware packages.
fetchware upgrade-all
And if you don't want to run it system wide as root, you can add it to your user
crontab by pasting the snippet below in to your crontab by executing C<crontab -e>.
# Check for updates using fetchware every night at 2:30AM.
# Minute Hour Day of Month Month Day of Week Command
# (0-59) (0-23) (1-31) (1-12 or Jan-Dec) (0-6 or Sun-Sat)
30 2 * * * fetchware upgrade-all
=back
=head1 App::Fetchware'S FETCHWAREFILE CONFIGURATION OPTIONS
App::Fetchware has many configuration options. Most were briefly described in
the section L<MANUALLY CREATING A App::Fetchware FETCHWAREFILE>. All of them are
detailed below.
=head2 program 'Program Name';
C<program> simply gives this Fetchwarefile a name. It is availabe to fetchware
after parsing your Fetchwarefile, and is used to name your Fetchwarefile when
using C<fetchware new>. It is required just like C<lookup_url>, C<mirror>,
perhaps C<filter>, and some method to verify downloads are.
=head2 filter 'perl regex here';
Specifies a Perl regular expression that fetchware uses when it determines what
the latest version of a program is. It simply compares each file in the
directory listing specified in your C<lookup_url> to this regular expression,
and only matching files are allowed to pass through to the next part of
fetchware that looks for source code archives to download.
See L<perlretut> for details on how to use and create Perl regular expressions;
however, actual regex know how is not really needed just paste verbatim text
between the single quotes C<'>. For example, C<filter 'httpd-2.2';> will cause
fetchware to only download Apache 2.2 instead of the version for Windows or
whatever is in the weird httpd-deps-* package.
=head2 temp_dir '/tmp';
C<temp_dir> tells fetchware where to store fetchware's temporary working
directory that it uses to download, verify, unarchive, build, and install your
software. By default it uses your system temp directory, which is whatever
directory L<File::Temp's> tempdir() decides to use, which is whatever
L<File::Spec>'s tmpdir() decides to use.
=head2 fetchware_db_path '~/.fetchwaredb';
C<fetchware_db_path> tells fetchware to use a different directory other
than its default directory to store the installed fetchware package for the
particular fetchware package that this option is specified in your
Fetchwarefile. Fetchware's default is C</var/log/fetchware> on Unix when run as
root, and something like C</home/[username]/.local/share/Perl/dist/fetchware/>
when run nonroot.
This option is B<not> recommended unless you only want to change it for just one
fetchware package, because fetchware also consults the
C<FETCHWARE_DATABASE_PATH> environment variable that you should set in your
shell startup files if you want to change this globally for all of your
fetchware packages. For sh/bash like shells use:
export FETCHWARE_DATABASE_PATH='/your/path/here'
=head2 user 'nobody';
Tells fetchware what user it should drop privileges to. The default is
C<nobody>, but you can specify a different username with this configuration
option if you would like to.
Dropping privileges allows fetchware to avoid downloading files and executing
anything inside the downloaded archive as root. Except of course the commands
needed to install the software, which will still need root to able to write
to system directories. This improves security, because the downloaded software
won't have sytem privileges until after it is verified, providing that what you
downloaded is exactly what the author uploaded.
Note this only works for unix like systems, and is not used on Windows and
other non-unix systems.
Also note, that if you are running fetchware on Unix even if you do not specify
the C<user> configuration option to configure what user you will drop privileges
to, fetchware will still drop privileges using the ubiquitous C<nobody> user.
If you do B<not> want to drop privileges, then you must use the C<stay_root>
configuration option as described below.
=head2 stay_root 'On';
Tells fetchware to B<not> drop privileges. Dropping privileges when run as root
is fetchware's default behavior. It improves security, and allows fetchware to
avoid exposing the root account by downloading files as root.
Do B<not> use this feature unless you are absolutely sure you need it.
=over
=item SECURITY NOTICE
stay_root, when turned on, causes fetchware to not drop privileges when
fetchware looks up, downloads, verifies, and builds your program. Instead,
fetchware will stay root through the entire build cycle, which needlessly
exposes the root account when downloading files from the internet. These files
may come from trusted mirrors, but mirrors can, and do get cracked:
L<http://www.itworld.com/security/322169/piwik-software-installer-rigged-back-door-following-website-compromise?page=0,0>
L<http://www.networkworld.com/news/2012/092612-compromised-sourceforge-mirror-distributes-backdoored-262815.html>
L<http://www.csoonline.com/article/685037/wordpress-warns-server-admins-of-trojans>
L<http://www.computerworld.com/s/article/9233822/Hackers_break_into_two_FreeBSD_Project_servers_using_stolen_SSH_keys>
=back
=head2 lookup_url 'ftp://somedomain.com/some/path
This configuration option specifies a url of a FTP or HTTP or local (file://)
directory listing that fetchware can download, and use to determine what actual
file to download and perhaps also what version of that program to download if
more than one version is available as some mirrors delete old versions and only
keep the latest one.
This url is used for:
=over
=item 1. To determine what the actual download url is for the latest version of this program
=item 2. As the base url to also download a cryptographic signature (ends in .asc) or a SHA-1 or MD5 signature to verify the contents match what the SHA-1 or MD5 checksum is.
=back
You can use the C<mirror> configuration option to specify additional mirrors.
However, those mirrors will only be used to download the large software
archives. Only the lookup_url will be used to download directory listings to
check for new versions, and to download digital signatures or checksums to
verify downloads.
=head2 lookup_method 'timestamp';
Fetchware has two algorithms it uses to determine what version of your program
to download:
=over
=item timestamp
The timestamp algorithm simply uses the mtime (last modification time) that is
availabe in FTP and HTTP directory listings to determine what file in the
directory is the newest. C<timestamp> is also the default option, and is the one
used if C<lookup_method> is not specified.
=item versionstring
Versionstring parses out the version numbers that each downloadable program has,
and uses them to determine the downloadable archive with the highest version
number, which should also be the newest and best version of the archive to use.
=back
lib/App/Fetchware.pm view on Meta::CPAN
install your software or not install your software, but instead prints out the
path of its build directory, so that you can QA test or review the software before
you install it.
=over
=item NOTICE
C<no_install> is a boolean configuration option, which just means its
values are limited to either true or false. True values are C<'True'>, C<'On'>,
C<1>, and false values are C<'False'>, C<'Off'>, and C<0>. All other values are
syntax errors.
=back
=head2 mirror 'somemirror0.com/some/optional/path';
Your Fetchwarefile needs to have at least one mirror specified. Although you can
specify as many as you want to.
This configuration option, unlike all the others, can be specified more than
once. So, for example you could put:
mirror 'somemirror1.com';
mirror 'somemirror2.com';
mirror 'somemirror3.com';
mirror 'somemirror4.com';
mirror 'somemirror5.com';
When fetchware downloads files or directories it will try each one of these
mirrors in order, and only fail if all attempts at all mirrors fail.
If you specify a path in addition to just the hostname, then fetchware will try
to get whatever it wants to download at that alternate path as well.
mirror 'somemirror6./com/alternate/path';
=head1 FURTHER CUSTOMIZING YOUR FETCHWAREFILE
Because fetchware's configuration files, its Fetchwarefiles, are little Perl
programs, you have the full power of Perl at your disposal to customize
fetchware's behavior to match what you need fetchware to do to install your
source code distributions.
Not only can you use arbitrary Perl code in your Fetchwarefile to customize
fetchware for programs that don't follow most FOSS mirroring's unwritten
standards or use a totally different build system, you can also create a
fetchware extension. Creating a fetchware extension even allows you to turn your
extension into a proper CPAN distribution, and upload it to CPAN to share it
with everybody else. See the section below,
L<CREATING A FETCHWARE EXTENSION>, for full details.
=head2 How Fetchware's configuration options are made
Each configuration option is created with L<App::Fetchware::CreateConfigOptions>
This package's import() is a simple code generator that generates configuration
subroutines. These subroutines have the same names as fetchware's configuration
options, because that is exactly what they are. Perl's
L<Prototypes|perlsub/Prototypes> are used in the code that is generated, so
that you can remove the parentheses typically required around each configuration
subroutine. This turns what looks like a function call into what could
believably be considered a configuration file syntax.
These prototypes turn:
lookup_url('http://somemirror.com/some/path');
Into:
lookup_url 'http://somemirror.com/some/path';
Perl's prototypes are not perfect. The single quotes and semicolon are still
required, but the lack of parens instantly makes it look much more like a
configuration file syntax, then an actual programming language.
=head2 The magic of C<use App::Fetchware;>
The real magic power behind turning a Perl program into a configuration file
sytax comes from the C<use App::Fetchware;> line. This line is single handedly
responsible for making this work. This one line imports all of the configuration
subroutines that make up fetchware's configuration file syntax. And this
mechanism is also behind fetchware's extension mechanism. (To use a
App::Fetchware extension, you just C<use> it. Like
C<use App::FetchwareX::HTMLPageSync;>. That's all there is to it. This I<other>
App::Fetchware is responsible for exporting subroutines of the same names as
those that make up App::Fetchware's API. These subroutines are listed in the
section L<FETCHWAREFILE API SUBROUTINES> as well as their helper subroutines.
See the section below L<CREATING A FETCHWARE EXTENSION> for more information on
how to create App::Fetchware extensions.
=head2 So how do I add some custom Perl code to customize my Fetchwarefile?
You use hook() to override one of fetchware's API subroutines. Then when
fetchware goes to call that subroutine, your own subroutine is called in its
place. You can hook() as many of fetchware's API subroutines as you need to.
=over
Remember your replackement subroutine B<must> take the exact same arguments, and
return the same outputs that the standard fetchware API subroutines do!
All of the things these subroutines return are later used as parameters to later
API subroutines, so failing to return a correct value may cause fetchware to
fail.
=back
=head3 hook()
# Inside a Fetchwarefile...
hook lookup => sub {
# Your own custom lookup handler goes here!
};
hook() allows you to replace fetchware's API subroutines with whatever Perl
code reference you want to. But it B<must> take the same arguments that each API
subroutine takes, and provide the same return value. See the section
L<FETCHWAREFILE API SUBROUTINES> for the details of what the API subroutine's
parameters are, and what their return values should be.
hook() should be used sparingly, and only if you really know what you're doing,
( run in 2.042 seconds using v1.01-cache-2.11-cpan-39bf76dae61 )