App-Fetchware
view release on metacpan or search on metacpan
lib/App/Fetchware.pm view on Meta::CPAN
use warnings;
# CPAN modules making Fetchwarefile better.
use File::Spec::Functions qw(catfile splitpath splitdir file_name_is_absolute);
use Path::Class;
use Data::Dumper;
use File::Copy 'cp';
use HTML::TreeBuilder;
use Scalar::Util qw(blessed looks_like_number);
use Digest::SHA;
use Digest::MD5;
#use Crypt::OpenPGP::KeyRing;
#use Crypt::OpenPGP;
use Archive::Tar;
use Archive::Zip qw(:ERROR_CODES :CONSTANTS);
use Cwd 'cwd';
use Sub::Mage;
use URI::Split qw(uri_split uri_join);
use Text::ParseWords 'quotewords';
use File::Temp 'tempfile';
use Term::ReadLine;
use Term::UI;
use App::Fetchware::Util ':UTIL';
use App::Fetchware::Config ':CONFIG';
# Enable Perl 6 knockoffs, and use 5.10.1, because smartmatching and other
# things in 5.10 were changed in 5.10.1+.
use 5.010001;
# Set up Exporter to bring App::Fetchware's API to everyone who use's it
# including fetchware's ability to let you rip into its guts, and customize it
# as you need.
use Exporter qw( import );
# By default fetchware exports its configuration file like subroutines.
#
# These days popular dogma considers it bad to import stuff without being asked
# to do so, but App::Fetchware is meant to be a configuration file that is both
# human readable, and most importantly flexible enough to allow customization.
# This is done by making the configuration file a perl source code file called a
# Fetchwarefile that fetchware simply executes with eval.
our @EXPORT = qw(
program
filter
temp_dir
fetchware_db_path
user
prefix
configure_options
make_options
build_commands
install_commands
uninstall_commands
lookup_url
lookup_method
gpg_keys_url
gpg_sig_url
sha1_url
md5_url
user_agent
verify_method
no_install
verify_failure_ok
user_keyring
stay_root
mirror
config
new
new_install
check_syntax
start
lookup
download
verify
unarchive
build
install
end
uninstall
upgrade
hook
);
# These tags allow you to replace some or all of fetchware's default behavior to
# install unusual software.
our %EXPORT_TAGS = (
# No OVERRIDE_START OVERRIDE_END because start() does *not* use any helper
# subs that could be beneficial to override()rs.
OVERRIDE_NEW => [qw(
extension_name
fetchwarefile_name
opening_message
get_lookup_url
download_lookup_url
get_mirrors
get_verification
get_filter_option
append_to_fetchwarefile
prompt_for_other_options
append_options_to_fetchwarefile
edit_manually
)],
OVERRIDE_NEW_INSTALL => [qw(
ask_to_install_now_to_test_fetchwarefile
)],
OVERRIDE_CHECK_SYNTAX => [qw(
check_config_options
)],
OVERRIDE_LOOKUP => [qw(
get_directory_listing
parse_directory_listing
determine_download_path
ftp_parse_filelist
http_parse_filelist
file_parse_filelist
lookup_by_timestamp
lookup_by_versionstring
lookup_determine_downloadpath
)],
OVERRIDE_DOWNLOAD => [qw(
determine_package_path
)],
OVERRIDE_VERIFY => [qw(
gpg_verify
sha1_verify
md5_verify
digest_verify
)],
OVERRIDE_UNARCHIVE => [qw(
check_archive_files
list_files
list_files_tar
list_files_zip
unarchive_package
unarchive_tar
unarchive_zip
)],
OVERRIDE_BUILD => [qw(
run_star_commands
run_configure
)],
OVERRIDE_INSTALL => [qw(
chdir_unless_already_at_path
)],
OVERRIDE_UNINSTALL => [qw()],
OVERRIDE_UPGRADE => [qw()],
);
# OVERRIDE_ALL is simply all other tags combined.
@{$EXPORT_TAGS{OVERRIDE_ALL}} = map {@{$_}} values %EXPORT_TAGS;
# *All* entries in @EXPORT_TAGS must also be in @EXPORT_OK.
our @EXPORT_OK = @{$EXPORT_TAGS{OVERRIDE_ALL}};
###BUGALERT### Add strict argument checking to App::Fetchware's API subroutines
#to check for not being called correctly to aid extension debugging.
###BUGALERT### Recommend installing http://gpg4win.org if you use fetchware on
# Windows so you have gpg support.
# _make_config_sub() is an internal subroutine that only App::Fetchware and
# App::Fetchware::CreateConfigOptions should use. Use
# App::Fetchware::CreateConfigOptions to create any configuration option
# subroutines that you want your fetchware extensions to have.
#=head2 _make_config_sub()
#
# _make_config_sub($name, $one_or_many_values)
#
#A function factory that builds many functions that are the exact same, but have
#different names. It supports three types of functions determined by
#_make_config_sub()'s second parameter. It's first parameter is the name of that
#function. This is the subroutine that builds all of Fetchwarefile's
#configuration subroutines such as lookupurl, mirror, fetchware, etc....
#
#=over
#=item LIMITATION
#
#_make_config_sub() creates subroutines that have prototypes, but in order for
#perl to honor those prototypes perl B<must> know about them at compile-time;
lib/App/Fetchware.pm view on Meta::CPAN
#will die with an error message.
#
#Function created with C<$CONFIG{$name} = $value;> inside the generated function that
#is named $name.
#
#=item * 'ONEARRREF'
#
#Generates a function with the name of _make_config_sub()'s first parameter that
#can B<only> be called one time per Fetchwarefile. And just like C<'ONE'> above
#if called more than once it will throw an exception. However, C<'ONEARRREF'> can
#be called with a list of values just like C<'MANY'> can, but it can still only
#be called once like C<'ONE'>.
#
#=item * 'MANY'
#
#Generates a function with the name of _make_config_sub()'s first parameter that
#can be called more than just once. This option is only used by fetchware's
#C<mirror()> API call.
#
#Function created with C<push @{$CONFIG{$name}}, $value;> inside the generated function that
#is named $name.
#
#=item * 'BOOLEAN'
#
#Generates a function with the name of _make_config_sub()'s first parameter that
#can be called only once just like 'ONE' can be, but it also only support true or
#false values. What is true and false is the same as in perl, with the exception
#that /false/i and /off/i are also false.
#
#Function created the same way as 'ONE''s are, but with /false/i and /off/i
#mutated into a Perl accepted false value (they're turned into zeros.).
#
#=back
#
#=back
#
#All API subroutines fetchware provides to Fetchwarefile's are generated by
#_make_config_sub() except for fetchware() and override().
#
#=cut
my @api_functions = (
[ program => 'ONE' ],
[ filter => 'ONE' ],
[ temp_dir => 'ONE' ],
[ fetchware_db_path => 'ONE' ],
[ user => 'ONE' ],
[ prefix => 'ONE' ],
[ configure_options=> 'ONEARRREF' ],
[ make_options => 'ONEARRREF' ],
[ build_commands => 'ONEARRREF' ],
[ install_commands => 'ONEARRREF' ],
[ uninstall_commands => 'ONEARRREF' ],
[ lookup_url => 'ONE' ],
[ lookup_method => 'ONE' ],
[ gpg_keys_url => 'ONE' ],
[ gpg_sig_url => 'ONE' ],
[ sha1_url => 'ONE' ],
[ md5_url => 'ONE' ],
[ user_agent => 'ONE' ],
[ verify_method => 'ONE' ],
[ mirror => 'MANY' ],
[ no_install => 'BOOLEAN' ],
[ verify_failure_ok => 'BOOLEAN' ],
[ stay_root => 'BOOLEAN' ],
[ user_keyring => 'BOOLEAN' ],
);
# Loop over the list of options needed by _make_config_sub() to generated the
# needed API functions for Fetchwarefile.
for my $api_function (@api_functions) {
_make_config_sub(@{$api_function});
}
sub _make_config_sub {
my ($name, $one_or_many_values, $callers_package) = @_;
# Obtain caller's package name, so that the new configuration subroutine
# can be created in the caller's package instead of our own. Use the
# specifed $callers_package if the caller specified one. This allows
# create_config_options() to reuse _make_config_sub() by passing in its
# caller to _make_config_sub().
my $package = $callers_package // caller;
die <<EOD unless defined $name;
App-Fetchware: internal syntax error: _make_config_sub() was called without a
name. It must receive a name parameter as its first paramter. See perldoc
App::Fetchware.
EOD
unless ($one_or_many_values eq 'ONE'
or $one_or_many_values eq 'ONEARRREF',
or $one_or_many_values eq 'MANY'
or $one_or_many_values eq 'BOOLEAN') {
die <<EOD;
App-Fetchware: internal syntax error: _make_config_sub() was called without a
one_or_many_values parameter as its second parameter. Or the parameter it was
called with was invalid. Only 'ONE', 'MANY', and 'BOOLEAN' are acceptable
values. See perldoc App::Fetchware.
EOD
}
if ($one_or_many_values eq 'ONE') {
my $eval = <<'EOE';
package $package;
sub $name (@) {
my $value = shift;
die <<EOD if defined config('$name');
App-Fetchware: internal syntax error: $name was called more than once in this
Fetchwarefile. Currently only mirror supports being used more than once in a
Fetchwarefile, but you have used $name more than once. Please remove all calls
to $name but one. See perldoc App::Fetchware.
EOD
unless (@_) {
config('$name', $value);
} else {
die <<EOD;
App-Fetchware: internal syntax error. $name was called with more than one
option. $name only supports just one option such as '$name 'option';'. It does
not support more than one option such as '$name 'option', 'another option';'.
Please chose one option not both, or combine both into one option. See perldoc
lib/App/Fetchware.pm view on Meta::CPAN
fetchware_database_path specifies an alternate path for fetchware to use to
store the fetchware package that 'fetchware install' creates, and that
'fetchware upgrade' uses to upgrade this fetchware package.
EOA
prefix => <<EOA,
prefix specifies what base path your software will be installed under. This
only works for software that uses GNU AutoTools to configure itself, it uses
./configure.
EOA
configure_options => <<EOA,
configure_options specifes what options fetchware should pass to ./configure
when it configures your software. This option only works for software that
uses GNU AutoTools.
EOA
make_options => <<EOA,
make_options specifes what options fetchware should pass to make when make is
run to build and install your software.
EOA
build_commands => <<EOA,
build_commands specifies what commands fetchware should execute to build your
software.
EOA
install_commands => <<EOA,
install_commands specifies what commands fetchware should execute to install
your software.
EOA
uninstall_commands => <<EOA,
uninstall_commands specifies what commands fetchware should execute to uninstall
your software.
EOA
lookup_url => <<EOA,
lookup_url specifes the url that fetchware uses to determine what what
versions of your program are available. It should point to a directory listing
instead of a specific file.
EOA
lookup_method => <<EOA,
lookup_method specifies how fetchware determines what version of your program
to install. The default is the 'timestamp' algorithm, and then to try the
'versionstring' algorithm if 'timestamp' fails. lookup_method specifies which
one you would like to use. Only the strings 'timestamp' and 'versionstring'
are allowed options.
EOA
gpg_keys_url => <<EOA,
gpg_keys_url specifies the url that fetchware will use to download the author's
KEYS file that it uses for gpg verification.
EOA
gpg_sig_url => <<EOA,
gpg_sig_url specifies the url that fetchware uses to download digital
signatures of this program. They're files that usually end .asc.
EOA
sha1_url => <<EOA,
sha1_url specfies the url that fetchware uses to download sha1sum files of
this program. This url should be the program's main download site instead of a
mirror, because a hacked mirror could alter the sha1sum on that mirror.
EOA
md5_url => <<EOA,
md5_url specfies the url that fetchware uses to download md5sum files of
this program. This url should be the program's main download site instead of a
mirror, because a hacked mirror could alter the md5sum on that mirror.
EOA
verify_method => <<EOA,
verify_method specifes a specific method that fetchware should use to verify
your program. This method can be 'gpg', 'sha1', or 'md5'.
EOA
no_install => <<EOA,
no_install specifies that this software should not be installed. Instead, the
install step is skipped, and fetchware prints to STDOUT where it downloaded,
verified, and built your program. no_install must be a true or false value.
EOA
verify_failure_ok => <<EOA,
verify_failure_ok specifies that fetchware should not stop installing your
software and terminate with an error message if fetchware fails to verify your
software. You should never set this to true. Doing so could cause fetchware to
install software that may have been compromised, or had malware inserted into
it. Never use this option unless the author or maintainer of this program does
not gpg sign or checksum his software.
EOA
user_keyring => <<EOA,
users_keyring if enabled causes fetchware to use the user's own gpg keyring
instead of fetchware's own keyring.
EOA
mirror => <<EOA
The mirror configuration option provides fetchware with alternate servers to
try to download this program from. This option is used when the server
specified in the url options in this file is unavailable or times out.
EOA
}
);
###INSANEFEATUREENHANCEMENT### Prompt for name of program, and do a fuzzy
#search on CPAN for that program under
#App::Fetchware::FetchwarefileX::UpCasedProgName. Consider using the meta
#CPAN API. And if it exists ask user if they wanna use that one instead of
#autogening one.
#
#Perhaps create a 'fetchwarefile' command to download and look at
#fetchwarefiles from CPAN, and then install them, and/or perhaps upload
#them pausing to ask for the user's PAUSE credentials!!!!!!!!!
extension_name(__PACKAGE__);
my $opening_message = <<EOM;
Fetchware's new command is reasonably sophisticated, and is smart enough to
determine based on the lookup_url you provide if it can autogenerate a
Fetchwarefile for you. If Fetchware cannot, then it will ask you more
questions regarding the information it requires to be able to build a
installable fetchware package for you. After that, fetchware will ask you if
you would like to edit the Fetchwarefile, fetchware has created for you in an
editor. If you say yes, fetchware will open a editor for you, but if you say
no, fetchware will skip the custom editing. Next, fetchware will create a test
Fetchwarefile for you, and ask you if you would like to test it by trying to
install it now. If you say yes, fetchware will install it, and if you say no,
then fetchware will print the location of the Fetchwarefile it created for
you to later use to install your application.
EOM
opening_message($opening_message);
# Ask user for name of program unless the user provided one at command
# line such as fetchware new <programname>.
$program_name = fetchwarefile_name(program => $program_name);
vmsg "Determined name of your program to be [$program_name]";
$fetchwarefile->config_options(program => $program_name);
vmsg "Appended program [$program_name] configuration option to Fetchwarefile";
my $lookup_url = get_lookup_url($term);
vmsg "Asked user for lookup_url [$lookup_url] from user.";
$fetchwarefile->config_options(lookup_url => $lookup_url);
vmsg "Appended lookup_url [$lookup_url] configuration option to Fetchwarefile";
vmsg "Downloaded lookup_url [$lookup_url]";
my $filename_listing = download_lookup_url($term, $lookup_url);
vmsg "Downloaded lookup_url's directory listing";
vmsg Dumper($filename_listing);
my $mirrors_hashref = get_mirrors($term, $filename_listing);
vmsg "Added mirrors to your Fetchwarefile.";
vmsg Dumper($mirrors_hashref);
my $verify_hashref = get_verification($term, $filename_listing, $lookup_url);
vmsg "Added verification settings to Fetchwarefile.";
vmsg Dumper($verify_hashref);
my $filter_hashref = get_filter_option($term, $filename_listing);
vmsg "Added [$filter_hashref->{filter}] filter setting to Fetchwarefile.";
$fetchwarefile->config_options(
%$mirrors_hashref,
%$verify_hashref,
%$filter_hashref
);
###BUGALERT### Ask to parrallelize make with make_options???
###BUGALERT### Verify prefix is writable by current user, who will
#presumably be the user who will install the package now and later.
###BUGALERT### Ask user for a prefix if their running nonroot???
vmsg 'Prompting for other options that may be needed.';
my $other_options_hashref = prompt_for_other_options($term,
temp_dir => {
prompt => <<EOP,
What temp_dir configuration option would you like?
EOP
print_me => <<EOP
temp_dir is the directory where fetchware creates a temporary directory that
stores all of the temporary files it creates while it is building your software.
The default directory is /tmp on Unix systems and C:\\temp on Windows systems.
EOP
},
user => {
prompt => <<EOP,
What user configuration option would you like?
EOP
print_me => <<EOP
user specifies what user fetchware will drop priveleges to on Unix systems
capable of doing so. This allows fetchware to download files from the internet
with user priveleges, and not do anything as the administrative root user until
after the downloaded software package has been verified as exactly the same as
the author of the package intended it to be. If you use this option, the only
thing that is run as root is 'make install' or whatever this package's
install_commands configuratio option is.
EOP
},
prefix => {
prompt => <<EOP,
What prefix configuration option would you like?
EOP
print_me => <<EOP
prefix specifies the base path that will be used to install this software. The
default is /usr/local, which is acceptable for most unix users. Please note that
this difective only works for software packages that use GNU AutoTools, software
that uses ./configure --prefix=<your prefix will go here> to change the prefix.
EOP
},
configure_options => {
prompt => <<EOP,
What configure_options configuration option would you like?
EOP
print_me => <<EOP
configure_options specifies what options fetchware should add when it configures
this software package for you. A list of possible options can be obtained by
running unarchiving the software package that corresponds to this Fetchwarefile,
and running the command './configure --help'. These options vary from software
package to software package. Please note that this option only works for GNU
AutoTools based software distributions, ones that use ./configure to configure
the software.
EOP
},
make_options => {
prompt => <<EOP,
lib/App/Fetchware.pm view on Meta::CPAN
},
install_commands => {
prompt => <<EOP,
What install_commands configuration option would you like?
EOP
print_me => <<EOP
install_commands specifies what commands fetchware will run to install your
software package. Fetchware's default is simply 'make install', which is good
for most programs. If you're software package uses something other than
fetchware's default of GNU AutoTools, then you may need to change this
configuration option to specify what you would like instead. Specify multiple
build commands in single quotes with a comma between them:
'make test', 'make install'
EOP
},
uninstall_commands => {
prompt => <<EOP,
What uninstall_commands configuration option would you like?
EOP
print_me => <<EOP,
uninstall_commands specifes what commands fetchware will run to uninstall your
software pacakge. The default is 'make uninstall,' which works for some GNU
AutoTools packages, but not all. If your software package does not have a 'make
uninstall' make target, but it has some other command that can uninstall it,
then please specify it using uninstall_commands so fetchware can uninstall it.
EOP
},
lookup_method => {
prompt => <<EOP,
What lookup_method configuration option would you like?
EOP
print_me => <<EOP
lookup_method specifies what how fetchware determines if a new version of your
software package is available. The available algorithms are 'timstamp' and
'versionstring'. 'timestamp' uses the timestamp listed in the FTP or HTTP
listing, and uses the software package that is the newest by filesystem
timestamp. The 'versionstring' algorithm uses the filename of the files in the
FTP or HTTP listing. It parses out the version information, sorts it highest to
lowest, and then picks the highest version of your software package. The default
is try 'timestamp' and if that doesn't work, then try 'versionstring'.
EOP
},
gpg_keys_url => {
prompt => <<EOP,
What gpg_keys_url configuration option would you like?
EOP
print_me => <<EOP
gpg_keys_url specifies a url similar to lookup_url in that it should specify a
directory instead a specific file. It is used to download KEYS files, which
contain your program author's gpg keys to import into gpg.
EOP
},
gpg_sig_url => {
prompt => <<EOP,
What gpg_sig_url configuration option would you like?
EOP
print_me => <<EOP
gpg_sig_url specifies a url similar to lookup_url in that it should specify a
directory instead a specific file. It is used to download gpg signatures to
verify your software package.
EOP
},
sha1_url => {
prompt => <<EOP,
What sha1_url configuration option would you like?
EOP
print_me => <<EOP
sha1_url specifies a url similar to lookup_url in that it should specify a
directory instead of a specific file. It is separate from lookup_url, because
you should download software from mirrors, but checksums from the original
vendor's server, because checksums are easily replaced on a mirror by a hacker
if the mirror gets hacked.
EOP
},
md5_url => {
prompt => <<EOP,
What md5_url configuration option would you like?
EOP
print_me => <<EOP,
md5_url specifies a url similar to lookup_url in that it should specify a
directory instead of a specific file. It is separate from lookup_url, because
you should download software from mirrors, but checksums from the original
vendor's server, because checksums are easily replaced on a mirror by a hacker
if the mirror gets hacked.
EOP
},
verify_method => {
prompt => <<EOP,
What verify_method configuration option would you like?
EOP
print_me => <<EOP,
verify_method specifies what method of verification fetchware should use to
ensure the software you have downloaded has not been tampered with. The default
is to try gpg verification, then sha1, and then finally md5, and if they all
fail an error message is printed and fetchware exits, because if your software
package cannot be verified, then it should not be installed. This configuration
option allows you to remove the warnings by specifying a specific way of
verifying your software has not been tampered with. To disable verification set
the 'verify_failure_ok' configuration option to true.
EOP
},
###BUGALERT### replace no_install config su with a command line option that
#would be the opposite of --force???
# Nah! Leave it! Just create a command line option for it too!
no_install => {
prompt => <<EOP,
Would you like to enable the no_install configuration option?
EOP
###BUGALERT### no_install is not currently implemented properly!!!
print_me => <<EOP
no_install is a true or false option, whoose acceptable values include 1
or 0, true or falue, On or Off. It's default value is false, but if you enable
it, then fetchware will not install your software package, and instead it will
simply download, verify, and build it. And then it will print out the full path
of the directory it built your software package in.
EOP
###BUGALERT### Add support for a check regex, so that I can ensure
#that what the user enters will be either true or false!!!
},
verify_failure_ok => {
prompt => <<EOP,
Would you like to enable the verify_failure_ok configuration option?
EOP
print_me => <<EOP
verify_failure_ok is a true or false option, whoose acceptable values include 1
or 0, true or falue, On or Off. It's default value is false, but if you enable
it, then fetchware will not print an error message and exit if verification
fails for your software package. Please note that you should never use this
option, because it makes it possible for fetchware to install source code that
may have been tampered with.
EOP
},
users_keyring => {
prompt => <<EOP,
Would you like to enable users_keyring configuration option?
EOP
print_me => <<EOP
users_keyring when enabled causes fetchware to use the user who calls
fetchware's gpg keyring instead of fetchware's own gpg keyring. Useful for
source code distributions that do not provide an easily accessible KEYS file.
Just remember to import the author's keys into your gpg keyring with gpg
--import.
EOP
},
);
vmsg 'User entered the following options.';
vmsg Dumper($other_options_hashref);
# Append all other options to the Fetchwarefile.
$fetchwarefile->config_options(%$other_options_hashref);
vmsg 'Appended all other options listed above to Fetchwarefile.';
my $edited_fetchwarefile = edit_manually($term, $fetchwarefile);
vmsg <<EOM;
Asked user if they would like to edit their generated Fetchwarefile manually.
EOM
# Generate Fetchwarefile.
if (blessed($edited_fetchwarefile)
and
$edited_fetchwarefile->isa('App::Fetchware::Fetchwarefile')) {
# If edit_manually() did not modify the Fetchwarefile, then generate
# it.
$fetchwarefile = $fetchwarefile->generate();
} else {
# If edit_manually() modified the Fetchwarefile, then do not
# generate it, and replace the Fetchwarefile object with the new
# string that represents the user's edited Fetchwarefile.
$fetchwarefile = $edited_fetchwarefile;
}
# Whatever variables the new() API subroutine returns are written via a pipe
# back to the parent, and then the parent reads the variables back, and
# makes then available to new_install(), back in the parent, as arguments.
return $program_name, $fetchwarefile;
}
sub extension_name {
# Use a state variable to keep $extension_name's value between calls.
state $extension_name;
lib/App/Fetchware.pm view on Meta::CPAN
should point to.
The mirror should be a URL in standard browser format such as [ftp://a.mirror/].
FTP, HTTP, and local file:// mirrors are supported. All other formats are not
supported.
EOP
prompt => 'Please enter the URL of your mirror: ',
allow => qr!^(ftp|http|file)://!,
);
# Append mirror to $fetchwarefile.
push @mirrors, $mirror;
if (
$term->ask_yn(
print_me => <<EOP,
In addition to the one required mirror that you must define in order for
fetchware to function properly, you may specify additonal mirros that fetchware
will use if the mirror you've already specified is unreachable or download
attempts using that mirror fail.
EOP
prompt => 'Would you like to add any additional mirrors? ',
default => 'n',
)
) {
# Prompt for first mirror outside loop, because if you just hit enter or
# type done, then the above text will be appended to your fetchwarefile,
# but you'll be able to skip actually adding a mirror.
my $first_mirror = $term->get_reply(
prompt => 'Type in URL of mirror or done to continue: ',
allow => qr!^(ftp|http|file)://!,
);
# Append $first_mirror to $fetchwarefile.
push @mirrors, $first_mirror;
while (1) {
my $mirror_or_done = $term->get_reply(
prompt => 'Type in URL of mirror or done to continue: ',
default => 'done',
allow => qr!(^(ftp|http|file)://)|done!,
);
if ($mirror_or_done eq 'done') {
last;
} else {
# Append $mirror_or_done to $fetchwarefile.
push @mirrors, $mirror_or_done;
}
}
}
return {mirror => \@mirrors};
}
sub get_verification {
my ($term, $filename_listing, $lookup_url) = @_;
my %options;
my %available_verify_methods;
# Determine what types of verification are available.
for my $file_and_timestamp (@$filename_listing) {
if ($file_and_timestamp->[0] =~ /\.(asc|sig|sign)$/) {
$available_verify_methods{gpg}++;
} elsif ($file_and_timestamp->[0] =~ /\.sha1?$/) {
$available_verify_methods{sha1}++;
} elsif ($file_and_timestamp->[0] =~ /\.md5$/) {
$available_verify_methods{md5}++;
}
}
my $verify_configed_flag = 0;
#If gpg is available prefer it over the others.
if (exists $available_verify_methods{gpg}
and defined $available_verify_methods{gpg}
and $available_verify_methods{gpg} > 0
) {
msg <<EOM;
gpg digital signatures found. Using gpg verification.
EOM
$options{verify_method} = 'gpg';
# Search for a KEYS file to use to import the author's keys.
if (grep {$_->[0] eq 'KEYS'} @$filename_listing) {
msg <<EOM;
KEYS file found using lookup_url. Adding gpg_keys_url to your Fetchwarefile.
EOM
# Add 'KEYS' or '/KEYS' to $lookup_url's path.
my ($scheme, $auth, $path, $query, $fragment) =
uri_split($lookup_url);
$path = catfile($path, 'KEYS');
$lookup_url = uri_join($scheme, $auth, $path, $query, $fragment);
$options{gpg_keys_url} = $lookup_url;
$verify_configed_flag++;
} else {
msg <<EOM;
KEYS file *not* found!
EOM
# Since autoconfiguration of KEYS failed, try asking the user if
# they would like to import the author's key themselves into their
# own keyring and have fetchware use that.
if (
$term->ask_yn(prompt =>
q{Would you like to import the author's key yourself after fetchware completes? },
default => 'n',
print_me => <<EOP,
Automatic KEYS file discovery failed. Fetchware needs the author's keys to
download and import into its own keyring, or you may specify the option
user_keyring, which if true will cause fetchware to use the user who runs
fetchware's keyring instead of fetchware's own keyring. But you, the user, needs
to import the author's keys into your own gpg keyring. You can do this now in a
separate shell, or after you finish configuring this Fetchwarefile. Just run the
command [gpg --import <name of file>].
EOP
)
) {
$options{user_keyring} = 'On';
$verify_configed_flag++;
}
# And if the user does not want to, then fallback to sha1 and/or md5
# if they're defined, which is done below.
}
}
# Only try sha1 and md5 if gpg failed.
unless ($verify_configed_flag == 1) {
if (exists $available_verify_methods{sha1}
and defined $available_verify_methods{sha1}
and $available_verify_methods{sha1} > 0
) {
msg <<EOM;
SHA1 checksums found. Using SHA1 verification.
EOM
$options{verify_method} = 'sha1';
} elsif (exists $available_verify_methods{md5}
and defined $available_verify_methods{md5}
and $available_verify_methods{md5} > 0
) {
msg <<EOM;
MD5 checksums found. Using MD5 verification.
EOM
$options{verify_method} = 'md5';
} else {
# Print a huge long nasty warning even include links to news stories
# of mirrors actually getting hacked and serving malware, which
# would be detected and prevented with proper verification enabled.
# Ask user if they would like to continue installing fetchware even if
# verification fails, and then enable the verify_failure_ok option.
if (
$term->ask_yn(prompt => <<EOP,
Would you like fetchware to ignore the fact that it is unable to verify the
authenticity of any downloads it makes? Are you ok with possibly downloading
viruses, worms, rootkits, or any other malware, and installing it possibly even
as root?
EOP
default => 'n',
print_me => <<EOP,
Automatic verification of your fetchware package has failed! Fetchware is
capable of ignoring the error, and installing software packages anyway using its
verify_failure_ok configuration option. However, installing software packages
without verifying that they have not been tampered with could allow hackers to
potentially install malware onto your computer. Don't think this is *not*
possible or do you think its extremely unlikely? Well, it's actually
surprisingly common:
1. http://arstechnica.com/security/2012/09/questions-abound-as-malicious-phpmyadmin-backdoor-found-on-sourceforge-site/
Discusses how a mirror for sourceforge was hacked, and the phpMyAdmin
software package on that mirror was modified to spread malware.
2. http://www.geek.com/news/major-open-source-code-repository-hacked-for-months-says-fsf-551344/
Discusses how FSF's gnu.org ftp download site was hacked.
3. http://arstechnica.com/security/2012/11/malicious-code-added-to-open-source-piwik-following-website-compromise/
Discusses how Piwiki's wordpress software was hacked, and downloads of
Piwiki had malicious code inserted into them.
4. http://www.theregister.co.uk/2011/03/21/php_server_hacked/
Discusses how php's wiki.php.org server was hacked yielding credentials to
php's source code repository.
Download mirrors *do* get hacked. Do not make the mistake, and think that it is
not possible. It is possible, and it does happen, so please properly configure
your Fetchwarefile to enable fetchware to verify that the downloaded software is
the same what the author uploaded.
EOP
)
) {
# If the user is ok with not properly verifying downloads, then
# ignore the failure, and install anyway.
$options{verify_failure_ok} = 'On';
} else {
# Otherwise, throw an exception.
die <<EOD;
fetchware: Fetchware *must* be able to verify any software packages that it
downloads. The Fetchwarefile that you were creating could not do this, because
you failed to specify how fetchware can verify its downloads. Please rerun
fetchware new again, and this time be sure to specify a gpg_keys_url, specify
user_keyring to use your own gpg keyring, or answer yes to the question
regarding adding verify_failure_ok to your Fetchwarefile to make failing
verificaton acceptable to fetchware.
EOD
}
}
}
return \%options;
}
sub get_filter_option {
my $term = shift;
# $filename_listing is an array of [$filename, $timestamp] arrays.
my $filename_listing = shift;
msg <<EOS;
Analyzing the lookup_url you provided to determine if fetchware can use it to
successfully determine when new versions of your software are released.
EOS
my $filter;
if (grep {$_->[0] =~ /^(CURRENT|LATEST)[_-]IS[_-].+/} @$filename_listing) {
# There is only one version in the lookup_url directory listing, so
# I do not need a filter option.
msg <<EOS;
* The lookup_url you gave fetchware includes a CURRENT_IS or a LATEST_IS file
that tells fetchware and regular users what the latest version is. Because of
this we can be reasonable sure that a filter option is not needed, so I'll skip
asking for one. You can provide one later if you need to provide one, when
fetchware prompts you for any custom options you may want to use.
EOS
} else {
# There is a CURRENT_IS_<ver_num> or LATEST_IS_<ver_num> file that tells
# you what the latest version is.
###BUGALERT### Why is this line in both sections of the if statement??? Inside
#this else block means that a CURRENT_IS or LATEST-IS was *not* found??? Fix
#this!!!!!!
msg <<EOS;
* The directory listing of your lookup_url has a CURRENT_IS_<ver_num> or
LATEST_IS_<ver_num> file that specifies the latest version, which means that
your program's corresponding Fetchwarefile does not need a filter option. If you
still would like to provide one, you can do so later on, when fetchware allows
you to define any additional configuration options.
EOS
my $what_a_filter_is = <<EOA;
Fetchware needs you to provide a filter option, which is a pattern that fetchware
compares each file in the directory listing of your lookup_url to to determine
which version of your program to install.
Directories will have other junk files in them or even completely different
programs that could confuse fetchware, and even potentially cause it to install
a different program. Therefore, you should also add the program name to the
begining of your filter. For example if you program is apache, then your filter
should include the name of apache on mirror sites, which is actually:
httpd
For example, Apache's lookup_url has three versions in the same lookup_url
directory listing. These are 2.4, 2.2, and 2.0. Without the filter option
fetchware would choose the highest, which would be 2.4, which is the latest
version. However, you may want to stick with the older and perhaps more stable
lib/App/Fetchware.pm view on Meta::CPAN
##DELME## if (config('lookup_url') =~ m!^file://!) {
##DELME## # Must prepend scheme, so that download() knows how to retrieve this
##DELME## # file with download_file(), which requires a URL that must begin
##DELME## # with a scheme, and file:// is the scheme for local files.
##DELME## $fl->[0] =~ s/"file://$fl->[0]";
}
die <<EOD;
App-Fetchware: run-time error. Fetchware failed to determine what URL it should
use to download your software. This URL is based on the lookup_url you
specified. See perldoc App::Fetchware.
EOD
}
sub download {
my ($temp_dir, $download_path) = @_;
# Ensure we're passed just a path, and *not* a full URL.
die <<EOD if $download_path =~ m!(?:http|ftp|file)://!;
App-Fetchware: download() has been passed a full URL *not* only a path.
download() should only be called with a path never a full URL. The URL you
specified was [$download_path]
EOD
vmsg <<EOM;
Using [$download_path] as basis for determined our download_url using the user
supplied mirrors.
EOM
msg "Downloading from url [$download_path] to temp dir [$temp_dir]";
my $downloaded_file_path = download_file(PATH => $download_path);
vmsg "Downloaded file to [$downloaded_file_path]";
my $package_path = determine_package_path($temp_dir, $downloaded_file_path);
msg "Determined package path to be [$package_path]";
return $package_path;
}
sub determine_package_path {
my ($tempdir, $filename) = @_;
# return $package_path, which stores the full path of where the file
# HTTP::Tiny downloaded.
###BUGALERT### $tempdir is no longer used, so remove it from
#determine_package_path() and probably download() too.
return catfile(cwd(), $filename)
}
sub verify {
my ($download_path, $package_path) = @_;
msg "Verifying the downloaded package [$package_path]";
my $retval;
unless (defined(config('verify_method'))) {
# if gpg fails try
# sha and if it fails try
# md5 and if it fails die
msg 'Trying to use gpg to cyptographically verify downloaded package.';
my ($gpg_err, $sha_err, $md5_err);
eval {$retval = gpg_verify($download_path)};
$gpg_err = $@;
if ($gpg_err) {
msg <<EOM;
Cyptographic verification using gpg failed!
GPG verification error [
$@
]
EOM
warn $gpg_err;
}
if (! $retval or $gpg_err) {
msg <<EOM;
Trying SHA1 verification of downloaded package.
EOM
eval {$retval = sha1_verify($download_path, $package_path)};
$sha_err = $@;
if ($sha_err) {
msg <<EOM;
SHA1 verification failed!
SHA1 verificaton error [
$@
]
EOM
warn $sha_err;
}
if (! $retval or $sha_err) {
msg <<EOM;
Trying MD5 verification of downloaded package.
EOM
eval {$retval = md5_verify($download_path, $package_path)};
$md5_err = $@;
if ($md5_err) {
msg <<EOM;
MD5 verification failed!
MD5 verificaton error [
$@
]
EOM
warn $md5_err;
}
}
if (! $retval or $md5_err) {
die <<EOD unless config('verify_failure_ok');
App-Fetchware: run-time error. Fetchware failed to verify your downloaded
software package. You can rerun fetchware with the --force option or add
[verify_failure_ok 'True';] to your Fetchwarefile. See the section VERIFICATION
FAILED in perldoc fetchware.
EOD
}
if (config('verify_failure_ok')) {
warn <<EOW;
App-Fetchware: run-time warning. Fetchware failed to verify the integrity of you
downloaded file [$package_path]. This is ok, because you asked Fetchware to
ignore its errors when it tries to verify the integrity of your downloaded file.
You can also ignore the errors Fetchware printed out abover where it tried to
verify your downloaded file. See perldoc App::Fetchware.
EOW
vmsg <<EOM;
Verification Failed! But you asked to ignore verification failures, so this
failure is not fatal.
EOM
return 'warned due to verify_failure_ok'
}
}
} elsif (config('verify_method') =~ /gpg/i) {
vmsg <<EOM;
You selected gpg cryptographic verification. Verifying now.
EOM
###BUGALERT### Should trap the exception {gpg,sha1,md5}_verify()
#throws, and then add that error to the one here, otherwise the
#error message here is never seen.
gpg_verify($download_path)
or die <<EOD unless config('verify_failure_ok');
App-Fetchware: run-time error. You asked fetchware to only try to verify your
package with gpg or openpgp, but they both failed. See the warning above for
their error message. See perldoc App::Fetchware.
EOD
} elsif (config('verify_method') =~ /sha1?/i) {
vmsg <<EOM;
You selected SHA1 checksum verification. Verifying now.
EOM
sha1_verify($download_path, $package_path)
or die <<EOD unless config('verify_failure_ok');
App-Fetchware: run-time error. You asked fetchware to only try to verify your
package with sha, but it failed. See the warning above for their error message.
See perldoc App::Fetchware.
EOD
} elsif (config('verify_method') =~ /md5/i) {
vmsg <<EOM;
You selected MD5 checksum verification. Verifying now.
EOM
md5_verify($download_path, $package_path)
or die <<EOD unless config('verify_failure_ok');
App-Fetchware: run-time error. You asked fetchware to only try to verify your
package with md5, but it failed. See the warning above for their error message.
See perldoc App::Fetchware.
EOD
} else {
die <<EOD;
App-Fetchware: run-time error. Your fetchware file specified a wrong
verify_method option. The only supported types are 'gpg', 'sha', 'md5', but you
specified [@{[config('verify_method')]}]. See perldoc App::Fetchware.
EOD
}
msg 'Verification succeeded.';
}
sub gpg_verify {
my $download_path = shift;
my $keys_file;
# Attempt to download KEYS file in lookup_url's containing directory.
# If that fails, try gpg_keys_url if defined.
# Import downloaded KEYS file into a local gpg keyring using gpg command.
# Determine what URL to use to download the signature file *only* from
# lookup_url's host, so that we only download the signature from the
# project's main mirror.
# Download it.
# gpg verify the sig using the downloaded and imported keys in our local
# keyring.
# Skip downloading and importing keys if we're called from inside a
# fetchware package, which should already have a copy of our package's
# KEYS file.
unless (config('user_keyring')
or (-e './pubring.gpg' and -e './secring.gpg')) {
# Obtain a KEYS file listing everyone's key that signs this distribution.
if (defined config('gpg_keys_url')) {
$keys_file = no_mirror_download_file(config('gpg_keys_url'));
} else {
eval {
$keys_file = no_mirror_download_file(config('lookup_url'). '/KEYS');
};
die <<EOD if $@;
App-Fetchware: Fetchware was unable to download the gpg_key_url you specified or
that fetchware tried appending asc, sig, or sign to [@{[config('lookup_url')]}].
It needs to download this file to properly verify you software package. This is
a fatal error, because failing to verify packages is a perferable default over
potentially installing compromised ones. If failing to verify your software
package is ok to you, then you may disable verification by adding
verify_failure_ok 'On'; to your Fetchwarefile. See perldoc App::Fetchware.
EOD
}
# Import downloaded KEYS file into a local gpg keyring using gpg
# command.
eval {
# Add --homedir option if needed.
if (config('user_keyring')) {
run_prog('gpg', '--import', $keys_file);
} else {
run_prog('gpg', '--homedir', '.', '--import', $keys_file);
}
1;
} or msg <<EOM;
App-Fetchware: Warning: gpg exits nonzero when importing large KEY files such as
Apache's. However, despite exiting nonzero gpg still manages to import most of
the keys into its keyring. It only exits nonzero, because some of the keys in
the KEYS file had errors, and these key's errors were enough to cause gpg to
exit nonzero, but not enough to cause it to completely fail importing the keys.
EOM
}
# Download Signature using lookup_url.
my $sig_file;
my (undef, undef, $path, undef, undef) = uri_split($download_path);
my ($scheme, $auth, undef, undef, undef) = uri_split(config('lookup_url'));
my $sig_url;
for my $ext (qw(asc sig sign)) {
eval {
$sig_url = uri_join($scheme, $auth, "$path.$ext", undef, undef);
$sig_file = no_mirror_download_file($sig_url);
};
# If the file was downloaded stop trying other extensions.
last if defined $sig_file;
}
die <<EOD if not defined $sig_file;
App-Fetchware: Fetchware was unable to download the gpg_sig_url you specified or
that fetchware tried appending asc, sig, or sign to [$sig_url]. It needs
to download this file to properly verify you software package. This is a fatal
error, because failing to verify packages is a perferable default over
potentially installing compromised ones. If failing to verify your software
package is ok to you, then you may disable verification by adding
verify_failure_ok 'On'; to your Fetchwarefile. See perldoc App::Fetchware.
EOD
###BUGALERT### # Use Crypt::OpenPGP if its installed.
###BUGALERT### if (eval {use Crypt::OpenPGP}) {
##DOESNTWORK?? # Build a pubring needed for verify.
##DOESNTWORK?? my $pubring = Crypt::OpenPGP::KeyRing->new();
##DOESNTWORK?? my $secring = Crypt::OpenPGP::KeyRing->new();
##DOESNTWORK??
##DOESNTWORK?? # Turn on gpg compatibility just in case its needed.
##DOESNTWORK?? my $pgp = Crypt::OpenPGP->new(
##DOESNTWORK?? Compat => 'GnuPG',
##DOESNTWORK?? PubRing => $pubring,
##DOESNTWORK?? SecRing => $secring,
##DOESNTWORK?? # Automatically download public keys as needed.
##DOESNTWORK?? AutoKeyRetrieve => 1,
##DOESNTWORK?? # Use this keyserver to download them from.
##DOESNTWORK?? KeyServer => 'pool.sks-keyservers.net',
##DOESNTWORK?? );
##DOESNTWORK??
##DOESNTWORK?? # Verify the downloaded file.
##DOESNTWORK?? my $retval = $pgp->verify(SigFile => $sig_file, Files => $CONFIG{PackagePath});
##DOESNTWORK?? if ($retval == 0) {
##DOESNTWORK?? warn "Crypt::OpenPGP failed due to invalid signature.";
##DOESNTWORK?? # return failure, because Fetchware failed to verify the downloaded
##DOESNTWORK?? # file.
##DOESNTWORK?? return undef;
##DOESNTWORK?? } elsif ($retval) {
##DOESNTWORK?? return 'Package verified';
##DOESNTWORK?? } else {
##DOESNTWORK?? # print warning about $pgp errstr message.
##DOESNTWORK?? my $errstr = $pgp->errstr();
##DOESNTWORK?? warn "Crypt::OpenPGP failed with message: [$errstr]";
##DOESNTWORK?? # return failure, because Fetchware failed to verify the downloaded
##DOESNTWORK?? # file.
##DOESNTWORK?? return undef;
##DOESNTWORK?? }
###BUGALERT### } else {
###BUGALERT### ###BUGALERT### eval the run_prog()'s below & add better error reporting in
###BUGALERT### ###BUGALERT### if Crypt::OpenPGP works ok remove gpg support & this if &
###BUGALERT### }
#IPC::System::Simple dependency.
#my standard format.
# Use automatic key retrieval & a cool pool of keyservers
###BUGALERT## Give Crypt::OpenPGP another try with
#pool.sks-keyservers.net
###BUGALERT### Should I cache the files gpg puts in its "homedir"? They
#are the public keys that verify this fetchware package. Or should they
#always be downloaded on demand as they are now??? But if verify() can
#have keys cached inside the fetchware package does that mean that I
#should open up this as an API for fetchware extensions????? I don't
#know. I'll have to think more about this issue.
#run_prog('gpg', '--keyserver', 'pool.sks-keyservers.net',
# '--keyserver-options', 'auto-key-retrieve=1',
# '--homedir', '.', "$sig_file");
# Verify sig.
# Add --homedir option if needed.
if (config('user_keyring')) {
run_prog('gpg', '--verify', $sig_file);
} else {
run_prog('gpg', '--homedir', '.', '--verify', $sig_file);
}
# Return true indicating the package was verified.
return 'Package Verified';
}
sub sha1_verify {
my ($download_path, $package_path) = @_;
return digest_verify('SHA-1', $download_path, $package_path);
}
sub md5_verify {
my ($download_path, $package_path) = @_;
return digest_verify('MD5', $download_path, $package_path);
}
sub digest_verify {
my ($digest_type, $download_path, $package_path) = @_;
# Turn SHA-1 into sha1 & MD5 into md5.
my $digest_ext = $digest_type;
$digest_ext = lc $digest_type;
$digest_ext =~ s/-//g;
##subify get_sha_sum()
my $digest_file;
# Obtain a sha sum file.
if (defined config("${digest_ext}_url")) {
my (undef, undef, $path, undef, undef) = uri_split($download_path);
my ($scheme, $auth, undef, undef, undef) =
uri_split(config("${digest_ext}_url"));
my $digest_url = uri_join($scheme, $auth, $path, undef, undef);
msg "Downloading $digest_ext digest using [$digest_url.$digest_ext]";
$digest_file = no_mirror_download_file("$digest_url.$digest_ext");
} else {
eval {
my (undef, undef, $path, undef, undef) = uri_split($download_path);
my ($scheme, $auth, undef, undef, undef) =
uri_split(config('lookup_url'));
my $digest_url = uri_join($scheme, $auth, $path, undef, undef);
msg "Downloading $digest_ext digest using [$digest_url.$digest_ext]";
$digest_file = no_mirror_download_file("$digest_url.$digest_ext");
};
if ($@) {
die <<EOD;
App-Fetchware: Fetchware was unable to download the $digest_type sum it needs to
download to properly verify you software package. This is a fatal error, because
failing to verify packages is a perferable default over potentially installin
compromised ones. If failing to verify your software package is ok to you, then
you may disable verification by adding verify_failure_ok 'On'; to your
Fetchwarefile. See perldoc App::Fetchware.
EOD
}
}
###BUGALERT###subify calc_sum()
# Open the downloaded software archive for reading.
my $package_fh = safe_open($package_path, <<EOD);
App-Fetchware: run-time error. Fetchware failed to open the file it downloaded
while trying to read it in order to check its MD5 sum. The file was
[$package_path]. See perldoc App::Fetchware.
EOD
# Do Digest type checking myself, because until Digest.pm 1.17,
# Digest->new() could run any Perl code you specify or a user does causing
# the security hole. Instead of use Digest 1.17, just avoid it altogether.
my $digest;
if ($digest_type eq 'MD5') {
$digest = Digest::MD5->new();
} elsif ($digest_type eq 'SHA-1') {
$digest = Digest::SHA->new();
} else {
die <<EOD;
EOD
}
# Digest requires the filehandle to have binmode set.
binmode $package_fh;
my $calculated_digest;
eval {
# Add the file for digesting.
$digest->addfile($package_fh);
# Actually digest it.
$calculated_digest = $digest->hexdigest();
};
if ($@) {
die <<EOD;
App-Fetchware: run-time error. Digest::$digest_type croak()ed an error [$@].
See perldoc App::Fetchware.
EOD
}
close $package_fh or die <<EOD;
App-Fetchware: run-time error Fetchware failed to close the file
[$package_path] after opening it for reading. See perldoc App::Fetchware.
EOD
###subify compare_sums();
# Open the downloaded software archive for reading.
my $digest_fh = safe_open($digest_file, <<EOD);
App-Fetchware: run-time error. Fetchware failed to open the $digest_type file it
downloaded while trying to read it in order to check its $digest_type sum. The file was
[$digest_file]. See perldoc App::Fetchware.
EOD
# Will only check the first checksum it finds.
while (<$digest_fh>) {
next if /^\s+$/; # skip whitespace only lines just in case.
my @fields = split ' '; # Defaults to $_, which is filled in by <>
# Search the @fields for a regex that is either 32 hex for md5 or 40 hex
# for sha1.
my ($checksum) = grep /^[0-9a-f]{32}(?:[0-9a-f]{8})?$/i, @fields;
# Skip trying to verify the $checksum if we failed to find it in this
# line, and instead skip to the next line in the checksum file to try to
# find a $checksum.
next unless defined $checksum;
if ($checksum eq $calculated_digest) {
return 'Package verified';
# Sometimes a = is appended to make it 32bits.
} elsif ("$checksum=" eq $calculated_digest) {
return 'Package verified';
}
}
close $digest_fh;
# Return failure, because fetchware failed to verify by checksum
return undef;
}
sub unarchive {
my $package_path = shift;
msg "Unarchiving the downloaded package [$package_path]";
my ($format, $files) = list_files($package_path);
{ # Encloseing block for $", which prints a \n between each array element.
local $" = "\n";
vmsg <<EOM;
Files are:
[
@$files
]
EOM
} # Enclosing block for $"
# Ensure no files starting with an absolute path get extracted
# And determine $build_path.
my $build_path = check_archive_files($files);
vmsg "Unarchiving $format archive [$package_path].";
unarchive_package($format, $package_path);
msg "Determined build path to be [$build_path]";
return $build_path;
}
sub list_files {
my $package_path = shift;
# List files based on archive format.
my $files;
my $format;
if ($package_path =~ /\.(t(gz|bz|xz|Z))|(tar\.(gz|bz2|xz|Z))|.fpkg$/) {
$format = 'tar';
vmsg <<EOM;
Listing files in your tar format archive [$package_path].
EOM
$files = list_files_tar($package_path);
} elsif ($package_path =~ /\.zip$/) {
$format = 'zip';
vmsg <<EOM;
Listing files in your zip format archive [$package_path].
EOM
$files = list_files_zip($package_path);
} else {
die <<EOD;
lib/App/Fetchware.pm view on Meta::CPAN
Striped the new download url [$download_path_basename] and the installed
package's [$upgrade_name_basename] of their file extensions.
EOM
# Check if $upgrade_name_basename and $download_path_basename are eq, and if
# they are return false indicating that this program should not be upgraded,
# because the version available for upgrading is the same as the currently
# installed version.
return 0 if $upgrade_name_basename eq $download_path_basename;
# Transform both competing filenames into a string of version numbers.
# Use lookup_by_versionstring() to determine which version of the same
# program is "newer."
my $sorted_file_names = lookup_by_versionstring(
[
[$upgrade_name_basename, 'placeholder'],
[$download_path_basename, 'placeholder'],
]
);
if ($sorted_file_names->[0][0] eq $download_path_basename
# Make sure cmd_upgrade() does not upgrade when the latest version is
# the same as the currently installed version ($upgrade_name_basename).
and $sorted_file_names->[0][0] ne $upgrade_name_basename) {
# The latest version we can download ($download_path_basename) is newer
# than the currently installed version ($upgrade_name_basename), so we
# should upgrade.
return 1;
} else {
# Currenlty installed version ($upgrade_name_basename) is equal to the
# latest version available for download ($download_path_basename), so
# return false indicating that we sould not upgrade.
return 0;
}
}
sub check_syntax {
# Use check_config_options() to run config() a bunch of times to check the
# already parsed Fetchwarefile.
return check_config_options(
BothAreDefined => [ [qw(build_commands)],
[qw(prefix configure_options make_options)] ],
Mandatory => [ 'program', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a program configuration
option. Please add one, and try again.
EOM
Mandatory => [ 'mirror', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a mirror configuration
option. Please add one, and try again.
EOM
Mandatory => [ 'lookup_url', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a lookup_url configuration
option. Please add one, and try again.
EOM
ConfigOptionEnum => ['lookup_method', [qw(timestamp versionstring)] ],
ConfigOptionEnum => ['verify_method', [qw(gpg sha1 md5)] ],
);
}
sub check_config_options {
my @args = @_;
my @both_are_defined;
my @mandatory;
my @config_option_enum;
# Process arguments, and check that they were specified correctly.
# Loop over @args 2 at a time hence the $i += 2 instead of $i++.
for( my $i = 0; $i < @args; $i += 2 ) {
my( $type, $AnB ) = @args[ $i, $i+1 ];
die <<EOD unless ref $AnB eq 'ARRAY';
App-Fetchware: check_config_options()'s even arguments must be an array
reference. Please correct your arguments, and try again.
EOD
die <<EOD unless @$AnB == 2;
App-Fetchware: check_config_options()'s even arguments must be an array
reference with exactly two elements in it. Please correct and try again.
EOD
if ($type eq 'BothAreDefined') {
push @both_are_defined, $AnB;
} elsif ($type eq 'Mandatory') {
push @mandatory, $AnB;
} elsif ($type eq 'ConfigOptionEnum') {
push @config_option_enum, $AnB;
} else {
die <<EOD;
App-Fetchware: check_config_options() only supports types 'BothAreDefined',
'Mandatory', and 'ConfigOptionEnum.' Please specify one of these, and try again.
EOD
}
}
# Process @both_are_defined by checking if both of the elements in the
# provided arrayrefs are "both defined", and if they are "both defined"
# throw an exception.
for my $AnB (@both_are_defined) {
my ($A, $B) = @$AnB;
my @A_defined;
my @B_defined;
# Check which ones are defined in both $A and $B
{
# the config() call will call the specified strings of which many
# are expected to be uninitialized. Because we expect them to be
# uninitialized, we use that behavior to determine if they have been
# specified in the users Fetchwarefile, and if an option was not
# specified, then undef is returned by config(). Since, we expect
# lots of undef warnings, we'll disable them.
no warnings 'uninitialized';
@A_defined = grep {config($_)} @$A;
lib/App/Fetchware.pm view on Meta::CPAN
=pod
=head1 NAME
App::Fetchware - App::Fetchware is Fetchware's API used to make extensions.
=head1 VERSION
version 1.016
=head1 SYNOPSIS
### App::Fetchware's use inside a Fetchwarefile.
### See fetchware's new command for an easy way to create Fetchwarefiles.
use App::Fetchware;
# Only program, lookup_url, one or more mirrors, and some method of
# verification are required.
program 'Your program';
lookup_url 'http://whatevermirror.your/program/is/on';
gpg_keys_url 'http://whatevermirror.your/program/gpg/key/url.asc';
mirror 'http://whatevermirror1.your/program/is/on';
mirror 'http://whatevermirror2.your/program/is/on';
mirror 'http://whatevermirror3.your/program/is/on';
mirror 'http://whatevermirror4.your/program/is/on';
mirror 'http://whatevermirror5.your/program/is/on';
# filter is not required, but is often needed to tell fetchware which
# program in the lookup_url directory or what specific version you would
# want to install. For example, Apache maintains 3 versions 2.0, 2.2, and
# 2.4. filter is what allows you to select which version you want fetchware
# to use.
filter 'version-2.0';
# Below are some popular options that may interest you.
make_options '-j 4';
### This is how Fetchwarefile's can replace lookup()'s or any other
### App::Fetchware API subroutine's default behavior.
### Remember your coderef must take the same parameters and return the same
### values.
hook lookup => sub {
# Callback that replaces lookup()'s behavior.
# Callback receives the same arguments as lookup(), and it must return
# the same number and type of arguments that lookup() returns.
return $download_path;
};
### See EXTENDING App::Fetchware WITH A MODULE for details on how to extend
### fetchware with a module to install software that cannot be expressed
### using App::Fetchware's configuration file syntax.
=head1 DESCRIPTION
App::Fetchware represents fetchware's API. For ducumentation on how to use
App::Fetchware's fetchware command line interface see L<fetchware>.
It is the heart and soul of fetchware where all of fetchware's main behaviors
are kept. It is fetchware's API, which consists of the subroutines new(),
new_install(), check_syntax(), start(), lookup(), download(), verify(),
unarchive(), build(), install(), uninstall(), upgrade() and end().
App::Fetchware stores both details about C<fetchware>'s configuration file
syntax, documents how to create a fetchware extension, and documents the
internal workings of how App::Fetchware implements C<fetchware>'s package
management behavior:
=over
=item *
For details on App::Fetchware's configuration file syntax see the section L<CREATING A App::Fetchware FETCHWAREFILE> and the section L<MANUALLY CREATING A App::Fetchware FETCHWAREFILE> for more details, and how to create one in a text editor without ...
=item *
If the needs of your program overcome the capabilities of App::Fetchware's configuration options, then see the section
L<FURTHER CUSTOMIZTING YOUR FETCHWAREFILE> for details on how to overcome those
limitations.
=item *
Straightforward and more complicated Fetchware file examples are available in
the L<EXAMPLE FETCHWAREFILES> section.
=item *
For instructions on how to create a fetchware extension see the section
L<CREATING A FETCHWARE EXTENSION>.
=item *
For details on App::Fetchware's API that is useful for those customizing their
Fetchwarefile's and to those who are implementing a fetchware extension please
see the section L<FETCHWAREFILE API SUBROUTINES>.
=back
=head1 CREATING A App::Fetchware FETCHWAREFILE
In order to create a new fetchware package, you need to create a new
Fetchwarefile. You can easily do this with the C<fetchware new> command, which
works as follows.
=over
=item 1. This command will ask you a few questions, and use the answers you provide to create a Fetchwarefile for you.
=item 2. After it does so, it gives you a chance to edit its autogenerated Fetchwarefile manually in an editor of your choice.
=item 3. Afterwards, it will ask you if you would like to go ahead and use your newly created Fetchwarefile to install your new program as a fetchware package. If you answer yes, the default, it will install it, but if you anwer no; instead, it will...
=back
You can also create your Fetchwarefile manually in a text editor if you want to.
See the section L</MANUALLY CREATING A App::Fetchware FETCHWAREFILE> for the
details. Some programs require greater customization of Fetchware's behavior
than is available in its configuration options in these cases see the section
L</FURTHER CUSTOMIZTING YOUR FETCHWAREFILE> for the specific details on how to
make fetchware do what it needs to do to manage your source code distributions.
lib/App/Fetchware.pm view on Meta::CPAN
The syntax for setting configuration options is easy. It's just the name of the
configuration option you want to specify like so:
program
And then you add a space, and then whatever value you want it to have in quotes.
program 'Apache';
And don't forget the semicolon C<;> on then end. The semicolon is required
You can use comments as needed to help document you Fetchwarefile like so:
# Fetchwarefile for Apache.
program 'Apache';
=over
=item B<1. Name it>
Use your text editor to create a file with a C<.Fetchwarefile> file extension.
Use of this convention is not required, but it makes it obvious what type of
file it is. Then, just copy and paste the example text below, and replace
C<[program]> with what you choose the name of your proram to be.
C<program> is simply a configuration option that simply names your
Fetchwarefile. It is not actually used for anything other than to name your
Fetchwarefile to document what program or behavior this Fetchwarefile manages.
use App::Fetchware;
# [program] - explain what [program] does.
program '[program]';
Fetchwarefiles are actually small, well structured, Perl programs that can
contain arbitrary perl code to customize fetchware's behavior, or, in most
cases, simply specify a number of fetchware or a fetchware extension's
configuration options. Below is my filled in example App::Fetchware
fetchwarefile.
use App::Fetchware;
# apache 2.2 - Web Server.
program 'apache-2.2';
Notice the C<use App::Fetchware;> line at the top. That line is
absolutely critical for this Fetchwarefile to work properly, because it is what
allows fetchware to use Perl's own syntax as a nice easy to use syntax for
Fetchwarefiles. If you do not use the matching C<use App::Fetchware;> line,
then fetchware will spit out crazy errors from Perl's own compiler listing all
of the syntax errors you have. If you ever receive that error, just ensure you
have the correct C<use App::Fetchware;> line at the top of your
Fetchwarefile.
=item B<2. Determine your lookup_url>
At the heart of App::Fetchware is its C<lookup_url>, which is
the URL to the FTP or HTTP mirror you want App::Fetchware to use to obtain a
directory listing to see if a new version of your program is available for
download. To figure this out just use your browser to find the program you
want fetchware to manage for you's Web site. Skip over the download link, and
instead look for the gpg, sha1, or md5 verify links, and copy and paste one of
those between the single quotes above in the lookup_url. Then delete the file
portion--from right to left until you reach a C</>. This is necessary, because
fetchware uses the lookup_url as a basis to download your the gpg, sha1, or md5
digital signatures or checksums to ensure that the packages fetchware downloads
and installs are exactly the same as the ones the author uploads.
lookup_url '';
And then after you copy the url.
lookup_url 'http://www.apache.org/dist/httpd/';
=item B<3. Determine your filter configuration option>
The C<filter> option specifies a L<perl regex|perlretut> that is matched against
the list of the files in the directory you specify in your C<lookup_url>. This
sorts through the directory to pick out which program or even which version of
the same program you want this Fetchwarefile to manage.
This is needed, because some programs such as L<Apache|http://httpd.apache.org>
have multiple versions available at the same time, so you would need to specify
which version of apache you want to download. So, you'd specify
C<filter 'httpd-2.2';> to make fetchware download and manage Apache 2.2, or you
could specify C<filter 'httpd-2.4';> to specify the newer 2.4 series version.
This option also exists to allow fetchware to pick out our program if you
specify a mirror directory that has more than one program in it. If you do this,
then fetchware can use C<filter> to pick out the program you want to download
and install from the crowd.
Just write your perl regex in between the single quotes C<'> below. You don't
need to master regular expressions to specify this option. Just specify the name
of the program and/or the main part of the version number. B<Do not> specify the
entire version number or fetchware will never update your program properly.
filter '';
And then after you type in the text pattern.
filter 'httpd-2.2';
=item B<4. Add mandatory verification settings>
Verification of software downloads is mandatory, because fetchware, in order to
install the software that is downloaded, must execute the build and installation
scripts on your computer sometimes even as the root administrator! Therefore,
fetchware will refuse to build and install any software package that cannot be
verified. This limitation can be bypassed by setting the C<verify_failure_ok>
configuration option to true, but this is B<not> recommended.
Instead, if standard verification fails, please set up one or more of the
configuration options below that may allow verification to succeed if the author
has his download site set up differently then fetchware expects.
=over
=item gpg_keys_url - Should list a URL to a file most likely named C<KEYS> that
contains versions of the author's gpg verification keys that is suitable to be
imported into gpg using C<gpg --import [name of file]>. An example would be:
gpg_keys_url 'http://www.apache.org/dist/httpd/KEYS';
=item users_keyring - Tells fetchware to use the user who calls fetchware's gpg
keyring instead of fetchware's own keyring. This is handy for when you want to
install a program, but the author has no easily accessible C<KEYS> file, but the
author has listed his gpg key on his Website. With this option, you can import
this key into your own keyring using C<gpg --import [name of file]>, and then
specify this option in your Fetchwarefile as shown below.
users_keyring 'On';
=item gpg_sig_url - Should list a URL to a directory (not a file) that has files
with the same names as the software archives that contain your program, but with
a C<.asc>, C<.sig>, or C<.sign> file extension. An example would be:
gpg_sig_url 'http://www.apache.org/dist/httpd/';
=item sha1_url - Should list a URL to a directory (not a file) that has files
with the same names as the software archives that contain your program, but with
a C<.sha> or C<.sha1> file extension. An example would be:
sha1_url 'http://www.apache.org/dist/httpd/';
=item md5_url - Should list a URL to a directory (not a file) that has files
with the same names as the software archives that contain your program, but with
a C<.md5> file extension. An example would be:
md5_url 'http://www.apache.org/dist/httpd/';
=item NOTICE: There is no configuration option to change what filename fetchware
uses. You're stuck with its default of what fetchware determines your
$download_path to be with the appropriate C<.asc>, C<sha1>, or C<.md5> added
to it.
=back
Just copy and paste the example above replacing the example between the single
quotes C<'> with the actual value you need.
=item B<5. Specify at least one mirror>
Because fetchware's C<lookup_url> B<must> be the author's main mirror instead of
a 3rd party mirror for verification purposes, you must also add a mirror option
that specifies one 3rd party mirror. I recommend picking one near your physical
geographical location or at least in your own country or one close by.
C<mirror> can be specified more than once, you you can have more than one
mirror. An example is below.
mirror 'http://apache.mesi.com.ar//httpd/';
mirror 'http://apache.osuosl.org//httpd/';
mirror 'ftp://apache.mirrors.pair.com//httpd/';
mirror 'http://mirrors.sonic.net/apache//httpd/';
mirror 'http://apache.mirrors.lucidnetworks.net//';
You can specify as many mirrors as you want to. You could perhaps include all
the mirrors your source code distribution has. And the mirrors are tried in the
order they are specified in your Fetchwarefile.
=item B<6. Specifiy other options>
That's all there is to it unless you need to further customize App::Fetchware's
behavior to modify how your program is installed.
If your Fetchwarefile is now finished, you can install your new Fetchwarefile
as a fetchware package with:
fetchware install [path to your new fetchwarefile]
Or you can futher customize it further as shown next if needed.
=item B<7. Optionally add build and install settings>
If you want to specify additional settings the first to choose from are the
build and install settings. These settings control how fetchware builds and
installs your software. They are briefly listed below. For further details see
the section L<App::Fetchware FETCHWAREFILE CONFIGURATION OPTIONS>.
=over
=item B<temp_dir> - Specifies the temporary directory fetchware will use to create its own working temporary directory where it downloads, unarchives, builds, and then installs your program from a directory inside this directory.
=item B<user> - (UNIX only) - Specifies a non-root user to drop privileges to when downloading, verifying, unarchive, and building your program. Root priveedges are kept in the parent process for install if needed.
=item B<prefix> - Specifies the --prefix option for AutoTools (./configure) based programs.
=item B<configure_options> - Specifies any additional options that fetchware should give to AutoTools when it runs ./configure to configure your program before it is built and installed.
=item B<make_options> - Specifies any command line options you would like to provide to make when make is run to build and install your software. C<-j 4> is quite popular to do a paralled make to build and install the program faster.
=item B<build_commands> - Specifies a list of commands that fetchware will use to build your program. You only need this option if your program uses a build system other than AutoTools such as C<cmake> or perhaps a custom one like Perl's C<Configure>
=item B<install_commands> - Specifies a list of commands that fetchware will use to install your program. You only need this option if your program uses a build system other than AutoTools such as C<cmake> or perhaps a custom one like Perl's C<Config...
=item B<uninstall_commands> - Specifies a list of commands that fetchware will
use to I<uninstall> your program. You only need this option if your source code
distribution does not provide a C<make uninstall> target, which not every source
code distribution does.
=item B<no_install> - Specifies a boolean (true or false) value to turn off fetchware installing the software it has downloaded, verified, unarchvied, and built. If you specify a true argument (1 or 'True' or 'On'), then fetchware will C<not> install...
=back
Just copy and paste the example below replacing C<[new_directive]> with the name
of the new directive you would like to add, and fill in the space between the
single quotes C<'>.
[new_directive] '';
After pasting it should look like.
[new_directive] '~/wallpapers';
=back
=head1 USING YOUR App::Fetchware FETCHWAREFILE WITH FETCHWARE
After you have
L<created your Fetchwarefile|/"MANUALLY CREATING A App::Fetchware FETCHWAREFILE">
as shown above you need to actually use the fetchware command line program to
install, upgrade, or uninstall your App::Fetchware Fetchwarefile.
=over
=item B<install>
A C<fetchware install [path/to/Fetchwarefile]> while using a App::Fetchware
Fetchwarefile causes fetchware to install the program specified in your
fetchwarefile to your computer as you have specified any build or install
options.
=item B<upgrade>
A C<fetchware upgrade [installed program name]> while using a App::Fetchware
Fetchwarefile will simply run the same thing as install all over again, which
ill upgrade your program if a new version is available.
=item B<uninstall>
A C<fetchware uninstall [installed program name]> will cause fetchware to run
the command C<make uninstall>, or run the commands specified by the
C<uninstall_commands> configuration option. C<make uninstall> is only available
from some programs that use AutoTools such as ctags, but apache, for example,
also uses AutoTools, but does not provide a uninstall make target. Apache for
example, therefore, cannot be uninstalled by fetchware automatically.
=item B<upgrade-all>
A C<fetchware upgrade-all> will cause fetchware to run C<fetchware upgrade> for
all installed packages that fetchware is tracking in its internal fetchware
database. This command can be used to have fetchware upgrade all currently
installed programs that fetchware installed.
If you would like C<fetchware upgrade-all> to be run every night automatically
by cron, then just create a file say fetchware with the contents below in it,
and add it to /etc/cron.daily.
#!/bin/sh
# Update all already installed fetchware packages.
fetchware upgrade-all
And if you don't want to run it system wide as root, you can add it to your user
crontab by pasting the snippet below in to your crontab by executing C<crontab -e>.
# Check for updates using fetchware every night at 2:30AM.
# Minute Hour Day of Month Month Day of Week Command
# (0-59) (0-23) (1-31) (1-12 or Jan-Dec) (0-6 or Sun-Sat)
30 2 * * * fetchware upgrade-all
=back
=head1 App::Fetchware'S FETCHWAREFILE CONFIGURATION OPTIONS
App::Fetchware has many configuration options. Most were briefly described in
the section L<MANUALLY CREATING A App::Fetchware FETCHWAREFILE>. All of them are
detailed below.
=head2 program 'Program Name';
C<program> simply gives this Fetchwarefile a name. It is availabe to fetchware
after parsing your Fetchwarefile, and is used to name your Fetchwarefile when
using C<fetchware new>. It is required just like C<lookup_url>, C<mirror>,
perhaps C<filter>, and some method to verify downloads are.
=head2 filter 'perl regex here';
Specifies a Perl regular expression that fetchware uses when it determines what
the latest version of a program is. It simply compares each file in the
directory listing specified in your C<lookup_url> to this regular expression,
and only matching files are allowed to pass through to the next part of
fetchware that looks for source code archives to download.
See L<perlretut> for details on how to use and create Perl regular expressions;
however, actual regex know how is not really needed just paste verbatim text
between the single quotes C<'>. For example, C<filter 'httpd-2.2';> will cause
fetchware to only download Apache 2.2 instead of the version for Windows or
whatever is in the weird httpd-deps-* package.
=head2 temp_dir '/tmp';
C<temp_dir> tells fetchware where to store fetchware's temporary working
directory that it uses to download, verify, unarchive, build, and install your
software. By default it uses your system temp directory, which is whatever
directory L<File::Temp's> tempdir() decides to use, which is whatever
L<File::Spec>'s tmpdir() decides to use.
=head2 fetchware_db_path '~/.fetchwaredb';
C<fetchware_db_path> tells fetchware to use a different directory other
than its default directory to store the installed fetchware package for the
particular fetchware package that this option is specified in your
Fetchwarefile. Fetchware's default is C</var/log/fetchware> on Unix when run as
root, and something like C</home/[username]/.local/share/Perl/dist/fetchware/>
when run nonroot.
This option is B<not> recommended unless you only want to change it for just one
fetchware package, because fetchware also consults the
C<FETCHWARE_DATABASE_PATH> environment variable that you should set in your
shell startup files if you want to change this globally for all of your
fetchware packages. For sh/bash like shells use:
export FETCHWARE_DATABASE_PATH='/your/path/here'
=head2 user 'nobody';
Tells fetchware what user it should drop privileges to. The default is
C<nobody>, but you can specify a different username with this configuration
option if you would like to.
Dropping privileges allows fetchware to avoid downloading files and executing
anything inside the downloaded archive as root. Except of course the commands
needed to install the software, which will still need root to able to write
to system directories. This improves security, because the downloaded software
won't have sytem privileges until after it is verified, providing that what you
downloaded is exactly what the author uploaded.
Note this only works for unix like systems, and is not used on Windows and
other non-unix systems.
Also note, that if you are running fetchware on Unix even if you do not specify
the C<user> configuration option to configure what user you will drop privileges
to, fetchware will still drop privileges using the ubiquitous C<nobody> user.
If you do B<not> want to drop privileges, then you must use the C<stay_root>
configuration option as described below.
=head2 stay_root 'On';
Tells fetchware to B<not> drop privileges. Dropping privileges when run as root
is fetchware's default behavior. It improves security, and allows fetchware to
avoid exposing the root account by downloading files as root.
Do B<not> use this feature unless you are absolutely sure you need it.
=over
=item SECURITY NOTICE
stay_root, when turned on, causes fetchware to not drop privileges when
fetchware looks up, downloads, verifies, and builds your program. Instead,
fetchware will stay root through the entire build cycle, which needlessly
exposes the root account when downloading files from the internet. These files
may come from trusted mirrors, but mirrors can, and do get cracked:
L<http://www.itworld.com/security/322169/piwik-software-installer-rigged-back-door-following-website-compromise?page=0,0>
L<http://www.networkworld.com/news/2012/092612-compromised-sourceforge-mirror-distributes-backdoored-262815.html>
L<http://www.csoonline.com/article/685037/wordpress-warns-server-admins-of-trojans>
L<http://www.computerworld.com/s/article/9233822/Hackers_break_into_two_FreeBSD_Project_servers_using_stolen_SSH_keys>
=back
=head2 lookup_url 'ftp://somedomain.com/some/path
This configuration option specifies a url of a FTP or HTTP or local (file://)
directory listing that fetchware can download, and use to determine what actual
file to download and perhaps also what version of that program to download if
more than one version is available as some mirrors delete old versions and only
keep the latest one.
This url is used for:
=over
=item 1. To determine what the actual download url is for the latest version of this program
=item 2. As the base url to also download a cryptographic signature (ends in .asc) or a SHA-1 or MD5 signature to verify the contents match what the SHA-1 or MD5 checksum is.
=back
You can use the C<mirror> configuration option to specify additional mirrors.
However, those mirrors will only be used to download the large software
archives. Only the lookup_url will be used to download directory listings to
check for new versions, and to download digital signatures or checksums to
verify downloads.
=head2 lookup_method 'timestamp';
Fetchware has two algorithms it uses to determine what version of your program
to download:
=over
=item timestamp
The timestamp algorithm simply uses the mtime (last modification time) that is
availabe in FTP and HTTP directory listings to determine what file in the
directory is the newest. C<timestamp> is also the default option, and is the one
used if C<lookup_method> is not specified.
=item versionstring
Versionstring parses out the version numbers that each downloadable program has,
and uses them to determine the downloadable archive with the highest version
number, which should also be the newest and best version of the archive to use.
=back
=head2 gpg_keys_url 'lookup_url.com/some/path';
Specifies a file not a directory URL for a C<KEYS> file that lists all of the
authors' gpg keys for fetchware to download and import before using them to
verify the downloaded software package.
If you come accross a software package whoose author uses gpg to sign his
software packages, but he does not include it in the form of a file on his main
mirror, then you can specify the C<user_keyring> option. This option forces
fetchware to use the user who runs fetchware's keyring instead of fetchware's
own keyring. This way you can then import the author's key into your own
keyring, and have fetchware use that keyring that already has the author's key
in it to verify your downloads.
=head2 user_keyring 'On';
When enabled fetchware will use the user who runs fetchware's keyring instead of
fetchware's own keyring. Fetchware uses its own keyring to avoid adding cruft to
your own keyring.
This is needed when the author of a software package does not maintain a KEYS
file that can easily be downloaded and imported into gpg. This option allows you
to import the author's key manually into your own gpg keyring, and then
fetchware will use your own keyring instead of its own to verify your downloads.
=over
=item LIMITAITON
C<user_keyring> when set to true requires that the user that fetchware is
running under have a real gpg keyring with keys that have been imported into it.
This is not the case B<unless> the C<user> option has been specified with a user
account with a proper home directory and gpg keyring for gpg to use. Because of
this limitation if you need to specify C<user_keyring> be sure to also specify
the C<user> option to specify a I<real> user account instead of the default fake
one C<nobody>.
Typically you would import the keys into your own user accounts gpg keyring, and
then you would specify your own username with the C<user> option to tell
fetchware to drop privs to your own user account to have access to your own gpg
keys.
=back
=head2 gpg_sig_url 'mirror.com/some/path';
Specifies an alternate url to use to download the cryptographic signature that
goes with your program. This is usually a file with the same name as the
download url with a C<.asc> file extension added on. Fetchware will also append
the extensions C<sig> and C<sign> if C<.asc> is not found, because some pgp
programs and authors use these extensions too.
=head2 sha1_url 'mastermirror.com/some/path';
Specifies an alternate url to download the SHA-1 checksum. This checksum is used
to verify the integrity of the archive that fetchware downloads.
You B<must> specify the master mirror site, which is your programs main mirror
site, because if you download it from a mirror, its possible that both the
archive and the checksum could have been tampered with.
=head2 md5_url 'mastermirror.com/some/path';
Specifies an alternate url to download the MD5 checksum. This checksum is used
to verify the integrity of the archive that fetchware downloads.
You B<must> specify the master mirror site, which is your programs main mirror
site, because if you download it from a mirror, its possible that both the
archive and the checksum could have been tampered with.
=head2 user_agent 'Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0'
Specifies what C<user_agent> you would like fetchware to pretend to be when
downloading files using the HTTP protocol. Some sites annoying prevent some
user agents from working while allowing others. This allows you to pretend to
be a real browser such as Firefox if you need to.
=head2 verify_method 'gpg';
Chooses a method to verify your program. The default is to try C<gpg>, then
C<sha1>, and finally C<md5>, and if all three fail, then the default is to exit
fetchware with an error message, because it is insecure to install archives that
cannot be verified. The availabel options are:
=over
=item gpg - Uses the gpg program to cryptographically verify that the program you downloaded is exactly the same as its author uploaded it.
=item sha1 - Uses the SHA-1 hash function to verify the integrity of the download. This is much less secure than gpg.
=item md5 - Uses the MD5 hash function to verify the integrity of the download. This is much less secure than gpg.
=back
=head2 verify_failure_ok 'True';
Fetchware's default regarding failing to verify your downloaded Archive with
gpg, sha1, or md5 is to exit with an error message, because installing software
that cannot be cryptographically verified should never be done.
=over
=item SECURITY NOTICE
However, if the author of a program you want to use fetchware to manage for you
does not offer a gpg, sha1, or md5 file to verify its integrity, then you can
use this option to force Fetchware to install this program anyway. However, do
not enable this option lightly. Please scour the program's mirrors and homepage
to see which C<gpg_keys_url>, C<gpg_sig_url>, C<sha1_url>, C<md5_url>, or
C<user_keyring> you can use to ensure that your archive is verified before it is
compiled and installed. Even mirrors from sites large and small get hacked
regularly:
L<http://www.itworld.com/security/322169/piwik-software-installer-rigged-back-door-following-website-compromise?page=0,0>
L<http://www.networkworld.com/news/2012/092612-compromised-sourceforge-mirror-distributes-backdoored-262815.html>
L<http://www.csoonline.com/article/685037/wordpress-warns-server-admins-of-trojans>
L<http://www.computerworld.com/s/article/9233822/Hackers_break_into_two_FreeBSD_Project_servers_using_stolen_SSH_keys>
So, Please give searching for a C<gpg_keys_url>, C<gpg_sig_url>, C<sha1_url>,
C<md5_url>, or C<user_keyring> for your program another try before simply
enabling this option.
=back
=over
=item NOTICE
C<verify_failure_ok> is a boolean configuration option, which just means its
values are limited to either true or false. True values are C<'True'>, C<'On'>,
C<1>, and false values are C<'False'>, C<'Off'>, and C<0>. All other values are
syntax errors.
=back
=head2 prefix '/opt/';
Controls the AutoTools C<./configuration --prefix=...> option, which allows you
to change the base directory that most software (software that uses AutoTools)
uses as the base directory for when they install themselves.
For example, most programs copy binaries to C<prefix/bin>, documentation to
C<prefix/docs>, manpages to C<prefix/man>, and so on.
=over
=item WARNING: C<prefix> only supports source code distributions that use GNU
AutoTools. These can easily be determined by the presence of certain files in
the the distributions main directory such as C<configure>, C<configure.in>, and
C<acinclude.m4>, and others. So, if your program uses a different build system
just include that system's version of AutoTools' C<--prefix> option in your
C<build_commands> configuration option.
=back
=head2 configure_options '--datadir=/var/mysql';
Provides options to AutoTools C<./configure> program that configures the source
code for building. Most programs don't need this, but some like Apache and MySQL
need lots of options to configure them properly. In order to provide multiple
options do not separate them with spaces; instead, separate them with commas and
keep single quotes C<'> around them like in the example below.
configure_options '--datadir=/var/mysql', '--mandir=/opt/man',
'--enable-module=example';
This option is B<not> compatible with C<build_commands>. If you use
C<build_commands>, than this option will B<not> be used.
=over
=item WARNING: C<configure_options> only supports source code distributions that use GNU
AutoTools. These can easily be determined by the presence of certain files in
the the distributions main directory such as C<configure>, C<configure.in>, and
C<acinclude.m4>, and others. So, if your program uses a different build system
just include that system's version of AutoTools' C<./configure> program in your
C<build_commands> configuration option.
=back
=head2 make_options '-j4';
This option exists mostly just to enable parallel make using the C<-j> jobs
option. But any list of options make accepts will work here too. Separate them
using commas and surround each one with single quotes C<'> like in the example
above.
=head2 build_commands './configure', 'make';
lib/App/Fetchware.pm view on Meta::CPAN
However, that is not quite right, because some of App::Fetchware's API
subroutines take important arguments and return important arguments that are
then passed to other API subroutines later on. So, your I<replacement> lookup()
B<must> take the same arguments and B<must> return the same values that the
other App::Fetchware subroutines may expect to be passed to them. So, let's fix
lookup(). Just check lookup()'s documentation to see what its arguments are and
what it returns by checking out the section L<FETCHWAREFILE API SUBROUTINES>:
hook lookup => sub {
# lookup does not take any arguments.
# Your replacement for lookup() goes here.
# Must return the same thing that the original lookup() does, so
# download() and everything else works the same way.
return $download_path;
};
Some App::Fetchware API subroutines take arguments, so be sure to account for
them:
hook download => sub {
# Take same args as App::Fetchware's download() does.
my $download_path = shift;
# Your replacement for download() goes here.
# Must return the same things as App::Fetchware's download()
return $package_path;
};
If changing lookup()'s behavior or one of the other App::Fetchware
subroutines, and you only want to change part of its behavior, then consider
importing one of the C<:OVERRIDE_*> export tags. These tags exist for most of the
App::Fetchware API subroutines, and are listed below along with what helper
subroutines they import with them. To check their documentation see the section
L<FETCHWAREFILE API SUBROUTINES>.
=over
=item WARNING
If you specify a C<OVERRIDE_*> export tag to C<App::Fetchware> be sure to add
the C<:DEFAULT> export tag to B<also> export C<App::Fetchware>'s default
exports, which must be properly exported for fetchware to work properly.
=back
=over
=item L<OVERRIDE_LOOKUP|lookup() API REFERENCE> -
L</get_directory_listing()>, L</parse_directory_listing()>,
L</determine_download_path()>, L</ftp_parse_filelist()>, L</http_parse_filelist()>,
L</file_parse_filelist()>, L</lookup_by_timestamp()>,
L</lookup_by_versionstring()>, L</lookup_determine_downloadpath()>
=item L<OVERRIDE_DOWNLOAD|download() API REFERENCE> -
L</determine_package_path()>
=item L<OVERRIDE_VERIFY|verify() API REFERENCE> - L</gpg_verify()>,
L</sha1_verify()>, L</md5_verify()>, L</digest_verify()>
=item L<OVERRIDE_UNARCHIVE|unarchive() API REFERENCE> -
L</check_archive_files()>, L</list_files()>, L</list_files_tar()>,
L</list_files_zip()>, L</unarchive_package()>, L</unarchive_tar()>,
L</unarchive_zip()>
=item L<OVERRIDE_BUILD|build() API REFERENCE> - L</run_star_commands()> and
L</run_configure()>.
=item L<OVERRIDE_INSTALL|install() API REFERENCE> -
L</chdir_unless_already_at_path()>.
=item OVERRIDE_UNINSTALL - uninstall() uses build()'s and install()'s API's, but
does not add any subroutines of its own..
=back
An example:
use App::Fetchware qw(:DEFAULT :OVERRIDE_LOOKUP);
...
hook lookup => sub {
...
# ...Download a directory listing....
# Use same lookup alorithms that lookup() uses.
return lookup_by_versionstring($filename_listing);
# Return what lookup() needs to return.
return $download_path;
};
Feel free to specify a list of the specifc subroutines that you need to avoid
namespace polution, or install and use L<Sub::Import> if you demand more control
over imports.
=head2 A real example
See the section L<EXAMPLE FETCHWAREFILES> for real examples of using hook() to
change fetchware's behavior enough to make it work properly with different
source code distributions that are popular.
=head1 EXAMPLE FETCHWAREFILES
Below are example Fetchwarefiles. They use a range of features to show you what
Fetchware can do, and how it's Fetchwarefile can be manipulated to work with any
source-code distribution.
=head2 Apache Web Server
This Apache Fetchwarefile includes a few extra mirrors just in case one is down.
The fairly common C<-j 4> C<make_option> to make the build go faster, and a
gigantic C<configure_options> telling C<./configure> excatly how I want my
Apache built and configured. It also uses a heredoc to make its
C<configure_options> configuration option much more legible.
=over
use App::Fetchware;
program 'Apache';
lookup_url 'http://www.apache.org/dist/httpd/';
filter 'httpd-2.2';
mirror 'http://apache.mirrors.pair.com/httpd/';
mirror 'http://mirrors.ibiblio.org/apache/httpd/';
mirror 'ftp://apache.cs.utah.edu/apache.org/httpd/';
verify_method 'gpg';
gpg_keys_url 'http://www.apache.org/dist/httpd/KEYS';
make_options '-j 4';
prefix '/home/dly/software/apache2.2';
# You can use heredocs to make gigantic options like this one more legible.
configure_options <<EOO;
--with-mpm=prefork
--enable-modules="access alias auth autoindex cgi logio log_config status vhost_alias userdir rewrite ssl"
--enable-so
EOO
=back
=head2 NGINX Web Server
nginx has its distribution set up differently than Apache, so some changes are
needed. First, nginx does not seem to use any mirrors at all, which means
nginx's Fetchwarefile is going to look kind of stupid with the same exact URL
being used for both the C<lookup_url> and the C<mirror>, but such a
configuration is supported. Next, nginx does not have a KEYS file, but it does
list it's developer's keys on its Website. So, they have to be imported manually
into your keyring, and then specify the C<user_keyring> option to switch
fetchware from usings its own keyring to using your own keyring. Also, note the
comment regarding having to use the C<user> option to specify a real user
account. This is needed, because the verify step is done by fetchware's child
after that child drops its root privileges. Th default user is nobody, and
nobody has no real home, and therefore no keyring, so gpg won't be able to read
the keys you ask it to by using the C<user_keyring> option; therefore, C<user>
must be specified to change it to a real user, whoose keyring has had these keys
imported into it. Also, worth noting that this nginx configuration does not use
a C<filter> option. This is not actually needed, because the only source-code
packages availabe at the C<lookup_url> are the nginx software packages
themselves, but it might be a good idea to include one, because the nginx
developers could always change how their download server is structured. So,
including it is always a good idea.
=over
use App::Fetchware;
program 'nginx';
# lookup_url and mirror are the same thing, because nginx does not seem to have
# mirrors. Fetchware, however, requires one, so the same URL is simply
# duplicated.
lookup_url 'http://nginx.org/download/';
mirror 'http://nginx.org/download/';
# Must add the developers public keys to my own keyring. These keys are
# availabe from http://nginx.org/en/pgp_keys.html Do this with:
# gpg \
# --fetch-keys http://nginx.org/keys/aalexeev.key\
# --fetch-keys http://nginx.org/keys/is.key\
# --fetch-keys http://nginx.org/keys/mdounin.key\
# --fetch-keys http://nginx.org/keys/maxim.key\
# --fetch-keys http://nginx.org/keys/sb.key\
# --fetch-keys http://nginx.org/keys/glebius.key\
# --fetch-keys http://nginx.org/keys/nginx_signing.key
# You might think you could just set gpg_keys_url to the nginx-signing.key key,
# but that won't work, because like apache different releases are signed by
# different people. Perhaps I could change gpg_keys_url to be like mirror where
# you can specify more than one option?
user_keyring 'On';
# user_keyring specifies to use the user's own keyring instead of fetchware's.
# But fetchware drops privileges by default using he user 'nobody.' nobody is
# nobody, so that user account does not have a home directory for gpg to read a
# keyring from. Therefore, I'm using my own account instead.
user 'dly';
# The other option, which is commented out below, is to use root's own keyring,
# and the no_install option to ensure that root uses its own keyring instead of
# nobody's.
# noinstall 'On';
verify_method 'gpg';
=back
=head2 PHP Programming Language
PHP annoyingly uses a custom Web application on each of its mirror sites to
serve HTTP downloads. No simple directory listing is available. Therefore, to
use php with fetchware, custom C<lookup>, C<download>, and C<verify> hooks are
needed that override fetchware's internal behavior to customize fetchware as
needed so that it can work with how PHP's site is up.
The C<lookup> hook downloads and parses the L<http://www.php.net/downloads.php>
page, which lists files availabe for download. This file is parsed using
L<HTML::TreeBuilder> to determine the latest version. The MD5 sum is also parsed
out to verify the downloaded file as well.
The C<download> hook is only needed, because http_download_file() presumes that
the last part of the path is the filename you're downloading. And this is
annoyingly not the case with the way PHP has its downloading system set up.
The C<verify> hook just uses L<Digest::MD5> to calculate the md5sum of the
downloaded file, and compares it with the one C<lookup> parses out.
=over
use App::Fetchware qw(
:OVERRIDE_LOOKUP
:OVERRIDE_DOWNLOAD
:OVERRIDE_VERIFY
:DEFAULT
);
use App::Fetchware::Util ':UTIL';
use HTML::TreeBuilder;
use URI::Split qw(uri_split uri_join);
use Data::Dumper;
use HTTP::Tiny;
program 'php';
lookup_url 'http://us1.php.net/downloads.php';
mirror 'http://us1.php.net';
mirror 'http://us2.php.net';
mirror 'http://www.php.net';
# php does *not* use a standard http or ftp mirrors for downloads. Instead, it
# uses its Web site, and some sort of application to download files using URLs
# such as: http://us1.php.net/get/php-5.5.3.tar.bz2/from/this/mirror
#
# Bizarrely a URL like
# http://us1.php.net/get/php-5.5.3.tar.bz2/from/us2.php.net/mirror
# gets you the same page, but on a different mirror. Weirdly, these are direct
# downloads without any HTTP redirects using 300 codes, but direct downloads.
#
# This is why using fetchware with php you needs a custom lookup handler.
# The files you download are resolved to a [http://us1.php.net/distributions/...]
# directory, but trying to access a apache styple auto index at that url fails
# with a rediret back to downloads.php.
my $md5sum;
hook lookup => sub {
die <<EOD unless config('lookup_url') =~ m!^http://!;
php.Fetchwarefile: Only http:// lookup_url's and mirrors are supported. Please
only specify a http lookup_url or mirror.
EOD
msg "Downloading lookup_url [@{[config('lookup_url')]}].";
my $dir_list = download_dirlist(config('lookup_url'));
vmsg "Parsing HTML page listing php releases.";
my $tree = HTML::TreeBuilder->new_from_content($dir_list);
# This parsing code assumes that the latest version of php is the first one
# we find, which seems like a dependency that's unlikely to change.
my $download_path;
$tree->look_down(
_tag => 'a',
sub {
my $h = shift;
my $link = $h->as_text();
# Is the link a php download link or something to ignore.
if ($link =~ /tar\.(gz|bz2|xz)|(tgz|tbz2|txz)/) {
# Set $download_path to this tags href, which should be
# something like: /get/php-5.5.3.tar.bz2/from/a/mirror
if (exists $h->{href} and defined $h->{href}) {
$download_path = $h->{href};
} else {
die <<EOD;
php.Fetchwarefile: A path should be found in this link [$link], but there is no
path it in. No href [$h->{href}].
EOD
}
# Find and save the $md5sum for the verify hook below.
# It should be 6 elements over, so it should be the sixth index
# in the @right array below (remember to start counting from 0.).
my @right = $h->right();
my $md5_span_tag = $right[5];
$md5sum = $md5_span_tag->as_text();
$md5sum =~ s/md5:\s+//; # Ditch md5 header.
}
}
);
# Delete the $tree, so perl can garbage collect it.
$tree = $tree->delete;
# Determine and return a proper $download_path.
# Switch it from [/from/a/mirror] to [/from/this/mirror], so the mirror will
# actually return the file to download.
$download_path =~ s!/a/!/this/!;
vmsg "Determined download path to be [$download_path]";
return $download_path;
};
# I also must hook download(), because fetchware presumes that the filename of
# the downloaded file is the last part of the $path, but that is not the case
# with the path php uses for file downloads, because it ends in mirror, which is
# *not* the name of the file; therefore, I must hook download() to fix this
# problem.
hook download => sub {
my ($temp_dir, $download_path) = @_;
my $http = HTTP::Tiny->new();
my $response;
for my $mirror (config('mirror')) {
my ($scheme, $auth, $path, $query, $fragment) = uri_split($mirror);
my $url = uri_join($scheme, $auth, $download_path, undef, undef);
msg <<EOM;
Downloading path [$download_path] using mirror [$mirror].
EOM
$response = $http->get($url);
# Only download it once.
last if $response->{success};
}
die <<EOD unless $response->{success};
php.Fetchwarefile: Failed to download the download path [$download_path] using
the mirrors [@{[config('mirror')]}]. The response was:
[@{[Dumper($response->{headers})]}].
EOD
die <<EOD unless length $response->{content};
php.Fetchwarefile: Didn't actually download anything. The length of what was
downloaded is zero. status [$response->{status}] reason [$response->{reason}]
HTTP headers [@{[Dumper($response->{headers})]}].
EOD
msg 'File downloaded successfully.';
# Determine $filename from $download_path
my @paths = split('/', $download_path);
my ($filename) = grep /php/, @paths;
vmsg "Filename determined to be [$filename]";
open(my $fh, '>', $filename) or die <<EOD;
php.Fetchwarefile: Failed to open [$filename] for writing. OS error [$!].
EOD
print $fh $response->{content};
close $fh or die <<EOD;
php.Fetchwarefile: Huh close($filename) failed! OS error [$!].
EOD
my $package_path = determine_package_path($temp_dir, $filename);
vmsg "Package path determined to be [$package_path].";
return $package_path
};
# The above lookup hook parses out the md5sum on the php downloads.php web
# site, and stores it in $md5sum, which is used in the the verify hook below.
hook verify => sub {
# Don't need the $download_path, because lookup above did that work for us.
# $package_path is the actual php file that we need to ensure its md5
# matches the one lookup determined.
my ($download_path, $package_path) = @_;
msg "Verifying [$package_path] using md5.";
dir <<EOD if not defined $md5sum;
php.Fetchwarefile: lookup failed to figure out the md5sum for verify to use to
verify that the php version [$package_path] matches the proper md5sum.
The md5sum was [$md5sum].
EOD
my $package_fh = safe_open($package_path, <<EOD);
php.Fetchwarefile: Can not open the php package [$package_path]. The OS error
was [$!].
EOD
# Calculate the downloaded php file's md5sum.
my $digest = Digest::MD5->new();
$digest->addfile($package_fh);
my $calculated_digest = $digest->hexdigest();
die <<EOD unless $md5sum eq $calculated_digest;
php.Fetchwarefile: MD5sum comparison failed. The calculated md5sum
[$calculated_digest] does not match the one parsed of php.net's Web site
[$md5sum]! Do not trust this downloaded file! Perhaps there's a bug somewhere,
or perhaps the php mirror you downloaded this php package from has been hacked.
Mirrors do get hacked occasionally, so it is very much possible.
EOD
msg "ms5sums [$md5sum] [$calculated_digest] match.";
return 'Package Verified';
};
=back
=head2 PHP Programming Language using its git VCS instead of download mirrors.
PHP like most open source software you can easily download off the internet uses
a version control system to track changes to its source code. This source code
repository is basically the same thing as a normal source code distribution
would be except VCS commands like C<git pull> are used to update it instead of
checking a mirror for a new version. The Fetchwarefile below for php customizes
Fetchware to work with php's VCS instead of the traditional downloading of
actual source code archives.
It overrides lookup() to use a local git repo stored in the $git_repo_dir
variable. To create a repo just clone php's git repo (see
http://us1.php.net/git.php for details.). It runs git pull to update the repo,
and then it runs git tags, and ditches some older junk tags, and finds only the
tags used for new versions of php. These are sorted using the C<versonstring>
lookup() algorithm, and the latest one is returned.
download() uses C<git checkout [latesttag]> to "download" php by simply changing
the working directory to the latest tag. verify() uses git's cool C<verify-tag>
command to verify the gpg signature. unarchive() is updated to do nothing since
there is no archive to unarchive. However, because we reuse build(), archive()
must return a $build_path that build() will change its directory to. start() and
end() are also overridden, because managing a temporary directory is not needed,
so, instead, they just do a C<git checkout master> to switch from whatever the
latest tag is back to master, because git pull bases what it does on what branch
you're in, so we must actually be a real branch to update git.
=over
# php-using-git.Fetchwarefile: example fetchwarefile using php's git repo
# for lookup(), download(), and verify() functionality.
use App::Fetchware qw(:DEFAULT :OVERRIDE_LOOKUP);
use App::Fetchware::Util ':UTIL';
use Cwd 'cwd';
# The directory where the php source code's local git repo is.
my $git_repo_dir = '/home/dly/Desktop/Code/php-src';
# By default Fetchware drops privs, and since the source code repo is stored in
# the user dly's home directory, I should drop privs to dly, so that I have
# permission to access it.
user 'dly';
# Determine latest version by using the tags developers create to determine the
# latest version.
hook lookup => sub {
# chdir to git repo.
chdir $git_repo_dir or die <<EOD;
php.Fetchwarefile: Failed to chdir to git repo at
[$git_repo_dir].
OS error [$!].
EOD
# Pull latest changes from php git repo.
run_prog('git pull');
# First determine latest version that is *not* a development version.
# And chomp off their newlines.
chomp(my @tags = `git tag`);
# Now sort @tags for only ones that begin with 'php-'.
@tags = grep /^php-/, @tags;
# Ditch release canidates (RC, alphas and betas.
@tags = grep { $_ !~ /(RC\d+|beta\d+|alpha\d+)$/ } @tags;
# Sort the tags to find the latest one.
# This is quite brittle, but it works nicely.
@tags = sort { $b cmp $a } @tags;
# Return $download_path, which is only just the latest tag, because that's
# all I need to know to download it using git by checking out the tag.
my $download_path = $tags[0];
return $download_path;
};
# Just checkout the latest tag to "download" it.
hook download => sub {
my ($temp_dir, $download_path) = @_;
# The latest tag is the download path see lookup.
my $latest_tag = $download_path;
# checkout the $latest_tag to download it.
run_prog('git checkout', "$latest_tag");
my $package_path = cwd();
return $package_path;
};
# You must manually add php's developer's gpg keys to your gpg keyring. Do
# this by going to the page: http://us1.php.net/downloads.php . At the
# bottom the gpg key "names are listed such as "7267B52D" or "5DA04B5D."
# These are their key "names." Use gpg to download them and import them into
# your keyring using: gpg --keyserver pgp.mit.edu --recv-keys [key id]
hook verify => sub {
my ($download_path, $package_path) = @_;
# the latest tag is the download path see lookup.
my $latest_tag = $download_path;
# Run git verify-tag to verify the latest tag
my $success = eval { run_prog('git verify-tag', "$latest_tag"); 1;};
# If the git verify-tag fails, *and* verify_failure_ok has been turned on,
# then ignore the thrown exception, but print an annoying message.
unless (defined $success and $success) {
unless (config('verify_failure_ok')) {
msg <<EOM;
Verification failure ok, becuase you've configured fetchware to continue even
if it cannot verify its downloads. Please reconsider, because mirror and source
code repos do get hacked. The exception that was caught was:
[$@]
EOM
}
}
};
hook unarchive => sub {
# there is nothing to archive due to use of git.
do_nothing(); # But return the $build_path, which is the cwd().
my $build_path = $git_repo_dir;
return $build_path;
};
# It's a git tag, so it lacks an already generated ./configure, so I must use
# ./buildconf to generate one. But it won't work on php releases, so I have to
# force it with --force to convince ./buildconf to run autoconf to generate the
# ./configure program to configure php for building.
build_commands './buildconf --force', './configure', 'make';
# Add any custom configure options that you may want to add to customize
# your build of php, or control what php extensions get built.
#configure_options '--whatever you --need ok';
# start() creates a tempdir in most cases this is exactly what you want, but
# because this Fetchwarefile is using git instead. I don't need to bother with
# creating a temporary directory.
hook start => sub {
# But checkout master anyway that way the repo can be in a known good state
# so lookup()'s git pull can succeed.
run_prog('git checkout master');
};
# Switch the local php repo back to the master branch to make using it less
# crazy. Furthermore, when using git pull to update the repo git uses what
# branch your on, and if I've checked out a tag, I'm not actually on a branch
# anymore; therefore, I must switch back to master, so that the git pull when
# this fetchwarefile is run again will still work.
hook end => sub {
run_prog('git checkout master');
};
=back
=head2 MariaDB Database
This example MariaDB Fetchwarefile parses the MariaDB download page to determine
what the latest version is based on what C<filter> option you set up. Once this
is determined, the download path is created based on the weird path that MariaDB
uses on its mirrors.
Like PHP MariaDB uses some annoying software on their Web site to presumably
track downloads. This software makes use of AJAX, which is vastly beyone the
capabilities of HTML::TreeBuilder to parse, because it needs a working
JavaScript environment. Therefore, the example Fetchwarefile below has no way of
verifying the MySQL downloads. This could be fixed by using a Perl Web scraping
module that can deal with JavaScript.
=over
use App::Fetchware;
program 'MariaDB';
# MariaDB uses ccache, which wants to create a ~/.ccache cache, which it can't
# do when it's running as nobody, so use a real user account to ensure ccache
# has a cache directory it can write to.
user 'dly';
lookup_url 'https://downloads.mariadb.org/';
# Below are the two USA mirrors where I live. Customize them as you need based
# on the mirrors listed on the download page (https://downloads.mariadb.org/ and
# then click on which version you want, and then click on the various mirrors
# by country. All you need is the scheme (ftp:// or http:// part) and the
# hostname without a slash (ftp.osuosl.org or mirror.jmu.edu). Not the full path
# for each mirror.
mirror 'http://ftp.osuosl.org';
mirror 'http://mirror.jmu.edu';
# The filter option is key to the custom lookup hook working correctly. It must
# represent the text that corresponds to the latest GA release of MariaDB
# available. It should be 'Download 5.5' for 5.5 or 'Download 10.0' for the
# newver but not GA 10.0 version of MariaDB.
filter 'Download 5.5';
hook lookup => sub {
vmsg "Downloading HTML download page listing MariaDB releases.";
my $dir_list = http_download_dirlist(config('lookup_url'));
vmsg "Parsing HTML page listing MariaDB releases.";
my $tree = HTML::TreeBuilder->new_from_content($dir_list);
# This parsing code assumes that the latest version of php is the first one
# we find, which seems like a dependency that's unlikely to change.
my @version_number;
$tree->look_down(
_tag => 'a',
sub {
my $h = shift;
my $link = $h->as_text();
# Find the filter which should be "Download\s[LATESTVERSION]"
my $filter = config('filter');
if ($link =~ /$filter/) {
# Parse out the version number.
# It's just the second space separated field.
push @version_number, (split ' ', $link)[1];
}
}
);
# Delete the $tree, so perl can garbage collect it.
$tree = $tree->delete;
# Only one version should be found.
die <<EOD if @version_number > 1;
mariaDB.Fetchwarefile: multiple version numbers detected. You should probably
refine your filter option and try again. Filter [@{[config('filter')]}].
Versions found [@version_number].
EOD
# Construct a download path using $version_number[0].
my $filename = 'mariadb-' . $version_number[0] . '.tar.gz';
# Return a proper $download_path, so That I do not have to hook download(),
# but can reuse Fetchware's download() subroutine.
my $weird_prefix = '/mariadb-' . $version_number[0] . '/kvm-tarbake-jaunty-x86/';
my $download_path = '/pub/mariadb' . $weird_prefix .$filename;
return $download_path;
};
# Make verify() failing to verify MariaDB ok, because parsing out the MD5 sum
# would require a Web scraper that supports javascript, which HTML::TreeBuilder
# obviously does not.
verify_failure_ok 'On';
# Use build_commands to configure fetchware to use MariaDB's BUILD script to
# build it. See https://mariadb.com/kb/en/generic-build-instructions/ for
# instructions on the different BUILD cmake scripts that are available.
build_commands 'BUILD/compile-pentium64-max';
# Use install_commands to tell fetchware how to install it. I could leave this
# out, but it nicely documents what command is needed to install MariaDB
# properly.
install_commands 'make install';
=back
=head2 PostgreSQL Database
Below is a example Fetchwarefile that overrides lookup() to determine the latest
version, but manages to avoid overriding anything else. It uses the same style
as the rest downloading an HTML page that lists the version numbers on it
somewhere. Then it parses the HTML with HTML::TreeBuilder. It populates an
array, and then uses L<App::Fetchware>'s lookup_by_versionstring() to determine
which version is the latest one. This is then concatenated with a bunch of other
stuff to determine the $download_path.
MD5 verification is supported by simply specifying a C<md5_url> option, because
by default fetchware uses the C<lookup_url> to determine where to download the
md5sum from, but that won't work with PostgreSQL, because it's download system
has the md5sum on the download C<mirror> instead of the C<lookup_url>.
=over
use App::Fetchware qw(:DEFAULT :OVERRIDE_LOOKUP);
use App::Fetchware::Util ':UTIL';
use HTML::TreeBuilder;
program 'postgres';
# The Postgres file browser URL lists the available versions of Postgres.
lookup_url 'http://www.postgresql.org/ftp/source/';
# Mirror URL where the file browser links to download them from.
my $mirror = 'http://ftp.postgresql.org';
mirror $mirror;
# The Postgres file browser URL that is used for the lookup_url lists version
# numbers of Postgres like v9.3.0. this lookup hook parses out the list of
# theses numbers, determines the latest one, and constructs a $download_path to
# return for download to use to download based on what I set my mirror to.
hook lookup => sub {
my $dir_list = no_mirror_download_dirlist(config('lookup_url'));
my $tree = HTML::TreeBuilder->new_from_content($dir_list);
# Parse out version number directories.
my @ver_nums;
my @list_context = $tree->look_down(
_tag => 'a',
sub {
my $h = shift;
my $link = $h->as_text();
# Is this link a version number or something to ignore?
if ($link =~ /^v\d+\.\d+(.\d+)?$/) {
# skip version numbers that are beta's, alpha's or release
# candidates (rc).
return if $link =~ /beta|alpha|rc/i;
# Strip useless "v" that just gets in the way later when I
# create the $download_path.
$link =~ s/^v//;
push @ver_nums, $link;
}
}
);
# Turn @ver_num into the array of arrays that lookup_by_versionstring()
# needs its arguments to be in.
my $directory_listing = do {
my $arrayref_of_arrays_directory_listing = [];
for my $ver_num (@ver_nums) {
push @$arrayref_of_arrays_directory_listing,
[$ver_num];
}
$arrayref_of_arrays_directory_listing;
};
# Find latest version.
my $latest_ver = lookup_by_versionstring($directory_listing);
# Return $download_path.
my $download_path = '/pub/source/'. "v$latest_ver->[0][0]" .
"/postgresql-$latest_ver->[0][0].tar.bz2";
return $download_path;
};
# MD5sums are stored on the download site, so use them to verify the package.
verify_method 'md5';
# But they are *not* stored on the original "lookup_url" site, so I must provide
# a md5_url pointing to the download site.
md5_url $mirror;
=back
=head1 CREATING A FETCHWARE EXTENSION
=over
=item WARNING
Currently, fetchware's extension system is B<BETA>, and unlikely to change, but
changes may happen most likely just in the form of bug fixes. This, however, is
unlikely, but it could happen. Most likely, just some minor bug fixes will occur
to the API not any major changes or refactors or rewrites. I'm considering
"genericizing" make_test_dist() so Fetchware extension authors can reuse it to
test their extensions.
=back
Fetchware's main program C<fetchware> uses App::Fetchware's short and simple API
to implement fetchware's default behavior; however, other styles of source code
distributions exist on the internet that may not fit inside App::Fetchware's
capabilities. That is why, in addition to its flexible configuration file
syntax, fetchware allows modules other than App::Fetchware to provide it with
its behavior.
=head2 How the API works
When fetchware installs or upgrades something it executes the API subroutines
check_syntax(), start(), lookup(), download(), verify(), unarchive(), build(),
install(), and end() in that order. And when fetchware uninstalls an installed
package it executes the API subroutines check_syntax(), start(), unarchive(),
uninstall(), and end(). Upgrade is basically the exact same thing as install(),
but it compares version numbers using the upgrade() API subroutine. And
C<fetchware new> obviously just calls new() and new_install().
=head2 Extending App::Fetchware
This API can be overridden inside a user created Fetchwarefile by using hook()
as L<explained above|/So how do I add some custom Perl code to customize my Fetchwarefile?>.
hook() simply takes a Perl code reference that takes the same parameters, and
returns the same results that the subroutine that you've hook()ed takes and
returns.
For more extensive changes you can create a App::Fetchware module that
I<"subclasses"> App::Fetchware. Now App::Fetchware is not an object-oriented
module, so you cannot use L<parent> or L<base> to add it to your program's
inheritance tree using C<@INC>. You can, however, use
L<App::Fetchware::ExportAPI> to import whatever subroutines from App::Fetchware
that you want to reuse such as start() and end(), and then simply implement the
remaining subroutines that make up App::Fetchware's API. Just like the
C<CODEREF> extensions mentioned above, you must take the same arguments and
return the same values that fetchware expects or using your App::Fetchware
extension will blow up in your face.
This is described in much more detail below in L<CHANGING FETCHWARE'S BEHAVIOR>.
=head2 Essential Terminology
App::Fetchware manages to behave like an object oriented module would with
regards to its extension system without actually using perl's object-oriented
features especially using @INC for subclassing, which App::Fetchware does not
use.
The same terminology as used in OOP is also used here in App::Fetchware, because
the concepts are nearly the same--they're simply implemented differently.
=head3 API subroutines
These are the subroutines that App::Fetchware implements, and that fetchware
uses to implement its desired behavior. They are new(), new_install(),
check_syntax(), start(), lookup(), download(), verify(), unarchive(), build(),
install(), uninstall(), upgrade(), and end(). All must be
implemented or "inherited" from App::Fetchware using L<App::Fetchware::ExportAPI> as
discussed below in L<Implement your fetchware extension.> in a App::Fetchware
L<subclass>.
=head3 override
Means the same thing it does in object-oriented programming. Changing the
definition of a method/subroutine without changing its name. In OOP this is
simply done by subclassing something, and then defining one of the methods that
are in the superclass in the subclass. In App::Fetchware extensions this is done
by simply defining a subroutine with the same name as one or more of the API
subroutines that are defined above. There is no method resolution order and
C<@INC> is not consulted.
=head3 subclass
Means the same thing it does in object-oriented programming. Taking one class
and replacing it with another class. Only since App::Fetchware is not
object-oriented, it is implemented differently. You simply use
L<App::Fetchware::ExportAPI> to specify the L<API subroutines> that you are
B<not> going to override, and then actually implement the remaining subroutines,
so that your App::Fetchware I<subclass> has the same interface that
App::Fetchware does.
To create a fetchware extension you must understand how they work:
=over
=item 1. First a Fetchwarefile is created, and what module implements App:Fetchware's API is declared with a C<use App::Fetchware...;> line. This line is C<use App::Fetchware> for default Fetchwarefiles that use App::Fetchware to provide C<fetchware>...
=item 2. To use a fetchware extension, you simply specify the fetchware
extension you want to use with a C<use App::FetchwareX::...;> instead of specifying
C<use App::Fetchware> line in your Fetchwarefile. You B<must> replace the
App::Fetchware import with the extension's. Both cannot be present. Fetchware
will exit with an error if you use more than one App::Fetchware line without
specifying specific subroutines in all but one of them.
=item 3. Then when C<fetchware> parses this Fetchwarefile when you use it to install, upgrade, or uninstall something, This C<use App::FetchwareX::...;> line is what imports App::Fetchware's API subroutines into C<fetchware>'s namespace.
=back
That's all there is to it. That simple C<use App::Fetchware...;> imports from
App::Fetchware or a App::Fetchware extension such as
App::FetchwareX::HTMLPageSync the API subroutines (such as start(), lookup(),
..., install(), and uninstall()) C<fetchware> needs to use to install, upgrade,
or uninstall whatever program your Fetchwarefile specifies.
After understanding how they work, simply follow the instructons and consider
the recommendations below. Obviously, knowing Perl is required. A great place to
start is chromatic's
L<Modern Perl|http://www.onyxneon.com/books/modern_perl/index.html>.
=head2 Develop your idea keeping in mind fetchware's package manager metaphor
Fetchware is a package manager like apt-get, yum, or slackpkg. It I<installs>,
I<upgrades>, or I<uninstalls> programs. Fetchware is not a Plack for command
line programs. It has a rather specific API meant to fit its package manager
metaphor. So, keep this in mind when developing your idea for a fetchware
extension.
=head2 Map your extension's behavior to App::Fetchware's API
App::Fetchware has a specific behavior consisting of just a few subroutines with
specific names that take specific arguments, and return specific values. This
API is how you connect your extension to fetchware.
Just consider the description's of App::Fetchware's API below, and perhaps
consult their full documentation in L<FETCHWAREFILE API SUBROUTINES>.
=over
=item B<my ($program_name, $fetchwarefile) = new($term, $program_name)> -
Along with new_install() implements Fetchware's new command for helping users
create new Fetchwarefiles using a Q&Z wizard interface.
=item B<my $fetchware_package_path = new_install($program_name, $fetchwarefile)>
- new()'s sister subroutine for implementing the Q&A wizard interface for
Fetchware's new command. Just used to keep root to make
ask_to_install_now_to_test_fetchwarefile() work for system level installs while
keeping drop privs enabled by default.
=item B<'Syntax Ok' = check_syntax()> - Checks the user's Fetchwarefile, but only
at Fetchware's level. Perl level syntax errors are not checked. PPI is not used
to parse the file. Instead only high-level Fetchware specific syntax errors are
checked.
=item B<my $temp_dir = start(KeepTempDir => 0 | 1)> - Gives your extension a chance to do anything needed before the rest of the API subroutines get called. App::Fetchware's C<start()> manages App::Fetchware's temporary directory creation. If you wo...
=item B<my $download_url = lookup()> - Determines and returns a download url that C<download()> receives and uses to download the archive for the program.o
=item B<my $package_path = download($tempd_dir, $download_url)> - Downloads its provided $download_url argument.
=item B<verify($download_url, $package_path)> - Verifies the integrity of your downloaded archive using gpg, sha1, or md5.
=item B<my $build_path = unarchive($package_path)> - Unpacks the downloaded archive.
=item B<build($build_path)> - Configures and compiles the downloaded archive.
=item B<install()> - Installs the compiled archive.
=item B<end()> - Cleans up the temporary directory that start() created. Can be overridden to do any other clean up tasks that your archive needs.
=item B<uninstall($build_path)> - Uninstalls an already installed program installed with the same App::Fetchware extension.
=item B<$upgrade = upgrade($download_path, $fetchware_package_path)> - uses its
two arguments to determine if a new version is available to upgrade to, or if
the currently installed version is the latest version, and no additional
upgrades are needed.
=back
Also, keep in mind the order in which these subroutines are called, what
arguments they receive, and what their expected return value is.
=over
=item B<new> = Just new() and new_install().
=item B<install> - check_syntax(), start(), lookup(), download(), verify(), unarchive(), build(), install(), and end().
=item B<upgrade> - check_syntax(), start(), lookup(), I<upgrade()>, download(),
verify(), unarchive(), build(), install(), and end().
=item B<upgrade-all> - Is the same as upgrade, because it just call upgrade for
each fetchware package that is installed in the fetchware package database.
=item B<uninstall> - You might think its just uninstall(), but it does not.
check_syntax(), start(), unarchive(), uninstall(), and end().
=back
Use the above overview of App::Fetchware's API to design what each API
subroutine keeping in mind its arguments and what its supposed to return.
=head2 Determine your fetchware extension's Fetchwarefile configuration options.
App::Fetchware has various configuration options such as C<temp_dir>, C<prefix>,
and so on. Chances are your fetchware extension will also need such
configuration options. These are easily created with
L<App::Fetchware::CreateConfigOptions>, which manufactures these to order for
your convenience. There are four different types of configuration options:
=over
=item ONE - Takes only one argument, and can only be used once.
=item ONEARRREF - Can only be called once, but can take multiple agruments at once.
=item MANY - Takes only one argument, but can be called more than once. The only example is mirror.
=item BOOLEAN - Takes only one arguement, and can only be called once just like
ONE. The difference is that BOOLEANs are limited to only I<boolean> true or
false values such as C<'On'> or C<'Off'>, C<'True'> or C<'False'>, or C<1> or
C<0>. App::Fetchware examples include C<no_install> and C<vefify_failure_ok>.
=back
Using the documentation above and perhaps also the documentation for
L<App::Fetchware::CreateConfigOptions>, determine the names of your
configuration options, and what type of configuraton options they will be.
=head2 Implement your fetchware extension.
Since you've designed your new fetchware extension, now it's time to code it up.
The easiest way to do so, is to just take an existing extension, and just copy
and paste it, and then delete its specifics to create a simple extension
skeleton. Then just follow the steps below to fill in this skeleton with the
specifics needed for your fetchware extension.
=over
=item 1. Set up proper exports and imports.
Because fetchware needs your fetchware extension to export all of the
subroutines that make up the fetchware's API, and any configuration
options (as Perl subroutines) your extension will use, fetchware uses the helper
packages L<App::Fetchware::ExportAPI> and L<App::Fetchware::CreateConfigOptions>
to easily manage setting all of this up for you.
First, use L<App::Fetchware::ExportAPI> to be sure to export all of fetchware's
API subroutines. This package is also capable of "inheriting" any of
App::Fetchware's API subroutines that you would like to keep. An example.
# Use App::Fetchware::ExportAPI to set up proper exports this fetchware
# extension.
use App::Fetchware::ExportAPI KEEP => [qw(new_install start end)],
OVERRIDE => [qw(new check_syntax lookup download verify
unarchive build install uninstall upgrade)]
;
There two types of subroutines ExportAPI helps you with:
=over
=item *
B<KEEP> - Specifies which API subroutines you will "keep" or perhaps "inherit"
from App::Fetchware. Usually this is just start() and end() to manage the
C<temp_dir> for you.
=item *
B<OVERRIDE> = Specifies which API subroutines you will "override," or implement
yourself.
=back
Second, use L<App::Fetchware::CreateConfigOptions> to create all of the
configuration options (such as C<temp_dir>, C<no_install>, and so on.) you want
your fetchware extension to have.
There are four types of configuration options.
=over
=item C<ONE> - Take one an only ever one argument, and can only be
called once per Fetchwarefile.
I<Examples:> C<temp_file>, C<prefix>, and C<lookup_url>.
=item C<ONEARRREF> - Takes one or more arguments like C<MANY>, but unlike
C<MANY> can only be called once.
I<Examples:> C<configure_options>, C<make_options>, and C<build_commands>.
=item C<MANY> - Takes one or more arguments, and can be called more than once.
If called more than once, then second call's arguments are I<added> to the
existing list of arguments.
I<Examples:> C<mirror>.
=item C<BOOLEAN> - Just like C<ONE> except it will convert /off/i and /false/i
to 0 to support more than just Perl's 0 or undef being false.
I<Examples:> C<verify_failure_ok>, C<no_install>, and C<stay_root>.
=back
Additionally, there is another type of configuration option that only
App::Fetchware::CreateConfigOptions uses:
=over
=item C<IMPORT> - Allows you to import already defined configuration options
from App::Fetchware into your Fetchware extension. For exmaple, you might also
want a C<temp_dir> or C<no_install> configuration option, and the C<IMPORT> type
allows you to easily import one and document the fact that it is imported.
An example.
use App::Fetchware::CreateConfigOptions
IMPORT => [qw(temp_dir no_install)],
ONE => [qw(repository directory)],
ONEARRREF => [qw(build_options install_options)],
BOOLEAN => [qw(delete_after_download)],
;
These 2 simple use()'s are all it takes to set up proper exports for your
fetchware extension.
=item 2. Code any App::Fetchware API subroutines that you won't be reusing from App::Fetchware.
Use their API documentation from the section L<FETCHWAREFILE API SUBROUTINES> to
ensure that you use the correct subroutine names, arguments and return the
correct value as well.
An example for overriding lookup() is below.
=head2 lookup()
my $download_url = lookup();
# New lookup docs go here....
=cut
sub lookup {
# New code for new lookup() goes here....
# Return the required $download_url.
return $download_url;
}
=back
=back
=head3 Use Fetchware's Own Libraries to Save Developement Time.
Fetchware includes many libraries to save development time. These libraries are
well tested by Fetchware's own test suite, so you too can use them to save
development time in your own App::Fetchware extensions.
These libraries are:
lib/App/Fetchware.pm view on Meta::CPAN
L<original_cwd()|App::Fetchware::Util/original_cwd()>, and
L<cleanup_tempdir()|App::Fetchware::Util/cleanup_tempdir()>. Only needed if you
don't reuse App::Fetchware's own start() and end() subroutines that make use of
thse temporary directory subroutines for you.
=over
=item * L<create_tempdir()|App::Fetchware::Util/create_tempdir()> creates and
chdir()'s into a temporary directory using File::Temp's tempdir() function. It
also deals with creating a Fetchware semaphore file to keep C<fetchware clean>
from deleting any still needed temporary directories.
=item * L<original_cwd()|App::Fetchware::Util/original_cwd()> simply returns
what fetchware's current working directory was before create_tempdir() created
and chdir()'d into the temporary directory.
=item * L<cleanup_tempdir()|App::Fetchware::Util/cleanup_tempdir()> deals with
closing the fetchware semaphore file.
=back
=back
=item L<Test::Fetchware>
Test::Fetchware includes utility subroutines that fetchware itself uses to test
itself, and they are shared with extension writers through this module.
=over
=item * L<eval_ok()|Test::Fetchware/eval_ok()> - A poor man's Test::Exception
in one simple subroutine. Why require every user to install a dependency only
used for testing when one simple subroutine does the trick.
=item * L<print_ok()|Test::Fetchware/print_ok()> - A poor man's Test::Output in
one simple subroutine.
=item *
L<skip_all_unless_release_testing()|Test::Fetchware/skip_all_unless_release_testing()>
- Does just what it's name says. If fetchware's internal release/author only
Environment variables are set, only then will any Test::More subtests that call
this subroutine skip the entire subtest. This is used to skip running tests that
install real programs on the testing computer's system. Many of Fetchware's
tests actually install a real program such as Apache, and I doubt any
Fetchware user would like to have Apache installed and uninstalled a
bagillion times when they install Fetchware. Use this subroutine in your own
App::Fetchware extension's to keep that from happening.
=item * L<make_clean()|Test::Fetchware/make_clean()> - Just run_prog('make',
'clean') in the current working directory just as its name suggests.
=item * L<make_test_dist()|Test::Fetchware/make_test_dist()> - Use this
subroutine or craft your own similar subroutine, if you need more flexibility,
to test actually installing a program on user's systems that just happens to
execute all of the proper installation commands, but supplies installation
commands that don't actually install anything. It's used to test actually
installing software without actually installing any software.
=item * L<md5sum_file()|Test::Fetchware/md5sum_file()> - Used to provide a
md5sum file for make_test_dist() create software packages in order to pass
fetchware's verify checks.
=item * L<verbose_on()|Test::Fetchware/verbose_on()> - Make's all vmsg()'s
actually print to the screen even if -v or --verbose was not actually
provided on the command line. Used to aid debugging.
=back
=item L<App::Fetchware::Config>
App::Fetchware::Config stores and manages fetchware's parsed configuration file.
parse_fetchwarefile() from L<fetchware> does the actual parsing, but it stores
the configuration file inside App::Fetchware::Config. Use the subroutines below
to access any configuration file options that you create with
L<App::Fetchware::CreateConfigOptions> to customize your fetchware extension.
Also feel free to reuse any names of App::Fetchware configuration subroutines
such as C<temp_dir> or C<lookup_url>
=over
=item L<config()|App::Fetchware::Config/config()> - Sets and gets values from
the currently parsed fetchware configuration file. If there is one argument,
then it returns that configuration options value or undef if there is none.If
there are more than one argument, then the first argument is what configuration
option to use, and the rest of the arguments are what values to set that
configuration option to.
=item L<config_iter()|App::Fetchware::Config/config_iter()> - returns a
configuration I<iterator>. that when I<kicked> (called, like
C<$config_iter-E<gt>()>) will return one value from the specifed
configuration option. Can be kicked any number of times, but once the number of
configuration values is exhausted the iterator will return undef.
=item L<config_replace()|App::Fetchware::Config/config_replace()> - config() is
used to I<set> configuration options, and once set they I<cannot> be changed by
config(). This is meant to catch and reduce errors. But sometimes, mostly in
test suites, you need to change the value of a configuration option. That's
what config_replace() is for.
=item L<config_delete()|App::Fetchware::Config/config_delete()> - deletes the
specified configuration option. Mostly just used for testing.
=item L<__clear_CONFIG()|App::Fetchware::Config/__clear_CONFIG()> - An internal
only subroutine that should be only used when it is really really needed. It
I<clears> (deletes) the entire internal hash that the configuration options are
stored in. It really should only be used during testing to clear
App::Fetchware::Config's intenal state between tests.
=item L<debug_CONFIG()|App::Fetchware::Config/debug_CONFIG()> - prints
App::Fetchware::Config's internal state directly to STDOUT. Meant for debugging
only in your test suite.
=back
=item L<App::Fetchware::Fetchwarefile>
Helper OO class for new() API subroutine. Allows you to programatically build a
Fetchwarefile for the user using a small API instead of manually concatenating
tons of strings constantly worrying about wordwrap and whitespace.
=over
lib/App/Fetchware.pm view on Meta::CPAN
to add the actual configuration options and their values to the Fetchwarefile
object.
=item L<generate()|App::Fetchware::Fetchwarefile/generate()> - Returns the
"generated" Fetchwarefile. This method does all of the string concatenation for
you in order to create the Fetchwarefile specfied in your new() and
config_options() calls.
=back
=item L<App::Fetchware's OVERRIDE_* export tags.|FETCHWAREFILE API SUBROUTINES>
App::Fetchware's main API subroutines, especially the crazy complicated ones
such as lookup(), are created by calling and passing data among many component
subroutines. This is done to make testing much much easier, and to allow
App::Fetchware extensions to also use some or most of these component
subroutines when they override a App::Fetchware API subroutine.
=over
=item L<new()'s OVERRIDE_NEW export tag.|new() API REFERENCE>
The export tag exports all of the helper subroutines new() uses to implement its
functionality. Some like get_lookup_url(), get_verification(), and
get_filter_option() are quite specific to App::Fetchware, but extension_name,
fetchwarefile_name(), opening_message(), prompt_for_other_options(), and
edit_manually() are nice and generic, and should be appropriate for any
Fetchware extension.
=item L<new_install()'s OVERRIDE_NEW_INSTALL export tag.|new_install() API REFERENCE>
new_install() only exports ask_to_install_now_to_test_fetchwarefile(), and most
Fetchware extensions should probably just "inherit" new_install(), because
new_install()'s only real purpose was making
ask_to_install_now_to_test_fetchwarefile() work when fetchware drops privs.
=item L<check_syntax()'s OVERRIDE_CHECK_SYNTAX export tag|check_syntax() API REFERENCE>
check_syntax() only has the check_config_options() helper subroutine that is
meant for reuse by Fetchware extensions, so they can check their syntax just as
Fetchware itself does.
=item L<lookup()'s OVERRIDE_LOOKUP export tag.|lookup() API REFERENCE>
This export tag is the largest, and perhaps the most important, because it
implements fetchware's ability to determine if a new version of your software
package is available. Its default is just a clever use of HTTP and FTP directory
listings.
See the section L<lookup() API REFERENCE> for more details on how to use these
subroutines to determine if new versions of your software is available
automatically.
=item L<download()'s OVERRIDE_DOWNLOAD export tag.|download() API REFERENCE>
Only exports the subroutine determine_package_path(), which simply comcatenates
a $tempdir with a $filename to return a properl $package_path, which unarchive
later uses. This is mostly its own subroutine to better document how this is
done, and to allow easier code reuse.
=item L<verify()'s OVERRIDE_VERIFY export tag.|verify() API REFERENCE>
Exports a family of subroutines to verify via MD5, SHA1, or GPG the integrity of
your downloaded package. MD5 and SHA1 are supported for legacy reasons. All
software packages should be GPG signed for much much much better security. GPG
signatures when verified actually prove that the software package you downloaded
is exactly what the author of that software package created, whereas MD5 and
SHA1 sums just verify that you downloaded the bunch of bits in the same order
that they are stored on the server.
digest_verify() an be used to add support for any other Digest::* modules that
CPAN has a Digest based module for that correctly follow Digest's API.
=item L<unarchive()'s OVERRIDE_UNARCHIVE export tag.|unarchive() API REFERENCE>
Exports subroutines that will help you unarchive software packages in tar and
zip format. The most important part to remember is to use list_files() to list
the files in your archive, and pass that list to check_archive_files() to ensure
that the archive will not overwrite any system files, and contains no absolute
paths that could cause havok on your system. unarchive_package() does the actual
unarchiving of software packages.
=item L<build()'s OVERRIDE_BUILD export tag.|build() API REFERENCE>
Provides run_star_commands(), which is meant to execute common override commands
that fetchware provides with the C<build_commands>, C<install_commands>, and
C<uninstall_commands> configuration file directives. These directives are of
type C<ONEARRREF> where they can only be called once, but you can supply a comma
separated list of commands that fetchware will install instead of the standard
commands default AutoTools commands (build() => ./configure, make; install() =>
make install; uninstall() => ./configure, make uninstall). See its
L<documentation|run_star_commands(config('*_commands'));> for more details.
=item L<install()'s OVERRIDE_INSTALL export tag.|install() API REFERENCE>
install() only exports chdir_unless_already_at_path(), which is of limited use.
install() also uses build()'s run_star_commands().
=item uninstall()'s OVERRDIE_UNINSTALL export tag.
uninstall() actually has no exports of its own, but it does make use of
build() and install()'s exports.
=item upgrade()'s OVERRIDE_UNINSTALL export tag.
upgrade() also has no exports of its own, and does not use anyone elses.
=back
=back
=head2 Write your fetchware extension's documentation
Fill in all of the skeleton's missing POD to ensure that fetchware extension has
enough documentation to make it easy for user's to use your fetchware extension.
Be sure to document:
=over
=item * All of Perl's standard POD sections such as SYNOPSIS, DESCRIPTION, AUTHOR, and all of the others. See L<perlpodstyle> for more details.
=item * Give each subroutine its own chunk of POD before it explaining its arguments, any App::Fetchware configuration options it uses, and what its return value is.
=item * Be sure to document both its external interface, its Fetchwarefile, and its internal interface, what subroutines it has and uses.
=back
=head2 Write tests for your fetchware extension
Use perls venerable Test::More, and whatever other Perl TAP testing modules you
need to be sure your fetchware extension works as expected.
L<Test::Fetchware/> has a few testing subroutines that fetchware itself uses
in its test suite that you may find helpful. These include:
=over
=item L<Test::Fetchware/eval_ok()> - A poor man's Test::Exception. Captures any
exceptions that are thrown, and compares them to the provided exception text or
regex.
=item L<Test::Fetchware/print_ok()> - A poor man's Test::Output. Captures
STDOUT, and compares it to the provided text.
=item L<Test::Fetchware/skip_all_unless_release_testing()> - Fetchware is a
package manager, but who wants software installed on their computer just to test
it? This subroutine marks test files or subtests that should be skipped unless
fetchware's extensive FETCHWARE_RELEASE_TESTING environement variables are set.
This funtionality is described next.
=item L<Test::Fetchware/make_clean()> - Just runs C<make clean> in the current
directory.
=item L<Test::Fetchware/make_test_dist()> - Creates a temporary distribution
that is used for testing. This temporary distribution contains a C<./configure>
and a C<Makefile> that create no files, but can still be executed in the
standard AutoTools way.
=item L<Test::Fetchware/md5sum_file()> - Just md5sum's a file so verify() can be
tested.
=item L<Test::Fetchware/expected_filename_listing()> - Returns a string of crazy
Test::Deep subroutines to test filename listings. Not quite as useful as the
rest, but may come in handy if you're only changing the front part of lookup().
=back
Your tests should make use of fetchware's own C<FETHWARE_RELEASE_TESTING>
environment variable that controls with the help of
skip_all_unless_release_testing() if and where software is actually installed.
This is done, because everyone who installs fetchware or your fetchware
extension is really gonna freak out if its test suite installs apache or ctags
just to test its package manager functionality. To use it:
=over
=item 1. Set up an automated way of enabling FETCHWARE_RELEASE_TESTING.
Just paste the frt() bash shell function below. Translating this to your
favorite shell should be pretty straight forward. Do not just copy and paste it.
You'll need to customize the specific C<FETCHWARE_*> environment variables to
whatever mirrors you want to use or whatever actual programs you want to test
with. And you'll have to point the local (file://) urls to directories that
actually exist on your computer.
# Sets FETCHWARE_RELEASE_TESTING env vars for fully testing fetchware.
frt() {
if [ -z "$FETCHWARE_RELEASE_TESTING" ]
then
echo -n 'Setting fetchware_release_testing environment variables...';
export FETCHWARE_RELEASE_TESTING='***setting this will install software on your computer!!!!!!!***'
export FETCHWARE_FTP_LOOKUP_URL='ftp://carroll.cac.psu.edu/pub/apache/httpd'
export FETCHWARE_HTTP_LOOKUP_URL='http://www.apache.org/dist/httpd/'
export FETCHWARE_FTP_MIRROR_URL='ftp://carroll.cac.psu.edu/pub/apache/httpd'
export FETCHWARE_HTTP_MIRROR_URL='http://mirror.cc.columbia.edu/pub/software/apache//httpd/'
export FETCHWARE_FTP_DOWNLOAD_URL='ftp://carroll.cac.psu.edu/pub/apache/httpd/httpd-2.2.26.tar.bz2'
export FETCHWARE_HTTP_DOWNLOAD_URL='http://mirrors.ibiblio.org/apache//httpd/httpd-2.2.26.tar.bz2'
export FETCHWARE_LOCAL_URL='file:///home/dly/software/httpd-2.2.22.tar.bz2'
export FETCHWARE_LOCAL_ZIP_URL='file:///home/dly/software/ctags-zip/ctags58.zip'
export FETCHWARE_LOCAL_BUILD_URL='/home/dly/software/ctags-5.8.tar.gz'
export FETCHWARE_LOCAL_UPGRADE_URL='file:///home/dly/software/fetchware-upgrade'
export FETCHWARE_NONROOT_USER='YOURUSERNAME'
echo 'done.'
else
echo -n 'Deleting fetchware_release_testing environment variables...';
unset FETCHWARE_RELEASE_TESTING
unset FETCHWARE_FTP_LOOKUP_URL
unset FETCHWARE_HTTP_LOOKUP_URL
unset FETCHWARE_FTP_MIRROR_URL
unset FETCHWARE_HTTP_MIRROR_URL
unset FETCHWARE_FTP_DOWNLOAD_URL
unset FETCHWARE_HTTP_DOWNLOAD_URL
unset FETCHWARE_LOCAL_URL
unset FETCHWARE_LOCAL_ZIP_URL
unset FETCHWARE_LOCAL_BUILD_URL
unset FETCHWARE_LOCAL_UPGRADE_URL
unset FETCHWARE_NONROOT_USER
echo 'done.'
fi
lib/App/Fetchware.pm view on Meta::CPAN
=head3 get_lookup_url()
my $lookup_url = get_lookup_url($term);
Uses $term argument as a L<Term::ReadLine>/L<Term::UI> object to interactively
explain what a lookup_url is, and to ask the user to provide one and press
enter.
=head3 download_lookup_url()
my $filename_listing = download_lookup_url($term, $lookup_url);
Attempts to download the lookup_url the user provides. Returns it after parsing
it using parse_directory_listing() from L<App::Fetchware> that lookup() itself
uses.
=head3 get_mirrors()
my $mirrors_hashref = get_mirrors($term, $filename_listing);
# $mirrors_hashref = (
# mirrors => [
# 'ftp://some.mirror/mirror',
# 'http://some.mirror/mirror',
# 'file://some.mirror/mirror',
# ],
# );
Asks the user to specify at least one mirror to use to download their archives.
It also reiterates to the user that the C<lookup_url> should point to the
author's original download site, and B<not> a 3rd party mirror, because md5sums,
sha1sums, and gpg signatures should B<only> be downloaded from the author's
download site to avoid them being modified by a hacked 3rd party mirror. While
C<mirror> should be configured to point to a 3rd party mirror to lessen the load
on the author's offical download site.
After the user enters at least one mirror, get_mirrors() asks the user if they
would like to add any additional mirrors, and it adds them if the user specifies
them.
The list of the mirrors the user specified is returned as a hash with only one
key C<mirror>, and a value that is an arrayref of mirrors that the user has
specified. The caller, then should call append_options_to_fetchwarefile() to add
this list of mirrors to the user's Fetchwarefile.
=head3 get_verification()
my $verification_hashref = get_verification($term, $filename_listing, $lookup_url);
# $verification_hashref = (
# gpg_keys_url => 'http://main.mirror/distdir',
# verification_method => 'gpg',
# );
Parses $filename_listing to determine what type of verification is available.
Prefering gpg, but falling back on sha1, and then md5 if gpg is not available.
If the type is gpg, then get_verification() will ask the user to specify a
C<gpg_keys_url>, which is required for gpg, because fetchware needs to be able
to import the needed keys to be able to use those keys to verify package
downloads. If this URL is not provided by the author, then get_verification()
will ask the user if they would like to import the author's key into their own
gpg public keyring. If they would, then get_verification() will use the
C<user_keyring> C<'On'> option to use the user's public keyring instead of
fetchware's own keyring. And if the user does not want to use their own gpg
public keyring, then get_verification will fall back to sha1 or md5 setting
C<verify_method> to sha1 or md5 as needed.
Also, adds a gpg_keys_url option if a C<KEYS> file is found in
$filename_listing.
If no verification methods are available, fetchware will print a big nasty
warning message, and offer to use C<verify_failure_ok> to make such a failure
cause fetchware to continue installing your software.
Returns a hashref of options for the user's Fetchwarefile. You're responsible
for calling append_options_to_fetchwarefile() to add them to the user's
Fetchwarefile, or perhaps the caller could analyze them in some way, before
adding them if needed. The keys are the names of the configuration options, and
the values are their values.
=head3 get_filter_option()
$filter_hashref = get_filter_option($term, $filename_listing);
# $filter_hashref = (
# filter => 'user specfied filter option',
# );
Analyzes $filename_listing and asks the user whatever questions are needed by
fetchware to determine if a C<filter> configuration option is needed, and if it
is what it should be. C<filter> is simply a perl regex that the list of files
that fetchware downloads is checked against, and only files that match this
regex will fetchware consider to be the latest version of the software package
that you want to install. The C<filter> option is needed, because some mirrors
will have multiple software packages in the same directory or multitple
different versions of one piece of software in the same directory. An example
would be Apache, which has Apache versions 2.0, 2.2, and 2.4 all in the same
directory. The C<filter> option is how you differentiate between them.
If a filter was provided by the user than it is returned as a hashref with
C<filter> as the key for use with append_options_to_fetchwarefile(), or for
further analysis by extension authors.
=head3 prompt_for_other_options()
prompt_for_other_options($term,
temp_dir => {
prompt => <<EOP,
What temp_dir configuration option would you like?
EOP
print_me => <<EOP
temp_dir is the directory where fetchware creates a temporary directory that
stores all of the temporary files it creates while it is building your software.
The default directory is /tmp on Unix systems and C:\\temp on Windows systems.
EOP
},
...
);
Accepts a Term::Readline/Term::UI object as an argument to use to ask the user
questions, and a gigantic hash of hashes in list form. The hash of hashes,
%option_description, argument incluedes the C<prompt> and C<print_me> options
that are then passed through to Term::UI to ask the user what argument they want
for each specified option in the %option_description hash.
The user's answers are tallied up an returned as a hash reference.
=head3 edit_manually()
$fetchwarefile = edit_manually($term, $fetchwarefile);
edit_manually() asks the user if they would like to edit the specified
lib/App/Fetchware.pm view on Meta::CPAN
=back
=back
Downloads $download_path to C<tempdir 'whatever/you/specify';> or to
whatever File::Spec's tempdir() method tries. Supports ftp and http URLs as well
as local files specified like in browsers using C<file://>
Also, returns $package_path, which is used by unarchive() as the path to the
archive for unarchive() to untar or unzip.
=over
=item LIMITATIONS
Uses Net::FTP and HTTP::Tiny to download ftp and http files. No other types of
downloading are supported, and fetchware is stuck with whatever limitations or
bugs Net::FTP or HTTP::Tiny impose.
=back
=over
=item drop_privs() NOTES
This section notes whatever problems you might come accross implementing and
debugging your Fetchware extension due to fetchware's drop_privs mechanism.
See L<Util's drop_privs() subroutine for more info|App::Fetchware::Util/drop_privs()>.
=over
=item *
Under drop_privs() download() is executed in the child with reduced privileges.
=back
=back
=head2 download() API REFERENCE
The subroutines below are used by download() to provide the download
functionality for fetchware. If you have overridden the download() handler, you
may want to use some of these subroutines so that you don't have to copy and
paste anything from download.
App::Fetchware is B<not> object-oriented; therefore, you B<can not> subclass
App::Fetchware to extend it!
=head3 determine_package_path()
my $package_path = determine_package_path($tempdir, $filename)
Determines what $package_path is based on the provided $tempdir and
$filename. $package_path is the path used by unarchive() to unarchive the
software distribution download() downloads.
$package_path is returned to caller.
=head2 verify()
verify($download_path, $package_path)
=over
=item Configuration subroutines used:
=over
=item gpg_keys_url
=item user_keyring
=item gpg_sig_url
=item sha1_url
=item md5_url
=item verify_method
=item verify_failure_ok
=item user_agent
=back
=back
Verifies the downloaded package stored in $package_path by downloading
$download_path.{asc,sha1,md5}> and comparing the two together. Uses the
helper subroutines C<{gpg,sha1,md5,digest}_verify()>.
=over
=item LIMITATIONS
Uses gpg command line, and the interface to gpg is a little brittle.
Crypt::OpenPGP is buggy and not currently maintainted again, so fetchware cannot
make use of it, so were stuck with using the command line gpg program.
=back
=over
=item drop_privs() NOTES
This section notes whatever problems you might come accross implementing and
debugging your Fetchware extension due to fetchware's drop_privs mechanism.
See L<Util's drop_privs() subroutine for more info|App::Fetchware::Util/drop_privs()>.
=over
=item *
Under drop_privs() verify() is executed in the child with reduced privileges.
=back
=back
=head2 verify() API REFERENCE
The subroutines below are used by verify() to provide the verify
functionality for fetchware. If you have overridden the verify() handler, you
may want to use some of these subroutines so that you don't have to copy and
paste anything from verify().
App::Fetchware is B<not> object-oriented; therefore, you B<can not> subclass
App::Fetchware to extend it!
=head3 gpg_verify()
'Package Verified' = gpg_verify($download_path);
Uses the command-line program C<gpg> to cryptographically verify that the file
you download is the same as the file the author uploaded. It uses public-key
priviate-key cryptography. The author signs his software package using gpg or
some other OpenPGP compliant program creating a digital signature file with the
same filename as the software package, but usually with a C<.asc> file name
extension. gpg_verify() downloads the author's keys, imports them into
fetchware's own keyring unless the user sets C<user_keyring> to true in his
Fetchwarefile. Then Fetchware downloads a digital signature that usually
ends in C<.asc>. Afterwards, fetchware uses the gpg command line program to
verify the digital signature. gpg_verify returns true if successful, and throws
an exception otherwise.
You can use C<gpg_keys_url> to specify the URL of a file where the author has
uploaded his keys. And the C<gpg_sig_url> can be used to setup an alternative
location of where the C<.asc> digital signature is stored.
=head3 sha1_verify()
'Package verified' = sha1_verify($download_path, $package_path);
undef = sha1_verify($download_path, $package_path);
Verifies the downloaded software archive's integrity using the SHA Digest
specified by the C<sha_url 'ftp://sha.url/package.sha'> config option. Returns
true for sucess dies on error.
=over
=item SECURITY NOTE
If an attacker cracks a mirror and modifies a software package, they can also
modify the MD5 sum of that software package on that B<same mirror>. Because of
this limitation MD5 sums can only tell you if the software package was corrupted
while downloading. This can actually happen as I've had it happen to me once.
If your stuck with using MD5 sum, because your software package does not provide
gpg signing, I recommend that you download your SHA1 sums (and MD5 sums) from
your software package's master mirror. For example, Apache provides MD5 and SHA1
sums, but it does not mirror them--you must download them directly from Apache's
servers. To do this specify a C<sha1_url 'master.mirror/package.sha1';> in your
Fetchwarefile.
=back
=head3 md5_verify()
'Package verified' = md5_verify($download_path, $package_path);
undef = md5_verify($download_path, $package_path);
Verifies the downloaded software archive's integrity using the MD5 Digest
specified by the C<md5_url 'ftp://sha.url/package.sha'> config option. Returns
true for sucess and dies on error.
=over
=item SECURITY NOTE
If an attacker cracks a mirror and modifies a software package, they can also
modify the MD5 sum of that software package on that B<same mirror>. Because of
this limitation MD5 sums can only tell you if the software package was corrupted
while downloading. This can actually happen as I've had it happen to me once.
If your stuck with using MD5 sum, because your software package does not provide
gpg signing, I recommend that you download your MD5 sums (and SHA1 sums) from
your software package's master mirror. For example, Apache provides MD5 and SHA1
sums, but it does not mirror them--you must download them directly from Apache's
servers. To do this specify a C<md5_url 'master.mirror/package.md5';> in your
Fetchwarefile.
=back
=head3 digest_verify()
'Package verified' = digest_verify($digest_type, $download_path, $package_path);
undef = digest_verify($digest_type, $download_path, $package_path);
Verifies the downloaded software archive's integrity using the specified
$digest_type, which also determines the
C<"$digest_type_url" 'ftp://sha.url/package.sha'> config option. Returns
true for sucess and returns false for failure.
=over
=item OVERRIDE NOTE
If you need to override verify() in your Fetchwarefile to change the type of
digest used, you can do this easily, because digest_verify() uses L<Digest>,
which supports a number of Digest::* modules of different Digest algorithms.
Simply do this by override verify() to call
C<digest_verify('Digest's name for your Digest::* algorithm');>
=back
=over
=item SECURITY NOTE
If an attacker cracks a mirror and modifies a software package, they can also
modify the $digest_type sum of that software package on that B<same mirror>.
Because of this limitation $digest_type sums can only tell you if the software
package was corrupted while downloading. This can actually happen as I've had
it happen to me once.
If your stuck with using $digest_type sum, because your software package does
not provide gpg signing, I recommend that you download your $digest_type sums
(and SHA1 sums) from your software package's master mirror. For example, Apache
provides MD5 and SHA1 sums, but it does not mirror them--you must download them
directly from Apache's servers. To do this specify a
C<$digest_type_url 'master.mirror/package.$digest_type';>' in your Fetchwarefile.
=back
=head2 unarchive()
my $build_path = unarchive($package_path)
=over
=item Configuration subroutines used:
=over
=item none
=back
=back
Uses L<Archive::Tar> or L<Archive::Zip> to turn .tar.{gz,bz2,xz} or .zip into a
directory. Is intelligent enough to warn if the archive being unarchived does
not contain B<all> of its files in a single directory like nearly all software
packages do. Uses $package_path as the archive to unarchive, and returns
$build_path.
=over
=item drop_privs() NOTES
This section notes whatever problems you might come accross implementing and
debugging your Fetchware extension due to fetchware's drop_privs mechanism.
See L<Util's drop_privs() subroutine for more info|App::Fetchware::Util/drop_privs()>.
=over
=item *
Under drop_privs() unarchive() is executed in the child with reduced privileges.
=back
lib/App/Fetchware.pm view on Meta::CPAN
=item Configuration subroutines used:
=over
=item none
=back
=back
Calls check_config_options() to check for the following syntax errors in
Fetchwarefiles. Note by the time check_syntax() has been called
parse_fetchwarefile() has already parsed the Fetchwarefile, and any syntax
errors in the user's Fetchwarefile will have already been reported by Perl.
This may seem like a bug, but it's not. Do you really want to try to use regexes
or something to try to parse the Fetchwarefile reliably, and then report errors
to users? Or add PPI of all insane Perl modules as a dependency just to write
syntax checking code that most of the time says the syntax is Ok anyway, and
therefore a complete waste of time and effort? I don't want to deal with any of
that insanity.
Instead, check_syntax() uses config() to examine the already parsed
Fetchwarefile for "higher level" syntax errors. Syntax errors that are
B<Fetchware> syntax errors instead of just Perl syntax errors.
For yours and my own convienience I created check_config_options() helper
subroutine. Its data driven, and will check Fetchwarefile's for three different
types of common syntax errors that occur in App::Fetchware's Fetchwarefile
syntax. These errors are more at the level of I<logic errors> than actual syntax
errors. See its POD below for additional details.
Below briefly lists what App::Fetchware's implementation of check_syntax()
checks.
=over
=item * Mandatory configuration options
=over
=item * program, lookup_url, and mirror are required for all Fetchwarefiles.
=back
=item * Conflicting configuration options
=over
=item * build_commands conflicts with any of prefix, configure_options, and
make_options.
=back
=item * Ensures some options have only allowable options specified.
=over
=item * lookup_method can only have 'timestamp' or 'versionstring'. as options.
=item * And verify_method can only have 'gpg', 'sha1', or 'md5' specified.
=back
=back
=over
=item drop_privs() NOTES
This section notes whatever problems you might come accross implementing and
debugging your Fetchware extension due to fetchware's drop_privs mechanism.
See L<Util's drop_privs() subroutine for more info|App::Fetchware::Util/drop_privs()>.
=over
=item *
check_syntax() is run in the parent process before even start() has run, so no
temporary directory is available for use.
=back
=back
=head2 check_syntax() API REFERENCE
check_syntax() uses check_config_options() to do the heavy lifting of Set Theory
to check that you do not use some configuration options that are not compatible
with other ones that you have used.
=head3 check_config_options()
check_config_options(
BothAreDefined => [ [qw(build_commands)],
[qw(prefix configure_options make_options)] ],
Mandatory => [ 'program', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a program configuration
option. Please add one, and try again.
EOM
Mandatory => [ 'mirror', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a mirror configuration
option. Please add one, and try again.
EOM
Mandatory => [ 'lookup_url', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a lookup_url configuration
option. Please add one, and try again.
EOM
ConfigOptionEnum => ['lookup_method', [qw(timestamp versionstring)] ],
ConfigOptionEnum => ['verify_method', [qw(gpg sha1 md5)] ],
);
Uses config() to test that no configuration options that clash with each other
are used.
It's parameters are specified in a list with an even number of parameters. Each
group of 2 parameters specifies a type of test that check_config_options() will
test for. There are three types of tests. Also, note that the parameters are
specified as a list not as a hash, because multiple "keys" are allowed, and hash
keys must be unique; therefore, the parameters are a list instead of a hash.
=over
=item BothAreDefined
Is used to test if one or more elements from both provided arrayrefs are defined
at the same time. This is needed, because if you specify C<build_commands> any
value you specify for C<prefix>, C<configure_options>, C<make_options> will be
ignored, which may not be what you expect or want, so BothAreDefined is used to
check for this.
Also, unlike C<Mandatory> and C<ConfigOptionEnum> this syntax checker does not
take a string argument that specifies an error message, because it takes the two
other values that you specifiy, and uses them to fill in its own error message.
=item Mandatory
Is used to check for mandatory options, which just means that these options
absolutely must be specified in user's Fetchwarefiles, and if they are not, then
the provided error message is thrown as an exception.
=item ConfigOptionEnum
Tests that enumerations are valid. For example, C<lookup_method> can only take
two values C<timestamp> or C<versionstring>, and ConfigOptionEnum is used to
test for this.
=back
=head2 end()
end();
=over
=item Configuration subroutines used:
=over
=item none
=back
=back
end() is called after all of the other main fetchware subroutines such as
lookup() are called. It's job is to cleanup after everything else. It just calls
cleanup_tempdir(), which mostly just closes the C<fetchware.sem> fetchware
semaphore file used to lock each fetchware temporary directory so C<fetchware
clean> does not delete it out from under an concurrently running Fetchware
lib/App/Fetchware.pm view on Meta::CPAN
=back
=back
=back
=head1 FAQ
=head2 Why doesn't fetchware and App::Fetchware use OO or Moose?
One of my goals for fetchware was that its guts be pragmatic. I wanted it to
consist of a bunch of subroutines that simply get executed in a specific order.
And these subroutines should be small and do one and only one thing to make
understanding them easier and to make testing them a breeze.
OO is awesome for massive systems that need to take advandtage of inheritance or
perhaps composition. But fetchware is small and simple, and I wanted its
implementation to also be small and simple. It is mostly just two four
thousand line programs with some smaller utility files. If it were OO there
would be half or even a whole dozen number of smaller files, and they would have
complicated relationships with each other. I did not want to bother with
needless abstration that would just make fetchware more complicated. It is a
simple command line program, so it should be written as one.
Moose was out, because I don't need any of its amazing features. I could use
those amazing features, but fetchware's simple nature does not demand them.
Also, unless Moose and its large chunk of dependencies are in your OS's file
system cache, Moose based command line apps take a few seconds to start,
because perl has to do a bunch of IO on slow disks to read in Moose and its
dependencies. I don't want to waste those few seconds. Plus fetchware is a
program not intended for use by experienced Perl developers like dzil is, which
does use Moose, and has a few second start up overhead, which is acceptable to
its author and maintainer. I also use it, and I'm ok with it, but those few
seconds might be off putting to users who have no Perl or Moose knowledge.
=head2 What's up with the insane fetchware extension mechanism?
Extension systems are always insane. dzil, for example, uses a configuration
file where you list names of dzil plugins, and for each name dzil loads that
plugin, and figures out what dzil roles it does, then when dzil executes any
commands you give it, dzil executes all of those roles that the plugins
registered inside you configuration file.
I wanted a configuration file free of declaring what plugins you're using. I
wanted it to be easy to use. dzil is for Perl programmers, so it requiring some
knowledge of Perl and Moose is ok. But fetchware is for end users or perhaps
system administrators not Perl programmers, so something easier is needed.
The extension mechanism was designed for ease of use by people who use your
fetchware extension. And it is. Just "use" whatever fetchware extension you want
in your Fetchwarefile, and then supply whatever configuration options you
need.
This extension mechanism is also very easy for Perl programmers, because you're
basically I<subclassing> App::Fetchware, only you do it using
L<App::Fetchware::ExportAPI> and L<App::Fetchware::CreateConfigOptions>. See
section L<Implement your fetchware extension.> for full details.
=head2 How do I fix the verification failed error.
Fetchware is designed to always attempt to verify the software archives it
downloads even if you failed to configure fetchware's verification settings. It
will try to guess what those setting should be using simple heuristics. First it
will try gpg verificaton, then sha1 verification, and finally md5 verification.
If all fail, then fetchware exit failure with an appropriate error message.
When you get this error message
L<read fetchware's documentation on how to set this up|/4. Add mandatory verification settings>.
=head2 How do I make fetchware log to a file instead of STDOUT?
You can't fetchware does not have any log file support. However, you can simply
redirect STDOUT to a file to make your shell redirect STDOUT to a file for you.
fetchware install <some-program.Fetchwarefile> > fetchware.log
This would not prevent any error messages from STDERR being printed to your
screen for that:
fetchware install <some-program.Fetchwarefile> 2>&1 fetchware.log
And to throw away all messages use:
fetchware -q install <some-progra.Fetchwarefile>
or use the shell
fetchware install <some-program.Fetchwarefile 2>&1 /dev/null
=head2 Why don't you use Crypt::OpenPGP instead of the gpg command line program?
I tried to use Crypt::OpenPGP, but I couldn't get it to work. And getting gpg to
work was a breeze after digging through its manpage to find the right command
line options that did what I need it to.
Also, unfortunately Crypt::OpenPGP is buggy, out-of-date, and seems to have
lost another maintainer. If it ever gets another maintainer, who fixes the newer
bugs, perhaps I'll add support for Crypt::OpenPGP again. Because of how
fetchware works it needs to use supported but not popular options of
Crypt::OpenPGP, which may be where the bugs preventing it from working reside.
Supporting Crypt::OpenPGP is still on my TODO list. It's just not very high on
that list. Patches are welcome to add support for it, and the old code is still
there commented out, but it needs updating if anyone is interested.
In the meantime if you're on Windows without simple access to a gpg command line
program, try installing gpg from the L<gpg4win project|http://gpg4win.org/>,
which packages up gpg and a bunch of other tools for easier use on Windows.
=head2 Does fetchware support Windows?
Yes and no. I intend to support Windows, but right now I'm running Linux, and my
Windows virtual machine is broken, so I can't easily test it on Windows. The
codebase makes heavy use of File::Spec and Path::Class, so all of its file
operations should work on Windows.
I currently have not tested fetchware on Windows. There are probably some test
failures on Windows, but Windows support should be just a few patches away.
So the real answer is no, but I intend to support Windows in a future release.
( run in 0.878 second using v1.01-cache-2.11-cpan-39bf76dae61 )