App-Fetchware
view release on metacpan or search on metacpan
t/App-Fetchware-new-get_filter_option
t/App-Fetchware-new-get_mirrors
t/App-Fetchware-new-get_verification
t/App-Fetchware-new-prompt_for_other_options
t/App-Fetchware-new.t
t/App-Fetchware-new_install.t
t/App-Fetchware-start.t
t/App-Fetchware-unarchive.t
t/App-Fetchware-uninstall.t
t/App-Fetchware-upgrade.t
t/App-Fetchware-verify.t
t/App-FetchwareX-HTMLPageSync-new
t/App-FetchwareX-HTMLPageSync-new-ask_about_keep_destination_directory
t/App-FetchwareX-HTMLPageSync-new-get_destination_directory
t/App-FetchwareX-HTMLPageSync-new-get_html_page_url
t/App-FetchwareX-HTMLPageSync.t
t/Test-Fetchware.t
t/author-pod-coverage.t
t/author-pod-syntax.t
t/bin-fetchware-Fetchwarefile.t
t/bin-fetchware-clean.t
bin/fetchware view on Meta::CPAN
#it.
#If no filename was
#provided or the filename doesn't exist then, cmd_install() calls new() to create
#and install a new fetchware package.
sub cmd_install {
# These variables must be shared back to the parent from the child using
# pipe_{write,read}_newline().
my $P_build_path;
###BUGALERT### After verifying basic functionality of cmd_install wrap
#subroutine contents in a for my $filename (pop @ARGV) loop to try to
#install all given arguments that arn't command line options as parsed by
#GetOpt::Long.
### Add this loop in run(), so there is just one loop to test.
my $filename = shift;
my $output;
if (defined($filename) and -e $filename) {
msg "Starting fetchware install to install [$filename]";
bin/fetchware view on Meta::CPAN
#download_urlif possible, compare with the one in the fetchware
#package database, and call exit right here if the current version
#is already installed unless of course --force is used!!!
my $download_url = lookup();
my $package_path = download($temp_dir, $download_url);
###BUGALERT### Add support for caching the key files gpg creates to
#the fetchwarefile, and for actually using them later on inside the
#fpkg.
verify($download_url, $package_path);
$P_build_path = unarchive($package_path);
build($P_build_path);
# Tell the parent, root, process the values of the variables the
# child calculated in this coderef, and write them across this pipe
# back to the parent
write_dropprivs_pipe($write_pipe, $P_build_path);
}, config('user')
bin/fetchware view on Meta::CPAN
# Return the name of the uninstalled package's full path fetchware's
# database.
return $fetchware_package_path;
}
###BUGALERT### Move cmd_new() before install()?????
###BUGALERT### Print out fetchware's assumptions it makes about what FTP & hTTP
#lookup_url's look like, versionstring's assumptions, timestamp's assumptions,
#verify's assumptions, and so on. If not here in new() at least do it in the
#POD documentation.
###BUGALERT### Support ~/.Fetchwarefile, or whatever File::HomeDir wants it to
#be. Test if ~/.Fetchwarefile exists, if it does do nothing, but if it does not
#exist then prompt the user to fill one out!!!
############BUGALERT########################BUGALERT##################
############BUGALERT########################BUGALERT##################
###BUGALERT### Modify analyze_lookup_listing() to print the directory listing
#for the user to peruse, and have the user choose what program they want to
#install from the listing. Then use that as the basis for the filter option.
bin/fetchware view on Meta::CPAN
# ($P_fetchware_package_path).
my $P_upgrade = upgrade($P_download_path, $P_fetchware_package_path);
if ($P_upgrade) {
msg 'New version available upgrading now.';
my $package_path = download($temp_dir, $P_download_path);
###BUGALERT### Add support for caching the key files gpg creates to the
#fetchwarefile, and for actually using them later on inside the fpkg.
verify($P_download_path, $package_path);
$P_build_path = unarchive($package_path);
build($P_build_path);
} else {
# If a new version is not available, then the child should do
# nothing, and let the parent call end() to clean up below.
# Set $P_build_path to something that will fail, and give a decent
# error message just in case.
$P_build_path = 'Build Path not set because upgrade not needed.';
bin/fetchware view on Meta::CPAN
msg 'Downloading and unarchiving specified distribution.';
my $download_url = lookup();
my $package_path = download(cwd(), $download_url);
###BUGALERT### Add support for caching the key files gpg creates to
#the fetchwarefile, and for actually using them later on inside the
#fpkg.
verify($download_url, $package_path);
my $build_path = unarchive($package_path);
# end() is *not* run, because the point of look is to lookup,
# download, and unarchive, and then actually "look" at the files,
# and running end() would delete them.
# Compose the $P_look_path. A simple catfile($temp_dir, $build_path)
# should work, but don't forget about drop_privs() extra temporary
# directory when run as root! To avoid this problem of the $P_look_path
bin/fetchware view on Meta::CPAN
with your Fetchware package (*.fpkg). The error was [$@].
EOD
# Ensure that the specified App::Fetchware implementation exports the proper
# subroutines.
my %api_subs = (
start => 1,
lookup => 1,
download => 1,
verify => 1,
unarchive => 1,
build => 1,
install => 1,
uninstall => 1,
upgrade => 1,
check_syntax => 1,
end => 1,
);
# Determine if all of the @api_subs are in sublist, the list of all subs in
bin/fetchware view on Meta::CPAN
if (not exists $intersection{$api_sub}
or not defined $intersection{$api_sub}
or ($intersection{$api_sub} == 0)
) {
push @missing_api_subs, $api_sub;
}
}
die <<EOD;
fetchware: The App::Fetchware module you choose in your fetchwarefile does not
properly export the necessary subroutines fetchware needs it to. These include:
start(), lookup(), download(), verify, unarchive(), build(), install(),
uninstall(), and end().
The missing subroutines are [@missing_api_subs].
EOD
}
# Call App::Fetchware's check_syntax() (or a App::Fetchware extension's).
check_syntax();
return 'Evaled config file successfully';
}
bin/fetchware view on Meta::CPAN
Fetchware is a package manager for source code distributions. It takes advantage
of the fact that coincidentially I<most> source code distributions follow the same
conventions. Most use FTP and HTTP mirrors. Most use AutoTools or at least just
a few commands that you execute in sequence to configure, build, and install the
program.
Fetchware harnesses these conventions to create a powerful and flexible package
manager for source code distributions. It includes a simple, powerful, and
flexible configuration syntax stored in files called C<Fetchwarefile>s. These
C<Fetchwarefile>s specify the required mandatory configuration options,
C<program>, C<lookup>, C<mirror>, and a method of verifying your program. And
they also specify any additional optional configuration options.
To create a new Fetchwarefile to install a source code distribution use the
L<fetchware new|/new> command. It will ask you a bunch of questions, and based
on your answers and fetchware's assumptions fetchware will automagically create
a new Fetchwarefile for you. Then it will ask if you would like fetchware to
install it for you.
If your source code distribution exceeds fetchware's new command's capabilities,
then see the section L<MANUALLY CREATING A App::Fetchware FETCHWAREFILE> in
bin/fetchware view on Meta::CPAN
=item L<checkinstall|http://www.debian-administration.org/articles/147>
Run like C<checkinstall make install> will detect what files are copied where
during installation, and will create a slackware, debian, or redhat package
based on this information.
=item L<paco|http://paco.sourceforge.net/>
Provides very similar functionality to fetchware, but lacks fetchware's lookup
and verify mechanisms. Includes its own package management functionality.
=back
=back
As far as fetchware one day supporting some sort of hack like checkinstall or
paco use, I'm against it. I'd prefer everyone just adding a C<make uninstall> to
their Makefiles. But it is on my todo list, and I may add similar functionality
in the future, but I'll make no promises. Until then consider using the
C<prefix> and C<uninstall_commands> hack.
bin/fetchware view on Meta::CPAN
=item *
Lookup to see if a new version is available (lookup())
=item *
Downloading the archive (download())
=item *
Verifying that the downloaded file is the same one the author uploaded (verify())
=item *
Unarchiving the package (unarchive())
=item *
Building and installing it (build() and install())
=item *
bin/fetchware view on Meta::CPAN
List just globs all files in the fetchware database directory as returned by
fetchware_database_path(), and prints them to STDOUT. It does not let you
specify a Perl regex, or a keyword or anything yet, because I'm currently unsure
about the security ramifications of doing so. This feature may be added in the
future.
=item look
look just does the first part of install(). It parses whatever Fetchwarefile it
gets passed to it, then it does the start(), lookup(), download(), verify(), and
unarchive() parts of install(). Then look prints the path of this directory, and
exits.
=item clean
Clean just deletes all fetchware temp files and directories in the system
temp_dir. These files and directories all start with C<fetchware-*> or
C<Fetchwarefile-*>.
=item help
lib/App/Fetchware.pm view on Meta::CPAN
build_commands
install_commands
uninstall_commands
lookup_url
lookup_method
gpg_keys_url
gpg_sig_url
sha1_url
md5_url
user_agent
verify_method
no_install
verify_failure_ok
user_keyring
stay_root
mirror
config
new
new_install
check_syntax
start
lookup
download
verify
unarchive
build
install
end
uninstall
upgrade
hook
);
lib/App/Fetchware.pm view on Meta::CPAN
http_parse_filelist
file_parse_filelist
lookup_by_timestamp
lookup_by_versionstring
lookup_determine_downloadpath
)],
OVERRIDE_DOWNLOAD => [qw(
determine_package_path
)],
OVERRIDE_VERIFY => [qw(
gpg_verify
sha1_verify
md5_verify
digest_verify
)],
OVERRIDE_UNARCHIVE => [qw(
check_archive_files
list_files
list_files_tar
list_files_zip
unarchive_package
unarchive_tar
unarchive_zip
)],
lib/App/Fetchware.pm view on Meta::CPAN
[ build_commands => 'ONEARRREF' ],
[ install_commands => 'ONEARRREF' ],
[ uninstall_commands => 'ONEARRREF' ],
[ lookup_url => 'ONE' ],
[ lookup_method => 'ONE' ],
[ gpg_keys_url => 'ONE' ],
[ gpg_sig_url => 'ONE' ],
[ sha1_url => 'ONE' ],
[ md5_url => 'ONE' ],
[ user_agent => 'ONE' ],
[ verify_method => 'ONE' ],
[ mirror => 'MANY' ],
[ no_install => 'BOOLEAN' ],
[ verify_failure_ok => 'BOOLEAN' ],
[ stay_root => 'BOOLEAN' ],
[ user_keyring => 'BOOLEAN' ],
);
# Loop over the list of options needed by _make_config_sub() to generated the
# needed API functions for Fetchwarefile.
for my $api_function (@api_functions) {
_make_config_sub(@{$api_function});
}
lib/App/Fetchware.pm view on Meta::CPAN
sha1_url => <<EOA,
sha1_url specfies the url that fetchware uses to download sha1sum files of
this program. This url should be the program's main download site instead of a
mirror, because a hacked mirror could alter the sha1sum on that mirror.
EOA
md5_url => <<EOA,
md5_url specfies the url that fetchware uses to download md5sum files of
this program. This url should be the program's main download site instead of a
mirror, because a hacked mirror could alter the md5sum on that mirror.
EOA
verify_method => <<EOA,
verify_method specifes a specific method that fetchware should use to verify
your program. This method can be 'gpg', 'sha1', or 'md5'.
EOA
no_install => <<EOA,
no_install specifies that this software should not be installed. Instead, the
install step is skipped, and fetchware prints to STDOUT where it downloaded,
verified, and built your program. no_install must be a true or false value.
EOA
verify_failure_ok => <<EOA,
verify_failure_ok specifies that fetchware should not stop installing your
software and terminate with an error message if fetchware fails to verify your
software. You should never set this to true. Doing so could cause fetchware to
install software that may have been compromised, or had malware inserted into
it. Never use this option unless the author or maintainer of this program does
not gpg sign or checksum his software.
EOA
user_keyring => <<EOA,
users_keyring if enabled causes fetchware to use the user's own gpg keyring
instead of fetchware's own keyring.
EOA
mirror => <<EOA
lib/App/Fetchware.pm view on Meta::CPAN
vmsg "Downloaded lookup_url [$lookup_url]";
my $filename_listing = download_lookup_url($term, $lookup_url);
vmsg "Downloaded lookup_url's directory listing";
vmsg Dumper($filename_listing);
my $mirrors_hashref = get_mirrors($term, $filename_listing);
vmsg "Added mirrors to your Fetchwarefile.";
vmsg Dumper($mirrors_hashref);
my $verify_hashref = get_verification($term, $filename_listing, $lookup_url);
vmsg "Added verification settings to Fetchwarefile.";
vmsg Dumper($verify_hashref);
my $filter_hashref = get_filter_option($term, $filename_listing);
vmsg "Added [$filter_hashref->{filter}] filter setting to Fetchwarefile.";
$fetchwarefile->config_options(
%$mirrors_hashref,
%$verify_hashref,
%$filter_hashref
);
###BUGALERT### Ask to parrallelize make with make_options???
###BUGALERT### Verify prefix is writable by current user, who will
#presumably be the user who will install the package now and later.
###BUGALERT### Ask user for a prefix if their running nonroot???
vmsg 'Prompting for other options that may be needed.';
my $other_options_hashref = prompt_for_other_options($term,
temp_dir => {
lib/App/Fetchware.pm view on Meta::CPAN
contain your program author's gpg keys to import into gpg.
EOP
},
gpg_sig_url => {
prompt => <<EOP,
What gpg_sig_url configuration option would you like?
EOP
print_me => <<EOP
gpg_sig_url specifies a url similar to lookup_url in that it should specify a
directory instead a specific file. It is used to download gpg signatures to
verify your software package.
EOP
},
sha1_url => {
prompt => <<EOP,
What sha1_url configuration option would you like?
EOP
print_me => <<EOP
sha1_url specifies a url similar to lookup_url in that it should specify a
directory instead of a specific file. It is separate from lookup_url, because
you should download software from mirrors, but checksums from the original
lib/App/Fetchware.pm view on Meta::CPAN
What md5_url configuration option would you like?
EOP
print_me => <<EOP,
md5_url specifies a url similar to lookup_url in that it should specify a
directory instead of a specific file. It is separate from lookup_url, because
you should download software from mirrors, but checksums from the original
vendor's server, because checksums are easily replaced on a mirror by a hacker
if the mirror gets hacked.
EOP
},
verify_method => {
prompt => <<EOP,
What verify_method configuration option would you like?
EOP
print_me => <<EOP,
verify_method specifies what method of verification fetchware should use to
ensure the software you have downloaded has not been tampered with. The default
is to try gpg verification, then sha1, and then finally md5, and if they all
fail an error message is printed and fetchware exits, because if your software
package cannot be verified, then it should not be installed. This configuration
option allows you to remove the warnings by specifying a specific way of
verifying your software has not been tampered with. To disable verification set
the 'verify_failure_ok' configuration option to true.
EOP
},
###BUGALERT### replace no_install config su with a command line option that
#would be the opposite of --force???
# Nah! Leave it! Just create a command line option for it too!
no_install => {
prompt => <<EOP,
Would you like to enable the no_install configuration option?
EOP
###BUGALERT### no_install is not currently implemented properly!!!
print_me => <<EOP
no_install is a true or false option, whoose acceptable values include 1
or 0, true or falue, On or Off. It's default value is false, but if you enable
it, then fetchware will not install your software package, and instead it will
simply download, verify, and build it. And then it will print out the full path
of the directory it built your software package in.
EOP
###BUGALERT### Add support for a check regex, so that I can ensure
#that what the user enters will be either true or false!!!
},
verify_failure_ok => {
prompt => <<EOP,
Would you like to enable the verify_failure_ok configuration option?
EOP
print_me => <<EOP
verify_failure_ok is a true or false option, whoose acceptable values include 1
or 0, true or falue, On or Off. It's default value is false, but if you enable
it, then fetchware will not print an error message and exit if verification
fails for your software package. Please note that you should never use this
option, because it makes it possible for fetchware to install source code that
may have been tampered with.
EOP
},
users_keyring => {
prompt => <<EOP,
Would you like to enable users_keyring configuration option?
lib/App/Fetchware.pm view on Meta::CPAN
return {mirror => \@mirrors};
}
sub get_verification {
my ($term, $filename_listing, $lookup_url) = @_;
my %options;
my %available_verify_methods;
# Determine what types of verification are available.
for my $file_and_timestamp (@$filename_listing) {
if ($file_and_timestamp->[0] =~ /\.(asc|sig|sign)$/) {
$available_verify_methods{gpg}++;
} elsif ($file_and_timestamp->[0] =~ /\.sha1?$/) {
$available_verify_methods{sha1}++;
} elsif ($file_and_timestamp->[0] =~ /\.md5$/) {
$available_verify_methods{md5}++;
}
}
my $verify_configed_flag = 0;
#If gpg is available prefer it over the others.
if (exists $available_verify_methods{gpg}
and defined $available_verify_methods{gpg}
and $available_verify_methods{gpg} > 0
) {
msg <<EOM;
gpg digital signatures found. Using gpg verification.
EOM
$options{verify_method} = 'gpg';
# Search for a KEYS file to use to import the author's keys.
if (grep {$_->[0] eq 'KEYS'} @$filename_listing) {
msg <<EOM;
KEYS file found using lookup_url. Adding gpg_keys_url to your Fetchwarefile.
EOM
# Add 'KEYS' or '/KEYS' to $lookup_url's path.
my ($scheme, $auth, $path, $query, $fragment) =
uri_split($lookup_url);
$path = catfile($path, 'KEYS');
$lookup_url = uri_join($scheme, $auth, $path, $query, $fragment);
$options{gpg_keys_url} = $lookup_url;
$verify_configed_flag++;
} else {
msg <<EOM;
KEYS file *not* found!
EOM
# Since autoconfiguration of KEYS failed, try asking the user if
# they would like to import the author's key themselves into their
# own keyring and have fetchware use that.
if (
$term->ask_yn(prompt =>
q{Would you like to import the author's key yourself after fetchware completes? },
lib/App/Fetchware.pm view on Meta::CPAN
user_keyring, which if true will cause fetchware to use the user who runs
fetchware's keyring instead of fetchware's own keyring. But you, the user, needs
to import the author's keys into your own gpg keyring. You can do this now in a
separate shell, or after you finish configuring this Fetchwarefile. Just run the
command [gpg --import <name of file>].
EOP
)
) {
$options{user_keyring} = 'On';
$verify_configed_flag++;
}
# And if the user does not want to, then fallback to sha1 and/or md5
# if they're defined, which is done below.
}
}
# Only try sha1 and md5 if gpg failed.
unless ($verify_configed_flag == 1) {
if (exists $available_verify_methods{sha1}
and defined $available_verify_methods{sha1}
and $available_verify_methods{sha1} > 0
) {
msg <<EOM;
SHA1 checksums found. Using SHA1 verification.
EOM
$options{verify_method} = 'sha1';
} elsif (exists $available_verify_methods{md5}
and defined $available_verify_methods{md5}
and $available_verify_methods{md5} > 0
) {
msg <<EOM;
MD5 checksums found. Using MD5 verification.
EOM
$options{verify_method} = 'md5';
} else {
# Print a huge long nasty warning even include links to news stories
# of mirrors actually getting hacked and serving malware, which
# would be detected and prevented with proper verification enabled.
# Ask user if they would like to continue installing fetchware even if
# verification fails, and then enable the verify_failure_ok option.
if (
$term->ask_yn(prompt => <<EOP,
Would you like fetchware to ignore the fact that it is unable to verify the
authenticity of any downloads it makes? Are you ok with possibly downloading
viruses, worms, rootkits, or any other malware, and installing it possibly even
as root?
EOP
default => 'n',
print_me => <<EOP,
Automatic verification of your fetchware package has failed! Fetchware is
capable of ignoring the error, and installing software packages anyway using its
verify_failure_ok configuration option. However, installing software packages
without verifying that they have not been tampered with could allow hackers to
potentially install malware onto your computer. Don't think this is *not*
possible or do you think its extremely unlikely? Well, it's actually
surprisingly common:
1. http://arstechnica.com/security/2012/09/questions-abound-as-malicious-phpmyadmin-backdoor-found-on-sourceforge-site/
Discusses how a mirror for sourceforge was hacked, and the phpMyAdmin
software package on that mirror was modified to spread malware.
2. http://www.geek.com/news/major-open-source-code-repository-hacked-for-months-says-fsf-551344/
Discusses how FSF's gnu.org ftp download site was hacked.
3. http://arstechnica.com/security/2012/11/malicious-code-added-to-open-source-piwik-following-website-compromise/
Discusses how Piwiki's wordpress software was hacked, and downloads of
Piwiki had malicious code inserted into them.
4. http://www.theregister.co.uk/2011/03/21/php_server_hacked/
Discusses how php's wiki.php.org server was hacked yielding credentials to
php's source code repository.
Download mirrors *do* get hacked. Do not make the mistake, and think that it is
not possible. It is possible, and it does happen, so please properly configure
your Fetchwarefile to enable fetchware to verify that the downloaded software is
the same what the author uploaded.
EOP
)
) {
# If the user is ok with not properly verifying downloads, then
# ignore the failure, and install anyway.
$options{verify_failure_ok} = 'On';
} else {
# Otherwise, throw an exception.
die <<EOD;
fetchware: Fetchware *must* be able to verify any software packages that it
downloads. The Fetchwarefile that you were creating could not do this, because
you failed to specify how fetchware can verify its downloads. Please rerun
fetchware new again, and this time be sure to specify a gpg_keys_url, specify
user_keyring to use your own gpg keyring, or answer yes to the question
regarding adding verify_failure_ok to your Fetchwarefile to make failing
verificaton acceptable to fetchware.
EOD
}
}
}
return \%options;
}
lib/App/Fetchware.pm view on Meta::CPAN
# return $package_path, which stores the full path of where the file
# HTTP::Tiny downloaded.
###BUGALERT### $tempdir is no longer used, so remove it from
#determine_package_path() and probably download() too.
return catfile(cwd(), $filename)
}
sub verify {
my ($download_path, $package_path) = @_;
msg "Verifying the downloaded package [$package_path]";
my $retval;
unless (defined(config('verify_method'))) {
# if gpg fails try
# sha and if it fails try
# md5 and if it fails die
msg 'Trying to use gpg to cyptographically verify downloaded package.';
my ($gpg_err, $sha_err, $md5_err);
eval {$retval = gpg_verify($download_path)};
$gpg_err = $@;
if ($gpg_err) {
msg <<EOM;
Cyptographic verification using gpg failed!
GPG verification error [
$@
]
EOM
warn $gpg_err;
}
if (! $retval or $gpg_err) {
msg <<EOM;
Trying SHA1 verification of downloaded package.
EOM
eval {$retval = sha1_verify($download_path, $package_path)};
$sha_err = $@;
if ($sha_err) {
msg <<EOM;
SHA1 verification failed!
SHA1 verificaton error [
$@
]
EOM
warn $sha_err;
}
if (! $retval or $sha_err) {
msg <<EOM;
Trying MD5 verification of downloaded package.
EOM
eval {$retval = md5_verify($download_path, $package_path)};
$md5_err = $@;
if ($md5_err) {
msg <<EOM;
MD5 verification failed!
MD5 verificaton error [
$@
]
EOM
warn $md5_err;
}
}
if (! $retval or $md5_err) {
die <<EOD unless config('verify_failure_ok');
App-Fetchware: run-time error. Fetchware failed to verify your downloaded
software package. You can rerun fetchware with the --force option or add
[verify_failure_ok 'True';] to your Fetchwarefile. See the section VERIFICATION
FAILED in perldoc fetchware.
EOD
}
if (config('verify_failure_ok')) {
warn <<EOW;
App-Fetchware: run-time warning. Fetchware failed to verify the integrity of you
downloaded file [$package_path]. This is ok, because you asked Fetchware to
ignore its errors when it tries to verify the integrity of your downloaded file.
You can also ignore the errors Fetchware printed out abover where it tried to
verify your downloaded file. See perldoc App::Fetchware.
EOW
vmsg <<EOM;
Verification Failed! But you asked to ignore verification failures, so this
failure is not fatal.
EOM
return 'warned due to verify_failure_ok'
}
}
} elsif (config('verify_method') =~ /gpg/i) {
vmsg <<EOM;
You selected gpg cryptographic verification. Verifying now.
EOM
###BUGALERT### Should trap the exception {gpg,sha1,md5}_verify()
#throws, and then add that error to the one here, otherwise the
#error message here is never seen.
gpg_verify($download_path)
or die <<EOD unless config('verify_failure_ok');
App-Fetchware: run-time error. You asked fetchware to only try to verify your
package with gpg or openpgp, but they both failed. See the warning above for
their error message. See perldoc App::Fetchware.
EOD
} elsif (config('verify_method') =~ /sha1?/i) {
vmsg <<EOM;
You selected SHA1 checksum verification. Verifying now.
EOM
sha1_verify($download_path, $package_path)
or die <<EOD unless config('verify_failure_ok');
App-Fetchware: run-time error. You asked fetchware to only try to verify your
package with sha, but it failed. See the warning above for their error message.
See perldoc App::Fetchware.
EOD
} elsif (config('verify_method') =~ /md5/i) {
vmsg <<EOM;
You selected MD5 checksum verification. Verifying now.
EOM
md5_verify($download_path, $package_path)
or die <<EOD unless config('verify_failure_ok');
App-Fetchware: run-time error. You asked fetchware to only try to verify your
package with md5, but it failed. See the warning above for their error message.
See perldoc App::Fetchware.
EOD
} else {
die <<EOD;
App-Fetchware: run-time error. Your fetchware file specified a wrong
verify_method option. The only supported types are 'gpg', 'sha', 'md5', but you
specified [@{[config('verify_method')]}]. See perldoc App::Fetchware.
EOD
}
msg 'Verification succeeded.';
}
sub gpg_verify {
my $download_path = shift;
my $keys_file;
# Attempt to download KEYS file in lookup_url's containing directory.
# If that fails, try gpg_keys_url if defined.
# Import downloaded KEYS file into a local gpg keyring using gpg command.
# Determine what URL to use to download the signature file *only* from
# lookup_url's host, so that we only download the signature from the
# project's main mirror.
# Download it.
# gpg verify the sig using the downloaded and imported keys in our local
# keyring.
# Skip downloading and importing keys if we're called from inside a
# fetchware package, which should already have a copy of our package's
# KEYS file.
unless (config('user_keyring')
or (-e './pubring.gpg' and -e './secring.gpg')) {
# Obtain a KEYS file listing everyone's key that signs this distribution.
if (defined config('gpg_keys_url')) {
$keys_file = no_mirror_download_file(config('gpg_keys_url'));
} else {
eval {
$keys_file = no_mirror_download_file(config('lookup_url'). '/KEYS');
};
die <<EOD if $@;
App-Fetchware: Fetchware was unable to download the gpg_key_url you specified or
that fetchware tried appending asc, sig, or sign to [@{[config('lookup_url')]}].
It needs to download this file to properly verify you software package. This is
a fatal error, because failing to verify packages is a perferable default over
potentially installing compromised ones. If failing to verify your software
package is ok to you, then you may disable verification by adding
verify_failure_ok 'On'; to your Fetchwarefile. See perldoc App::Fetchware.
EOD
}
# Import downloaded KEYS file into a local gpg keyring using gpg
# command.
eval {
# Add --homedir option if needed.
if (config('user_keyring')) {
run_prog('gpg', '--import', $keys_file);
} else {
lib/App/Fetchware.pm view on Meta::CPAN
$sig_url = uri_join($scheme, $auth, "$path.$ext", undef, undef);
$sig_file = no_mirror_download_file($sig_url);
};
# If the file was downloaded stop trying other extensions.
last if defined $sig_file;
}
die <<EOD if not defined $sig_file;
App-Fetchware: Fetchware was unable to download the gpg_sig_url you specified or
that fetchware tried appending asc, sig, or sign to [$sig_url]. It needs
to download this file to properly verify you software package. This is a fatal
error, because failing to verify packages is a perferable default over
potentially installing compromised ones. If failing to verify your software
package is ok to you, then you may disable verification by adding
verify_failure_ok 'On'; to your Fetchwarefile. See perldoc App::Fetchware.
EOD
###BUGALERT### # Use Crypt::OpenPGP if its installed.
###BUGALERT### if (eval {use Crypt::OpenPGP}) {
##DOESNTWORK?? # Build a pubring needed for verify.
##DOESNTWORK?? my $pubring = Crypt::OpenPGP::KeyRing->new();
##DOESNTWORK?? my $secring = Crypt::OpenPGP::KeyRing->new();
##DOESNTWORK??
##DOESNTWORK?? # Turn on gpg compatibility just in case its needed.
##DOESNTWORK?? my $pgp = Crypt::OpenPGP->new(
##DOESNTWORK?? Compat => 'GnuPG',
##DOESNTWORK?? PubRing => $pubring,
##DOESNTWORK?? SecRing => $secring,
##DOESNTWORK?? # Automatically download public keys as needed.
##DOESNTWORK?? AutoKeyRetrieve => 1,
##DOESNTWORK?? # Use this keyserver to download them from.
##DOESNTWORK?? KeyServer => 'pool.sks-keyservers.net',
##DOESNTWORK?? );
##DOESNTWORK??
##DOESNTWORK?? # Verify the downloaded file.
##DOESNTWORK?? my $retval = $pgp->verify(SigFile => $sig_file, Files => $CONFIG{PackagePath});
##DOESNTWORK?? if ($retval == 0) {
##DOESNTWORK?? warn "Crypt::OpenPGP failed due to invalid signature.";
##DOESNTWORK?? # return failure, because Fetchware failed to verify the downloaded
##DOESNTWORK?? # file.
##DOESNTWORK?? return undef;
##DOESNTWORK?? } elsif ($retval) {
##DOESNTWORK?? return 'Package verified';
##DOESNTWORK?? } else {
##DOESNTWORK?? # print warning about $pgp errstr message.
##DOESNTWORK?? my $errstr = $pgp->errstr();
##DOESNTWORK?? warn "Crypt::OpenPGP failed with message: [$errstr]";
##DOESNTWORK?? # return failure, because Fetchware failed to verify the downloaded
##DOESNTWORK?? # file.
##DOESNTWORK?? return undef;
##DOESNTWORK?? }
###BUGALERT### } else {
###BUGALERT### ###BUGALERT### eval the run_prog()'s below & add better error reporting in
###BUGALERT### ###BUGALERT### if Crypt::OpenPGP works ok remove gpg support & this if &
###BUGALERT### }
#IPC::System::Simple dependency.
#my standard format.
# Use automatic key retrieval & a cool pool of keyservers
###BUGALERT## Give Crypt::OpenPGP another try with
#pool.sks-keyservers.net
###BUGALERT### Should I cache the files gpg puts in its "homedir"? They
#are the public keys that verify this fetchware package. Or should they
#always be downloaded on demand as they are now??? But if verify() can
#have keys cached inside the fetchware package does that mean that I
#should open up this as an API for fetchware extensions????? I don't
#know. I'll have to think more about this issue.
#run_prog('gpg', '--keyserver', 'pool.sks-keyservers.net',
# '--keyserver-options', 'auto-key-retrieve=1',
# '--homedir', '.', "$sig_file");
# Verify sig.
# Add --homedir option if needed.
if (config('user_keyring')) {
run_prog('gpg', '--verify', $sig_file);
} else {
run_prog('gpg', '--homedir', '.', '--verify', $sig_file);
}
# Return true indicating the package was verified.
return 'Package Verified';
}
sub sha1_verify {
my ($download_path, $package_path) = @_;
return digest_verify('SHA-1', $download_path, $package_path);
}
sub md5_verify {
my ($download_path, $package_path) = @_;
return digest_verify('MD5', $download_path, $package_path);
}
sub digest_verify {
my ($digest_type, $download_path, $package_path) = @_;
# Turn SHA-1 into sha1 & MD5 into md5.
my $digest_ext = $digest_type;
$digest_ext = lc $digest_type;
$digest_ext =~ s/-//g;
##subify get_sha_sum()
my $digest_file;
# Obtain a sha sum file.
if (defined config("${digest_ext}_url")) {
lib/App/Fetchware.pm view on Meta::CPAN
my (undef, undef, $path, undef, undef) = uri_split($download_path);
my ($scheme, $auth, undef, undef, undef) =
uri_split(config('lookup_url'));
my $digest_url = uri_join($scheme, $auth, $path, undef, undef);
msg "Downloading $digest_ext digest using [$digest_url.$digest_ext]";
$digest_file = no_mirror_download_file("$digest_url.$digest_ext");
};
if ($@) {
die <<EOD;
App-Fetchware: Fetchware was unable to download the $digest_type sum it needs to
download to properly verify you software package. This is a fatal error, because
failing to verify packages is a perferable default over potentially installin
compromised ones. If failing to verify your software package is ok to you, then
you may disable verification by adding verify_failure_ok 'On'; to your
Fetchwarefile. See perldoc App::Fetchware.
EOD
}
}
###BUGALERT###subify calc_sum()
# Open the downloaded software archive for reading.
my $package_fh = safe_open($package_path, <<EOD);
App-Fetchware: run-time error. Fetchware failed to open the file it downloaded
while trying to read it in order to check its MD5 sum. The file was
lib/App/Fetchware.pm view on Meta::CPAN
EOD
# Will only check the first checksum it finds.
while (<$digest_fh>) {
next if /^\s+$/; # skip whitespace only lines just in case.
my @fields = split ' '; # Defaults to $_, which is filled in by <>
# Search the @fields for a regex that is either 32 hex for md5 or 40 hex
# for sha1.
my ($checksum) = grep /^[0-9a-f]{32}(?:[0-9a-f]{8})?$/i, @fields;
# Skip trying to verify the $checksum if we failed to find it in this
# line, and instead skip to the next line in the checksum file to try to
# find a $checksum.
next unless defined $checksum;
if ($checksum eq $calculated_digest) {
return 'Package verified';
# Sometimes a = is appended to make it 32bits.
} elsif ("$checksum=" eq $calculated_digest) {
return 'Package verified';
}
}
close $digest_fh;
# Return failure, because fetchware failed to verify by checksum
return undef;
}
sub unarchive {
my $package_path = shift;
msg "Unarchiving the downloaded package [$package_path]";
lib/App/Fetchware.pm view on Meta::CPAN
EOM
Mandatory => [ 'mirror', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a mirror configuration
option. Please add one, and try again.
EOM
Mandatory => [ 'lookup_url', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a lookup_url configuration
option. Please add one, and try again.
EOM
ConfigOptionEnum => ['lookup_method', [qw(timestamp versionstring)] ],
ConfigOptionEnum => ['verify_method', [qw(gpg sha1 md5)] ],
);
}
sub check_config_options {
my @args = @_;
lib/App/Fetchware.pm view on Meta::CPAN
### fetchware with a module to install software that cannot be expressed
### using App::Fetchware's configuration file syntax.
=head1 DESCRIPTION
App::Fetchware represents fetchware's API. For ducumentation on how to use
App::Fetchware's fetchware command line interface see L<fetchware>.
It is the heart and soul of fetchware where all of fetchware's main behaviors
are kept. It is fetchware's API, which consists of the subroutines new(),
new_install(), check_syntax(), start(), lookup(), download(), verify(),
unarchive(), build(), install(), uninstall(), upgrade() and end().
App::Fetchware stores both details about C<fetchware>'s configuration file
syntax, documents how to create a fetchware extension, and documents the
internal workings of how App::Fetchware implements C<fetchware>'s package
management behavior:
=over
=item *
lib/App/Fetchware.pm view on Meta::CPAN
have the correct C<use App::Fetchware;> line at the top of your
Fetchwarefile.
=item B<2. Determine your lookup_url>
At the heart of App::Fetchware is its C<lookup_url>, which is
the URL to the FTP or HTTP mirror you want App::Fetchware to use to obtain a
directory listing to see if a new version of your program is available for
download. To figure this out just use your browser to find the program you
want fetchware to manage for you's Web site. Skip over the download link, and
instead look for the gpg, sha1, or md5 verify links, and copy and paste one of
those between the single quotes above in the lookup_url. Then delete the file
portion--from right to left until you reach a C</>. This is necessary, because
fetchware uses the lookup_url as a basis to download your the gpg, sha1, or md5
digital signatures or checksums to ensure that the packages fetchware downloads
and installs are exactly the same as the ones the author uploads.
lookup_url '';
And then after you copy the url.
lib/App/Fetchware.pm view on Meta::CPAN
And then after you type in the text pattern.
filter 'httpd-2.2';
=item B<4. Add mandatory verification settings>
Verification of software downloads is mandatory, because fetchware, in order to
install the software that is downloaded, must execute the build and installation
scripts on your computer sometimes even as the root administrator! Therefore,
fetchware will refuse to build and install any software package that cannot be
verified. This limitation can be bypassed by setting the C<verify_failure_ok>
configuration option to true, but this is B<not> recommended.
Instead, if standard verification fails, please set up one or more of the
configuration options below that may allow verification to succeed if the author
has his download site set up differently then fetchware expects.
=over
=item gpg_keys_url - Should list a URL to a file most likely named C<KEYS> that
contains versions of the author's gpg verification keys that is suitable to be
lib/App/Fetchware.pm view on Meta::CPAN
If you want to specify additional settings the first to choose from are the
build and install settings. These settings control how fetchware builds and
installs your software. They are briefly listed below. For further details see
the section L<App::Fetchware FETCHWAREFILE CONFIGURATION OPTIONS>.
=over
=item B<temp_dir> - Specifies the temporary directory fetchware will use to create its own working temporary directory where it downloads, unarchives, builds, and then installs your program from a directory inside this directory.
=item B<user> - (UNIX only) - Specifies a non-root user to drop privileges to when downloading, verifying, unarchive, and building your program. Root priveedges are kept in the parent process for install if needed.
=item B<prefix> - Specifies the --prefix option for AutoTools (./configure) based programs.
=item B<configure_options> - Specifies any additional options that fetchware should give to AutoTools when it runs ./configure to configure your program before it is built and installed.
=item B<make_options> - Specifies any command line options you would like to provide to make when make is run to build and install your software. C<-j 4> is quite popular to do a paralled make to build and install the program faster.
=item B<build_commands> - Specifies a list of commands that fetchware will use to build your program. You only need this option if your program uses a build system other than AutoTools such as C<cmake> or perhaps a custom one like Perl's C<Configure>
=item B<install_commands> - Specifies a list of commands that fetchware will use to install your program. You only need this option if your program uses a build system other than AutoTools such as C<cmake> or perhaps a custom one like Perl's C<Config...
lib/App/Fetchware.pm view on Meta::CPAN
App::Fetchware has many configuration options. Most were briefly described in
the section L<MANUALLY CREATING A App::Fetchware FETCHWAREFILE>. All of them are
detailed below.
=head2 program 'Program Name';
C<program> simply gives this Fetchwarefile a name. It is availabe to fetchware
after parsing your Fetchwarefile, and is used to name your Fetchwarefile when
using C<fetchware new>. It is required just like C<lookup_url>, C<mirror>,
perhaps C<filter>, and some method to verify downloads are.
=head2 filter 'perl regex here';
Specifies a Perl regular expression that fetchware uses when it determines what
the latest version of a program is. It simply compares each file in the
directory listing specified in your C<lookup_url> to this regular expression,
and only matching files are allowed to pass through to the next part of
fetchware that looks for source code archives to download.
See L<perlretut> for details on how to use and create Perl regular expressions;
however, actual regex know how is not really needed just paste verbatim text
between the single quotes C<'>. For example, C<filter 'httpd-2.2';> will cause
fetchware to only download Apache 2.2 instead of the version for Windows or
whatever is in the weird httpd-deps-* package.
=head2 temp_dir '/tmp';
C<temp_dir> tells fetchware where to store fetchware's temporary working
directory that it uses to download, verify, unarchive, build, and install your
software. By default it uses your system temp directory, which is whatever
directory L<File::Temp's> tempdir() decides to use, which is whatever
L<File::Spec>'s tmpdir() decides to use.
=head2 fetchware_db_path '~/.fetchwaredb';
C<fetchware_db_path> tells fetchware to use a different directory other
than its default directory to store the installed fetchware package for the
particular fetchware package that this option is specified in your
Fetchwarefile. Fetchware's default is C</var/log/fetchware> on Unix when run as
lib/App/Fetchware.pm view on Meta::CPAN
file to download and perhaps also what version of that program to download if
more than one version is available as some mirrors delete old versions and only
keep the latest one.
This url is used for:
=over
=item 1. To determine what the actual download url is for the latest version of this program
=item 2. As the base url to also download a cryptographic signature (ends in .asc) or a SHA-1 or MD5 signature to verify the contents match what the SHA-1 or MD5 checksum is.
=back
You can use the C<mirror> configuration option to specify additional mirrors.
However, those mirrors will only be used to download the large software
archives. Only the lookup_url will be used to download directory listings to
check for new versions, and to download digital signatures or checksums to
verify downloads.
=head2 lookup_method 'timestamp';
Fetchware has two algorithms it uses to determine what version of your program
to download:
=over
=item timestamp
lib/App/Fetchware.pm view on Meta::CPAN
Versionstring parses out the version numbers that each downloadable program has,
and uses them to determine the downloadable archive with the highest version
number, which should also be the newest and best version of the archive to use.
=back
=head2 gpg_keys_url 'lookup_url.com/some/path';
Specifies a file not a directory URL for a C<KEYS> file that lists all of the
authors' gpg keys for fetchware to download and import before using them to
verify the downloaded software package.
If you come accross a software package whoose author uses gpg to sign his
software packages, but he does not include it in the form of a file on his main
mirror, then you can specify the C<user_keyring> option. This option forces
fetchware to use the user who runs fetchware's keyring instead of fetchware's
own keyring. This way you can then import the author's key into your own
keyring, and have fetchware use that keyring that already has the author's key
in it to verify your downloads.
=head2 user_keyring 'On';
When enabled fetchware will use the user who runs fetchware's keyring instead of
fetchware's own keyring. Fetchware uses its own keyring to avoid adding cruft to
your own keyring.
This is needed when the author of a software package does not maintain a KEYS
file that can easily be downloaded and imported into gpg. This option allows you
to import the author's key manually into your own gpg keyring, and then
fetchware will use your own keyring instead of its own to verify your downloads.
=over
=item LIMITAITON
C<user_keyring> when set to true requires that the user that fetchware is
running under have a real gpg keyring with keys that have been imported into it.
This is not the case B<unless> the C<user> option has been specified with a user
account with a proper home directory and gpg keyring for gpg to use. Because of
this limitation if you need to specify C<user_keyring> be sure to also specify
lib/App/Fetchware.pm view on Meta::CPAN
Specifies an alternate url to use to download the cryptographic signature that
goes with your program. This is usually a file with the same name as the
download url with a C<.asc> file extension added on. Fetchware will also append
the extensions C<sig> and C<sign> if C<.asc> is not found, because some pgp
programs and authors use these extensions too.
=head2 sha1_url 'mastermirror.com/some/path';
Specifies an alternate url to download the SHA-1 checksum. This checksum is used
to verify the integrity of the archive that fetchware downloads.
You B<must> specify the master mirror site, which is your programs main mirror
site, because if you download it from a mirror, its possible that both the
archive and the checksum could have been tampered with.
=head2 md5_url 'mastermirror.com/some/path';
Specifies an alternate url to download the MD5 checksum. This checksum is used
to verify the integrity of the archive that fetchware downloads.
You B<must> specify the master mirror site, which is your programs main mirror
site, because if you download it from a mirror, its possible that both the
archive and the checksum could have been tampered with.
=head2 user_agent 'Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0'
Specifies what C<user_agent> you would like fetchware to pretend to be when
downloading files using the HTTP protocol. Some sites annoying prevent some
user agents from working while allowing others. This allows you to pretend to
be a real browser such as Firefox if you need to.
=head2 verify_method 'gpg';
Chooses a method to verify your program. The default is to try C<gpg>, then
C<sha1>, and finally C<md5>, and if all three fail, then the default is to exit
fetchware with an error message, because it is insecure to install archives that
cannot be verified. The availabel options are:
=over
=item gpg - Uses the gpg program to cryptographically verify that the program you downloaded is exactly the same as its author uploaded it.
=item sha1 - Uses the SHA-1 hash function to verify the integrity of the download. This is much less secure than gpg.
=item md5 - Uses the MD5 hash function to verify the integrity of the download. This is much less secure than gpg.
=back
=head2 verify_failure_ok 'True';
Fetchware's default regarding failing to verify your downloaded Archive with
gpg, sha1, or md5 is to exit with an error message, because installing software
that cannot be cryptographically verified should never be done.
=over
=item SECURITY NOTICE
However, if the author of a program you want to use fetchware to manage for you
does not offer a gpg, sha1, or md5 file to verify its integrity, then you can
use this option to force Fetchware to install this program anyway. However, do
not enable this option lightly. Please scour the program's mirrors and homepage
to see which C<gpg_keys_url>, C<gpg_sig_url>, C<sha1_url>, C<md5_url>, or
C<user_keyring> you can use to ensure that your archive is verified before it is
compiled and installed. Even mirrors from sites large and small get hacked
regularly:
L<http://www.itworld.com/security/322169/piwik-software-installer-rigged-back-door-following-website-compromise?page=0,0>
L<http://www.networkworld.com/news/2012/092612-compromised-sourceforge-mirror-distributes-backdoored-262815.html>
lib/App/Fetchware.pm view on Meta::CPAN
So, Please give searching for a C<gpg_keys_url>, C<gpg_sig_url>, C<sha1_url>,
C<md5_url>, or C<user_keyring> for your program another try before simply
enabling this option.
=back
=over
=item NOTICE
C<verify_failure_ok> is a boolean configuration option, which just means its
values are limited to either true or false. True values are C<'True'>, C<'On'>,
C<1>, and false values are C<'False'>, C<'Off'>, and C<0>. All other values are
syntax errors.
=back
=head2 prefix '/opt/';
Controls the AutoTools C<./configuration --prefix=...> option, which allows you
to change the base directory that most software (software that uses AutoTools)
lib/App/Fetchware.pm view on Meta::CPAN
=item L<OVERRIDE_LOOKUP|lookup() API REFERENCE> -
L</get_directory_listing()>, L</parse_directory_listing()>,
L</determine_download_path()>, L</ftp_parse_filelist()>, L</http_parse_filelist()>,
L</file_parse_filelist()>, L</lookup_by_timestamp()>,
L</lookup_by_versionstring()>, L</lookup_determine_downloadpath()>
=item L<OVERRIDE_DOWNLOAD|download() API REFERENCE> -
L</determine_package_path()>
=item L<OVERRIDE_VERIFY|verify() API REFERENCE> - L</gpg_verify()>,
L</sha1_verify()>, L</md5_verify()>, L</digest_verify()>
=item L<OVERRIDE_UNARCHIVE|unarchive() API REFERENCE> -
L</check_archive_files()>, L</list_files()>, L</list_files_tar()>,
L</list_files_zip()>, L</unarchive_package()>, L</unarchive_tar()>,
L</unarchive_zip()>
=item L<OVERRIDE_BUILD|build() API REFERENCE> - L</run_star_commands()> and
L</run_configure()>.
=item L<OVERRIDE_INSTALL|install() API REFERENCE> -
lib/App/Fetchware.pm view on Meta::CPAN
use App::Fetchware;
program 'Apache';
lookup_url 'http://www.apache.org/dist/httpd/';
filter 'httpd-2.2';
mirror 'http://apache.mirrors.pair.com/httpd/';
mirror 'http://mirrors.ibiblio.org/apache/httpd/';
mirror 'ftp://apache.cs.utah.edu/apache.org/httpd/';
verify_method 'gpg';
gpg_keys_url 'http://www.apache.org/dist/httpd/KEYS';
make_options '-j 4';
prefix '/home/dly/software/apache2.2';
# You can use heredocs to make gigantic options like this one more legible.
configure_options <<EOO;
--with-mpm=prefork
--enable-modules="access alias auth autoindex cgi logio log_config status vhost_alias userdir rewrite ssl"
--enable-so
EOO
lib/App/Fetchware.pm view on Meta::CPAN
nginx has its distribution set up differently than Apache, so some changes are
needed. First, nginx does not seem to use any mirrors at all, which means
nginx's Fetchwarefile is going to look kind of stupid with the same exact URL
being used for both the C<lookup_url> and the C<mirror>, but such a
configuration is supported. Next, nginx does not have a KEYS file, but it does
list it's developer's keys on its Website. So, they have to be imported manually
into your keyring, and then specify the C<user_keyring> option to switch
fetchware from usings its own keyring to using your own keyring. Also, note the
comment regarding having to use the C<user> option to specify a real user
account. This is needed, because the verify step is done by fetchware's child
after that child drops its root privileges. Th default user is nobody, and
nobody has no real home, and therefore no keyring, so gpg won't be able to read
the keys you ask it to by using the C<user_keyring> option; therefore, C<user>
must be specified to change it to a real user, whoose keyring has had these keys
imported into it. Also, worth noting that this nginx configuration does not use
a C<filter> option. This is not actually needed, because the only source-code
packages availabe at the C<lookup_url> are the nginx software packages
themselves, but it might be a good idea to include one, because the nginx
developers could always change how their download server is structured. So,
including it is always a good idea.
lib/App/Fetchware.pm view on Meta::CPAN
user_keyring 'On';
# user_keyring specifies to use the user's own keyring instead of fetchware's.
# But fetchware drops privileges by default using he user 'nobody.' nobody is
# nobody, so that user account does not have a home directory for gpg to read a
# keyring from. Therefore, I'm using my own account instead.
user 'dly';
# The other option, which is commented out below, is to use root's own keyring,
# and the no_install option to ensure that root uses its own keyring instead of
# nobody's.
# noinstall 'On';
verify_method 'gpg';
=back
=head2 PHP Programming Language
PHP annoyingly uses a custom Web application on each of its mirror sites to
serve HTTP downloads. No simple directory listing is available. Therefore, to
use php with fetchware, custom C<lookup>, C<download>, and C<verify> hooks are
needed that override fetchware's internal behavior to customize fetchware as
needed so that it can work with how PHP's site is up.
The C<lookup> hook downloads and parses the L<http://www.php.net/downloads.php>
page, which lists files availabe for download. This file is parsed using
L<HTML::TreeBuilder> to determine the latest version. The MD5 sum is also parsed
out to verify the downloaded file as well.
The C<download> hook is only needed, because http_download_file() presumes that
the last part of the path is the filename you're downloading. And this is
annoyingly not the case with the way PHP has its downloading system set up.
The C<verify> hook just uses L<Digest::MD5> to calculate the md5sum of the
downloaded file, and compares it with the one C<lookup> parses out.
=over
use App::Fetchware qw(
:OVERRIDE_LOOKUP
:OVERRIDE_DOWNLOAD
:OVERRIDE_VERIFY
:DEFAULT
);
lib/App/Fetchware.pm view on Meta::CPAN
# something like: /get/php-5.5.3.tar.bz2/from/a/mirror
if (exists $h->{href} and defined $h->{href}) {
$download_path = $h->{href};
} else {
die <<EOD;
php.Fetchwarefile: A path should be found in this link [$link], but there is no
path it in. No href [$h->{href}].
EOD
}
# Find and save the $md5sum for the verify hook below.
# It should be 6 elements over, so it should be the sixth index
# in the @right array below (remember to start counting from 0.).
my @right = $h->right();
my $md5_span_tag = $right[5];
$md5sum = $md5_span_tag->as_text();
$md5sum =~ s/md5:\s+//; # Ditch md5 header.
}
}
);
lib/App/Fetchware.pm view on Meta::CPAN
my $package_path = determine_package_path($temp_dir, $filename);
vmsg "Package path determined to be [$package_path].";
return $package_path
};
# The above lookup hook parses out the md5sum on the php downloads.php web
# site, and stores it in $md5sum, which is used in the the verify hook below.
hook verify => sub {
# Don't need the $download_path, because lookup above did that work for us.
# $package_path is the actual php file that we need to ensure its md5
# matches the one lookup determined.
my ($download_path, $package_path) = @_;
msg "Verifying [$package_path] using md5.";
dir <<EOD if not defined $md5sum;
php.Fetchwarefile: lookup failed to figure out the md5sum for verify to use to
verify that the php version [$package_path] matches the proper md5sum.
The md5sum was [$md5sum].
EOD
my $package_fh = safe_open($package_path, <<EOD);
php.Fetchwarefile: Can not open the php package [$package_path]. The OS error
was [$!].
EOD
# Calculate the downloaded php file's md5sum.
my $digest = Digest::MD5->new();
lib/App/Fetchware.pm view on Meta::CPAN
actual source code archives.
It overrides lookup() to use a local git repo stored in the $git_repo_dir
variable. To create a repo just clone php's git repo (see
http://us1.php.net/git.php for details.). It runs git pull to update the repo,
and then it runs git tags, and ditches some older junk tags, and finds only the
tags used for new versions of php. These are sorted using the C<versonstring>
lookup() algorithm, and the latest one is returned.
download() uses C<git checkout [latesttag]> to "download" php by simply changing
the working directory to the latest tag. verify() uses git's cool C<verify-tag>
command to verify the gpg signature. unarchive() is updated to do nothing since
there is no archive to unarchive. However, because we reuse build(), archive()
must return a $build_path that build() will change its directory to. start() and
end() are also overridden, because managing a temporary directory is not needed,
so, instead, they just do a C<git checkout master> to switch from whatever the
latest tag is back to master, because git pull bases what it does on what branch
you're in, so we must actually be a real branch to update git.
=over
# php-using-git.Fetchwarefile: example fetchwarefile using php's git repo
# for lookup(), download(), and verify() functionality.
use App::Fetchware qw(:DEFAULT :OVERRIDE_LOOKUP);
use App::Fetchware::Util ':UTIL';
use Cwd 'cwd';
# The directory where the php source code's local git repo is.
my $git_repo_dir = '/home/dly/Desktop/Code/php-src';
# By default Fetchware drops privs, and since the source code repo is stored in
# the user dly's home directory, I should drop privs to dly, so that I have
# permission to access it.
lib/App/Fetchware.pm view on Meta::CPAN
my $package_path = cwd();
return $package_path;
};
# You must manually add php's developer's gpg keys to your gpg keyring. Do
# this by going to the page: http://us1.php.net/downloads.php . At the
# bottom the gpg key "names are listed such as "7267B52D" or "5DA04B5D."
# These are their key "names." Use gpg to download them and import them into
# your keyring using: gpg --keyserver pgp.mit.edu --recv-keys [key id]
hook verify => sub {
my ($download_path, $package_path) = @_;
# the latest tag is the download path see lookup.
my $latest_tag = $download_path;
# Run git verify-tag to verify the latest tag
my $success = eval { run_prog('git verify-tag', "$latest_tag"); 1;};
# If the git verify-tag fails, *and* verify_failure_ok has been turned on,
# then ignore the thrown exception, but print an annoying message.
unless (defined $success and $success) {
unless (config('verify_failure_ok')) {
msg <<EOM;
Verification failure ok, becuase you've configured fetchware to continue even
if it cannot verify its downloads. Please reconsider, because mirror and source
code repos do get hacked. The exception that was caught was:
[$@]
EOM
}
}
};
hook unarchive => sub {
# there is nothing to archive due to use of git.
lib/App/Fetchware.pm view on Meta::CPAN
This example MariaDB Fetchwarefile parses the MariaDB download page to determine
what the latest version is based on what C<filter> option you set up. Once this
is determined, the download path is created based on the weird path that MariaDB
uses on its mirrors.
Like PHP MariaDB uses some annoying software on their Web site to presumably
track downloads. This software makes use of AJAX, which is vastly beyone the
capabilities of HTML::TreeBuilder to parse, because it needs a working
JavaScript environment. Therefore, the example Fetchwarefile below has no way of
verifying the MySQL downloads. This could be fixed by using a Perl Web scraping
module that can deal with JavaScript.
=over
use App::Fetchware;
program 'MariaDB';
# MariaDB uses ccache, which wants to create a ~/.ccache cache, which it can't
# do when it's running as nobody, so use a real user account to ensure ccache
lib/App/Fetchware.pm view on Meta::CPAN
# Construct a download path using $version_number[0].
my $filename = 'mariadb-' . $version_number[0] . '.tar.gz';
# Return a proper $download_path, so That I do not have to hook download(),
# but can reuse Fetchware's download() subroutine.
my $weird_prefix = '/mariadb-' . $version_number[0] . '/kvm-tarbake-jaunty-x86/';
my $download_path = '/pub/mariadb' . $weird_prefix .$filename;
return $download_path;
};
# Make verify() failing to verify MariaDB ok, because parsing out the MD5 sum
# would require a Web scraper that supports javascript, which HTML::TreeBuilder
# obviously does not.
verify_failure_ok 'On';
# Use build_commands to configure fetchware to use MariaDB's BUILD script to
# build it. See https://mariadb.com/kb/en/generic-build-instructions/ for
# instructions on the different BUILD cmake scripts that are available.
build_commands 'BUILD/compile-pentium64-max';
# Use install_commands to tell fetchware how to install it. I could leave this
# out, but it nicely documents what command is needed to install MariaDB
# properly.
install_commands 'make install';
lib/App/Fetchware.pm view on Meta::CPAN
};
# Find latest version.
my $latest_ver = lookup_by_versionstring($directory_listing);
# Return $download_path.
my $download_path = '/pub/source/'. "v$latest_ver->[0][0]" .
"/postgresql-$latest_ver->[0][0].tar.bz2";
return $download_path;
};
# MD5sums are stored on the download site, so use them to verify the package.
verify_method 'md5';
# But they are *not* stored on the original "lookup_url" site, so I must provide
# a md5_url pointing to the download site.
md5_url $mirror;
=back
=head1 CREATING A FETCHWARE EXTENSION
=over
lib/App/Fetchware.pm view on Meta::CPAN
Fetchware's main program C<fetchware> uses App::Fetchware's short and simple API
to implement fetchware's default behavior; however, other styles of source code
distributions exist on the internet that may not fit inside App::Fetchware's
capabilities. That is why, in addition to its flexible configuration file
syntax, fetchware allows modules other than App::Fetchware to provide it with
its behavior.
=head2 How the API works
When fetchware installs or upgrades something it executes the API subroutines
check_syntax(), start(), lookup(), download(), verify(), unarchive(), build(),
install(), and end() in that order. And when fetchware uninstalls an installed
package it executes the API subroutines check_syntax(), start(), unarchive(),
uninstall(), and end(). Upgrade is basically the exact same thing as install(),
but it compares version numbers using the upgrade() API subroutine. And
C<fetchware new> obviously just calls new() and new_install().
=head2 Extending App::Fetchware
This API can be overridden inside a user created Fetchwarefile by using hook()
as L<explained above|/So how do I add some custom Perl code to customize my Fetchwarefile?>.
lib/App/Fetchware.pm view on Meta::CPAN
features especially using @INC for subclassing, which App::Fetchware does not
use.
The same terminology as used in OOP is also used here in App::Fetchware, because
the concepts are nearly the same--they're simply implemented differently.
=head3 API subroutines
These are the subroutines that App::Fetchware implements, and that fetchware
uses to implement its desired behavior. They are new(), new_install(),
check_syntax(), start(), lookup(), download(), verify(), unarchive(), build(),
install(), uninstall(), upgrade(), and end(). All must be
implemented or "inherited" from App::Fetchware using L<App::Fetchware::ExportAPI> as
discussed below in L<Implement your fetchware extension.> in a App::Fetchware
L<subclass>.
=head3 override
Means the same thing it does in object-oriented programming. Changing the
definition of a method/subroutine without changing its name. In OOP this is
simply done by subclassing something, and then defining one of the methods that
lib/App/Fetchware.pm view on Meta::CPAN
at Fetchware's level. Perl level syntax errors are not checked. PPI is not used
to parse the file. Instead only high-level Fetchware specific syntax errors are
checked.
=item B<my $temp_dir = start(KeepTempDir => 0 | 1)> - Gives your extension a chance to do anything needed before the rest of the API subroutines get called. App::Fetchware's C<start()> manages App::Fetchware's temporary directory creation. If you wo...
=item B<my $download_url = lookup()> - Determines and returns a download url that C<download()> receives and uses to download the archive for the program.o
=item B<my $package_path = download($tempd_dir, $download_url)> - Downloads its provided $download_url argument.
=item B<verify($download_url, $package_path)> - Verifies the integrity of your downloaded archive using gpg, sha1, or md5.
=item B<my $build_path = unarchive($package_path)> - Unpacks the downloaded archive.
=item B<build($build_path)> - Configures and compiles the downloaded archive.
=item B<install()> - Installs the compiled archive.
=item B<end()> - Cleans up the temporary directory that start() created. Can be overridden to do any other clean up tasks that your archive needs.
=item B<uninstall($build_path)> - Uninstalls an already installed program installed with the same App::Fetchware extension.
lib/App/Fetchware.pm view on Meta::CPAN
=back
Also, keep in mind the order in which these subroutines are called, what
arguments they receive, and what their expected return value is.
=over
=item B<new> = Just new() and new_install().
=item B<install> - check_syntax(), start(), lookup(), download(), verify(), unarchive(), build(), install(), and end().
=item B<upgrade> - check_syntax(), start(), lookup(), I<upgrade()>, download(),
verify(), unarchive(), build(), install(), and end().
=item B<upgrade-all> - Is the same as upgrade, because it just call upgrade for
each fetchware package that is installed in the fetchware package database.
=item B<uninstall> - You might think its just uninstall(), but it does not.
check_syntax(), start(), unarchive(), uninstall(), and end().
=back
Use the above overview of App::Fetchware's API to design what each API
lib/App/Fetchware.pm view on Meta::CPAN
packages L<App::Fetchware::ExportAPI> and L<App::Fetchware::CreateConfigOptions>
to easily manage setting all of this up for you.
First, use L<App::Fetchware::ExportAPI> to be sure to export all of fetchware's
API subroutines. This package is also capable of "inheriting" any of
App::Fetchware's API subroutines that you would like to keep. An example.
# Use App::Fetchware::ExportAPI to set up proper exports this fetchware
# extension.
use App::Fetchware::ExportAPI KEEP => [qw(new_install start end)],
OVERRIDE => [qw(new check_syntax lookup download verify
unarchive build install uninstall upgrade)]
;
There two types of subroutines ExportAPI helps you with:
=over
=item *
B<KEEP> - Specifies which API subroutines you will "keep" or perhaps "inherit"
lib/App/Fetchware.pm view on Meta::CPAN
=item C<MANY> - Takes one or more arguments, and can be called more than once.
If called more than once, then second call's arguments are I<added> to the
existing list of arguments.
I<Examples:> C<mirror>.
=item C<BOOLEAN> - Just like C<ONE> except it will convert /off/i and /false/i
to 0 to support more than just Perl's 0 or undef being false.
I<Examples:> C<verify_failure_ok>, C<no_install>, and C<stay_root>.
=back
Additionally, there is another type of configuration option that only
App::Fetchware::CreateConfigOptions uses:
=over
=item C<IMPORT> - Allows you to import already defined configuration options
from App::Fetchware into your Fetchware extension. For exmaple, you might also
lib/App/Fetchware.pm view on Meta::CPAN
=item * L<make_test_dist()|Test::Fetchware/make_test_dist()> - Use this
subroutine or craft your own similar subroutine, if you need more flexibility,
to test actually installing a program on user's systems that just happens to
execute all of the proper installation commands, but supplies installation
commands that don't actually install anything. It's used to test actually
installing software without actually installing any software.
=item * L<md5sum_file()|Test::Fetchware/md5sum_file()> - Used to provide a
md5sum file for make_test_dist() create software packages in order to pass
fetchware's verify checks.
=item * L<verbose_on()|Test::Fetchware/verbose_on()> - Make's all vmsg()'s
actually print to the screen even if -v or --verbose was not actually
provided on the command line. Used to aid debugging.
=back
=item L<App::Fetchware::Config>
App::Fetchware::Config stores and manages fetchware's parsed configuration file.
lib/App/Fetchware.pm view on Meta::CPAN
subroutines to determine if new versions of your software is available
automatically.
=item L<download()'s OVERRIDE_DOWNLOAD export tag.|download() API REFERENCE>
Only exports the subroutine determine_package_path(), which simply comcatenates
a $tempdir with a $filename to return a properl $package_path, which unarchive
later uses. This is mostly its own subroutine to better document how this is
done, and to allow easier code reuse.
=item L<verify()'s OVERRIDE_VERIFY export tag.|verify() API REFERENCE>
Exports a family of subroutines to verify via MD5, SHA1, or GPG the integrity of
your downloaded package. MD5 and SHA1 are supported for legacy reasons. All
software packages should be GPG signed for much much much better security. GPG
signatures when verified actually prove that the software package you downloaded
is exactly what the author of that software package created, whereas MD5 and
SHA1 sums just verify that you downloaded the bunch of bits in the same order
that they are stored on the server.
digest_verify() an be used to add support for any other Digest::* modules that
CPAN has a Digest based module for that correctly follow Digest's API.
=item L<unarchive()'s OVERRIDE_UNARCHIVE export tag.|unarchive() API REFERENCE>
Exports subroutines that will help you unarchive software packages in tar and
zip format. The most important part to remember is to use list_files() to list
the files in your archive, and pass that list to check_archive_files() to ensure
that the archive will not overwrite any system files, and contains no absolute
paths that could cause havok on your system. unarchive_package() does the actual
unarchiving of software packages.
lib/App/Fetchware.pm view on Meta::CPAN
This funtionality is described next.
=item L<Test::Fetchware/make_clean()> - Just runs C<make clean> in the current
directory.
=item L<Test::Fetchware/make_test_dist()> - Creates a temporary distribution
that is used for testing. This temporary distribution contains a C<./configure>
and a C<Makefile> that create no files, but can still be executed in the
standard AutoTools way.
=item L<Test::Fetchware/md5sum_file()> - Just md5sum's a file so verify() can be
tested.
=item L<Test::Fetchware/expected_filename_listing()> - Returns a string of crazy
Test::Deep subroutines to test filename listings. Not quite as useful as the
rest, but may come in handy if you're only changing the front part of lookup().
=back
Your tests should make use of fetchware's own C<FETHWARE_RELEASE_TESTING>
environment variable that controls with the help of
lib/App/Fetchware.pm view on Meta::CPAN
# $verification_hashref = (
# gpg_keys_url => 'http://main.mirror/distdir',
# verification_method => 'gpg',
# );
Parses $filename_listing to determine what type of verification is available.
Prefering gpg, but falling back on sha1, and then md5 if gpg is not available.
If the type is gpg, then get_verification() will ask the user to specify a
C<gpg_keys_url>, which is required for gpg, because fetchware needs to be able
to import the needed keys to be able to use those keys to verify package
downloads. If this URL is not provided by the author, then get_verification()
will ask the user if they would like to import the author's key into their own
gpg public keyring. If they would, then get_verification() will use the
C<user_keyring> C<'On'> option to use the user's public keyring instead of
fetchware's own keyring. And if the user does not want to use their own gpg
public keyring, then get_verification will fall back to sha1 or md5 setting
C<verify_method> to sha1 or md5 as needed.
Also, adds a gpg_keys_url option if a C<KEYS> file is found in
$filename_listing.
If no verification methods are available, fetchware will print a big nasty
warning message, and offer to use C<verify_failure_ok> to make such a failure
cause fetchware to continue installing your software.
Returns a hashref of options for the user's Fetchwarefile. You're responsible
for calling append_options_to_fetchwarefile() to add them to the user's
Fetchwarefile, or perhaps the caller could analyze them in some way, before
adding them if needed. The keys are the names of the configuration options, and
the values are their values.
=head3 get_filter_option()
lib/App/Fetchware.pm view on Meta::CPAN
=head3 determine_package_path()
my $package_path = determine_package_path($tempdir, $filename)
Determines what $package_path is based on the provided $tempdir and
$filename. $package_path is the path used by unarchive() to unarchive the
software distribution download() downloads.
$package_path is returned to caller.
=head2 verify()
verify($download_path, $package_path)
=over
=item Configuration subroutines used:
=over
=item gpg_keys_url
=item user_keyring
=item gpg_sig_url
=item sha1_url
=item md5_url
=item verify_method
=item verify_failure_ok
=item user_agent
=back
=back
Verifies the downloaded package stored in $package_path by downloading
$download_path.{asc,sha1,md5}> and comparing the two together. Uses the
helper subroutines C<{gpg,sha1,md5,digest}_verify()>.
=over
=item LIMITATIONS
Uses gpg command line, and the interface to gpg is a little brittle.
Crypt::OpenPGP is buggy and not currently maintainted again, so fetchware cannot
make use of it, so were stuck with using the command line gpg program.
=back
lib/App/Fetchware.pm view on Meta::CPAN
This section notes whatever problems you might come accross implementing and
debugging your Fetchware extension due to fetchware's drop_privs mechanism.
See L<Util's drop_privs() subroutine for more info|App::Fetchware::Util/drop_privs()>.
=over
=item *
Under drop_privs() verify() is executed in the child with reduced privileges.
=back
=back
=head2 verify() API REFERENCE
The subroutines below are used by verify() to provide the verify
functionality for fetchware. If you have overridden the verify() handler, you
may want to use some of these subroutines so that you don't have to copy and
paste anything from verify().
App::Fetchware is B<not> object-oriented; therefore, you B<can not> subclass
App::Fetchware to extend it!
=head3 gpg_verify()
'Package Verified' = gpg_verify($download_path);
Uses the command-line program C<gpg> to cryptographically verify that the file
you download is the same as the file the author uploaded. It uses public-key
priviate-key cryptography. The author signs his software package using gpg or
some other OpenPGP compliant program creating a digital signature file with the
same filename as the software package, but usually with a C<.asc> file name
extension. gpg_verify() downloads the author's keys, imports them into
fetchware's own keyring unless the user sets C<user_keyring> to true in his
Fetchwarefile. Then Fetchware downloads a digital signature that usually
ends in C<.asc>. Afterwards, fetchware uses the gpg command line program to
verify the digital signature. gpg_verify returns true if successful, and throws
an exception otherwise.
You can use C<gpg_keys_url> to specify the URL of a file where the author has
uploaded his keys. And the C<gpg_sig_url> can be used to setup an alternative
location of where the C<.asc> digital signature is stored.
=head3 sha1_verify()
'Package verified' = sha1_verify($download_path, $package_path);
undef = sha1_verify($download_path, $package_path);
Verifies the downloaded software archive's integrity using the SHA Digest
specified by the C<sha_url 'ftp://sha.url/package.sha'> config option. Returns
true for sucess dies on error.
=over
=item SECURITY NOTE
If an attacker cracks a mirror and modifies a software package, they can also
modify the MD5 sum of that software package on that B<same mirror>. Because of
lib/App/Fetchware.pm view on Meta::CPAN
If your stuck with using MD5 sum, because your software package does not provide
gpg signing, I recommend that you download your SHA1 sums (and MD5 sums) from
your software package's master mirror. For example, Apache provides MD5 and SHA1
sums, but it does not mirror them--you must download them directly from Apache's
servers. To do this specify a C<sha1_url 'master.mirror/package.sha1';> in your
Fetchwarefile.
=back
=head3 md5_verify()
'Package verified' = md5_verify($download_path, $package_path);
undef = md5_verify($download_path, $package_path);
Verifies the downloaded software archive's integrity using the MD5 Digest
specified by the C<md5_url 'ftp://sha.url/package.sha'> config option. Returns
true for sucess and dies on error.
=over
=item SECURITY NOTE
If an attacker cracks a mirror and modifies a software package, they can also
modify the MD5 sum of that software package on that B<same mirror>. Because of
lib/App/Fetchware.pm view on Meta::CPAN
If your stuck with using MD5 sum, because your software package does not provide
gpg signing, I recommend that you download your MD5 sums (and SHA1 sums) from
your software package's master mirror. For example, Apache provides MD5 and SHA1
sums, but it does not mirror them--you must download them directly from Apache's
servers. To do this specify a C<md5_url 'master.mirror/package.md5';> in your
Fetchwarefile.
=back
=head3 digest_verify()
'Package verified' = digest_verify($digest_type, $download_path, $package_path);
undef = digest_verify($digest_type, $download_path, $package_path);
Verifies the downloaded software archive's integrity using the specified
$digest_type, which also determines the
C<"$digest_type_url" 'ftp://sha.url/package.sha'> config option. Returns
true for sucess and returns false for failure.
=over
=item OVERRIDE NOTE
If you need to override verify() in your Fetchwarefile to change the type of
digest used, you can do this easily, because digest_verify() uses L<Digest>,
which supports a number of Digest::* modules of different Digest algorithms.
Simply do this by override verify() to call
C<digest_verify('Digest's name for your Digest::* algorithm');>
=back
=over
=item SECURITY NOTE
If an attacker cracks a mirror and modifies a software package, they can also
modify the $digest_type sum of that software package on that B<same mirror>.
Because of this limitation $digest_type sums can only tell you if the software
package was corrupted while downloading. This can actually happen as I've had
lib/App/Fetchware.pm view on Meta::CPAN
make_options.
=back
=item * Ensures some options have only allowable options specified.
=over
=item * lookup_method can only have 'timestamp' or 'versionstring'. as options.
=item * And verify_method can only have 'gpg', 'sha1', or 'md5' specified.
=back
=back
=over
=item drop_privs() NOTES
This section notes whatever problems you might come accross implementing and
lib/App/Fetchware.pm view on Meta::CPAN
EOM
Mandatory => [ 'mirror', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a mirror configuration
option. Please add one, and try again.
EOM
Mandatory => [ 'lookup_url', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a lookup_url configuration
option. Please add one, and try again.
EOM
ConfigOptionEnum => ['lookup_method', [qw(timestamp versionstring)] ],
ConfigOptionEnum => ['verify_method', [qw(gpg sha1 md5)] ],
);
Uses config() to test that no configuration options that clash with each other
are used.
It's parameters are specified in a list with an even number of parameters. Each
group of 2 parameters specifies a type of test that check_config_options() will
test for. There are three types of tests. Also, note that the parameters are
specified as a list not as a hash, because multiple "keys" are allowed, and hash
keys must be unique; therefore, the parameters are a list instead of a hash.
lib/App/Fetchware.pm view on Meta::CPAN
in your Fetchwarefile, and then supply whatever configuration options you
need.
This extension mechanism is also very easy for Perl programmers, because you're
basically I<subclassing> App::Fetchware, only you do it using
L<App::Fetchware::ExportAPI> and L<App::Fetchware::CreateConfigOptions>. See
section L<Implement your fetchware extension.> for full details.
=head2 How do I fix the verification failed error.
Fetchware is designed to always attempt to verify the software archives it
downloads even if you failed to configure fetchware's verification settings. It
will try to guess what those setting should be using simple heuristics. First it
will try gpg verificaton, then sha1 verification, and finally md5 verification.
If all fail, then fetchware exit failure with an appropriate error message.
When you get this error message
L<read fetchware's documentation on how to set this up|/4. Add mandatory verification settings>.
=head2 How do I make fetchware log to a file instead of STDOUT?
lib/App/Fetchware/CreateConfigOptions.pm view on Meta::CPAN
#provided a list when you call it such as
#C<install_commands './configure', 'make', 'make install'> would create a
#configuration option with three values. However, if a ONEARRREF is called more
#than once, and exception is also thrown.
#
#=item 3. MANY - Stores many values one at a time just like ONEARRREF, but can
#also be called any number of times, and values are appended to any already
#existing ones.
#
#=item 4. BOOLEAN - Stores true or false values such as C<stay_root 'On'> or
#C<verify_failure_ok 1> or C<no_install 'True'>
#
#=back
#
#In addition to App::Fetchware's types, _create_config_options() features an
#additional type:
#
#=over
#
#=item 5. IMPORT - This option is documented only for completness, because it is
#recommended that you use L<ExportAPI|App::Fetchware::ExportAPI>, and any
lib/App/Fetchware/CreateConfigOptions.pm view on Meta::CPAN
App::Fetchware::CreateConfigOptions - Used by fetchware extensions to create their configuration options.
=head1 VERSION
version 1.016
=head1 SYNOPSIS
use App::Fetchware::ExportAPI KEEP => [qw(start end)],
OVERRIDE =>
[qw(lookup download verify unarchive build install uninstall)];
=head1 DESCRIPTION
App::Fetchware::ExportAPI is a utility helper class for fetchware extensions. It
makes it easy to ensure that your fetchware extension implements or imports all
of App::Fetchware's required API subroutines.
See section L<App::Fetchware/CREATING A FETCHWARE EXTENSION> in App::Fetchware's
documentation for more information on how to create your very own fetchware
extension.
lib/App/Fetchware/CreateConfigOptions.pm view on Meta::CPAN
provided a list when you call it such as
C<install_commands './configure', 'make', 'make install'> would create a
configuration option with three values. However, if a ONEARRREF is called more
than once, and exception is also thrown.
=item 3. MANY - Stores many values one at a time just like ONEARRREF, but can
also be called any number of times, and values are appended to any already
existing ones.
=item 4. BOOLEAN - Stores true or false values such as C<stay_root 'On'> or
C<verify_failure_ok 1> or C<no_install 'True'>
=back
In addition to App::Fetchware's types, import() features an
additional type:
=over
=item 5. IMPORT - the C<IMPORT> this hash key does not create new configuration
lib/App/Fetchware/ExportAPI.pm view on Meta::CPAN
# the magic happen.
clone(import => (from => 'Exporter', to => $callers_package_name));
my %api_subs = (
check_syntax => 0,
new => 0,
new_install => 0,
start => 0,
lookup => 0,
download => 0,
verify => 0,
unarchive => 0,
build => 0,
install => 0,
end => 0,
uninstall => 0,
upgrade => 0,
);
# Check %opts for correctness.
for my $sub_type (@opts{qw(KEEP OVERRIDE)}) {
lib/App/Fetchware/ExportAPI.pm view on Meta::CPAN
App::Fetchware::ExportAPI - Used by fetchware extensions to export their API subroutines.
=head1 VERSION
version 1.016
=head1 SYNOPSIS
use App::Fetchware::ExportAPI KEEP => [qw(start end new_install)],
OVERRIDE =>
[qw(new lookup download verify unarchive build install uninstall
upgrade check_syntax)];
=head1 DESCRIPTION
App::Fetchware::ExportAPI is a utility helper class for fetchware extensions. It
makes it easy to ensure that your fetchware extension implements or imports all
of App::Fetchware's required API subroutines.
See section L<App::Fetchware/CREATING A FETCHWARE EXTENSION> in App::Fetchware's
documentation for more information on how to create your very own fetchware
lib/App/Fetchware/ExportAPI.pm view on Meta::CPAN
"inherited" Fetchware API subroutines from App::Fetchware, and also setting up
the caller's exports, so that the caller also exports all of Fetchware's API
subroutines.
=head2 import()
# You don't actually call import() unless you're doing something weird.
# Instead, use calls import for you.
use App::Fetchware::ExportAPI KEEP => [qw(start end)],
OVERRIDE =>
[qw(lookup download verify unarchive build install uninstall)];
# But if you really do need to run import() itself.
BEGIN {
require App::Fetchware::ExportAPI;
App::Fetchware::ExportAPI->import(KEEP => [qw(start end)],
OVERRIDE =>
[qw(lookup download verify unarchive build install uninstall)]
);
}
Adds fetchware's API subroutines (new(), new_install(), check_syntax(), start(),
lookup(), download(), verify(), unarchive(), build(), install(), end(),
uninstall(), and upgrade()) to the caller()'s @EXPORT. It also imports
L<Exporter>'s import() subroutine to the caller's package, so that the caller
as a proper import() subroutine that Perl will use when someone uses your
fetchware extension in their fetchware extension. Used by fetchware extensions
to easily add fetchware's API subroutines to your extension's package exports.
The C<KEEP> type is how fetchware extensions I<inherit> whatever API subroutines that they
want to reuse from App::Fetchware, while C<OVERRIDE> specifies which API
subroutines this Fetchware extension will implement itself or "override".
lib/App/FetchwareX/HTMLPageSync.pm view on Meta::CPAN
# specified below can be import()ed properly.
use App::Fetchware::ExportAPI
# KEEP or "inherit" new_install, because I want my new_install to just call
# ask_to_install_now_to_test_fetchwarefile(), and App::Fetchware's does that
# already for me. And start() and end() are to create and manage the
# temporary directory for me, so I don't have to worry about polluting the
# current working directory with temporary files.
KEEP => [qw(new_install start end)],
# OVERRIDE everything else.
OVERRIDE =>
[qw(new check_syntax lookup download verify unarchive build install
uninstall upgrade)]
;
# Use App::Fetchware::CreateconfigOptions to build our App::Fetchware
# configuration options for us. These are subroutines with correct prototypes to
# turn a perl code file into something that resembles a configuration file.
use App::Fetchware::CreateConfigOptions
ONE => [qw(
page_name
lib/App/FetchwareX/HTMLPageSync.pm view on Meta::CPAN
@{[@download_file_paths]}
]
EOM
# AKA $package_path.
return \@download_file_paths;
}
sub verify {
vmsg <<EOM;
Skipping verify subroutine, because HTMLPageSync does not need to verify anything
EOM
do_nothing();
}
sub unarchive {
vmsg <<EOM;
Skipping unarchive subroutine, because HTMLPageSync does not need to unarchive
anything
lib/App/FetchwareX/HTMLPageSync.pm view on Meta::CPAN
};
=head2 App::FetchwareX::HTMLPageSync App::Fetchware-like API.
my $temp_file = start();
my $download_url = lookup();
download($temp_dir, $download_url);
verify($download_url, $package_path);
unarchive($package_path);
build($build_path);
install();
uninstall($build_path);
=head1 MOTIVATION
lib/App/FetchwareX/HTMLPageSync.pm view on Meta::CPAN
=head2 download()
download($temp_dir, $download_url);
download() uses App::Fetchware's utility function download_http_url() to
download all of the urls that lookup() returned. If the user specifed a
C<user_agent> configuration option, then that option is passed along to
download_http_url()'s call to HTTP::Tiny.
=head2 verify()
verify($download_url, $package_path);
verify() simply calls App::Fetchware's :UTIL subroutine do_nothing(), which as
you can tell from its name does nothing, but return. The reason for the useless
do_nothing() call is simply for better documentation, and standardizing how to
override a App::Fetchware API subroutine in order for it to do nothing at all,
so that you can prevent the original App::Fetchware subroutine from doing what
it normally does.
=head2 unarchive()
unarchive();
unarchive() does nothing by calling App::Fetchware's :UTIL subroutine
do_nothing(), which does nothing.
=head2 build()
build($build_path);
build() does the same thing as verify(), and that is nothing by calling
App::Fetchware's do_nothing() subroutine to better document the fact
that it does nothing.
=head2 install()
install($package_path);
install() takes the $package_path, which is really an array ref of the paths
of the files that download() copied, and copies them the the user specified
destination directory, C<destination_directory>.
lib/App/FetchwareX/HTMLPageSync.pm view on Meta::CPAN
lookup() is overridden, and downloads the C<html_page_url>, which is the main
configuration option that HTMLPageSync uses. Then lookup() parses that
C<html_page_url>, and determines what the download urls should be. If the
C<html_trebuilder_callback> and C<download_links_callbacks> exist, then they are
called to customize lookup()'s default bahavior. See their descriptions below.
=head3 download()
download() downloads the array ref of download links that lookup() returns.
=head3 verify()
verify() is overridden to do nothing.
=head3 unarchive()
verify() is overridden to do nothing.
=head3 build()
build() is overridden to do nothing.
=head3 install()
install() takes its argument, which is an arrayref of of the paths of the
files that were downloaded to the tempdir created by start(), and copies them to
the user's provided C<destination_directory>.
lib/Test/Fetchware.pm view on Meta::CPAN
# Every Fetchwarefile needs a lookup_url...
lookup_url 'file://$opts{destination_directory}';
# ...and a mirror.
mirror 'file://$opts{destination_directory}';
# Need to filter out the cruft.
filter '$opts{file_name}';
# Just use MD5 to verify it.
verify_method 'md5';
EOF
}
if (not defined $opts{configure}) {
$opts{configure} = <<EOF;
#!/bin/sh
# A Test ./configure file for testing Fetchware's install, upgrade, and so on
# functionality.
lib/Test/Fetchware.pm view on Meta::CPAN
else. There is no corresponding verbose_off(). Just a vebose_on().
Meant to be used in test suites, so that you can see any vmsg()s that print
during testing for debugging purposes.
=head2 export_ok()
export_ok($sorted_subs, $sorted_export);
my @api_subs
= qw(start lookup download verify unarchive build install uninstall);
export_ok(\@api_subs, \@TestPackage::EXPORT);
Just loops over C<@{$sorted_subs}>, and array ref, and ensures that each one
matches the same element of C<@{$sorted_export}>. You do not have to pre sort
these array refs, because export_ok() will copy them, and sort that copy of
them. Uses Test::More's pass() or fail() for each element in the arrays.
=head2 end_ok()
Because end() no longer uses File::Temp's cleanup() to delete B<all> temporary
t/App-Fetchware-ExportAPI.t view on Meta::CPAN
subtest 'Test _export_api() success.' => sub {
package TestPackage;
use App::Fetchware::Util ':UTIL';
# I must load App::Fetchware, because export_api() will try to copy its API
# subs into Test::Package's namespace.
use App::Fetchware ();
my $caller = 'TestPackage';
my @api_subs
= qw(check_syntax new new_install start lookup download verify unarchive build install end uninstall upgrade);
App::Fetchware::ExportAPI::_export_api($caller, KEEP => \@api_subs);
package main;
# Test that _export_api() exports Exporter's import() into its caller's
# package. This is *extremely* important, because if this does not happen,
# then the caller's package will not have a import(), and will be unable to
# export its fetchware API subroutines, and won't work properly.
ok(TestPackage->can('import'),
'checked _export_api() import() method creation.');
export_ok(\@api_subs, \@TestPackage::EXPORT);
package TestPackage2;
use App::Fetchware::Util ':UTIL';
sub check_syntax { return 'nothing'; }
sub new { return 'nothing'; }
sub new_install { return 'nothing'; }
sub start { return 'nothing'; }
sub lookup { return 'nothing'; }
sub download { return 'nothing'; }
sub verify { return 'nothing'; }
sub unarchive { return 'nothing'; }
sub build { return 'nothing'; }
sub install { return 'nothing'; }
sub end {return 'nothing'; }
sub uninstall { return 'nothing'; }
sub upgrade { return 'nothing'; }
$caller = 'TestPackage2';
App::Fetchware::ExportAPI::_export_api($caller, OVERRIDE => \@api_subs);
t/App-Fetchware-ExportAPI.t view on Meta::CPAN
subtest 'Test import() success.' => sub {
# Must call import() as a class method.
package TestPackage3;
use App::Fetchware::Util ':UTIL';
# I must load App::Fetchware, because export_api() will try to copy its API
# subs into Test::Package's namespace.
use App::Fetchware ();
my @api_subs
= qw(check_syntax new new_install start lookup download verify unarchive build install end uninstall upgrade);
App::Fetchware::ExportAPI->import(KEEP => \@api_subs);
package main;
# Test that _export_api() exports Exporter's import() into its caller's
# package. This is *extremely* important, because if this does not happen,
# then the caller's package will not have a import(), and will be unable to
# export its fetchware API subroutines, and won't work properly.
ok(TestPackage3->can('import'),
'checked _export_api() import() method creation.');
export_ok(\@api_subs, \@TestPackage3::EXPORT);
package TestPackage4;
use App::Fetchware::Util ':UTIL';
sub check_syntax { return 'nothing'; }
sub new { return 'nothing'; }
sub new_install { return 'nothing'; }
sub start { return 'nothing'; }
sub lookup { return 'nothing'; }
sub download { return 'nothing'; }
sub verify { return 'nothing'; }
sub unarchive { return 'nothing'; }
sub build { return 'nothing'; }
sub install { return 'nothing'; }
sub end {return 'nothing'; }
sub uninstall { return 'nothing'; }
sub upgrade { return 'nothing'; }
App::Fetchware::ExportAPI->import(OVERRIDE => \@api_subs);
package main;
# Test that _export_api() exports Exporter's import() into its caller's
t/App-Fetchware-ExportAPI.t view on Meta::CPAN
subtest 'Test use App::Fetchware::ExportAPI.' => sub {
package TestPackage5;
use App::Fetchware::Util ':UTIL';
# I must load App::Fetchware, because export_api() will try to copy its API
# subs into Test::Package's namespace.
use App::Fetchware ();
my @api_subs
= qw(check_syntax new new_install start lookup download verify unarchive build install end uninstall upgrade);
use App::Fetchware::ExportAPI
KEEP => [qw(check_syntax new new_install start lookup download verify unarchive build install end uninstall upgrade)];
# For debugging--you can't debug begin blocks and use's.
# require App::Fetchware::ExportAPI;
# App::Fetchware::ExportAPI->import(KEEP => \@api_subs);
package main;
# Test that _export_api() exports Exporter's import() into its caller's
# package. This is *extremely* important, because if this does not happen,
# then the caller's package will not have a import(), and will be unable to
# export its fetchware API subroutines, and won't work properly.
ok(TestPackage5->can('import'),
t/App-Fetchware-ExportAPI.t view on Meta::CPAN
export_ok(\@api_subs, \@TestPackage5::EXPORT);
package TestPackage6;
use App::Fetchware::Util ':UTIL';
sub check_syntax { return 'nothing'; }
sub new { return 'nothing'; }
sub new_install { return 'nothing'; }
sub start { return 'nothing'; }
sub lookup { return 'nothing'; }
sub download { return 'nothing'; }
sub verify { return 'nothing'; }
sub unarchive { return 'nothing'; }
sub build { return 'nothing'; }
sub install { return 'nothing'; }
sub end {return 'nothing'; }
sub uninstall { return 'nothing'; }
sub upgrade { return 'nothing'; }
use App::Fetchware::ExportAPI
OVERRIDE => [qw(check_syntax new new_install start lookup download verify unarchive build install end uninstall upgrade)];
# For debugging--you can't debug begin blocks and use's.
# require App::Fetchware::ExportAPI;
# App::Fetchware::ExportAPI->import(OVERRIDE => \@api_subs);
package main;
# Test that _export_api() exports Exporter's import() into its caller's
# package. This is *extremely* important, because if this does not happen,
# then the caller's package will not have a import(), and will be unable to
# export its fetchware API subroutines, and won't work properly.
ok(TestPackage6->can('import'),
t/App-Fetchware-check_syntax.t view on Meta::CPAN
my $retval = check_config_options(Mandatory => [ 'program', <<EOM ],);
App-Fetchware: Your Fetchwarefile must specify a program configuration
option. Please add one, and try again.
EOM
is($retval, 'Syntax Ok', 'checked check_config_options() Mandatory success.');
__clear_CONFIG();
# ConfigOptionEnum only triggers when the specified option, verify_method in
# this case, has been specified. Therefore, I must specify verify_method in
# order to test ConfigOptionEnum.
config(verify_method => 'notgpg');
eval_ok(sub{check_config_options(
ConfigOptionEnum => ['verify_method', [qw(gpg sha1 md5)] ],)
}, <<EOD, 'checked check_config_options() ConfigOptionEnum exception');
App-Fetchware: You specified the option [verify_method], but failed to specify only
one of its acceptable values [gpg sha1 md5]. Please change the value you
specified [notgpg] to one of the acceptable ones listed above, and try again.
EOD
__clear_CONFIG();
config(verify_method => 'gpg');
is(check_config_options(
ConfigOptionEnum => ['verify_method', [qw(gpg sha1 md5)] ],),
'Syntax Ok', 'checked check_config_options() ConfigOptionEnum success.');
__clear_CONFIG();
# Define a test Fetchwarefile with just config().
config(program => 'Some Program');
config(lookup_url => 'scheme://fake.url');
config(mirror => 'scheme://fake.url');
is(check_config_options(
t/App-Fetchware-check_syntax.t view on Meta::CPAN
EOM
Mandatory => [ 'mirror', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a mirror configuration
option. Please add one, and try again.
EOM
Mandatory => [ 'lookup_url', <<EOM ],
App-Fetchware: Your Fetchwarefile must specify a lookup_url configuration
option. Please add one, and try again.
EOM
ConfigOptionEnum => ['lookup_method', [qw(timestamp versionstring)] ],
ConfigOptionEnum => ['verify_method', [qw(gpg sha1 md5)] ],
), 'Syntax Ok', 'checked check_config_options() all options together.');
};
subtest 'test check_syntax()' => sub {
is(check_syntax(), 'Syntax Ok',
'checked check_syntax() success.');
};
t/App-Fetchware-config-file.t view on Meta::CPAN
configure_options 'test';
make_options 'test';
build_commands 'test';
install_commands 'test';
lookup_url 'test';
lookup_method 'test';
gpg_sig_url 'test';
gpg_keys_url 'test';
sha1_url 'test';
md5_url 'test';
verify_method 'test';
no_install 'test';
verify_failure_ok 'test';
stay_root 'test';
user_keyring 'test';
debug_CONFIG();
for my $config_sub (qw(
temp_dir
user
prefix
configure_options
make_options
build_commands
install_commands
lookup_url
lookup_method
gpg_sig_url
verify_method
no_install
verify_failure_ok
stay_root
)) {
is(config($config_sub), 'test', "checked config sub $config_sub");
}
# Test 'MANY' config subs.
mirror 'test';
mirror 'test';
mirror 'test';
mirror 'test';
t/App-Fetchware-config-file.t view on Meta::CPAN
[ make_options => 'ONEARRREF' ],
[ build_commands => 'ONEARRREF' ],
[ install_commands => 'ONEARRREF' ],
[ uninstall_commands => 'ONEARRREF' ],
[ lookup_url => 'ONE' ],
[ lookup_method => 'ONE' ],
[ gpg_keys_url => 'ONE' ],
[ gpg_sig_url => 'ONE' ],
[ sha1_url => 'ONE' ],
[ md5_url => 'ONE' ],
[ verify_method => 'ONE' ],
[ mirror => 'MANY' ],
[ no_install => 'BOOLEAN' ],
[ verify_failure_ok => 'BOOLEAN' ],
[ stay_root => 'BOOLEAN' ],
[ user_keyring => 'BOOLEAN' ],
);
{ no strict 'refs';
for my $config_sub (@onearrref_or_not) {
if ($config_sub->[1] eq 'ONE'
or $config_sub->[1] eq 'BOOLEAN') {
# eval_ok( sub {eval "$config_sub->[0] 'onevalue', 'twovalues';"},
t/App-Fetchware-lookup.t view on Meta::CPAN
$ENV{FETCHWARE_HTTP_LOOKUP_URL}
) {
# Clear %CONFIG, so I can call lookup_url again.
__clear_CONFIG();
# Set download type.
# Make this a FETCHWARE_FTP_REMOTE env var in frt().
lookup_url "$lookup_url";
# Must also specify a program config option.
program 'testin';
mirror "$lookup_url";
verify_failure_ok 'On';
# Test get_directory_listing().
$lookup_url =~ m!^(ftp|http)(:?://.*)?!;
my $scheme = $1;
ok(get_directory_listing(),
"checked get_directory_listing() $scheme success");
}
};
t/App-Fetchware-lookup.t view on Meta::CPAN
};
subtest 'test parse_directory_listing()' => sub {
skip_all_unless_release_testing();
# Clear App::Fetchware's %CONFIG variable.
__clear_CONFIG();
###BUGALERT### Add loop after http_parse_listing() is finished to test this
#sub's http functionality too.
lookup_url $ENV{FETCHWARE_FTP_LOOKUP_URL};
# Must also specify program, mirror, and a verify method.
program 'testin';
mirror $ENV{FETCHWARE_FTP_LOOKUP_URL};
verify_failure_ok 'On';
# Do the stuff parse_directory_listing() depends on.
my $directory_listing = get_directory_listing();
cmp_deeply(parse_directory_listing($directory_listing),
eval(expected_filename_listing()),
'checked parse_directory_listing() ftp success.');
};
t/App-Fetchware-lookup.t view on Meta::CPAN
# testing.
my $test_lookup_url = $ENV{FETCHWARE_FTP_LOOKUP_URL};
subtest 'test lookup_determine_downloadpath()' => sub {
skip_all_unless_release_testing();
# Clear App::Fetchware's %CONFIG variable.
__clear_CONFIG();
lookup_url "$test_lookup_url";
# Must also specify program, mirror, and a verify method.
program 'testin';
mirror "$test_lookup_url";
verify_failure_ok 'On';
# Select one of the different apache versions 'httpd-2.{0,2,3}'.
filter 'httpd-2.2';
# Test lookup_determine_downloadpath() with 'CURRENT_IS_VER_NO' in the
# file listing.
###BUGALERT### Refactor out static crap like $current_file_list.
my $current_file_list =
[
[ 'CURRENT-IS-2.2.21', '999910051831' ],
t/App-Fetchware-lookup.t view on Meta::CPAN
skip_all_unless_release_testing();
# Clear App::Fetchware's %CONFIG variable.
__clear_CONFIG();
# This must be set for lookup() to work on Apache's mirror format.
filter 'httpd-2.2';
# Set needed config variables.
lookup_url "$test_lookup_url";
# Must also specify program, mirror, and a verify method.
program 'testin';
mirror "$test_lookup_url";
verify_failure_ok 'On';
my $directory_listing = get_directory_listing();
my $filename_listing = parse_directory_listing($directory_listing);
like(determine_download_path($filename_listing),
qr{httpd-2.2.\d+?.tar.bz2},
'checked determine_download_path() success.');
# Clear App::Fetchware's %CONFIG variable so I can test it with custom
# lookup_methods.
__clear_CONFIG();
# This must be set for lookup() to work on Apache's mirror format.
filter 'httpd-2.2';
# Set needed config variables.
lookup_url "$test_lookup_url";
# Must also specify program, mirror, and a verify method.
program 'testin';
mirror "$test_lookup_url";
verify_failure_ok 'On';
lookup_method 'versionstring';
$directory_listing = get_directory_listing();
$filename_listing = parse_directory_listing($directory_listing);
like(determine_download_path($filename_listing),
qr{httpd-2.2.\d+?.tar.bz2},
'checked determine_download_path() success.');
t/App-Fetchware-lookup.t view on Meta::CPAN
skip_all_unless_release_testing();
# Clear App::Fetchware's %CONFIG variable.
__clear_CONFIG();
# This must be set for lookup() to work on Apache's mirror format.
filter 'httpd-2.2';
# Set needed config variables.
lookup_url $ENV{FETCHWARE_FTP_LOOKUP_URL};
# Must also specify program, mirror, and a verify method.
program 'testin';
mirror $ENV{FETCHWARE_FTP_LOOKUP_URL};
verify_failure_ok 'On';
like(lookup(),
qr{httpd-2.2.\d+?.tar.bz2},
'checked lookup_determine_downloadurl() success.');
};
# Remove this or comment it out, and specify the number of tests, because doing
t/App-Fetchware-lookup.t view on Meta::CPAN
<h2><a name="sig">PGP Signatures</a></h2>
<p>All of the release distribution packages have been digitally signed
(using PGP or GPG) by the Apache Group members that constructed them.
There will be an accompanying <SAMP><EM>distribution</EM>.asc</SAMP> file
in the same directory as the distribution. The PGP keys can be found
at the MIT key repository and within this project's
<a href="http://www.apache.org/dist/httpd/KEYS">KEYS file</a>.</p>
<p>Always use the signature files to verify the authenticity
of the distribution, <i>e.g.</i>,</p>
<pre>
% pgpk -a KEYS
% pgpv httpd-2.2.8.tar.gz.asc
<i>or</i>,
% pgp -ka KEYS
% pgp httpd-2.2.8.tar.gz.asc
<i>or</i>,
% gpg --import KEYS
% gpg --verify httpd-2.2.8.tar.gz.asc
</pre>
<p>We offer MD5 hashes as an alternative to validate the integrity
of the downloaded files. A unix program called <code>md5</code> or
<code>md5sum</code> is included in many unix distributions. It is
also available as part of <a
href="http://www.gnu.org/software/textutils/textutils.html">GNU
Textutils</a>. Windows users can get binary md5 programs from <a
href="http://www.fourmilab.ch/md5/">here</a>, <a
href="http://www.pc-tools.net/win32/freeware/console/">here</a>, or
t/App-Fetchware-new-get_verification view on Meta::CPAN
my $lookup_url = $ENV{FETCHWARE_HTTP_LOOKUP_URL};
my $verification_hashref = get_verification($term, $current_file_list, $lookup_url);
ok(exists $verification_hashref->{user_keyring},
'checked get_verification() keyring success.');
like($verification_hashref->{user_keyring}, qr/on|true/i,
'checked get_verification() keyring value.');
ok(exists $verification_hashref->{verify_method},
'checked get_verification method sucess.');
like($verification_hashref->{verify_method}, qr/gpg/i,
'checked get_verification() verify_method value.');
# Spit out # of tests run.
done_testing();
# Print a bogus "prompt" to keep Expect from freaking out, because it presumes
# the prompt works like it does in a shell, but fetchware new is not a shell.
print "Bogus shell: \n";
# Because we're in a child process not the same one that is running the main
# test suite, if any tests fail this failure will not be reported back to our
t/App-Fetchware-new.t view on Meta::CPAN
# First test that the command produced the correct outout.
expect_like(qr/Automatic KEYS file discovery failed. Fetchware needs the/,
'checked get_verification() received correct mirror prompt');
# Have Expect print an example URL.
expect_send('Y',
'check get_verification() sent manual KEYS file Y.');
expect_like(qr<\[y/N\]|Automatic verification of your fetchware package has failed!>,
'check get_verification() received no verify prompt.');
expect_send('Y',
'checked get_verification() sent no verify Y.');
expect_quit();
###BUGALERT### Add tests for get_verification()'s other branches.
};
subtest 'test get_filter_option() success' => sub {
skip_all_unless_release_testing();
t/App-Fetchware-verify.t view on Meta::CPAN
#!perl
# App-Fetchware-verify.t tests App::Fetchware's verify() subroutine, which gpg
# verifies your downloaded archive if possible. If not it will also try md5/sha.
# Pretend to be bin/fetchware, so that I can test App::Fetchware as though
# bin/fetchware was calling it.
package fetchware;
use strict;
use warnings;
use 5.010001;
# Set a umask of 022 just like bin/fetchware does. Not all fetchware tests load
# bin/fetchware, and so all fetchware tests must set a umask of 0022 to ensure
t/App-Fetchware-verify.t view on Meta::CPAN
# and compile-time, and prototypes are properly honored."
BEGIN { use_ok('App::Fetchware', qw(:DEFAULT :OVERRIDE_VERIFY)); }
# Print the subroutines that App::Fetchware imported by default when I used it.
note("App::Fetchware's default imports [@App::Fetchware::EXPORT]");
subtest 'OVERRIDE_VERIFY exports what it should' => sub {
my @expected_overide_verify_exports = qw(
gpg_verify
sha1_verify
md5_verify
digest_verify
);
# sort them to make the testing their equality very easy.
my @sorted_verify_tag = sort @{$App::Fetchware::EXPORT_TAGS{OVERRIDE_VERIFY}};
@expected_overide_verify_exports = sort @expected_overide_verify_exports;
is_deeply(\@sorted_verify_tag, \@expected_overide_verify_exports,
'checked for correct OVERRIDE_VERIFY @EXPORT_TAG');
};
my $download_path;
my $package_path;
subtest 'Do verify() prereqs.' => sub {
skip_all_unless_release_testing();
# Call start() to create & cd to a tempdir, so end() called later can delete
# all of the files that will be downloaded.
my $temp_dir = start();
# Use config subs to determine a $download_path based on the current version
# of apache instead of updating this manually.
program 'httpd-2.2';
lookup_url 'http://www.apache.org/dist/httpd';
mirror 'http://apache.mirrors.pair.com/httpd';
filter 'httpd-2.2';
t/App-Fetchware-verify.t view on Meta::CPAN
$package_path = download($temp_dir, $download_path);
ok(-e $temp_dir,
'checked successful start().');
ok(defined $download_path,
'checked successful lookup().');
ok(-e $package_path,
'checked successful download().');
};
subtest 'test digest_verify()' => sub {
skip_all_unless_release_testing();
for my $digest_type (qw(SHA-1 MD5)) {
note("TYPE[$digest_type]");
ok(digest_verify($digest_type, $download_path, $package_path),
"checked digest_verify($digest_type) success.");
eval_ok(sub {
digest_verify($digest_type, $download_path,
'./doesntexistunlessyoucreateitbutdontdothat');
}, <<EOE, "checked digest_verify($digest_type) package path failure");
App-Fetchware: run-time error. Fetchware failed to open the file it downloaded
while trying to read it in order to check its MD5 sum. The file was
[./doesntexistunlessyoucreateitbutdontdothat]. See perldoc App::Fetchware.
EOE
# Undo last tests change to make it fail, so now it'll succeed.
###HOWTOTEST###eval_ok(sub {}, <<EOE, 'checked md5_verify() md5 croaked with error'
###HOWTOTEST### eval_ok(sub {}, <<EOD, 'checked md5_verify() failed to open downloaded md5m file');
###HOWTOTEST###App-Fetchware: run-time error. Fetchware failed to open the md5sum file it
###HOWTOTEST###downloaded while trying to read it in order to check its MD5 sum. The file was
###HOWTOTEST###[$md5_file]. See perldoc App::Fetchware.
###HOWTOTEST###EOE
# Test failure by setting $package_path to the wrong thing.
# Create a useless file to test against.
my ($tf_fh, $tf_name)
=
tempfile("fetchware-test-$$-XXXXXXXXXX", TMPDIR => 1, UNLINK => 1);
ok(-e $tf_name,
'checked tempfile creation to test digest_verify() failure()');
print $tf_fh 'SOME CRAP TO ACTUALLY MD5/SHA1 SUM!!!!!';
close $tf_fh;
is(digest_verify($digest_type, $download_path, $tf_name), undef,
"checked digest_verify($digest_type) failure");
eval_ok(sub {
digest_verify($digest_type,
'ftp://fake.url/will.fail', $package_path);
}, <<EOE, "checked digest_verify($digest_type) download digest failure");
App-Fetchware: Fetchware was unable to download the $digest_type sum it needs to
download to properly verify you software package. This is a fatal error, because
failing to verify packages is a perferable default over potentially installin
compromised ones. If failing to verify your software package is ok to you, then
you may disable verification by adding verify_failure_ok 'On'; to your
Fetchwarefile. See perldoc App::Fetchware.
EOE
} # End for.
};
subtest 'test md5_verify()' => sub {
skip_all_unless_release_testing();
ok(md5_verify($download_path, $package_path),
"checked md5_verify() success.");
md5_url 'http://www.apache.org/dist/httpd/';
ok(md5_verify($download_path, $package_path),
'checked md5_verify() md5_url success.');
};
subtest 'test sha1_verify()' => sub {
skip_all_unless_release_testing();
ok(sha1_verify($download_path, $package_path),
"checked sha1_verify() success.");
sha1_url 'http://www.apache.org/dist/httpd/';
ok(sha1_verify($download_path, $package_path),
'checked sha1_verify() sha_url success.');
};
subtest 'test gpg_verify()' => sub {
skip_all_unless_release_testing();
# Clean the gunk of of %CONFIG.
__clear_CONFIG();
# Test gpg_verify() success.
lookup_url 'http://www.apache.org/dist/httpd';
ok(gpg_verify($download_path), 'checked gpg_verify() success');
# Test gpg_verify() success using gpg_keys_url.
gpg_keys_url config('lookup_url') . '/KEYS';
ok(gpg_verify($download_path), 'checked gpg_verify() success');
eval_ok(sub {
gpg_verify('ftp://fake.url/will.fail');
}, <<EOE, 'checked gpg_verify() download gpg_sig_url failure');
App-Fetchware: Fetchware was unable to download the gpg_sig_url you specified or
that fetchware tried appending asc, sig, or sign to [http://www.apache.org/will.fail.sign]. It needs
to download this file to properly verify you software package. This is a fatal
error, because failing to verify packages is a perferable default over
potentially installing compromised ones. If failing to verify your software
package is ok to you, then you may disable verification by adding
verify_failure_ok 'On'; to your Fetchwarefile. See perldoc App::Fetchware.
EOE
};
subtest 'test verify()' => sub {
skip_all_unless_release_testing();
# test verify_method
# test gpg verify_method
# test sha1 verify_method
# test md5 verify_method
# Specify a DownloadURL to test some gpg_verify() guessing magic.
for my $verify_method (qw(gpg sha md5)) {
config_replace('verify_method', "$verify_method");
eval {verify($download_path, $package_path)};
unless ($@) {
pass("checked verify() verify_method $verify_method");
} else {
fail("checked verify() verify_method $verify_method");
}
}
# test using copied gpg_verify setup from above.
eval {verify($download_path, $package_path)};
note("exe[$@]");
unless ($@) {
pass("checked verify() automatic method gpg");
} else {
fail("checked verify() automatic method gpg");
}
# test for skiping gpg & using sha1. Can't find a site that does this.
###BUGALERT### Figure out how to test for this. I may have to wait until I
#implement testing webserver to download files from using maybe
#Test::Fake::HTTPD or something else.
###HOWTOTEST?? eval {verify()};
###HOWTOTEST?? unless ($@) {
###HOWTOTEST?? pass("checked verify() automatic method sha");
###HOWTOTEST?? } else {
###HOWTOTEST?? fail("checked verify() automatic method sha");
###HOWTOTEST?? }
# test using just a plain old md5sum.
# Use postgressql to test for only a md5, though I should find a smaller
# progject that packages up md5 correctly.
# Must temporarily change the lookup_url.
my $old_lookup = config('lookup_url');
config_replace(lookup_url => 'http://ftp.postgresql.org/pub/source/');
my $postgres_download_path =
'http://ftp.postgresql.org/pub/source/v9.2.4/postgresql-9.2.4.tar.bz2';
eval {verify(
$postgres_download_path, download_file($postgres_download_path)
)};
unless ($@) {
pass("checked verify() automatic method md5");
} else {
die $@;
fail("checked verify() automatic method md5");
}
config_replace(lookup_url => $old_lookup);
# Clear CONFIG for next run.
__clear_CONFIG();
# test verify failure with verify_failure_ok Off.
eval_ok(sub {verify('ftp://fake.url/doesnt/exist.ever',
$package_path)}, <<EOE, 'checked verify() failure');
App-Fetchware: run-time error. Fetchware failed to verify your downloaded
software package. You can rerun fetchware with the --force option or add
[verify_failure_ok 'True';] to your Fetchwarefile. See the section VERIFICATION
FAILED in perldoc fetchware.
EOE
# test verify_failure_ok
###BUGALERT### Must test success & failure with this option.
verify_failure_ok 'On';
note("vfo[@{[config('verify_failure_ok')]}]");
is(verify('ftp://fake.url/doesnt/exist.ever', $package_path),
'warned due to verify_failure_ok',
'checked verify() verify_failure_ok');
# Test an invalid verify_method.
verify_method 'invalid';
eval_ok(sub {verify($download_path, $package_path)},
<<EOE, 'checked verify() invalid verify_method');
App-Fetchware: run-time error. Your fetchware file specified a wrong
verify_method option. The only supported types are 'gpg', 'sha', 'md5', but you
specified [invalid]. See perldoc App::Fetchware.
EOE
config_delete('verify_method');
};
subtest 'Call end() to clean up temporary directory.' => sub {
# Skip this simple test, because start() is only caled with the other
# prereques, which are skipped except during release testing.
skip_all_unless_release_testing();
# Call end() to delete temp dir created by start().
ok(end(),
'cleared out fetchware temporary directory.');
};
###BUGALERT###Add a subtest that uses make_test_dist() and md5sum_file() to test
#verify() functionality on non FETCHWARE_RELEASE_TESTING systems.
# Remove this or comment it out, and specify the number of tests, because doing
# so is more robust than using this, but this is better than no_plan.
#done_testing();
t/App-FetchwareX-HTMLPageSync.t view on Meta::CPAN
is($download_file_paths->[0],
file($ENV{FETCHWARE_HTTP_DOWNLOAD_URL})->basename(),
'checked download() success.');
ok(-e $download_file_paths->[0],
'checked download()ed file existence');
};
subtest 'test HTMLPageSync verify()' => sub {
is(verify('dummy', 'args'), undef, 'checked verify() success.');
};
subtest 'test HTMLPageSync unarchive()' => sub {
is(unarchive(), undef, 'checked unarchive() success.');
};
subtest 'test HTMLPageSync build()' => sub {
is(verify('dummy arg'), undef, 'checked build() success.');
};
subtest 'test HTMLPageSync install()' => sub {
skip_all_unless_release_testing();
ok(install($download_file_paths),
'checked install() success');
# install() needs an array reference.
t/bin-fetchware-Fetchwarefile.t view on Meta::CPAN
my $apache_fetchwarefile = <<'EOF';
use App::Fetchware;
program 'Apache';
lookup_url 'http://www.apache.org/dist/httpd/';
filter 'httpd-2.2';
mirror 'http://mirrors.ibiblio.org/apache/httpd/';
mirror 'ftp://apache.cs.utah.edu/apache.org/httpd/';
verify_method 'gpg';
gpg_keys_url 'http://www.apache.org/dist/httpd/KEYS';
make_options '-j 4';
prefix '/home/dly/software/apache2.2';
# You can use heredocs to make gigantic options like this one more legible.
configure_options <<EOO;
--with-mpm=prefork
--enable-modules="access alias auth autoindex cgi logio log_config status vhost_alias userdir rewrite ssl"
--enable-so
EOO
t/bin-fetchware-Fetchwarefile.t view on Meta::CPAN
user_keyring 'On';
# user_keyring specifies to use the user's own keyring instead of fetchware's.
# But fetchware drops privileges by default using he user 'nobody.' nobody is
# nobody, so that user account does not have a home directory for gpg to read a
# keyring from. Therefore, I'm using my own account instead.
user 'dly';
# The other option, which is commented out below, is to use root's own keyring,
# and the no_install option to ensure that root uses its own keyring instead of
# nobody's.
# noinstall 'On';
verify_method 'gpg';
EOF
# Create a tempfile to store the Fetchwarefile in.
my ($fh, $filename) = tempfile("fetchware-test-$$-XXXXXXXXXXX", TMPDIR => 1,
UNLINK => 1);
# Write the $apache_fetchwarefile to disk, so bin/fetchware can access it.
print $fh $nginx_fetchwarefile;
close $fh; # Close $fh to ensure its contents make it out to disk.
# Just execute bin/fetchware install with the newly created
t/bin-fetchware-Fetchwarefile.t view on Meta::CPAN
# something like: /get/php-5.5.3.tar.bz2/from/a/mirror
if (exists $h->{href} and defined $h->{href}) {
$download_path = $h->{href};
} else {
die <<EOD;
php.Fetchwarefile: A path should be found in this link [$link], but there is no
path it in. No href [$h->{href}].
EOD
}
# Find and save the $md5sum for the verify hook below.
# It should be 3 elements over, so it should be the third index
# in the @right array below (remember to start counting 2 0.).
my @right = $h->right();
# Left for the next time the page annoyingly, arbitrarily changes :)
#local $Data::Dumper::Maxdepth = 3; # Only show 3 "levels" of crap.
#use Test::More;
#diag("RIGHT[");
#for my $i (0..$#right) {
# diag("TAG#[$i]");
# diag explain \@right;
t/bin-fetchware-Fetchwarefile.t view on Meta::CPAN
my $package_path = determine_package_path($temp_dir, $filename);
vmsg "Package path determined to be [$package_path].";
return $package_path
};
# The above lookup hook parses out the md5sum on the php downloads.php web
# site, and stores it in $md5sum, which is used in the the verify hook below.
hook verify => sub {
# Don't need the $download_path, because lookup above did that work for us.
# $package_path is the actual php file that we need to ensure its md5
# matches the one lookup determined.
my ($download_path, $package_path) = @_;
msg "Verifying [$package_path] using md5.";
die <<EOD if not defined $md5sum;
php.Fetchwarefile: lookup failed to figure out the md5sum for verify to use to
verify that the php version [$package_path] matches the proper md5sum.
The md5sum was [$md5sum].
EOD
my $package_fh = safe_open($package_path, <<EOD);
php.Fetchwarefile: Can not open the php package [$package_path]. The OS error
was [$!].
EOD
# Calculate the downloaded php file's md5sum.
my $digest = Digest::MD5->new();
t/bin-fetchware-Fetchwarefile.t view on Meta::CPAN
ok(run_prog(qw!perl -I lib bin/fetchware install!, $filename),
'Checked PHP Fetchwarefile success.');
};
subtest 'test PHP git Fetchwarefile success' => sub {
my $php_git_fetchwarefile = <<'EOF';
# php-using-git.Fetchwarefile: example fetchwarefile using php's git repo
# for lookup(), download(), and verify() functionality.
use App::Fetchware qw(:DEFAULT :OVERRIDE_LOOKUP);
use App::Fetchware::Util ':UTIL';
use Cwd 'cwd';
# The directory where the php source code's local git repo is.
my $git_repo_dir = '/home/dly/Desktop/Code/php-src';
# By default Fetchware drops privs, and since the source code repo is stored in
# the user dly's home directory, I should drop privs to dly, so that I have
# permission to access it.
user 'dly';
# Turn on verify failure ok, because the current version as of this writing (php
# 5.4.5) is signed by someone who has not shared uploaded their gpg key to a
# keyserver somewhere making git verifgy-tag fail, because I can't find the key.
# Hopefully, php bug# 65840 will result in this being fixed.
verify_failure_ok 'On';
# Determine latest version by using the tags developers create to determine the
# latest version.
hook lookup => sub {
# chdir to git repo.
chdir $git_repo_dir or die <<EOD;
php.Fetchwarefile: Failed to chdir to git repo at
[$git_repo_dir].
OS error [$!].
EOD
t/bin-fetchware-Fetchwarefile.t view on Meta::CPAN
my $package_path = cwd();
return $package_path;
};
# You must manually add php's developer's gpg keys to your gpg keyring. Do
# this by going to the page: http://us1.php.net/downloads.php . At the
# bottom the gpg key "names are listed such as "7267B52D" or "5DA04B5D."
# These are their key "names." Use gpg to download them and import them into
# your keyring using: gpg --keyserver pgp.mit.edu --recv-keys [key id]
hook verify => sub {
my ($download_path, $package_path) = @_;
# the latest tag is the download path see lookup.
my $latest_tag = $download_path;
# Run git verify-tag to verify the latest tag
my $success = eval { run_prog('git verify-tag', "$latest_tag"); 1;};
# If the git verify-tag fails, *and* verify_failure_ok has been turned on,
# then ignore the thrown exception, but print an annoying message.
unless (defined $success and $success) {
unless (config('verify_failure_ok')) {
msg <<EOM;
Verification failure ok, becuase you've configured fetchware to continue even
if it cannot verify its downloads. Please reconsider, because mirror and source
code repos do get hacked. The exception that was caught was:
[$@]
EOM
}
}
};
hook unarchive => sub {
# there is nothing to archive due to use of git.
t/bin-fetchware-Fetchwarefile.t view on Meta::CPAN
# Construct a download path using $version_number[0].
my $filename = 'mariadb-' . $version_number[0] . '.tar.gz';
# Return a proper $download_path, so That I do not have to hook download(),
# but can reuse Fetchware's download() subroutine.
my $weird_prefix = '/mariadb-' . $version_number[0] . '/kvm-tarbake-jaunty-x86/';
my $download_path = '/pub/mariadb' . $weird_prefix .$filename;
return $download_path;
};
# Make verify() failing to verify MariaDB ok, because parsing out the MD5 sum
# would require a Web scraper that supports javascript, which HTML::TreeBuilder
# obviously does not.
verify_failure_ok 'On';
# Use build_commands to configure fetchware to use MariaDB's BUILD script to
# build it.
build_commands 'BUILD/compile-pentium64-max';
# Use install_commands to tell fetchware how to install it. I could leave this
# out, but it nicely documents what command is needed to install MariaDB
# properly.
install_commands 'make install';
EOF
t/bin-fetchware-Fetchwarefile.t view on Meta::CPAN
};
# Find latest version.
my $latest_ver = lookup_by_versionstring($directory_listing);
# Return $download_path.
my $download_path = '/pub/source/'. "v$latest_ver->[0][0]" .
"/postgresql-$latest_ver->[0][0].tar.bz2";
return $download_path;
};
# MD5sums are stored on the download site, so use them to verify the package.
verify_method 'md5';
# But they are *not* stored on the original "lookup_url" site, so I must provide
# a md5_url pointing to the download site.
md5_url $mirror;
EOF
# Create a tempfile to store the Fetchwarefile in.
my ($fh, $filename) = tempfile("fetchware-test-$$-XXXXXXXXXXX", TMPDIR => 1,
UNLINK => 1);
# Write the $apache_fetchwarefile to disk, so bin/fetchware can access it.
print $fh $postgresql_fetchwarefile;
t/bin-fetchware-uninstall.t view on Meta::CPAN
program 'ctags';
lookup_url '$lookup_url';
# Must provide a mirror as well, so just use the same thing as the lookup_url.
mirror '$lookup_url';
filter 'ctags';
###BUGALERT### Add local verification for this.
verify_failure_ok 'On';
# FETCHWARE_LOCAL_URL is in a user's home directory, so drop_privs() default of
# nobody has no access to this directory hence need for user config option.
user '$ENV{FETCHWARE_NONROOT_USER}';
EOF
note('FETCHWAREFILE');
note("$fetchwarefile");
my $fetchwarefile_path = create_test_fetchwarefile($fetchwarefile);
t/bin-fetchware-upgrade-all.t view on Meta::CPAN
program 'ctags';
lookup_url '$ENV{FETCHWARE_LOCAL_UPGRADE_URL}';
mirror '$ENV{FETCHWARE_LOCAL_UPGRADE_URL}';
filter 'ctags';
# Disable verification, because ctags provides none.
verify_failure_ok 'On';
# user needed when root, because nobody won't haver permissions to access local
# user's directory where the local files above for loookup_url and mirror are.
user 'dly';
EOF
my @fetchware_packages;
for my $fetchwarefile ($apache_fetchwarefile, $ctags_fetchwarefile) {
note('FETCHWAREFILE');
note("$fetchwarefile");
( run in 0.855 second using v1.01-cache-2.11-cpan-73692580452 )