App-get_flash_videos
view release on metacpan or search on metacpan
lib/FlashVideo/VideoPreferences.pm
lib/FlashVideo/VideoPreferences/Account.pm
lib/FlashVideo/VideoPreferences/Quality.pm
Makefile.PL
MANIFEST This list of files
MANIFEST.SKIP
README
t/google_video_search.t
t/json.t
t/load.t
t/prefs.t
t/rtmpdownloader.t
t/title_to_filename.t
t/url.t
t/urls
t/utils.t
utils/autoplay.sh
utils/combine-head
utils/combine-header
utils/combine-perl.pl
utils/uncompress-flash.pl
get_flash_videos view on Meta::CPAN
}
}
else {
@download_urls = @urls;
}
my $download_count = 0;
# Construct a preferences object for these downloads, currently just based on
# the command line options.
my $prefs = FlashVideo::VideoPreferences->new(%opt);
foreach my $url (@download_urls) {
if (download($url, $prefs, @download_urls - $download_count)) {
$download_count++;
}
}
if($download_count == 0) {
info "Couldn't download any videos.";
exit 1;
} elsif($download_count != @download_urls) {
info "Problems downloading some videos.";
exit 2;
}
exit 0;
sub download {
my($url, $prefs, $remaining) = @_;
$url = "http://$url" if $url !~ m!^\w+:!;
# Might be downloading from a site that uses Brightcove or other similar
# Flash RTMP streaming server. These are handled differently. Need to get
# the page to determine this.
info "Downloading $url";
my $browser = FlashVideo::Mechanize->new;
$browser->get($url);
get_flash_videos view on Meta::CPAN
# for "mature" videos.)
if (!$browser->success and !$browser->response->is_redirect) {
error "Couldn't download '$url': " . $browser->response->status_line;
}
# Figure out what package we need to use to get either the HTTP URL or
# rtmpdump data for the video.
my($package, $possible_url) = FlashVideo::URLFinder->find_package($url, $browser);
my($actual_url, @suggested_fnames) = eval {
$package->find_video($browser, $possible_url, $prefs);
};
if(!$actual_url) {
if($@ =~ /^Must have | requires /) {
my $error = "$@";
$error =~ s/at $0.*//;
print STDERR "$error" . REQ_INFO;
return 0;
} else {
print STDERR "Error: $@" . FRIENDLY_FAILURE;
lib/FlashVideo/Generic.pm view on Meta::CPAN
use strict;
use FlashVideo::Utils;
use URI;
use URI::Escape qw(uri_unescape);
my $video_re = qr!http[-:/a-z0-9%_.?=&]+@{[EXTENSIONS]}
# Grab any params that might be used for auth..
(?:\?[-:/a-z0-9%_.?=&]+)?!xi;
sub find_video {
my ($self, $browser, $embed_url, $prefs) = @_;
# First strategy - identify all the Flash video files, and download the
# biggest one. Yes, this is hacky.
if (!$browser->success) {
$browser->get($browser->response->header('Location'));
die "Couldn't download URL: " . $browser->response->status_line
unless $browser->success;
}
my ($possible_filename, $actual_url, $title);
lib/FlashVideo/Generic.pm view on Meta::CPAN
RE: for my $regex(
qr{(?si)<embed.*?flashvars=["']?([^"'>]+)},
qr{(?si)<embed.*?src=["']?([^"'>]+)},
qr{(?si)<a[^>]* href=["']?([^"'>]+?@{[EXTENSIONS]})},
qr{(?si)<object[^>]*>.*?<param [^>]*value=["']?([^"'>]+)},
qr{(?si)<object[^>]*>(.*?)</object>},
# Attempt to handle scripts using flashvars / swfobject
qr{(?si)<script[^>]*>(.*?)</script>}) {
for my $param($browser->content =~ /$regex/gi) {
(my $url, $possible_filename, $filename_is_reliable) = find_file_param($browser->clone, $param, $prefs);
if($url) {
my $resolved_url = url_exists($browser->clone, $url);
if($resolved_url) {
$actual_url = $resolved_url;
last RE;
}
}
}
}
if(!$actual_url) {
for my $iframe($browser->content =~ /<iframe[^>]+src=["']?([^"'>]+)/gi) {
$iframe = URI->new_abs($iframe, $browser->uri);
debug "Found iframe: $iframe";
my $sub_browser = $browser->clone;
$sub_browser->get($iframe);
($actual_url) = eval { $self->find_video($sub_browser, undef, $prefs) };
}
}
}
my @filenames;
return $actual_url, $possible_filename if $filename_is_reliable;
$possible_filename =~ s/\?.*//;
# The actual filename, provided it looks like it might be reasonable
lib/FlashVideo/Generic.pm view on Meta::CPAN
my $filename = uri_unescape(File::Basename::basename(URI->new($flv_url)->path()));
$filename =~ s/\.flv$//i;
return ($flv_url, $filename);
}
return;
}
sub find_file_param {
my($browser, $param, $prefs) = @_;
for my $file($param =~ /(?:video|movie|file|path)_?(?:href|src|url)?['"]?\s*[=:,]\s*['"]?([^&'" ]+)/gi,
$param =~ /(?:config|playlist|options)['"]?\s*[,:=]\s*['"]?(http[^'"&]+)/gi,
$param =~ /['"=](.*?@{[EXTENSIONS]})/gi,
$param =~ /([^ ]+@{[EXTENSIONS]})/gi,
$param =~ /SWFObject\(["']([^"']+)/) {
debug "Found $file";
my ($actual_url, $filename, $filename_is_reliable) = guess_file($browser, $file, '', $prefs);
if(!$actual_url && $file =~ /\?(.*)/) {
# Maybe we have query params?
debug "Trying query param on $1";
for my $query_param(split /[;&]/, $1) {
my($query_key, $query_value) = split /=/, $query_param;
debug "Found $query_value from $query_key";
($actual_url, $filename, $filename_is_reliable)
= guess_file($browser, $query_value, '', $prefs);
last if $actual_url;
}
}
if($actual_url) {
my $possible_filename = $filename || (split /\//, $actual_url)[-1];
return $actual_url, $possible_filename, $filename_is_reliable;
}
}
if($param =~ m{(rtmp://[^ &"']+)}) {
info "This looks like RTMP ($1), no generic support yet..";
}
return;
}
sub guess_file {
my($browser, $file, $once, $prefs) = @_;
# Contains lots of URI encoding, so try escaping..
$file = uri_unescape($file) if scalar(() = $file =~ /%[A-F0-9]{2}/gi) > 3;
my $orig_uri = URI->new_abs($file, $browser->uri);
info "Guessed $orig_uri trying...";
if($orig_uri) {
my $uri = url_exists($browser->clone, $orig_uri);
if($uri) {
# Check to see if this URL is for a supported site.
my ($package, $url) = FlashVideo::URLFinder->find_package($uri,
$browser->clone);
if($package && $package ne __PACKAGE__) {
debug "$uri is supported by $package.";
(my $browser_on_supported_site = $browser->clone())->get($uri);
return $package->find_video($browser_on_supported_site, $uri, $prefs), 1;
}
my $content_type = $browser->response->header("Content-type");
if($content_type =~ m!^(text|application/xml)!) {
# Just in case someone serves the video itself as text/plain.
$browser->add_header("Range", "bytes=0-10000");
$browser->get($uri);
$browser->delete_header("Range");
lib/FlashVideo/Generic.pm view on Meta::CPAN
# If this looks like HTML we have no hope of guessing right, so
# give up now.
return if $browser->content =~ /<html[^>]*>/i;
if($browser->content =~ m!($video_re)!) {
# Found a video URL
return $1;
} elsif(!defined $once
&& $browser->content =~ m!(http[-:/a-zA-Z0-9%_.?=&]+)!i) {
# Try once more, one level deeper..
return guess_file($browser, $1, 1, $prefs);
} else {
info "Tried $uri, but no video URL found";
}
} elsif($content_type =~ m!application/! && $uri ne $orig_uri) {
# We were redirected, maybe something in the new URL?
return((find_file_param($browser, $uri))[0]);
} else {
return $uri->as_string;
}
} elsif(not defined $once) {
# Try using the location of the .swf file as the base, if it's different.
if($browser->content =~ /["']([^ ]+\.swf)/) {
my $swf_uri = URI->new_abs($1, $browser->uri);
if($swf_uri) {
my $new_uri = URI->new_abs($file, $swf_uri);
debug "Found SWF: $swf_uri -> $new_uri";
if($new_uri ne $uri) {
return guess_file($browser, $new_uri, 1, $prefs);
}
}
}
}
}
return;
}
1;
lib/FlashVideo/Site/Apple.pm view on Meta::CPAN
# Part of get-flash-videos. See get_flash_videos for copyright.
package FlashVideo::Site::Apple;
use strict;
use FlashVideo::Utils;
sub find_video {
my ($self, $browser, $embed_url, $prefs) = @_;
if(!FlashVideo::Downloader->check_file($browser->content)) {
# We weren't given a quicktime link, so find one..
my @urls = $browser->content =~ /['"]([^'"]+\.mov)(?:\?[^'"]+)?['"]/g;
die "No .mov URLs found on page" unless @urls;
debug "Found URLs: @urls";
my $redirect_url = $prefs->quality->choose(map {
/(\d+p?)\.mov/ && {
url => $_,
resolution => $prefs->quality->format_to_resolution($1)
}
} @urls
)->{url};
$browser->get($redirect_url);
}
my $url = $self->handle_mov($browser);
my $filename = ($url->path =~ m{([^/]+)$})[0];
lib/FlashVideo/Site/Bing.pm view on Meta::CPAN
# Part of get-flash-videos. See get_flash_videos for copyright.
package FlashVideo::Site::Bing;
use strict;
use FlashVideo::Utils;
sub find_video {
my ($self, $browser, $embed_url, $prefs) = @_;
my $count = 0;
while((my $location = $browser->response->header("Location")) && $count++ < 5) {
$browser->get($location);
}
my $title;
if ($browser->content =~ /sourceFriendly:\s*'([^']+)'[\s\S]+?\s*title:\s*'([^']+)'/) {
$title = "$1 - $2";
}
lib/FlashVideo/Site/Dailymotion.pm view on Meta::CPAN
# Part of get-flash-videos. See get_flash_videos for copyright.
package FlashVideo::Site::Dailymotion;
use strict;
use FlashVideo::Utils;
use URI::Escape;
sub find_video {
my ($self, $browser, $embed_url, $prefs) = @_;
if ($browser->content =~ /content.is.not.available.for.your.country/i) {
error "Can't (yet) download this video because it's not available " .
"in your area";
exit 1;
}
$browser->allow_redirects;
$browser->content =~ /<h1[^>]*>(.*?)<\//;
lib/FlashVideo/Site/Dailymotion.pm view on Meta::CPAN
if($data =~ /videotitle=([^&]+)/) {
$filename = title_to_filename(uri_unescape($1));
}
}
if(!$video) {
# Sometimes dailymotion actually embeds another site, so check that..
my($package, $possible_url) = FlashVideo::URLFinder->find_package($browser->uri, $browser);
if($package ne __PACKAGE__) {
return $package->find_video($browser, $possible_url, $prefs);
}
}
die "Couldn't find video parameter." unless $video;
my @streams;
for(split /\|\|/, $video) {
my($path, $type) = split /@@/;
my($width, $height) = $path =~ /(\d+)x(\d+)/;
lib/FlashVideo/Site/Pbs.pm view on Meta::CPAN
Programs that don't work yet:
- http://www.pbs.org/wgbh/pages/frontline/woundedplatoon/view/
- http://www.pbs.org/wgbh/roadshow/rmw/RMW-003_200904F02.html
TODO:
- subtitles
=cut
sub find_video {
my ($self, $browser, $embed_url, $prefs) = @_;
die "Must have Crypt::Rijndael installed to download from PBS"
unless eval { require Crypt::Rijndael };
my ($media_id) = $browser->uri->as_string =~ m[
^http://video\.pbs\.org/video/(\d+)
]x;
unless (defined $media_id) {
($media_id) = $browser->content =~ m[
http://video\.pbs\.org/widget/partnerplayer/(\d+)
lib/FlashVideo/Site/Pbs.pm view on Meta::CPAN
($media_id) = $browser->content =~ m[var videoUrl = "([^"]+)"];
}
unless (defined $media_id) {
my ($pap_id, $youtube_id) = $browser->content =~ m[
\bDetectFlashDecision\ \('([^']+)',\ '([^']+)'\);
]x;
if ($youtube_id) {
debug "Youtube ID found, delegating to Youtube plugin\n";
my $url = "http://www.youtube.com/v/$youtube_id";
require FlashVideo::Site::Youtube;
return FlashVideo::Site::Youtube->find_video($browser, $url, $prefs);
}
}
die "Couldn't find media_id\n" unless defined $media_id;
debug "media_id: $media_id\n";
$browser->get("http://video.pbs.org/videoPlayerInfo/$media_id");
my $xml = $browser->content;
$xml =~ s/&/&/g;
my $href = from_xml($xml);
lib/FlashVideo/Site/Seesaw.pm view on Meta::CPAN
use HTML::Entities qw(decode_entities);
use URI::Escape qw(uri_escape);
my @res = (
{ name => "lowResUrl", resolution => [ 512, 288 ] },
{ name => "stdResUrl", resolution => [ 672, 378 ] },
{ name => "highResUrl", resolution => [ 1024, 576 ] }
);
sub find_video {
my ($self, $browser, $page_url, $prefs) = @_;
# The videoplayerinfo info URL now appears as the Nth parameter to
# player.init(), so just look for the videoplayerinfo directly, rather
# than looking for player.init and the first parameter.
my $player_info = ($browser->content =~ m{(/videoplayerinfo/\d+[^"]+)"})[0];
# Remove escaped slashes
(my $content = $browser->content) =~ s{\\/}{/}g;
# Grab title and normalise
lib/FlashVideo/Site/Seesaw.pm view on Meta::CPAN
my @urls;
for my $res(@res) {
if($browser->content =~ /$res->{name}":\["([^"]+)/) {
push @urls, { %$res, url => $1 };
}
}
die "No video URLs found" unless @urls;
my $rtmp = $prefs->quality->choose(@urls);
my($app, $playpath, $query) = $rtmp->{url} =~ m{^\w+://[^/]+/(\w+/\w+)(/[^?]+)(\?.*)};
my $prefix = "mp4";
$prefix = "flv" if $playpath =~ /\.flv$/;
if ($prefs->subtitles) {
if ($browser->content =~ m{"subtitleLocation":\["([^"]+)"\]}) {
my $subtitles_url = $1;
if ($subtitles_url =~ m{^/}) {
$subtitles_url = "http://www.seesaw.com$subtitles_url";
}
debug "Got Seesaw subtitles URL: $subtitles_url";
$browser->get($subtitles_url);
lib/FlashVideo/Site/Stickam.pm view on Meta::CPAN
# Part of get-flash-videos. See get_flash_videos for copyright.
package FlashVideo::Site::Stickam;
use strict;
use FlashVideo::Utils;
sub find_video {
my($self, $browser, $embed_url, $prefs) = @_;
my $perfomer_id;
if ($browser->content =~ /profileUserId=(\d+)/) {
$perfomer_id = $1;
}
else {
die "Can't get performer ID";
}
lib/FlashVideo/Site/Truveo.pm view on Meta::CPAN
# Part of get-flash-videos. See get_flash_videos for copyright.
package FlashVideo::Site::Truveo;
use strict;
use FlashVideo::Utils;
sub find_video {
my($self, $browser, $embed_url, $prefs) = @_;
my($videourl) = $browser->content =~ /var videourl = "(.*?)"/;
# Maybe we were given a direct URL..
$videourl = $embed_url
if !$videourl && $browser->uri->host eq 'xml.truveo.com';
die "videourl not found" unless $videourl;
$browser->get($videourl);
if($browser->content =~ /url=(http:.*?)["']/) {
my $redirect = url_exists($browser, $1);
$browser->get($redirect);
my($package, $possible_url) = FlashVideo::URLFinder->find_package($redirect, $browser);
die "Recursion detected" if $package eq __PACKAGE__;
return $package->find_video($browser, $possible_url, $prefs);
} else {
die "Redirect URL not found";
}
}
1;
lib/FlashVideo/Site/Youtube.pm view on Meta::CPAN
{ id => 22, resolution => [1280, 720] },
{ id => 35, resolution => [854, 480] },
{ id => 34, resolution => [640, 360] },
{ id => 18, resolution => [480, 270] },
{ id => 5, resolution => [400, 224] },
{ id => 17, resolution => [176, 144] },
{ id => 13, resolution => [176, 144] },
);
sub find_video {
my ($self, $browser, $embed_url, $prefs) = @_;
if($embed_url !~ m!youtube\.com/watch!) {
$browser->get($embed_url);
if ($browser->response->header('Location') =~ m!/swf/.*video_id=([^&]+)!
|| $embed_url =~ m!/v/([-_a-z0-9]+)!i
|| $browser->uri =~ m!v%3D([-_a-z0-9]+)!i) {
# We ended up on a embedded SWF or other redirect page
$embed_url = "http://www.youtube.com/watch?v=$1";
$browser->get($embed_url);
}
}
if (!$browser->success) {
verify_age($browser, $prefs);
}
my $title = extract_info($browser)->{meta_title};
if (!$title and
$browser->content =~ /<div id="vidTitle">\s+<span ?>(.+?)<\/span>/ or
$browser->content =~ /<div id="watch-vid-title">\s*<div ?>(.+?)<\/div>/) {
$title = $1;
}
# If the page contains fmt_url_map, then process this. With this, we
# don't require the 't' parameter.
if ($browser->content =~ /["']fmt_url_map["']:\s{0,3}(["'][^"']+["'])/) {
debug "Using fmt_url_map method from page ($1)";
return $self->download_fmt_map($prefs, $browser, $title, {}, @{from_json $1});
}
my $video_id;
if ($browser->content =~ /(?:var pageVideoId =|(?:CFG_)?VIDEO_ID'?\s*:)\s*'(.+?)'/
|| $embed_url =~ /v=([^&]+)/) {
$video_id = $1;
} else {
check_die($browser, "Couldn't extract video ID");
}
lib/FlashVideo/Site/Youtube.pm view on Meta::CPAN
} else {
die "Couldn't extract SWF URL";
}
my $rtmp_url = $info{conn};
if($info{fmt_stream_map}) {
my $fmt_stream_map = parse_youtube_format_url_map($info{fmt_stream_map}, 1);
# Sort by quality...
my $preferred_quality = $prefs->quality->choose(map { $fmt_stream_map->{$_->{id}}
? { resolution => $_->{resolution}, url => $fmt_stream_map->{$_->{id}} }
: () } @formats);
$rtmp_url = $preferred_quality->{url};
}
return {
flv => title_to_filename($title),
rtmp => $rtmp_url,
swfhash($browser, $swf_url)
};
} elsif($info{fmt_url_map}) {
debug "Using fmt_url_map method from info";
return $self->download_fmt_map($prefs, $browser, $title, \%info, $info{fmt_url_map});
}
}
# Try old get_video method, just incase.
return download_get_video($browser, $prefs, $video_id, $title, $t);
}
sub download_fmt_map {
my($self, $prefs, $browser, $title, $info, $fmt_map) = @_;
my $fmt_url_map = parse_youtube_format_url_map($fmt_map);
if (!$title and $browser->uri->as_string =~ m'/user/.*?#') {
# This is a playlist and getting the video title without the ID is
# practically impossible because multiple videos are referenced in the
# page. However, the encrypted (apparently) video ID is included in the
# URL.
my $video_id = (split /\//, $browser->uri->fragment)[-1];
my %info = get_youtube_video_info($browser->clone, $video_id);
$title = $info->{title};
}
# Sort by quality...
my $preferred_quality = $prefs->quality->choose(map { $fmt_url_map->{$_->{id}}
? { resolution => $_->{resolution}, url => $fmt_url_map->{$_->{id}} }
: () } @formats);
$browser->allow_redirects;
return $preferred_quality->{url}, title_to_filename($title, "mp4");
}
sub download_get_video {
my($browser, $prefs, $video_id, $title, $t) = @_;
my $fetcher = sub {
my($url, $filename) = @_;
$url = url_exists($browser->clone, $url, 1);
return $url, $filename if $url;
return;
};
my @formats_to_try = @formats;
while(my $fmt = $prefs->quality->choose(@formats_to_try)) {
# Remove from the list
@formats_to_try = grep { $_ != $fmt } @formats_to_try;
# Try it..
my @ret = $fetcher->("http://www.youtube.com/get_video?fmt=$fmt->{id}&video_id=$video_id&t=$t",
title_to_filename($title, "mp4"));
return @ret if @ret;
}
# Otherwise try without an ID
lib/FlashVideo/Site/Youtube.pm view on Meta::CPAN
$alert =~ s/(^\s+|\s+$)//g;
$message .= "\nYouTube: $alert";
error $message;
exit 1;
} else {
die "$message\n";
}
}
sub verify_age {
my($browser, $prefs) = @_;
my $orig_uri = $browser->uri;
if ($browser->response->code == 303
&& $browser->response->header('Location') =~ m!/verify_age|/accounts/!) {
my $confirmation_url = $browser->response->header('Location');
$browser->get($confirmation_url);
if($browser->content =~ /has_verified=1/) {
my($verify_url) = $browser->content =~ /href="(.*?has_verified=1)"/;
$verify_url = decode_entities($verify_url);
$browser->get($verify_url);
# Great that worked, otherwise probably does want a login
return if $browser->response->code == 200;
}
# Lame age verification page - yes, we are grown up, please just give
# us the video!
my $account = $prefs->account("youtube", <<EOT);
Unfortunately, due to Youtube being lame, you have to have
an account to download this video. (See the documentation for how to configure
~/.netrc)
EOT
unless ($account->username and $account->password) {
error "You must supply Youtube account details.";
exit 1;
}
#!perl
use Test::More tests => 24;
BEGIN {
use_ok("FlashVideo::VideoPreferences");
}
my $prefs = FlashVideo::VideoPreferences->new;
isa_ok $prefs, FlashVideo::VideoPreferences;
# ---- Quality ----
my $q = $prefs->quality;
isa_ok $q, FlashVideo::VideoPreferences::Quality;
# Default is high quality
is $q->name, "high";
# Check formats are understood
# Standard names
is_deeply $q->format_to_resolution("720p"), [1280, 720, "high"];
is_deeply $q->format_to_resolution("480p"), [640, 480, "medium"];
( run in 1.977 second using v1.01-cache-2.11-cpan-0bb4e1dffa6 )