App-Chart
view release on metacpan or search on metacpan
unused/YahooOld.pm view on Meta::CPAN
# looking at a quote from the various international XX.finance.yahoo.com.
# For the zone for the date however you need to be watching at midnight
# GMT, where it ticks over (at all the international XX.finance.yahoo.com).
# quote_parse_div_date ($str) returns an iso YYYY-MM-DD date string for a
# dividend $str coming from quote.csv data, or undef if none. There are
# several different formats,
# "Jan 7" # finance.yahoo.com
# " 5 Jan" # au.finance, uk.finance
# "24-Sep-04" # ABB.AX on finance.yahoo.com
# "24 Sep, 2004" # ABB.AX on au.finance
# "Sep 24, 2004" # ABB.AX on ca.finance
#
# An error is thrown for an unrecognised string, don't want some new form to
# end up with dividends silently forgotten.
#
sub quote_parse_div_date {
my ($str) = @_;
if (DEBUG) { print "quote_parse_div_date() '$str'\n"; }
if (! defined $str || $str eq 'N/A' || $str eq '') {
return undef; # no info
}
my ($ss,$mm,$hh,$day,$month,$year,$zone) = Date::Parse::strptime ($str);
$month++;
if ($year) {
$year += 1900;
if ($year < 2000) { # "04" returned as 1904, bump to 2004
$year += 100;
}
} else {
# year not given, try nearest
$year = App::Chart::Download::month_to_nearest_year ($month);
}
if (! Date::Calc::check_date ($year, $month, $day)) {
warn "Yahoo invalid dividend date '$str'";
}
return App::Chart::ymd_to_iso ($year, $month, $day);
}
#------------------------------------------------------------------------------
# Latest Quotes by CSV
# wget -S -O /dev/stdout 'http://download.finance.yahoo.com/d/quotes.csv?f=snc4b3b2d1t1oml1c1vqdx&e=.csv&s=GM'
#
# sub latest_download {
# my ($symbol_list) = @_;
# App::Chart::Download::status (__('Yahoo quotes'));
#
# # App::Chart::Download::verbose_message ("Yahoo crumb $crumb cookies\n"
# # . $jar->as_string);
#
# my $crumb_errors = 0;
# SYMBOL: foreach my $symbol (@$symbol_list) {
# my $tdate = daily_available_tdate ($symbol);
#
# App::Chart::Download::status(__('Yahoo quote'), $symbol);
#
# my $lo_timet = tdate_to_unix($tdate - 4);
# my $hi_timet = tdate_to_unix($tdate + 2);
#
# my $data = daily_cookie_data($symbol);
# if (! defined $data) {
# print "Yahoo $symbol does not exist\n";
# next SYMBOL;
# }
# my $crumb = URI::Escape::uri_escape($data->{'crumb'});
# my $jar = http_cookies_from_string($data->{'cookies'} // '');
#
# my $events = 'history';
# my $url = "http://query1.finance.yahoo.com/v7/finance/download/"
# . URI::Escape::uri_escape($symbol)
# . "?period1=$lo_timet&period2=$hi_timet&interval=1d&events=$events&crumb=$crumb";
#
# my $resp = App::Chart::Download->get ($url,
# allow_401 => 1,
# allow_404 => 1,
# cookie_jar => $jar,
# );
# if ($resp->code == 401) {
# if (++$crumb_errors >= 2) { die "Yahoo: crumb authorization failed"; }
# App::Chart::Database->write_extra ('', 'yahoo-daily-cookies', undef);
# redo SYMBOL;
# }
# if ($resp->code == 404) {
# print "Yahoo $symbol does not exist\n";
# next SYMBOL;
# }
#
# App::Chart::Download::write_latest_group
# (latest_parse($symbol,$resp,$tdate));
# }
# }
#
# sub latest_parse {
# my ($symbol, $resp, $tdate) = @_;
#
# my $h = { source => __PACKAGE__,
# resp => $resp,
# prefer_decimals => 2,
# date_format => 'ymd' };
# daily_parse($symbol,$resp,$h);
#
# my $data = $h->{'data'};
# @$data = sort {$a->{'date'} cmp $b->{'date'}} @$data;
#
# my $this = (@$data ? $data->[-1] : {});
# $this->{'symbol'} = $symbol;
# $this->{'last_date'} = delete $this->{'date'};
# my $last = $this->{'last'} = delete $this->{'close'};
# if (defined $last && @$data >= 2) {
# my $prev = $data->[-2]->{'close'};
# if (defined $prev) {
# $this->{'change'} = decimal_subtract($last, $prev);
# }
# }
# @$data = ($this);
# return $h;
# }
unused/YahooOld.pm view on Meta::CPAN
# info => \@info,
# dividends => \@dividends };
#
# require Text::CSV_XS;
# my $csv = Text::CSV_XS->new;
#
# foreach my $line (App::Chart::Download::split_lines ($content)) {
# $csv->parse($line);
# my ($symbol, $name, $exchange, $currency, $ex_date, $pay_date, $amount)
# = $csv->fields();
#
# $ex_date = quote_parse_div_date ($ex_date);
# $pay_date = quote_parse_div_date ($pay_date);
#
# push @info, { symbol => $symbol,
# name => $name,
# currency => $currency,
# exchange => $exchange };
# # circa 2015 the "d" dividend amount field is "N/A" when after the
# # dividend payment (with "r1" pay date "N/A" too)
# if ($ex_date && $amount ne 'N/A' && $amount != 0) {
# push @dividends, { symbol => $symbol,
# ex_date => $ex_date,
# pay_date => $pay_date,
# amount => $amount };
# }
# }
# return $h;
# }
#--------------------------------------------------------------------
# Past Weblink info page
#
# For reference, past was like http://finance.yahoo.com/q?s=BHP.AX
# but that's been taken away so as to break everybody's bookmarks.
#-----------------------------------------------------------------------------
# v7 cookies and crumb
# Download Data
#
# This uses the historical prices page like
#
# https://finance.yahoo.com/quote/AMP.AX/history?p=AMP.AX
#
# which puts a cookie like
#
# Set-Cookie: B=fab5sl9cqn2rd&b=3&s=i3; expires=Sun, 03-Sep-2018 04:56:13 GMT; path=/; domain=.yahoo.com
#
# and contains buried within 1.5 mbytes of hideous script
#
# <script type="application/json" data-sveltekit-fetched data-url="https://query1.finance.yahoo.com/v1/test/getcrumb?lang=en-US&region=US" data-ttl="59">{"status":200,"statusText":"OK","headers":{},"body":"DKVWQE/ggh4"}</script>
#
# Any \u002F or similar is escaped "/" character or similar.
# The crumb is included in a CSV download query like the following
# (alas can't use http, it redirects to https)
#
# https://query1.finance.yahoo.com/v7/finance/download/AMP.AX?period1=1503810440&period2=1504415240&interval=1d&events=history&crumb=hdDX/HGsZ0Q
#
# period1 is the start time, period2 the end time, both as Unix seconds
# since 1 Jan 1970. Not sure of the timezone needed. Some experiments
# suggest it depends on the timezone of the symbol. http works as well as
# https. The result is like
#
# Date,Open,High,Low,Close,Adj Close,Volume
# 2017-09-07,30.299999,30.379999,30.000000,30.170000,30.170000,3451099
#
# The "9999s" are some bad rounding off to what would be usually at most
# 3 (maybe 4?) decimal places.
#
# Response is 404 if no such symbol, 401 unauthorized if no cookie or crumb.
#
# "events=div" gives dividends like
#
# Date,Dividends
# 2017-08-11,0.161556
#
# "events=div" gives splits like, for a consolidation (GXY.AX)
#
# Date,Stock Splits
# 2017-05-22,1/5
#
#----------------
# For reference, there's a "v8" which is json format (%7C = "|")
#
# https://query2.finance.yahoo.com/v8/finance/chart/IBM?formatted=true&lang=en-US®ion=US&period1=1504028419&period2=1504428419&interval=1d&events=div%7Csplit&corsDomain=finance.yahoo.com
#
# This doesn't require a cookie and crumb, has some info like symbol
# timezone. The numbers look like they're rounded through 32-bit single
# precision floating point, for example "142.55999755859375" which is 142.55
# in a 23-bit mantissa. log(14255000)/log(2) = 23.76 bits
# Are they about the same precision as the CSV ?
#
# FIXME: All prices look like they're split-adjusted, which is ok if that's
# what you want and are downloading a full data set, but bad for incremental
# since you don't know when a change is applied.
#
App::Chart::DownloadHandler->new
(name => __('Yahoo'),
pred => $download_pred,
available_tdate_by_symbol => \&daily_available_tdate,
available_tdate_extra => 2,
proc => \&daily_download,
chunk_size => 150);
sub daily_available_tdate {
my ($symbol) = @_;
# Sep 2017: daily data is present for the current day's trade, during the
# trading session. Try reckoning it complete at 6pm.
return App::Chart::Download::tdate_today_after
(18,0, App::Chart::TZ->for_symbol ($symbol));
# return App::Chart::Download::tdate_today_after
# (10,30, App::Chart::TZ->for_symbol ($symbol))
# - 1;
}
sub daily_download {
my ($symbol_list) = @_;
App::Chart::Download::status (__('Yahoo daily data'));
# App::Chart::Download::verbose_message ("Yahoo crumb $crumb cookies\n"
# . $jar->as_string);
my $crumb_errors = 0;
SYMBOL: foreach my $symbol (@$symbol_list) {
my $lo_tdate = App::Chart::Download::start_tdate_for_update (@$symbol_list);
my $hi_tdate = daily_available_tdate ($symbol);
App::Chart::Download::status
(__('Yahoo data'), $symbol,
App::Chart::Download::tdate_range_string ($lo_tdate, $hi_tdate));
my $lo_timet = tdate_to_unix($lo_tdate - 2);
my $hi_timet = tdate_to_unix($hi_tdate + 2);
my $data = daily_cookie_data($symbol);
if (! defined $data) {
print "Yahoo $symbol no daily cookie data\n";
next SYMBOL;
}
my $crumb = URI::Escape::uri_escape($data->{'crumb'});
my $jar = http_cookies_from_string($data->{'cookies'} // '');
my $h = { source => __PACKAGE__,
prefer_decimals => 2,
date_format => 'ymd',
};
foreach my $elem (['history',\&daily_parse],
['div', \&daily_parse_div],
['split', \&daily_parse_split]) {
my ($events,$parse) = @$elem;
my $url = "https://query1.finance.yahoo.com/v7/finance/download/"
. URI::Escape::uri_escape($symbol)
. "?period1=$lo_timet"
. "&period2=$hi_timet"
. "&interval=1d"
. "&events=$events"
. "&crumb=$crumb"
;
my $resp = App::Chart::Download->get ($url,
allow_401 => 1,
allow_404 => 1,
# cookie_jar => $jar,
);
if ($resp->code == 401) {
if (++$crumb_errors >= 2) {
die "Yahoo: crumb authorization failed";
}
App::Chart::Database->write_extra ('', 'yahoo-daily-cookies', undef);
redo SYMBOL;
}
if ($resp->code == 404) {
print "Yahoo $symbol does not exist\n";
next SYMBOL;
}
$parse->($symbol,$resp,$h, $hi_tdate);
}
### $h
App::Chart::Download::write_daily_group ($h);
}
}
sub daily_parse {
my ($symbol, $resp, $h, $hi_tdate) = @_;
my @data = ();
$h->{'data'} = \@data;
my $hi_tdate_iso;
if (defined $hi_tdate){ $hi_tdate_iso = App::Chart::tdate_to_iso($hi_tdate); }
my $body = $resp->decoded_content (raise_error => 1);
my @line_list = App::Chart::Download::split_lines($body);
unless ($line_list[0] =~ /^Date,Open,High,Low,Close,Adj Close,Volume/) {
( run in 0.742 second using v1.01-cache-2.11-cpan-64ef6c95b5d )