App-Chart
view release on metacpan or search on metacpan
unused/Yahoo-v7.pm view on Meta::CPAN
# recheck_days => 14);
sub info_url {
my ($symbol) = @_;
return 'https://query2.finance.yahoo.com/v1/finance/search?q='
. URI::Escape::uri_escape ($symbol)
. '&enableFuzzyQuery=false';
}
sub info_download {
my ($symbol_list) = @_;
foreach my $symbol (@$symbol_list) {
App::Chart::Download::status(__('Yahoo info'), $symbol);
my $url = info_url($symbol);
my $resp = App::Chart::Download->get ($url);
my $h = info_parse($resp);
$h->{'recheck_list'} = [ $symbol ];
App::Chart::Download::write_daily_group ($h);
}
}
sub info_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (raise_error => 1);
my @info;
my $h = { source => __PACKAGE__,
info => \@info };
require JSON;
my $J = JSON::from_json($content) // {};
my $quotes = $J->{'quotes'} // [];
if (my $e = $quotes->[0]) {
my $symbol = $e->{'symbol'};
my $name = $e->{'shortname'};
if (defined $name) {
# ASX shares have the symbol repeated at the end of shortname,
# like "BHP FPO [BHP]". Seems unnecessary, so strip that.
my $end = '[' . App::Chart::symbol_sans_suffix ($symbol) .']';
$name =~ s/\s*\Q$end\E$//;
}
push @info, { symbol => $symbol,
name => $name,
exchange => $e->{'exchange'},
};
}
return $h;
}
#------------------------------------------------------------------------------
# Latest
#
# This uses for example
#
# https://query1.finance.yahoo.com/v7/finance/chart/BHP.AX?period1=1718841600&period2=1719532800&interval=1d&events=history&close=unadjusted
#
# periodi1 and period2 are Unix style seconds since 1 Jan 1970 GMT.
#
# https://stackoverflow.com/questions/47076404/currency-helper-of-yahoo-sorry-unable-to-process-request-at-this-time-erro
# ->
# https://stackoverflow.com/questions/47064776/has-yahoo-suddenly-today-terminated-its-finance-download-api
App::Chart::LatestHandler->new
(pred => $latest_pred,
proc => \&latest_download,
max_symbols => 1); # downloads go 1 at a time
sub latest_download {
my ($symbol_list) = @_;
foreach my $symbol (@$symbol_list) {
my $tdate = daily_available_tdate ($symbol);
App::Chart::Download::status(__('Yahoo quote'), $symbol);
my $lo_timet = tdate_to_unix($tdate - 4);
my $hi_timet = tdate_to_unix($tdate + 2);
my $events = 'history';
my $url = "https://query1.finance.yahoo.com/v7/finance/chart/"
. URI::Escape::uri_escape($symbol)
."?period1=$lo_timet"
."&period2=$hi_timet"
."&interval=1d"
."&events=$events"
."&close=unadjusted";
# unknown symbol is 404 with json error details
#
my $resp = App::Chart::Download->get ($url, allow_404 => 1,);
App::Chart::Download::write_latest_group
(latest_parse($symbol,$resp,$tdate));
}
}
sub latest_parse {
my ($symbol, $resp, $tdate) = @_;
my $content = $resp->decoded_content (raise_error => 1);
require JSON;
my $json = JSON::from_json($content);
my %record = (symbol => $symbol,
);
my $h = { source => __PACKAGE__,
resp => $resp,
prefer_decimals => 2,
date_format => 'ymd',
data => [ \%record ],
};
if (defined (my $error = $json->{'chart'}->{'error'}->{'code'})) {
$record{'error'} = $error;
}
if (my $result = $json->{'chart'}->{'result'}->[0]) {
my $meta = $result->{'meta'}
// die "Yahoo JSON oops, no meta";
$record{'currency'} = $meta->{'currency'},
$record{'exchange'} = $meta->{'exchangeName'},
my $symbol_timezone = App::Chart::TZ->for_symbol ($symbol);
my $timestamps = $result->{'timestamp'}
// die "Yahoo JSON oops, no timestamp";
if (@$timestamps) {
# timestamps are time of last trade, as can be seen by looking at
# something with low enough volume, eg. RMX.AX
#
if (defined (my $timet = $timestamps->[-1])) {
($record{'last_date'}, $record{'last_time'})
= $symbol_timezone->iso_date_time($timet);
}
if (my $indicators = $result->{'indicators'}->{'quote'}->[0]) {
foreach my $key ('open','high','low') {
if (my $aref = $indicators->{$key}) {
$record{$key} = crunch_trailing_nines($aref->[$#$timestamps]);
}
}
if (my $aref = $indicators->{'volume'}) {
$record{'volume'} = $aref->[$#$timestamps];
}
if (my $aref = $indicators->{'close'}) {
my $last = $record{'last'}
= crunch_trailing_nines($aref->[$#$timestamps]);
# "change" from second last timestamp, if there is one.
# As of Nov 2017, XAUUSD=X only ever gives a single latest
# quote from v7, no previous day to compare.
#
if (defined $last
&& scalar(@$timestamps) >= 2
&& defined(my $prev = $aref->[$#$timestamps - 1])) {
$record{'change'}
= App::Chart::decimal_sub($last, crunch_trailing_nines($prev));
}
}
}
}
if (defined $record{'last_date'}
&& (my $splits = $result->{'events'}->{'splits'})) {
while (my ($timet, $href) = each %$splits) {
my $split_date = $symbol_timezone->iso_date($timet);
if ($split_date eq $record{'last_date'}) {
__x('Split {ratio}', ratio => $href->{'splitRatio'})
}
}
}
}
return $h;
}
#-----------------------------------------------------------------------------
# Download Data
#
# This uses the historical prices page like
#
# https://finance.yahoo.com/quote/AMP.AX/history?p=AMP.AX
#
# which puts a cookie like
#
# Set-Cookie: B=fab5sl9cqn2rd&b=3&s=i3; expires=Sun, 03-Sep-2018 04:56:13 GMT; path=/; domain=.yahoo.com
#
# and contains buried within 1.5 mbytes of hideous script
#
# <script type="application/json" data-sveltekit-fetched data-url="https://query1.finance.yahoo.com/v1/test/getcrumb?lang=en-US&region=US" data-ttl="59">{"status":200,"statusText":"OK","headers":{},"body":"DKVWQE/ggh4"}</script>
#
# Any \u002F or similar is escaped "/" character or similar.
# The crumb is included in a CSV download query like the following
# (alas can't use http, it redirects to https)
#
# https://query1.finance.yahoo.com/v7/finance/download/AMP.AX?period1=1503810440&period2=1504415240&interval=1d&events=history&crumb=hdDX/HGsZ0Q
#
# period1 is the start time, period2 the end time, both as Unix seconds
# since 1 Jan 1970. Not sure of the timezone needed. Some experiments
# suggest it depends on the timezone of the symbol. http works as well as
# https. The result is like
#
# Date,Open,High,Low,Close,Adj Close,Volume
# 2017-09-07,30.299999,30.379999,30.000000,30.170000,30.170000,3451099
#
# The "9999s" are some bad rounding off to what would be usually at most
# 3 (maybe 4?) decimal places.
#
# Response is 404 if no such symbol, 401 unauthorized if no cookie or crumb.
#
# "events=div" gives dividends like
#
# Date,Dividends
# 2017-08-11,0.161556
#
# "events=div" gives splits like, for a consolidation (GXY.AX)
#
# Date,Stock Splits
# 2017-05-22,1/5
#
#----------------
# For reference, there's a "v8" which is json format (%7C = "|")
#
# https://query2.finance.yahoo.com/v8/finance/chart/IBM?formatted=true&lang=en-US®ion=US&period1=1504028419&period2=1504428419&interval=1d&events=div%7Csplit&corsDomain=finance.yahoo.com
#
# This doesn't require a cookie and crumb, has some info like symbol
# timezone. The numbers look like they're rounded through 32-bit single
# precision floating point, for example "142.55999755859375" which is 142.55
# in a 23-bit mantissa. log(14255000)/log(2) = 23.76 bits
# Are they about the same precision as the CSV ?
#
# FIXME: All prices look like they're split-adjusted, which is ok if that's
# what you want and are downloading a full data set, but bad for incremental
# since you don't know when a change is applied.
#
App::Chart::DownloadHandler->new
(name => __('Yahoo'),
pred => $download_pred,
available_tdate_by_symbol => \&daily_available_tdate,
available_tdate_extra => 2,
url_and_cookiejar_func => \&daily_url_and_cookiejar,
proc => \&daily_download,
chunk_size => 150);
sub daily_available_tdate {
my ($symbol) = @_;
# Sep 2017: daily data is present for the current day's trade, during the
# trading session. Try reckoning it complete at 6pm.
return App::Chart::Download::tdate_today_after
(18,0, App::Chart::TZ->for_symbol ($symbol));
# return App::Chart::Download::tdate_today_after
# (10,30, App::Chart::TZ->for_symbol ($symbol))
# - 1;
}
sub daily_download {
my ($symbol_list) = @_;
App::Chart::Download::status (__('Yahoo daily data'));
# App::Chart::Download::verbose_message ("Yahoo crumb $crumb cookies\n"
# . $jar->as_string);
my $crumb_errors = 0;
SYMBOL: foreach my $symbol (@$symbol_list) {
my $lo_tdate = App::Chart::Download::start_tdate_for_update (@$symbol_list);
my $hi_tdate = daily_available_tdate ($symbol);
App::Chart::Download::status
(__('Yahoo data'), $symbol,
App::Chart::Download::tdate_range_string ($lo_tdate, $hi_tdate));
my $lo_timet = tdate_to_unix($lo_tdate - 2);
my $hi_timet = tdate_to_unix($hi_tdate + 2);
# my $data = cookie_and_crumb_data();
# if (! defined $data) {
# print "Yahoo $symbol no daily cookie data\n";
# next SYMBOL;
# }
# my $crumb = URI::Escape::uri_escape($data->{'crumb'});
# my $jar = http_cookies_from_string($data->{'cookies'} // '');
my $h = { source => __PACKAGE__,
prefer_decimals => 2,
date_format => 'ymd',
};
foreach my $elem (['history',\&daily_parse_v8],
# ['div', \&daily_parse_div],
# ['split', \&daily_parse_split]
) {
my ($events,$parse) = @$elem;
my $url = "https://query1.finance.yahoo.com/v7/finance/chart/"
. URI::Escape::uri_escape($symbol)
. "?period1=$lo_timet"
. "&period2=$hi_timet"
. "&interval=1d"
. "&events=$events"
# . "&crumb=$crumb"
;
my $resp = App::Chart::Download->get ($url,
allow_401 => 1,
allow_404 => 1,
# cookie_jar => $jar,
);
if ($resp->code == 401) {
App::Chart::Download::verbose_message ($resp->as_string . "\n");
if (++$crumb_errors >= 0) {
die "Yahoo: crumb authorization failed";
}
App::Chart::Database->write_extra ('', 'yahoo-daily-cookies', undef);
redo SYMBOL;
}
if ($resp->code == 404) {
print "Yahoo $symbol does not exist\n";
next SYMBOL;
}
$parse->($symbol,$resp,$h, $hi_tdate);
}
### $h
App::Chart::Download::write_daily_group ($h);
}
}
sub daily_parse_v8 {
my ($symbol, $resp) = @_;
my $hi_tdate = daily_available_tdate ($symbol);
my @data = ();
my $h = { source => __PACKAGE__,
prefer_decimals => 2, # default
date_format => 'ymd',
data => \@data,
};
my $content = $resp->decoded_content (raise_error => 1);
require JSON;
my $json = JSON::from_json($content);
my $result = $json->{'chart'}->{'result'}->[0];
my $meta = $result->{'meta'} // {};
my $meta_symbol = $meta->{'symbol'} // '';
if ($meta_symbol ne $symbol) {
die "Yahoo JSON oops, symbol wanted $symbol got $meta_symbol";
}
# Trading in pence Sterling is "GBp", such as TSCO.L
$h->{'currencies'}->{$symbol} = $meta->{'currency'};
$h->{'exchanges'}->{$symbol} = $meta->{'exchangeName'};
if (defined(my $decimals = $meta->{'priceHint'})) {
$h->{'prefer_decimals'} = $decimals;
}
my $gmtoffset = $meta->{'gmtoffset'};
my $timet_to_iso = sub {
my ($t) = @_;
return (defined $t ? POSIX::strftime('%Y-%m-%d', gmtime($t + $gmtoffset)) : undef);
};
my $timestamps = $result->{'timestamp'};
my $quote= $result->{'indicators'}->{'quote'}->[0];
my $opens = $quote->{'open'} // [];
my $highs = $quote->{'high'} // [];
my $lows = $quote->{'low'} // [];
my $closes = $quote->{'close'} // [];
my $volumes = $quote->{'volume'} // [];
foreach my $i (0 .. $#$timestamps) {
my $date = $timet_to_iso->($timestamps->[$i]);
if (App::Chart::Download::iso_to_tdate_floor($date) > $hi_tdate) {
# Current day's trading shows in the data.
# Don't enter it in the database until close of trade.
next;
}
push @data, { symbol => $symbol,
date => $date,
open => crunch_trailing_nines($opens->[$i]),
( run in 0.515 second using v1.01-cache-2.11-cpan-8f98c5d2c55 )