App-Chart
view release on metacpan or search on metacpan
unused/Yahoo-v7.pm view on Meta::CPAN
#
# This uses the info pages like
#
# https://query2.finance.yahoo.com/v1/finance/search?q=CSCO&enableFuzzyQuery=false
#
# which is a JSON format of various company information etc.
#
# The JSON looks like
# {"explains":[],
# "count":7,
# "quotes":[{"exchange":"NMS",
# "shortname":"Cisco Systems, Inc.",
# "quoteType":"EQUITY",
# "symbol":"CSCO",
# ...
#
# There can be multiple exchanges in the quotes list, use the first.
# Other fields include
#
# longname occasionally shorter than shortname actually
# but use shortname
#
# exchDisp longer display name for the exchange,
# eg. exchange=NMS and exchDisp=NASDAQ.
# or exchange=ASX and exchDisp=Australian.
# Think the exchange code more useful.
#
# The URL has an optional &newsCount=0 for no news events, but think
# that's the default anyway.
# FIXME: Disabled temporarily.
# App::Chart::DownloadHandler->new
# (name => __('Yahoo info'),
# key => 'Yahoo-info',
# pred => $download_pred,
# proc => \&info_download,
# recheck_days => 14);
sub info_url {
my ($symbol) = @_;
return 'https://query2.finance.yahoo.com/v1/finance/search?q='
. URI::Escape::uri_escape ($symbol)
. '&enableFuzzyQuery=false';
}
sub info_download {
my ($symbol_list) = @_;
foreach my $symbol (@$symbol_list) {
App::Chart::Download::status(__('Yahoo info'), $symbol);
my $url = info_url($symbol);
my $resp = App::Chart::Download->get ($url);
my $h = info_parse($resp);
$h->{'recheck_list'} = [ $symbol ];
App::Chart::Download::write_daily_group ($h);
}
}
sub info_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (raise_error => 1);
my @info;
my $h = { source => __PACKAGE__,
info => \@info };
require JSON;
my $J = JSON::from_json($content) // {};
my $quotes = $J->{'quotes'} // [];
if (my $e = $quotes->[0]) {
my $symbol = $e->{'symbol'};
my $name = $e->{'shortname'};
if (defined $name) {
# ASX shares have the symbol repeated at the end of shortname,
# like "BHP FPO [BHP]". Seems unnecessary, so strip that.
my $end = '[' . App::Chart::symbol_sans_suffix ($symbol) .']';
$name =~ s/\s*\Q$end\E$//;
}
push @info, { symbol => $symbol,
name => $name,
exchange => $e->{'exchange'},
};
}
return $h;
}
#------------------------------------------------------------------------------
# Latest
#
# This uses for example
#
# https://query1.finance.yahoo.com/v7/finance/chart/BHP.AX?period1=1718841600&period2=1719532800&interval=1d&events=history&close=unadjusted
#
# periodi1 and period2 are Unix style seconds since 1 Jan 1970 GMT.
#
# https://stackoverflow.com/questions/47076404/currency-helper-of-yahoo-sorry-unable-to-process-request-at-this-time-erro
# ->
# https://stackoverflow.com/questions/47064776/has-yahoo-suddenly-today-terminated-its-finance-download-api
App::Chart::LatestHandler->new
(pred => $latest_pred,
proc => \&latest_download,
max_symbols => 1); # downloads go 1 at a time
sub latest_download {
my ($symbol_list) = @_;
foreach my $symbol (@$symbol_list) {
my $tdate = daily_available_tdate ($symbol);
App::Chart::Download::status(__('Yahoo quote'), $symbol);
my $lo_timet = tdate_to_unix($tdate - 4);
my $hi_timet = tdate_to_unix($tdate + 2);
my $events = 'history';
my $url = "https://query1.finance.yahoo.com/v7/finance/chart/"
. URI::Escape::uri_escape($symbol)
."?period1=$lo_timet"
."&period2=$hi_timet"
."&interval=1d"
."&events=$events"
."&close=unadjusted";
# unknown symbol is 404 with json error details
#
my $resp = App::Chart::Download->get ($url, allow_404 => 1,);
App::Chart::Download::write_latest_group
(latest_parse($symbol,$resp,$tdate));
}
}
sub latest_parse {
my ($symbol, $resp, $tdate) = @_;
my $content = $resp->decoded_content (raise_error => 1);
require JSON;
my $json = JSON::from_json($content);
my %record = (symbol => $symbol,
);
my $h = { source => __PACKAGE__,
resp => $resp,
prefer_decimals => 2,
date_format => 'ymd',
data => [ \%record ],
};
if (defined (my $error = $json->{'chart'}->{'error'}->{'code'})) {
$record{'error'} = $error;
}
if (my $result = $json->{'chart'}->{'result'}->[0]) {
my $meta = $result->{'meta'}
// die "Yahoo JSON oops, no meta";
$record{'currency'} = $meta->{'currency'},
$record{'exchange'} = $meta->{'exchangeName'},
my $symbol_timezone = App::Chart::TZ->for_symbol ($symbol);
my $timestamps = $result->{'timestamp'}
// die "Yahoo JSON oops, no timestamp";
if (@$timestamps) {
# timestamps are time of last trade, as can be seen by looking at
# something with low enough volume, eg. RMX.AX
#
if (defined (my $timet = $timestamps->[-1])) {
($record{'last_date'}, $record{'last_time'})
= $symbol_timezone->iso_date_time($timet);
}
if (my $indicators = $result->{'indicators'}->{'quote'}->[0]) {
foreach my $key ('open','high','low') {
if (my $aref = $indicators->{$key}) {
$record{$key} = crunch_trailing_nines($aref->[$#$timestamps]);
}
}
if (my $aref = $indicators->{'volume'}) {
$record{'volume'} = $aref->[$#$timestamps];
}
if (my $aref = $indicators->{'close'}) {
my $last = $record{'last'}
= crunch_trailing_nines($aref->[$#$timestamps]);
# "change" from second last timestamp, if there is one.
# As of Nov 2017, XAUUSD=X only ever gives a single latest
# quote from v7, no previous day to compare.
#
if (defined $last
&& scalar(@$timestamps) >= 2
&& defined(my $prev = $aref->[$#$timestamps - 1])) {
$record{'change'}
= App::Chart::decimal_sub($last, crunch_trailing_nines($prev));
}
}
}
unused/Yahoo-v7.pm view on Meta::CPAN
# my $data = cookie_and_crumb_data();
# if (! defined $data) {
# print "Yahoo $symbol no daily cookie data\n";
# next SYMBOL;
# }
# my $crumb = URI::Escape::uri_escape($data->{'crumb'});
# my $jar = http_cookies_from_string($data->{'cookies'} // '');
my $h = { source => __PACKAGE__,
prefer_decimals => 2,
date_format => 'ymd',
};
foreach my $elem (['history',\&daily_parse_v8],
# ['div', \&daily_parse_div],
# ['split', \&daily_parse_split]
) {
my ($events,$parse) = @$elem;
my $url = "https://query1.finance.yahoo.com/v7/finance/chart/"
. URI::Escape::uri_escape($symbol)
. "?period1=$lo_timet"
. "&period2=$hi_timet"
. "&interval=1d"
. "&events=$events"
# . "&crumb=$crumb"
;
my $resp = App::Chart::Download->get ($url,
allow_401 => 1,
allow_404 => 1,
# cookie_jar => $jar,
);
if ($resp->code == 401) {
App::Chart::Download::verbose_message ($resp->as_string . "\n");
if (++$crumb_errors >= 0) {
die "Yahoo: crumb authorization failed";
}
App::Chart::Database->write_extra ('', 'yahoo-daily-cookies', undef);
redo SYMBOL;
}
if ($resp->code == 404) {
print "Yahoo $symbol does not exist\n";
next SYMBOL;
}
$parse->($symbol,$resp,$h, $hi_tdate);
}
### $h
App::Chart::Download::write_daily_group ($h);
}
}
sub daily_parse_v8 {
my ($symbol, $resp) = @_;
my $hi_tdate = daily_available_tdate ($symbol);
my @data = ();
my $h = { source => __PACKAGE__,
prefer_decimals => 2, # default
date_format => 'ymd',
data => \@data,
};
my $content = $resp->decoded_content (raise_error => 1);
require JSON;
my $json = JSON::from_json($content);
my $result = $json->{'chart'}->{'result'}->[0];
my $meta = $result->{'meta'} // {};
my $meta_symbol = $meta->{'symbol'} // '';
if ($meta_symbol ne $symbol) {
die "Yahoo JSON oops, symbol wanted $symbol got $meta_symbol";
}
# Trading in pence Sterling is "GBp", such as TSCO.L
$h->{'currencies'}->{$symbol} = $meta->{'currency'};
$h->{'exchanges'}->{$symbol} = $meta->{'exchangeName'};
if (defined(my $decimals = $meta->{'priceHint'})) {
$h->{'prefer_decimals'} = $decimals;
}
my $gmtoffset = $meta->{'gmtoffset'};
my $timet_to_iso = sub {
my ($t) = @_;
return (defined $t ? POSIX::strftime('%Y-%m-%d', gmtime($t + $gmtoffset)) : undef);
};
my $timestamps = $result->{'timestamp'};
my $quote= $result->{'indicators'}->{'quote'}->[0];
my $opens = $quote->{'open'} // [];
my $highs = $quote->{'high'} // [];
my $lows = $quote->{'low'} // [];
my $closes = $quote->{'close'} // [];
my $volumes = $quote->{'volume'} // [];
foreach my $i (0 .. $#$timestamps) {
my $date = $timet_to_iso->($timestamps->[$i]);
if (App::Chart::Download::iso_to_tdate_floor($date) > $hi_tdate) {
# Current day's trading shows in the data.
# Don't enter it in the database until close of trade.
next;
}
push @data, { symbol => $symbol,
date => $date,
open => crunch_trailing_nines($opens->[$i]),
high => crunch_trailing_nines($highs->[$i]),
low => crunch_trailing_nines($lows->[$i]),
close => crunch_trailing_nines($closes->[$i]),
volume => $volumes->[$i] };
}
my $events = $result->{'events'} // {};
### $events
# Eg. BHP.AX
# date 1709766000
# amount 1.096196
#
foreach my $dividend (values %{$events->{'dividends'} // {}}) {
push @{$h->{'dividends'}},
{ symbol => $symbol,
ex_date => $timet_to_iso->($dividend->{'date'}),
amount => $dividend->{'amount'},
};
}
# eg. 1:30 split from NKLA.MX on 21 Jun 2024 is like
# date 1719322200
# numerator 1
# denominator 30
# splitRatio 1:30
# closing price (unadjusted) drops from 193.80 to 6.47
# new 30 shares = denominator
# older prices when displayed adjusted are factor
# old/new = numerator/denominator
#
foreach my $split (values %{$events->{'splits'} // {}}) {
push @{$h->{'splits'}},
{ symbol => $symbol,
date => $timet_to_iso->($split->{'date'}),
new => $split->{'denominator'},
old => $split->{'numerator'},
};
}
return $h;
}
sub daily_parse {
my ($symbol, $resp, $h, $hi_tdate) = @_;
my @data = ();
$h->{'data'} = \@data;
my $hi_tdate_iso;
if (defined $hi_tdate){ $hi_tdate_iso = App::Chart::tdate_to_iso($hi_tdate); }
my $body = $resp->decoded_content (raise_error => 1);
my @line_list = App::Chart::Download::split_lines($body);
unless ($line_list[0] =~ /^Date,Open,High,Low,Close,Adj Close,Volume/) {
die "Yahoo: unrecognised daily data headings: " . $line_list[0];
}
shift @line_list;
foreach my $line (@line_list) {
my ($date, $open, $high, $low, $close, $adj_volume, $volume)
= split (/,/, $line);
$date = daily_date_fixup ($symbol, $date);
if (defined $hi_tdate_iso && $date gt $hi_tdate_iso) {
# Sep 2017: There's a daily data record during the trading day, but
# want to write the database only at the end of trading.
### skip date after hi_tdate ...
next;
}
# Sep 2017 have seen "null,null,null,...", maybe for non-trading days
foreach my $field ($open, $high, $low, $close, $adj_volume, $volume) {
if ($field eq 'null') {
$field = undef;
}
}
if ($index_pred->match($symbol)) {
# In the past indexes which not calculated intraday had
# open==high==low==close and volume==0, eg. ^WIL5. Use the close
# alone in this case, with the effect of drawing line segments instead
# of OHLC or Candle figures with no range.
if (defined $open && defined $high && defined $low && defined $close
&& $open == $high && $high == $low && $low == $close && $volume == 0){
$open = undef;
$high = undef;
$low = undef;
}
} else {
# In the past shares with no trades had volume==0,
# open==low==close==bid price, and high==offer price, from some time
# during the day, maybe the end of day. Zap all the prices in this
# case.
#
# For a public holiday it might be good to zap the volume to undef
# too, but don't have anything to distinguish holiday, suspension,
# delisting vs just no trades.
#
# On the ASX when shares are suspended the bid/offer can be crossed as
# usual for pre-open auction, and this gives high<low. For a part-day
# suspension then can have volume!=0 in this case too. Don't want to
# show a high<low, so massage high/low to open/close range if the high
# looks like a crossed offer.
if (defined $high && defined $low && $high < $low) {
$high = App::Chart::max_maybe ($open, $close);
$low = App::Chart::min_maybe ($open, $close);
}
if (defined $open && defined $low && defined $close && defined $volume
&& $open == $low && $low == $close && $volume == 0) {
$open = undef;
$high = undef;
$low = undef;
$close = undef;
}
}
push @data, { symbol => $symbol,
date => $date,
open => crunch_trailing_nines($open),
high => crunch_trailing_nines($high),
low => crunch_trailing_nines($low),
close => crunch_trailing_nines($close),
volume => $volume };
}
return $h;
}
sub daily_parse_div {
my ($symbol, $resp, $h) = @_;
my @dividends = ();
$h->{'dividends'} = \@dividends;
my $body = $resp->decoded_content (raise_error => 1);
my @line_list = App::Chart::Download::split_lines($body);
# Date,Dividends
# 2015-11-04,1.4143
# 2016-05-17,1.41428
# 2017-05-16,1.4143
# 2016-11-03,1.4143
unless ($line_list[0] =~ /^Date,Dividends/) {
warn "Yahoo: unrecognised dividend headings: " . $line_list[0];
return;
}
shift @line_list;
foreach my $line (@line_list) {
my ($date, $amount) = split (/,/, $line);
push @dividends, { symbol => $symbol,
ex_date => daily_date_fixup ($symbol, $date),
amount => $amount };
}
return $h;
}
sub daily_parse_split {
my ($symbol, $resp, $h) = @_;
my @splits = ();
$h->{'splits'} = \@splits;
my $body = $resp->decoded_content (raise_error => 1);
my @line_list = App::Chart::Download::split_lines($body);
# For example GXY.AX split so $10 shares become $2
# Date,Stock Splits
# 2017-05-22,1:5
#
# In the past it was a "/" instad of ":"
# 2017-05-22,1/5
unless ($line_list[0] =~ /^Date,Stock Splits/) {
warn "Yahoo: unrecognised split headings: " . $line_list[0];
return;
}
shift @line_list;
foreach my $line (@line_list) {
my ($date, $ratio) = split (/,/, $line);
my ($old, $new) = split m{[:/]}, $ratio;
push @splits, { symbol => $symbol,
date => daily_date_fixup ($symbol, $date),
new => $new,
old => $old };
}
return $h;
}
# ENHANCE-ME: The digits are 23-bit float formatted badly, or so it seems.
# That makes it 23 / (log(10)/log(2)) = 6.92 many high digits good.
# Maybe round-to-nearest of high 7 digits, or maybe only high 6 depending
# where the bit range falls (how many bits used by the high digit).
#
# $str is a string like "30.299999"
# Return it with trailing 9s turned into trailing 0s.
sub crunch_trailing_nines {
my ($str) = @_;
if (defined $str) {
$str =~ s/(\....(99|00)).*/$1/; # trailing garbage
if ($str =~ /(.*)\.(....9+)$/) {
$str = decimal_add_low($str,1);
} elsif ($str =~ /(.*)\.(....*01)$/) {
$str = decimal_add_low($str,-1);
}
if ($str =~ /(.*)\./) {
my $ilen = length($1);
my $decimals = ($ilen >= 4 ? 2
: $ilen == 3 ? 3
: 4);
$str = round_decimals($str,$decimals);
}
$str = pad_decimals($str, 2);
}
return $str;
}
sub decimal_add_low {
my ($str, $add) = @_;
### decimal_add_low(): "$str add $add"
unused/Yahoo-v7.pm view on Meta::CPAN
# which is wanted.
#
# As of April 2024, some User-Agent strings result in 503 Service Unavailable.
# Doesn't seem to affect other download parts, just this cookie/crumb
# getting part. "Mozilla/5.0" works.
#
sub cookie_and_crumb_data {
my ($symbol) = @_;
my $key = 'yahoo-cookie-and-crumb';
my $str = App::Chart::Database->read_extra ('', $key);
my $h = eval ($str || '{}');
$h = {};
unless (App::Chart::Download::timestamp_within
($h->{'timestamp'}, 3 * 86400)) {
App::Chart::Download::status ('Yahoo cookie');
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
my $user_agent = 'Mozilla/5.0';
{
# first to get a cookie
my $resp = App::Chart::Download->get
('https://www.yahoo.com/',
user_agent => $user_agent,
cookie_jar => $jar);
### jar: $jar->as_string
App::Chart::Download::verbose_message ("Yahoo cookies: "
. $jar->as_string);
}
App::Chart::Download::status ('Yahoo auth crumb');
my $resp = App::Chart::Download->get
('https://finance.yahoo.com/quote/IBM/history/?p=IBM',
user_agent => $user_agent,
cookie_jar => $jar);
my $crumb = crumb_parse($resp);
App::Chart::Download::verbose_message ("Yahoo crumb: $crumb");
my $cookies_str = $jar->as_string;
$h = { crumb => $crumb,
cookies => $cookies_str,
timestamp => App::Chart::Download::timestamp_now(),
};
my $str = Data_Dumper_str($h);
App::Chart::Database->write_extra ('', $key, $str);
}
return $h;
}
sub crumb_parse {
my ($resp) = @_;
# script like, with backslash escaping on "\uXXXX"
# "user":{"age":0,"crumb":"8OyCBPyO4ZS"
# The form prior to about July 2023 was
# "user":{"crumb":"hdDX\u002FHGsZ0Q",
# The form prior to about January 2023 was
# "CrumbStore":{"crumb":"hdDX\u002FHGsZ0Q"}
# The form prior to about May 2024 was
# "RequestPlugin":{"user":{"age":0,"crumb":"8OyCBPyO4ZS"
#
my $content = $resp->decoded_content (raise_error => 1);
$content =~ /getcrumb.*?"body":"([^"]*)"/
or die "Yahoo getcrumb not found in parse";
return App::Chart::Yahoo::javascript_string_unquote($1);
}
sub Data_Dumper_str {
my ($h) = @_;
my $dumper = Data::Dumper->new ([$h], ['var']);
$dumper->Indent(1);
$dumper->Terse(1);
$dumper->Sortkeys(1);
return $dumper->Dump;
}
# $str is an ISO date string like 2017-11-05
# It is date GMT of 9:30am in the timezone of $symbol.
# Return the date in the symbol timezone.
#
sub daily_date_fixup {
my ($symbol, $str) = @_;
### daily_date_fixup: "$symbol $str"
my ($year, $month, $day) = App::Chart::iso_to_ymd ($str);
my $timezone = App::Chart::TZ->for_symbol($symbol);
if (timezone_gmtoffset_at_ymd($timezone, $year, $month, $day+1)
<= - (10*60+20)*60) {
my $adate = App::Chart::ymd_to_adate ($year, $month, $day);
$str = App::Chart::adate_to_iso ($adate+1);
my $today = $timezone->iso_date();
if ($str gt $today) {
$str = $today;
}
}
return $str;
}
sub timezone_gmtoffset_at_ymd {
my ($timezone, $year, $month, $day) = @_;
my $timet = $timezone->call(\&POSIX::mktime,
0, 0, 0, $day, $month-1, $year-1900);
my ($sec,$min,$hour,$gmt_day) = gmtime($timet);
return $sec + 60*$min + 3600*$hour + 86400*($gmt_day - $day);
}
# Return seconds since 00:00:00, 1 Jan 1970 GMT.
sub tdate_to_unix {
my ($tdate) = @_;
my $adate = App::Chart::tdate_to_adate ($tdate);
return ($adate + 4)*86400;
}
# $str is a string from previous HTTP::Cookies ->as_string()
# Return a new HTTP::Cookies object with that content.
sub http_cookies_from_string {
my ($str) = @_;
require File::Temp;
my $fh = File::Temp->new (TEMPLATE => 'chart-XXXXXX',
TMPDIR => 1);
print $fh "#LWP-Cookies-1.0\n", $str or die;
close $fh or die;
( run in 0.775 second using v1.01-cache-2.11-cpan-39bf76dae61 )