App-Chart
view release on metacpan or search on metacpan
devel/MON.pm view on Meta::CPAN
my ($line) = @_;
my @ret;
while ($line =~ m{<t[hd]>(.*?)</t[hd]>}g) {
push @ret, $1;
}
return @ret;
}
sub daily_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (raise_error => 1);
# lines like
# <tr><th>Date</th><th>Class Symbol</th>...
my @lines = split /\n/, $content;
@lines = grep /^<tr>/, @lines;
# 'InstrumentSymbol' includes month like CGBU17
my @headings;
{
lib/App/Chart/Barchart.pm view on Meta::CPAN
my $resp = App::Chart::Download->get ($url,
cookie_jar => cookie_jar(),
referer => $url);
my $h = ifutpage_parse ($commodity, $suffix, $resp);
App::Chart::Download::write_latest_group ($h);
}
sub ifutpage_parse {
my ($commodity, $suffix, $resp) = @_;
my $content = $resp->decoded_content (raise_error => 1);
my @data = ();
my $h = { source => __PACKAGE__,
resp => $resp,
front_month => 1,
data => \@data };
# eg. " <B>CRUDE OIL</B> Delayed Futures -20:10 - Sunday, 19 June"
# " <B>SIMEX NIKKEI 225</B> Delayed Futures -18:20 - Tuesday, 12 December</td>"
# " <B>OATS </B> Daily Futures - Friday, 20 April </td>
lib/App/Chart/Barchart.pm view on Meta::CPAN
sub fiveday_parse {
my ($symbol, $resp) = @_;
my @data;
my $h = { source => __PACKAGE__,
prefer_decimals => 2,
date_format => 'mdy',
data => \@data };
my $content = $resp->decoded_content (raise_error => 1);
# message in table when no info for given symbol (eg. an old month/year)
if ($content =~ /In order to form an extended quote/) {
return $h;
}
# eg. <h1 class="fl" id="symname">Crude Oil WTI December 2016 (CLZ16)</h1>
#
$content =~ m{<h1[^>]* id="symname">(([^<\r\n]*?) [a-z]+ [0-9]+)}si
or die "Barchart fiveday heading not matched";
lib/App/Chart/Barchart.pm view on Meta::CPAN
my $resp = App::Chart::Download->get ($url,
cookie_jar => $jar,
referer => $url);
return (intraday_resp_to_url ($resp, $symbol),
cookie_jar => $jar,
referer => $url);
}
# separate func for offline testing ...
sub intraday_resp_to_url {
my ($resp, $symbol) = @_;
my $content = $resp->decoded_content (raise_error => 1);
require HTML::LinkExtor;
my $parser = HTML::LinkExtor->new(undef, $resp->base);
$parser->parse($content);
$parser->eof;
# eg.
# </map><img src="/cache/bde71ebe23ddac66f2d25081b1b5f953.png"
# must match some of the link target name since there's other images in
lib/App/Chart/Download.pm view on Meta::CPAN
if ($resp->is_success
|| ($options{'allow_401'} && $code == 401)
|| ($options{'allow_404'} && $code == 404)
|| (($etag || $lastmod) && $code == 304)
|| http_code_is_allowed($options{'allow_http_codes'}, $code)) {
substatus (__('processing'));
return $resp;
} else {
my $error = $resp->status_line . "\n";
if ($code == 500) {
$error .= $resp->decoded_content;
unless ($error =~ /\n$/s) { $error .= "\n"; }
}
croak "Cannot download $url\n" . $error;
}
}
#------------------------------------------------------------------------------
my $last_status = ''; # without substatus addition
lib/App/Chart/Google.pm view on Meta::CPAN
}
sub daily_parse {
my ($symbol, $resp) = @_;
my @data = ();
my $h = { source => __PACKAGE__,
prefer_decimals => 2,
date_format => 'ymd', # eg. '3-Oct-08'
data => \@data };
my $body = $resp->decoded_content (raise_error => 1);
my @line_list = App::Chart::Download::split_lines($body);
if ($line_list[0] !~ /^Date,Open,High,Low,Close,Volume/i) {
die "Google: unrecognised daily data headings: " . $line_list[0];
}
shift @line_list;
foreach my $line (@line_list) {
my ($date, $open, $high, $low, $close, $volume)
= split (/,/, $line);
lib/App/Chart/IntradayHandler.pm view on Meta::CPAN
my $req = HTTP::Request->new ('GET', $url, \@headers);
$ua->prepare_request ($req);
### IntradayHandler request: $req->as_string
my $resp = $ua->request
($req,
sub {
my ($chunk, $resp, $protobj) = @_;
$resp->add_content($chunk);
# if the message is deflate/gzip/etc compressed then decoded_content
# returns undef until the whole image -- but don't worry about that
# until we get a server sending us compressed images
#
my $image = $resp->decoded_content(charset=>'none');
if ($image && length $image >= 256) {
App::Chart::Intraday::write_intraday_image (symbol => $symbol,
mode => $mode,
image => $image);
}
});
if ($resp->is_success) {
$image = $resp->decoded_content(charset=>'none',raise_error=>1);
} else {
$error = __x('Error: {status_line}',
status_line => $resp->status_line);
}
}
require App::Chart::Intraday;
App::Chart::Intraday::write_intraday_image (symbol => $symbol,
mode => $mode,
image => $image,
resp => $resp,
lib/App/Chart/Pagebits.pm view on Meta::CPAN
etag => $h->{'ETag'},
last_modified => $h->{'Last-Modified'},
allow_404 => $options{'allow_404'},
user_agent => $options{'user_agent'},
);
if ($options{'allow_404'} && $resp->code == 404) {
return undef;
}
if ($resp->is_success) {
my $content = $resp->decoded_content (raise_error=>1);
$h = $parse->($content, $resp);
$h->{'ETag'} = scalar $resp->header('ETag');
$h->{'Last-Modified'} = $resp->last_modified;
}
$h->{'timestamp'} = App::Chart::Download::timestamp_now();
my $dumper = Data::Dumper->new ([$h], ['var']);
$dumper->Indent(1);
$dumper->Terse(1);
$dumper->Sortkeys(1);
lib/App/Chart/Suffix/ATH.pm view on Meta::CPAN
symbol => $symbol));
my $url = 'http://www.ase.gr/content/en/marketdata/stocks/prices/Share_SearchResults.asp?share='
. URI::Escape::uri_escape (App::Chart::symbol_sans_suffix ($symbol));
my $resp = App::Chart::Download->get ($url);
App::Chart::Download::write_daily_group (last30_parse ($resp));
}
}
sub last30_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (raise_error => 1);
my @data = ();
my $h = { source => __PACKAGE__,
currency => 'EUR',
last_download => 1,
cost_key => 'athens-last30',
date_format => 'dmy',
resp => $resp,
data => \@data };
lib/App/Chart/Suffix/ATH.pm view on Meta::CPAN
App::Chart::DownloadHandler::DividendsPage->new
(name => __('ATHEX dividends'),
pred => $pred,
url => DIVIDENDS_URL,
parse => \÷nds_parse,
key => 'ATH-dividends');
sub dividends_parse {
my ($resp) = @_;
my $body = $resp->decoded_content (raise_error => 1);
my @dividends = ();
my $h = { source => __PACKAGE__,
resp => $resp,
date_format => 'dmy',
# amounts are like "0.360", trim to 2 decimals
prefer_decimals => 2,
dividends => \@dividends };
# "Price in €" reaches here as wide char \x{20AC}, probably, maybe,
lib/App/Chart/Suffix/BEN.pm view on Meta::CPAN
sub quote_date_time {
# (tm->adate-time-within (localtime (current-time) (timezone-bendigo))
# #,(hms->seconds 9 30 0)
# #,(hms->seconds 11 30 0)))
}
sub bendigo_quote {
sub quote_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (raise_error => 1);
my @data = ();
my $h = { source => __PACKAGE__,
resp => $resp,
date_format => 'dmy',
data => \@data };
require HTML::TableExtract;
my $te = HTML::TableExtract->new
lib/App/Chart/Suffix/CAS.pm view on Meta::CPAN
my @dividends = ();
my $h = { source => __PACKAGE__,
resp => $resp,
url_tags_key => 'CAS-dividends',
currency => 'MAD',
date_format => 'dmy', # eg. "26.05.00"
dividends => \@dividends };
if (! $resp->is_success) {
return $h;
}
my $content = $resp->decoded_content (raise_error=>1);
require HTML::TableExtract;
my $te = HTML::TableExtract->new
(headers => [ qr/Dividend.*Brut/is,
# qr/Num.*Coupon/is, # coupon number
qr/tachement.*la.*Bourse/is ]);
$te->parse($content);
if (! $te->tables) {
die 'Casablanca dividends table not matched';
}
lib/App/Chart/Suffix/GHA.pm view on Meta::CPAN
my $h = daily_parse ($resp);
App::Chart::Download::write_daily_group ($h);
}
App::Chart::Download::write_daily_group ($newest_h);
}
# return values ([$code,$tdate],...) which are the available sessions, by
# session code and tdate, sorted from oldest to newest tdate
sub daily_sessions {
my ($resp) = @_;
my $content = $resp->decoded_content(raise_error=>1);
my @ret = ();
# Eg. <option value="2425"> 18/01/2007 </option>
while ($content =~ m{<option +value=\"([0-9]+)\"> *([0-9]+/[0-9]+/[0-9][0-9][0-9][0-9]) *</option>}g) {
my $code = $1;
my $tdate = App::Chart::ymd_to_tdate_floor
(Date::Calc::Decode_Date_EU ($2)); # d/m/y
push @ret, [$code, $tdate];
}
return sort {$a->[1] <=> $b->[1]} @ret;
}
sub daily_parse {
my ($resp) = @_;
my $content = $resp->decoded_content(raise_error=>1);
my @data = ();
my $h = { source => __PACKAGE__,
url => GSE_DAILY_URL,
resp => $resp,
currency => 'GHC',
date_format => 'dmy',
data => \@data };
# Eg. "24/04/2006\n\t\t</font> Trading Results"
lib/App/Chart/Suffix/GHA.pm view on Meta::CPAN
# my ($symbol) = @_;
# return 'http://www.gse.com.gh/marketstat/officiallist_details.asp?Symbol='
# . URI::Escape::uri_escape (App::Chart::symbol_sans_suffix ($symbol));
# }
#
# sub name_parse {
# my ($symbol, $resp) = @_;
#
# # eg. " <td><strong> <span class="just">2241 <strong> Trading Session </strong></span>for:<font color="#D98713"> Accra Brewery Company Ltd.</font></strong></td>"
#
# my $body = $resp->decoded_content (raise_error => 1);
# $body =~ s/<[^>]*>//g; # no html
# $body =~ /Trading Session.*for:([^\n]*)/
# or die "GSE: company name not found for '$symbol'";
# my $name = App::Chart::collapse_whitespace ($1);
# return { name => $name };
# }
1;
__END__
lib/App/Chart/Suffix/LJ.pm view on Meta::CPAN
my @data = ();
my $h = { source => __PACKAGE__,
url_tags_key => 'LJ-BTStecajEUR',
resp => $resp,
currency => 'EUR',
date_format => 'dmy',
suffix => '.TSP',
data => \@data };
# charset not specified, but is codepage 1250
my $content = $resp->decoded_content (raise_error => 1,
default_charset => 'cp1250');
$content =~ s/\r//g;
# 0001 file format marker
# $content =~ /^ 0001 110 /
# or die 'Ljubljana: BTS file missing 0001 id line';
# my $date = txt_to_date ($content);
foreach my $line (split /\n+/, $content) {
lib/App/Chart/Suffix/LJ.pm view on Meta::CPAN
return sprintf 'http://www.ljse.si/cgi-bin/jve.cgi?doc=2137&date1=%02d.%02d.%04d&date2=%02d.%02d.%04d&IndexCode=%s&SecurityId=%s&csv=1',
$lo_day, $lo_month, $lo_year,
$hi_day, $hi_month, $hi_year,
($index ? $symbol : ''), # index symbol, or empty
($index ? 'X' : $symbol); # share symbol, or dummy
}
sub podatki_parse {
my ($symbol, $resp) = @_;
my $content = $resp->decoded_content (raise_error => 1);
my $index = ($symbol =~ /^\^/); # if an index symbol
my @data = ();
my $h = { source => __PACKAGE__,
currency => 'EUR',
date_format => 'mdy', # eg. 'Jul 28. 2006'
# last_download => 1,
data => \@data };
foreach my $line (split /\n/, $content) {
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
require App::Chart::UserAgent;
require HTTP::Cookies;
my $ua = App::Chart::UserAgent->instance->clone;
my $jar = HTTP::Cookies->new;
$ua->cookie_jar ($jar);
my $login_url = LOGIN_URL;
$login_url = 'http://localhost/Login.aspx';
my $resp = App::Chart::Download->get ($login_url, ua => $ua);
my $content = $resp->decoded_content(raise_error=>1);
my $form = HTML::Form->parse($content, $login_url)
or die "LME login page not a form";
# these are literal "$" in the field name
$form->value ("_logIn\$_userID", $username);
$form->value ("_logIn\$_password", $password);
my $req = $form->click();
$ua->requests_redirectable ([]);
$resp = $ua->request ($req);
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
#-----------------------------------------------------------------------------
# Daily price page parsing
sub daily_parse {
my ($resp, $want_tdate) = @_;
my @data = ();
my $h = { source => __PACKAGE__,
currency => 'USD',
data => \@data };
my $content = $resp->decoded_content (raise_error => 1);
$content = mung_1x1_tables ($content);
# Eg. "Official Prices, US$ per tonne for\n\t\t19 September 2008"
# Eg. "LME Official Prices, US$ per tonne for 18 September 2008"
#
$content =~ /Prices.*?for\s*\n?\s*([0-9]{1,2}\s+[A-Za-z]+\s+[0-9][0-9][0-9][0-9])/i
or die "LME daily: date not found";
my $date = App::Chart::Download::Decode_Date_EU_to_iso ($1);
if (defined $want_tdate) {
my $want_date = App::Chart::tdate_to_iso($want_tdate);
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
'Tin' => 'SN',
'Nickel' => 'NI',
'Far East' => 'FF', # steel
'Med' => 'FM', # steel
'Averages' => undef,
'Plastic Avg' => undef,
'Averages inc. Euro Eq' => undef);
sub monthxls_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (charset => 'none', raise_error => 1);
require Spreadsheet::ParseExcel;
require Spreadsheet::ParseExcel::Utility;
my @data = ();
my $h = { source => __PACKAGE__,
cover_pred => $pred,
data => \@data };
my $excel = Spreadsheet::ParseExcel::Workbook->Parse (\$content);
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
App::Chart::Download::status (__x('LME volumes {filename}',
filename => $filename));
my $resp = App::Chart::Download->get ($url);
my $h = volume_parse ($resp);
App::Chart::Download::write_daily_group ($h);
}
}
sub volume_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (charset => 'none', raise_error => 1);
require Spreadsheet::ParseExcel;
require Spreadsheet::ParseExcel::Utility;
my @data = ();
my $h = { source => __PACKAGE__,
data => \@data };
my $excel = Spreadsheet::ParseExcel::Workbook->Parse (\$content);
my $sheet = $excel->Worksheet (0);
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
type => $type),
url => $type_to_daily_url{$type},
key => "lme-daily-latest-$type",
freq_days => 0,
timezone => App::Chart::TZ->london,
parse => \&daily_latest_parse);
}
sub daily_latest_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (raise_error => 1);
my $h = daily_parse ($resp);
return { h => $h,
date => $h->{'data'}->[0]->{'date'},
url => $resp->uri->as_string,
content => $content };
}
1;
__END__
lib/App/Chart/Suffix/MGEX.pm view on Meta::CPAN
. URI::Escape::uri_escape (App::Chart::symbol_sans_suffix ($symbol))
. $intraday_mode_to_data{$mode};
App::Chart::Download::verbose_message ("Intraday page", $url);
my $resp = App::Chart::Download->get ($url);
return barchart_customer_resp_to_url ($resp, $symbol);
}
# separate func for offline testing ...
sub barchart_customer_resp_to_url {
my ($resp, $symbol) = @_;
my $content = $resp->decoded_content (raise_error => 1);
# This doesn't use java_document_write() (in Finance::Quote::MGEX) and
# HTML::LinkExtor because the width/height in the document.write() in the
# <img> bit is computed, which java_document_write() can't handle. The
# src="" part of it is constant though. There's only a single <img> to
# pick out.
if ($content =~ /<img src="([^"]+)"/i) {
return $1; # url
}
if ($content =~ /no chart available/i) {
lib/App/Chart/Suffix/MLC.pm view on Meta::CPAN
URI::Escape::uri_escape ($fund),
$lo_day, $lo_month, $lo_year,
$hi_day, $hi_month, $hi_year);
}
# Lines like:
# historicalProduct1funds[1]="MLC Property Securities Fund,MasterKey Superannuation (Gold Star),29 March 2007,64.71567,0.00000";
#
sub parse {
my ($symbol, $resp) = @_;
my $content = $resp->decoded_content(raise_error=>1);
my @data = ();
my $h = { source => __PACKAGE__,
resp => $resp,
currency => 'AUD',
data => \@data };
while ($content =~ /^historicalProduct1funds.*=\"(.*)\"/mg) {
### $&
my ($fund, $product, $date, $price) = split /,/, $1;
lib/App/Chart/Suffix/NZ.pm view on Meta::CPAN
my $h = { source => __PACKAGE__,
resp => $resp,
dividends => \@dividends,
copyright_key => 'NZ-dividends-copyright',
copyright => 'https://www.nzx.com/meta-pages/terms-of-use',
date_format => 'ymd', # ISO 2024-09-09
prefer_decimals => 2,
};
# note: want wide-chars for HTML::TableExtract parse
my $content = $resp->decoded_content (raise_error => 1);
$content =~ /<script id="__NEXT_DATA__".*?>(.*?)<\/script>/i
or die "NZX dividends cannot find JSON table";
my $str = $1;
my $json = JSON::from_json($1) // {};
my $props = $json->{'props'} // {};
my $pageProps = $props->{'pageProps'} // {};
my $data = $pageProps->{'data'} // {};
# marketInstruments array of hashrefs like
lib/App/Chart/Suffix/RBA.pm view on Meta::CPAN
sub threeday_parse {
my ($resp) = @_;
my @data = ();
my $h = { url => RBA_EXCHANGE_URL,
copyright => RBA_COPYRIGHT_URL,
source => __PACKAGE__,
resp => $resp,
cover_pred => $pred,
data => \@data };
my $content = $resp->decoded_content(raise_error=>1);
# mung <tr id="USD"> to add <td>USD</td> so it appears in the TableExtract
$content =~ s{<tr>}{<tr><td></td>}ig;
$content =~ s{(<tr +id="([^"]*)">)}{$1<td>$2</td>}ig;
require HTML::TableExtract;
my $te = HTML::TableExtract->new
(
# is now a <caption>
# headers => ['Units of foreign currency per'],
lib/App/Chart/Suffix/RBA.pm view on Meta::CPAN
'SARD' => 'ZAR', # South African rand
'SARY' => 'SAR', # Saudi riyal
'SF' => 'CHF', # Swiss franc
'SK' => 'SEK', # Swedish krona
# 'SDR' # special drawing right
);
sub monthly_parse {
my ($resp, $stop_iso) = @_;
### RBA monthly_parse() ...
my $content = $resp->decoded_content(raise_error=>1);
my @data = ();
my $h = { source => __PACKAGE__,
copyright => RBA_COPYRIGHT_URL,
data => \@data };
require Spreadsheet::ParseExcel;
require Spreadsheet::ParseExcel::Utility;
my $excel = Spreadsheet::ParseExcel::Workbook->Parse (\$content);
lib/App/Chart/Suffix/RBA.pm view on Meta::CPAN
#
# https://www.rba.gov.au/Statistics/HistoricalExchangeRates/2003to2007.xls
#
# The files aren't huge (500k upwards) but there's a lot of cells, which
# makes Spreadsheet::ParseExcel fairly slow and eat up about 50Mb of core
# (as of version 0.32), even before getting to the $h data build.
sub xls_parse {
my ($resp) = @_;
### RBA xls_parse() ...
my $content = $resp->decoded_content(raise_error=>1);
my @data = ();
my $h = { source => __PACKAGE__,
copyright => RBA_COPYRIGHT_URL,
data => \@data };
require Spreadsheet::ParseExcel;
require Spreadsheet::ParseExcel::Utility;
my $excel = Spreadsheet::ParseExcel::Workbook->Parse (\$content);
lib/App/Chart/Suffix/RBA.pm view on Meta::CPAN
#
# exchange page
# AED CAD CHF CNY EUR GBP HKD IDR INR JPY KRW MYR NZD PGK PHP SDR SGD THB TWD TWI USD VND
use constant RBA_CURRENT_CSV_URL =>
'https://www.rba.gov.au/statistics/tables/csv/f11.1-data.csv';
sub csv_parse {
my ($resp) = @_;
### RBA csv_parse() ...
my $content = $resp->decoded_content(raise_error=>1);
my @data = ();
my $h = { source => __PACKAGE__,
copyright => RBA_COPYRIGHT_URL,
date_format => 'dmy',
data => \@data };
my @currencies;
foreach my $line (split /\n/, $content) {
my @fields = split /,\s*/, $line;
lib/App/Chart/Suffix/TGE.pm view on Meta::CPAN
filename => $filename));
my $resp = App::Chart::Download->get ($url,
url_tags_key => 'TGE-day');
if (! $resp->is_success) {
# not modified, no new data
return 1;
}
my $got_tdate = csv_tdate ($resp);
if ($got_tdate == $start_tdate) {
# got the expected data, process it
my $content = $resp->decoded_content (charset => 'none');
my $h = csv_parse ($content);
$h->{'url_tags_key'} = 'TGE-day';
return 1;
} elsif ($got_tdate < $avail_tdate) {
# got something older, there's no new data
return 1;
} else {
return 0;
}
}
lib/App/Chart/Suffix/TGE.pm view on Meta::CPAN
($end_year, $end_month, $end_day, -1, 0, 1);
return sprintf 'http://www.tge.or.jp/data/down_load/%s01%02d%02d%02d%02d%02d%02d.zip',
lc($commodity),
$start_year % 100, $start_month, $start_day,
$end_year % 100, $end_month, $end_day;
}
sub zip_parse {
my ($resp) = @_;
my $zipstr = $resp->decoded_content (charset => 'none', raise_error => 1);
my $h;
require Archive::Zip;
require IO::String;
my $zip = Archive::Zip->new;
my $io = IO::String->new ($zipstr);
$zip->readFromFileHandle ($io);
foreach my $member ($zip->members) {
my $csv = $member->contents;
lib/App/Chart/Suffix/TSP.pm view on Meta::CPAN
# zeros. Could prefer_decimals to strip them, but may as well leave at
# current precision.
#
my @data = ();
my $h = { source => __PACKAGE__,
currency => 'USD',
date_format => 'ymd',
suffix => '.TSP',
data => \@data };
my $content = $resp->decoded_content (raise_error=>1);
### $content
my @lines = App::Chart::Download::split_lines ($content);
my ($date, @symbols) = split /,/, $lines[0];
lc($date) eq 'date'
or die 'TSP csv doesn\'t start with "date": ', $lines[0];
shift @lines;
### @symbols
# "L Income" becomes uppercase "LINCOME.TSP" (documented in chart.texi
lib/App/Chart/Suffix/TSP.pm view on Meta::CPAN
# # zeros. Could prefer_decimals to strip them, but may as well leave at
# # current precision.
# #
# my @data = ();
# my $h = { source => __PACKAGE__,
# currency => 'USD',
# date_format => 'mdy',
# suffix => '.TSP',
# data => \@data };
#
# my $content = $resp->decoded_content (raise_error=>1);
# ### $content
#
# require HTML::TableExtract;
# my $te = HTML::TableExtract->new (headers => [qr/^Date$/i],
# keep_headers => 1,
# slice_columns => 0);
# $te->parse($content);
# my $ts = $te->first_table_found();
# #### $ts
# if (! $ts) { die 'TSP price table not found'; }
lib/App/Chart/UserAgent.pm view on Meta::CPAN
} elsif (defined $options{'agent'} && $options{'agent'} =~ / $/) {
# ends in space, append our default
$options{'agent'} .= $class->_agent;
}
my $self = $class->SUPER::new (timeout => 60, # seconds, default
%options);
# with lwp bit appended
# $self->agent ();
# ask for everything decoded_content() accepts
$self->default_header ('Accept-Encoding' => scalar HTTP::Message::decodable());
# trace on redirect ...
return $self;
}
sub _agent {
my ($class) = @_;
return "Chart/$App::Chart::VERSION " . $class->SUPER::_agent;
lib/App/Chart/UserAgent.pm view on Meta::CPAN
} elsif ($App::Chart::option{'verbose'} >= 1) {
print $o->method," ",$o->url,"\n";
}
}
if ($phase eq 'response_done') {
if ( $App::Chart::option{'verbose'} >= 1) {
print $o->status_line,"\n";
}
if ( $App::Chart::option{'verbose'} >= 2) {
print $o->headers->as_string,"\n";
my $content = $o->decoded_content;
if (length $content <= 1000) {
print "content:\n", $content, "\n\n";
}
}
}
return $self->SUPER::run_handlers ($phase, $o);
}
# use Data::Dumper;
# *LWP::Protocol::http::SocketMethods::configure = sub {
lib/App/Chart/UserAgent.pm view on Meta::CPAN
Connection caching, currently just 1 kept open at any time.
=item *
C<User-Agent> identification header.
=item *
C<Accept-Encoding> header with C<HTTP::Message::decodable()> to let the
server send gzip etc. This means all responses should be accessed with
C<< $resp->decoded_content() >>, not raw C<content()>.
=item *
Progress and redirection messages (back through C<App::Chart::Download>).
=back
=head1 FUNCTIONS
=over 4
lib/App/Chart/Yahoo.pm view on Meta::CPAN
. URI::Escape::uri_escape ($symbol)
. '&enableFuzzyQuery=false';
}
sub info_parse {
my ($symbol, $resp) = @_;
my @info;
my $h = { source => __PACKAGE__,
info => \@info };
my $content = $resp->decoded_content (raise_error => 1);
my $json = JSON::decode_json($content) // {};
my $quotes = $json->{'quotes'} // [];
my $e = $quotes->[0] // {};
# Should have symbol in the data equal to $symbol requested.
# May want to be relaxed about that, as the Yahoo server allows
# different upper/lower case.
#
# my $quotes_symbol = $e->{'symbol'} // '';
# if ($quotes_symbol ne $symbol) {
lib/App/Chart/Yahoo.pm view on Meta::CPAN
# unknown symbol is 404 with JSON error details
#
my $resp = App::Chart::Download->get ($url, allow_404 => 1,);
App::Chart::Download::write_latest_group
(latest_parse($symbol,$resp,$tdate));
}
}
sub latest_parse {
my ($symbol, $resp, $tdate) = @_;
my $content = $resp->decoded_content (raise_error => 1);
my $json = JSON::from_json($content);
my %record = (symbol => $symbol,
);
my $h = { source => __PACKAGE__,
resp => $resp,
prefer_decimals => 2,
date_format => 'ymd',
data => [ \%record ],
};
lib/App/Chart/Yahoo.pm view on Meta::CPAN
sub daily_parse {
my ($symbol, $resp) = @_;
my $hi_tdate = daily_available_tdate ($symbol);
my @data = ();
my $h = { source => __PACKAGE__,
prefer_decimals => 2, # default
date_format => 'ymd',
data => \@data,
};
my $content = $resp->decoded_content (raise_error => 1);
my $json = JSON::decode_json($content);
my $result = $json->{'chart'}->{'result'}->[0];
my $meta = $result->{'meta'} // {};
# Should have symbol in the data equal to $symbol requested.
# May want to be relaxed about that, as the Yahoo server allows
# different upper/lower case. The intention is to have all
# symbols in the database as a canonical form.
#
maybe/Build-PL view on Meta::CPAN
'HTML::TreeBuilder' => 0,
'IO::String' => 0,
'List::MoreUtils' => 0,
# need 1.16 for bind_textdomain_filter() to mung gettext
# strings
'Locale::TextDomain' => '1.16',
# dunno what version actually needed, at least 5.803 for a
# working "decoded_content".
LWP => '5.803',
'Math::Round' => 0,
'Module::Load' => 0,
# need 1.60 for fixups to initializing in locales like
# de_DE with "." as the monetary thousands sep
'Number::Format' => '1.60',
'Perl6::Slurp' => 0,
misc/t-float.pl view on Meta::CPAN
use Data::Dumper;
use File::Slurp 'slurp';
use App::Chart::Download;
use App::Chart::Float;
{
my $resp = HTTP::Response->new(200,'OK');
my $content = slurp (<~/chart/samples/float/CML.zip>);
$resp->content($content);
die if ($resp->decoded_content(charset=>'none') ne $content);
my $h = App::Chart::Float::zip_parse ('CML.zip', $resp, 'float-indiv-zip');
# print Dumper ($h);
App::Chart::Download::write_daily_group ($h);
exit 0;
}
{
require App::Chart::Gtk2::Symlist::All;
require App::Chart::Gtk2::Symlist::Glob;
require App::Chart::DownloadCost;
misc/t-http.pl view on Meta::CPAN
my $file = File::Slurp::slurp ('foo.gz');
print length($file),"\n";
my $chunk = substr ($file, 0, length($file)/2);
my $resp = HTTP::Message->new;
$resp->add_content($chunk);
$resp->push_header('Content-Encoding' => 'gzip');
print $resp->headers->as_string;
my $image = $resp->decoded_content(charset=>'none');
print length ($image);
misc/t-tsp.pl view on Meta::CPAN
{
my $url;
$url = 'https://www.tsp.gov/data/fund-price-history.csv?startdate=2023-02-01&enddate=2023-02-07&Lfunds=1&InvFunds=1&download=1';
$url = 'https://www.tsp.gov/data/fund-price-history.csv?startdate=2023-01-30&enddate=2023-02-03';
$App::Chart::option{'verbose'} = 2;
my $resp = App::Chart::Download->get
($url,
user_agent => App::Chart::Suffix::TSP::TSP_USER_AGENT,
referer => 'https://www.tsp.gov/share-price-history/');
my $content = $resp->decoded_content (charset => 'none');
### $content
my $h = App::Chart::Suffix::TSP::parse($resp);
print "h= ",Dumper($h);
exit 0;
}
{
# my $content = slurp ($ENV{'HOME'}.'/chart/samples/tsp/share-prices.html');
# my $content = slurp ("$ENV{HOME}/chart/samples/tsp/sharePriceHistory.shtml.2");
misc/t-tsp.pl view on Meta::CPAN
;
}
{
require App::Chart::Download;
my $hi_tdate = App::Chart::Download::tdate_today();
my $lo_tdate = $hi_tdate - 10;
my $resp = App::Chart::Suffix::TSP::get_chunk (['LINCOME.TSP'],
$lo_tdate,
$hi_tdate);
print $resp->headers->as_string;
print $resp->decoded_content (charset => 'none');
exit 0;
}
{
print App::Chart::Suffix::TSP::symbol_to_name('G.TSP'),"\n";
print App::Chart::Suffix::TSP::symbol_to_name('LINCOME.TSP'),"\n";
exit 0;
}
{
misc/t-yahoo.pl view on Meta::CPAN
."&period2=$hi_timet"
."&interval=1d"
."&events=". URI::Escape::uri_escape('div|split')
."&close=unadjusted";
print "$url\n";
require App::Chart::UserAgent;
my $ua = App::Chart::UserAgent->instance;
my $resp = $ua->get($url);
my $content = $resp->decoded_content (raise_error => 1, charset => 'none');
File::Slurp::write_file('/tmp/z', {err_mode=>'carp'}, $content);
$content = $resp->decoded_content (raise_error => 1);
my $decoded = JSON::from_json($content);
print JSON->new->pretty->encode($decoded);
exit 0;
}
{
# v1 info parse
my $filename = "$ENV{HOME}/chart/samples/yahoo/v1-info-CSCO.json";
$filename = "$ENV{HOME}/chart/samples/yahoo/v1-info-ORD.AX.json";
$filename =~ /v1-info-([A-Z.]+)\.json/ or die;
my $symbol = $1;
require HTTP::Response;
my $resp = HTTP::Response->new(200,'OK');
misc/t-yahoo.pl view on Meta::CPAN
my $jar = HTTP::Cookies->new;
my $user_agent = 'Mozilla/5.0';
my $resp = App::Chart::Download->get
($url,
user_agent => $user_agent,
cookie_jar => $jar,
redirect_ok => 0);
### headers: $resp->headers->as_string
print "jar string: ",$jar->as_string,"\n";
print "content:\n";
my $content = $resp->decoded_content (raise_error => 1, charset => 'none');
print $content,"\n";
exit 0;
}
{
$App::Chart::option{'verbose'} = 2;
my $h = App::Chart::Yahoo::cookie_and_crumb_data();
print Dumper($h);
exit 0;
}
misc/t-yahoo.pl view on Meta::CPAN
."&period2=$hi_timet"
."&interval=1d"
."&events=". URI::Escape::uri_escape('div|split')
."&close=unadjusted";
print "$url\n";
require App::Chart::UserAgent;
my $ua = App::Chart::UserAgent->instance;
my $resp = $ua->get($url);
my $content = $resp->decoded_content (raise_error => 1, charset => 'none');
File::Slurp::write_file('/tmp/z', {err_mode=>'carp'}, $content);
$content = $resp->decoded_content (raise_error => 1);
print $content;
exit 0;
}
{
# v8 dates, as of June 2024
my $t = 1504013400; # IBM
my $tz = "America/New_York";
my $gmtoff = -14400;
misc/t-yahoo.pl view on Meta::CPAN
my $h = App::Chart::Yahoo::exchanges_data ();
print Dumper(\$h);
exit 0;
}
{
# exchanges_parse() from file
#
my $filename = $ENV{'HOME'}.'/chart/samples/yahoo/exchanges.html';
$filename = $ENV{'HOME'}.'/chart/samples/yahoo/SLN2310.html';
$filename = $ENV{'HOME'}.'/chart/samples/yahoo/exchanges-data-providers-yahoo-finance-sln2310.html';
my $decoded_content = File::Slurp::read_file ($filename, {binmode => ':utf8'});
my $h = App::Chart::Yahoo::exchanges_parse ($decoded_content);
print Dumper(\$h);
exit 0;
}
{
# v8 download size
# https://query2.finance.yahoo.com/v8/finance/chart/TSCO.L?period1=1701140528&period2=1718766128&interval=1d&events=div%7Csplit
# &corsDomain=finance.yahoo.com
exit 0;
misc/t-yahoo.pl view on Meta::CPAN
$events = "split";
$events = "div";
$events = "history";
$events = "history|split";
my $url = "https://query2.finance.yahoo.com/v8/finance/chart/$symbol?formatted=true&lang=en-US®ion=US&period1=$start&period2=$end&interval=1d&events=$events&corsDomain=finance.yahoo.com";
require App::Chart::UserAgent;
my $ua = App::Chart::UserAgent->instance;
my $resp = $ua->get($url);
my $content = $resp->decoded_content (raise_error => 1, charset => 'none');
File::Slurp::write_file('/tmp/z', {err_mode=>'carp'}, $content);
$content = $resp->decoded_content (raise_error => 1);
print $content;
exit 0;
}
{
# v7 in June 2024
# https://finance.yahoo.com/quote/AMP.AX/history?p=AMP.AX
# redirect to:
# https://finance.yahoo.com/quote/AMP.AX/history/?p=AMP.AX
misc/t-yahoo.pl view on Meta::CPAN
my $url = "https://query1.finance.yahoo.com/v7/finance/download/$symbol?period1=$start&period2=$end&interval=1d®ion-AU&events=$events&crumb=$crumb";
print "$url\n";
my $resp = $ua->get($url);
print "\n";
my $resp_size = length($resp->as_string);
print "size $resp_size\n";
print $resp->status_line,"\n";
print $resp->headers->as_string;
my $content = $resp->decoded_content (raise_error => 1, charset => 'none');
File::Slurp::write_file('/tmp/y', {err_mode=>'carp'}, $content);
$content = $resp->decoded_content (raise_error => 1);
print $content;
exit 0;
}
{
# daily_parse_split()
require HTTP::Response;
my $filename = "$ENV{HOME}/chart/samples/yahoo/GXY-split.csv";
my $resp = HTTP::Response->new();
my $content = slurp ($filename);
misc/t-yahoo.pl view on Meta::CPAN
."?period1=$start&period2=$end&interval=1d&events=$events";
# seem to be defaults:
# &includeTimestamps=true
# &indicators=quote
require App::Chart::UserAgent;
my $ua = App::Chart::UserAgent->instance;
my $resp = $ua->get($url);
my $content = $resp->decoded_content (raise_error => 1, charset => 'none');
File::Slurp::write_file('/tmp/z', {err_mode=>'carp'}, $content);
$content = $resp->decoded_content (raise_error => 1);
print $content;
exit 0;
}
misc/t-yahoo.pl view on Meta::CPAN
});
my $crumb;
if (0) {
my $url = "https://finance.yahoo.com/quote/$symbol/history?p=$symbol";
my $resp = $ua->get($url);
my $resp_size = length($resp->as_string);
print "size $resp_size\n";
print $resp->status_line,"\n";
print $resp->headers->as_string;
my $content = $resp->decoded_content (raise_error => 1, charset => 'none');
File::Slurp::write_file('/tmp/x', {err_mode=>'carp'}, $content);
$content = $resp->decoded_content (raise_error => 1);
$content =~ /"CrumbStore":\{"crumb":"([^"]*)"}/;
$crumb = $1;
print "crumb raw $crumb\n";
$crumb = App::Chart::Yahoo::javascript_string_unquote($crumb);
$jar->save('/tmp/cookie3.txt');
} else {
$crumb = 'GmF7zbT8OWV';
$jar->load('/tmp/cookie3.txt');
}
print "crumb decoded $crumb\n";
print "cookies\n";
print $jar->as_string;
print "\n";
my $events;
$events = 'history';
$events = 'div';
$events = 'split,history';
print "now CSV\n";
misc/t-yahoo.pl view on Meta::CPAN
my $start = $end - 86400*5;
my $url = "https://query1.finance.yahoo.com/v7/finance/download/$symbol?period1=$start&period2=$end&interval=1d&events=$events&crumb=$crumb";
print "$url\n";
my $resp = $ua->get($url);
print "\n";
my $resp_size = length($resp->as_string);
print "size $resp_size\n";
print $resp->status_line,"\n";
print $resp->headers->as_string;
my $content = $resp->decoded_content (raise_error => 1, charset => 'none');
File::Slurp::write_file('/tmp/y', {err_mode=>'carp'}, $content);
$content = $resp->decoded_content (raise_error => 1);
print $content;
exit 0;
}
{
# timegm 00:00:00 AAPL last day missed, TSCO.L ok, BHP.AX ok
# end +86399 AAPL ok, TSCO.L and BHP.AX extra day
my $symbol;
$symbol = 'BHP.AX';
$symbol = 'AAPL';
misc/t-yahoo.pl view on Meta::CPAN
App::Chart::ymd_to_tdate_floor (2017,9,8));
### $url
### jar: $jar->as_string
my $resp = App::Chart::Download->get($url,
cookie_jar => $jar,
allow_404 => 1);
my $resp_size = length($resp->as_string);
print "size $resp_size\n";
print $resp->status_line,"\n";
print $resp->headers->as_string;
my $content = $resp->decoded_content (raise_error => 1);
print $content;
print "\n";
exit 0;
}
{
# App::Chart::Database->write_extra ('', 'yahoo-daily-cookies', undef);
$App::Chart::option{'verbose'} = 2;
misc/t-yahoo.pl view on Meta::CPAN
exit 0;
}
{
App::Chart::Database->add_symbol ('ETR.AX');
my $resp = HTTP::Response->new();
my $content = slurp ($ENV{'HOME'}.'/chart/samples/yahoo/etr.csv');
$resp->{'_rc'} = 200;
$resp->content($content);
die if ($resp->decoded_content(charset=>'none') ne $content);
my $h = App::Chart::Yahoo::daily_parse ('ETR.AX', $resp);
print Dumper ($h);
App::Chart::Download::write_daily_group ($h);
exit 0;
}
{
my $content = slurp ('/nosuchfile');
exit 0;
}
unused/CommSec.pm view on Meta::CPAN
my ($resp, $months) = @_;
my @data = ();
my $h = { source => __PACKAGE__,
currency => 'AUD',
suffix => '.AX',
prefer_decimals => 2,
date_format => 'dmy',
resp => $resp,
data => \@data };
my $body = $resp->decoded_content(raise_error=>1);
if ($body =~ /server error/i) {
# an unknown symbol
return $h;
}
$h->{'cost_key'} = INDIV_PERMONTH_COST_KEY;
$h->{'cost_value'} = int (length($body) / $months);
# Sample line
#
unused/CommSec.pm view on Meta::CPAN
my ($resp) = @_;
my @data = ();
my $h = { source => __PACKAGE__,
currency => 'AUD',
prefer_decimals => 2,
date_format => 'ymd',
resp => $resp,
data => \@data,
cost_key => WHOLEDAY_COST_KEY };
my $body = $resp->decoded_content(raise_error=>1);
if ($body =~ /server error/i) {
# an unknown symbol
return $h;
}
# Sample line, 5 Sep 2008
# BHP,080905,3640,3720,3630,3700,14390282
#
foreach my $line (App::Chart::Download::split_lines($body)) {
my ($symbol, $date, $open, $high, $low, $close, $volume)
unused/Float.pm view on Meta::CPAN
sub zip_parse {
my ($url, $resp) = @_;
my @data = ();
my $h = { source => __PACKAGE__,
currency => 'AUD',
prefer_decimals => 2,
# cover_pred => $pred,
# cover_date =>
data => \@data };
my $body = $resp->decoded_content (charset=>'none',raise_error=>1);
$h->{'cost_value'} = length ($body);
# unknown symbol or no data for a particular date gives a zip file with no
# members
$body = unzip_one ($body);
if (! $body) { return undef; }
my $max_date = tdate_to_yyyymmdd (available_tdate());
foreach my $line (App::Chart::Download::split_lines($body)) {
unused/NZX-old.pm view on Meta::CPAN
parse => \÷nds_parse,
key => 'NZ-dividends',
# low priority so prices fetched first
priority => -10);
sub dividends_parse {
my ($resp) = @_;
### NZ dividends_parse() ...
# note: want wide-chars for HTML::TableExtract parse
my $body = $resp->decoded_content (raise_error => 1);
### $body
my @dividends = ();
my $h = { source => __PACKAGE__,
resp => $resp,
dividends => \@dividends,
copyright_key => 'NZ-dividends-copyright',
copyright => 'http://www.nzx.com/terms',
date_format => 'dmy', # dates like "27 Sep 2018"
prefer_decimals => 2,
unused/TradingRoom.pm view on Meta::CPAN
my ($resp, $symbol) = @_;
my @data = ();
my $h = { source => __PACKAGE__,
currency => 'AUD',
suffix => '.AX',
prefer_decimals => 2,
date_format => 'dmy',
resp => $resp,
data => \@data };
my $body = $resp->decoded_content(raise_error=>1);
# Like
#
# Date,Open,High,Low,Close,Volume,Cumulative Dilution Factor
# 25-Jun-2015,94.6100,94.9000,94.5900,94.8000,7581,1
#
# 4 decimals so can be fractions of a cent.
# Unknown symbol is heading but no data lines.
#
my @lines = App::Chart::Download::split_lines($body);
unused/Yahoo-v7.pm view on Meta::CPAN
my $url = info_url($symbol);
my $resp = App::Chart::Download->get ($url);
my $h = info_parse($resp);
$h->{'recheck_list'} = [ $symbol ];
App::Chart::Download::write_daily_group ($h);
}
}
sub info_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (raise_error => 1);
my @info;
my $h = { source => __PACKAGE__,
info => \@info };
require JSON;
my $J = JSON::from_json($content) // {};
my $quotes = $J->{'quotes'} // [];
if (my $e = $quotes->[0]) {
my $symbol = $e->{'symbol'};
my $name = $e->{'shortname'};
unused/Yahoo-v7.pm view on Meta::CPAN
#
my $resp = App::Chart::Download->get ($url, allow_404 => 1,);
App::Chart::Download::write_latest_group
(latest_parse($symbol,$resp,$tdate));
}
}
sub latest_parse {
my ($symbol, $resp, $tdate) = @_;
my $content = $resp->decoded_content (raise_error => 1);
require JSON;
my $json = JSON::from_json($content);
my %record = (symbol => $symbol,
);
my $h = { source => __PACKAGE__,
resp => $resp,
prefer_decimals => 2,
date_format => 'ymd',
data => [ \%record ],
unused/Yahoo-v7.pm view on Meta::CPAN
sub daily_parse_v8 {
my ($symbol, $resp) = @_;
my $hi_tdate = daily_available_tdate ($symbol);
my @data = ();
my $h = { source => __PACKAGE__,
prefer_decimals => 2, # default
date_format => 'ymd',
data => \@data,
};
my $content = $resp->decoded_content (raise_error => 1);
require JSON;
my $json = JSON::from_json($content);
my $result = $json->{'chart'}->{'result'}->[0];
my $meta = $result->{'meta'} // {};
my $meta_symbol = $meta->{'symbol'} // '';
if ($meta_symbol ne $symbol) {
die "Yahoo JSON oops, symbol wanted $symbol got $meta_symbol";
}
# Trading in pence Sterling is "GBp", such as TSCO.L
unused/Yahoo-v7.pm view on Meta::CPAN
return $h;
}
sub daily_parse {
my ($symbol, $resp, $h, $hi_tdate) = @_;
my @data = ();
$h->{'data'} = \@data;
my $hi_tdate_iso;
if (defined $hi_tdate){ $hi_tdate_iso = App::Chart::tdate_to_iso($hi_tdate); }
my $body = $resp->decoded_content (raise_error => 1);
my @line_list = App::Chart::Download::split_lines($body);
unless ($line_list[0] =~ /^Date,Open,High,Low,Close,Adj Close,Volume/) {
die "Yahoo: unrecognised daily data headings: " . $line_list[0];
}
shift @line_list;
foreach my $line (@line_list) {
my ($date, $open, $high, $low, $close, $adj_volume, $volume)
= split (/,/, $line);
unused/Yahoo-v7.pm view on Meta::CPAN
close => crunch_trailing_nines($close),
volume => $volume };
}
return $h;
}
sub daily_parse_div {
my ($symbol, $resp, $h) = @_;
my @dividends = ();
$h->{'dividends'} = \@dividends;
my $body = $resp->decoded_content (raise_error => 1);
my @line_list = App::Chart::Download::split_lines($body);
# Date,Dividends
# 2015-11-04,1.4143
# 2016-05-17,1.41428
# 2017-05-16,1.4143
# 2016-11-03,1.4143
unless ($line_list[0] =~ /^Date,Dividends/) {
warn "Yahoo: unrecognised dividend headings: " . $line_list[0];
unused/Yahoo-v7.pm view on Meta::CPAN
ex_date => daily_date_fixup ($symbol, $date),
amount => $amount };
}
return $h;
}
sub daily_parse_split {
my ($symbol, $resp, $h) = @_;
my @splits = ();
$h->{'splits'} = \@splits;
my $body = $resp->decoded_content (raise_error => 1);
my @line_list = App::Chart::Download::split_lines($body);
# For example GXY.AX split so $10 shares become $2
# Date,Stock Splits
# 2017-05-22,1:5
#
# In the past it was a "/" instad of ":"
# 2017-05-22,1/5
unless ($line_list[0] =~ /^Date,Stock Splits/) {
unused/Yahoo-v7.pm view on Meta::CPAN
# script like, with backslash escaping on "\uXXXX"
# "user":{"age":0,"crumb":"8OyCBPyO4ZS"
# The form prior to about July 2023 was
# "user":{"crumb":"hdDX\u002FHGsZ0Q",
# The form prior to about January 2023 was
# "CrumbStore":{"crumb":"hdDX\u002FHGsZ0Q"}
# The form prior to about May 2024 was
# "RequestPlugin":{"user":{"age":0,"crumb":"8OyCBPyO4ZS"
#
my $content = $resp->decoded_content (raise_error => 1);
$content =~ /getcrumb.*?"body":"([^"]*)"/
or die "Yahoo getcrumb not found in parse";
return App::Chart::Yahoo::javascript_string_unquote($1);
}
sub Data_Dumper_str {
my ($h) = @_;
my $dumper = Data::Dumper->new ([$h], ['var']);
$dumper->Indent(1);
$dumper->Terse(1);
( run in 0.525 second using v1.01-cache-2.11-cpan-26ccb49234f )