App-Chart
view release on metacpan or search on metacpan
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
'https://secure.lme.com/Data/Community/Login.aspx?ReturnUrl=%2fData%2fcommunity%2findex.aspx';
#
# The result is a cookie ".ASPXAUTH" recorded under "lme-cookie-jar" in the
# database ready for subsequent use. An extra cookie with a dummy domain,
#
use constant LOGIN_DOMAIN => 'chart-lme-logged-in.local';
#
# is used to note success. Not sure how long a login is supposed to last
# (the server doesn't put an expiry on the cookie), but for now consider it
# expired after an hour,
#
use constant LOGIN_EXPIRY_SECONDS => 3600;
#
# create and return a new HTTP::Cookies which is the jar in the database
sub login_read_jar {
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
my $str = App::Chart::Database->read_extra ('', 'lme-cookie-jar');
if ($str) { http_cookies_set_string ($jar, $str); }
return $jar;
}
# $jar is a HTTP::Cookies object, save it to the database
sub login_write_jar {
my ($jar) = @_;
App::Chart::Database->write_extra ('', 'lme-cookie-jar',
http_cookies_get_string ($jar));
}
# return true if we're still logged in
sub login_is_logged_in {
my $jar = login_read_jar();
my $login_timestamp = jar_get_login_timestamp ($jar);
return App::Chart::Download::timestamp_within ($login_timestamp,
LOGIN_EXPIRY_SECONDS);
}
sub login_ensure {
if (login_is_logged_in()) { return; }
App::Chart::Download::status (__('LME login'));
App::Chart::Database->write_extra ('', 'lme-cookie-jar', undef);
my $username = App::Chart::Database->preference_get ('lme-username', undef);
my $password = App::Chart::Database->preference_get ('lme-password', '');
if (! defined $username || $username eq '') {
die 'No LME username set in preferences';
}
require App::Chart::UserAgent;
require HTTP::Cookies;
my $ua = App::Chart::UserAgent->instance->clone;
my $jar = HTTP::Cookies->new;
$ua->cookie_jar ($jar);
my $login_url = LOGIN_URL;
$login_url = 'http://localhost/Login.aspx';
my $resp = App::Chart::Download->get ($login_url, ua => $ua);
my $content = $resp->decoded_content(raise_error=>1);
my $form = HTML::Form->parse($content, $login_url)
or die "LME login page not a form";
# these are literal "$" in the field name
$form->value ("_logIn\$_userID", $username);
$form->value ("_logIn\$_password", $password);
my $req = $form->click();
$ua->requests_redirectable ([]);
$resp = $ua->request ($req);
# The POST is to the Login.aspx page and success is a redirect to the main
# data page /Data/community/index.aspx. So failure is anything other than
# 302, or no Location, or a Location but containing "Login".
if ($resp->code != 302
|| ! $resp->header ('Location')
|| $resp->header ('Location') =~ /Login/) {
die "LME: login failed";
}
jar_set_login_timestamp ($jar);
login_write_jar ($jar);
}
sub jar_get_login_timestamp {
my ($jar) = @_;
my $login_timestamp;
$jar->scan(sub {
my ($version, $key, $val, $path, $domain, $port, $path_spec,
$secure, $expires, $discard, $hash) = @_;
if ($domain eq LOGIN_DOMAIN && $key eq 'timestamp') {
$login_timestamp = $val;
}
});
return $login_timestamp;
}
sub jar_set_login_timestamp {
my ($jar) = @_;
$jar->set_cookie (0, # version
'timestamp', # key
App::Chart::Download::timestamp_now(), # value
'/', # path
LOGIN_DOMAIN, # domain
0, # port
0, # path_spec
0, # secure
LOGIN_EXPIRY_SECONDS, # maxage
0); # discard
}
#-----------------------------------------------------------------------------
# Daily data
# return tdate for available daily report
#
sub daily_available_date {
my ($symbol) = @_;
my $type = type($symbol);
if ($type eq 'metals') {
# http://www.lme.co.uk/who_how_ringtimes.asp
# Prices after second ring session each trading day, which would be
# 16:15 maybe, try at 16:30.
return App::Chart::Download::weekday_date_after_time
(16,30, App::Chart::TZ->london, -1);
}
if ($type eq 'plastics') {
# https://secure.lme.com/Data/community/Dataprices_daily_prices_plastics.aspx
# per prices page, available at 2am the following day
return App::Chart::Download::weekday_date_after_time
(2,0, App::Chart::TZ->london, -1);
}
if ($type eq 'steels') {
# per prices page, available at 2am the following day
return App::Chart::Download::weekday_date_after_time
(2,0, App::Chart::TZ->london, -1);
}
die;
}
#-----------------------------------------------------------------------------
# Daily price page parsing
sub daily_parse {
my ($resp, $want_tdate) = @_;
my @data = ();
my $h = { source => __PACKAGE__,
currency => 'USD',
data => \@data };
my $content = $resp->decoded_content (raise_error => 1);
$content = mung_1x1_tables ($content);
# Eg. "Official Prices, US$ per tonne for\n\t\t19 September 2008"
# Eg. "LME Official Prices, US$ per tonne for 18 September 2008"
#
$content =~ /Prices.*?for\s*\n?\s*([0-9]{1,2}\s+[A-Za-z]+\s+[0-9][0-9][0-9][0-9])/i
or die "LME daily: date not found";
my $date = App::Chart::Download::Decode_Date_EU_to_iso ($1);
if (defined $want_tdate) {
my $want_date = App::Chart::tdate_to_iso($want_tdate);
if ($date ne $want_tdate) {
die "LME daily: didn't get expected date, got $date want $want_tdate";
}
}
require HTML::TableExtract;
my $te = HTML::TableExtract->new (headers => [qr/PP.*Global/is],
keep_headers => 1,
slice_columns => 0);
$te->parse($content);
my $ts = $te->first_table_found();
if (! $ts) {
$te = HTML::TableExtract->new (headers => [qr/COPPER|STEEL/i],
keep_headers => 1,
slice_columns => 0);
$te->parse($content);
$ts = $te->first_table_found()
|| die "LME daily: prices table not found";
}
my $rows = $ts->rows();
my $lastrow = $#$rows;
my $lastcol = $#{$rows->[0]};
my @column;
my @column_commodity;
my @column_name;
foreach my $c (2 .. $lastcol) {
my $commodity = $rows->[0]->[$c] || next;
my $name;
if ($commodity =~ /ALUMINIUM ALLOY/i) { $commodity = 'AA'; }
elsif ($commodity =~ /ALUMINIUM/i) { $commodity = 'AH'; }
elsif ($commodity =~ /COPPER/i) { $commodity = 'CA'; }
elsif ($commodity =~ /LEAD/i) { $commodity = 'PB'; }
elsif ($commodity =~ /NICKEL/i) { $commodity = 'NI'; }
elsif ($commodity =~ /TIN/i) { $commodity = 'SN'; }
elsif ($commodity =~ /ZINC/i) { $commodity = 'ZS'; }
elsif ($commodity =~ /NASAAC/i) { $commodity = 'NI'; }
elsif ($commodity =~ /STEEL.*MEDITERRANEAN/s) { $commodity = 'FM'; }
elsif ($commodity =~ /STEEL.*FAR EAST/s) { $commodity = 'FF'; }
elsif ($commodity =~ /^([A-Z][A-Z])\s+(.*)/is) { $commodity = $1; $name = $2; }
else { next; }
push @column, $c;
push @column_commodity, $commodity;
push @column_name, $name;
}
if (DEBUG) { require Data::Dumper;
print "columns ", Data::Dumper::Dumper(\@column);
print "columns ", Data::Dumper::Dumper(\@column_commodity);
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
# Return tdate of anticipated available montly .xls download, that being
# the end of the previous month.
#
# Don't know exactly when a new month full of data becomes available,
# assume here midnight at the start of the second trading day of the new
# month.
#
sub monthxls_available_tdate {
my $tdate = App::Chart::Download::tdate_today
(App::Chart::TZ->london);
$tdate--; # not until second business day into this month
$tdate = tdate_start_of_month ($tdate);
return $tdate - 1; # last day of previous month
}
sub monthxls_download {
my ($symbol_list) = @_;
if (DEBUG) { print "LME ",@$symbol_list,"\n"; }
my $lo_tdate = App::Chart::Download::start_tdate_for_update (@$symbol_list);
my $hi_tdate = monthxls_available_tdate();
my @files = grep {$_->{'url'} !~ /volume/i} historical_xls_files();
my $files = App::Chart::Download::choose_files (\@files, $lo_tdate, $hi_tdate);
foreach my $f (@$files) {
my $url = $f->{'url'};
require File::Basename;
my $filename = File::Basename::basename($url);
App::Chart::Download::status (__x('LME data {filename}',
filename => $filename));
my $resp = App::Chart::Download->get ($url);
my $h = monthxls_parse ($resp);
App::Chart::Download::write_daily_group ($h);
}
}
sub tdate_start_of_month {
my ($tdate) = @_;
my ($year,$month,$day) = App::Chart::tdate_to_ymd ($tdate);
return App::Chart::ymd_to_tdate_ceil ($year, $month, 1);
}
my %monthxls_sheet_to_commodity =
('Copper' => 'CA',
'Al. Alloy' => 'AA',
'NASAAC' => 'NA',
'Zinc' => 'ZS',
'Lead' => 'PB',
'Pr. Aluminium' => 'AH',
'Tin' => 'SN',
'Nickel' => 'NI',
'Far East' => 'FF', # steel
'Med' => 'FM', # steel
'Averages' => undef,
'Plastic Avg' => undef,
'Averages inc. Euro Eq' => undef);
sub monthxls_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (charset => 'none', raise_error => 1);
require Spreadsheet::ParseExcel;
require Spreadsheet::ParseExcel::Utility;
my @data = ();
my $h = { source => __PACKAGE__,
cover_pred => $pred,
data => \@data };
my $excel = Spreadsheet::ParseExcel::Workbook->Parse (\$content);
foreach my $sheet (@{$excel->{Worksheet}}) {
my $sheet_name = $sheet->{'Name'};
if (DEBUG) { print "Sheet: $sheet_name\n"; }
my $commodity;
if ($sheet_name =~ /^[A-Z][A-Z]$/) {
# plastics symbol
$commodity = $sheet_name;
} elsif (exists $monthxls_sheet_to_commodity{$sheet_name}) {
$commodity = $monthxls_sheet_to_commodity{$sheet_name}
// next; # undef for ignored sheets
} else {
warn "LME: unrecognised month data sheet: $sheet_name\n";
next;
}
my ($minrow, $maxrow) = $sheet->RowRange;
my ($mincol, $maxcol) = $sheet->ColRange;
my $heading_row = $minrow;
my $date_col;
my $seller_col;
HEADING: for (;; $heading_row++) {
if ($heading_row > $maxrow) { die "LME: headings row not found\n"; }
for ($seller_col = $mincol; $seller_col <= $maxcol; $seller_col++) {
my $cell = $sheet->Cell($heading_row,$seller_col) // next;
my $str = $cell->Value;
if (DEBUG >= 2) { print " cell $heading_row,$seller_col $str\n"; }
if ($str =~ /SELLER/i) { last HEADING; }
}
}
$date_col = $seller_col - 2;
if (DEBUG) { print " heading row $heading_row seller col $seller_col\n"; }
my @column_num = ();
my @column_symbol = ();
for (my $col = $seller_col; $col+2 <= $maxcol; $col += 3) {
my $cell = $sheet->Cell($heading_row,$col) || last;
$cell->Value =~ /SELLER/i or next;
my $period = $sheet->Cell($heading_row-1,$col)->Value;
if (DEBUG >= 2) { print " col=$col period=$period\n"; }
if ($period =~ /cash/i) {
$period = '';
} elsif ($period =~ /([0-9]+).*(months|mths)/i) {
$period = $1;
} elsif ($period eq '') {
last;
} else {
die "LME: month sheet '$sheet_name' heading row=$heading_row col=$col period unrecognised: '$period'\n";
}
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
my $symbol = $column_symbol[$i];
# unformatted value gets '1490.00' instead of '$1,490.00'
my $seller = $sheet->Cell($row,$col)->{'Val'};
push @data, { symbol => $symbol,
date => $date,
close => $seller,
};
}
}
if (! $seen_date) {
die "LME month data: no dates found in sheet '$sheet_name'";
}
}
my $date = $data[0]->{'date'};
my ($year, $month, $day) = App::Chart::iso_to_ymd ($date);
$h->{'cover_lo_date'} = App::Chart::ymd_to_iso ($year, $month, 1);
($year, $month, $day) = Date::Calc::Add_Delta_YMD ($year, $month, $day,
0, 1, -1);
$h->{'cover_hi_date'} = App::Chart::ymd_to_iso ($year, $month, $day);
return $h;
}
#-----------------------------------------------------------------------------
# download - volume xls files
#
# This crunches files like
# http://www.lme.co.uk/downloads/volumes_Jan_08.xls
#
# App::Chart::DownloadHandler->new
# (name => __('LME month volumes'),
# pred => $pred,
# proc => \&volume_download,
# # backto => \&volume_backto,
# available_tdate => \&monthxls_available_tdate);
sub volume_download {
my ($symbol_list) = @_;
my $lo_tdate = App::Chart::Download::start_tdate_for_update (@$symbol_list);
my $hi_tdate = monthxls_available_tdate();
my @files = grep {$_->{'url'} =~ /volume/i} historical_xls_files();
my $files = App::Chart::Download::choose_files (\@files, $lo_tdate, $hi_tdate);
foreach my $f (@$files) {
my $url = $f->{'url'};
require File::Basename;
my $filename = File::Basename::basename($url);
App::Chart::Download::status (__x('LME volumes {filename}',
filename => $filename));
my $resp = App::Chart::Download->get ($url);
my $h = volume_parse ($resp);
App::Chart::Download::write_daily_group ($h);
}
}
sub volume_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (charset => 'none', raise_error => 1);
require Spreadsheet::ParseExcel;
require Spreadsheet::ParseExcel::Utility;
my @data = ();
my $h = { source => __PACKAGE__,
data => \@data };
my $excel = Spreadsheet::ParseExcel::Workbook->Parse (\$content);
my $sheet = $excel->Worksheet (0);
if (DEBUG) { print "Sheet: ",$sheet->{'Name'},"\n"; }
my ($minrow, $maxrow) = $sheet->RowRange;
my ($mincol, $maxcol) = $sheet->ColRange;
# headings are like "AAFUT" for Aluminium Alloy, find that row
my $heading_row;
HEADINGROW: foreach my $row ($minrow .. $maxrow) {
foreach my $col ($mincol .. $maxcol) {
my $cell = $sheet->Cell($row,$col) or next;
if ($cell->Value =~ /FUT$/) {
$heading_row = $row;
last HEADINGROW;
}
}
}
if (! $heading_row) { die 'LME Volumes: unrecognised headings'; }
if (DEBUG) { print " heading row $heading_row\n"; }
# look for each "AAFUT" etc column in the heading row
my @column_num = ();
my @column_symbol = ();
foreach my $col ($mincol .. $maxcol) {
my $cell = $sheet->Cell($heading_row,$col) // next; # skip empties
$cell->{'Type'} eq 'Text' or next; # skip dates in heading
my $str = $cell->Value;
$str =~ /(.*)FUT$/ or next;
my $commodity = $1;
push @column_num, $col;
push @column_symbol, $commodity . '.LME';
}
my $seen_date = 0;
foreach my $row ($heading_row+1 .. $maxrow) {
my $date;
# Jan 2008 has 'Date' type in column 1
# May 2008 onwards has text d-Mmm-yy in column 0
my $datecell = $sheet->Cell($row,0);
if ($datecell->{'Type'} eq 'Text') {
$date = App::Chart::Download::Decode_Date_EU_to_iso($datecell->{'Val'},1);
# skip blanks at end, avoid "Total"
if (! defined $date) { next; }
} else {
$datecell = $sheet->Cell($row,1);
# skip blanks at end, avoid "Total"
$datecell->{'Type'} eq 'Date' or next;
# default format is like 31-Jan-08, go straight to ISO to be unambiguous
$date = Spreadsheet::ParseExcel::Utility::ExcelFmt
('yyyy-mm-dd', $datecell->{'Val'}, $excel->{'Flg1904'});
}
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
}
return \%sm;
}
sub daily_download_one {
my ($type, $tdate, $l) = @_;
require HTML::Form;
my $content = $l->{'content'};
my $url = $l->{'url'};
my $form = HTML::Form->parse($content, $url)
or die "LME metals page not a form";
my ($year, $month, $day) = App::Chart::tdate_to_ymd ($tdate);
# these are literal "$" in the field name
$form->value ("_searchForm\$_lstdate", $day);
$form->value ("_searchForm\$_lstmonth", $month);
$form->value ("_searchForm\$_lstyear", $year);
App::Chart::Download::status
(__x('LME daily {type} {date}',
type => $type,
date => App::Chart::Download::tdate_range_string ($tdate)));
require App::Chart::UserAgent;
require HTTP::Cookies;
my $ua = App::Chart::UserAgent->instance->clone;
$ua->requests_redirectable ([]);
my $jar = HTTP::Cookies->new;
$ua->cookie_jar ($jar);
my $req = $form->click();
my $resp = $ua->request ($req);
if (! $resp->is_success) {
die "Cannot download $url\n",$resp->headers->as_string,"\n";
}
return $resp;
}
my %type_to_daily_url
= (metals => 'https://secure.lme.com/Data/community/Dataprices_daily_metals.aspx',
plastics => 'https://secure.lme.com/Data/community/Dataprices_daily_prices_plastics.aspx',
steels => 'https://secure.lme.com/Data/community/Dataprices_Steels_OfficialPrices.aspx');
sub daily_latest {
my ($type) = @_;
require App::Chart::Pagebits;
return App::Chart::Pagebits::get
(name => __x('LME daily latest {type}',
type => $type),
url => $type_to_daily_url{$type},
key => "lme-daily-latest-$type",
freq_days => 0,
timezone => App::Chart::TZ->london,
parse => \&daily_latest_parse);
}
sub daily_latest_parse {
my ($resp) = @_;
my $content = $resp->decoded_content (raise_error => 1);
my $h = daily_parse ($resp);
return { h => $h,
date => $h->{'data'}->[0]->{'date'},
url => $resp->uri->as_string,
content => $content };
}
1;
__END__
#-----------------------------------------------------------------------------
# download - daily
#
#
# LST has elements (SYMBOL NAME TDATE BUY-STR SELL-STR MDATE) per
# `daily-html-parse'
#
# The sell price is used. The report for cash prices has the seller marked
# as the settlement and for the forwards the historical files can be seen
# with the seller price.
#
(define (daily-process symbol-list lst)
(download-process
#:module (_ "LME")
#:symbol-list symbol-list
#:currency "USD"
#:row-list
(map (lambda (row)
(receive-list (symbol name tdate buy sell mdate)
row
(list #:tdate tdate
#:mdate mdate
#:commodity (chart-symbol-commodity symbol)
#:close sell)))
lst)))
(define (lme-daily-download symbol-list type)
(define selector (case type
((metals) lme-metal-symbol?)
((plastics) lme-plastics-symbol?)))
(set! symbol-list (filter selector symbol-list))
(if (not (null? symbol-list))
(let* ((end-data (assq-ref (daily-latest-info type) 'data))
(end-tdate (if end-data
(data-tdate end-data)
(daily-available-tdate type))))
(set! symbol-list
(download-also symbol-list #:selector selector))
# only go back 25 days for LMEX or others without yearly data,
# since at 70kbytes per day it quickly becomes slow
#
(do ((t (apply min (map (lambda (symbol)
(download-start-tdate symbol #:initial 25))
( run in 1.810 second using v1.01-cache-2.11-cpan-437f7b0c052 )