App-Chart
view release on metacpan or search on metacpan
"Gtk2::Ex::TreeViewBits" : "8",
"Gtk2::Ex::Units" : "13",
"Gtk2::Ex::WidgetBits" : "10",
"Gtk2::Ex::WidgetCursor" : "8",
"Gtk2::Ex::WidgetEvents" : "21",
"Gtk2::Pango" : "0",
"HTML::Form" : "0",
"HTML::LinkExtor" : "0",
"HTML::TableExtract" : "0",
"HTML::TreeBuilder" : "0",
"HTTP::Cookies" : "0",
"HTTP::Message" : "5.814",
"HTTP::Request" : "0",
"IO::String" : "1.07",
"JSON" : "0",
"LWP" : "5.814",
"LWP::UserAgent" : "0",
"Lisp::Printer" : "0",
"Lisp::Reader" : "0.04",
"Lisp::Symbol" : "0",
"List::MoreUtils" : "0.24",
Gtk2::Ex::TreeViewBits: '8'
Gtk2::Ex::Units: '13'
Gtk2::Ex::WidgetBits: '10'
Gtk2::Ex::WidgetCursor: '8'
Gtk2::Ex::WidgetEvents: '21'
Gtk2::Pango: '0'
HTML::Form: '0'
HTML::LinkExtor: '0'
HTML::TableExtract: '0'
HTML::TreeBuilder: '0'
HTTP::Cookies: '0'
HTTP::Message: '5.814'
HTTP::Request: '0'
IO::String: '1.07'
JSON: '0'
LWP: '5.814'
LWP::UserAgent: '0'
Lisp::Printer: '0'
Lisp::Reader: '0.04'
Lisp::Symbol: '0'
List::MoreUtils: '0.24'
Makefile.PL view on Meta::CPAN
# version 1.19 for N__() in scalar context
# (Locale::Messages in 1.16 didn't declare a version, it was only in
# Locale::TextDomain until 1.17 or 1.18 or something)
'Locale::TextDomain' => '1.19',
'Locale::Messages' => '1.19',
'LWP' => '5.814', # version 5.814 for new HTTP::Message::decodable()
# parts of lwp used explicitly, some of which are separate now
'LWP::UserAgent' => 0,
'HTTP::Cookies' => 0,
'HTTP::Message' => '5.814',
'HTTP::Request' => 0,
'HTML::Form' => 0,
'Math::Round' => 0,
'Module::Find' => 0.06, # version 0.06 avoids duplicates
'Module::Util' => 0,
'Module::Pluggable' => 0,
# version 1.60 for fixups to initializing in locales like de_DE with "."
lib/App/Chart/Barchart.pm view on Meta::CPAN
my ($setup_pred) = @_;
$latest_pred->add ($setup_pred);
$intraday_pred->add ($setup_pred);
$fiveday_pred->add ($setup_pred);
}
#-----------------------------------------------------------------------------
# various
sub cookie_jar {
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
# Set-Cookie: bcad_int=1; path=/; domain=barchart.com;
$jar->set_cookie(undef, # version
'bcad_int', # key
'1', # value
'/', # path
'.barchart.com'); # domain
return $jar;
}
#-----------------------------------------------------------------------------
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
if ($symbol =~ /^SN/) { return 'http://www.lme.co.uk/tin.asp' }
if ($symbol =~ /^ZS/) { return 'http://www.lme.co.uk/zinc.asp' }
if ($symbol =~ /^F/) { return 'http://www.lme.co.uk/steel.asp' }
if ($symbol =~ /^P/) { return 'http://www.lme.co.uk/plastics.asp' }
if ($symbol =~ /^L/) { return 'http://www.lme.co.uk/plastics.asp' }
return undef;
});
#-----------------------------------------------------------------------------
# HTTP::Cookies extras
# $jar is a HTTP::Cookies object, read $str into it with $jar->load (which
# would normally read from a file)
#
sub http_cookies_set_string {
my ($jar, $str) = @_;
my $fh = File::Temp->new (TEMPLATE => 'chart-cookie-jar-XXXXXX',
TMPDIR => 1);
if (DEBUG) { print "cookie set tempfile ",$fh->filename,"\n"; }
print $fh $str;
close $fh or die;
$jar->load ($fh->filename);
}
# $jar is a HTTP::Cookies object, return a string which is $jar->save output
# (which would normally go to a file)
#
sub http_cookies_get_string {
my ($jar) = @_;
my $fh = File::Temp->new (TEMPLATE => 'chart-cookie-jar-XXXXXX',
TMPDIR => 1);
if (DEBUG) { print "cookie get $fh tempfile ",$fh->filename,"\n"; }
$jar->save ($fh->filename);
close $fh or die;
# not certain if File::Temp 0.21 blessed handle is ok, use the filename
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
#
use constant LOGIN_DOMAIN => 'chart-lme-logged-in.local';
#
# is used to note success. Not sure how long a login is supposed to last
# (the server doesn't put an expiry on the cookie), but for now consider it
# expired after an hour,
#
use constant LOGIN_EXPIRY_SECONDS => 3600;
#
# create and return a new HTTP::Cookies which is the jar in the database
sub login_read_jar {
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
my $str = App::Chart::Database->read_extra ('', 'lme-cookie-jar');
if ($str) { http_cookies_set_string ($jar, $str); }
return $jar;
}
# $jar is a HTTP::Cookies object, save it to the database
sub login_write_jar {
my ($jar) = @_;
App::Chart::Database->write_extra ('', 'lme-cookie-jar',
http_cookies_get_string ($jar));
}
# return true if we're still logged in
sub login_is_logged_in {
my $jar = login_read_jar();
my $login_timestamp = jar_get_login_timestamp ($jar);
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
App::Chart::Download::status (__('LME login'));
App::Chart::Database->write_extra ('', 'lme-cookie-jar', undef);
my $username = App::Chart::Database->preference_get ('lme-username', undef);
my $password = App::Chart::Database->preference_get ('lme-password', '');
if (! defined $username || $username eq '') {
die 'No LME username set in preferences';
}
require App::Chart::UserAgent;
require HTTP::Cookies;
my $ua = App::Chart::UserAgent->instance->clone;
my $jar = HTTP::Cookies->new;
$ua->cookie_jar ($jar);
my $login_url = LOGIN_URL;
$login_url = 'http://localhost/Login.aspx';
my $resp = App::Chart::Download->get ($login_url, ua => $ua);
my $content = $resp->decoded_content(raise_error=>1);
my $form = HTML::Form->parse($content, $login_url)
or die "LME login page not a form";
lib/App/Chart/Suffix/LME.pm view on Meta::CPAN
$form->value ("_searchForm\$_lstdate", $day);
$form->value ("_searchForm\$_lstmonth", $month);
$form->value ("_searchForm\$_lstyear", $year);
App::Chart::Download::status
(__x('LME daily {type} {date}',
type => $type,
date => App::Chart::Download::tdate_range_string ($tdate)));
require App::Chart::UserAgent;
require HTTP::Cookies;
my $ua = App::Chart::UserAgent->instance->clone;
$ua->requests_redirectable ([]);
my $jar = HTTP::Cookies->new;
$ua->cookie_jar ($jar);
my $req = $form->click();
my $resp = $ua->request ($req);
if (! $resp->is_success) {
die "Cannot download $url\n",$resp->headers->as_string,"\n";
}
return $resp;
}
lib/App/Chart/Yahoo.pm view on Meta::CPAN
# today's date and it changes though the course of trading.
# Don't know how or whether pre-market or post-market trading is
# applied, or what happens to a few futures or similar which might
# even be 24 hour trading.
#
# Dividends and splits seem to only appear on the ex date.
# That's disappointing since upcoming dividends often influence price
# action. (Which they shouldn't since everyone knows in advance.)
#
#
# Cookies and Crumb:
#
# In recent times there's been some protocol hoops to jump through.
# It seemed to be sometimes on the v7 download, maybe always on v11.
# The v7 JSON has been fine asking for the latest few days daily data,
# but not sure about bigger historical data.
#
# The hoops consisted of
#
# - Fetch one of the finance.yahoo.com web pages to get a
# HTTP Set-Cookie header.
# As of September 2024, there seems no such cookie any more
# (only a general user-tracking one for www.yahoo.com).
# - Maybe answer the ridiculous EU cookie consent on the page.
# Maybe that depends on where your IP seems to be from.
# (Not that an IP reliably indicates legal jurisdiction.)
# - Look deep within script in that page for a "crumb" string.
# Or maybe a further "getcrumb" web fetch, but seems result
# crumb is embedded in the page.
# - On each data download, HTTP Cookie header, and URL crumb
# field.
#
# Presumably this is designed either to track user activity, or
# as a level of difficulty to stop what Yahoo had said was
# widespread mis-use (contrary to personal use terms).
#
package App::Chart::Yahoo;
use 5.010;
misc/t-barchart.pl view on Meta::CPAN
my $uri = URI->new($url);
require HTTP::Request;
my @headers = (Referer => $url);
my $req = HTTP::Request->new ('GET', $url, \@headers, undef);
print $req->uri,"\n";
require HTTP::Response;
my $resp = HTTP::Response->new
(200, 'OK',
[ 'Set-Cookie', 'bcad_int=1; path=/; domain=barchart.com;' ]);
$resp->request ($req);
print $resp->as_string,"\n";
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
# $jar->set_cookie(1, # version
# 'bcad_int', # key
# '1', # value
# '/', # path
# 'barchart.com'); # domain
$jar->extract_cookies ($resp);
print $jar->as_string, "\n";
require App::Chart::UserAgent;
my $ua = App::Chart::UserAgent->instance;
print HTTP::Cookies::_host($req,$uri),"\n";
# $ua->prepare_request ($req);
$jar->add_cookie_header($req);
print "req now\n", $req->as_string, "\n";
exit 0;
}
misc/t-lme.pl view on Meta::CPAN
{
my $resp = HTTP::Response->new(200,'OK');
my $content = slurp (<~/chart/samples/lme/volumes_May_08.xls>);
$resp->content($content);
my $h = App::Chart::Suffix::LME::volume_parse ($resp);
print Dumper($h);
exit 0;
}
{
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
my $ts = App::Chart::Suffix::LME::jar_get_login_timestamp($jar);
print Dumper($ts);
App::Chart::Suffix::LME::jar_set_login_timestamp($jar);
print "Jar as_string:\n",$jar->as_string;
require URI;
require HTTP::Request;
my $uri = URI->new;
$uri->scheme ('http');
misc/t-yahoo.pl view on Meta::CPAN
my $h = App::Chart::Yahoo::daily_parse ($symbol, $resp);
App::Chart::Download::crunch_h ($h);
print Dumper(\$h);
exit 0;
}
{
# cookie
# cf Finance::Quote::YahooJSON
# https://www.yahoo.com/
# Set-Cookie: RRC=st=1725958300186.3&cnt=1; path=/; domain=.www.yahoo.com; HttpOnly
# Redirect to https://au.yahoo.com/?p=us
#
# https://finance.yahoo.com/quote/IBM/history/?p=IBM
# no cookie
my $url = 'https://www.yahoo.com/';
$url = 'https://finance.yahoo.com';
$url = 'https://finance.yahoo.com/quote/IBM/history/?p=IBM';
$App::Chart::option{'verbose'} = 2;
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
my $user_agent = 'Mozilla/5.0';
my $resp = App::Chart::Download->get
($url,
user_agent => $user_agent,
cookie_jar => $jar,
redirect_ok => 0);
### headers: $resp->headers->as_string
print "jar string: ",$jar->as_string,"\n";
print "content:\n";
my $content = $resp->decoded_content (raise_error => 1, charset => 'none');
misc/t-yahoo.pl view on Meta::CPAN
}
{
# cf
# https://gist.github.com/Mister-Meeseeks/df985c5e3abb1be88004319f11ebe3fb/raw/a6541d2a7cda6376ad0fc20fecd3388c7ada49ca/pullYahoo.sh
#
# "CrumbStore":{"crumb":"pwrULw9Alv\u002F"}
require File::Slurp;
require HTTP::Cookies;
require App::Chart::UserAgent;
my $symbol = 'GXY.AX';
my $ua = App::Chart::UserAgent->instance;
my $jar = HTTP::Cookies->new;
$ua->cookie_jar ($jar);
$ua->add_handler (request_send => sub {
my ($request, $ua, $headers) = @_;
print $request->method," ",$request->uri,"\n";
print $request->headers->as_string,"\n";
return;
});
my $crumb;
# not yet
# is (App::Chart::symbol_source_help ('COPPER.LME'),
# __p('manual-node','London Metal Exchange'));
# is (App::Chart::Suffix::LME::Mmm_yyy_str_to_mdate ('January_1970'), 0);
# is (App::Chart::Suffix::LME::Mmm_yyy_str_to_mdate ('Mar_1970'), 2);
# is (App::Chart::Suffix::LME::Mmm_yyy_str_to_mdate ('Jan_1971'), 12);
{
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
App::Chart::Suffix::LME::jar_set_login_timestamp($jar);
my $ts = App::Chart::Suffix::LME::jar_get_login_timestamp($jar);
ok (App::Chart::Download::timestamp_within ($ts, 3600),
'current timestamp within 1 hour of now');
}
sub jar_count_cookies {
my ($jar) = @_;
my $count = 0;
$jar->scan(sub { $count++ });
return $count;
}
{
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
my $str = App::Chart::Suffix::LME::http_cookies_get_string ($jar);
diag explain $str;
ok (1, 'jar empty -> string');
App::Chart::Suffix::LME::http_cookies_set_string ($jar, $str);
ok (1, 'empty string -> jar');
App::Chart::Suffix::LME::jar_set_login_timestamp($jar);
$str = App::Chart::Suffix::LME::http_cookies_get_string ($jar);
#------------------------------------------------------------------------------
# http_cookies_from_string()
# {
# # round trip ->as_string(), primarily to see that the hack in
# # http_cookies_from_string() is ok
# #
# my $str = <<'HERE';
# Set-Cookie3: foo="bar"; path="/"; domain=.example.com; path_spec; expires="2037-06-02 20:00:00Z"; version=0
# HERE
# my $cookies = App::Chart::Yahoo::http_cookies_from_string($str);
# my $got = $cookies->as_string;
# ok ($got,$str);
# }
#------------------------------------------------------------------------------
# $index_pred
unused/Yahoo-v7.pm view on Meta::CPAN
# number of decimals. Eg. 2 decimals for dollars and cents, but
# allowing for trading in fractions of a cent which is 3 decimals.
#
# In daily data download, current day trading in progress shows in
# today's date and changes though the course of trading until being
# fixed maybe at marked close, or maybe the next day. Don't know
# how pre-market or post-market trading is applied, or what happens
# if a few futures or similar might even be 24 hour trading.
#
#
# Cookies and Crumb:
#
# There's been some protocol hoops to jump through in recent times.
# It seems to be sometimes on the v7 form, maybe always on v11.
# The v7 had seemed fine asking for the latest few days daily data,
# and it's possible lately needs nothing for any amount.
#
# The hoops consist of
#
# - Fetch one of the finance.yahoo.com web pages to get a
# HTTP Set-Cookie header.
# - Maybe answer the ridiculous EU cookie consent on the page.
# Maybe that depends where your IP looks like it's from.
# - Look deep within script in that page for a "crumb" string.
# Or maybe a further "getcrumb" web fetch, but seems result
# is embedded in the page.
# - On each data download, HTTP Cookie header, and URL crumb
# field.
#
# Presumably this is designed as a level of difficulty, to stop the
# past quotes and data that could be had from a single URL (and which
# Yahoo apparently found was widely abused beyond personal use).
#
package App::Chart::Yahoo;
use 5.010;
unused/Yahoo-v7.pm view on Meta::CPAN
#-----------------------------------------------------------------------------
# Download Data
#
# This uses the historical prices page like
#
# https://finance.yahoo.com/quote/AMP.AX/history?p=AMP.AX
#
# which puts a cookie like
#
# Set-Cookie: B=fab5sl9cqn2rd&b=3&s=i3; expires=Sun, 03-Sep-2018 04:56:13 GMT; path=/; domain=.yahoo.com
#
# and contains buried within 1.5 mbytes of hideous script
#
# <script type="application/json" data-sveltekit-fetched data-url="https://query1.finance.yahoo.com/v1/test/getcrumb?lang=en-US&region=US" data-ttl="59">{"status":200,"statusText":"OK","headers":{},"body":"DKVWQE/ggh4"}</script>
#
# Any \u002F or similar is escaped "/" character or similar.
# The crumb is included in a CSV download query like the following
# (alas can't use http, it redirects to https)
#
# https://query1.finance.yahoo.com/v7/finance/download/AMP.AX?period1=1503810440&period2=1504415240&interval=1d&events=history&crumb=hdDX/HGsZ0Q
unused/Yahoo-v7.pm view on Meta::CPAN
$got = 0;
$str .= '.';
}
if ((my $add = $decimals - $got) > 0) {
$str .= '0' x $add;
}
return $str;
}
# return a hashref
# { cookies => string, # in format HTTP::Cookies ->as_string()
# crumb => string
# }
#
# If no such $symbol then return undef;
#
# Any $symbol which exists is good enough to get a crumb for all later use.
# Could hard-code something likely here, but better to go from the symbol
# which is wanted.
#
# As of April 2024, some User-Agent strings result in 503 Service Unavailable.
unused/Yahoo-v7.pm view on Meta::CPAN
sub cookie_and_crumb_data {
my ($symbol) = @_;
my $key = 'yahoo-cookie-and-crumb';
my $str = App::Chart::Database->read_extra ('', $key);
my $h = eval ($str || '{}');
$h = {};
unless (App::Chart::Download::timestamp_within
($h->{'timestamp'}, 3 * 86400)) {
App::Chart::Download::status ('Yahoo cookie');
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
my $user_agent = 'Mozilla/5.0';
{
# first to get a cookie
my $resp = App::Chart::Download->get
('https://www.yahoo.com/',
user_agent => $user_agent,
cookie_jar => $jar);
### jar: $jar->as_string
App::Chart::Download::verbose_message ("Yahoo cookies: "
. $jar->as_string);
unused/Yahoo-v7.pm view on Meta::CPAN
return $sec + 60*$min + 3600*$hour + 86400*($gmt_day - $day);
}
# Return seconds since 00:00:00, 1 Jan 1970 GMT.
sub tdate_to_unix {
my ($tdate) = @_;
my $adate = App::Chart::tdate_to_adate ($tdate);
return ($adate + 4)*86400;
}
# $str is a string from previous HTTP::Cookies ->as_string()
# Return a new HTTP::Cookies object with that content.
sub http_cookies_from_string {
my ($str) = @_;
require File::Temp;
my $fh = File::Temp->new (TEMPLATE => 'chart-XXXXXX',
TMPDIR => 1);
print $fh "#LWP-Cookies-1.0\n", $str or die;
close $fh or die;
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
$jar->load($fh->filename);
return $jar;
}
#------------------------------------------------------------------------------
# undo javascript string backslash quoting in STR, per
#
# https://developer.mozilla.org/en/JavaScript/Guide/Values,_Variables,_and_Literals#String_Literals
#
unused/YahooOld.pm view on Meta::CPAN
#-----------------------------------------------------------------------------
# v7 cookies and crumb
# Download Data
#
# This uses the historical prices page like
#
# https://finance.yahoo.com/quote/AMP.AX/history?p=AMP.AX
#
# which puts a cookie like
#
# Set-Cookie: B=fab5sl9cqn2rd&b=3&s=i3; expires=Sun, 03-Sep-2018 04:56:13 GMT; path=/; domain=.yahoo.com
#
# and contains buried within 1.5 mbytes of hideous script
#
# <script type="application/json" data-sveltekit-fetched data-url="https://query1.finance.yahoo.com/v1/test/getcrumb?lang=en-US&region=US" data-ttl="59">{"status":200,"statusText":"OK","headers":{},"body":"DKVWQE/ggh4"}</script>
#
# Any \u002F or similar is escaped "/" character or similar.
# The crumb is included in a CSV download query like the following
# (alas can't use http, it redirects to https)
#
# https://query1.finance.yahoo.com/v7/finance/download/AMP.AX?period1=1503810440&period2=1504415240&interval=1d&events=history&crumb=hdDX/HGsZ0Q
unused/YahooOld.pm view on Meta::CPAN
push @splits, { symbol => $symbol,
date => daily_date_fixup ($symbol, $date),
new => $new,
old => $old };
}
return $h;
}
# return a hashref
# { cookies => string, # in format HTTP::Cookies ->as_string()
# crumb => string
# }
#
# If no such $symbol then return undef;
#
# Any $symbol which exists is good enough to get a crumb for all later use.
# Could hard-code something likely here, but better to go from the symbol
# which is wanted.
#
# As of April 2024, some User-Agent strings result in 503 Service Unavailable.
unused/YahooOld.pm view on Meta::CPAN
# The form prior to about January 2023 was
# "CrumbStore":{"crumb":"hdDX\u002FHGsZ0Q"}
# The form prior to about May 2024 was
# "RequestPlugin":{"user":{"age":0,"crumb":"8OyCBPyO4ZS"
#
$content =~ /getcrumb.*?"body":"([^"]*)"/
or die "Yahoo daily data: getcrumb not found in parse";
my $crumb = App::Chart::Yahoo::javascript_string_unquote($1);
# header like
# Set-Cookie: B=fab5sl9cqn2rd&b=3&s=i3; expires=Sun, 03-Sep-2018 04:56:13 GMT; path=/; domain=.yahoo.com
#
# Expiry time is +1 year. Who knows whether would really work that long.
#
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
$jar->extract_cookies($resp);
my $cookies_str = $jar->as_string;
App::Chart::Download::verbose_message ("Yahoo new crumb $crumb\n"
. $cookies_str);
return { crumb => $crumb,
cookies => $cookies_str };
}
# $str is an ISO date string like 2017-11-05
unused/YahooOld.pm view on Meta::CPAN
return $str;
}
sub timezone_gmtoffset_at_ymd {
my ($timezone, $year, $month, $day) = @_;
my $timet = $timezone->call(\&POSIX::mktime,
0, 0, 0, $day, $month-1, $year-1900);
my ($sec,$min,$hour,$gmt_day) = gmtime($timet);
return $sec + 60*$min + 3600*$hour + 86400*($gmt_day - $day);
}
# $str is a string from previous HTTP::Cookies ->as_string()
# Return a new HTTP::Cookies object with that content.
sub http_cookies_from_string {
my ($str) = @_;
require File::Temp;
my $fh = File::Temp->new (TEMPLATE => 'chart-XXXXXX',
TMPDIR => 1);
print $fh "#LWP-Cookies-1.0\n", $str or die;
close $fh or die;
require HTTP::Cookies;
my $jar = HTTP::Cookies->new;
$jar->load($fh->filename);
return $jar;
}
# (String quoting for parsing of <script> in HTML to get crumb.)
# undo javascript string backslash quoting in STR, per
#
# https://developer.mozilla.org/en/JavaScript/Guide/Values,_Variables,_and_Literals#String_Literals
#
# Encode::JavaScript::UCS does \u, but not the rest
( run in 1.342 second using v1.01-cache-2.11-cpan-e9199f4ba4c )