looks results from the CPAN

looks

App-Chart

view release on metacpan or search on metacpan


# This file is part of Chart.
#
# Chart is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3, or (at your option) any later version.
#
# Chart is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Chart.  If not, see <http://www.gnu.org/licenses/>.


# Protocols:
#
# Don't know the full state of what Yahoo intends to offer.
# But there's a range of "v7", "v8", "v11" URL forms.  These might be
# meant as data sources for Yahoo client-side viewing software.
#
# Here currently using the v8 method, as it seems to be without
# protocol level hoops.  Presumably it's for personal use and possibly
# after creating a Yahoo account -- even if it works without doing so
# or logging in.
#
#
# Data Format:
#
# Each of these methods can apparently return results in either CSV or
# JSON format.
#
# The JSON form, maybe CSV too, has prices put through
# single-precision floats, ie. 24 bit mantissa, which causes some
# price strings like 123.44999999 instead of 123.45.
# 
# The parse here tries to massage that back to an apparent intended
# number of decimals.  Eg. 2 decimals for dollars and cents, but
# allowing for trading in fractions of a cent which is 3 decimals.
# 
# In daily data download, current day trading in progress shows in
# today's date and changes though the course of trading until being
# fixed maybe at marked close, or maybe the next day.  Don't know
# how pre-market or post-market trading is applied, or what happens
# if a few futures or similar might even be 24 hour trading.
#
#
# Cookies and Crumb:
#
# There's been some protocol hoops to jump through in recent times.
# It seems to be sometimes on the v7 form, maybe always on v11.
# The v7 had seemed fine asking for the latest few days daily data,
# and it's possible lately needs nothing for any amount.
#
# The hoops consist of
#
#     - Fetch one of the finance.yahoo.com web pages to get a
#       HTTP Set-Cookie header.
#     - Maybe answer the ridiculous EU cookie consent on the page.
#       Maybe that depends where your IP looks like it's from.
#     - Look deep within script in that page for a "crumb" string.
#       Or maybe a further "getcrumb" web fetch, but seems result
#       is embedded in the page.
#     - On each data download, HTTP Cookie header, and URL crumb
#       field.
#
# Presumably this is designed as a level of difficulty, to stop the
# past quotes and data that could be had from a single URL (and which
# Yahoo apparently found was widely abused beyond personal use).
#


package App::Chart::Yahoo;
use 5.010;
use strict;
use warnings;
use Carp;
use Date::Calc;
use Date::Parse;
use List::Util qw (min max);
use POSIX ();
use Time::Local;
use URI::Escape;
use Locale::TextDomain ('App-Chart');

use Tie::TZ;
use App::Chart;
use App::Chart::Database;
use App::Chart::Download;
use App::Chart::DownloadHandler;
use App::Chart::DownloadHandler::IndivChunks;
use App::Chart::IntradayHandler;
use App::Chart::Latest;
use App::Chart::Sympred;
use App::Chart::TZ;
use App::Chart::Weblink;

# uncomment this to run the ### lines
use Smart::Comments;

use constant DEBUG => 0;


# .X or .XY or no suffix
our $yahoo_pred = App::Chart::Sympred::Proc->new
  (sub {
     my ($symbol) = @_;
     return ($symbol !~ /\.(FQ|LJ)$/
             && $symbol =~ /[.=]..?$|^[^.]+$/);
   });

my $download_pred = App::Chart::Sympred::Any->new ($yahoo_pred);
our $latest_pred  = App::Chart::Sympred::Any->new ($yahoo_pred);
our $index_pred   = App::Chart::Sympred::Regexp->new (qr/^\^|^0.*\.SS$/);
my $futures_pred  = App::Chart::Sympred::Any->new;

# overridden by specific nodes
App::Chart::setup_source_help
  ($yahoo_pred, __p('manual-node','Yahoo Finance'));

unused/Yahoo-v7.pm view on Meta::CPAN

  return $delay;
}

# exchanges_data() return a hashref of exchange delay data like
#   { '.AX' => 20, '.BI' => 15 }
# which means .AX quotes are delayed by 20 minutes.
#
sub exchanges_data {
  require App::Chart::Pagebits;
  return App::Chart::Pagebits::get
    (name      => __('Yahoo exchanges page'),
     url       => EXCHANGES_URL,
     key       => 'yahoo-quote-delays',
     freq_days => EXCHANGES_UPDATE_DAYS,
     parse     => \&exchanges_parse);
}
sub exchanges_parse {
  my ($content) = @_;
  my $h = {};

  require HTML::TableExtract;
  my $te = HTML::TableExtract->new (headers => ['Suffix', 'Delay']);
  $te->parse($content);
  if (! $te->tables) {
    warn "Yahoo exchanges page unrecognised, assuming 15 min quote delay";
    return $h;
  }

  foreach my $row ($te->rows) {
    my $suffix = $row->[0];
    my $delay = $row->[1];
    next if ($suffix eq 'N/A');

    # eg "15 min"
    # or "15 min**"  with footnote
    #
    if ($delay =~ /^(\d+) min/) {
      $delay = $1;
    } elsif ($delay =~ /real/i) {
      $delay = 0;
    } else {
      warn "Yahoo exchanges page unrecognised delay: \"$delay\"\n";
      next;
    }

    $h->{$suffix} = $delay + 0;
  }
  return $h;
}


#-----------------------------------------------------------------------------
# Info - Share Names
# 
# This uses the info pages like
#
#    https://query2.finance.yahoo.com/v1/finance/search?q=CSCO&enableFuzzyQuery=false
# 
# which is a JSON format of various company information etc.
# 
# The JSON looks like
#     {"explains":[],
#      "count":7,
#      "quotes":[{"exchange":"NMS",
#                 "shortname":"Cisco Systems, Inc.",
#                 "quoteType":"EQUITY",
#                 "symbol":"CSCO",
#                 ...
#                 
# There can be multiple exchanges in the quotes list, use the first.
# Other fields include
# 
#     longname    occasionally shorter than shortname actually
#                 but use shortname
#                 
#     exchDisp    longer display name for the exchange,
#                 eg. exchange=NMS and exchDisp=NASDAQ.
#                 or  exchange=ASX and exchDisp=Australian.
#                 Think the exchange code more useful.
# 
# The URL has an optional &newsCount=0 for no news events, but think
# that's the default anyway.

# FIXME: Disabled temporarily.
# App::Chart::DownloadHandler->new
#   (name         => __('Yahoo info'),
#    key          => 'Yahoo-info',
#    pred         => $download_pred,
#    proc         => \&info_download,
#    recheck_days => 14);

sub info_url {
  my ($symbol) = @_;
  return 'https://query2.finance.yahoo.com/v1/finance/search?q='
    . URI::Escape::uri_escape ($symbol)
    . '&enableFuzzyQuery=false';
}

sub info_download {
  my ($symbol_list) = @_;

  foreach my $symbol (@$symbol_list) {
    App::Chart::Download::status(__('Yahoo info'), $symbol);
    my $url = info_url($symbol);
    my $resp = App::Chart::Download->get ($url);
    my $h = info_parse($resp);
    $h->{'recheck_list'} = [ $symbol ];
    App::Chart::Download::write_daily_group ($h);
  }
}

sub info_parse {
  my ($resp) = @_;
  my $content = $resp->decoded_content (raise_error => 1);
  my @info;
  my $h = { source    => __PACKAGE__,
            info      => \@info };
  require JSON;
  my $J = JSON::from_json($content) // {};
  my $quotes = $J->{'quotes'} // [];
  if (my $e = $quotes->[0]) {

unused/Yahoo-v7.pm view on Meta::CPAN

  my ($symbol, $resp, $h, $hi_tdate) = @_;
  my @data = ();
  $h->{'data'} = \@data;
  my $hi_tdate_iso;
  if (defined $hi_tdate){ $hi_tdate_iso = App::Chart::tdate_to_iso($hi_tdate); }

  my $body = $resp->decoded_content (raise_error => 1);
  my @line_list = App::Chart::Download::split_lines($body);

  unless ($line_list[0] =~ /^Date,Open,High,Low,Close,Adj Close,Volume/) {
    die "Yahoo: unrecognised daily data headings: " . $line_list[0];
  }
  shift @line_list;

  foreach my $line (@line_list) {
    my ($date, $open, $high, $low, $close, $adj_volume, $volume)
      = split (/,/, $line);

    $date = daily_date_fixup ($symbol, $date);
    if (defined $hi_tdate_iso && $date gt $hi_tdate_iso) {
      # Sep 2017: There's a daily data record during the trading day, but
      # want to write the database only at the end of trading.
      ### skip date after hi_tdate ...
      next;
    }

    # Sep 2017 have seen "null,null,null,...", maybe for non-trading days
    foreach my $field ($open, $high, $low, $close, $adj_volume, $volume) {
      if ($field eq 'null') {
        $field = undef;
      }
    }

    if ($index_pred->match($symbol)) {
      # In the past indexes which not calculated intraday had
      # open==high==low==close and volume==0, eg. ^WIL5.  Use the close
      # alone in this case, with the effect of drawing line segments instead
      # of OHLC or Candle figures with no range.

      if (defined $open && defined $high && defined $low && defined $close
          && $open == $high && $high == $low && $low == $close && $volume == 0){
        $open = undef;
        $high = undef;
        $low = undef;
      }

    } else {
      # In the past shares with no trades had volume==0,
      # open==low==close==bid price, and high==offer price, from some time
      # during the day, maybe the end of day.  Zap all the prices in this
      # case.
      #
      # For a public holiday it might be good to zap the volume to undef
      # too, but don't have anything to distinguish holiday, suspension,
      # delisting vs just no trades.
      #
      # On the ASX when shares are suspended the bid/offer can be crossed as
      # usual for pre-open auction, and this gives high<low.  For a part-day
      # suspension then can have volume!=0 in this case too.  Don't want to
      # show a high<low, so massage high/low to open/close range if the high
      # looks like a crossed offer.

      if (defined $high && defined $low && $high < $low) {
        $high = App::Chart::max_maybe ($open, $close);
        $low  = App::Chart::min_maybe ($open, $close);
      }

      if (defined $open && defined $low && defined $close && defined $volume
          && $open == $low && $low == $close && $volume == 0) {
        $open  = undef;
        $high  = undef;
        $low   = undef;
        $close = undef;
      }
    }

    push @data, { symbol => $symbol,
                  date   => $date,
                  open   => crunch_trailing_nines($open),
                  high   => crunch_trailing_nines($high),
                  low    => crunch_trailing_nines($low),
                  close  => crunch_trailing_nines($close),
                  volume => $volume };
  }
  return $h;
}
sub daily_parse_div {
  my ($symbol, $resp, $h) = @_;
  my @dividends = ();
  $h->{'dividends'} = \@dividends;

  my $body = $resp->decoded_content (raise_error => 1);
  my @line_list = App::Chart::Download::split_lines($body);

  # Date,Dividends
  # 2015-11-04,1.4143
  # 2016-05-17,1.41428
  # 2017-05-16,1.4143
  # 2016-11-03,1.4143

  unless ($line_list[0] =~ /^Date,Dividends/) {
    warn "Yahoo: unrecognised dividend headings: " . $line_list[0];
    return;
  }
  shift @line_list;

  foreach my $line (@line_list) {
    my ($date, $amount) = split (/,/, $line);

    push @dividends, { symbol  => $symbol,
                       ex_date => daily_date_fixup ($symbol, $date),
                       amount  => $amount };
  }
  return $h;
}
sub daily_parse_split {
  my ($symbol, $resp, $h) = @_;
  my @splits = ();
  $h->{'splits'} = \@splits;

  my $body = $resp->decoded_content (raise_error => 1);

( run in 1.247 second using v1.01-cache-2.11-cpan-97f6503c9c8 )