view release on metacpan or  search on metacpan

lib/App/Chart/Suffix/TGE.pm  view on Meta::CPAN

# used when that's all that's needed to update (which will be the case if
# you're updating every day).
#
# The zip is about 35kbytes and has the same most recent day as the csv,
# plus the past 12 months.  The filename is a date range
# (XX01yymmddyymmdd.zip), but there's only one file, you can't get an
# arbitrary range.
#
# The zip filename is formed from today's date, or a day or two earlier,
# but if we don't hit it that way there's a fallback to fetch and parse the
# actual download page.  The pattern in the filename is pretty clear, so
# perhaps that fallback is unnecessary.
#
# There's an ETag / Last-Modified on the year file, but it's hardly needed
# since the dates are in the name.

App::Chart::DownloadHandler->new
  (name   => __('TGE'),
   pred   => $pred,
   proc   => \&download,
   backto => undef,
   available_tdate => \&available_tdate,
   by_commodity    => 1);

sub download {
  my ($symbol_list) = @_;
  download_day ($symbol_list)
    || download_year ($symbol_list);

  # (download-name-from-latest (_ "TGE") symbol-list))
}

# return tdate for available download data
# the .csv downloads have been seen with a Last-Modified headers
#     Wed, 24 Aug 2005 01:00:20 GMT      == 10am tokyo
#     Thu, 01 Sep 2005 22:33:05 GMT      ==  7am tokyo
# so try at 10:05am tokyo
#
sub available_tdate {
  App::Chart::Download::weekday_tdate_after_time
      (10,5, App::Chart::TZ->tokyo, -1);
}

# do a download of the last day .csv file for $symbol_list, if that would
# update all those
# return true if updated successfully, or false if not (should use zip instead)
# 
sub download_day {
  my ($symbol_list) = @_;

  my $commodity = App::Chart::symbol_commodity ($symbol_list->[0]);
  my $avail_tdate = available_tdate();
  my $start_tdate = App::Chart::Download::start_tdate_for_update(@$symbol_list);

  # use csv if all of $symbol_list just wanting $avail_tdate
  if ($start_tdate < $avail_tdate) {
    return 0;
  }
  my $filename = "\L$commodity\E01.csv";
  my $url = 'http://www.tge.or.jp/data/down_load/'
    . URI::Escape::uri_escape ($filename);

  App::Chart::Download::status (__x('TGE data {filename}',
                                   filename => $filename));
  my $resp = App::Chart::Download->get ($url,
                                       url_tags_key => 'TGE-day');
  if (! $resp->is_success) {
    # not modified, no new data
    return 1;
  }
  my $got_tdate = csv_tdate ($resp);
  if ($got_tdate == $start_tdate) {
    # got the expected data, process it
    my $content = $resp->decoded_content (charset => 'none');
    my $h = csv_parse ($content);
    $h->{'url_tags_key'} = 'TGE-day';
    return 1;
  } elsif ($got_tdate < $avail_tdate) {
    # got something older, there's no new data
    return 1;
  } else {
    return 0;
  }
}

sub download_year {
  my ($symbol_list) = @_;
  my $commodity = App::Chart::symbol_commodity ($symbol_list->[0]);
  my $avail_tdate = available_tdate();


  # try 0, -1, -2 direct filenames, then finally download page
  # (allowing for new COMM_NUM values too)
  #
  download_year_attempt (download_tdate_url ($commodity, $avail_tdate))
    || download_year_attempt (download_tdate_url ($commodity, $avail_tdate - 1))
      || download_year_attempt (download_tdate_url ($commodity, $avail_tdate - 2))
        || download_year_attempt (download_page ($commodity));
}

# return true if successful
sub download_year_attempt {
  my ($commodity, $url) = @_;

  $url =~ m{/([^/]+)$/};
  my $filename = $1;
  App::Chart::Download::status (__x('TGE data {filename}',
                                   filename => $filename));

  my $resp = App::Chart::Download->get ($url, allow_404 => 1);
  if (! $resp->is_success) { return 0; }

  # got the expected data, process it
  my $h = zip_parse ($resp);
  $h->{'url_tags_key'} = 'TGE-day';
  return 1;
}

# eg. http://www.tge.or.jp/data/down_load/co01040610050609.zip
#     for Jun/10/2004 - Jun/09/2005
sub download_tdate_url {