API-Docker
view release on metacpan or search on metacpan
lib/API/Docker/Role/HTTP.pm view on Meta::CPAN
package API::Docker::Role::HTTP;
# ABSTRACT: HTTP transport role for Docker Engine API
our $VERSION = '0.004';
use Moo::Role;
use IO::Socket::UNIX;
use IO::Socket::INET;
# For the sysread method on a plain filehandle: _pull calls it as a method so
# that IO::Socket::SSL's own gets picked up rather than the builtin. See _pull.
use IO::Handle;
use Socket qw( SOL_SOCKET SO_RCVTIMEO );
# How a read that delivered nothing says it ran out of time rather than out of
# stream (EAGAIN/EWOULDBLOCK), and how it says it was interrupted rather than
# either (EINTR). See _pull and _timed_out.
use Errno qw( EAGAIN EWOULDBLOCK EINTR );
use JSON::MaybeXS qw( encode_json decode_json );
use Scalar::Util qw( looks_like_number );
use Path::Tiny;
use Carp qw( croak shortmess );
use Log::Any qw( $log );
use API::Docker::Error::HTTP;
use API::Docker::Error::Stream;
use API::Docker::Error::Timeout;
use API::Docker::Error::Truncated;
use namespace::clean;
requires 'host';
requires 'api_version';
requires 'tls';
requires 'cert_path';
requires 'tls_insecure';
# Docker stream frame types, indexed by the first byte of the frame header.
my @STREAM_TYPE = qw( stdin stdout stderr );
# A field name is an RFC 9110 token and nothing else. Anything outside this
# set -- CR, LF, a space, a colon -- is rejected rather than stripped; see
# _assert_header_name.
my $HEADER_NAME = qr/\A[0-9A-Za-z!#\$%&'*+.^_`|~-]+\z/;
# The request-target path is caller data -- a container name, an image
# reference -- spliced straight into the request line as /v$version$path, so a
# byte the line's own grammar reads rewrites the request rather than naming a
# resource: CR or LF ends the line, a space opens the HTTP-version field, and
# a ? or # opens the query or fragment. It is held to the RFC 3986 origin-form
# path character set -- unreserved, the sub-delims, and : @ % / -- and
# rejected, not sanitised, for the reason a header name is (see
# _assert_request_path). Query parameters carry the ? and everything after it
# and are assembled separately below, each element run through _uri_encode.
my $REQUEST_PATH = qr{\A[A-Za-z0-9\-._~:/\@!\$&'()*+,;=%]*\z};
# The three units a response can be cut into, one option each. A request picks
# one of them, or none and gets the buffered path; see _stream_handler.
my @STREAM_OPTION = qw( on_event on_frame on_chunk );
# What a response body has to start with to be worth handing to decode_json.
# An object or an array is not the whole of JSON: the engine answers several
# endpoints with a bare JSON scalar, and a `null` used to come back as the
# four-character string 'null'. See _request.
my $JSON_BODY = qr/\A\s*(?:[\[\{"]|-?[0-9]|true|false|null)/;
# How much is asked for per sysread. Strictly an upper bound -- sysread
# returns what has arrived rather than filling to it (see _pull), so on a live
# feed a call typically comes back with one burst, and asking for 64K costs
# nothing but the size of the buffer it lands in.
my $READ_SIZE = 64 * 1024;
has read_timeout => (
is => 'ro',
);
has connect_timeout => (
is => 'ro',
);
has _socket => (
is => 'lazy',
clearer => '_clear_socket',
);
# The connect timeout and the endpoint it belongs to, on their way to
# _build__socket. It is a lazy builder and so cannot be handed an argument,
# and the value is per request rather than per client -- hence rw, with
# _reconnect as the only writer, setting it immediately before the build and
# clearing it immediately after. Unset is the whole of the old behaviour: no
# Timeout on any constructor, and the plain croak on a failure.
has _pending_connect => (
is => 'rw',
init_arg => undef,
);
sub _build__socket {
my ($self) = @_;
my $host = $self->host;
my $pending = $self->_pending_connect;
my $timeout = $pending ? $pending->{timeout} : undef;
if ($host =~ m{^unix://(.+)$}) {
my $path = $1;
$log->debugf("Connecting to Unix socket: %s", $path);
my $sock = IO::Socket::UNIX->new(
Peer => $path,
Type => SOCK_STREAM,
$timeout ? (Timeout => $timeout) : (),
);
unless ($sock) {
# Asked before anything else can touch $@ or $!, which is the whole of
# the evidence; see _connect_expired.
$self->_croak_connect_timeout($pending, 'unix://' . $path)
if $self->_connect_expired($timeout);
croak "Cannot connect to Unix socket $path: $!";
}
return $sock;
}
elsif ($host =~ m{^tcp://([^:]+):(\d+)$}) {
my ($addr, $port) = ($1, $2);
unless ($self->tls) {
lib/API/Docker/Role/HTTP.pm view on Meta::CPAN
# Same resolution, same reason.
my $connect_timeout = $self->_connect_timeout_value(
exists $opts{connect_timeout}
? $opts{connect_timeout} : $self->connect_timeout);
# The endpoint without its query string, for the same reason the >= 400
# croak uses that form.
my $ctx = { endpoint => $endpoint, timeout => $timeout };
my $sock = $self->_reconnect(
{ timeout => $connect_timeout, endpoint => $endpoint });
# Applied here rather than in _build__socket because the value is per
# request, not per client: a socket is opened and closed for each one, so
# this is the connection the option belongs to.
$self->_apply_read_timeout($sock, $timeout);
print $sock $request;
# Reading can croak, and now does so from further in than it used to: an
# on_event stream raises Error::Stream at the event that reports the
# failure, and an on_frame stream refuses a header that is not one. Without
# the eval those exceptions leave the socket open until the next request
# replaces it, so it is closed on the way out either way and the exception
# re-raised unchanged.
my $response;
my $ok = eval { $response = $handler
? $self->_read_streaming_response($sock, $method, $handler, $ctx)
: $self->_read_response($sock, $method, $ctx); 1 };
my $err = $@;
close $sock;
$self->_clear_socket;
die $err unless $ok;
my ($status_code, $status_text, $headers, $body, $summary) = @$response;
$log->debugf("Response: %s %s", $status_code, $status_text);
# The status line and the response headers are metadata the return value
# cannot carry: it is the decoded body and nothing else, so 204 and 304 are
# both undef and a header holding the payload -- HEAD
# /containers/{id}/archive answers with an empty body and
# X-Docker-Container-Path-Stat -- is unreachable. They go into a hash the
# caller supplies, so no existing caller's return shape changes. Filled
# before the croak below, so an eval'ing caller can still read the status.
if (my $out = $opts{response}) {
%$out = (
status => $status_code,
reason => $status_text,
headers => $headers,
);
}
if ($status_code >= 400) {
my $error_msg = $body;
my $data;
if ($body && $body =~ /^\s*[\{\[]/) {
eval {
$data = decode_json($body);
# Docker answers with {"message":...}. Podman answers a failed push
# with the stream shape instead -- {"errorDetail":{"message":...},
# "error":...} and no message key at all -- so without these two
# fallbacks the whole JSON object became the croak text (karr k13).
my $detail = ref $data->{errorDetail} eq 'HASH'
? $data->{errorDetail}{message} : undef;
$error_msg = $data->{message} // $detail // $data->{error} // $body;
};
}
# Carp hands a reference straight back rather than decorating it, so this
# croak is a die with an object -- hence the location captured by hand,
# which names the same frame a croak of a plain string would have named.
# message . location is byte for byte what the string croak produced,
# newline-terminated engine messages included: Carp appends the suffix
# after the newline rather than skipping it (karr k50).
my $error = API::Docker::Error::HTTP->new(
message => "Docker API error ($status_code): $error_msg",
location => shortmess(''),
status => $status_code,
reason => $status_text,
body => $body // '',
data => $data,
);
croak $error;
}
# A streamed request has handed every unit to the callback already and kept
# none of them, so there is no body left to decode and return. What the
# caller cannot know otherwise is how the stream ended, and that is what
# comes back instead.
return $summary if $summary;
# Zero bytes is a different answer in each shape a request can ask for, so
# the two options that promise one are answered before the empty-body check
# rather than after it. `raw` promises the response bytes and a body of no
# bytes is '', which a caller can take length() of; `ndjson` promises an
# ArrayRef of events even for a stream carrying a single object, so a stream
# that carried none is []. Returning undef for both broke each promise
# exactly where the engine legitimately says nothing.
$body = '' unless defined $body;
# The framed endpoints (logs, attach, exec/start) carry arbitrary bytes
# that must not be mistaken for JSON -- a TTY container printing a JSON
# line would otherwise come back decoded.
return $body if $opts{raw};
# Streaming endpoints (/build, /images/create, /images/*/push) always
# return an ArrayRef of events, even when the stream carried exactly one
# object. See _decode_stream.
if ($opts{ndjson}) {
my $events = $self->_decode_stream($body);
# A failed build, pull or push is HTTP 200 with the failure buried in the
# stream, so the status line above cannot catch it. Opt out for a stream
# whose objects are engine data rather than the outcome of one operation.
$self->_assert_no_stream_error($endpoint, $events)
if $opts{croak_on_error} // 1;
return $events;
}
# Nothing was asked of the body's shape and there is no body, so there is
# nothing to hand back. 204 says so in the status line and is taken at its
# word even if bytes follow it.
return undef if $status_code == 204 || $body eq '';
# A body that is JSON is decoded, whichever JSON value it is. The guard was
# `{` or `[` alone, which returned a body that is a bare JSON scalar as its
# own bytes: `null` came back as the four-character string 'null'. The
# engine sends exactly that where a Go nil slice or pointer is the whole
# response -- GET /plugins/privileges for a plugin that demands nothing,
# GET /containers/{id}/changes for a container that changed nothing -- and
# the string is neither the ArrayRef those endpoints document nor anything
# a caller can iterate.
#
# The eval decides, not the pattern: a plain-text body that happens to
# start with one of these characters fails to decode and is returned as
# itself. So must the eval's success, not its result -- decode_json('null')
# is a successful decode to undef.
if ($body =~ $JSON_BODY) {
my $decoded;
return $decoded if eval { $decoded = decode_json($body); 1 };
}
return $body;
}
sub _decode_stream {
my ($self, $body) = @_;
# Newline-delimited JSON: one object per line. A literal newline cannot
# occur inside a JSON string, so splitting on lines is safe.
my @events;
for my $line (split /\r?\n/, $body) {
next unless $line =~ /\S/;
my $event = eval { decode_json($line) };
push @events, $event if defined $event;
}
# Fall back to the whole body for a stream that is not newline-framed
# (a single pretty-printed object), so nothing is silently dropped.
unless (@events) {
my $event = eval { decode_json($body) };
push @events, $event if defined $event;
}
return \@events;
}
sub _assert_no_stream_error {
my ($self, $endpoint, $events) = @_;
for my $event (@$events) {
next unless ref $event eq 'HASH';
my $detail = $event->{errorDetail};
next unless defined $detail;
# errorDetail is a HashRef carrying the message. The engine sends a flat
# `error` next to it with the same text; that is the fallback, not the
# trigger -- the trigger is errorDetail, and nothing else.
my $reason = ref $detail eq 'HASH' ? $detail->{message} : undef;
$reason = $event->{error} unless defined $reason && length $reason;
$reason = 'no message given' unless defined $reason && length $reason;
# Engine messages end in a newline, and Carp appends no location to a
# message that already does.
$reason =~ s/\s+\z//;
# Carp hands a reference straight back rather than decorating it, so this
# croak is a die with an object -- hence the location captured by hand,
# which names the same frame a croak of a plain string would have named.
# The object goes into a variable first: `croak CLASS->new(...)` is
# indirect object syntax and parses as CLASS->croak(new(...)).
my $error = API::Docker::Error::Stream->new(
message => 'Docker API stream error (' . $endpoint . '): ' . $reason,
events => $events,
location => shortmess(''),
);
croak $error;
}
return;
}
sub _assert_header_name {
my ($self, $name) = @_;
return if defined $name && $name =~ $HEADER_NAME;
my $display = defined $name ? $name : '';
$display =~ s/([^\x20-\x7E])/sprintf('\\x%02X', ord $1)/ge;
croak __PACKAGE__ . '->_request invalid header name "' . $display . '": a '
. 'header name must be an RFC 9110 token (letters, digits and '
lib/API/Docker/Role/HTTP.pm view on Meta::CPAN
# daemon closes the connection at best, and forever if it does not. So the
# body is not read for HEAD, whatever the headers promise.
return '' if defined $method && uc($method) eq 'HEAD';
if ($headers->{'transfer-encoding'}
&& lc($headers->{'transfer-encoding'}) eq 'chunked') {
return $self->_read_chunked($sock, $ctx);
}
if (defined $headers->{'content-length'}) {
my $len = $self->_assert_content_length($ctx, $headers->{'content-length'});
return '' unless $len > 0;
my $body = '';
# What a timeout hands over instead of dropping: see
# API::Docker::Error::Timeout/partial. localised so the context goes back
# to carrying nothing once this body is done with.
local $ctx->{partial} = \$body;
# An announced length is a promise, and a stream that ends before it is
# kept is truncation rather than the end of the body; see _read_exact.
$self->_read_exact($sock, $len, \$body, $ctx, 'content-length',
'the body');
return $body;
}
# Read until the daemon closes. This used to be a `local $/; <$sock>` slurp;
# it is a loop over the same primitive as the other two branches now, which
# is what karr k60 needed and what the timeout wanted anyway -- with $/ undef
# a whole body and a truncated one are both just bytes, so the slurp's own
# result could never say which it was. This is the path karr k52's hang is
# on, an attach whose buffered frames arrive and whose socket then never
# closes.
#
# And the one shape with no completeness check to make: the response
# announced no end, so the close IS the end (karr k64). Treating an EOF here
# as truncation would make every attach, every logs(follow) and every
# exec/start fail on the daemon hanging up, which is how all three finish.
my $body = '';
# What a timeout hands over instead of dropping; see the content-length
# branch above.
local $ctx->{partial} = \$body;
while (1) {
my ($n, $buf) = $self->_read_bytes($sock, $READ_SIZE, $ctx);
last unless $n;
$body .= $buf;
}
return $body;
}
# The incremental sibling of _read_response. Same [status, reason, headers,
# body] shape, plus a fifth element: the summary of what the callback was
# handed. The body it returns is empty -- that is the point, nothing is kept --
# except on the two paths that fall back to reading whole, which return undef
# as the summary instead so _request treats them exactly as before.
sub _read_streaming_response {
my ($self, $sock, $method, $handler, $ctx) = @_;
$ctx ||= {};
my ($status_code, $status_text, $headers) = @{ $self->_read_head($sock, $ctx) };
# Neither of these is a stream. A >= 400 body is a short JSON object naming
# the failure and _request has to croak with it, so it is read whole and the
# callback never sees it; a HEAD response has no body at all.
if ($status_code >= 400 || (defined $method && uc($method) eq 'HEAD')) {
return [$status_code, $status_text, $headers,
$self->_read_body($sock, $headers, $method, $ctx), undef];
}
# Set only here, past the two branches above, so a timeout while reading an
# error body is still reported in bytes rather than in units nothing
# delivered. From this point on an expiry carries the callback's own summary
# instead: the units are with the caller already, and how many is the part it
# cannot know otherwise.
local $ctx->{summary} = $handler->{summary};
my $feed = $handler->{feed};
my $more = 1;
# There is deliberately nothing here to hand the callback the bytes of the
# read that expires. karr k59 needed that hook because PerlIO's read() could
# come back with data *and* EAGAIN at once, so a stall could land with the
# whole response read and none of it fed; sysread cannot (karr k60). Every
# byte reaches the callback in the pull that delivered it, and the pull that
# expires delivers none -- so the property k59 established now holds by
# construction instead of by rescue.
if ($headers->{'transfer-encoding'}
&& lc($headers->{'transfer-encoding'}) eq 'chunked') {
while ($more) {
my $chunk_header = $self->_read_line($sock, $ctx);
my $chunk_size = $self->_assert_chunk_header($ctx, $chunk_header);
last if $chunk_size == 0;
my $read = 0;
while ($read < $chunk_size) {
my ($n, $buf) = $self->_read_bytes($sock, $chunk_size - $read, $ctx);
last unless $n;
$read += $n;
# Fed per read rather than per completed chunk. A chunk is the
# daemon's framing, not the caller's -- the engine is free to send an
# hour of log output as one chunk -- so waiting for the whole of one
# would reintroduce exactly the buffering this path exists to avoid.
$more = $feed->($buf);
last unless $more;
}
# Every truncation check on this path is guarded by $more, and that is
# the whole of what distinguishes the two ways a streamed chunk ends
# early: the daemon ran out, or the callback said stop. A caller that
# stopped left the rest of the chunk unread on purpose (karr k64).
$self->_croak_truncated($ctx, phase => 'chunk-data', piece => 'a chunk',
expected => $chunk_size, received => $read)
if $more && $read < $chunk_size;
last unless $more;
# The CRLF that terminates the chunk data. Skipped when the caller
# stopped mid-chunk: the socket is closed straight after, and the
# remaining bytes of that chunk are still unread in front of it.
$self->_assert_chunk_terminator($ctx, $self->_read_line($sock, $ctx));
}
}
elsif (defined $headers->{'content-length'}) {
my $len = $self->_assert_content_length($ctx, $headers->{'content-length'});
my $read = 0;
while ($more && $read < $len) {
my $want = $len - $read;
$want = $READ_SIZE if $want > $READ_SIZE;
my ($n, $buf) = $self->_read_bytes($sock, $want, $ctx);
last unless $n;
$read += $n;
$more = $feed->($buf);
}
$self->_croak_truncated($ctx, phase => 'content-length',
piece => 'the body', expected => $len, received => $read)
if $more && $read < $len;
}
else {
while ($more) {
my ($n, $buf) = $self->_read_bytes($sock, $READ_SIZE, $ctx);
last unless $n;
$more = $feed->($buf);
}
}
# Only a stream the daemon ended has a tail worth flushing, or a leftover
# worth complaining about. One the caller stopped has bytes in the carry
# buffer by construction, and treating those as truncation would turn every
# early stop into an error.
$handler->{finish}->() unless $handler->{stopped}->();
return [$status_code, $status_text, $headers, '', $handler->{summary}->()];
}
# One unit per call, and the unit is whichever of the three the caller asked
# for. The engine's streaming endpoints do not share one: /events and the
# build/pull/push progress streams are newline-delimited JSON, logs and
# exec/start are 8-byte-framed, and an image export is bytes with no structure
# above them at all. Forcing one unit on all three would mean handing two of
# them back undecoded and calling it streaming.
#
# The three decoders differ only in how they cut the byte stream up; the carry
# buffer, the delivery and the stop handling below are common to all of them.
sub _stream_handler {
my ($self, $endpoint, $option, $cb, $croak_on_error) = @_;
my $carry = '';
my $delivered = 0;
my $stopped = 0;
# Stopping is an explicit call, not a return value, and the callback's
# return value is deliberately never looked at. Every truthiness convention
# has a silent failure mode here: `push @got, $_[0]` returns a count and
# `$last = $event->{status}` returns whatever the engine said -- and a
# container event's status is literally 'stop'. Both would end the stream by
# accident and hand back a truncated one with no diagnostic. A closure the
# caller has to invoke cannot be produced by accident.
my $stop = sub { $stopped = 1; return };
my $deliver = sub {
my ($unit) = @_;
$delivered++;
$cb->($unit, $stop);
return !$stopped;
};
my ($feed, $finish);
if ($option eq 'on_chunk') {
# No carry: the bytes as they arrive are the unit, so there is no boundary
# to reassemble across.
$feed = sub {
my ($bytes) = @_;
return 1 unless defined $bytes && length $bytes;
return $deliver->($bytes);
};
$finish = sub { return };
}
elsif ($option eq 'on_event') {
my $emit_line = sub {
my ($line) = @_;
$line =~ s/\r\z//;
return 1 unless $line =~ /\S/;
my $event = eval { decode_json($line) };
return 1 unless defined $event;
# Checked per event rather than over the finished list, so a failed
# build croaks at the event that reports it instead of when the daemon
# eventually closes. The Error::Stream then carries that one event: a
# callback stream keeps no history, having been given all of it already.
$self->_assert_no_stream_error($endpoint, [$event]) if $croak_on_error;
return $deliver->($event);
};
$feed = sub {
my ($bytes) = @_;
$carry .= $bytes;
# A JSON string cannot contain a literal newline, so a newline in the
# buffer always ends an event -- and everything after the last one is
# an event still arriving, which stays in the carry for the next read.
while ((my $idx = index($carry, "\n")) >= 0) {
my $line = substr($carry, 0, $idx, '');
substr($carry, 0, 1, '');
return 0 unless $emit_line->($line);
}
return 1;
};
$finish = sub {
# A last event with no trailing newline is a complete event, not a
# truncated one: the daemon closing is what ended it.
return unless length $carry;
my $line = $carry;
$carry = '';
$emit_line->($line);
return;
};
}
else {
$feed = sub {
my ($bytes) = @_;
$carry .= $bytes;
while (length($carry) >= 8) {
my ($type, $pad1, $pad2, $pad3, $size) = unpack 'C4 N', substr($carry, 0, 8);
croak __PACKAGE__ . '->_request on_frame: not a framed stream (header '
. 'byte 0 is ' . $type . ', bytes 1-3 are ' . $pad1 . '/' . $pad2
. '/' . $pad3 . '). A callback stream cannot sniff its own framing '
. 'the way the buffered path does -- that needs the whole body, '
. 'which is what is not being kept. Declare an unframed stream with '
. 'tty => 1'
if $type > $#STREAM_TYPE || $pad1 || $pad2 || $pad3;
# The header is complete but the payload is not yet: leave the whole
# frame in the carry and wait for the rest of it. This is the case a
# per-chunk reader gets wrong -- an 8-byte header can be split across
# two chunks just as easily as a payload can.
last if length($carry) < 8 + $size;
my $frame = {
stream => $STREAM_TYPE[$type],
data => substr($carry, 8, $size),
};
substr($carry, 0, 8 + $size, '');
return 0 unless $deliver->($frame);
}
return 1;
};
$finish = sub {
return unless length $carry;
croak __PACKAGE__ . '->_request on_frame: the daemon closed mid-frame, '
. 'leaving ' . length($carry) . ' bytes that do not complete one';
};
}
return {
feed => $feed,
finish => $finish,
stopped => sub { $stopped },
summary => sub { { delivered => $delivered, stopped => $stopped ? 1 : 0 } },
};
}
lib/API/Docker/Role/HTTP.pm view on Meta::CPAN
stream => $STREAM_TYPE[$type],
data => substr($body, $pos + 8, $size),
};
$pos += 8 + $size;
}
return undef unless @frames;
return \@frames;
}
1;
__END__
=pod
=encoding UTF-8
=head1 NAME
API::Docker::Role::HTTP - HTTP transport role for Docker Engine API
=head1 VERSION
version 0.004
=head1 SYNOPSIS
package MyDockerClient;
use Moo;
has host => (is => 'ro', required => 1);
has api_version => (is => 'ro');
has tls => (is => 'ro', default => 0);
has cert_path => (is => 'ro');
has tls_insecure => (is => 'ro', default => 0);
with 'API::Docker::Role::HTTP';
# Now use get, post, put, delete_request, head methods
my $data = $self->get('/containers/json');
=head1 DESCRIPTION
This role provides HTTP transport for the Docker Engine API. It implements
HTTP/1.1 communication over Unix sockets and TCP sockets without depending on
heavy HTTP client libraries like LWP.
Features:
=over
=item * Unix socket transport (C<unix://...>)
=item * TCP socket transport (C<tcp://host:port>), in the clear or over TLS
with client certificates (L</"TLS on a tcp:// connection">)
=item * HTTP/1.1 chunked transfer encoding
=item * Automatic JSON encoding/decoding
=item * Newline-delimited JSON event streams (C<< ndjson => 1 >>), including
the failures the engine reports inside an HTTP 200 body
=item * Demultiplexing of the Docker stream format (L</stream_frames>)
=item * Incremental delivery of a response through a per-request callback, so
the endpoints that never close are usable at all (L</"Streaming a response as
it arrives">)
=item * Request/response logging via L<Log::Any>
=item * Automatic connection management
=back
Consuming classes must provide C<host>, C<api_version>, C<tls>, C<cert_path>
and C<tls_insecure> attributes. The last three are read only by the C<tcp://>
branch of the socket builder, and only when TLS is asked for, but the contract
is stated once rather than probed for at connect time.
A C<unix://> connection is a local socket with no wire to protect and is never
encrypted; it ignores all three attributes, and L<API::Docker> refuses the
combination at construction rather than letting a request for an encrypted
transport be answered with an unencrypted one. A C<tcp://> connection is
B<plaintext unless C<< tls => 1 >>>, which is the whole of the difference --
see L</"TLS on a tcp:// connection">.
=head2 TLS on a tcp:// connection
C<< tls => 1 >> replaces the L<IO::Socket::INET> connection with an
L<IO::Socket::SSL> one and changes nothing else: the same request writer, the
same reader, the same everything above the socket.
my $docker = API::Docker->new(
host => 'tcp://dockerhost:2376',
tls => 1,
cert_path => '/home/me/.docker',
);
=head3 What the certificates are, and where
C<cert_path> names a directory in the layout the C<docker> CLI writes, and
each of the three files is used if it is there:
=over
=item * F<ca.pem> - the trust anchor the daemon's certificate is checked
against
=item * F<cert.pem> and F<key.pem> - this client's certificate and private
key, sent when the daemon asks the client to identify itself
=back
The two halves of the client certificate go together: one of them present
without the other is a croak, because a key with no certificate proves nothing
and a certificate with no key cannot be used. A directory holding only
F<ca.pem> is fine -- that is a daemon this client verifies but does not
authenticate to. A C<cert_path> that names nothing is a croak: it is read only
once TLS was asked for, and at that point a path pointing nowhere means the
caller believes certificates are in use that are not.
lib/API/Docker/Role/HTTP.pm view on Meta::CPAN
hand. The better answer to that is nearly always F<ca.pem>: a self-signed
certificate is its own CA and can be used as the anchor directly.
=head3 The dependency
L<IO::Socket::SSL> is a B<recommended>, not a required, dependency, and it is
loaded at the moment the first TLS connection is opened. It brings in
L<Net::SSLeay>, which is XS compiled against libssl, and the C<unix://>
transport -- local Docker, rootless Podman, the default -- never needs any of
it; requiring it would make this client unbuildable on a machine with no
OpenSSL headers for the sake of a transport it is not using. Without it,
C<< tls => 1 >> croaks naming the module and how to install it, at the same
point every other connection failure is reported.
=head2 read_timeout
Seconds of silence after which a request gives up and croaks with an
L<API::Docker::Error::Timeout>. C<undef> -- the default, and what every
existing caller gets -- means no timeout at all and is the behaviour this
distribution has always had. C<0> means the same and is the way to say it
explicitly, so a client carrying a default can be opted out of per request.
my $docker = API::Docker->new(read_timeout => 30);
$docker->system->using(read_timeout => 0)->events; # this one may wait
Per request it is an option of L</get>, L</post>, L</put>, L</delete_request>
and L</head>. A resource class carries it through
L<API::Docker::Role::Using/using>, which clones the class rather than taking
it per method -- up for a slow endpoint, down for a stream that should not
stall, off with C<0>.
See L</"Bounding a request that never ends"> for what it does and does not
cover, and L<API::Docker/"What a timeout covers"> for the same question
asked of both bounds at once.
=head2 connect_timeout
Seconds after which opening the connection gives up and croaks with an
L<API::Docker::Error::Timeout> whose C<< ->phase >> is C<'connect'>. C<undef>
-- the default, and what every existing caller gets -- means no bound and is
the behaviour this distribution has always had; C<0> means the same and is the
way to say it explicitly.
my $docker = API::Docker->new(connect_timeout => 5);
$docker->system->using(connect_timeout => 0)->version; # may wait
Separate from L</read_timeout> rather than folded into it, because the two
bound different things and want different numbers: a connect is either
immediate or broken, while a read is waiting on work the daemon has to do.
Per request it is an option of L</get>, L</post>, L</put>, L</delete_request>
and L</head>. A resource class carries it through
L<API::Docker::Role::Using/using>. See L</"Bounding the connection itself">
for what it does on each transport, which is not the same thing on all
three.
=head2 get
my $data = $client->get($path, %opts);
Perform HTTP GET request. Returns decoded JSON or raw response body.
Options:
=over
=item * C<params> - HashRef of query parameters; a HashRef value is JSON-encoded
=item * C<headers> - HashRef of extra HTTP headers, e.g.
C<< { 'X-Registry-Auth' => $b64 } >>
=item * C<ndjson> - Parse the body as newline-delimited JSON and always
return an ArrayRef of events, even for a stream carrying a single object.
Named for the format rather than C<stream>, which is already a query
parameter of C</events> and C</containers/{id}/stats>. An C<errorDetail>
event in such a stream croaks; see L</"Failure inside a 200 response">
=item * C<croak_on_error> - Default true, and only consulted with
C<< ndjson => 1 >>. Set it false for a stream whose objects are engine data
rather than the outcome of one operation -- C</events> is the only such
endpoint here
=item * C<raw> - Never decode the body; return the response bytes verbatim
=item * C<response> - HashRef the status line and the response headers are
written into; see L</"Reading the status line and the response headers">
=item * C<on_event>, C<on_frame>, C<on_chunk> - CodeRef called with each unit
of the response as it arrives, instead of the body being buffered and
returned. At most one of the three; see L</"Streaming a response as it
arrives">
=item * C<read_timeout> - Seconds of silence after which this request gives up
and croaks with an L<API::Docker::Error::Timeout>. Overrides the
L</read_timeout> attribute; C<0> means no timeout. See L</"Bounding a request
that never ends">
=item * C<connect_timeout> - Seconds after which opening the connection gives
up and croaks with an L<API::Docker::Error::Timeout> whose C<< ->phase >> is
C<'connect'>. Overrides the L</connect_timeout> attribute; C<0> means no
bound. See L</"Bounding the connection itself">
=item * C<headers> names are validated, not sanitised; see
L</"Header names are rejected, header values are stripped">
=back
=head2 Bounding a request that never ends
Nothing above stops a request waiting forever. C<Connection: close> asks the
daemon to hang up when it is done, and the readers wait for that -- so a
daemon that has nothing more to send and does not hang up leaves the client
blocked with no way out. That is not hypothetical: attaching to a container
that has B<already exited> answers, delivers the buffered frames and then
holds the connection open indefinitely on rootless Podman (karr k52), and
C</containers/{id}/stats> opened on a running container does not end when that
container exits on Docker -- it degrades into zero-filled readings and keeps
going (karr k59).
L</read_timeout> bounds that:
# Give up after two seconds of silence rather than waiting forever.
my $frames = $docker->containers->using(read_timeout => 2)->attach($id);
=head3 It is an idle timeout, not a deadline
The clock measures the time since the last byte arrived, not the time since
the request started. A stream that keeps producing runs as long as it likes;
one that stops producing is cut off. That distinction is the whole point --
both hangs above deliver data first and stall afterwards, so a bound on the
total time would have to be set longer than any legitimate stream, and a bound
on the time to the first byte would never fire at all.
lib/API/Docker/Role/HTTP.pm view on Meta::CPAN
=item * C<tcp://> -- a real bound. Against a host that drops SYNs, an unbounded
connect waits for the kernel's own timeout, which on Linux is over two
minutes; C<< connect_timeout => 2 >> gave up after 2.00s. This is the case the
option exists for.
=item * C<unix://> -- a bound, but it does not wait. A connect to a Unix socket
whose listen backlog is full blocks: measured against a listener with
C<< Listen => 1 >> and nobody accepting, still blocked after 8 seconds. With a
C<connect_timeout> set it fails at once instead, with C<EAGAIN> -- because
C<IO::Socket> performs a timed connect non-blocking, and an C<AF_UNIX> connect
has no in-progress state to wait on. So the hang is gone, at the price of not
tolerating even a momentary backlog. A socket path that does not exist is
C<ENOENT> either way and is not affected.
=item * TLS -- bounds the TCP connect only. The handshake that follows it runs
on the connected socket, before L</read_timeout>'s C<SO_RCVTIMEO> is applied,
and is not covered by either.
=back
An expiry croaks with an L<API::Docker::Error::Timeout> carrying
C<< ->phase >> C<'connect'>, C<< ->timeout >> the value that expired and an
empty C<< ->partial >> -- there is no response to have part of. Every other
connect failure croaks with the plain string it always did: a refused
connection, a missing socket path and a rejected certificate are diagnoses,
not timeouts, and rewriting them as one would name a cause the caller cannot
act on.
=head2 Streaming a response as it arrives
Without one of these options a request is read whole, then parsed. That is
right for a request/response endpoint and wrong for every endpoint whose point
is that it keeps going: C<< logs(follow => 1) >>, C</events> with no C<until>
and C</containers/{id}/stats> with no C<< stream => 0 >> never return, because
the daemon never closes and there is nothing else to wait for.
A callback is half the answer -- it decides what to do with each unit, and it
can stop. L</"Bounding a request that never ends"> is the other half, for the
stream that stops arriving without ever ending.
Pass a callback and the body is handed over piece by piece instead:
my $summary = $client->get('/events',
croak_on_error => 0,
on_event => sub {
my ($event, $stop) = @_;
print $event->{status}, "\n";
$stop->() if $event->{status} eq 'destroy';
},
);
$summary; # { delivered => 7, stopped => 1 }
=head3 One unit per call, and three units to choose from
The engine's streaming endpoints do not share a natural unit, so there is an
option per unit and a request picks one:
=over
=item * C<on_event> - one decoded HashRef per newline-delimited JSON object.
For C</events> and the C</build>, C</images/create>, C</images/*/push>
progress streams
=item * C<on_frame> - one C<< { stream => ..., data => ... } >> HashRef per
demultiplexed frame of the Docker stream format. For
C<< /containers/{id}/logs >> and C<< /exec/{id}/start >>; normally reached
through L</stream_frames> rather than directly
=item * C<on_chunk> - the response bytes as they arrive, undecoded and
unbuffered. For an image export, and for anything with no structure this role
knows about
=back
Passing two of them croaks before the request is sent: they are three shapes
different endpoints have, not three views of one stream.
=head3 Saying stop
The callback is called as C<< $cb->($unit, $stop) >> and its return value is
ignored. To end the stream it calls C<< $stop->() >>; C<_request> checks after
the callback returns, delivers nothing further, and comes back.
An explicit closure rather than a return value, because every truthiness
convention has a silent failure mode here. C<< sub { push @got, $_[0] } >>
returns a count and C<< sub { $last = $event->{status} } >> returns whatever
the engine said -- and a container event's C<status> is literally C<stop>.
Under either polarity one of those ends the stream by accident and hands back
a truncated one with nothing to show for it. A closure the caller has to
invoke cannot be produced by accident.
=head3 What comes back
A streamed request returns a summary HashRef, not the body:
{ delivered => 7, stopped => 1 }
C<delivered> is how many units went to the callback; C<stopped> is 1 when the
callback ended the stream and 0 when the daemon did. Nothing is accumulated
along the way -- an unbounded feed must not cost memory in proportion to how
long it runs, and the caller has been handed every unit already. What it could
not otherwise know is how the stream ended, and that is what the summary says.
Only a complete unit is buffered while it is still arriving: the current
ndjson line or the current frame. A line, and equally an 8-byte frame header,
can be split across two chunks or two reads, so partial ones are carried
forward rather than decoded early.
=head3 How often the callback is called
Once per unit the daemon has finished sending, as soon as the bytes that
complete it have arrived -- not once per read of a fixed size, and not once
at the end.
That is worth stating because it was not true before karr k60. The reads were
C<read()>, which is C<fread>-shaped: it loops until it has the length it was
asked for or the stream ends, rather than returning what has arrived. On the
raw-stream endpoints -- C<attach>, C<< logs(follow => 1) >>, C<exec/start>,
which carry neither a C<Content-Length> nor chunked encoding -- the reader
asks for 64K, so nothing reached the callback until 64K had accumulated or the
daemon hung up. On a stream that never ends, nothing reached it at all.
Measured on an C<AF_UNIX> socket pair with no daemon involved, a peer writing
three frames 0.15s apart and then closing:
before: 1 call at 0.45s (the moment it closed)
after: 3 calls at 0.15s, 0.30s, 0.45s
Every read now goes through one C<sysread> and a buffer this role keeps
itself, which is also why the status line and the headers are read the same
way: PerlIO's read-ahead put the first bytes of the body somewhere the body
reader could not get at them, so the header reads had to move too or those
bytes would have been dropped.
=head3 What is not streamed
A response with status >= 400 is read whole and croaked with as always: it is
a short JSON object naming a failure, not a stream, and the callback never
sees it. C<response> is still filled. A C<HEAD> response has no body, so a
callback on one is never called and C<undef> comes back as usual.
With C<on_event>, C<croak_on_error> works as it does for C<ndjson> -- except
that the check runs per event, so a failed build croaks at the event that
reports it instead of when the daemon eventually closes. The
L<API::Docker::Error::Stream> then carries that one event in C<< ->events >>
rather than the whole stream: the callback was handed the rest as it arrived,
and none of it was kept.
C<on_frame> requires the stream to be framed. The buffered path decides
framing by walking the whole body (see L</"Detecting a framed stream">), which
is exactly what a streamed one does not have, so an unframed stream has to
declare itself with C<< tty => 1 >> to L</stream_frames> and an undeclared one
that turns out not to be framed croaks. A stream the daemon cuts off mid-frame
croaks too -- there is no whole body left to fall back to raw with. Neither
applies after a C<< $stop->() >>, which leaves a partial unit in the buffer by
construction.
=head2 Reading the status line and the response headers
The return value is the decoded body and nothing else, which leaves two things
the engine said unreachable: the status code, and the response headers. Pass a
HashRef as C<response> to get them:
my %res;
my $data = $client->post("/containers/$id/start", undef,
response => \%res);
$res{status}; # 204
$res{reason}; # 'No Content'
$res{headers}{'api-version'}; # header names are lowercased
The hash is overwritten on every call and filled B<before> the C<< >= 400 >>
croak, so a caller that wraps the request in C<eval> can still read the status
of a failed one. The return value is unaffected, so passing C<response> never
changes what a method hands back.
Two things need it. The engine answers a state change that did nothing with
B<304 Not Modified> -- starting a running container, stopping a stopped one --
which carries no body, exactly like the 204 of a change that did happen; see
L<API::Docker::API::Containers/start>. And C<< HEAD /containers/{id}/archive >>
carries its whole payload in the C<X-Docker-Container-Path-Stat> header, with
no body to return at all.
=head2 Failure on the status line
A status of 400 or above croaks with an L<API::Docker::Error::HTTP>. The
message is the engine's C<message> field, its C<errorDetail.message>, its flat
C<error> key or the raw body, in that order of preference, wrapped as
C<Docker API error (STATUS): REASON> -- the same text this croak has always
carried, and the object stringifies to it byte for byte, Carp's location
suffix included. Code that catches C<$@> as a string cannot tell the
difference and needs no change.
What the object adds is C<< $err->status >>. The message is engine-specific
prose: killing a stopped container answers 409 with C<can only kill running
containers ... container state improper> on rootless Podman 5.4.2, while
Docker's own example for that case reads C<Container E<lt>idE<gt> is not
running>. Anything that had to tell "no such container" from "wrong state"
lib/API/Docker/Role/HTTP.pm view on Meta::CPAN
the response is unread because the caller said so, and every check on the
streaming path is skipped once it has.
=head3 Against a timeout, and against a status
L<API::Docker::Error::Timeout> is the daemon going B<quiet> for longer than a
bound the caller asked for; this is the daemon B<closing> mid-response, and
needs no bound to be noticed. The two share a contract -- neither ever returns
a short body, and both hand over what arrived -- and are separate classes
because only one of them is about an option, and only one of them can fire on
a response that would have completed.
A response whose status is 400 or above raises this rather than an
L<API::Docker::Error::HTTP> when it is B<its> body that was cut short, which
is the rule the timeout already follows in the same place: the transport
cannot tell a caller what the engine said when it did not finish saying it.
C<< ->partial >> holds the part of the error body that did arrive.
=head2 Header names are rejected, header values are stripped
A CR or LF in a header B<value> is stripped and the value is flattened onto
its own line. A header B<name> that is not an RFC 9110 token is refused with
a croak instead.
The asymmetry is deliberate. A value can pick up a stray newline honestly --
C<MIME::Base64::encode_base64> wraps its output by default, and a token pasted
out of a file brings its line ending along -- and flattening it preserves what
the caller meant. A name is a literal the programmer wrote; there is no benign
way for one to contain CR, LF, a space or a colon, and quietly rewriting
C<< "X-Foo\r\nX-Bar" >> into C<X-FooX-Bar> would put a header on the wire
under a name nobody asked for. Validating against the token grammar also
catches the separators that would corrupt the request without injecting
anything.
=head2 A request path is rejected, not sanitised
The C<$path> given to L</get>, L</post>, L</put>, L</delete_request>, L</head>
and C<_request> is spliced straight into the request line as
C<< $method /v$version$path HTTP/1.1 >>, and it carries caller data: the
resource methods build it by interpolation -- C<< "/containers/$id/json" >>,
C<< "/images/$name/push" >> -- so a container name or an image reference the
user typed ends up in the request line unescaped. A byte the line's own
grammar reads therefore rewrites the request rather than naming a resource: a
CR or LF ends the line and opens a header of its own, a space starts the
HTTP-version field, and a C<?> or C<#> opens the query string or fragment.
So the path is checked against the RFC 3986 origin-form character set --
unreserved, the sub-delims, and C<:> C<@> C<%> C<< / >>, which is the set an
image reference lives in -- and a path outside it is refused with a croak
before anything reaches the wire, the same treatment and for the same reason a
header name gets. Sanitising is not on the table here: percent-encoding the
path at this layer cannot tell a separator from data, so it would either
mangle every C<< / >> and C<:> or leave the injection open. Query parameters
belong in C<params>, which is assembled separately and runs each element
through C<_uri_encode>.
=head2 post
my $data = $client->post($path, $body, %opts);
Perform HTTP POST request. C<$body> is automatically JSON-encoded if provided.
Options: C<params>, C<headers>, C<ndjson>, C<croak_on_error>, C<raw>,
C<response> and the C<on_event>/C<on_frame>/C<on_chunk> callbacks as for
L</get>, plus C<raw_body> and C<content_type> for sending a non-JSON payload
such as a build context tarball.
=head2 put
my $data = $client->put($path, $body, %opts);
Perform HTTP PUT request. C<$body> is automatically JSON-encoded if provided.
Options: C<params>, C<headers>, C<ndjson>, C<croak_on_error>, C<raw>,
C<response> and the C<on_event>/C<on_frame>/C<on_chunk> callbacks as for
L</get>, plus C<raw_body> and C<content_type> for sending a non-JSON payload
-- C<< containers->put_archive >> uses both to send a tar stream.
=head2 delete_request
my $data = $client->delete_request($path, %opts);
Perform HTTP DELETE request.
Options: C<params> (hashref of query parameters).
=head2 head
my %res;
$client->head("/containers/$id/archive",
params => { path => '/etc/hostname' },
response => \%res,
);
my $stat = decode_json(decode_base64($res{headers}{'x-docker-container-path-stat'}));
Perform HTTP HEAD request. Always returns C<undef>: a HEAD response has no
body by definition, so everything it says is in the status line and the
headers, and C<response> is the only way to reach them.
The body is not read even when the response announces one. A HEAD response
repeats the header fields the equivalent GET would send, C<Content-Length>
among them, and then sends nothing -- reading it would block on bytes that
never arrive. Measured against Podman 5.4.2 (API 1.41),
C<< HEAD /containers/{id}/archive >> in fact announces no length at all, only
C<X-Docker-Container-Path-Stat> -- but an engine that does announce one is not
waited on either.
Options: C<params>, C<headers> and C<response> as for L</get>.
=head2 stream_frames
my $frames = $client->stream_frames('GET', "/containers/$id/logs", %opts);
Perform a request against one of the engine's framed endpoints
(C<< /containers/{id}/logs >>, C<< /exec/{id}/start >>) and return an ArrayRef
of frames:
[ { stream => 'stdout', data => "OUT\n" },
{ stream => 'stderr', data => "ERR\n" } ]
C<stream> is C<stdout>, C<stderr> or C<stdin> for a multiplexed stream, and
C<raw> for an unframed one. It is always a plain string, so callers never need
a defined-check. Joining the payloads gives the plain text:
my $text = join '', map { $_->{data} } @$frames;
The response body is never JSON-decoded, so a container printing JSON lines is
returned verbatim.
Options are those of C<_request> (C<params>, C<body>, C<headers>), plus:
=over
=item * C<tty> - Skip demultiplexing and return the body as a single C<raw>
frame. Set it when the container or exec instance was created with a TTY and
its output is binary; see L</"Detecting a framed stream"> for why.
=item * C<on_frame> - CodeRef called with each frame as it arrives instead of
the whole ArrayRef being returned at the end; see below.
=back
=head2 Following a framed stream
With C<on_frame> the frames are handed over as they arrive and the return
value is the summary HashRef described in L</"Streaming a response as it
arrives">, not an ArrayRef:
my $summary = $client->stream_frames('GET', "/containers/$id/logs",
params => { follow => 1, stdout => 1, stderr => 1 },
on_frame => sub {
my ($frame, $stop) = @_;
print $frame->{data};
$stop->() if $frame->{data} =~ /listening on/;
},
);
This is the only way to use C<< follow => 1 >> at all: without it the request
does not return until the container exits.
The frame shape is the same either way, C<tty> included -- a TTY stream
arrives as a series of C<< { stream => 'raw', ... } >> frames rather than the
single one the buffered path builds, so a caller still never branches on it.
C<tty> is a declaration here rather than the hint it is on the buffered path.
Deciding framing from the bytes needs the whole body, which is precisely what
is not being kept; so an unframed stream must say so, and one that does not
and is not framed croaks instead of inventing frames from its payload.
=head2 Detecting a framed stream
A container created without a TTY produces the Docker stream format -- an
8-byte header per frame (byte 0 the stream type, bytes 4-7 a big-endian uint32
payload length) followed by that many payload bytes. With a TTY there is no
header and the payload is raw pty output.
The engine is supposed to distinguish the two with the response C<Content-Type>
(C<application/vnd.docker.multiplexed-stream> against
C<application/vnd.docker.raw-stream>), but that signal is not dependable.
Measured against Podman 5.4.2 (API 1.41): C<< GET /containers/{id}/logs >>
sends no C<Content-Type> at all, for either kind of container, and
C<< POST /exec/{id}/start >> sends C<application/vnd.docker.raw-stream> for
both -- including the non-TTY exec whose body is in fact multiplexed. Trusting
the header would therefore hand frame headers to the caller on that engine.
The framing is decided from the bytes instead. The body is walked as frames:
each header must have a stream type of 0, 1 or 2, three zero bytes after it,
( run in 1.680 second using v1.01-cache-2.11-cpan-364913b4093 )