view release on metacpan or  search on metacpan

lib/DBIx/QuickDB/Watcher.pm  view on Meta::CPAN

        master_pid  => $mpid,
        data_dir    => $ddir,
        server_pid  => $spid,
        signal      => $ssig,
        fast_signal => $fsig,
        delete_data => $delete_data,
        owner_kill  => $owner_kill,
        kill        => $kill,
        hup         => $hup,
    );
}

sub _do_watch {
    my $class = shift;

    $0 = 'db-quick-watcher';

    my %params = @ARGV;

    my $kill = $params{kill} // '';
    my $hup  = $params{hup}  // 0;
    local $SIG{TERM} = sub { $kill = 'TERM' };
    local $SIG{INT}  = sub { $kill = 'INT' };
    local $SIG{USR1} = sub { $kill = 'FAST_TERM' };
    local $SIG{HUP}  = sub { $hup  = 1 };

    # watch() blocked the teardown signals before exec so a stop/eliminate/
    # fast_eliminate racing startup stays pending rather than being discarded.
    # Now that the handlers above are installed, unblock them -- any pending
    # teardown fires here and sets $kill before we enter the watch loop.
    POSIX::sigprocmask(POSIX::SIG_UNBLOCK(), POSIX::SigSet->new(POSIX::SIGUSR1(), POSIX::SIGINT(), POSIX::SIGTERM()));

    my $blah;
    close(STDIN);
    open(STDIN, '<', \$blah) or warn "$!";

    my $master_pid  = $params{master_pid} or die "No master pid provided";
    my $server_pid  = $params{server_pid} or die "No server pid provided";
    my $data_dir    = $params{data_dir}   or die "No data dir provided";
    my $signal      = $params{signal} // 'TERM';
    my $fast_signal = $params{fast_signal} // 'KILL';
    my $delete_data = $params{delete_data} // 0;
    my $owner_kill  = $params{owner_kill} // 'TERM';

    my $reaped = 0;
    my $hupped = 0;
    while (!$kill) {
        if ($hup && !$hupped) {
            close(STDOUT);
            open(STDOUT, '>', \$blah) or warn "$!";
            close(STDERR);
            open(STDERR, '>', \$blah) or warn "$!";
            $hupped = 1;
        }

        sleep 0.1;

        # Nothing else reaps the server, so a crashed one would linger as a
        # zombie -- and kill(0) succeeds on a zombie, which made start() report
        # it alive and wait out the whole timeout. Keep watching after the reap:
        # exiting here would run teardown and delete the logs start() needs.
        $reaped ||= $class->_reap_server($server_pid, $data_dir);

        next if kill(0, $master_pid);
        $kill = $owner_kill;
    }

    unless (eval { $class->_watcher_terminate(send_sig => $signal, fast_sig => $fast_signal, got_sig => $kill, pid => $server_pid, dir => $data_dir, delete_data => $delete_data, already_reaped => $reaped); 1 }) {
        # warn $err, not $@: entering an eval BLOCK clears $@, which stripped
        # the message off every teardown failure.
        my $err = $@;
        eval { warn $err };
        POSIX::_exit(1);
    }

    POSIX::_exit(0);
}

sub server_exit_status_file {
    my $class = shift;
    my ($dir) = @_;
    return "$dir/server-exit-status";
}

# Non-blocking reap; writes the wait status to server_exit_status_file() and
# returns true once collected. Never dies -- this runs in the watch loop, and
# taking the watcher down would orphan the server.
sub _reap_server {
    my $class = shift;
    my ($pid, $dir) = @_;

    my ($got, $status);
    {
        local $?;
        $got    = waitpid($pid, WNOHANG);
        $status = $?;
    }

    # Only a real reap counts. Negative is ECHILD, which yields no status and
    # cannot distinguish "auto-reaped" from "never our child"; teardown keys off
    # this to decide whether it may skip the kill, so do not overclaim.
    return 0 unless $got > 0;

    my $file = $class->server_exit_status_file($dir);
    if (open(my $fh, '>', $file)) {
        print $fh "$status\n";
        close($fh);
    }

    return 1;
}

sub spawn {
    my $self = shift;

    croak "Extra spawn" if $self->{+SERVER_PID};

    my $db   = $self->{+DB};
    my $args = $self->{+ARGS} || [];

lib/DBIx/QuickDB/Watcher.pm  view on Meta::CPAN

}

# stop(), eliminate(), and detach() each signal the watcher process by pid. Once
# ANY terminal teardown has been initiated, the watcher is exiting (or already
# gone) and the OS may recycle its pid to an unrelated process -- notably a
# sibling database's postmaster. A second signal would then land on the wrong
# process and shut down a live server out from under its owner. So once a stop
# or eliminate has been sent, never signal this pid again. (Data-dir cleanup for
# a stopped database is handled by Driver::DESTROY, not by a second signal.)
sub stop {
    my $self = shift;
    return if $self->{+STOPPED}++ || $self->{+ELIMINATED};
    my $pid = $self->{+WATCHER_PID} or return;
    $self->_latch_server_reaped;
    kill('INT', $pid);
}

sub eliminate {
    my $self = shift;
    return if $self->{+ELIMINATED}++ || $self->{+STOPPED};
    my $pid = $self->{+WATCHER_PID} or return;
    $self->_latch_server_reaped;
    kill('TERM', $pid);
}

# Like eliminate(), but the watcher kills the server immediately with the
# driver's fast_stop_sig (SIGKILL by default, or a clean immediate-shutdown
# signal) rather than attempting a graceful shutdown. Sets ELIMINATED so the
# normal teardown signals are never also sent to this (possibly
# soon-to-be-recycled) pid.
sub fast_eliminate {
    my $self = shift;
    return if $self->{+ELIMINATED}++ || $self->{+STOPPED};
    my $pid = $self->{+WATCHER_PID} or return;
    $self->_latch_server_reaped;
    kill('USR1', $pid);
}

sub detach {
    my $self = shift;
    return if $self->{+DETACHED}++;
    return if $self->{+STOPPED} || $self->{+ELIMINATED};
    my $pid = $self->{+WATCHER_PID} or return;
    kill('HUP', $pid);
}

sub wait {
    my $self = shift;
    my $pid = $self->{+WATCHER_PID} or return;

    # Give the watcher long enough to finish a graceful shutdown. The watcher
    # escalates to SIGKILL on the server after QDB_STOP_GRACE and then BLOCKS
    # until the server is reaped, so this must outlast the watcher's own
    # escalation schedule (grace + grace/2, plus slack).
    my $timeout = _stop_grace() * 2 + 2;

    # A watcher that outlives $timeout is almost never hung -- the usual
    # cause is a server the kernel has not been able to kill yet (e.g. stuck
    # in disk-sleep under heavy I/O), with the watcher dutifully blocking on
    # the post-SIGKILL reap. Killing the watcher at that point orphans a
    # still-alive server, so stop() would return "success" while the data dir
    # is locked by a live postmaster/mysqld and the next start on that dir
    # fails on the stale lock file. So past $timeout we only warn, keep
    # waiting on a much longer leash, and SIGKILL the watcher purely as a
    # last resort. Tunable via QDB_STOP_LEASH (extra seconds past $timeout).
    my $extra = $ENV{QDB_STOP_LEASH};
    $extra = 60 unless defined($extra) && $extra =~ /^\d+$/ && $extra > 0;
    my $leash = $timeout + $extra;

    my ($warned, $nuked);
    my $start = time;
    while(kill(0, $pid)) {
        my $waited = time - $start;

        if ($waited > $timeout && !$warned++) {
            warn "Watcher (pid $pid) did not finish within ${timeout}s; the server is probably stuck mid-shutdown, waiting up to ${extra}s longer for it to die";
        }

        if ($waited > $leash && !$nuked++) {
            warn "Watcher (pid $pid) still running after ${leash}s, killing it; the server may survive as an orphan";
            kill('KILL', $pid);
            $start = time;    # from here just wait for the SIGKILL to land
        }

        sleep 0.02;
    }

    # The watcher has exited; forget its pid so no later teardown signal (e.g.
    # from DESTROY) can land on a recycled pid now owned by another process.
    delete $self->{+WATCHER_PID};

    # The server may have outlived the watcher, so verify with a read-only
    # kill(0). Never send a real signal here -- the pid may have been recycled,
    # and a false "alive" should cost a warning, not a wrong kill.
    #
    # Skipped only when the watch loop recorded an exit status, which proves it
    # reaped the server; the pid may since have been recycled, and probing it
    # would report a false "alive" and stall for the whole timeout. A missing
    # data dir proves nothing here -- teardown removes a disposable dir whether
    # or not it managed to stop the server.
    $self->_latch_server_reaped;

    if (!$self->{+SERVER_REAPED} and my $spid = $self->{+SERVER_PID}) {
        my $sstart = time;
        while (kill(0, $spid)) {
            if (time - $sstart > $timeout) {
                warn "Server (pid $spid) still appears to be alive after its watcher exited; its data dir may still be locked";
                last;
            }
            sleep 0.02;
        }
    }
}

sub DESTROY {
    my $self = shift;

    if ($self->{+MASTER_PID} == $$) {
        $self->eliminate;
        $self->wait;
    }
    else {
        unlink($self->{+LOG_FILE}) if $self->{+LOG_FILE};
    }
}

1;

__END__

=pod

=encoding UTF-8

=head1 NAME

DBIx::QuickDB::Watcher - Daemon that sits between main process and the server.

=head1 DESCRIPTION

When a database is spun up a 'db-quick-watcher' process is started. This
process has 1 job: Make sure cleanup happens. This process is a daemon
completely disconnected from the process that requested the database, and the
db-server is a process under it.

If this process detects that your main process goes away (exited, killed, etc)
this process will kill the database server and delete the data dir, then exit.

The main process can also send signals to this one to make it stop, clean up,
etc.

=head1 METHODS

=over 4

=item $path = DBIx::QuickDB::Watcher->server_exit_status_file($instance_dir)

Path of the file the watcher writes when it reaps the database server, holding
the raw wait status.

The owning process is not the server's parent, so it cannot C<waitpid> and
cannot tell a crashed server from a slow one -- C<kill(0)> succeeds on a zombie.
This file is how it finds out. L<DBIx::QuickDB::Driver/start> polls for it to
fail fast on a server that died during startup, and removes it before spawning a
new server so a previous crash cannot fail the next start.

=back