view release on metacpan or  search on metacpan

lib/DBIx/QuickDB/Watcher.pm  view on Meta::CPAN

#
# $root is only passed by tests; it defaults to the load-time capture.
sub _watcher_inc_args {
    my ($root) = @_ ? @_ : ($LOADED_FROM);

    my @dirs;
    push @dirs => $root if defined $root;

    # @INC hooks (refs) cannot be expressed as -I.
    push @dirs => grep { !ref($_) && defined($_) && length($_) } @INC;

    # Absolutizing the rest is hygiene only -- this runs in the forked watcher,
    # which shares the caller's cwd, so a relative entry would resolve the same
    # either way. $LOADED_FROM above is what actually pins the module.
    my (@out, %seen);
    for my $dir (@dirs) {
        my $abs = abs_path($dir);
        $dir = $abs if defined($abs) && length($abs);
        next if $seen{$dir}++;
        push @out => $dir;
    }

    return map {"-I$_"} @out;
}

sub watch {
    my $self = shift;
    my ($wh) = @_;

    $0 = 'db-quick-watcher';

    my $kill = '';
    my $hup = 0;
    local $SIG{TERM} = sub { $kill = 'TERM' };
    local $SIG{INT}  = sub { $kill = 'INT' };
    local $SIG{USR1} = sub { $kill = 'FAST_TERM' };
    local $SIG{HUP} = sub { $hup = 1 };

    # Own child reaping before spawning. An inherited SIGCHLD of IGNORE makes
    # the kernel reap the server first, losing its exit status. Protects the
    # window between spawn() and the exec below.
    local $SIG{CHLD} = 'DEFAULT';

    my $start_pid = $$;
    my $pid = $self->spawn();
    print $wh "$pid\n";
    close($wh);

    my $mpid = $self->{+MASTER_PID};
    my $spid = $self->{+SERVER_PID} or die "No server pid";

    my $ddir = $self->{+DATA_DIR};
    my $ssig = $self->{+DB}->stop_sig // 'TERM';
    my $fsig = $self->{+DB}->fast_stop_sig // 'KILL';
    my $delete_data = $self->{+DB}->should_cleanup ? 1 : 0;
    my $owner_kill  = $delete_data && $self->{+DB}->fast_destroy
        ? 'FAST_TERM'
        : 'TERM';

    # Block (rather than ignore) every teardown signal across the exec. A
    # blocked signal stays *pending* instead of being discarded, so a stop(),
    # eliminate(), or fast_eliminate() that races server startup -- arriving
    # after the socket is up (so the caller's start() has returned) but before
    # _do_watch has installed its handlers -- is not lost: _do_watch unblocks
    # them once the handlers are in place and a pending one fires immediately.
    #
    # SIGTERM/SIGINT used to be SIG_IGN'd here instead, on the wrong belief
    # that an ignored signal is "held" across exec -- it is DISCARDED. A
    # stop() landing in the exec window (perl startup + module load in the
    # fresh watcher) was silently dropped, the watcher never learned it should
    # stop, and the caller's wait() eventually gave up and killed the watcher,
    # orphaning a still-running server whose data dir stayed locked. The
    # build-then-immediately-stop pattern (DBIx::QuickDB::Pool) hit that
    # window constantly.
    POSIX::sigprocmask(POSIX::SIG_BLOCK(), POSIX::SigSet->new(POSIX::SIGUSR1(), POSIX::SIGINT(), POSIX::SIGTERM()));

    exec(
        $^X, _watcher_inc_args(),

        '-e' => "require DBIx::QuickDB::Watcher; DBIx::QuickDB::Watcher->_do_watch()",

        master_pid  => $mpid,
        data_dir    => $ddir,
        server_pid  => $spid,
        signal      => $ssig,
        fast_signal => $fsig,
        delete_data => $delete_data,
        owner_kill  => $owner_kill,
        kill        => $kill,
        hup         => $hup,
    );
}

sub _do_watch {
    my $class = shift;

    $0 = 'db-quick-watcher';

    my %params = @ARGV;

    my $kill = $params{kill} // '';
    my $hup  = $params{hup}  // 0;
    local $SIG{TERM} = sub { $kill = 'TERM' };
    local $SIG{INT}  = sub { $kill = 'INT' };
    local $SIG{USR1} = sub { $kill = 'FAST_TERM' };
    local $SIG{HUP}  = sub { $hup  = 1 };

    # watch() blocked the teardown signals before exec so a stop/eliminate/
    # fast_eliminate racing startup stays pending rather than being discarded.
    # Now that the handlers above are installed, unblock them -- any pending
    # teardown fires here and sets $kill before we enter the watch loop.
    POSIX::sigprocmask(POSIX::SIG_UNBLOCK(), POSIX::SigSet->new(POSIX::SIGUSR1(), POSIX::SIGINT(), POSIX::SIGTERM()));

    my $blah;
    close(STDIN);
    open(STDIN, '<', \$blah) or warn "$!";

    my $master_pid  = $params{master_pid} or die "No master pid provided";
    my $server_pid  = $params{server_pid} or die "No server pid provided";
    my $data_dir    = $params{data_dir}   or die "No data dir provided";
    my $signal      = $params{signal} // 'TERM';
    my $fast_signal = $params{fast_signal} // 'KILL';
    my $delete_data = $params{delete_data} // 0;
    my $owner_kill  = $params{owner_kill} // 'TERM';

    my $reaped = 0;
    my $hupped = 0;
    while (!$kill) {
        if ($hup && !$hupped) {
            close(STDOUT);
            open(STDOUT, '>', \$blah) or warn "$!";
            close(STDERR);
            open(STDERR, '>', \$blah) or warn "$!";
            $hupped = 1;
        }

        sleep 0.1;

        # Nothing else reaps the server, so a crashed one would linger as a
        # zombie -- and kill(0) succeeds on a zombie, which made start() report
        # it alive and wait out the whole timeout. Keep watching after the reap:
        # exiting here would run teardown and delete the logs start() needs.
        $reaped ||= $class->_reap_server($server_pid, $data_dir);

        next if kill(0, $master_pid);
        $kill = $owner_kill;
    }

    unless (eval { $class->_watcher_terminate(send_sig => $signal, fast_sig => $fast_signal, got_sig => $kill, pid => $server_pid, dir => $data_dir, delete_data => $delete_data, already_reaped => $reaped); 1 }) {
        # warn $err, not $@: entering an eval BLOCK clears $@, which stripped
        # the message off every teardown failure.
        my $err = $@;
        eval { warn $err };
        POSIX::_exit(1);
    }

    POSIX::_exit(0);
}

sub server_exit_status_file {
    my $class = shift;
    my ($dir) = @_;
    return "$dir/server-exit-status";
}

# Non-blocking reap; writes the wait status to server_exit_status_file() and
# returns true once collected. Never dies -- this runs in the watch loop, and
# taking the watcher down would orphan the server.

lib/DBIx/QuickDB/Watcher.pm  view on Meta::CPAN

# stop(), eliminate(), and detach() each signal the watcher process by pid. Once
# ANY terminal teardown has been initiated, the watcher is exiting (or already
# gone) and the OS may recycle its pid to an unrelated process -- notably a
# sibling database's postmaster. A second signal would then land on the wrong
# process and shut down a live server out from under its owner. So once a stop
# or eliminate has been sent, never signal this pid again. (Data-dir cleanup for
# a stopped database is handled by Driver::DESTROY, not by a second signal.)
sub stop {
    my $self = shift;
    return if $self->{+STOPPED}++ || $self->{+ELIMINATED};
    my $pid = $self->{+WATCHER_PID} or return;
    $self->_latch_server_reaped;
    kill('INT', $pid);
}

sub eliminate {
    my $self = shift;
    return if $self->{+ELIMINATED}++ || $self->{+STOPPED};
    my $pid = $self->{+WATCHER_PID} or return;
    $self->_latch_server_reaped;
    kill('TERM', $pid);
}

# Like eliminate(), but the watcher kills the server immediately with the
# driver's fast_stop_sig (SIGKILL by default, or a clean immediate-shutdown
# signal) rather than attempting a graceful shutdown. Sets ELIMINATED so the
# normal teardown signals are never also sent to this (possibly
# soon-to-be-recycled) pid.
sub fast_eliminate {
    my $self = shift;
    return if $self->{+ELIMINATED}++ || $self->{+STOPPED};
    my $pid = $self->{+WATCHER_PID} or return;
    $self->_latch_server_reaped;
    kill('USR1', $pid);
}

sub detach {
    my $self = shift;
    return if $self->{+DETACHED}++;
    return if $self->{+STOPPED} || $self->{+ELIMINATED};
    my $pid = $self->{+WATCHER_PID} or return;
    kill('HUP', $pid);
}

sub wait {
    my $self = shift;
    my $pid = $self->{+WATCHER_PID} or return;

    # Give the watcher long enough to finish a graceful shutdown. The watcher
    # escalates to SIGKILL on the server after QDB_STOP_GRACE and then BLOCKS
    # until the server is reaped, so this must outlast the watcher's own
    # escalation schedule (grace + grace/2, plus slack).
    my $timeout = _stop_grace() * 2 + 2;

    # A watcher that outlives $timeout is almost never hung -- the usual
    # cause is a server the kernel has not been able to kill yet (e.g. stuck
    # in disk-sleep under heavy I/O), with the watcher dutifully blocking on
    # the post-SIGKILL reap. Killing the watcher at that point orphans a
    # still-alive server, so stop() would return "success" while the data dir
    # is locked by a live postmaster/mysqld and the next start on that dir
    # fails on the stale lock file. So past $timeout we only warn, keep
    # waiting on a much longer leash, and SIGKILL the watcher purely as a
    # last resort. Tunable via QDB_STOP_LEASH (extra seconds past $timeout).
    my $extra = $ENV{QDB_STOP_LEASH};
    $extra = 60 unless defined($extra) && $extra =~ /^\d+$/ && $extra > 0;
    my $leash = $timeout + $extra;

    my ($warned, $nuked);
    my $start = time;
    while(kill(0, $pid)) {
        my $waited = time - $start;

        if ($waited > $timeout && !$warned++) {
            warn "Watcher (pid $pid) did not finish within ${timeout}s; the server is probably stuck mid-shutdown, waiting up to ${extra}s longer for it to die";
        }

        if ($waited > $leash && !$nuked++) {
            warn "Watcher (pid $pid) still running after ${leash}s, killing it; the server may survive as an orphan";
            kill('KILL', $pid);
            $start = time;    # from here just wait for the SIGKILL to land
        }

        sleep 0.02;
    }

    # The watcher has exited; forget its pid so no later teardown signal (e.g.
    # from DESTROY) can land on a recycled pid now owned by another process.
    delete $self->{+WATCHER_PID};

    # The server may have outlived the watcher, so verify with a read-only
    # kill(0). Never send a real signal here -- the pid may have been recycled,
    # and a false "alive" should cost a warning, not a wrong kill.
    #
    # Skipped only when the watch loop recorded an exit status, which proves it
    # reaped the server; the pid may since have been recycled, and probing it
    # would report a false "alive" and stall for the whole timeout. A missing
    # data dir proves nothing here -- teardown removes a disposable dir whether
    # or not it managed to stop the server.
    $self->_latch_server_reaped;

    if (!$self->{+SERVER_REAPED} and my $spid = $self->{+SERVER_PID}) {
        my $sstart = time;
        while (kill(0, $spid)) {
            if (time - $sstart > $timeout) {
                warn "Server (pid $spid) still appears to be alive after its watcher exited; its data dir may still be locked";
                last;
            }
            sleep 0.02;
        }
    }
}

sub DESTROY {
    my $self = shift;

    if ($self->{+MASTER_PID} == $$) {
        $self->eliminate;
        $self->wait;
    }
    else {
        unlink($self->{+LOG_FILE}) if $self->{+LOG_FILE};
    }
}

1;

__END__

=pod

=encoding UTF-8

=head1 NAME

DBIx::QuickDB::Watcher - Daemon that sits between main process and the server.

=head1 DESCRIPTION

When a database is spun up a 'db-quick-watcher' process is started. This
process has 1 job: Make sure cleanup happens. This process is a daemon
completely disconnected from the process that requested the database, and the
db-server is a process under it.

If this process detects that your main process goes away (exited, killed, etc)
this process will kill the database server and delete the data dir, then exit.

The main process can also send signals to this one to make it stop, clean up,
etc.

=head1 METHODS

=over 4

=item $path = DBIx::QuickDB::Watcher->server_exit_status_file($instance_dir)

Path of the file the watcher writes when it reaps the database server, holding
the raw wait status.

The owning process is not the server's parent, so it cannot C<waitpid> and
cannot tell a crashed server from a slow one -- C<kill(0)> succeeds on a zombie.
This file is how it finds out. L<DBIx::QuickDB::Driver/start> polls for it to
fail fast on a server that died during startup, and removes it before spawning a
new server so a previous crash cannot fail the next start.

=back