Developer-Dashboard
view release on metacpan or search on metacpan
lib/Developer/Dashboard/RuntimeManager.pm view on Meta::CPAN
label => 'Stop dashboard web service',
}
);
return _numeric_pid($pid);
}
$self->_send_signal( 'TERM', $pid ) if $pid;
$self->_send_signal( 'TERM', @listener_pids );
$self->_pkill_perl('^dashboard web:');
my @ajax_workers = $self->_managed_ajax_processes;
my @ajax_worker_pids = map { $_->{pid} } @ajax_workers;
$self->_send_signal( 'TERM', @ajax_worker_pids );
my @legacy_web_pids = map { $_->{pid} } $self->_find_legacy_web_processes;
$self->_send_signal( 'TERM', @legacy_web_pids );
my $watch_ajax_workers = @ajax_worker_pids ? 1 : 0;
for ( 1 .. 5 ) {
if ($watch_ajax_workers) {
@ajax_workers = $self->_managed_ajax_processes;
@ajax_worker_pids = map { $_->{pid} } @ajax_workers;
}
last if !$self->_wait_for_unix_web_shutdown(
pid => $pid,
listener_pids => \@listener_pids,
ajax_pids => \@ajax_worker_pids,
legacy_pids => \@legacy_web_pids,
);
sleep $self->_runtime_poll_interval;
}
if ( $pid && $self->_pid_is_running($pid) ) {
$self->_send_signal( 'KILL', $pid );
}
$self->_send_signal( 'KILL', @ajax_worker_pids );
my @still_listening = grep { kill 0, $_ } @listener_pids;
$self->_send_signal( 'KILL', @still_listening );
$self->_send_signal( 'KILL', grep { $self->_pid_is_running($_) } @legacy_web_pids );
sleep $self->_runtime_poll_interval;
my $released = $self->_wait_for_port_release($port);
if ( !$released && $port ) {
my @late_listeners = grep { kill 0, $_ } $self->_listener_pids_for_port($port);
$self->_send_signal( 'KILL', @late_listeners );
$self->_reap_child_processes(@late_listeners);
$self->_wait_for_port_release($port);
}
$self->_reap_child_processes( $pid, @listener_pids, @legacy_web_pids, @ajax_worker_pids );
$self->_cleanup_web_files;
$self->_progress_emit(
$progress,
{
task_id => 'stop_web',
status => 'done',
label => 'Stop dashboard web service',
}
);
return _numeric_pid($pid);
}
# _wait_for_unix_web_shutdown(%args)
# Checks whether Unix web shutdown still has managed web, ajax worker, legacy
# serve, or listener processes alive before escalation to KILL.
# Input: optional pid integer plus array references for listener_pids,
# ajax_pids, and legacy_pids.
# Output: boolean true when shutdown work is still pending.
sub _wait_for_unix_web_shutdown {
my ( $self, %args ) = @_;
my $pid = $args{pid};
my @listener_pids = @{ $args{listener_pids} || [] };
my @ajax_pids = @{ $args{ajax_pids} || [] };
my @legacy_pids = @{ $args{legacy_pids} || [] };
return 1 if defined $pid && $pid =~ /^\d+$/ && $pid > 0 && $self->_pid_is_running($pid);
return 1 if grep { defined $_ && $self->_pid_is_running($_) } @ajax_pids;
return 1 if grep { defined $_ && $self->_pid_is_running($_) } @legacy_pids;
return 1 if grep { defined $_ && /^\d+$/ && kill 0, $_ } @listener_pids;
return 0;
}
# _managed_ajax_processes()
# Returns dashboard Ajax singleton workers that belong to the current runtime so
# web stop and restart actions do not interfere with unrelated dashboard Ajax
# processes owned by the same user.
# Input: none.
# Output: list of ajax process hash references for the active runtime root.
sub _managed_ajax_processes {
my ($self) = @_;
my $runtime_root = $self->{paths} ? $self->{paths}->state_root : '';
my @matches;
for my $proc ( $self->_find_processes_by_prefix('dashboard ajax:') ) {
my $marker = $self->_read_process_env_marker( $proc->{pid}, 'DEVELOPER_DASHBOARD_RUNTIME_ROOT' );
next if defined $marker && $marker ne '' && $marker ne $runtime_root;
next if defined $marker && $marker eq '' && $runtime_root ne '';
next if !defined $marker && $self->_procfs_available && $runtime_root ne '';
push @matches, $proc;
}
return @matches;
}
# _numeric_pid($pid)
# Normalizes persisted pid values back to numeric scalars for lifecycle JSON
# output while preserving undef when no process id is available.
# Input: optional pid scalar.
# Output: numeric pid scalar or undef.
sub _numeric_pid {
my ($pid) = @_;
return undef if !defined $pid || $pid eq '';
return $pid =~ /^\d+$/ ? $pid + 0 : $pid;
}
# _reap_child_process($pid)
# Reaps one direct runtime child when it has already exited so background
# lifecycle helpers do not accumulate zombie processes.
# Input: process id integer.
# Output: boolean true when waitpid reaped the child.
sub _reap_child_process {
my ( $self, $pid ) = @_;
return 0 if !defined $pid || $pid !~ /^\d+$/ || $pid < 1;
my $waited = waitpid( $pid, 1 );
return $waited == $pid ? 1 : 0;
}
# _pid_is_running($pid)
# Determines whether one runtime-managed pid is still alive after opportunistic
# child reaping.
# Input: process id integer.
# Output: boolean true when the process is still running.
sub _pid_is_running {
my ( $self, $pid ) = @_;
return 0 if !defined $pid || $pid !~ /^\d+$/ || $pid < 1;
return 0 if $self->_reap_child_process($pid);
return 0 if ( $self->_read_process_state($pid) || '' ) eq 'Z';
return $self->_process_exists($pid) ? 1 : 0;
}
# _reap_child_processes(@pids)
# Reaps every direct child in one pid list when those children have already
# exited.
# Input: list of process id integers.
# Output: number of child processes reaped.
sub _reap_child_processes {
my ( $self, @pids ) = @_;
my $count = 0;
for my $pid (@pids) {
$count++ if $self->_reap_child_process($pid);
}
return $count;
}
# _wait_for_any_child_process($flags)
# Wraps waitpid for any direct child so the watchdog supervisor can reap exited
# adopted children without depending on implicit process cleanup.
# Input: waitpid flag integer such as WNOHANG.
# Output: reaped pid integer, zero when nothing is ready, or -1 when no child
# remains.
sub _wait_for_any_child_process {
my ( $self, $flags ) = @_;
return waitpid( -1, $flags );
}
# _reap_any_child_processes()
# Reaps every direct child that has already exited so long-lived runtime helper
# processes such as the collector watchdog do not accumulate zombies when they
# become the parent of dashboard-managed subprocesses.
# Input: none.
# Output: number of child processes reaped.
sub _reap_any_child_processes {
my ($self) = @_;
my $count = 0;
while (1) {
my $reaped = $self->_wait_for_any_child_process(1);
last if !defined $reaped || $reaped <= 0;
$count++;
}
return $count;
}
# _wait_for_windows_web_shutdown($pid, $port, $listener_pids)
# Checks whether the Windows-managed web process and its listener port have
# both gone away after shutdown signals were sent.
# Input: optional saved pid, optional listen port, and array reference of
# listener pids discovered from persisted state.
# Output: boolean true while the web runtime still appears alive.
sub _wait_for_windows_web_shutdown {
my ( $self, $pid, $port, $listener_pids ) = @_;
my @listener_pids = ref($listener_pids) eq 'ARRAY' ? @{$listener_pids} : ();
return 1 if $pid && kill 0, $pid;
return 1 if grep { kill 0, $_ } @listener_pids;
return 1 if $port && scalar $self->_listener_pids_for_port($port);
return 0;
}
# start_collectors()
# Starts configured non-manual collectors in the background.
# Input: none.
# Output: list of started collector hashes.
sub start_collectors {
my ( $self, %args ) = @_;
my $progress = $args{progress};
my %wanted = map { $_ => 1 } @{ $args{names} || [] };
my @jobs = @{ $self->{config}->collectors };
$self->_stop_disabled_collectors( jobs => \@jobs, progress => $progress, wanted => \%wanted );
my @started;
for my $job (@jobs) {
next if ref($job) ne 'HASH';
my $schedule = $job->{schedule} || ( $job->{cron} ? 'cron' : $job->{interval} ? 'interval' : 'manual' ); # uncoverable condition false the nested schedule ternary always yields a truthy string
my $name = $job->{name} || '(unnamed)';
if (%wanted) {
next if !$wanted{$name};
}
else {
next if $schedule eq 'manual';
}
next if $self->_collector_disabled($job);
$self->_progress_emit(
$progress,
{
task_id => "start_collector:$name",
status => 'running',
label => "Start collector $name",
}
);
my $pid = eval { $self->{runner}->start_loop($job) };
if ($@) {
my $error = $@;
chomp $error;
for my $started (@started) {
eval { $self->{runner}->stop_loop( $started->{name} ) };
}
$self->_progress_emit(
$progress,
{
task_id => "start_collector:$name",
status => 'failed',
label => "Start collector $name",
}
);
die "Failed to start collector '$name': $error\n";
}
if ( defined $pid && !$self->_collector_runtime_ready( $job->{name}, $pid ) ) {
for my $started (@started) {
eval { $self->{runner}->stop_loop( $started->{name} ) };
}
lib/Developer/Dashboard/RuntimeManager.pm view on Meta::CPAN
next;
}
my $loop_job = $self->_loop_job_for_named_start($job);
my $pid = eval { $self->{runner}->start_loop($loop_job) };
my $start_error = $@;
if ( !$start_error && defined $pid && !$self->_collector_runtime_ready( $name, $pid ) ) {
eval { $self->{runner}->stop_loop($name) };
$start_error = "Failed to keep collector '$name' running after watchdog restart\n";
}
if ($start_error) {
chomp $start_error;
$self->{collectors}->write_status(
$name,
{
running => 0,
watchdog_attention_required => 0,
watchdog_last_error => $start_error,
watchdog_last_unexpected_stop_at => $observed_at,
watchdog_last_unexpected_stop_at_epoch => $observed_at_epoch,
watchdog_restart_count => $restart_count,
watchdog_restart_window_started_at => $window_started_at,
watchdog_restart_window_started_at_epoch => $window_started_epoch,
watchdog_status => 'restart_failed',
}
);
$self->_log_collector_watchdog_event( $name, $start_error );
next;
}
$self->{collectors}->write_status(
$name,
{
running => 1,
watchdog_attention_required => 0,
watchdog_last_error => $stopped_stalled
? sprintf(
"Collector '%s' stopped making progress and was restarted by the watchdog",
$name
)
: undef,
watchdog_last_restart_at => $observed_at,
watchdog_last_restart_at_epoch => $observed_at_epoch,
watchdog_last_unexpected_stop_at => $observed_at,
watchdog_last_unexpected_stop_at_epoch => $observed_at_epoch,
watchdog_restart_count => $restart_count,
watchdog_restart_window_started_at => $window_started_at,
watchdog_restart_window_started_at_epoch => $window_started_epoch,
watchdog_status => 'running',
}
);
$self->_log_collector_watchdog_event( $name, "Watchdog restarted collector '$name' (attempt $restart_count)" );
push @{ $result{restarted} }, { name => $name, pid => $pid };
}
return \%result;
}
# _collector_stalled_for_watchdog($job, $status)
# Detects when a managed scheduled collector loop is alive but has stopped
# making progress long enough that the watchdog should recycle it.
# Input: collector job hash reference and collector status hash reference.
# Output: boolean true when the collector is stalled.
sub _collector_stalled_for_watchdog {
my ( $self, $job, $status ) = @_;
return 0 if ref($job) ne 'HASH';
return 0 if ref($status) ne 'HASH';
my $latest_epoch = $self->_collector_watchdog_last_progress_epoch($status);
return 0 if !$latest_epoch;
my $stale_after = $self->_collector_watchdog_stale_seconds($job);
return 0 if $stale_after < 1;
return time - $latest_epoch > $stale_after ? 1 : 0;
}
# _collector_watchdog_last_progress_epoch($status)
# Extracts the latest meaningful collector progress timestamp from persisted
# status fields so the watchdog can detect live-but-stalled collector loops.
# Input: collector status hash reference.
# Output: latest progress epoch integer or zero when no usable timestamp exists.
sub _collector_watchdog_last_progress_epoch {
my ( $self, $status ) = @_;
return 0 if ref($status) ne 'HASH';
my @epochs;
for my $field (qw(last_completed_at last_started_at last_run)) {
my $timestamp = $status->{$field};
next if !defined $timestamp || $timestamp eq '';
my $epoch = eval { $self->{collectors}->_iso8601_to_epoch($timestamp) };
next if !$epoch;
push @epochs, $epoch;
}
return 0 if !@epochs;
return ( sort { $b <=> $a } @epochs )[0];
}
# _collector_watchdog_stale_seconds($job)
# Builds the maximum no-progress window for a collector from its configured
# interval and timeout plus a small watchdog grace period.
# Input: collector job hash reference.
# Output: positive integer number of seconds.
sub _collector_watchdog_stale_seconds {
my ( $self, $job ) = @_;
$job ||= {};
my $interval = Developer::Dashboard::CollectorRunner::_effective_interval_seconds(
bless( {}, 'Developer::Dashboard::CollectorRunner' ),
$job,
);
my $timeout = defined $job->{timeout_ms} && $job->{timeout_ms} =~ /^\d+$/ && $job->{timeout_ms} > 0
? ( $job->{timeout_ms} / 1000 )
: defined $job->{timeout} && $job->{timeout} =~ /^(?:\d+|\d*\.\d+)$/ && $job->{timeout} > 0
? $job->{timeout}
: 30;
return int( $interval + $timeout + $self->_collector_stall_grace_seconds + 0.999999 );
}
# _collector_watchdog_window($status)
# Normalizes the collector watchdog restart window and counter from persisted
# collector status.
# Input: collector status hash reference.
# Output: restart count integer, window-start ISO-8601 string, and window-start epoch.
sub _collector_watchdog_window {
lib/Developer/Dashboard/RuntimeManager.pm view on Meta::CPAN
# _adopt_web_listener_pid(%args)
# Replaces the transient startup wrapper pid in persisted web state with the
# real listener pid once the PSGI server has rebound under Starman.
# Input: listener_pid integer and optional current runtime-state hash reference.
# Output: adopted listener pid integer or undef when nothing was updated.
sub _adopt_web_listener_pid {
my ( $self, %args ) = @_;
my $listener_pid = $self->_normalized_process_id( $args{listener_pid} );
return if !defined $listener_pid || $listener_pid !~ /^\d+$/ || $listener_pid < 1;
return if !$self->_same_pid_namespace($listener_pid);
my $state = ref( $args{state} ) eq 'HASH'
? { %{ $args{state} } }
: { %{ $self->web_state || {} } };
return if ( $state->{pid} || 0 ) == $listener_pid;
$state->{pid} = $listener_pid + 0;
$state->{status} = 'running';
$state->{updated_at} = _now_iso8601();
my $title = $self->_read_process_title($listener_pid);
$state->{process_name} = $title if defined $title && $title ne '';
$self->{files}->write( 'web_pid', "$listener_pid\n" );
$self->_write_web_state($state);
return $listener_pid;
}
# _normalized_process_id($pid)
# Normalizes one observed process id into a positive integer on platforms such
# as Windows where pseudo-fork bookkeeping can surface a negative startup pid.
# Input: optional process id scalar.
# Output: positive integer process id or the original scalar when it is not a
# numeric pid.
sub _normalized_process_id {
my ( $self, $pid ) = @_;
return $pid if !defined $pid;
return $pid if $pid !~ /^-?\d+$/;
return abs($pid);
}
# _web_runtime_matches_pid($running, $pid, $port)
# Determines whether one observed runtime record matches the expected startup
# pid closely enough to prove the replacement web service stayed up.
# Input: running runtime hash reference, startup pid integer, and requested
# TCP port integer.
# Output: boolean true when the observed runtime matches the expected startup.
sub _web_runtime_matches_pid {
my ( $self, $running, $pid, $port ) = @_;
return 0 if !$running || ref($running) ne 'HASH';
return 1 if ( $running->{pid} || 0 ) == $pid;
return 0 if !is_windows();
my $listener_port = 0;
$listener_port = $port if $port;
$listener_port = $running->{port} if !$listener_port && $running->{port};
return 0 if !$listener_port;
return 0 if ( $running->{port} || 0 ) != $listener_port;
return 1;
}
# _collector_runtime_ready($name, $pid)
# Confirms that a newly started collector loop became visible and stayed alive
# long enough to catch an immediate post-ready crash.
# Input: collector name string and process id integer.
# Output: boolean true when the collector loop became visible and survived the
# short confirmation window afterwards.
sub _collector_runtime_ready {
my ( $self, $name, $pid ) = @_;
return 0 if !defined $name || $name eq '';
return 0 if !defined $pid || $pid !~ /^\d+$/ || $pid < 1;
my $ready_polls = 0;
for ( 1 .. $self->_runtime_stability_polls ) {
my $state = $self->{runner}->can('loop_state') ? $self->{runner}->loop_state($name) : undef;
my $state_ready = $state
&& ( $state->{pid} || 0 ) == $pid
&& ( $state->{name} || $name ) eq $name
&& ( $state->{status} || '' ) =~ /^(?:starting|running|error)$/
&& kill( 0, $pid );
my ($running) = $state_ready
? ()
: grep { $_->{name} eq $name && ( $_->{pid} || 0 ) == $pid } $self->{runner}->running_loops;
if ( $state_ready || $running ) {
$ready_polls++;
return 1 if $ready_polls >= $self->_runtime_confirmation_polls;
}
elsif ($ready_polls) {
return 0;
}
sleep $self->_runtime_poll_interval;
}
return 0;
}
# _runtime_stability_polls()
# Returns the number of readiness polls used to prove that a replacement
# runtime had enough time to become visible before it is declared dead on
# arrival.
# Input: none.
# Output: positive integer poll count.
sub _runtime_stability_polls {
my $override = $ENV{DEVELOPER_DASHBOARD_RUNTIME_STABILITY_POLLS};
return $override if defined $override && $override =~ /^\d+$/ && $override > 0;
my $perl5opt = join ' ', grep { defined && $_ ne '' } @ENV{qw(PERL5OPT HARNESS_PERL_SWITCHES)};
return 300 if $perl5opt =~ /Devel::Cover/ || exists $INC{'Devel/Cover.pm'};
return 300;
}
# _runtime_confirmation_polls()
# Returns the number of consecutive ready polls required after startup first
# becomes visible before the runtime is declared stable.
# Input: none.
# Output: positive integer poll count.
sub _runtime_confirmation_polls {
my $override = $ENV{DEVELOPER_DASHBOARD_RUNTIME_CONFIRMATION_POLLS};
return $override if defined $override && $override =~ /^\d+$/ && $override > 0;
return 3;
}
# _runtime_poll_interval()
# Returns the sleep interval in seconds between runtime readiness polls.
( run in 1.559 second using v1.01-cache-2.11-cpan-14f38c9f855 )