Timeout results from the CPAN

Timeout

App-FuguVM

view release on metacpan or search on metacpan

lib/App/FuguVM/Guest.pm view on Meta::CPAN

# ex:ts=8 sw=4:
# $OpenBSD$
#
# Copyright (c) 2024 Dick Olsson <hi@senzilla.io>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

use v5.36;

# App::FuguVM::Guest - the lifecycle of one OpenBSD guest.
#
# The module creates, starts, waits for, stops, and destroys one
# guest. It drives QEMU over QMP, so the lifecycle verbs report what
# the hypervisor says and not what a sleep guessed.

package App::FuguVM::Guest;
our $VERSION = '0.3.0';

use App::FuguVM::Arch;
use App::FuguVM::Autoinstall;
use App::FuguVM::Config;
use App::FuguVM::Miniroot;
use App::FuguVM::Mirror;
use App::FuguVM::DiskCache;
use App::FuguVM::Disk;
use App::FuguVM::Console;
use App::FuguVM::Proxy;
use App::FuguVM::QMP;
use App::FuguVM::State;

use Fcntl qw(:flock);
use Fugu::File;
use Fugu::Random;
use Fugu::Process;
use Fugu::SSH;
use Fugu::Timeout;

use constant {
	EXIT_SUCCESS       => 0,
	EXIT_ERROR         => 1,
	EXIT_CONFIG_ERROR  => 3,
	EXIT_VM_RUNNING    => 5,
	EXIT_TIMEOUT       => 7,
	EXIT_EXPECT_FAILED => 9,

	MEMORY_DEFAULT => '1G',
	CPU_COUNT      => 2,

	# The port lock closes the window between the probe and the
	# record of both ports. A probe is quick, so a long wait means
	# a wedged holder, and the probe then runs without the lock.
	PORT_LOCK_TIMEOUT => 30,

	# The bound on one 'qemu --version' run.
	QEMU_VERSION_TIMEOUT => 10,
};

sub new ( $class, %args )
{
	my $self = bless {
		config   => $args{config},
		state    => $args{state},
		log      => $args{log},
		emulate  => $args{emulate}  // 0,
		no_cache => $args{no_cache} // 0,
	}, $class;

	return $self;
}

# The operation is idempotent. It makes sure that the VM runs.
sub up ($self)
{
	my $config = $self->{config};
	my $state  = $self->{state};
	my $log    = $self->{log};

	# The QEMU binary of the architecture must be on PATH before
	# any other work starts.
	return EXIT_CONFIG_ERROR if !$self->_require_qemu;

	return EXIT_ERROR if !$self->_check_installed_arch;

	# Check if the VM already runs
	if ( $self->_is_running ) {

		# The VM runs, but the SSH key is not installed or is not
		# current. This occurs when the first boot failed, or when
		# the key changed in the configuration.
		if ( $state->is_installed && $self->_needs_ssh_key_update ) {
			return $self->_complete_ssh_setup;
		}

		$log->info("VM '$config->{name}' is already running");
		return EXIT_SUCCESS;
	}

lib/App/FuguVM/Guest.pm view on Meta::CPAN

		}
		$state->clear_vm_pid;

		# The runtime record stays: the restart below uses the
		# same ports, and a sibling that resolves ports in this
		# window must still skip them.

		# Publish the installed disk as a cached base image. A VM
		# that was force stopped can leave the disk mid-write. Thus
		# the code skips that capture and does not publish it.
		if ( defined $cache_key ) {
			if ($clean_exit) {
				$self->_cache_store( $cache, $cache_key,
					$root_password );
			}
			else {
				$log->warning(
"Skipping image cache: installation VM was force stopped"
				);
			}
		}

		# The entry is published, or this run cannot publish it.
		# Either way a waiting sibling can proceed now.
		if ( defined $cache_lock ) {
			close $cache_lock;
			undef $cache_lock;
		}

		# Restart the VM without the install media
		$log->info("Restarting installed system...");
		$pid = $self->_start_qemu;    # No boot image, no exit_on_halt
		if ( !defined $pid ) {
			$log->error("Failed to restart VM");
			return EXIT_ERROR;
		}
		$log->info("Started $config->{name} (PID: $pid)");

		# Install the SSH authorized key for future key-based
		# authentication. Outside the expect mode a guest can
		# carry no configured key: the image must trust the key
		# of the operator already, and the wait below proves it.
		if ( $mode eq 'expect' || $self->_needs_ssh_key_update ) {
			return $self->_complete_ssh_setup;
		}
	}

	# The VM is installed. Check if the SSH key must be installed or
	# updated.
	if ( $self->_needs_ssh_key_update ) {

		# The SSH key is not installed, or it changed in the
		# configuration. Use password authentication to wait for
		# SSH. Then install the key.
		return $self->_complete_ssh_setup;
	}

	# Wait for SSH. An installed VM uses key-based authentication.
	$log->info("Waiting for SSH...");
	if ( !$self->wait_ssh(120) ) {
		$log->error("Timeout waiting for SSH");
		return EXIT_TIMEOUT;
	}

	$log->info("VM ready");
	return EXIT_SUCCESS;
}

# $self->_image_cache:
#	Return the installed-image cache for this VM's configured
#	cache_dir. Return undef when caching is off. 'up --no-cache'
#	turns caching off for a single invocation. 'image_cache no'
#	turns it off in the configuration. Both stop restore and save
#	together. A half-cached run would leave an overlay with a base
#	that nothing published.
sub _image_cache ($self)
{
	return if $self->{no_cache};

	my $enabled = $self->{config}{image_cache};
	return if defined $enabled && !$enabled;

	return App::FuguVM::DiskCache->new( $self->_cache_dir );
}

# $self->_verify_backing_chain:
#	Make sure that the backing image of the working disk, if the
#	disk has one, is present. Return true when the chain resolves.
#	On a break, log the missing file and a remedy, and return
#	false. Thus a pruned or evicted cache entry fails with an
#	explanation, not with an opaque QEMU open error at boot.
sub _verify_backing_chain ($self)
{
	my $config = $self->{config};
	my $state  = $self->{state};
	my $log    = $self->{log};

	my $disk    = App::FuguVM::Disk->new( $state->state_dir );
	my $backing = $disk->backing_file( $config->{name} );
	return 1 if !defined $backing;
	return 1 if -f $backing;

	$log->error("Backing image missing: $backing");

	my $cache_dir = $self->_cache_dir;
	if ( index( $backing, "$cache_dir/" ) == 0 ) {
		$log->error(
"The image cache no longer holds this disk's base image."
		);
	}
	$log->error("Run 'fuguvm destroy' and 'fuguvm up' to rebuild the VM.");

	return 0;
}

# $self->_cache_restore($cache, $key):
#	Create the working disk as an overlay on a cached base image.
#	Seed the state that the installation would have written. Return
#	true on a cache hit. Return false on a miss or on any failure.
#	In both cases the caller then installs from scratch.
sub _cache_restore ( $self, $cache, $key )

lib/App/FuguVM/Guest.pm view on Meta::CPAN

sub wait_ssh ( $self, $timeout = 120, $password = undef )
{
	my $ssh = Fugu::SSH->new(
		host => $self->connect_address,
		port => $self->ssh_port,
		user => 'root',
		( defined $password ? ( password => $password ) : () ),
	);

	return $ssh->wait_available($timeout);
}

# $self->_needs_ssh_key_update:
#	Check if the SSH key must be installed or updated. Return true
#	if no key is installed. Also return true if the configured key
#	differs from the installed key.
sub _needs_ssh_key_update ($self)
{
	my $config = $self->{config};
	my $state  = $self->{state};

	my $configured_key = $config->{ssh_pubkey};
	my $installed_key  = $state->get_installed_ssh_pubkey;

	# No key is configured. There is nothing to install.
	return 0 if !defined $configured_key || $configured_key eq '';

	# No key is installed yet
	return 1 if !defined $installed_key;

	# Compare the keys. Normalize the whitespace for the comparison.
	my $configured_normalized = $configured_key =~ s/\s+/ /gr;
	my $installed_normalized  = $installed_key  =~ s/\s+/ /gr;

	return $configured_normalized ne $installed_normalized;
}

# $self->_complete_ssh_setup():
#	Install or update the SSH key on the VM. The method
#	authenticates with the stored root password. It runs to recover
#	from a failed first boot, or when the configured SSH key
#	changed.
sub _complete_ssh_setup ($self)
{
	my $state  = $self->{state};
	my $config = $self->{config};
	my $log    = $self->{log};

	my $root_password = $state->get_root_password;
	if ( !defined $root_password ) {
		$log->error(
			"No root password stored - cannot complete SSH setup");
		return EXIT_ERROR;
	}

	$log->info("Updating SSH key...");

	# Wait for SSH with password authentication
	$log->info("Waiting for SSH...");
	if ( !$self->wait_ssh( 120, $root_password ) ) {
		$log->error("Timeout waiting for SSH");
		return EXIT_TIMEOUT;
	}

	# Install the SSH authorized key
	if ( !$self->_install_ssh_key($root_password) ) {
		$log->error("Failed to install SSH key");
		return EXIT_ERROR;
	}
	$log->info("SSH key installed");

	$log->info("VM ready");
	return EXIT_SUCCESS;
}

# $self->_install_ssh_key($password):
#	Install the SSH public key from the configuration into
#	authorized_keys. The method uses password authentication,
#	because the key is not yet installed.
sub _install_ssh_key ( $self, $password )
{
	my $config = $self->{config};
	my $state  = $self->{state};
	my $log    = $self->{log};

	# Get the SSH public key from the configuration
	my $ssh_pubkey = $config->{ssh_pubkey};
	if ( !defined $ssh_pubkey || $ssh_pubkey eq '' ) {
		$log->error("No ssh_pubkey configured in ~/.fuguvmrc");
		return 0;
	}

	# Connect with the password
	my $ssh = Fugu::SSH->new(
		host     => $self->connect_address,
		port     => $self->ssh_port,
		user     => 'root',
		password => $password,
	);

	# Create the .ssh directory
	my $result =
	    $ssh->run_command('mkdir -p /root/.ssh && chmod 700 /root/.ssh');
	if ( $result->{exit_code} != 0 ) {
		return 0;
	}

	# Write the authorized_keys file
	my $authkeys_content = $ssh_pubkey . "\n";
	if (
		$ssh->write_file(
			'/root/.ssh/authorized_keys', $authkeys_content,
			0600
		) != 0
	    )
	{
		return 0;
	}

	# Store the installed pubkey for a future comparison
	$state->mark_ssh_key_installed($ssh_pubkey);
	return 1;
}

# Graceful shutdown with a filesystem sync: sync through SSH, then
# power off through the ACPI power button, and report the result.
sub _graceful_shutdown ($self)
{
	my $log = $self->{log};

	# Do a best-effort filesystem sync over SSH before the code
	# pulls the power. The sync has a hard time bound. A wedged
	# guest must never stall the shutdown. Also, libssh2 does not
	# reliably obey its own timeout on the connect and handshake. A
	# failure or a timeout here is acceptable. The ACPI powerdown
	# below runs the orderly shutdown of the guest, which syncs. A
	# force stop is the ultimate fallback.
	$self->_bounded(
		Fugu::SSH::DEFAULT_TIMEOUT() + 5,
		sub {
			my $ssh = Fugu::SSH->new(
				host => $self->connect_address,
				port => $self->ssh_port,
				user => 'root',
			);
			return $ssh->run_command('sync; sync; sync');
		} );

	# Ask the guest to power off through the ACPI power button. Then
	# wait.
	if ( $self->_qmp_powerdown && $self->_wait_exit(60) ) {
		$log->info("Shutdown via ACPI powerdown");
		return 1;
	}

	return 0;
}

# $self->_bounded($seconds, $code):
#	Run $code under a hard wall-clock deadline, so a blocked guest
#	interaction cannot stall the caller. The guard itself is
#	Fugu::Timeout; this wrapper adds the log line.
sub _bounded ( $self, $seconds, $code )
{
	my $result = Fugu::Timeout::bounded( $seconds, $code );
	return $result if defined $result;

	$self->{log}->warning("Guest did not respond within ${seconds}s");
	return;
}

# $self->_ensure_proxy:
#	Start the caching proxy if it does not run, and return it. The
#	method returns undef when the proxy cannot start.
#
#	The lifecycle lives here and not in App::FuguVM::State, because a
#	state file must not start a process. That split is what removed
#	the require cycle between the two modules.
sub _ensure_proxy ($self)
{
	my $proxy = $self->_proxy;
	return $proxy if $proxy->is_running;

	# The spawn passes a fixed argument list, so the distfile cap
	# reaches the child through the environment.
	$ENV{FUGUVM_DISTFILE_LIMIT} = $self->{config}{distfile_cache} // 0;

	unless ( defined $proxy->start ) {
		$self->{log}->warning(
			'Proxy did not start: %s',
			$proxy->error // 'unknown'
		);
		return;
	}

	return $proxy;
}

# $self->_stop_proxy:
#	Stop the proxy if it runs. The method returns 1 when it stopped
#	one, and 0 when there was none.
sub _stop_proxy ($self)
{
	my $proxy = $self->_proxy;
	return 0 unless $proxy->is_running;

	$proxy->stop;

	return 1;
}

# $self->_proxy:
#	Build the proxy supervisor over this VM's state.
sub _proxy ($self)
{
	my $state = $self->{state};

	return App::FuguVM::Proxy->new(
		cache => App::FuguVM::Proxy::Cache->new(
			$self->_cache_dir, $self->{config}{distfile_cache} // 0
		),
		pidfile => $state->proxy_pidfile,
		store   => $state->store,
		logfile => $state->vm_state_dir . '/proxy.log',
		log     => $self->{log},

lib/App/FuguVM/Guest.pm view on Meta::CPAN

	# accelerator by the host capability.
	push @cmd, '-M', $arch->machine;
	push @cmd, $self->_accel_args;

	# Memory and CPU
	push @cmd, '-m',   $config->{memory} // MEMORY_DEFAULT;
	push @cmd, '-smp', CPU_COUNT;

	# EFI firmware of the architecture. Both machines boot through
	# EFI, so a start without a firmware file cannot work. Fail
	# with a message rather than boot into the wrong firmware.
	my $firmware = $self->_find_efi_firmware;
	if ( !defined $firmware ) {
		$self->{log}->error(
			sprintf( "No EFI firmware for %s guests found",
				$self->_arch->name ) );
		return;
	}
	my @args = $self->_firmware_args($firmware);
	return unless @args;
	push @cmd, @args;

	# The main disk with the safe cache mode. The writethrough mode
	# syncs on each write.
	my $disk_path = $state->disk_path;
	push @cmd, '-drive',
	    "file=$disk_path,format=qcow2,if=virtio,cache=writethrough";

	# Boot image (CD-ROM) for installation
	push @cmd, $self->_media_args($boot_image);

	# Network with port forwarding, and the serial console on
	# telnet
	my $console_port = $state->get_runtime->{console_port};
	push @cmd, $self->_network_args;
	push @cmd, $self->_serial_args;

	# QMP control socket
	my $qmp_path = $self->_qmp_socket_path;
	unlink $qmp_path if -S $qmp_path;
	push @cmd, '-qmp', "unix:$qmp_path,server,nowait";

	# PID file for reliable tracking
	push @cmd, '-pidfile', $state->vm_pidfile->path;

	# No graphics display (headless)
	push @cmd, '-display', 'none';

	# Use Fugu::Process to spawn QEMU
	my $log_file = $state->vm_state_dir . '/qemu.log';
	my $result   = Fugu::Process->spawn_command(
		cmd       => \@cmd,
		daemonize => 1,
		stdout    => $log_file,
		stderr    => $log_file,
	);

	return unless $result->{success};

	# Wait until QEMU writes the PID file
	my $pid = Fugu::Timeout::wait_until(
		5, 0.1,
		sub {
			my $qemu_pid = $state->get_vm_pid;
			return $qemu_pid
			    if defined $qemu_pid
			    && Fugu::Process->is_alive($qemu_pid);
			return;
		} );

	unless ( defined $pid ) {
		$self->_dump_qemu_log($log_file);
		return;
	}

	# Arm the crash detection: was_unclean_shutdown reports true
	# when the state says running and the process is gone.
	$state->mark_running;

	# Make sure that QEMU accepts console connections before the
	# installer tries to attach. A QEMU that exited at startup, for
	# example with a bad accelerator or missing firmware, leaves the
	# port closed. This check fails fast with the QEMU log, not with
	# a long telnet timeout later.
	if ( defined $boot_image
		&& !$self->_wait_console_ready( $console_port, 30 ) )
	{
		$self->{log}
		    ->error( 'QEMU console port %d not listening after start',
			$console_port );
		$self->_dump_qemu_log($log_file);
		return;
	}

	return $pid;
}

# $self->_media_args($boot_image):
#	Return the QEMU arguments of the install media, or an empty
#	list when no media is attached. An autoinstall reboots
#	itself, and the miniroot is still attached. -no-reboot makes
#	the reboot an exit, so the guest cannot install a second
#	time.
sub _media_args ( $self, $boot_image = undef )
{
	return () if !defined $boot_image;

	my @args =
	    ( '-drive', "file=$boot_image,format=raw,if=virtio,readonly=on" );
	push @args, '-no-reboot'
	    if ( $self->{config}{install_mode} // '' ) eq 'autoinstall';

	return @args;
}

# $self->_network_args:
#	Return the QEMU network arguments. The ports come from the
#	record that _resolve_ports wrote before the spawn: the public
#	accessors serve a running guest only, and QEMU does not run
#	yet. The forwarded port binds to the configured host address,
#	and the default is loopback.
sub _network_args ($self)
{
	my $bind_address = $self->bind_address;
	my $ssh_port     = $self->{state}->get_runtime->{ssh_port};

	return ( '-device', 'virtio-net-pci,netdev=net0', '-netdev',
		"user,id=net0,hostfwd=tcp:$bind_address:" . "$ssh_port-:22",
	);
}

# $self->_serial_args:
#	Return the QEMU serial-console arguments: a telnet listener on
#	the bind address and the recorded console port.
sub _serial_args ($self)
{
	my $bind_address = $self->bind_address;
	my $console_port = $self->{state}->get_runtime->{console_port};

	return ( '-serial',
		"tcp:$bind_address:$console_port,server,telnet,nowait" );
}

# $self->_wait_console_ready($port, $timeout):
#	Poll the console TCP port until it accepts a connection. Thus
#	the telnet of the installer attaches to a live console. QEMU
#	binds the port at startup, before the guest boots. Thus the
#	poll is quick when QEMU is healthy, and bounded when it is not.
sub _wait_console_ready ( $self, $port, $timeout )
{
	require IO::Socket::INET;

	my $ready = Fugu::Timeout::wait_until(
		$timeout, 0.2,
		sub {
			my $sock = IO::Socket::INET->new(
				PeerAddr => $self->connect_address,
				PeerPort => $port,
				Proto    => 'tcp',
				Timeout  => 2,
			);
			if ( defined $sock ) {
				$sock->close;
				return 'ready';
			}

			# Stop the wait early if QEMU already exited
			my $qemu_pid = $self->{state}->get_vm_pid;
			return 'gone'
			    if defined $qemu_pid
			    && !Fugu::Process->is_alive($qemu_pid);

			return;
		} );

	return defined $ready && $ready eq 'ready' ? 1 : 0;
}

# $self->_dump_qemu_log($log_file):
#	Show the tail of the QEMU log. Thus a startup failure is
#	visible in the CI output, and shell access to the runner is not
#	necessary.
sub _dump_qemu_log ( $self, $log_file )
{
	open my $fh, '<', $log_file or return;
	my @lines = <$fh>;
	close $fh;

	@lines = splice( @lines, -40 ) if @lines > 40;
	$self->{log}->error('QEMU log tail:');
	$self->{log}->error( '  %s', $_ ) for map { chomp; $_ } @lines;

	return;
}

# $self->_accel_args():
#	Return the accelerator arguments, with the matching CPU model.
#	Host CPU passthrough is only valid with hardware acceleration.
#	TCG needs the named model of the architecture. The choice
#	itself comes from accel.
sub _accel_args ($self)
{
	my $arch  = $self->_arch;
	my $accel = $self->accel;

	$self->{log}->debug("Using QEMU accelerator: $accel")
	    if $self->{log};

	return ( '-accel', $accel,
		'-cpu', $accel eq 'tcg' ? $arch->tcg_cpu : 'host' );
}

# $self->_arch:
#	Return the App::FuguVM::Arch object of the configured value,
#	and cache it. The configuration loader is the boundary of the
#	directive, so an unknown value here is a programming error.
sub _arch ($self)
{
	my $name = $self->{config}{arch};

lib/App/FuguVM/Guest.pm view on Meta::CPAN

			Listen    => 1,
		);
		next if !defined $sock;

		$sock->close;
		return $port;
	}

	return;
}

# $self->_taken_ports:
#	Return the recorded ports of every guest of the project, as a
#	hash reference keyed by port. The method enumerates the state
#	directory, like App::FuguVM::CLI::_disks_backed_by: a record
#	counts whether a 'vm' block still declares its guest or not.
sub _taken_ports ($self)
{
	my %taken;

	my $state_dir = $self->{state}->state_dir;
	return \%taken if !-d $state_dir;

	opendir my $dh, $state_dir or return \%taken;
	my @names = sort grep { !/^\./ && -d "$state_dir/$_" } readdir $dh;
	closedir $dh;

	for my $name (@names) {
		my $sibling = App::FuguVM::State->new( $state_dir, $name )
		    or next;
		my $runtime = $sibling->get_runtime;
		for my $directive (qw(ssh_port console_port)) {
			my $port = $runtime->{$directive};
			$taken{$port} = 1 if defined $port;
		}
	}

	return \%taken;
}

# $self->_lock_ports:
#	Return the locked handle of ports.lock, or undef on the
#	deadline. The lock file lives in the cache directory, so every
#	project that shares that directory probes one port at a time.
#	The record exclusion of _taken_ports covers this project only.
#	A collision with an other project surfaces when QEMU binds the
#	port, as a reported startup failure. The caller probes without
#	the lock when the deadline elapses: a wedged holder must not
#	fail a run.
sub _lock_ports ($self)
{
	my $dir = $self->_cache_dir;
	Fugu::File->ensure_dir($dir) or return;

	my $path = "$dir/ports.lock";
	open my $fh, '>>', $path or do {
		$self->{log}->warning("Cannot open $path: $!");
		return;
	};

	my $locked = Fugu::Timeout::bounded( PORT_LOCK_TIMEOUT,
		sub { flock $fh, LOCK_EX } );
	return $fh if $locked;

	close $fh;
	$self->{log}->warning("Port lock not acquired, probing without it");

	return;
}

# $self->_check_installed_arch:
#	Report if the configured architecture matches the installed
#	disk, once for up and start. A disk belongs to one
#	architecture, so a changed directive must not start the wrong
#	QEMU on an existing disk. An absent record cannot prove a
#	difference, so the check passes.
sub _check_installed_arch ($self)
{
	my $config = $self->{config};

	my $installed = $self->{state}->get_installed_arch;
	return 1 if !defined $installed || $installed eq $config->{arch};

	$self->{log}->error(
"The disk of '$config->{name}' holds an $installed installation, not $config->{arch}"
	);
	$self->{log}->error("Run 'fuguvm destroy' to rebuild the VM.");

	return 0;
}

# _host_arch():
#	Return the host machine architecture from uname.
sub _host_arch ()
{
	require POSIX;
	my @uname = POSIX::uname();
	return $uname[4] // '';
}

# $self->_find_efi_firmware:
#	Return the firmware of the architecture as { code, vars }, or
#	undef. The vars entry is the variable-store template beside
#	the code file. It is undef when -bios boots the code file
#	alone. A code file without its template is not usable, so the
#	search walks on.
sub _find_efi_firmware ($self)
{
	my $arch = $self->_arch;

	my @candidates = $arch->firmware_paths;
	push @candidates, glob( $arch->firmware_glob );

	for my $code (@candidates) {
		next unless -f $code;

		my $vars = $arch->firmware_vars_path($code);
		next if defined $vars && !-f $vars;

		return { code => $code, vars => $vars };
	}

( run in 1.583 second using v1.01-cache-2.11-cpan-85d3896f969 )