view release on metacpan or  search on metacpan

lib/App/FuguBench/Traces.pm  view on Meta::CPAN

# ex:ts=8 sw=4:
# $OpenBSD$
#
# Copyright (c) 2026 Dick Olsson <hi@senzilla.io>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

package App::FuguBench::Traces;
our $VERSION = '0.1.0';

use v5.34;
use warnings;
use experimental 'signatures';
no feature qw(indirect multidimensional bareword_filehandles);

use Cwd            ();
use File::Basename qw(basename);
use File::Spec     ();
use JSON::PP       ();

use Fugu::CLI qw(EXIT_SUCCESS EXIT_ERROR);
use Fugu::Sandbox;

# App::FuguBench::Traces - the traces verb.
#
# The verb measures the Claude Code sessions of one checkout. Claude
# Code keeps one trace directory for each working directory, under
# ~/.claude/projects/. The name of the directory is the absolute path
# of the working directory, and the harness replaces each character
# outside a letter, a digit and a hyphen with a hyphen.
#
# The verb derives that name from the checkout root, it matches the
# trace directories of the checkout, and it prints one line for each
# session in them that holds a request.
#
# The derivation cuts the root at the last .claude/worktrees/ marker,
# because a nested checkout holds the marker more than one time
# (TRACE-NAME-1). Three name forms belong to one checkout: the
# checkout itself, a worktree of it, and a project clone in either of
# them. The match takes the exact forms, so a sibling checkout, such
# as a backup, stays out (TRACE-NAME-2).
#
# The option --root names a trace root in place of the one under the
# home of the operator. The option --name replaces the derived name
# (TRACE-NAME-3).
#
# The columns are:
#
#     session  the first eight characters of the session identifier
#     start    the time of the first record of the session, in UTC
#     reqs     the requests of the main session
#     peak     the largest context of one request: the fresh input
#              tokens, the cache writes and the cache reads
#     out      the output tokens of the main session, thinking
#              included
#     panel    the rounds of the review panel
#     edits    the file edits of the main session after the first
#              panel launch: a file inside the checkout, outside
#              scratch/ and SCRATCHPAD*.md
#     sub-in   the input tokens of every sub-agent of the session
#     sub-out  the output tokens of every sub-agent
#     rev-peak the largest peak context of one panel reviewer
#
# One request writes one record for each content block, and each
# record carries the usage of the whole request. An early record can
# carry a partial count, so the verb takes the usage of the last
# record of a request (TRACE-USAGE-1). A tool_use block appears in one
# record only, so each block counts one time.
#
# The sub-in and sub-out columns hold every sub-agent together, so
# neither one measures one reviewer (TRACE-SUB-1). A panel launch is a
# tool_use block with an identifier, and each sub-agent trace has a
# sibling <agent>.meta.json that carries the identifier of its launch.
# The rev-peak column maps the launch identifiers of the panel to
# their traces, and it reports the largest peak of them (TRACE-SUB-2).
#
# This verb and the hook verb hold the Claude Code assumptions of the
# program, and every other verb is agent-agnostic (D-10).

# The marker of a worktree path, and the trace root of the harness
# under the home of the operator.
use constant MARKER   => '/.claude/worktrees/';
use constant PROJECTS => '.claude/projects';

# The format of one line. The columns are session, start, reqs, peak,
# out, panel, edits, sub-in, sub-out and rev-peak (TRACE-COLUMNS-2).
# The format lives in a variable: on the floor perl, printf reads a
# bareword in that place as a filehandle, and the pragma block of the
# file forbids one.
my $ROW = "%-8s  %-16s  %6s  %8s  %8s  %6s  %6s  %10s  %10s  %8s\n";

# The tools that change a file. The panel reviews a commit, so an edit
# of the main session after the first launch of a round is an edit
# that no reviewer saw (TRACE-PANEL-3).
my %EDIT = map { $_ => 1 } qw(Edit Write MultiEdit NotebookEdit);

# The scratch space of the repository. The panel writes its ledger
# under scratch/, an audit writes its findings to a SCRATCHPAD-<N>.md
# file, and .gitignore holds both. The name must start a path segment
# and the scratchpad must end the path, so myscratch/x.md,
# NOTSCRATCHPAD.md and SCRATCHPAD-3.md.bak stay edits.

lib/App/FuguBench/Traces.pm  view on Meta::CPAN

		    && ( $rec->{type} // q{} ) eq 'assistant';

		# An old trace carries no requestId, and its records
		# then count one by one, which is the safe direction.
		my $req = $rec->{requestId} // $rec->{uuid} // q{};
		push @order, $req unless exists $usage{$req};
		$usage{$req} = _usage_of($rec);

		for my $block ( @{ $rec->{message}{content} // [] } ) {
			next unless ( $block->{type} // q{} ) eq 'tool_use';

			if ( _is_panel($block) ) {
				$launched                      = 1;
				$round{$req}                   = 1;
				$row{launches}{ $block->{id} } = 1
				    if length( $block->{id} // q{} );
			}
			elsif (    $launched
				&& $EDIT{ $block->{name} // q{} } )
			{
				$row{edits}++
				    if _repo_edit( $block, $checkout );
			}
		}
	}
	close $fh;

	for my $req (@order) {
		my ( $in, $out ) = @{ $usage{$req} };
		$row{in}  += $in;
		$row{out} += $out;
		$row{peak} = $in if $in > $row{peak};
	}
	$row{reqs}  = scalar @order;
	$row{panel} = scalar keys %round;

	return \%row;
}

# _usage_of($rec):
#	The context and the output of one request, in that order. The
#	context is the input that the request paid for: the fresh
#	input, the cache writes and the cache reads (TRACE-COLUMNS-3).
sub _usage_of ($rec)
{
	my $u = $rec->{message}{usage} // {};
	my $in =
	    ( $u->{input_tokens}                // 0 ) +
	    ( $u->{cache_creation_input_tokens} // 0 ) +
	    ( $u->{cache_read_input_tokens}     // 0 );

	return [ $in, $u->{output_tokens} // 0 ];
}

# _repo_edit($block, $checkout):
#	True when one edit block changes a repository file. The target
#	is file_path, or notebook_path for a notebook.
#
#	An absolute target must sit inside the checkout, on a
#	directory boundary, so a write to the home of the operator, or
#	to a sibling such as <checkout>-backup, is no repository file.
#	A relative target sits inside it, because the path resolves
#	against the working directory of the session. A block with no
#	target counts as an edit, which is the safe direction
#	(TRACE-PANEL-3).
#
#	A target in scratch space is no repository file: a path under
#	scratch/, or a SCRATCHPAD*.md file.
sub _repo_edit ( $block, $checkout )
{
	my $input = $block->{input}     // {};
	my $path  = $input->{file_path} // $input->{notebook_path} // q{};
	return 0 if $path =~ m{\A/} && index( $path, "$checkout/" ) != 0;

	return $path =~ $SCRATCH ? 0 : 1;
}

# _is_panel($block):
#	One launch of one panel member. The type of the agent decides
#	first: a reviewer is a member, and another role, such as a
#	fixer, is not one. A catch-all type and an absent type name no
#	role, so the description decides. The panel of an early
#	session dispatched a catch-all agent, and the description of
#	each member names the panel (TRACE-PANEL-1).
sub _is_panel ($block)
{
	return 0 unless $LAUNCH{ $block->{name} // q{} };

	my $input = $block->{input}         // {};
	my $type  = $input->{subagent_type} // q{};
	return 1 if $type eq 'reviewer';
	return 0 if length $type && !$CATCHALL{$type};

	return ( $input->{description} // q{} ) =~ /panel/i ? 1 : 0;
}

# _meta_id($path):
#	The launch identifier of one sub-agent trace. The trace has a
#	sibling meta file, and the file names the tool_use block that
#	launched the sub-agent. A trace with no meta file gives the
#	empty string, which matches no launch (TRACE-SUB-2).
sub _meta_id ($path)
{
	my $meta = $path =~ s/\.jsonl\z/.meta.json/r;
	open my $fh, '<', $meta or return q{};
	my $text = do { local $/ = undef; <$fh> };
	close $fh;

	my $rec = eval { $JSON->decode( $text // q{} ) };

	return ref $rec eq 'HASH' ? $rec->{toolUseId} // q{} : q{};
}

1;