view release on metacpan or  search on metacpan

t/0044-full-weight-training.t  view on Meta::CPAN

    for my $step (1..3) {
        $optimizer->zero_grad();
        
        my $ctx = Lugh::Context->new(size => 64 * 1024 * 1024);
        my $logits = Lugh::Train->forward(
            inference  => $opt_inference,
            context    => $ctx,
            tokens     => [1, 72, 101],
            train_lora => 0,
            train_full => 1,
        );
        
        Lugh::Train->register_weight_tensors($logits, \@opt_weights);
        
        my $loss = Lugh::Train->cross_entropy_loss($ctx, $logits, [72, 101, 108]);
        $loss->backward();
        $optimizer->step();
    }
    
    pass('completed 3 optimizer training steps');
    
    done_testing();
};

# =============================================================================
# GRADIENT ACCUMULATION
# =============================================================================

subtest 'gradient accumulation' => sub {
    my $optimizer = Lugh::Optimizer::AdamW->new(lr => 0.01);
    $optimizer->add_param($_) for @weights;
    
    $optimizer->zero_grad();
    
    # Accumulate gradients from multiple forward/backward
    for my $i (1..3) {
        my $ctx = Lugh::Context->new(size => 64 * 1024 * 1024);
        my $logits = Lugh::Train->forward(
            inference  => $inference,
            context    => $ctx,
            tokens     => [1, 72, 101],
            train_lora => 0,
            train_full => 1,
        );
        
        Lugh::Train->register_weight_tensors($logits, \@weights);
        
        my $loss = Lugh::Train->cross_entropy_loss($ctx, $logits, [72, 101, 108]);
        lives_ok { $loss->backward() } "accumulation step $i backward";
    }
    
    lives_ok { $optimizer->step() } 'optimizer step after accumulation';
    
    done_testing();
};

# =============================================================================
# STRESS TEST - MANY ITERATIONS (REGRESSION TEST FOR MEMORY CORRUPTION)
# =============================================================================

subtest 'stress test many iterations' => sub {
    # This catches memory corruption that manifests after many iterations
    my $iterations = 50;
    
    for my $i (1..$iterations) {
        my $ctx = Lugh::Context->new(size => 64 * 1024 * 1024);
        
        my $logits = Lugh::Train->forward(
            inference  => $inference,
            context    => $ctx,
            tokens     => [1, 65, 66, 67],  # 4 tokens
            train_lora => 0,
            train_full => 1,
        );
        
        Lugh::Train->register_weight_tensors($logits, \@weights);
        
        my $loss = Lugh::Train->cross_entropy_loss($ctx, $logits, [65, 66, 67, 68]);
        
        eval { $loss->backward() };
        if ($@) {
            fail("iteration $i crashed: $@");
            last;
        }
    }
    
    pass("completed $iterations iterations without crash");
    
    done_testing();
};

sub min { $_[0] < $_[1] ? $_[0] : $_[1] }

done_testing();