view release on metacpan or  search on metacpan

lib/Langertha/Skeid/Proxy.pm  view on Meta::CPAN

    };

    if ($body->{stream}) {
      _proxy_openai_stream($c, $url, $body, $node_id, $started, $meta);
      return;
    }

    $c->render_later;
    _proxy_openai_json_async($c, $url, $body, $node_id, $started, $meta, sub {
      my ($res, $err, $status) = @_;
      return if $err;
      _render_upstream_response($c, $res, $node_id);
    });
  });
}

sub _handle_openai_embeddings {
  my ($c) = @_;
  my $body = $c->req->json;
  unless (ref($body) eq 'HASH') {
    $c->render(json => { error => { message => 'Invalid JSON body', type => 'invalid_request_error' } }, status => 400);
    return;
  }

  my $model = $body->{model} // '';
  my $api_key_id = _request_api_key_id($c);
  _begin_route_async($c, $model, sub {
    my ($route, $node_id, $started) = @_;
    return unless $route;

    my $url = _endpoint_url_for_node($route->{url}, '/embeddings');
    my $meta = {
      api_format => 'openai',
      endpoint   => '/v1/embeddings',
      api_key_id => $api_key_id,
      provider   => 'skeid',
      engine     => ($route->{engine} // 'openaibase'),
      model      => $model,
      route_url  => ($route->{url} // ''),
    };
    $c->render_later;
    _proxy_openai_json_async($c, $url, $body, $node_id, $started, $meta, sub {
      my ($res, $err, $status) = @_;
      return if $err;
      _render_upstream_response($c, $res, $node_id);
    });
  });
}

sub _handle_anthropic_messages {
  my ($c) = @_;
  my $body = $c->req->json;
  unless (ref($body) eq 'HASH') {
    $c->render(json => { error => { message => 'Invalid JSON body', type => 'invalid_request_error' } }, status => 400);
    return;
  }

  if ($body->{stream}) {
    $c->render(json => {
      error => {
        message => 'Anthropic streaming is not implemented in Skeid proxy yet',
        type    => 'not_supported_error',
      }
    }, status => 501);
    return;
  }

  my $openai_body = _anthropic_request_to_openai($body);
  my $model = $openai_body->{model} // '';
  my $api_key_id = _request_api_key_id($c);

  _begin_route_async($c, $model, sub {
    my ($route, $node_id, $started) = @_;
    return unless $route;

    my $url = _endpoint_url_for_node($route->{url}, '/chat/completions');
    my $meta = {
      api_format => 'anthropic',
      endpoint   => '/v1/messages',
      api_key_id => $api_key_id,
      provider   => 'skeid',
      engine     => ($route->{engine} // 'openaibase'),
      model      => $model,
      route_url  => ($route->{url} // ''),
    };
    $c->render_later;
    _proxy_openai_json_async($c, $url, $openai_body, $node_id, $started, $meta, sub {
      my ($res, $err, $status) = @_;
      return if $err;

      my $payload = _openai_response_to_anthropic($res, $model);
      $c->res->code($status || 200);
      $c->res->headers->header('x-skeid-node' => $node_id);
      $c->render(json => $payload);
    });
  });
}

sub _handle_ollama_chat {
  my ($c) = @_;
  my $body = $c->req->json;
  unless (ref($body) eq 'HASH') {
    $c->render(json => { error => 'Invalid JSON body' }, status => 400);
    return;
  }

  if ($body->{stream}) {
    $c->render(json => { error => 'Ollama streaming is not implemented in Skeid proxy yet' }, status => 501);
    return;
  }

  my $openai_body = {
    model => ($body->{model} // ''),
    messages => ($body->{messages} || []),
    (defined($body->{options}{temperature}) ? (temperature => 0 + $body->{options}{temperature}) : ()),
    (defined($body->{options}{num_predict})  ? (max_tokens  => 0 + $body->{options}{num_predict})  : ()),
    (defined($body->{tools}) ? (tools => $body->{tools}) : ()),
    (defined($body->{tool_choice}) ? (tool_choice => $body->{tool_choice}) : ()),
  };

  _begin_route_async($c, $openai_body->{model}, sub {
    my ($route, $node_id, $started) = @_;
    return unless $route;

    my $url = _endpoint_url_for_node($route->{url}, '/chat/completions');
    my $meta = {
      api_format => 'ollama',
      endpoint   => '/api/chat',
      api_key_id => _request_api_key_id($c),
      provider   => 'skeid',
      engine     => ($route->{engine} // 'openaibase'),
      model      => ($openai_body->{model} // ''),
      route_url  => ($route->{url} // ''),
    };
    $c->render_later;
    _proxy_openai_json_async($c, $url, $openai_body, $node_id, $started, $meta, sub {
      my ($res, $err, $status) = @_;
      return if $err;

      my $payload = _openai_response_to_ollama_chat($res);
      $c->res->code($status || 200);
      $c->res->headers->header('x-skeid-node' => $node_id);
      $c->render(json => $payload);
    });
  });
}

sub _begin_route {
  my ($c, $model) = @_;
  my $wait_timeout_ms = 0 + ($c->skeid->route_wait_timeout_ms // 0);
  my $wait_poll_ms = 0 + ($c->skeid->route_wait_poll_ms // 25);
  $wait_poll_ms = 1 if $wait_poll_ms < 1;
  $wait_timeout_ms = 0 if $wait_timeout_ms < 0;

  my $started = time;
  my $deadline = $started + ($wait_timeout_ms / 1000);
  my $last_state;
  my $last_node_id = '';

  while (1) {
    my $state = $c->skeid->call_function('route.state', { model => ($model // '') });
    $last_state = $state if ref($state) eq 'HASH';

    # Wrong model / no healthy nodes for this model: fail fast.
    if (!$last_state || !$last_state->{has_eligible}) {
      $c->render(json => {
        error => {
          message => "No healthy node available for model '$model'",