LLM Latency Estimator

export function llmLatency(input: {
  networkMs: number;
  queueAndPromptMs: number;
  outputTokens: number;
  tokensPerSecond: number;
  clientOverheadMs: number;
  tailBufferPercent: number;
}) {
  const timeToFirstTokenMs =
    input.networkMs + input.queueAndPromptMs;
  const generationMs =
    (Math.max(0, input.outputTokens - 1) /
      input.tokensPerSecond) *
    1000;
  const endToEndMs =
    timeToFirstTokenMs + generationMs + input.clientOverheadMs;

  return {
    timeToFirstTokenMs,
    generationMs,
    bufferedEndToEndMs:
      endToEndMs * (1 + input.tailBufferPercent / 100),
  };
}