Laminar evaluations SDK and CLI reference

abstract class Dataset<D, T> {
  public async slice(start: number, end: number): Promise<Datapoint<D, T>[]> {
    const result = [];
    for (let i = Math.max(start, 0); i < Math.min(end, await this.size()); i++) {
      result.push(await this.get(i));
    }
    return result;
  }
  public abstract size(): Promise<number> | number;
  public abstract get(index: number): Promise<Datapoint<D, T>> | Datapoint<D, T>;
}

type EvaluatorFunctionReturn = number | Record<string, number>;
type EvaluatorFunction<O, T> = (output: O, target?: T, ...args: any[]) =>
  EvaluatorFunctionReturn | Promise<EvaluatorFunctionReturn>;

async function evaluate<D, T, O>({
  data, executor, evaluators, groupName, name, config,
}: {
  /**
   * List of data points to evaluate. `data` is the input to the executor
   * function,
   * `target` is the input to the evaluator function.
   * `Dataset` is the base class for `LaminarDataset`
   */
  data: (Datapoint<D, T>[]) | Dataset<D, T>;
  /**
   * The executor function. Takes the data point + any additional arguments
   * and returns the output to evaluate.
   */
  executor: (data: D, ...args: any[]) => O | Promise<O>;
  /**
   * Evaluator functions and names. Each evaluator function takes the output of
   * the executor _and_ the target data, and returns a score. The score can be a
   * single number or a dict of string keys and number values. If the score is a
   * single number, it will be named after the evaluator function. Evaluator
   * function names must contain only letters, digits, hyphens, underscores,
   * or spaces.
   */
  evaluators: Record<string, EvaluatorFunction<O, T>>;
  /**
   * Name of the evaluation. If not provided, a random name will be assigned.
   */
  name?: string;
  /**
   * Optional group id of the evaluation. Only evaluations within the same
   * group_id can be visually compared. Defaults to "default".
   */
  groupName?: string;
  /**
   * Optional override configurations for the evaluator.
   */
  config?: EvaluatorConfig;
}): Promise<void>;

/**
 * Configuration for the Evaluator
 */
interface EvaluatorConfig {
  /**
   * The number of data points to evaluate in parallel at a time. Defaults to 5.
   */
  concurrencyLimit?: number;
  /**
   * The project API key to use for the evaluation. If not provided,
   * the API key from the environment variable `LMNR_PROJECT_API_KEY` will be used.
   */
  projectApiKey?: string;
  /**
   * The base URL of the Laminar API. If not provided, the default is
   * `https://api.lmnr.ai`. Useful with self-hosted Laminar instances.
   * Do NOT include the port in the URL, use `httpPort` and `grpcPort` instead.
   */
  baseUrl?: string;
  /**
   * The HTTP port of the Laminar API. If not provided, the default is 443.
   */
  httpPort?: number;
  /**
   * The gRPC port of the Laminar API. If not provided, the default is 8443.
   */
  grpcPort?: number;
  /**
   * Object with modules to instrument. If not provided, all
   * available modules are instrumented.
   * See {@link https://docs.lmnr.ai/tracing/automatic-instrumentation}
   */
  instrumentModules?: InitializeOptions['instrumentModules'];
  /**
   * If true, then the spans will not be batched.
   */
  traceDisableBatch?: boolean;
  /**
   * Timeout for trace export. Defaults to 30_000 (30 seconds), which is over
   * the default OTLP exporter timeout of 10_000 (10 seconds).
   */
  traceExportTimeoutMillis?: number;
  /**
   * Defines default log level for SDK and all instrumentations.
   */
  logLevel?: "debug" | "info" | "warn" | "error";

  /**
   * Maximum number of spans to export at a time. Defaults to 64.
   */
  traceExportBatchSize?: number;
}

import type {
  Datapoint,
  EvaluatorFunction,
} from '@lmnr-ai/lmnr';

type DataType = {
  question: string;
  user_id: string;
  timestamp: string;
}

type TargetType = {
  answer: string;
  reasoning?: string;
}

type OutputType = string;

const accuracy: EvaluatorFunction<OutputType, TargetType> =
  (output, target) => output === target.answer ? 1 : 0;

const data: Datapoint<DataType, TargetType> = {
  input: {
    question: "What is the capital of France?",
    user_id: "123",
    timestamp: "2021-01-01",
  },
  target: { answer: "Paris" },
}

evaluate<DataType, TargetType, OutputType>({
  data,
  executor: async (data) => {
    // Call an LLM to answer the question
    return "Paris";
  },
  evaluators: { accuracy },
});