abstract class Dataset<D, T> {
public async slice(start: number, end: number): Promise<Datapoint<D, T>[]> {
const result = [];
for (let i = Math.max(start, 0); i < Math.min(end, await this.size()); i++) {
result.push(await this.get(i));
}
return result;
}
public abstract size(): Promise<number> | number;
public abstract get(index: number): Promise<Datapoint<D, T>> | Datapoint<D, T>;
}
type EvaluatorFunctionReturn = number | Record<string, number>;
type EvaluatorFunction<O, T> = (output: O, target?: T, ...args: any[]) =>
EvaluatorFunctionReturn | Promise<EvaluatorFunctionReturn>;
async function evaluate<D, T, O>({
data, executor, evaluators, groupName, name, config,
}: {
/**
* List of data points to evaluate. `data` is the input to the executor
* function,
* `target` is the input to the evaluator function.
* `Dataset` is the base class for `LaminarDataset`
*/
data: (Datapoint<D, T>[]) | Dataset<D, T>;
/**
* The executor function. Takes the data point + any additional arguments
* and returns the output to evaluate.
*/
executor: (data: D, ...args: any[]) => O | Promise<O>;
/**
* Evaluator functions and names. Each evaluator function takes the output of
* the executor _and_ the target data, and returns a score. The score can be a
* single number or a dict of string keys and number values. If the score is a
* single number, it will be named after the evaluator function. Evaluator
* function names must contain only letters, digits, hyphens, underscores,
* or spaces.
*/
evaluators: Record<string, EvaluatorFunction<O, T>>;
/**
* Name of the evaluation. If not provided, a random name will be assigned.
*/
name?: string;
/**
* Optional group id of the evaluation. Only evaluations within the same
* group_id can be visually compared. Defaults to "default".
*/
groupName?: string;
/**
* Optional override configurations for the evaluator.
*/
config?: EvaluatorConfig;
}): Promise<void>;
/**
* Configuration for the Evaluator
*/
interface EvaluatorConfig {
/**
* The number of data points to evaluate in parallel at a time. Defaults to 5.
*/
concurrencyLimit?: number;
/**
* The project API key to use for the evaluation. If not provided,
* the API key from the environment variable `LMNR_PROJECT_API_KEY` will be used.
*/
projectApiKey?: string;
/**
* The base URL of the Laminar API. If not provided, the default is
* `https://api.lmnr.ai`. Useful with self-hosted Laminar instances.
* Do NOT include the port in the URL, use `httpPort` and `grpcPort` instead.
*/
baseUrl?: string;
/**
* The HTTP port of the Laminar API. If not provided, the default is 443.
*/
httpPort?: number;
/**
* The gRPC port of the Laminar API. If not provided, the default is 8443.
*/
grpcPort?: number;
/**
* Object with modules to instrument. If not provided, all
* available modules are instrumented.
* See {@link https://docs.lmnr.ai/tracing/automatic-instrumentation}
*/
instrumentModules?: InitializeOptions['instrumentModules'];
/**
* If true, then the spans will not be batched.
*/
traceDisableBatch?: boolean;
/**
* Timeout for trace export. Defaults to 30_000 (30 seconds), which is over
* the default OTLP exporter timeout of 10_000 (10 seconds).
*/
traceExportTimeoutMillis?: number;
/**
* Defines default log level for SDK and all instrumentations.
*/
logLevel?: "debug" | "info" | "warn" | "error";
/**
* Maximum number of spans to export at a time. Defaults to 64.
*/
traceExportBatchSize?: number;
}