Langsmith

Typescript Versionen

npm下载地址

Installation

npm install langchain @langchain/core @langchain/community @langchain/openai langsmith

LangChain所有第三方的库：链接

Datensatz erstellen

Manuell hinzufügen

Mengen hinzufügen

Verfolgung

import { ChatOpenAI } from "@langchain/openai";
import { ChatPromptTemplate } from "@langchain/core/prompts";
import { StringOutputParser } from "@langchain/core/output_parsers";

async function main() {
    const prompt = ChatPromptTemplate.fromMessages([
        ["system", "You are a helpful assistant. Please respond to the user's request only based on the given context."],
        ["user", "Question: {question}\nContext: {context}"],
    ]);
    const model = new ChatOpenAI({ modelName: "gpt-3.5-turbo" });
    const outputParser = new StringOutputParser();

    const chain = prompt.pipe(model).pipe(outputParser);

    const question = "Can you summarize this morning's meetings?"
    const context = "During this morning's meeting, we solved all world conflict."
    await chain.invoke({ question: question, context: context });
}

main();

Bewertungen

import { Client } from "langsmith";
import { StringOutputParser } from "@langchain/core/output_parsers";
import { ChatPromptTemplate } from "@langchain/core/prompts";
import { ChatOpenAI } from "@langchain/openai";
import { RunEvalConfig, runOnDataset } from "langchain/smith";
import { Run, Example } from "langsmith";
import { EvaluationResult } from "langsmith/evaluation";

async function main() {
  // 待测试的函数
  const llm = new ChatOpenAI({ modelName: "gpt-3.5-turbo", temperature: 0 });
  const prompt = ChatPromptTemplate.fromMessages([
    ["human", "Spit some bars about {question}."],
  ]);
  const chain = prompt.pipe(llm).pipe(new StringOutputParser());

  // 定义和执行评估器
  const mustMention = async ({ run, example, }: { run: Run; example?: Example; })
    : Promise<EvaluationResult> => {
    // const mustMention: string[] = example?.outputs?.must_contain ?? [];
    // const score = mustMention.every((phrase) =>
    //   run?.outputs?.output.includes(phrase)
    // );
    return {
      key: "must_mention",
      score: 99,
    };
  };

  const evalConfig: RunEvalConfig = {
    customEvaluators: [mustMention],
  };

  let datasetName = "Rap Battle Dataset";
  await runOnDataset(chain, datasetName, {
    evaluationConfig: evalConfig
  });

}

main()

Benutzerdefinierter Evaluator

const evalPrimary = async ({ run, example, }: { run: Run; example?: Example; }): Promise<EvaluationResult> => {
}

Das Beispiel in:

{
  "id": "e7252e52-c7af-43d5-aba8-4dcc5b153c9b",
  "created_at": "2024-03-27T11:20:55.679149+00:00",
  "modified_at": "2024-03-27T11:34:07.449831+00:00",
  "name": "#e725 @ ds-ajar-creche-31",
  "dataset_id": "2c8361fe-204f-4cb5-b2cb-9c7afffc452c",
  "source_run_id": null,
  "metadata": null,
  "inputs": { // 里面是自定义的内容
    "query": "xxx"
  },
  "outputs": { // 里面是自定义的内容
    "outPrimary": "yyy"
  }
}

Python Versionen

n / a