1
/*
2
Client for the conat server in server.ts.
3
*/
4

5
import { conat } from "@cocalc/conat/client";
6
import type { ChatOptions } from "@cocalc/util/types/llm";
7
import { isValidUUID } from "@cocalc/util/misc";
8
import { llmSubject } from "./server";
9

10
export async function llm(options: ChatOptions): Promise<string> {
11
  if (!options.system?.trim()) {
12
    // I noticed in testing that for some models they just fail, so let's be clear immediately.
13
    throw Error("the system prompt MUST be nonempty");
14
  }
15
  if (!isValidUUID(options.account_id)) {
16
    throw Error("account_id must be a valid uuid");
17
  }
18
  const subject = llmSubject({ account_id: options.account_id });
19

20
  let all = "";
21
  let lastSeq = -1;
22
  const cn = await conat();
23
  let { stream, ...opts } = options;
24
  for await (const resp of await cn.requestMany(subject, opts, {
25
    maxWait: opts.timeout ?? 1000 * 60 * 10,
26
  })) {
27
    if (resp.data == null) {
28
      // client code also expects null token to know when stream is done.
29
      stream?.(null);
30
      break;
31
    }
32
    const { error, text, seq } = resp.data;
33
    if (error) {
34
      throw Error(error);
35
    }
36
    if (lastSeq + 1 != seq) {
37
      throw Error("missed response");
38
    }
39
    lastSeq = seq;
40
    stream?.(text);
41
    all += text;
42
  }
43

44
  return all;
45
}
46

47