1
/*
2
 *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
3
 *  License: MS-RSL – see LICENSE.md for details
4
 */
5

6
/*
7
Functionality that mainly involves working with a specific project.
8
*/
9

10
import { join } from "path";
11
import { redux } from "@cocalc/frontend/app-framework";
12
import { appBasePath } from "@cocalc/frontend/customize/app-base-path";
13
import { dialogs } from "@cocalc/frontend/i18n";
14
import { getIntl } from "@cocalc/frontend/i18n/get-intl";
15
import { allow_project_to_run } from "@cocalc/frontend/project/client-side-throttle";
16
import { ensure_project_running } from "@cocalc/frontend/project/project-start-warning";
17
import { API } from "@cocalc/frontend/project/websocket/api";
18
import { connection_to_project } from "@cocalc/frontend/project/websocket/connect";
19
import {
20
  Configuration,
21
  ConfigurationAspect,
22
} from "@cocalc/frontend/project_configuration";
23
import { HOME_ROOT } from "@cocalc/util/consts/files";
24
import type { ApiKey } from "@cocalc/util/db-schema/api-keys";
25
import {
26
  isExecOptsBlocking,
27
  type ExecOpts,
28
  type ExecOutput,
29
} from "@cocalc/util/db-schema/projects";
30
import {
31
  coerce_codomain_to_numbers,
32
  copy_without,
33
  defaults,
34
  encode_path,
35
  is_valid_uuid_string,
36
  required,
37
} from "@cocalc/util/misc";
38
import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
39
import { DirectoryListingEntry } from "@cocalc/util/types";
40
import { WebappClient } from "./client";
41
import { throttle } from "lodash";
42
import { writeFile, type WriteFileOptions } from "@cocalc/conat/files/write";
43
import { readFile, type ReadFileOptions } from "@cocalc/conat/files/read";
44

45
export class ProjectClient {
46
  private client: WebappClient;
47
  private touch_throttle: { [project_id: string]: number } = {};
48

49
  constructor(client: WebappClient) {
50
    this.client = client;
51
  }
52

53
  private conatApi = (project_id: string) => {
54
    return this.client.conat_client.projectApi({ project_id });
55
  };
56

57
  // This can write small text files in one message.
58
  write_text_file = async (opts): Promise<void> => {
59
    await this.writeFile(opts);
60
  };
61

62
  // writeFile -- easily write **arbitrarily large text or binary files**
63
  // to a project from a readable stream or a string!
64
  writeFile = async (
65
    opts: WriteFileOptions & { content?: string },
66
  ): Promise<{ bytes: number; chunks: number }> => {
67
    if (opts.content != null) {
68
      // @ts-ignore -- typescript doesn't like this at all, but it works fine.
69
      opts.stream = new Blob([opts.content], { type: "text/plain" }).stream();
70
    }
71
    return await writeFile(opts);
72
  };
73

74
  // readFile -- read **arbitrarily large text or binary files**
75
  // from a project via a readable stream.
76
  // Look at the code below if you want to stream a file for memory
77
  // efficiency...
78
  readFile = async (opts: ReadFileOptions): Promise<Buffer> => {
79
    const chunks: Uint8Array[] = [];
80
    for await (const chunk of await readFile(opts)) {
81
      chunks.push(chunk);
82
    }
83
    return Buffer.concat(chunks);
84
  };
85

86
  read_text_file = async ({
87
    project_id,
88
    path,
89
  }: {
90
    project_id: string; // string or array of strings
91
    path: string; // string or array of strings
92
  }): Promise<string> => {
93
    return await this.conatApi(project_id).system.readTextFileFromProject({
94
      path,
95
    });
96
  };
97

98
  // Like "read_text_file" above, except the callback
99
  // message gives a url from which the file can be
100
  // downloaded using standard AJAX.
101
  read_file = (opts: {
102
    project_id: string; // string or array of strings
103
    path: string; // string or array of strings
104
    compute_server_id?: number;
105
  }): string => {
106
    const base_path = appBasePath;
107
    if (opts.path[0] === "/") {
108
      // absolute path to the root
109
      opts.path = HOME_ROOT + opts.path; // use root symlink, which is created by start_smc
110
    }
111
    let url = join(
112
      base_path,
113
      `${opts.project_id}/files/${encode_path(opts.path)}`,
114
    );
115
    if (opts.compute_server_id) {
116
      url += `?id=${opts.compute_server_id}`;
117
    }
118
    return url;
119
  };
120

121
  copy_path_between_projects = async (opts: {
122
    src_project_id: string; // id of source project
123
    src_path: string; // relative path of director or file in the source project
124
    target_project_id: string; // if of target project
125
    target_path?: string; // defaults to src_path
126
    overwrite_newer?: boolean; // overwrite newer versions of file at destination (destructive)
127
    delete_missing?: boolean; // delete files in dest that are missing from source (destructive)
128
    backup?: boolean; // make ~ backup files instead of overwriting changed files
129
    timeout?: number; // **timeout in milliseconds** -- how long to wait for the copy to complete before reporting "error" (though it could still succeed)
130
    exclude?: string[]; // list of patterns to exclude; this uses exactly the (confusing) rsync patterns
131
  }): Promise<void> => {
132
    await this.client.conat_client.hub.projects.copyPathBetweenProjects(opts);
133
  };
134

135
  // Set a quota parameter for a given project.
136
  // As of now, only user in the admin group can make these changes.
137
  set_quotas = async (opts: {
138
    project_id: string;
139
    memory?: number;
140
    memory_request?: number;
141
    cpu_shares?: number;
142
    cores?: number;
143
    disk_quota?: number;
144
    mintime?: number;
145
    network?: number;
146
    member_host?: number;
147
    always_running?: number;
148
  }): Promise<void> => {
149
    // we do some extra work to ensure all the quotas are numbers (typescript isn't
150
    // enough; sometimes client code provides strings, which can cause lots of trouble).
151
    const x = coerce_codomain_to_numbers(copy_without(opts, ["project_id"]));
152
    await this.client.conat_client.hub.projects.setQuotas({
153
      ...x,
154
      project_id: opts.project_id,
155
    });
156
  };
157

158
  websocket = async (project_id: string): Promise<any> => {
159
    const store = redux.getStore("projects");
160
    // Wait until project is running (or admin and not on project)
161
    await store.async_wait({
162
      until: () => {
163
        const state = store.get_state(project_id);
164
        if (state == null && redux.getStore("account")?.get("is_admin")) {
165
          // is admin so doesn't know project state -- just immediately
166
          // try, which  will cause project to run
167
          return true;
168
        }
169
        return state == "running";
170
      },
171
    });
172

173
    // get_my_group returns undefined when the various info to
174
    // determine this isn't yet loaded.  For some connections
175
    // this websocket function gets called before that info is
176
    // loaded, which can cause trouble.
177
    let group: string | undefined;
178
    await store.async_wait({
179
      until: () => (group = store.get_my_group(project_id)) != null,
180
    });
181
    if (group == "public") {
182
      throw Error("no access to project websocket");
183
    }
184
    return await connection_to_project(project_id);
185
  };
186

187
  api = async (project_id: string): Promise<API> => {
188
    return (await this.websocket(project_id)).api;
189
  };
190

191
  /*
192
    Execute code in a given project or associated compute server.
193

194
    Aggregate option -- use like this:
195

196
        webapp.exec
197
            aggregate: timestamp (or something else sequential)
198

199
    means: if there are multiple attempts to run the given command with the same
200
    time, they are all aggregated and run only one time by the project.   If requests
201
    comes in with a newer time, they all run in another group after the first
202
    one finishes.    The timestamp will usually come from something like the "last save
203
    time" (which is stored in the db), which they client will know.  This is used, e.g.,
204
    for operations like "run rst2html on this file whenever it is saved."
205
    */
206
  exec = async (opts: ExecOpts & { post?: boolean }): Promise<ExecOutput> => {
207
    if ("async_get" in opts) {
208
      opts = defaults(opts, {
209
        project_id: required,
210
        compute_server_id: undefined,
211
        async_get: required,
212
        async_stats: undefined,
213
        async_await: undefined,
214
        post: false, // if true, uses the POST api through nextjs instead of the websocket api.
215
        timeout: 30,
216
        cb: undefined,
217
      });
218
    } else {
219
      opts = defaults(opts, {
220
        project_id: required,
221
        compute_server_id: undefined,
222
        filesystem: undefined,
223
        path: "",
224
        command: required,
225
        args: [],
226
        max_output: undefined,
227
        bash: false,
228
        aggregate: undefined,
229
        err_on_exit: true,
230
        env: undefined,
231
        post: false, // if true, uses the POST api through nextjs instead of the websocket api.
232
        async_call: undefined, // if given use a callback interface instead of async
233
        timeout: 30,
234
        cb: undefined,
235
      });
236
    }
237

238
    const intl = await getIntl();
239
    const msg = intl.formatMessage(dialogs.client_project_exec_msg, {
240
      blocking: isExecOptsBlocking(opts),
241
      arg: isExecOptsBlocking(opts) ? opts.command : opts.async_get,
242
    });
243

244
    if (!(await ensure_project_running(opts.project_id, msg))) {
245
      return {
246
        type: "blocking",
247
        stdout: "",
248
        stderr: intl.formatMessage(dialogs.client_project_exec_start_first),
249
        exit_code: 1,
250
        time: 0,
251
      };
252
    }
253

254
    try {
255
      const ws = await this.websocket(opts.project_id);
256
      const exec_opts = copy_without(opts, ["project_id", "cb"]);
257
      const msg = await ws.api.exec(exec_opts);
258
      if (msg.status && msg.status == "error") {
259
        throw new Error(msg.error);
260
      }
261
      if (msg.type === "blocking") {
262
        delete msg.status;
263
      }
264
      delete msg.error;
265
      if (opts.cb == null) {
266
        return msg;
267
      } else {
268
        opts.cb(undefined, msg);
269
        return msg;
270
      }
271
    } catch (err) {
272
      if (opts.cb == null) {
273
        throw err;
274
      } else {
275
        if (!err.message) {
276
          // Important since err.message can be falsey, e.g., for Error(''), but toString will never be falsey.
277
          opts.cb(err.toString());
278
        } else {
279
          opts.cb(err.message);
280
        }
281
        return {
282
          type: "blocking",
283
          stdout: "",
284
          stderr: err.message,
285
          exit_code: 1,
286
          time: 0, // should be ignored; this is just to make typescript happy.
287
        };
288
      }
289
    }
290
  };
291

292
  // Directly compute the directory listing.  No caching or other information
293
  // is used -- this just sends a message over the websocket requesting
294
  // the backend node.js project process to compute the listing.
295
  directory_listing = async (opts: {
296
    project_id: string;
297
    path: string;
298
    compute_server_id: number;
299
    timeout?: number;
300
    hidden?: boolean;
301
  }): Promise<{ files: DirectoryListingEntry[] }> => {
302
    if (opts.timeout == null) opts.timeout = 15;
303
    const api = await this.api(opts.project_id);
304
    const listing = await api.listing(
305
      opts.path,
306
      opts.hidden,
307
      opts.timeout * 1000,
308
      opts.compute_server_id,
309
    );
310
    return { files: listing };
311
  };
312

313
  find_directories = async (opts: {
314
    project_id: string;
315
    query?: string; // see the -iwholename option to the UNIX find command.
316
    path?: string; // Root path to find directories from
317
    exclusions?: string[]; // paths relative to `opts.path`. Skips whole sub-trees
318
    include_hidden?: boolean;
319
  }): Promise<{
320
    query: string;
321
    path: string;
322
    project_id: string;
323
    directories: string[];
324
  }> => {
325
    opts = defaults(opts, {
326
      project_id: required,
327
      query: "*", // see the -iwholename option to the UNIX find command.
328
      path: ".", // Root path to find directories from
329
      exclusions: undefined, // Array<String> Paths relative to `opts.path`. Skips whole sub-trees
330
      include_hidden: false,
331
    });
332
    if (opts.path == null || opts.query == null)
333
      throw Error("bug -- cannot happen");
334

335
    const args: string[] = [
336
      opts.path,
337
      "-xdev",
338
      "!",
339
      "-readable",
340
      "-prune",
341
      "-o",
342
      "-type",
343
      "d",
344
      "-iwholename", // See https://github.com/sagemathinc/cocalc/issues/5502
345
      `'${opts.query}'`,
346
      "-readable",
347
    ];
348
    if (opts.exclusions != null) {
349
      for (const excluded_path of opts.exclusions) {
350
        args.push(
351
          `-a -not \\( -path '${opts.path}/${excluded_path}' -prune \\)`,
352
        );
353
      }
354
    }
355

356
    args.push("-print");
357
    const command = `find ${args.join(" ")}`;
358

359
    const result = await this.exec({
360
      // err_on_exit = false: because want this to still work even if there's a nonzero exit code,
361
      // which might happen if find hits a directory it can't read, e.g., a broken ~/.snapshots.
362
      err_on_exit: false,
363
      project_id: opts.project_id,
364
      command,
365
      timeout: 60,
366
      aggregate: Math.round(Date.now() / 5000), // aggregate calls into 5s windows, in case multiple clients ask for same find at once...
367
    });
368
    const n = opts.path.length + 1;
369
    let v = result.stdout.split("\n");
370
    if (!opts.include_hidden) {
371
      v = v.filter((x) => x.indexOf("/.") === -1);
372
    }
373
    v = v.filter((x) => x.length > n).map((x) => x.slice(n));
374
    return {
375
      query: opts.query,
376
      path: opts.path,
377
      project_id: opts.project_id,
378
      directories: v,
379
    };
380
  };
381

382
  // This is async, so do "await smc_webapp.configuration(...project_id...)".
383
  // for reuseInFlight, see https://github.com/sagemathinc/cocalc/issues/7806
384
  configuration = reuseInFlight(
385
    async (
386
      project_id: string,
387
      aspect: ConfigurationAspect,
388
      no_cache: boolean,
389
    ): Promise<Configuration> => {
390
      if (!is_valid_uuid_string(project_id)) {
391
        throw Error("project_id must be a valid uuid");
392
      }
393
      return (await this.api(project_id)).configuration(aspect, no_cache);
394
    },
395
  );
396

397
  touch_project = async (
398
    // project_id where activity occured
399
    project_id: string,
400
    // optional global id of a compute server (in the given project), in which case we also mark
401
    // that compute server as active, which keeps it running in case it has idle timeout configured.
402
    compute_server_id?: number,
403
  ): Promise<void> => {
404
    if (compute_server_id) {
405
      // this is throttled, etc. and is independent of everything below.
406
      touchComputeServer({
407
        project_id,
408
        compute_server_id,
409
        client: this.client,
410
      });
411
      // that said, we do still touch the project, since if a user is actively
412
      // using a compute server, the project should also be considered active.
413
    }
414

415
    const state = redux.getStore("projects")?.get_state(project_id);
416
    if (!(state == null && redux.getStore("account")?.get("is_admin"))) {
417
      // not trying to view project as admin so do some checks
418
      if (!(await allow_project_to_run(project_id))) return;
419
      if (!this.client.is_signed_in()) {
420
        // silently ignore if not signed in
421
        return;
422
      }
423
      if (state != "running") {
424
        // not running so don't touch (user must explicitly start first)
425
        return;
426
      }
427
    }
428

429
    // Throttle -- so if this function is called with the same project_id
430
    // twice in 3s, it's ignored (to avoid unnecessary network traffic).
431
    // Do not make the timeout long, since that can mess up
432
    // getting the hub-websocket to connect to the project.
433
    const last = this.touch_throttle[project_id];
434
    if (last != null && Date.now() - last <= 3000) {
435
      return;
436
    }
437
    this.touch_throttle[project_id] = Date.now();
438
    try {
439
      await this.client.conat_client.hub.db.touch({ project_id });
440
    } catch (err) {
441
      // silently ignore; this happens, e.g., if you touch too frequently,
442
      // and shouldn't be fatal and break other things.
443
      // NOTE: this is a bit ugly for now -- basically the
444
      // hub returns an error regarding actually touching
445
      // the project (updating the db), but it still *does*
446
      // ensure there is a TCP connection to the project.
447
    }
448
  };
449

450
  // Print sagews to pdf
451
  // The printed version of the file will be created in the same directory
452
  // as path, but with extension replaced by ".pdf".
453
  // Only used for sagews.
454
  print_to_pdf = async ({
455
    project_id,
456
    path,
457
    options,
458
    timeout,
459
  }: {
460
    project_id: string;
461
    path: string;
462
    timeout?: number; // client timeout -- some things can take a long time to print!
463
    options?: any; // optional options that get passed to the specific backend for this file type
464
  }): Promise<string> => {
465
    return await this.client.conat_client
466
      .projectApi({ project_id })
467
      .editor.printSageWS({ path, timeout, options });
468
  };
469

470
  create = async (opts: {
471
    title: string;
472
    description: string;
473
    image?: string;
474
    start?: boolean;
475
    // "license_id1,license_id2,..." -- if given, create project with these licenses applied
476
    license?: string;
477
    // never use pool
478
    noPool?: boolean;
479
  }): Promise<string> => {
480
    const project_id =
481
      await this.client.conat_client.hub.projects.createProject(opts);
482
    this.client.tracking_client.user_tracking("create_project", {
483
      project_id,
484
      title: opts.title,
485
    });
486
    return project_id;
487
  };
488

489
  realpath = async (opts: {
490
    project_id: string;
491
    path: string;
492
  }): Promise<string> => {
493
    return (await this.api(opts.project_id)).realpath(opts.path);
494
  };
495

496
  isdir = async ({
497
    project_id,
498
    path,
499
  }: {
500
    project_id: string;
501
    path: string;
502
  }): Promise<boolean> => {
503
    const { stdout, exit_code } = await this.exec({
504
      project_id,
505
      command: "file",
506
      args: ["-Eb", path],
507
      err_on_exit: false,
508
    });
509
    return !exit_code && stdout.trim() == "directory";
510
  };
511

512
  ipywidgetsGetBuffer = reuseInFlight(
513
    async (
514
      project_id: string,
515
      path: string,
516
      model_id: string,
517
      buffer_path: string,
518
    ): Promise<ArrayBuffer> => {
519
      const actions = redux.getEditorActions(project_id, path);
520
      return await actions.jupyter_actions.ipywidgetsGetBuffer(
521
        model_id,
522
        buffer_path,
523
      );
524
    },
525
  );
526

527
  // getting, setting, editing, deleting, etc., the  api keys for a project
528
  api_keys = async (opts: {
529
    project_id: string;
530
    action: "get" | "delete" | "create" | "edit";
531
    password?: string;
532
    name?: string;
533
    id?: number;
534
    expire?: Date;
535
  }): Promise<ApiKey[] | undefined> => {
536
    return await this.client.conat_client.hub.system.manageApiKeys(opts);
537
  };
538

539
  computeServers = (project_id) => {
540
    const cs = redux.getProjectActions(project_id)?.computeServers();
541
    if (cs == null) {
542
      // this happens if something tries to access the compute server info after the project
543
      // tab is closed.  It shouldn't do that.
544
      throw Error("compute server information not available");
545
    }
546
    return cs;
547
  };
548

549
  getServerIdForPath = async ({
550
    project_id,
551
    path,
552
  }): Promise<number | undefined> => {
553
    return await this.computeServers(project_id)?.getServerIdForPath(path);
554
  };
555

556
  // will return undefined if compute servers not yet initialized
557
  getServerIdForPathSync = ({ project_id, path }): number | undefined => {
558
    const cs = this.computeServers(project_id);
559
    if (cs?.state != "connected") {
560
      return undefined;
561
    }
562
    return cs.get(path);
563
  };
564
}
565

566
// (NOTE: this won't throw an exception)
567
const touchComputeServer = throttle(
568
  async ({ project_id, compute_server_id, client }) => {
569
    if (!compute_server_id) {
570
      // nothing to do
571
      return;
572
    }
573
    try {
574
      await client.async_query({
575
        query: {
576
          compute_servers: {
577
            project_id,
578
            id: compute_server_id,
579
            last_edited_user: client.server_time(),
580
          },
581
        },
582
      });
583
    } catch (err) {
584
      // just a warning -- if we can't connect then touching isn't something we should be doing anyways.
585
      console.log(
586
        "WARNING: failed to touch compute server -- ",
587
        { compute_server_id },
588
        err,
589
      );
590
    }
591
  },
592
  30000,
593
);
594

595
// Polyfill for Safari: Add async iterator support to ReadableStream if missing.
596
// E.g., this is missing in all versions of Safari as of May 2025 according to
597
//           https://caniuse.com/?search=ReadableStream%20async
598
// This breaks reading and writing files to projects, which is why this
599
// is here (e.g., the writeFile and readFile functions above).
600
// This might also matter for Jupyter.
601
// https://chatgpt.com/share/6827a476-dbe8-800e-9156-3326eb41baae
602
if (
603
  typeof ReadableStream !== "undefined" &&
604
  !ReadableStream.prototype[Symbol.asyncIterator]
605
) {
606
  ReadableStream.prototype[Symbol.asyncIterator] = function () {
607
    const reader = this.getReader();
608
    return {
609
      async next() {
610
        return reader.read();
611
      },
612
      async return() {
613
        reader.releaseLock();
614
        return { done: true };
615
      },
616
      [Symbol.asyncIterator]() {
617
        return this;
618
      },
619
    };
620
  };
621
}
622

623