1
/*
2
 *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
3
 *  License: MS-RSL – see LICENSE.md for details
4
 */
5

6
/*
7
project-actions: additional actions that are only available in the
8
backend/project, which "manages" everything.
9

10
This code should not *explicitly* require anything that is only
11
available in the project or requires node to run, so that we can
12
fully unit test it via mocking of components.
13

14
NOTE: this is also now the actions used by remote compute servers as well.
15
*/
16

17
import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data";
18
import * as immutable from "immutable";
19
import json_stable from "json-stable-stringify";
20
import { debounce } from "lodash";
21
import {
22
  JupyterActions as JupyterActions0,
23
  MAX_OUTPUT_MESSAGES,
24
} from "@cocalc/jupyter/redux/actions";
25
import { callback2, once } from "@cocalc/util/async-utils";
26
import * as misc from "@cocalc/util/misc";
27
import { OutputHandler } from "@cocalc/jupyter/execute/output-handler";
28
import { RunAllLoop } from "./run-all-loop";
29
import nbconvertChange from "./handle-nbconvert-change";
30
import type { ClientFs } from "@cocalc/sync/client/types";
31
import { kernel as createJupyterKernel } from "@cocalc/jupyter/kernel";
32
import { removeJupyterRedux } from "@cocalc/jupyter/kernel";
33
import { initConatService } from "@cocalc/jupyter/kernel/conat-service";
34
import { type DKV, dkv } from "@cocalc/conat/sync/dkv";
35
import { computeServerManager } from "@cocalc/conat/compute/manager";
36
import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
37

38
// see https://github.com/sagemathinc/cocalc/issues/8060
39
const MAX_OUTPUT_SAVE_DELAY = 30000;
40

41
// refuse to open an ipynb that is bigger than this:
42
const MAX_SIZE_IPYNB_MB = 150;
43

44
type BackendState = "init" | "ready" | "spawning" | "starting" | "running";
45

46
export class JupyterActions extends JupyterActions0 {
47
  private _backend_state: BackendState = "init";
48
  private lastSavedBackendState?: BackendState;
49
  private _initialize_manager_already_done: any;
50
  private _kernel_state: any;
51
  private _manager_run_cell_queue: any;
52
  private _running_cells: { [id: string]: string };
53
  private _throttled_ensure_positions_are_unique: any;
54
  private run_all_loop?: RunAllLoop;
55
  private clear_kernel_error?: any;
56
  private running_manager_run_cell_process_queue: boolean = false;
57
  private last_ipynb_save: number = 0;
58
  protected _client: ClientFs; // this has filesystem access, etc.
59
  public blobs: DKV;
60
  private computeServers?;
61

62
  private initBlobStore = async () => {
63
    this.blobs = await dkv(this.blobStoreOptions());
64
  };
65

66
  // uncomment for verbose logging of everything here to the console.
67
  //   dbg(f: string) {
68
  //     return (...args) => console.log(f, args);
69
  //   }
70

71
  public run_cell(
72
    id: string,
73
    save: boolean = true,
74
    no_halt: boolean = false,
75
  ): void {
76
    if (this.store.get("read_only")) {
77
      return;
78
    }
79
    const cell = this.store.getIn(["cells", id]);
80
    if (cell == null) {
81
      // it is trivial to run a cell that does not exist -- nothing needs to be done.
82
      return;
83
    }
84
    const cell_type = cell.get("cell_type", "code");
85
    if (cell_type == "code") {
86
      // when the backend is running code, just don't worry about
87
      // trying to parse things like "foo?" out. We can't do
88
      // it without CodeMirror, and it isn't worth it for that
89
      // application.
90
      this.run_code_cell(id, save, no_halt);
91
    }
92
    if (save) {
93
      this.save_asap();
94
    }
95
  }
96

97
  private set_backend_state(backend_state: BackendState): void {
98
    this.dbg("set_backend_state")(backend_state);
99

100
    /*
101
        The backend states, which are put in the syncdb so clients
102
        can display this:
103

104
         - 'init' -- the backend is checking the file on disk, etc.
105
         - 'ready' -- the backend is setup and ready to use; kernel isn't running though
106
         - 'starting' -- the kernel itself is actived and currently starting up (e.g., Sage is starting up)
107
         - 'running' -- the kernel is running and ready to evaluate code
108

109

110
         'init' --> 'ready'  --> 'spawning' --> 'starting' --> 'running'
111
                     /|\                                        |
112
                      |-----------------------------------------|
113

114
        Going from ready to starting happens first when a code execution is requested.
115
        */
116

117
    // Check just in case Typescript doesn't catch something:
118
    if (
119
      ["init", "ready", "spawning", "starting", "running"].indexOf(
120
        backend_state,
121
      ) === -1
122
    ) {
123
      throw Error(`invalid backend state '${backend_state}'`);
124
    }
125
    if (backend_state == "init" && this._backend_state != "init") {
126
      // Do NOT allow changing the state to init from any other state.
127
      throw Error(
128
        `illegal state change '${this._backend_state}' --> '${backend_state}'`,
129
      );
130
    }
131
    this._backend_state = backend_state;
132

133
    if (this.lastSavedBackendState != backend_state) {
134
      this._set({
135
        type: "settings",
136
        backend_state,
137
        last_backend_state: Date.now(),
138
      });
139
      this.save_asap();
140
      this.lastSavedBackendState = backend_state;
141
    }
142

143
    // The following is to clear kernel_error if things are working only.
144
    if (backend_state == "running") {
145
      // clear kernel error if kernel successfully starts and stays
146
      // in running state for a while.
147
      this.clear_kernel_error = setTimeout(() => {
148
        this._set({
149
          type: "settings",
150
          kernel_error: "",
151
        });
152
      }, 3000);
153
    } else {
154
      // change to a different state; cancel attempt to clear kernel error
155
      if (this.clear_kernel_error) {
156
        clearTimeout(this.clear_kernel_error);
157
        delete this.clear_kernel_error;
158
      }
159
    }
160
  }
161

162
  set_kernel_state = (state: any, save = false) => {
163
    this._kernel_state = state;
164
    this._set({ type: "settings", kernel_state: state }, save);
165
  };
166

167
  // Called exactly once when the manager first starts up after the store is initialized.
168
  // Here we ensure everything is in a consistent state so that we can react
169
  // to changes later.
170
  async initialize_manager() {
171
    if (this._initialize_manager_already_done) {
172
      return;
173
    }
174
    const dbg = this.dbg("initialize_manager");
175
    dbg();
176
    this._initialize_manager_already_done = true;
177

178
    dbg("initialize Jupyter Conat api handler");
179
    await this.initConatApi();
180

181
    dbg("initializing blob store");
182
    await this.initBlobStore();
183

184
    this.sync_exec_state = debounce(this.sync_exec_state, 2000);
185
    this._throttled_ensure_positions_are_unique = debounce(
186
      this.ensure_positions_are_unique,
187
      5000,
188
    );
189
    // Listen for changes...
190
    this.syncdb.on("change", this.backendSyncdbChange);
191

192
    this.setState({
193
      // used by the kernel_info function of this.jupyter_kernel
194
      start_time: this._client.server_time().valueOf(),
195
    });
196

197
    // clear nbconvert start on init, since no nbconvert can be running yet
198
    this.syncdb.delete({ type: "nbconvert" });
199

200
    // Initialize info about available kernels, which is used e.g., for
201
    // saving to ipynb format.
202
    this.init_kernel_info();
203

204
    // We try once to load from disk.  If it fails, then
205
    // a record with type:'fatal'
206
    // is created in the database; if it succeeds, that record is deleted.
207
    // Try again only when the file changes.
208
    await this._first_load();
209

210
    // Listen for model state changes...
211
    if (this.syncdb.ipywidgets_state == null) {
212
      throw Error("syncdb's ipywidgets_state must be defined!");
213
    }
214
    this.syncdb.ipywidgets_state.on(
215
      "change",
216
      this.handle_ipywidgets_state_change,
217
    );
218
  }
219

220
  private conatService?;
221
  private initConatApi = reuseInFlight(async () => {
222
    if (this.conatService != null) {
223
      this.conatService.close();
224
      this.conatService = null;
225
    }
226
    const service = (this.conatService = await initConatService({
227
      project_id: this.project_id,
228
      path: this.path,
229
    }));
230
    this.syncdb.on("closed", () => {
231
      service.close();
232
    });
233
  });
234

235
  private _first_load = async () => {
236
    const dbg = this.dbg("_first_load");
237
    dbg("doing load");
238
    if (this.is_closed()) {
239
      throw Error("actions must not be closed");
240
    }
241
    try {
242
      await this.loadFromDiskIfNewer();
243
    } catch (err) {
244
      dbg(`load failed -- ${err}; wait for file change and try again`);
245
      const path = this.store.get("path");
246
      const watcher = this._client.watch_file({ path });
247
      await once(watcher, "change");
248
      dbg("file changed");
249
      watcher.close();
250
      await this._first_load();
251
      return;
252
    }
253
    dbg("loading worked");
254
    this._init_after_first_load();
255
  };
256

257
  private _init_after_first_load = () => {
258
    const dbg = this.dbg("_init_after_first_load");
259

260
    dbg("initializing");
261
    // this may change the syncdb.
262
    this.ensure_backend_kernel_setup();
263

264
    this.init_file_watcher();
265

266
    this._state = "ready";
267
  };
268

269
  private backendSyncdbChange = (changes: any) => {
270
    if (this.is_closed()) {
271
      return;
272
    }
273
    const dbg = this.dbg("backendSyncdbChange");
274
    if (changes != null) {
275
      changes.forEach((key) => {
276
        switch (key.get("type")) {
277
          case "settings":
278
            dbg("settings change");
279
            var record = this.syncdb.get_one(key);
280
            if (record != null) {
281
              // ensure kernel is properly configured
282
              this.ensure_backend_kernel_setup();
283
              // only the backend should change kernel and backend state;
284
              // however, our security model allows otherwise (e.g., via TimeTravel).
285
              if (
286
                record.get("kernel_state") !== this._kernel_state &&
287
                this._kernel_state != null
288
              ) {
289
                this.set_kernel_state(this._kernel_state, true);
290
              }
291
              if (record.get("backend_state") !== this._backend_state) {
292
                this.set_backend_state(this._backend_state);
293
              }
294

295
              if (record.get("run_all_loop_s")) {
296
                if (this.run_all_loop == null) {
297
                  this.run_all_loop = new RunAllLoop(
298
                    this,
299
                    record.get("run_all_loop_s"),
300
                  );
301
                } else {
302
                  // ensure interval is correct
303
                  this.run_all_loop.set_interval(record.get("run_all_loop_s"));
304
                }
305
              } else if (
306
                !record.get("run_all_loop_s") &&
307
                this.run_all_loop != null
308
              ) {
309
                // stop it.
310
                this.run_all_loop.close();
311
                delete this.run_all_loop;
312
              }
313
            }
314
            break;
315
        }
316
      });
317
    }
318

319
    this.ensure_there_is_a_cell();
320
    this._throttled_ensure_positions_are_unique();
321
    this.sync_exec_state();
322
  };
323

324
  // ensure_backend_kernel_setup ensures that we have a connection
325
  // to the selected Jupyter kernel, if any.
326
  ensure_backend_kernel_setup = () => {
327
    const dbg = this.dbg("ensure_backend_kernel_setup");
328
    if (this.isDeleted()) {
329
      dbg("file is deleted");
330
      return;
331
    }
332

333
    const kernel = this.store.get("kernel");
334
    dbg("ensure_backend_kernel_setup", { kernel });
335

336
    let current: string | undefined = undefined;
337
    if (this.jupyter_kernel != null) {
338
      current = this.jupyter_kernel.name;
339
      if (current == kernel) {
340
        const state = this.jupyter_kernel.get_state();
341
        if (state == "error") {
342
          dbg("kernel is broken");
343
          // nothing to do -- let user ponder the error they should see.
344
          return;
345
        }
346
        if (state != "closed") {
347
          dbg("everything is properly setup and working");
348
          return;
349
        }
350
      }
351
    }
352

353
    dbg(`kernel='${kernel}', current='${current}'`);
354
    if (
355
      this.jupyter_kernel != null &&
356
      this.jupyter_kernel.get_state() != "closed"
357
    ) {
358
      if (current != kernel) {
359
        dbg("kernel changed -- kill running kernel to trigger switch");
360
        this.jupyter_kernel.close();
361
        return;
362
      } else {
363
        dbg("nothing to do");
364
        return;
365
      }
366
    }
367

368
    dbg("make a new kernel");
369

370
    // No kernel wrapper object setup at all. Make one.
371
    this.jupyter_kernel = createJupyterKernel({
372
      name: kernel,
373
      path: this.store.get("path"),
374
      actions: this,
375
    });
376

377
    if (this.syncdb.ipywidgets_state == null) {
378
      throw Error("syncdb's ipywidgets_state must be defined!");
379
    }
380
    this.syncdb.ipywidgets_state.clear();
381

382
    if (this.jupyter_kernel == null) {
383
      // to satisfy typescript.
384
      throw Error("jupyter_kernel must be defined");
385
    }
386
    dbg("kernel created -- installing handlers");
387

388
    // save so gets reported to frontend, and surfaced to user:
389
    // https://github.com/sagemathinc/cocalc/issues/4847
390
    this.jupyter_kernel.on("kernel_error", (error) => {
391
      this.set_kernel_error(error);
392
    });
393

394
    // Since we just made a new kernel, clearly no cells are running on the backend:
395
    this._running_cells = {};
396

397
    const toStart: string[] = [];
398
    this.store?.get_cell_list().forEach((id) => {
399
      if (this.store.getIn(["cells", id, "state"]) == "start") {
400
        toStart.push(id);
401
      }
402
    });
403

404
    dbg("clear cell run state");
405
    this.clear_all_cell_run_state();
406

407
    this.restartKernelOnClose = () => {
408
      // When the kernel closes, make sure a new kernel gets setup.
409
      if (this.store == null || this._state !== "ready") {
410
        // This event can also happen when this actions is being closed,
411
        // in which case obviously we shouldn't make a new kernel.
412
        return;
413
      }
414
      dbg("kernel closed -- make new one.");
415
      this.ensure_backend_kernel_setup();
416
    };
417

418
    this.jupyter_kernel.once("closed", this.restartKernelOnClose);
419

420
    // Track backend state changes other than closing, so they
421
    // are visible to user etc.
422
    // TODO: Maybe all these need to move to ephemeral table?
423
    // There's a good argument that recording these is useful though, so when
424
    // looking at time travel or debugging, you know what was going on.
425
    this.jupyter_kernel.on("state", (state) => {
426
      dbg("jupyter_kernel state --> ", state);
427
      switch (state) {
428
        case "off":
429
        case "closed":
430
          // things went wrong.
431
          this._running_cells = {};
432
          this.clear_all_cell_run_state();
433
          this.set_backend_state("ready");
434
          this.jupyter_kernel?.close();
435
          this.running_manager_run_cell_process_queue = false;
436
          delete this.jupyter_kernel;
437
          return;
438
        case "spawning":
439
        case "starting":
440
          this.set_connection_file(); // yes, fall through
441
        case "running":
442
          this.set_backend_state(state);
443
      }
444
    });
445

446
    this.jupyter_kernel.on("execution_state", this.set_kernel_state);
447

448
    this.handle_all_cell_attachments();
449
    dbg("ready");
450
    this.set_backend_state("ready");
451

452
    // Run cells that the user explicitly set to be running before the
453
    // kernel actually had finished starting up.
454
    // This must be done after the state is ready.
455
    if (toStart.length > 0) {
456
      for (const id of toStart) {
457
        this.run_cell(id);
458
      }
459
    }
460
  };
461

462
  set_connection_file = () => {
463
    const connection_file = this.jupyter_kernel?.get_connection_file() ?? "";
464
    this._set({
465
      type: "settings",
466
      connection_file,
467
    });
468
  };
469

470
  init_kernel_info = async () => {
471
    let kernels0 = this.store.get("kernels");
472
    if (kernels0 != null) {
473
      return;
474
    }
475
    const dbg = this.dbg("init_kernel_info");
476
    dbg("getting");
477
    let kernels;
478
    try {
479
      kernels = await get_kernel_data();
480
      dbg("success");
481
    } catch (err) {
482
      dbg(`FAILED to get kernel info: ${err}`);
483
      // TODO: what to do??  Saving will be broken...
484
      return;
485
    }
486
    this.setState({
487
      kernels: immutable.fromJS(kernels),
488
    });
489
  };
490

491
  async ensure_backend_kernel_is_running() {
492
    const dbg = this.dbg("ensure_backend_kernel_is_running");
493
    if (this._backend_state == "ready") {
494
      dbg("in state 'ready', so kick it into gear");
495
      await this.set_backend_kernel_info();
496
      dbg("done getting kernel info");
497
    }
498
    const is_running = (s): boolean => {
499
      if (this._state === "closed") {
500
        return true;
501
      }
502
      const t = s.get_one({ type: "settings" });
503
      if (t == null) {
504
        dbg("no settings");
505
        return false;
506
      } else {
507
        const state = t.get("backend_state");
508
        dbg(`state = ${state}`);
509
        return state == "running";
510
      }
511
    };
512
    await this.syncdb.wait(is_running, 60);
513
  }
514

515
  // onCellChange is called after a cell change has been
516
  // incorporated into the store after the syncdb change event.
517
  // - If we are responsible for running cells, then it ensures
518
  //   that cell gets computed.
519
  // - We also handle attachments for markdown cells.
520
  protected onCellChange(id: string, new_cell: any, old_cell: any) {
521
    const dbg = this.dbg(`onCellChange(id='${id}')`);
522
    dbg();
523
    // this logging could be expensive due to toJS, so only uncomment
524
    // if really needed
525
    // dbg("new_cell=", new_cell?.toJS(), "old_cell", old_cell?.toJS());
526

527
    if (
528
      new_cell?.get("state") === "start" &&
529
      old_cell?.get("state") !== "start"
530
    ) {
531
      this.manager_run_cell_enqueue(id);
532
      // attachments below only happen for markdown cells, which don't get run,
533
      // we can return here:
534
      return;
535
    }
536

537
    const attachments = new_cell?.get("attachments");
538
    if (attachments != null && attachments !== old_cell?.get("attachments")) {
539
      this.handle_cell_attachments(new_cell);
540
    }
541
  }
542

543
  protected __syncdb_change_post_hook(doInit: boolean) {
544
    if (doInit) {
545
      // Since just opening the actions in the project, definitely the kernel
546
      // isn't running so set this fact in the shared database.  It will make
547
      // things always be in the right initial state.
548
      this.syncdb.set({
549
        type: "settings",
550
        backend_state: "init",
551
        kernel_state: "idle",
552
        kernel_usage: { memory: 0, cpu: 0 },
553
      });
554
      this.syncdb.commit();
555

556
      // Also initialize the execution manager, which runs cells that have been
557
      // requested to run.
558
      this.initialize_manager();
559
    }
560
    if (this.store.get("kernel")) {
561
      this.manager_run_cell_process_queue();
562
    }
563
  }
564

565
  // Ensure that the cells listed as running *are* exactly the
566
  // ones actually running or queued up to run.
567
  sync_exec_state = () => {
568
    // sync_exec_state is debounced, so it is *expected* to get called
569
    // after actions have been closed.
570
    if (this.store == null || this._state !== "ready") {
571
      // not initialized, so we better not
572
      // mess with cell state (that is somebody else's responsibility).
573
      return;
574
    }
575

576
    const dbg = this.dbg("sync_exec_state");
577
    let change = false;
578
    const cells = this.store.get("cells");
579
    // First verify that all actual cells that are said to be running
580
    // (according to the store) are in fact running.
581
    if (cells != null) {
582
      cells.forEach((cell, id) => {
583
        const state = cell.get("state");
584
        if (
585
          state != null &&
586
          state != "done" &&
587
          state != "start" && // regarding "start", see https://github.com/sagemathinc/cocalc/issues/5467
588
          !this._running_cells?.[id]
589
        ) {
590
          dbg(`set cell ${id} with state "${state}" to done`);
591
          this._set({ type: "cell", id, state: "done" }, false);
592
          change = true;
593
        }
594
      });
595
    }
596
    if (this._running_cells != null) {
597
      const cells = this.store.get("cells");
598
      // Next verify that every cell actually running is still in the document
599
      // and listed as running.  TimeTravel, deleting cells, etc., can
600
      // certainly lead to this being necessary.
601
      for (const id in this._running_cells) {
602
        const state = cells.getIn([id, "state"]);
603
        if (state == null || state === "done") {
604
          // cell no longer exists or isn't in a running state
605
          dbg(`tell kernel to not run ${id}`);
606
          this._cancel_run(id);
607
        }
608
      }
609
    }
610
    if (change) {
611
      return this._sync();
612
    }
613
  };
614

615
  _cancel_run = (id: any) => {
616
    const dbg = this.dbg(`_cancel_run ${id}`);
617
    // All these checks are so we only cancel if it is actually running
618
    // with the current kernel...
619
    if (this._running_cells == null || this.jupyter_kernel == null) return;
620
    const identity = this._running_cells[id];
621
    if (identity == null) return;
622
    if (this.jupyter_kernel.identity == identity) {
623
      dbg("canceling");
624
      this.jupyter_kernel.cancel_execute(id);
625
    } else {
626
      dbg("not canceling since wrong identity");
627
    }
628
  };
629

630
  // Note that there is a request to run a given cell.
631
  // You must call manager_run_cell_process_queue for them to actually start running.
632
  protected manager_run_cell_enqueue(id: string) {
633
    if (this._running_cells?.[id]) {
634
      return;
635
    }
636
    if (this._manager_run_cell_queue == null) {
637
      this._manager_run_cell_queue = {};
638
    }
639
    this._manager_run_cell_queue[id] = true;
640
  }
641

642
  // properly start running -- in order -- the cells that have been requested to run
643
  protected async manager_run_cell_process_queue() {
644
    if (this.running_manager_run_cell_process_queue) {
645
      return;
646
    }
647
    this.running_manager_run_cell_process_queue = true;
648
    try {
649
      const dbg = this.dbg("manager_run_cell_process_queue");
650
      const queue = this._manager_run_cell_queue;
651
      if (queue == null) {
652
        //dbg("queue is null");
653
        return;
654
      }
655
      delete this._manager_run_cell_queue;
656
      const v: any[] = [];
657
      for (const id in queue) {
658
        if (!this._running_cells?.[id]) {
659
          v.push(this.store.getIn(["cells", id]));
660
        }
661
      }
662

663
      if (v.length == 0) {
664
        dbg("no non-running cells");
665
        return; // nothing to do
666
      }
667

668
      v.sort((a, b) =>
669
        misc.cmp(
670
          a != null ? a.get("start") : undefined,
671
          b != null ? b.get("start") : undefined,
672
        ),
673
      );
674

675
      dbg(
676
        `found ${v.length} non-running cell that should be running, so ensuring kernel is running...`,
677
      );
678
      this.ensure_backend_kernel_setup();
679
      try {
680
        await this.ensure_backend_kernel_is_running();
681
        if (this._state == "closed") return;
682
      } catch (err) {
683
        // if this fails, give up on evaluation.
684
        return;
685
      }
686

687
      dbg(
688
        `kernel is now running; requesting that each ${v.length} cell gets executed`,
689
      );
690
      for (const cell of v) {
691
        if (cell != null) {
692
          this.manager_run_cell(cell.get("id"));
693
        }
694
      }
695

696
      if (this._manager_run_cell_queue != null) {
697
        // run it again to process additional entries.
698
        setTimeout(this.manager_run_cell_process_queue, 1);
699
      }
700
    } finally {
701
      this.running_manager_run_cell_process_queue = false;
702
    }
703
  }
704

705
  // returns new output handler for this cell.
706
  protected _output_handler(cell) {
707
    const dbg = this.dbg(`_output_handler(id='${cell.id}')`);
708
    if (
709
      this.jupyter_kernel == null ||
710
      this.jupyter_kernel.get_state() == "closed"
711
    ) {
712
      throw Error("jupyter kernel must exist and not be closed");
713
    }
714
    this.reset_more_output(cell.id);
715

716
    const handler = new OutputHandler({
717
      cell,
718
      max_output_length: this.store.get("max_output_length"),
719
      max_output_messages: MAX_OUTPUT_MESSAGES,
720
      report_started_ms: 250,
721
      dbg,
722
    });
723

724
    dbg("setting up jupyter_kernel.once('closed', ...) handler");
725
    const handleKernelClose = () => {
726
      dbg("output handler -- closing due to jupyter kernel closed");
727
      handler.close();
728
    };
729
    this.jupyter_kernel.once("closed", handleKernelClose);
730
    // remove the "closed" handler we just defined above once
731
    // we are done waiting for output from this cell.
732
    // The output handler removes all listeners whenever it is
733
    // finished, so we don't have to remove this listener for done.
734
    handler.once("done", () =>
735
      this.jupyter_kernel?.removeListener("closed", handleKernelClose),
736
    );
737

738
    handler.on("more_output", (mesg, mesg_length) => {
739
      this.set_more_output(cell.id, mesg, mesg_length);
740
    });
741

742
    handler.on("process", (mesg) => {
743
      // Do not enable -- mesg often very large!
744
      //        dbg("handler.on('process')", mesg);
745
      if (
746
        this.jupyter_kernel == null ||
747
        this.jupyter_kernel.get_state() == "closed"
748
      ) {
749
        return;
750
      }
751
      this.jupyter_kernel.process_output(mesg);
752
      //  dbg("handler -- after processing ", mesg);
753
    });
754

755
    return handler;
756
  }
757

758
  manager_run_cell = (id: string) => {
759
    const dbg = this.dbg(`manager_run_cell(id='${id}')`);
760
    dbg(JSON.stringify(misc.keys(this._running_cells)));
761

762
    if (this._running_cells == null) {
763
      this._running_cells = {};
764
    }
765

766
    if (this._running_cells[id]) {
767
      dbg("cell already queued to run in kernel");
768
      return;
769
    }
770

771
    // It's important to set this._running_cells[id] to be true so that
772
    // sync_exec_state doesn't declare this cell done.  The kernel identity
773
    // will get set properly below in case it changes.
774
    this._running_cells[id] = this.jupyter_kernel?.identity ?? "none";
775

776
    const orig_cell = this.store.get("cells").get(id);
777
    if (orig_cell == null) {
778
      // nothing to do -- cell deleted
779
      return;
780
    }
781

782
    let input: string | undefined = orig_cell.get("input", "");
783
    if (input == null) {
784
      input = "";
785
    } else {
786
      input = input.trim();
787
    }
788

789
    const halt_on_error: boolean = !orig_cell.get("no_halt", false);
790

791
    if (this.jupyter_kernel == null) {
792
      throw Error("bug -- this is guaranteed by the above");
793
    }
794
    this._running_cells[id] = this.jupyter_kernel.identity;
795

796
    const cell: any = {
797
      id,
798
      type: "cell",
799
      kernel: this.store.get("kernel"),
800
    };
801

802
    dbg(`using max_output_length=${this.store.get("max_output_length")}`);
803
    const handler = this._output_handler(cell);
804

805
    // exponentiallyThrottledSaved calls this.syncdb?.save, but
806
    // it throttles the calls, and does so using exponential backoff
807
    // up to MAX_OUTPUT_SAVE_DELAY milliseconds.   Basically every
808
    // time exponentiallyThrottledSaved is called it increases the
809
    // interval used for throttling by multiplying saveThrottleMs by 1.3
810
    // until saveThrottleMs gets to MAX_OUTPUT_SAVE_DELAY.  There is no
811
    // need at all to do a trailing call, since other code handles that.
812
    let saveThrottleMs = 1;
813
    let lastCall = 0;
814
    const exponentiallyThrottledSaved = () => {
815
      const now = Date.now();
816
      if (now - lastCall < saveThrottleMs) {
817
        return;
818
      }
819
      lastCall = now;
820
      saveThrottleMs = Math.min(1.3 * saveThrottleMs, MAX_OUTPUT_SAVE_DELAY);
821
      this.syncdb?.save();
822
    };
823

824
    handler.on("change", (save) => {
825
      if (!this.store.getIn(["cells", id])) {
826
        // The cell was deleted, but we just got some output
827
        // NOTE: client shouldn't allow deleting running or queued
828
        // cells, but we still want to do something useful/sensible.
829
        // We put cell back where it was with same input.
830
        cell.input = orig_cell.get("input");
831
        cell.pos = orig_cell.get("pos");
832
      }
833
      this.syncdb.set(cell);
834
      // This is potentially very verbose -- don't due it unless
835
      // doing low level debugging:
836
      //dbg(`change (save=${save}): cell='${JSON.stringify(cell)}'`);
837
      if (save) {
838
        exponentiallyThrottledSaved();
839
      }
840
    });
841

842
    handler.once("done", () => {
843
      dbg("handler is done");
844
      this.store.removeListener("cell_change", cell_change);
845
      exec.close();
846
      if (this._running_cells != null) {
847
        delete this._running_cells[id];
848
      }
849
      this.syncdb?.save();
850
      setTimeout(() => this.syncdb?.save(), 100);
851
    });
852

853
    if (this.jupyter_kernel == null) {
854
      handler.error("Unable to start Jupyter");
855
      return;
856
    }
857

858
    const get_password = (): string => {
859
      if (this.jupyter_kernel == null) {
860
        dbg("get_password", id, "no kernel");
861
        return "";
862
      }
863
      const password = this.jupyter_kernel.store.get(id);
864
      dbg("get_password", id, password);
865
      this.jupyter_kernel.store.delete(id);
866
      return password;
867
    };
868

869
    // This is used only for stdin right now.
870
    const cell_change = (cell_id, new_cell) => {
871
      if (id === cell_id) {
872
        dbg("cell_change");
873
        handler.cell_changed(new_cell, get_password);
874
      }
875
    };
876
    this.store.on("cell_change", cell_change);
877

878
    const exec = this.jupyter_kernel.execute_code({
879
      code: input,
880
      id,
881
      stdin: handler.stdin,
882
      halt_on_error,
883
    });
884

885
    exec.on("output", (mesg) => {
886
      // uncomment only for specific low level debugging -- see https://github.com/sagemathinc/cocalc/issues/7022
887
      // dbg(`got mesg='${JSON.stringify(mesg)}'`);  // !!!☡ ☡ ☡  -- EXTREME DANGER ☡ ☡ ☡ !!!!
888

889
      if (mesg == null) {
890
        // can't possibly happen, of course.
891
        const err = "empty mesg";
892
        dbg(`got error='${err}'`);
893
        handler.error(err);
894
        return;
895
      }
896
      if (mesg.done) {
897
        // done is a special internal cocalc message.
898
        handler.done();
899
        return;
900
      }
901
      if (mesg.content?.transient?.display_id != null) {
902
        // See https://github.com/sagemathinc/cocalc/issues/2132
903
        // We find any other outputs in the document with
904
        // the same transient.display_id, and set their output to
905
        // this mesg's output.
906
        this.handleTransientUpdate(mesg);
907
        if (mesg.msg_type == "update_display_data") {
908
          // don't also create a new output
909
          return;
910
        }
911
      }
912

913
      if (mesg.msg_type === "clear_output") {
914
        handler.clear(mesg.content.wait);
915
        return;
916
      }
917

918
      if (mesg.content.comm_id != null) {
919
        // ignore any comm/widget related messages
920
        return;
921
      }
922

923
      if (mesg.content.execution_state === "idle") {
924
        this.store.removeListener("cell_change", cell_change);
925
        return;
926
      }
927
      if (mesg.content.execution_state === "busy") {
928
        handler.start();
929
      }
930
      if (mesg.content.payload != null) {
931
        if (mesg.content.payload.length > 0) {
932
          // payload shell message:
933
          // Despite https://ipython.org/ipython-doc/3/development/messaging.html#payloads saying
934
          // ""Payloads are considered deprecated, though their replacement is not yet implemented."
935
          // we fully have to implement them, since they are used to implement (crazy, IMHO)
936
          // things like %load in the python2 kernel!
937
          mesg.content.payload.map((p) => handler.payload(p));
938
          return;
939
        }
940
      } else {
941
        // Normal iopub output message
942
        handler.message(mesg.content);
943
        return;
944
      }
945
    });
946

947
    exec.on("error", (err) => {
948
      dbg(`got error='${err}'`);
949
      handler.error(err);
950
    });
951
  };
952

953
  reset_more_output = (id: string) => {
954
    if (id == null) {
955
      this.store._more_output = {};
956
    }
957
    if (this.store._more_output[id] != null) {
958
      delete this.store._more_output[id];
959
    }
960
  };
961

962
  set_more_output = (id: string, mesg: object, length: number): void => {
963
    if (this.store._more_output[id] == null) {
964
      this.store._more_output[id] = {
965
        length: 0,
966
        messages: [],
967
        lengths: [],
968
        discarded: 0,
969
        truncated: 0,
970
      };
971
    }
972
    const output = this.store._more_output[id];
973

974
    output.length += length;
975
    output.lengths.push(length);
976
    output.messages.push(mesg);
977

978
    const goal_length = 10 * this.store.get("max_output_length");
979
    while (output.length > goal_length) {
980
      let need: any;
981
      let did_truncate = false;
982

983
      // check if there is a text field, which we can truncate
984
      let len = output.messages[0].text?.length;
985
      if (len != null) {
986
        need = output.length - goal_length + 50;
987
        if (len > need) {
988
          // Instead of throwing this message away, let's truncate its text part.  After
989
          // doing this, the message is at least shorter than it was before.
990
          output.messages[0].text = misc.trunc(
991
            output.messages[0].text,
992
            len - need,
993
          );
994
          did_truncate = true;
995
        }
996
      }
997

998
      // check if there is a text/plain field, which we can thus also safely truncate
999
      if (!did_truncate && output.messages[0].data != null) {
1000
        for (const field in output.messages[0].data) {
1001
          if (field === "text/plain") {
1002
            const val = output.messages[0].data[field];
1003
            len = val.length;
1004
            if (len != null) {
1005
              need = output.length - goal_length + 50;
1006
              if (len > need) {
1007
                // Instead of throwing this message away, let's truncate its text part.  After
1008
                // doing this, the message is at least need shorter than it was before.
1009
                output.messages[0].data[field] = misc.trunc(val, len - need);
1010
                did_truncate = true;
1011
              }
1012
            }
1013
          }
1014
        }
1015
      }
1016

1017
      if (did_truncate) {
1018
        const new_len = JSON.stringify(output.messages[0]).length;
1019
        output.length -= output.lengths[0] - new_len; // how much we saved
1020
        output.lengths[0] = new_len;
1021
        output.truncated += 1;
1022
        break;
1023
      }
1024

1025
      const n = output.lengths.shift();
1026
      output.messages.shift();
1027
      output.length -= n;
1028
      output.discarded += 1;
1029
    }
1030
  };
1031

1032
  private init_file_watcher = () => {
1033
    const dbg = this.dbg("file_watcher");
1034
    dbg();
1035
    this._file_watcher = this._client.watch_file({
1036
      path: this.store.get("path"),
1037
      debounce: 1000,
1038
    });
1039

1040
    this._file_watcher.on("change", async () => {
1041
      dbg("change");
1042
      try {
1043
        await this.loadFromDiskIfNewer();
1044
      } catch (err) {
1045
        dbg("failed to load on change", err);
1046
      }
1047
    });
1048
  };
1049

1050
  /*
1051
    * Unfortunately, though I spent two hours on this approach... it just doesn't work,
1052
    * since, e.g., if the sync file doesn't already exist, it can't be created,
1053
    * which breaks everything.  So disabling for now and re-opening the issue.
1054
    _sync_file_mode: =>
1055
        dbg = @dbg("_sync_file_mode"); dbg()
1056
        * Make the mode of the syncdb file the same as the mode of the .ipynb file.
1057
        * This is used for read-only status.
1058
        ipynb_file  = @store.get('path')
1059
        locals =
1060
            ipynb_file_ro  : undefined
1061
            syncdb_file_ro : undefined
1062
        syncdb_file = @syncdb.get_path()
1063
        async.parallel([
1064
            (cb) ->
1065
                fs.access ipynb_file, fs.constants.W_OK, (err) ->
1066
                    * Also store in @_ipynb_file_ro to prevent starting kernel in this case.
1067
                    @_ipynb_file_ro = locals.ipynb_file_ro = !!err
1068
                    cb()
1069
            (cb) ->
1070
                fs.access syncdb_file, fs.constants.W_OK, (err) ->
1071
                    locals.syncdb_file_ro = !!err
1072
                    cb()
1073
        ], ->
1074
            if locals.ipynb_file_ro == locals.syncdb_file_ro
1075
                return
1076
            dbg("mode change")
1077
            async.parallel([
1078
                (cb) ->
1079
                    fs.stat ipynb_file, (err, stats) ->
1080
                        locals.ipynb_stats = stats
1081
                        cb(err)
1082
                (cb) ->
1083
                    * error if syncdb_file doesn't exist, which is GOOD, since
1084
                    * in that case we do not want to chmod which would create
1085
                    * that file as empty and blank it.
1086
                    fs.stat(syncdb_file, cb)
1087
            ], (err) ->
1088
                if not err
1089
                    dbg("changing syncb mode to match ipynb mode")
1090
                    fs.chmod(syncdb_file, locals.ipynb_stats.mode)
1091
                else
1092
                    dbg("error stating ipynb", err)
1093
            )
1094
        )
1095
    */
1096

1097
  // Load file from disk if it is  newer than
1098
  // the last we saved to disk.
1099
  private loadFromDiskIfNewer = async () => {
1100
    const dbg = this.dbg("loadFromDiskIfNewer");
1101
    // Get mtime of last .ipynb file that we explicitly saved.
1102

1103
    // TODO: breaking the syncdb typescript data hiding.  The
1104
    // right fix will be to move
1105
    // this info to a new ephemeral state table.
1106
    const last_ipynb_save = await this.get_last_ipynb_save();
1107
    dbg(`syncdb last_ipynb_save=${last_ipynb_save}`);
1108
    let file_changed;
1109
    if (last_ipynb_save == 0) {
1110
      // we MUST load from file the first time, of course.
1111
      file_changed = true;
1112
      dbg("file changed because FIRST TIME");
1113
    } else {
1114
      const path = this.store.get("path");
1115
      let stats;
1116
      try {
1117
        stats = await callback2(this._client.path_stat, { path });
1118
        dbg(`stats.mtime = ${stats.mtime}`);
1119
      } catch (err) {
1120
        // This err just means the file doesn't exist.
1121
        // We set the 'last load' to now in this case, since
1122
        // the frontend clients need to know that we
1123
        // have already scanned the disk.
1124
        this.set_last_load();
1125
        return;
1126
      }
1127
      const mtime = stats.mtime.getTime();
1128
      file_changed = mtime > last_ipynb_save;
1129
      dbg({ mtime, last_ipynb_save });
1130
    }
1131
    if (file_changed) {
1132
      dbg(".ipynb disk file changed ==> loading state from disk");
1133
      try {
1134
        await this.load_ipynb_file();
1135
      } catch (err) {
1136
        dbg("failed to load on change", err);
1137
      }
1138
    } else {
1139
      dbg("disk file NOT changed: NOT loading");
1140
    }
1141
  };
1142

1143
  // if also set load is true, we also set the "last_ipynb_save" time.
1144
  set_last_load = (alsoSetLoad: boolean = false) => {
1145
    const last_load = new Date().getTime();
1146
    this.syncdb.set({
1147
      type: "file",
1148
      last_load,
1149
    });
1150
    if (alsoSetLoad) {
1151
      // yes, load v save is inconsistent!
1152
      this.syncdb.set({ type: "settings", last_ipynb_save: last_load });
1153
    }
1154
    this.syncdb.commit();
1155
  };
1156

1157
  /* Determine timestamp of aux .ipynb file, and record it here,
1158
     so we know that we do not have to load exactly that file
1159
     back from disk. */
1160
  private set_last_ipynb_save = async () => {
1161
    let stats;
1162
    try {
1163
      stats = await callback2(this._client.path_stat, {
1164
        path: this.store.get("path"),
1165
      });
1166
    } catch (err) {
1167
      // no-op -- nothing to do.
1168
      this.dbg("set_last_ipynb_save")(`WARNING -- issue in path_stat ${err}`);
1169
      return;
1170
    }
1171

1172
    // This is ugly (i.e., how we get access), but I need to get this done.
1173
    // This is the RIGHT place to save the info though.
1174
    // TODO: move this state info to new ephemeral table.
1175
    try {
1176
      const last_ipynb_save = stats.mtime.getTime();
1177
      this.last_ipynb_save = last_ipynb_save;
1178
      this._set({
1179
        type: "settings",
1180
        last_ipynb_save,
1181
      });
1182
      this.dbg("stats.mtime.getTime()")(
1183
        `set_last_ipynb_save = ${last_ipynb_save}`,
1184
      );
1185
    } catch (err) {
1186
      this.dbg("set_last_ipynb_save")(
1187
        `WARNING -- issue in set_last_ipynb_save ${err}`,
1188
      );
1189
      return;
1190
    }
1191
  };
1192

1193
  private get_last_ipynb_save = async () => {
1194
    const x =
1195
      this.syncdb.get_one({ type: "settings" })?.get("last_ipynb_save") ?? 0;
1196
    return Math.max(x, this.last_ipynb_save);
1197
  };
1198

1199
  load_ipynb_file = async () => {
1200
    /*
1201
    Read the ipynb file from disk.  Fully use the ipynb file to
1202
    set the syncdb's state.  We do this when opening a new file, or when
1203
    the file changes on disk (e.g., a git checkout or something).
1204
    */
1205
    const dbg = this.dbg(`load_ipynb_file`);
1206
    dbg("reading file");
1207
    const path = this.store.get("path");
1208
    let content: string;
1209
    try {
1210
      content = await callback2(this._client.path_read, {
1211
        path,
1212
        maxsize_MB: MAX_SIZE_IPYNB_MB,
1213
      });
1214
    } catch (err) {
1215
      // possibly file doesn't exist -- set notebook to empty.
1216
      const exists = await callback2(this._client.path_exists, {
1217
        path,
1218
      });
1219
      if (!exists) {
1220
        content = "";
1221
      } else {
1222
        // It would be better to have a button to push instead of
1223
        // suggesting running a command in the terminal, but
1224
        // adding that took 1 second.  Better than both would be
1225
        // making it possible to edit huge files :-).
1226
        const error = `Error reading ipynb file '${path}': ${err.toString()}.  Fix this to continue.  You can delete all output by typing cc-jupyter-no-output [filename].ipynb in a terminal.`;
1227
        this.syncdb.set({ type: "fatal", error });
1228
        throw Error(error);
1229
      }
1230
    }
1231
    if (content.length === 0) {
1232
      // Blank file, e.g., when creating in CoCalc.
1233
      // This is good, works, etc. -- just clear state, including error.
1234
      this.syncdb.delete();
1235
      this.set_last_load(true);
1236
      return;
1237
    }
1238

1239
    // File is nontrivial -- parse and load.
1240
    let parsed_content;
1241
    try {
1242
      parsed_content = JSON.parse(content);
1243
    } catch (err) {
1244
      const error = `Error parsing the ipynb file '${path}': ${err}.  You must fix the ipynb file somehow before continuing, or use TimeTravel to revert to a recent version.`;
1245
      dbg(error);
1246
      this.syncdb.set({ type: "fatal", error });
1247
      throw Error(error);
1248
    }
1249
    this.syncdb.delete({ type: "fatal" });
1250
    await this.set_to_ipynb(parsed_content);
1251
    this.set_last_load(true);
1252
  };
1253

1254
  private fetch_jupyter_kernels = async () => {
1255
    const data = await get_kernel_data();
1256
    const kernels = immutable.fromJS(data as any);
1257
    this.setState({ kernels });
1258
  };
1259

1260
  save_ipynb_file = async ({
1261
    version = 0,
1262
    timeout = 15000,
1263
  }: {
1264
    // if version is given, waits (up to timeout ms) for syncdb to
1265
    // contain that exact version before writing the ipynb to disk.
1266
    // This may be needed to ensure that ipynb saved to disk
1267
    // reflects given frontend state.  This comes up, e.g., in
1268
    // generating the nbgrader version of a document.
1269
    version?: number;
1270
    timeout?: number;
1271
  } = {}) => {
1272
    const dbg = this.dbg("save_ipynb_file");
1273
    if (version && !this.syncdb.hasVersion(version)) {
1274
      dbg(`frontend needs ${version}, which we do not yet have`);
1275
      const start = Date.now();
1276
      while (true) {
1277
        if (this.is_closed()) {
1278
          return;
1279
        }
1280
        if (Date.now() - start >= timeout) {
1281
          dbg("timed out waiting");
1282
          break;
1283
        }
1284
        try {
1285
          dbg(`waiting for version ${version}`);
1286
          await once(this.syncdb, "change", timeout - (Date.now() - start));
1287
        } catch {
1288
          dbg("timed out waiting");
1289
          break;
1290
        }
1291
        if (this.syncdb.hasVersion(version)) {
1292
          dbg("now have the version");
1293
          break;
1294
        }
1295
      }
1296
    }
1297
    if (this.is_closed()) {
1298
      return;
1299
    }
1300
    dbg("saving to file");
1301

1302
    // Check first if file was deleted, in which case instead of saving to disk,
1303
    // we should terminate and clean up everything.
1304
    if (this.isDeleted()) {
1305
      dbg("ipynb file is deleted, so NOT saving to disk and closing");
1306
      this.close({ noSave: true });
1307
      return;
1308
    }
1309

1310
    if (this.jupyter_kernel == null) {
1311
      // The kernel is needed to get access to the blob store, which
1312
      // may be needed to save to disk.
1313
      this.ensure_backend_kernel_setup();
1314
      if (this.jupyter_kernel == null) {
1315
        // still not null?  This would happen if no kernel is set at all,
1316
        // in which case it's OK that saving isn't possible.
1317
        throw Error("no kernel so cannot save");
1318
      }
1319
    }
1320
    if (this.store.get("kernels") == null) {
1321
      await this.init_kernel_info();
1322
      if (this.store.get("kernels") == null) {
1323
        // This should never happen, but maybe could in case of a very
1324
        // messed up compute environment where the kernelspecs can't be listed.
1325
        throw Error(
1326
          "kernel info not known and can't be determined, so can't save",
1327
        );
1328
      }
1329
    }
1330
    dbg("going to try to save: getting ipynb object...");
1331
    const blob_store = this.jupyter_kernel.get_blob_store();
1332
    let ipynb = this.store.get_ipynb(blob_store);
1333
    if (this.store.get("kernel")) {
1334
      // if a kernel is set, check that it was sufficiently known that
1335
      // we can fill in data about it --
1336
      //   see https://github.com/sagemathinc/cocalc/issues/7286
1337
      if (ipynb?.metadata?.kernelspec?.name == null) {
1338
        dbg("kernelspec not known -- try loading kernels again");
1339
        await this.fetch_jupyter_kernels();
1340
        // and again grab the ipynb
1341
        ipynb = this.store.get_ipynb(blob_store);
1342
        if (ipynb?.metadata?.kernelspec?.name == null) {
1343
          dbg("kernelspec STILL not known: metadata will be incomplete");
1344
        }
1345
      }
1346
    }
1347
    dbg("got ipynb object");
1348
    // We use json_stable (and indent 1) to be more diff friendly to user,
1349
    // and more consistent with official Jupyter.
1350
    const data = json_stable(ipynb, { space: 1 });
1351
    if (data == null) {
1352
      dbg("failed -- ipynb not defined yet");
1353
      throw Error("ipynb not defined yet; can't save");
1354
    }
1355
    dbg("converted ipynb to stable JSON string", data?.length);
1356
    //dbg(`got string version '${data}'`)
1357
    try {
1358
      dbg("writing to disk...");
1359
      await callback2(this._client.write_file, {
1360
        path: this.store.get("path"),
1361
        data,
1362
      });
1363
      dbg("succeeded at saving");
1364
      await this.set_last_ipynb_save();
1365
    } catch (err) {
1366
      const e = `error writing file: ${err}`;
1367
      dbg(e);
1368
      throw Error(e);
1369
    }
1370
  };
1371

1372
  ensure_there_is_a_cell = () => {
1373
    if (this._state !== "ready") {
1374
      return;
1375
    }
1376
    const cells = this.store.get("cells");
1377
    if (cells == null || cells.size === 0) {
1378
      this._set({
1379
        type: "cell",
1380
        id: this.new_id(),
1381
        pos: 0,
1382
        input: "",
1383
      });
1384
      // We are obviously contributing content to this (empty!) notebook.
1385
      return this.set_trust_notebook(true);
1386
    }
1387
  };
1388

1389
  private handle_all_cell_attachments() {
1390
    // Check if any cell attachments need to be loaded.
1391
    const cells = this.store.get("cells");
1392
    cells?.forEach((cell) => {
1393
      this.handle_cell_attachments(cell);
1394
    });
1395
  }
1396

1397
  private handle_cell_attachments(cell) {
1398
    if (this.jupyter_kernel == null) {
1399
      // can't do anything
1400
      return;
1401
    }
1402
    const dbg = this.dbg(`handle_cell_attachments(id=${cell.get("id")})`);
1403
    dbg();
1404

1405
    const attachments = cell.get("attachments");
1406
    if (attachments == null) return; // nothing to do
1407
    attachments.forEach(async (x, name) => {
1408
      if (x == null) return;
1409
      if (x.get("type") === "load") {
1410
        if (this.jupyter_kernel == null) return; // try later
1411
        // need to load from disk
1412
        this.set_cell_attachment(cell.get("id"), name, {
1413
          type: "loading",
1414
          value: null,
1415
        });
1416
        let sha1: string;
1417
        try {
1418
          sha1 = await this.jupyter_kernel.load_attachment(x.get("value"));
1419
        } catch (err) {
1420
          this.set_cell_attachment(cell.get("id"), name, {
1421
            type: "error",
1422
            value: `${err}`,
1423
          });
1424
          return;
1425
        }
1426
        this.set_cell_attachment(cell.get("id"), name, {
1427
          type: "sha1",
1428
          value: sha1,
1429
        });
1430
      }
1431
    });
1432
  }
1433

1434
  // handle_ipywidgets_state_change is called when the project ipywidgets_state
1435
  // object changes, e.g., in response to a user moving a slider in the browser.
1436
  // It crafts a comm message that is sent to the running Jupyter kernel telling
1437
  // it about this change by calling send_comm_message_to_kernel.
1438
  private handle_ipywidgets_state_change = (keys): void => {
1439
    if (this.is_closed()) {
1440
      return;
1441
    }
1442
    const dbg = this.dbg("handle_ipywidgets_state_change");
1443
    dbg(keys);
1444
    if (this.jupyter_kernel == null) {
1445
      dbg("no kernel, so ignoring changes to ipywidgets");
1446
      return;
1447
    }
1448
    if (this.syncdb.ipywidgets_state == null) {
1449
      throw Error("syncdb's ipywidgets_state must be defined!");
1450
    }
1451
    for (const key of keys) {
1452
      const [, model_id, type] = JSON.parse(key);
1453
      dbg({ key, model_id, type });
1454
      let data: any;
1455
      if (type === "value") {
1456
        const state = this.syncdb.ipywidgets_state.get_model_value(model_id);
1457
        // Saving the buffers on change is critical since otherwise this breaks:
1458
        //  https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html#file-upload
1459
        // Note that stupidly the buffer (e.g., image upload) gets sent to the kernel twice.
1460
        // But it does work robustly, and the kernel and nodejs server processes next to each
1461
        // other so this isn't so bad.
1462
        const { buffer_paths, buffers } =
1463
          this.syncdb.ipywidgets_state.getKnownBuffers(model_id);
1464
        data = { method: "update", state, buffer_paths };
1465
        this.jupyter_kernel.send_comm_message_to_kernel({
1466
          msg_id: misc.uuid(),
1467
          target_name: "jupyter.widget",
1468
          comm_id: model_id,
1469
          data,
1470
          buffers,
1471
        });
1472
      } else if (type === "buffers") {
1473
        // TODO: we MIGHT need implement this... but MAYBE NOT.  An example where this seems like it might be
1474
        // required is by the file upload widget, but actually that just uses the value type above, since
1475
        // we explicitly fill in the widgets there; also there is an explicit comm upload message that
1476
        // the widget sends out that updates the buffer, and in send_comm_message_to_kernel in jupyter/kernel/kernel.ts
1477
        // when processing that message, we saves those buffers and make sure they are set in the
1478
        // value case above (otherwise they would get removed).
1479
        //    https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html#file-upload
1480
        // which creates a buffer from the content of the file, then sends it to the backend,
1481
        // which sees a change and has to write that buffer to the kernel (here) so that
1482
        // the running python process can actually do something with the file contents (e.g.,
1483
        // process data, save file to disk, etc).
1484
        // We need to be careful though to not send buffers to the kernel that the kernel sent us,
1485
        // since that would be a waste.
1486
      } else if (type === "state") {
1487
        // TODO: currently ignoring this, since it seems chatty and pointless,
1488
        // and could lead to race conditions probably with multiple users, etc.
1489
        // It happens right when the widget is created.
1490
        /*
1491
        const state = this.syncdb.ipywidgets_state.getModelSerializedState(model_id);
1492
        data = { method: "update", state };
1493
        this.jupyter_kernel.send_comm_message_to_kernel(
1494
          misc.uuid(),
1495
          model_id,
1496
          data
1497
        );
1498
        */
1499
      } else {
1500
        const m = `Jupyter: unknown type '${type}'`;
1501
        console.warn(m);
1502
        dbg(m);
1503
      }
1504
    }
1505
  };
1506

1507
  async process_comm_message_from_kernel(mesg: any): Promise<void> {
1508
    const dbg = this.dbg("process_comm_message_from_kernel");
1509
    // serializing the full message could cause enormous load on the server, since
1510
    // the mesg may contain large buffers.  Only do for low level debugging!
1511
    // dbg(mesg); // EXTREME DANGER!
1512
    // This should be safe:
1513
    dbg(JSON.stringify(mesg.header));
1514
    if (this.syncdb.ipywidgets_state == null) {
1515
      throw Error("syncdb's ipywidgets_state must be defined!");
1516
    }
1517
    await this.syncdb.ipywidgets_state.process_comm_message_from_kernel(mesg);
1518
  }
1519

1520
  capture_output_message(mesg: any): boolean {
1521
    if (this.syncdb.ipywidgets_state == null) {
1522
      throw Error("syncdb's ipywidgets_state must be defined!");
1523
    }
1524
    return this.syncdb.ipywidgets_state.capture_output_message(mesg);
1525
  }
1526

1527
  close_project_only() {
1528
    const dbg = this.dbg("close_project_only");
1529
    dbg();
1530
    if (this.run_all_loop) {
1531
      this.run_all_loop.close();
1532
      delete this.run_all_loop;
1533
    }
1534
    // this stops the kernel and cleans everything up
1535
    // so no resources are wasted and next time starting
1536
    // is clean
1537
    (async () => {
1538
      try {
1539
        await removeJupyterRedux(this.store.get("path"), this.project_id);
1540
      } catch (err) {
1541
        dbg("WARNING -- issue removing jupyter redux", err);
1542
      }
1543
    })();
1544

1545
    this.blobs?.close();
1546
  }
1547

1548
  // not actually async...
1549
  async signal(signal = "SIGINT"): Promise<void> {
1550
    this.jupyter_kernel?.signal(signal);
1551
  }
1552

1553
  handle_nbconvert_change(oldVal, newVal): void {
1554
    nbconvertChange(this, oldVal?.toJS(), newVal?.toJS());
1555
  }
1556

1557
  // Handle transient cell messages.
1558
  handleTransientUpdate = (mesg) => {
1559
    const display_id = mesg.content?.transient?.display_id;
1560
    if (!display_id) {
1561
      return false;
1562
    }
1563

1564
    let matched = false;
1565
    // are there any transient outputs in the entire document that
1566
    // have this display_id?  search to find them.
1567
    // TODO: we could use a clever data structure to make
1568
    // this faster and more likely to have bugs.
1569
    const cells = this.syncdb.get({ type: "cell" });
1570
    for (let cell of cells) {
1571
      let output = cell.get("output");
1572
      if (output != null) {
1573
        for (const [n, val] of output) {
1574
          if (val.getIn(["transient", "display_id"]) == display_id) {
1575
            // found a match -- replace it
1576
            output = output.set(n, immutable.fromJS(mesg.content));
1577
            this.syncdb.set({ type: "cell", id: cell.get("id"), output });
1578
            matched = true;
1579
          }
1580
        }
1581
      }
1582
    }
1583
    if (matched) {
1584
      this.syncdb.commit();
1585
    }
1586
  };
1587

1588
  getComputeServers = () => {
1589
    // we don't bother worrying about freeing this since it is only
1590
    // run in the project or compute server, which needs the underlying
1591
    // dkv for its entire lifetime anyways.
1592
    if (this.computeServers == null) {
1593
      this.computeServers = computeServerManager({
1594
        project_id: this.project_id,
1595
      });
1596
    }
1597
    return this.computeServers;
1598
  };
1599

1600
  getComputeServerIdSync = (): number => {
1601
    const c = this.getComputeServers();
1602
    return c.get(this.syncdb.path) ?? 0;
1603
  };
1604

1605
  getComputeServerId = async (): Promise<number> => {
1606
    const c = this.getComputeServers();
1607
    return (await c.getServerIdForPath(this.syncdb.path)) ?? 0;
1608
  };
1609
}
1610

1611