1
/*
2
 *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
3
 *  License: MS-RSL – see LICENSE.md for details
4
 */
5

6
/*
7
Jupyter actions -- these are the actions for the underlying document structure.
8
This can be used both on the frontend and the backend.
9
*/
10

11
// This was 10000 for a while and that caused regular noticeable problems:
12
//    https://github.com/sagemathinc/cocalc/issues/4590
13
const DEFAULT_MAX_OUTPUT_LENGTH = 250000;
14
//const DEFAULT_MAX_OUTPUT_LENGTH = 1000;
15

16
// Maximum number of output messages total.  If nmore, you have to click
17
// "Fetch additional output" to see them.
18
export const MAX_OUTPUT_MESSAGES = 500;
19
//export const MAX_OUTPUT_MESSAGES = 5;
20

21
// Limit blob store to 100 MB. This means you can have at most this much worth
22
// of recents images displayed in notebooks.  E.g, if you had a single
23
// notebook with more than this much in images, the oldest ones would
24
// start vanishing from output.  Also, this impacts time travel.
25
// WARNING: It is *not* at all difficult to hit fairly large sizes, e.g., 50MB+
26
// when working with a notebook, by just drawing a bunch of large plots.
27
const MAX_BLOB_STORE_SIZE = 100 * 1e6;
28

29
declare const localStorage: any;
30

31
import * as immutable from "immutable";
32
import { Actions } from "@cocalc/util/redux/Actions";
33
import { three_way_merge } from "@cocalc/sync/editor/generic/util";
34
import { callback2, retry_until_success } from "@cocalc/util/async-utils";
35
import * as misc from "@cocalc/util/misc";
36
import { delay } from "awaiting";
37
import * as cell_utils from "@cocalc/jupyter/util/cell-utils";
38
import { JupyterStore, JupyterStoreState } from "@cocalc/jupyter/redux/store";
39
import { Cell, KernelInfo } from "@cocalc/jupyter/types";
40
import { IPynbImporter } from "@cocalc/jupyter/ipynb/import-from-ipynb";
41
import type { JupyterKernelInterface } from "@cocalc/jupyter/types/project-interface";
42
import {
43
  char_idx_to_js_idx,
44
  codemirror_to_jupyter_pos,
45
  js_idx_to_char_idx,
46
} from "@cocalc/jupyter/util/misc";
47
import { SyncDB } from "@cocalc/sync/editor/db/sync";
48
import type { Client } from "@cocalc/sync/client/types";
49
import latexEnvs from "@cocalc/util/latex-envs";
50
import { jupyterApiClient } from "@cocalc/conat/service/jupyter";
51
import { type AKV, akv } from "@cocalc/conat/sync/akv";
52
import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
53

54
const { close, required, defaults } = misc;
55

56
/*
57
The actions -- what you can do with a jupyter notebook, and also the
58
underlying synchronized state.
59
*/
60

61
// no worries, they don't break react rendering even when they escape
62
const CellWriteProtectedException = new Error("CellWriteProtectedException");
63
const CellDeleteProtectedException = new Error("CellDeleteProtectedException");
64

65
type State = "init" | "load" | "ready" | "closed";
66

67
export abstract class JupyterActions extends Actions<JupyterStoreState> {
68
  public is_project: boolean;
69
  public is_compute_server?: boolean;
70
  readonly path: string;
71
  readonly project_id: string;
72
  private _last_start?: number;
73
  public jupyter_kernel?: JupyterKernelInterface;
74
  private last_cursor_move_time: Date = new Date(0);
75
  private _cursor_locs?: any;
76
  private _introspect_request?: any;
77
  protected set_save_status: any;
78
  protected _client: Client;
79
  protected _file_watcher: any;
80
  protected _state: State;
81
  protected restartKernelOnClose?: (...args: any[]) => void;
82
  public asyncBlobStore: AKV;
83

84
  public _complete_request?: number;
85
  public store: JupyterStore;
86
  public syncdb: SyncDB;
87
  private labels?: {
88
    math: { [label: string]: { tag: string; id: string } };
89
    fig: { [label: string]: { tag: string; id: string } };
90
  };
91

92
  public _init(
93
    project_id: string,
94
    path: string,
95
    syncdb: SyncDB,
96
    store: any,
97
    client: Client,
98
  ): void {
99
    this._client = client;
100
    const dbg = this.dbg("_init");
101
    dbg("Initializing Jupyter Actions");
102
    if (project_id == null || path == null) {
103
      // typescript should ensure this, but just in case.
104
      throw Error("type error -- project_id and path can't be null");
105
    }
106
    store.dbg = (f) => {
107
      return client.dbg(`JupyterStore('${store.get("path")}').${f}`);
108
    };
109
    this._state = "init"; // 'init', 'load', 'ready', 'closed'
110
    this.store = store;
111
    // @ts-ignore
112
    this.project_id = project_id;
113
    // @ts-ignore
114
    this.path = path;
115
    store.syncdb = syncdb;
116
    this.syncdb = syncdb;
117
    // the project client is designated to manage execution/conflict, etc.
118
    this.is_project = client.is_project();
119
    if (this.is_project) {
120
      this.syncdb.on("first-load", () => {
121
        dbg("handling first load of syncdb in project");
122
        // Clear settings the first time the syncdb is ever
123
        // loaded, since it has settings like "ipynb last save"
124
        // and trust, which shouldn't be initialized to
125
        // what they were before. Not doing this caused
126
        // https://github.com/sagemathinc/cocalc/issues/7074
127
        this.syncdb.delete({ type: "settings" });
128
        this.syncdb.commit();
129
      });
130
    }
131
    this.is_compute_server = client.is_compute_server();
132

133
    let directory: any;
134
    const split_path = misc.path_split(path);
135
    if (split_path != null) {
136
      directory = split_path.head;
137
    }
138

139
    this.setState({
140
      error: undefined,
141
      has_unsaved_changes: false,
142
      sel_ids: immutable.Set(), // immutable set of selected cells
143
      md_edit_ids: immutable.Set(), // set of ids of markdown cells in edit mode
144
      mode: "escape",
145
      project_id,
146
      directory,
147
      path,
148
      max_output_length: DEFAULT_MAX_OUTPUT_LENGTH,
149
    });
150

151
    this.syncdb.on("change", this._syncdb_change);
152

153
    this.syncdb.on("close", this.close);
154

155
    this.asyncBlobStore = akv(this.blobStoreOptions());
156

157
    // Hook for additional initialization.
158
    this.init2();
159
  }
160

161
  protected blobStoreOptions = () => {
162
    return {
163
      name: `jupyter:${this.path}`,
164
      project_id: this.project_id,
165
      config: {
166
        max_bytes: MAX_BLOB_STORE_SIZE,
167
      },
168
    } as const;
169
  };
170

171
  // default is to do nothing, but e.g., frontend browser client
172
  // does overload this to do a lot of additional init.
173
  protected init2(): void {
174
    // this can be overloaded in a derived class
175
  }
176

177
  // Only use this on the frontend, of course.
178
  protected getFrameActions() {
179
    return this.redux.getEditorActions(this.project_id, this.path);
180
  }
181

182
  sync_read_only = (): void => {
183
    if (this._state == "closed") return;
184
    const a = this.store.get("read_only");
185
    const b = this.syncdb?.is_read_only();
186
    if (a !== b) {
187
      this.setState({ read_only: b });
188
      this.set_cm_options();
189
    }
190
  };
191

192
  protected api = (opts: { timeout?: number } = {}) => {
193
    return jupyterApiClient({
194
      project_id: this.project_id,
195
      path: this.path,
196
      timeout: opts.timeout,
197
    });
198
  };
199

200
  protected dbg(f: string) {
201
    if (this.is_closed()) {
202
      // calling dbg after the actions are closed is possible; this.store would
203
      // be undefined, and then this log message would crash, which sucks.  It happened to me.
204
      // See https://github.com/sagemathinc/cocalc/issues/6788
205
      return (..._) => {};
206
    }
207
    return this._client.dbg(`JupyterActions("${this.path}").${f}`);
208
  }
209

210
  protected close_client_only(): void {
211
    // no-op: this can be defined in a derived class. E.g., in the frontend, it removes
212
    // an account_change listener.
213
  }
214

215
  public is_closed(): boolean {
216
    return this._state === "closed" || this._state === undefined;
217
  }
218

219
  public async close({ noSave }: { noSave?: boolean } = {}): Promise<void> {
220
    if (this.is_closed()) {
221
      return;
222
    }
223
    // ensure save to disk happens:
224
    //   - it will automatically happen for the sync-doc file, but
225
    //     we also need it for the ipynb file... as ipynb is unique
226
    //     in having two formats.
227
    if (!noSave) {
228
      await this.save();
229
    }
230
    if (this.is_closed()) {
231
      return;
232
    }
233

234
    if (this.syncdb != null) {
235
      this.syncdb.close();
236
    }
237
    if (this._file_watcher != null) {
238
      this._file_watcher.close();
239
    }
240
    if (this.is_project || this.is_compute_server) {
241
      this.close_project_only();
242
    } else {
243
      this.close_client_only();
244
    }
245
    // We *must* destroy the action before calling close,
246
    // since otherwise this.redux and this.name are gone,
247
    // which makes destroying the actions properly impossible.
248
    this.destroy();
249
    this.store.destroy();
250
    close(this);
251
    this._state = "closed";
252
  }
253

254
  public close_project_only() {
255
    // real version is in derived class that project runs.
256
  }
257

258
  set_error = (err: any): void => {
259
    if (this._state === "closed") return;
260
    if (err == null) {
261
      this.setState({ error: undefined }); // delete from store
262
      return;
263
    }
264
    if (typeof err != "string") {
265
      err = `${err}`;
266
    }
267
    const cur = this.store.get("error");
268
    // don't show the same error more than once
269
    if ((cur?.indexOf(err) ?? -1) >= 0) {
270
      return;
271
    }
272
    if (cur) {
273
      err = err + "\n\n" + cur;
274
    }
275
    this.setState({ error: err });
276
  };
277

278
  // Set the input of the given cell in the syncdb, which will also change the store.
279
  // Might throw a CellWriteProtectedException
280
  public set_cell_input(id: string, input: string, save = true): void {
281
    if (!this.store) return;
282
    if (this.store.getIn(["cells", id, "input"]) == input) {
283
      // nothing changed.   Note, I tested doing the above check using
284
      // both this.syncdb and this.store, and this.store is orders of magnitude faster.
285
      return;
286
    }
287
    if (this.check_edit_protection(id, "changing input")) {
288
      // note -- we assume above that there was an actual change before checking
289
      // for edit protection.  Thus the above check is important.
290
      return;
291
    }
292
    this._set(
293
      {
294
        type: "cell",
295
        id,
296
        input,
297
        start: null,
298
        end: null,
299
      },
300
      save,
301
    );
302
  }
303

304
  set_cell_output = (id: string, output: any, save = true) => {
305
    this._set(
306
      {
307
        type: "cell",
308
        id,
309
        output,
310
      },
311
      save,
312
    );
313
  };
314

315
  setCellId = (id: string, newId: string, save = true) => {
316
    let cell = this.store.getIn(["cells", id])?.toJS();
317
    if (cell == null) {
318
      return;
319
    }
320
    cell.id = newId;
321
    this.syncdb.delete({ type: "cell", id });
322
    this.syncdb.set(cell);
323
    if (save) {
324
      this.syncdb.commit();
325
    }
326
  };
327

328
  clear_selected_outputs = () => {
329
    this.deprecated("clear_selected_outputs");
330
  };
331

332
  // Clear output in the list of cell id's.
333
  // NOTE: clearing output *is* allowed for non-editable cells, since the definition
334
  // of editable is that the *input* is editable.
335
  // See https://github.com/sagemathinc/cocalc/issues/4805
336
  public clear_outputs(cell_ids: string[], save: boolean = true): void {
337
    const cells = this.store.get("cells");
338
    if (cells == null) return; // nothing to do
339
    for (const id of cell_ids) {
340
      const cell = cells.get(id);
341
      if (cell == null) continue;
342
      if (cell.get("output") != null || cell.get("exec_count")) {
343
        this._set({ type: "cell", id, output: null, exec_count: null }, false);
344
      }
345
    }
346
    if (save) {
347
      this._sync();
348
    }
349
  }
350

351
  public clear_all_outputs(save: boolean = true): void {
352
    this.clear_outputs(this.store.get_cell_list().toJS(), save);
353
  }
354

355
  private show_not_xable_error(x: string, n: number, reason?: string): void {
356
    if (n <= 0) return;
357
    const verb: string = n === 1 ? "is" : "are";
358
    const noun: string = misc.plural(n, "cell");
359
    this.set_error(
360
      `${n} ${noun} ${verb} protected from ${x}${
361
        reason ? " when " + reason : ""
362
      }.`,
363
    );
364
  }
365

366
  private show_not_editable_error(reason?: string): void {
367
    this.show_not_xable_error("editing", 1, reason);
368
  }
369

370
  private show_not_deletable_error(n: number = 1): void {
371
    this.show_not_xable_error("deletion", n);
372
  }
373

374
  public toggle_output(id: string, property: "collapsed" | "scrolled"): void {
375
    this.toggle_outputs([id], property);
376
  }
377

378
  public toggle_outputs(
379
    cell_ids: string[],
380
    property: "collapsed" | "scrolled",
381
  ): void {
382
    const cells = this.store.get("cells");
383
    if (cells == null) {
384
      throw Error("cells not defined");
385
    }
386
    for (const id of cell_ids) {
387
      const cell = cells.get(id);
388
      if (cell == null) {
389
        throw Error(`no cell with id ${id}`);
390
      }
391
      if (cell.get("cell_type", "code") == "code") {
392
        this._set(
393
          {
394
            type: "cell",
395
            id,
396
            [property]: !cell.get(
397
              property,
398
              property == "scrolled" ? false : true, // default scrolled to false
399
            ),
400
          },
401
          false,
402
        );
403
      }
404
    }
405
    this._sync();
406
  }
407

408
  public toggle_all_outputs(property: "collapsed" | "scrolled"): void {
409
    this.toggle_outputs(this.store.get_cell_ids_list(), property);
410
  }
411

412
  public set_cell_pos(id: string, pos: number, save: boolean = true): void {
413
    this._set({ type: "cell", id, pos }, save);
414
  }
415

416
  public moveCell(
417
    oldIndex: number,
418
    newIndex: number,
419
    save: boolean = true,
420
  ): void {
421
    if (oldIndex == newIndex) return; // nothing to do
422
    // Move the cell that is currently at position oldIndex to
423
    // be at position newIndex.
424
    const cell_list = this.store.get_cell_list();
425
    const newPos = cell_utils.moveCell({
426
      oldIndex,
427
      newIndex,
428
      size: cell_list.size,
429
      getPos: (index) =>
430
        this.store.getIn(["cells", cell_list.get(index) ?? "", "pos"]) ?? 0,
431
    });
432
    this.set_cell_pos(cell_list.get(oldIndex) ?? "", newPos, save);
433
  }
434

435
  public set_cell_type(
436
    id: string,
437
    cell_type: string = "code",
438
    save: boolean = true,
439
  ): void {
440
    if (this.check_edit_protection(id, "changing cell type")) return;
441
    if (
442
      cell_type !== "markdown" &&
443
      cell_type !== "raw" &&
444
      cell_type !== "code"
445
    ) {
446
      throw Error(
447
        `cell type (='${cell_type}') must be 'markdown', 'raw', or 'code'`,
448
      );
449
    }
450
    const obj: any = {
451
      type: "cell",
452
      id,
453
      cell_type,
454
    };
455
    if (cell_type !== "code") {
456
      // delete output and exec time info when switching to non-code cell_type
457
      obj.output = obj.start = obj.end = obj.collapsed = obj.scrolled = null;
458
    }
459
    this._set(obj, save);
460
  }
461

462
  public set_selected_cell_type(cell_type: string): void {
463
    this.deprecated("set_selected_cell_type", cell_type);
464
  }
465

466
  set_md_cell_editing = (id: string): void => {
467
    this.deprecated("set_md_cell_editing", id);
468
  };
469

470
  set_md_cell_not_editing = (id: string): void => {
471
    this.deprecated("set_md_cell_not_editing", id);
472
  };
473

474
  // Set which cell is currently the cursor.
475
  set_cur_id = (id: string): void => {
476
    this.deprecated("set_cur_id", id);
477
  };
478

479
  protected deprecated(f: string, ...args): void {
480
    const s = "DEPRECATED JupyterActions(" + this.path + ")." + f;
481
    console.warn(s, ...args);
482
  }
483

484
  private set_cell_list(): void {
485
    const cells = this.store.get("cells");
486
    if (cells == null) {
487
      return;
488
    }
489
    const cell_list = cell_utils.sorted_cell_list(cells);
490
    if (!cell_list.equals(this.store.get_cell_list())) {
491
      this.setState({ cell_list });
492
      this.store.emit("cell-list-recompute");
493
    }
494
  }
495

496
  private syncdb_cell_change = (id: string, new_cell: any): boolean => {
497
    const cells: immutable.Map<
498
      string,
499
      immutable.Map<string, any>
500
    > = this.store.get("cells");
501
    if (cells == null) {
502
      throw Error("BUG -- cells must have been initialized!");
503
    }
504

505
    let cell_list_needs_recompute = false;
506
    //this.dbg("_syncdb_cell_change")("#{id} #{JSON.stringify(new_cell?.toJS())}")
507
    let old_cell = cells.get(id);
508
    if (new_cell == null) {
509
      // delete cell
510
      this.reset_more_output(id); // free up memory locally
511
      if (old_cell != null) {
512
        const cell_list = this.store.get_cell_list().filter((x) => x !== id);
513
        this.setState({ cells: cells.delete(id), cell_list });
514
      }
515
    } else {
516
      // change or add cell
517
      old_cell = cells.get(id);
518
      if (new_cell.equals(old_cell)) {
519
        return false; // nothing to do
520
      }
521
      if (
522
        old_cell != null &&
523
        new_cell.get("start") > old_cell.get("start") &&
524
        !this.is_project &&
525
        !this.is_compute_server
526
      ) {
527
        // cell re-evaluated so any more output is no longer valid -- clear frontend state
528
        this.reset_more_output(id);
529
      }
530
      if (old_cell == null || old_cell.get("pos") !== new_cell.get("pos")) {
531
        cell_list_needs_recompute = true;
532
      }
533
      // preserve cursor info if happen to have it, rather than just letting
534
      // it get deleted whenever the cell changes.
535
      if (old_cell?.has("cursors")) {
536
        new_cell = new_cell.set("cursors", old_cell.get("cursors"));
537
      }
538
      this.setState({ cells: cells.set(id, new_cell) });
539
      if (this.store.getIn(["edit_cell_metadata", "id"]) === id) {
540
        this.edit_cell_metadata(id); // updates the state during active editing.
541
      }
542
    }
543

544
    this.onCellChange(id, new_cell, old_cell);
545
    this.store.emit("cell_change", id, new_cell, old_cell);
546

547
    return cell_list_needs_recompute;
548
  };
549

550
  _syncdb_change = (changes: any) => {
551
    if (this.syncdb == null) return;
552
    this.store.emit("syncdb-before-change");
553
    this.__syncdb_change(changes);
554
    this.store.emit("syncdb-after-change");
555
    if (this.set_save_status != null) {
556
      this.set_save_status();
557
    }
558
  };
559

560
  __syncdb_change = (changes: any): void => {
561
    if (
562
      this.syncdb == null ||
563
      changes == null ||
564
      (changes != null && changes.size == 0)
565
    ) {
566
      return;
567
    }
568
    const doInit = this._state === "init";
569
    let cell_list_needs_recompute = false;
570

571
    if (changes == "all" || this.store.get("cells") == null) {
572
      // changes == 'all' is used by nbgrader to set the state...
573
      // First time initialization, rather than some small
574
      // update.  We could use the same code, e.g.,
575
      // calling syncdb_cell_change, but that SCALES HORRIBLY
576
      // as the number of cells gets large!
577

578
      // this.syncdb.get() returns an immutable.List of all the records
579
      // in the syncdb database.   These look like, e.g.,
580
      //    {type: "settings", backend_state: "running", trust: true, kernel: "python3", …}
581
      //    {type: "cell", id: "22cc3e", pos: 0, input: "# small copy", state: "done"}
582
      let cells: immutable.Map<string, Cell> = immutable.Map();
583
      this.syncdb.get().forEach((record) => {
584
        switch (record.get("type")) {
585
          case "cell":
586
            cells = cells.set(record.get("id"), record);
587
            break;
588
          case "settings":
589
            if (record == null) {
590
              return;
591
            }
592
            const orig_kernel = this.store.get("kernel");
593
            const kernel = record.get("kernel");
594
            const obj: any = {
595
              trust: !!record.get("trust"), // case to boolean
596
              backend_state: record.get("backend_state"),
597
              last_backend_state: record.get("last_backend_state"),
598
              kernel_state: record.get("kernel_state"),
599
              metadata: record.get("metadata"), // extra custom user-specified metadata
600
              max_output_length: bounded_integer(
601
                record.get("max_output_length"),
602
                100,
603
                250000,
604
                DEFAULT_MAX_OUTPUT_LENGTH,
605
              ),
606
            };
607
            if (kernel !== orig_kernel) {
608
              obj.kernel = kernel;
609
              obj.kernel_info = this.store.get_kernel_info(kernel);
610
              obj.backend_kernel_info = undefined;
611
            }
612
            this.setState(obj);
613
            if (
614
              !this.is_project &&
615
              !this.is_compute_server &&
616
              orig_kernel !== kernel
617
            ) {
618
              this.set_cm_options();
619
            }
620

621
            break;
622
        }
623
      });
624

625
      this.setState({ cells, cell_list: cell_utils.sorted_cell_list(cells) });
626
      cell_list_needs_recompute = false;
627
    } else {
628
      changes.forEach((key) => {
629
        const type: string = key.get("type");
630
        const record = this.syncdb.get_one(key);
631
        switch (type) {
632
          case "cell":
633
            if (this.syncdb_cell_change(key.get("id"), record)) {
634
              cell_list_needs_recompute = true;
635
            }
636
            break;
637
          case "fatal":
638
            const error = record != null ? record.get("error") : undefined;
639
            this.setState({ fatal: error });
640
            // This check can be deleted in a few weeks:
641
            if (
642
              error != null &&
643
              error.indexOf("file is currently being read or written") !== -1
644
            ) {
645
              // No longer relevant -- see https://github.com/sagemathinc/cocalc/issues/1742
646
              this.syncdb.delete({ type: "fatal" });
647
              this.syncdb.commit();
648
            }
649
            break;
650

651
          case "nbconvert":
652
            if (this.is_project || this.is_compute_server) {
653
              // before setting in store, let backend start reacting to change
654
              this.handle_nbconvert_change(this.store.get("nbconvert"), record);
655
            }
656
            // Now set in our store.
657
            this.setState({ nbconvert: record });
658
            break;
659

660
          case "settings":
661
            if (record == null) {
662
              return;
663
            }
664
            const orig_kernel = this.store.get("kernel", null);
665
            const kernel = record.get("kernel");
666
            const obj: any = {
667
              trust: !!record.get("trust"), // case to boolean
668
              backend_state: record.get("backend_state"),
669
              last_backend_state: record.get("last_backend_state"),
670
              kernel_state: record.get("kernel_state"),
671
              kernel_error: record.get("kernel_error"),
672
              metadata: record.get("metadata"), // extra custom user-specified metadata
673
              connection_file: record.get("connection_file") ?? "",
674
              max_output_length: bounded_integer(
675
                record.get("max_output_length"),
676
                100,
677
                250000,
678
                DEFAULT_MAX_OUTPUT_LENGTH,
679
              ),
680
            };
681
            if (kernel !== orig_kernel) {
682
              obj.kernel = kernel;
683
              obj.kernel_info = this.store.get_kernel_info(kernel);
684
              obj.backend_kernel_info = undefined;
685
            }
686
            const prev_backend_state = this.store.get("backend_state");
687
            this.setState(obj);
688
            if (!this.is_project && !this.is_compute_server) {
689
              // if the kernel changes or it just started running – we set the codemirror options!
690
              // otherwise, just when computing them without the backend information, only a crude
691
              // heuristic sets the values and we end up with "C" formatting for custom python kernels.
692
              // @see https://github.com/sagemathinc/cocalc/issues/5478
693
              const started_running =
694
                record.get("backend_state") === "running" &&
695
                prev_backend_state !== "running";
696
              if (orig_kernel !== kernel || started_running) {
697
                this.set_cm_options();
698
              }
699
            }
700
            break;
701
        }
702
      });
703
    }
704
    if (cell_list_needs_recompute) {
705
      this.set_cell_list();
706
    }
707

708
    this.__syncdb_change_post_hook(doInit);
709
  };
710

711
  protected __syncdb_change_post_hook(_doInit: boolean) {
712
    // no-op in base class -- does interesting and different
713
    // things in project, browser, etc.
714
  }
715

716
  protected onCellChange(_id: string, _new_cell: any, _old_cell: any) {
717
    // no-op in base class.  This is a hook though
718
    // for potentially doing things when any cell changes.
719
  }
720

721
  ensure_backend_kernel_setup() {
722
    // nontrivial in the project, but not in client or here.
723
  }
724

725
  protected _output_handler(_cell: any) {
726
    throw Error("define in a derived class.");
727
  }
728

729
  /*
730
  WARNING: Changes via set that are made when the actions
731
  are not 'ready' or the syncdb is not ready are ignored.
732
  These might happen right now if the user were to try to do
733
  some random thing at the exact moment they are closing the
734
  notebook. See https://github.com/sagemathinc/cocalc/issues/4274
735
  */
736
  _set = (obj: any, save: boolean = true) => {
737
    if (
738
      // _set is called during initialization, so don't
739
      // require this._state to be 'ready'!
740
      this._state === "closed" ||
741
      this.store.get("read_only") ||
742
      (this.syncdb != null && this.syncdb.get_state() != "ready")
743
    ) {
744
      // no possible way to do anything.
745
      return;
746
    }
747
    // check write protection regarding specific keys to be set
748
    if (
749
      obj.type === "cell" &&
750
      obj.id != null &&
751
      !this.store.is_cell_editable(obj.id)
752
    ) {
753
      for (const protected_key of ["input", "cell_type", "attachments"]) {
754
        if (misc.has_key(obj, protected_key)) {
755
          throw CellWriteProtectedException;
756
        }
757
      }
758
    }
759
    //@dbg("_set")("obj=#{misc.to_json(obj)}")
760
    this.syncdb.set(obj);
761
    if (save) {
762
      this.syncdb.commit();
763
    }
764
    // ensure that we update locally immediately for our own changes.
765
    this._syncdb_change(
766
      immutable.fromJS([misc.copy_with(obj, ["id", "type"])]),
767
    );
768
  };
769

770
  // might throw a CellDeleteProtectedException
771
  _delete = (obj: any, save = true) => {
772
    if (this._state === "closed" || this.store.get("read_only")) {
773
      return;
774
    }
775
    // check: don't delete cells marked as deletable=false
776
    if (obj.type === "cell" && obj.id != null) {
777
      if (!this.store.is_cell_deletable(obj.id)) {
778
        throw CellDeleteProtectedException;
779
      }
780
    }
781
    this.syncdb.delete(obj);
782
    if (save) {
783
      this.syncdb.commit();
784
    }
785
    this._syncdb_change(immutable.fromJS([{ type: obj.type, id: obj.id }]));
786
  };
787

788
  public _sync = () => {
789
    if (this._state === "closed") {
790
      return;
791
    }
792
    this.syncdb.commit();
793
  };
794

795
  public save = async (): Promise<void> => {
796
    if (this.store.get("read_only") || this.isDeleted()) {
797
      // can't save when readonly or deleted
798
      return;
799
    }
800
    // Save the .ipynb file to disk.  Note that this
801
    // *changes* the syncdb by updating the last save time.
802
    try {
803
      // Make sure syncdb content is all sent to the project.
804
      // This does not actually save the syncdb file to disk.
805
      // This "save" means save state to backend.
806
      // We save two things -- first the syncdb state:
807
      await this.syncdb.save();
808
      if (this._state === "closed") return;
809

810
      // Second the .ipynb file state:
811
      // Export the ipynb file to disk, being careful not to actually
812
      // save it until the backend actually gets the given version and
813
      // has processed it!
814
      const version = this.syncdb.newestVersion();
815
      try {
816
        await this.api({ timeout: 5 * 60 * 1000 }).save_ipynb_file({ version });
817
      } catch (err) {
818
        console.log(`WARNING: ${err}`);
819
        throw Error(
820
          `There was a problem writing the ipynb file to disk -- ${err}`,
821
        );
822
      }
823
      if (this._state === ("closed" as State)) return;
824
      // Save our custom-format syncdb to disk.
825
      await this.syncdb.save_to_disk();
826
    } catch (err) {
827
      if (this._state === ("closed" as State)) return;
828
      if (err.toString().indexOf("no kernel with path") != -1) {
829
        // This means that the kernel simply hasn't been initialized yet.
830
        // User can try to save later, once it has.
831
        return;
832
      }
833
      if (err.toString().indexOf("unknown endpoint") != -1) {
834
        this.set_error(
835
          "You MUST restart your project to run the latest Jupyter server! Click 'Restart Project' in your project's settings.",
836
        );
837
        return;
838
      }
839
      this.set_error(err.toString());
840
    } finally {
841
      if (this._state === "closed") return;
842
      // And update the save status finally.
843
      if (typeof this.set_save_status === "function") {
844
        this.set_save_status();
845
      }
846
    }
847
  };
848

849
  save_asap = async (): Promise<void> => {
850
    if (this.syncdb != null) {
851
      await this.syncdb.save();
852
    }
853
  };
854

855
  private id_is_available(id: string): boolean {
856
    return this.store.getIn(["cells", id]) == null;
857
  }
858

859
  protected new_id(is_available?: (string) => boolean): string {
860
    while (true) {
861
      const id = misc.uuid().slice(0, 6);
862
      if (
863
        (is_available != null && is_available(id)) ||
864
        this.id_is_available(id)
865
      ) {
866
        return id;
867
      }
868
    }
869
  }
870

871
  insert_cell(delta: any): string {
872
    this.deprecated("insert-cell", delta);
873
    return "";
874
  }
875

876
  insert_cell_at(
877
    pos: number,
878
    save: boolean = true,
879
    id: string | undefined = undefined, // dangerous since could conflict (used by whiteboard)
880
  ): string {
881
    if (this.store.get("read_only")) {
882
      throw Error("document is read only");
883
    }
884
    const new_id = id ?? this.new_id();
885
    this._set(
886
      {
887
        type: "cell",
888
        id: new_id,
889
        pos,
890
        input: "",
891
      },
892
      save,
893
    );
894
    return new_id; // violates CQRS... (this *is* used elsewhere)
895
  }
896

897
  // insert a cell adjacent to the cell with given id.
898
  // -1 = above and +1 = below.
899
  insert_cell_adjacent(
900
    id: string,
901
    delta: -1 | 1,
902
    save: boolean = true,
903
  ): string {
904
    const pos = cell_utils.new_cell_pos(
905
      this.store.get("cells"),
906
      this.store.get_cell_list(),
907
      id,
908
      delta,
909
    );
910
    return this.insert_cell_at(pos, save);
911
  }
912

913
  delete_selected_cells = (sync = true): void => {
914
    this.deprecated("delete_selected_cells", sync);
915
  };
916

917
  delete_cells(cells: string[], sync: boolean = true): void {
918
    let not_deletable: number = 0;
919
    for (const id of cells) {
920
      if (this.store.is_cell_deletable(id)) {
921
        this._delete({ type: "cell", id }, false);
922
      } else {
923
        not_deletable += 1;
924
      }
925
    }
926
    if (sync) {
927
      this._sync();
928
    }
929
    if (not_deletable === 0) return;
930

931
    this.show_not_deletable_error(not_deletable);
932
  }
933

934
  // Delete all blank code cells in the entire notebook.
935
  delete_all_blank_code_cells(sync: boolean = true): void {
936
    const cells: string[] = [];
937
    for (const id of this.store.get_cell_list()) {
938
      if (!this.store.is_cell_deletable(id)) {
939
        continue;
940
      }
941
      const cell = this.store.getIn(["cells", id]);
942
      if (cell == null) continue;
943
      if (
944
        cell.get("cell_type", "code") == "code" &&
945
        cell.get("input", "").trim() == "" &&
946
        cell.get("output", []).length == 0
947
      ) {
948
        cells.push(id);
949
      }
950
    }
951
    this.delete_cells(cells, sync);
952
  }
953

954
  move_selected_cells = (delta: number) => {
955
    this.deprecated("move_selected_cells", delta);
956
  };
957

958
  undo = (): void => {
959
    if (this.syncdb != null) {
960
      this.syncdb.undo();
961
    }
962
  };
963

964
  redo = (): void => {
965
    if (this.syncdb != null) {
966
      this.syncdb.redo();
967
    }
968
  };
969

970
  in_undo_mode(): boolean {
971
    return this.syncdb?.in_undo_mode() ?? false;
972
  }
973

974
  public run_code_cell(
975
    id: string,
976
    save: boolean = true,
977
    no_halt: boolean = false,
978
  ): void {
979
    const cell = this.store.getIn(["cells", id]);
980
    if (cell == null) {
981
      // it is trivial to run a cell that does not exist -- nothing needs to be done.
982
      return;
983
    }
984
    const kernel = this.store.get("kernel");
985
    if (kernel == null || kernel === "") {
986
      // just in case, we clear any "running" indicators
987
      this._set({ type: "cell", id, state: "done" });
988
      // don't attempt to run a code-cell if there is no kernel defined
989
      this.set_error(
990
        "No kernel set for running cells. Therefore it is not possible to run a code cell. You have to select a kernel!",
991
      );
992
      return;
993
    }
994

995
    if (cell.get("state", "done") != "done") {
996
      // already running -- stop it first somehow if you want to run it again...
997
      return;
998
    }
999

1000
    // We mark the start timestamp uniquely, so that the backend can sort
1001
    // multiple cells with a simultaneous time to start request.
1002

1003
    let start: number = this._client.server_time().valueOf();
1004
    if (this._last_start != null && start <= this._last_start) {
1005
      start = this._last_start + 1;
1006
    }
1007
    this._last_start = start;
1008
    this.set_jupyter_metadata(id, "outputs_hidden", undefined, false);
1009

1010
    this._set(
1011
      {
1012
        type: "cell",
1013
        id,
1014
        state: "start",
1015
        start,
1016
        end: null,
1017
        // time last evaluation took
1018
        last:
1019
          cell.get("start") != null && cell.get("end") != null
1020
            ? cell.get("end") - cell.get("start")
1021
            : cell.get("last"),
1022
        output: null,
1023
        exec_count: null,
1024
        collapsed: null,
1025
        no_halt: no_halt ? no_halt : null,
1026
      },
1027
      save,
1028
    );
1029
    this.set_trust_notebook(true, save);
1030
  }
1031

1032
  clear_cell = (id: string, save = true) => {
1033
    const cell = this.store.getIn(["cells", id]);
1034

1035
    this._set(
1036
      {
1037
        type: "cell",
1038
        id,
1039
        state: null,
1040
        start: null,
1041
        end: null,
1042
        last:
1043
          cell?.get("start") != null && cell?.get("end") != null
1044
            ? cell?.get("end") - cell?.get("start")
1045
            : (cell?.get("last") ?? null),
1046
        output: null,
1047
        exec_count: null,
1048
        collapsed: null,
1049
      },
1050
      save,
1051
    );
1052
  };
1053

1054
  run_selected_cells = (): void => {
1055
    this.deprecated("run_selected_cells");
1056
  };
1057

1058
  public abstract run_cell(id: string, save?: boolean, no_halt?: boolean): void;
1059

1060
  run_all_cells = (no_halt: boolean = false): void => {
1061
    this.store.get_cell_list().forEach((id) => {
1062
      this.run_cell(id, false, no_halt);
1063
    });
1064
    this.save_asap();
1065
  };
1066

1067
  clear_all_cell_run_state = (): void => {
1068
    const { store } = this;
1069
    if (!store) {
1070
      return;
1071
    }
1072
    const cells = store.get("cells");
1073
    for (const id of store.get_cell_list()) {
1074
      const state = cells.getIn([id, "state"]);
1075
      if (state && state != "done") {
1076
        this._set(
1077
          {
1078
            type: "cell",
1079
            id,
1080
            state: "done",
1081
          },
1082
          false,
1083
        );
1084
      }
1085
    }
1086
    this.save_asap();
1087
  };
1088

1089
  // Run all cells strictly above the specified cell.
1090
  run_all_above_cell(id: string): void {
1091
    const i: number = this.store.get_cell_index(id);
1092
    const v: string[] = this.store.get_cell_list().toJS();
1093
    for (const id of v.slice(0, i)) {
1094
      this.run_cell(id, false);
1095
    }
1096
    this.save_asap();
1097
  }
1098

1099
  // Run all cells below (and *including*) the specified cell.
1100
  public run_all_below_cell(id: string): void {
1101
    const i: number = this.store.get_cell_index(id);
1102
    const v: string[] = this.store.get_cell_list().toJS();
1103
    for (const id of v.slice(i)) {
1104
      this.run_cell(id, false);
1105
    }
1106
    this.save_asap();
1107
  }
1108

1109
  public set_cursor_locs(locs: any[] = [], side_effect: boolean = false): void {
1110
    this.last_cursor_move_time = new Date();
1111
    if (this.syncdb == null) {
1112
      // syncdb not always set -- https://github.com/sagemathinc/cocalc/issues/2107
1113
      return;
1114
    }
1115
    if (locs.length === 0) {
1116
      // don't remove on blur -- cursor will fade out just fine
1117
      return;
1118
    }
1119
    this._cursor_locs = locs; // remember our own cursors for splitting cell
1120
    this.syncdb.set_cursor_locs(locs, side_effect);
1121
  }
1122

1123
  public split_cell(id: string, cursor: { line: number; ch: number }): void {
1124
    if (this.check_edit_protection(id, "splitting cell")) {
1125
      return;
1126
    }
1127
    // insert a new cell before the currently selected one
1128
    const new_id: string = this.insert_cell_adjacent(id, -1, false);
1129

1130
    // split the cell content at the cursor loc
1131
    const cell = this.store.get("cells").get(id);
1132
    if (cell == null) {
1133
      throw Error(`no cell with id=${id}`);
1134
    }
1135
    const cell_type = cell.get("cell_type");
1136
    if (cell_type !== "code") {
1137
      this.set_cell_type(new_id, cell_type, false);
1138
    }
1139
    const input = cell.get("input");
1140
    if (input == null) {
1141
      this.syncdb.commit();
1142
      return; // very easy case.
1143
    }
1144

1145
    const lines = input.split("\n");
1146
    let v = lines.slice(0, cursor.line);
1147
    const line: string | undefined = lines[cursor.line];
1148
    if (line != null) {
1149
      const left = line.slice(0, cursor.ch);
1150
      if (left) {
1151
        v.push(left);
1152
      }
1153
    }
1154
    const top = v.join("\n");
1155

1156
    v = lines.slice(cursor.line + 1);
1157
    if (line != null) {
1158
      const right = line.slice(cursor.ch);
1159
      if (right) {
1160
        v = [right].concat(v);
1161
      }
1162
    }
1163
    const bottom = v.join("\n");
1164
    this.set_cell_input(new_id, top, false);
1165
    this.set_cell_input(id, bottom, true);
1166
  }
1167

1168
  // Copy content from the cell below the given cell into the currently
1169
  // selected cell, then delete the cell below the given cell.
1170
  public merge_cell_below_cell(cell_id: string, save: boolean = true): void {
1171
    const next_id = this.store.get_cell_id(1, cell_id);
1172
    if (next_id == null) {
1173
      // no cell below given cell, so trivial.
1174
      return;
1175
    }
1176
    for (const id of [cell_id, next_id]) {
1177
      if (this.check_edit_protection(id, "merging cell")) return;
1178
    }
1179
    if (this.check_delete_protection(next_id)) return;
1180

1181
    const cells = this.store.get("cells");
1182
    if (cells == null) {
1183
      return;
1184
    }
1185

1186
    const input: string =
1187
      cells.getIn([cell_id, "input"], "") +
1188
      "\n" +
1189
      cells.getIn([next_id, "input"], "");
1190

1191
    const output0 = cells.getIn([cell_id, "output"]) as any;
1192
    const output1 = cells.getIn([next_id, "output"]) as any;
1193
    let output: any = undefined;
1194
    if (output0 == null) {
1195
      output = output1;
1196
    } else if (output1 == null) {
1197
      output = output0;
1198
    } else {
1199
      // both output0 and output1 are defined; need to merge.
1200
      // This is complicated since output is a map from string numbers.
1201
      output = output0;
1202
      let n = output0.size;
1203
      for (let i = 0; i < output1.size; i++) {
1204
        output = output.set(`${n}`, output1.get(`${i}`));
1205
        n += 1;
1206
      }
1207
    }
1208

1209
    this._delete({ type: "cell", id: next_id }, false);
1210
    this._set(
1211
      {
1212
        type: "cell",
1213
        id: cell_id,
1214
        input,
1215
        output: output != null ? output : null,
1216
        start: null,
1217
        end: null,
1218
      },
1219
      save,
1220
    );
1221
  }
1222

1223
  // Merge the given cells into one cell, which replaces
1224
  // the frist cell in cell_ids.
1225
  // We also merge all output, instead of throwing away
1226
  // all but first output (which jupyter does, and makes no sense).
1227
  public merge_cells(cell_ids: string[]): void {
1228
    const n = cell_ids.length;
1229
    if (n <= 1) return; // trivial special case.
1230
    for (let i = 0; i < n - 1; i++) {
1231
      this.merge_cell_below_cell(cell_ids[0], i == n - 2);
1232
    }
1233
  }
1234

1235
  // Copy the list of cells into our internal clipboard
1236
  public copy_cells(cell_ids: string[]): void {
1237
    const cells = this.store.get("cells");
1238
    let global_clipboard = immutable.List();
1239
    for (const id of cell_ids) {
1240
      global_clipboard = global_clipboard.push(cells.get(id));
1241
    }
1242
    this.store.set_global_clipboard(global_clipboard);
1243
  }
1244

1245
  public studentProjectFunctionality() {
1246
    return this.redux
1247
      .getStore("projects")
1248
      .get_student_project_functionality(this.project_id);
1249
  }
1250

1251
  public requireToggleReadonly(): void {
1252
    if (this.studentProjectFunctionality().disableJupyterToggleReadonly) {
1253
      throw Error("Toggling of write protection is disabled in this project.");
1254
    }
1255
  }
1256

1257
  /* write protection disables any modifications, entering "edit"
1258
     mode, and prohibits cell evaluations example: teacher handout
1259
     notebook and student should not be able to modify an
1260
     instruction cell in any way. */
1261
  public toggle_write_protection_on_cells(
1262
    cell_ids: string[],
1263
    save: boolean = true,
1264
  ): void {
1265
    this.requireToggleReadonly();
1266
    this.toggle_metadata_boolean_on_cells(cell_ids, "editable", true, save);
1267
  }
1268

1269
  set_metadata_on_cells = (
1270
    cell_ids: string[],
1271
    key: string,
1272
    value,
1273
    save: boolean = true,
1274
  ) => {
1275
    for (const id of cell_ids) {
1276
      this.set_cell_metadata({
1277
        id,
1278
        metadata: { [key]: value },
1279
        merge: true,
1280
        save: false,
1281
        bypass_edit_protection: true,
1282
      });
1283
    }
1284
    if (save) {
1285
      this.save_asap();
1286
    }
1287
  };
1288

1289
  public write_protect_cells(
1290
    cell_ids: string[],
1291
    protect: boolean,
1292
    save: boolean = true,
1293
  ) {
1294
    this.set_metadata_on_cells(cell_ids, "editable", !protect, save);
1295
  }
1296

1297
  public delete_protect_cells(
1298
    cell_ids: string[],
1299
    protect: boolean,
1300
    save: boolean = true,
1301
  ) {
1302
    this.set_metadata_on_cells(cell_ids, "deletable", !protect, save);
1303
  }
1304

1305
  // this prevents any cell from being deleted, either directly, or indirectly via a "merge"
1306
  // example: teacher handout notebook and student should not be able to modify an instruction cell in any way
1307
  public toggle_delete_protection_on_cells(
1308
    cell_ids: string[],
1309
    save: boolean = true,
1310
  ): void {
1311
    this.requireToggleReadonly();
1312
    this.toggle_metadata_boolean_on_cells(cell_ids, "deletable", true, save);
1313
  }
1314

1315
  // This toggles the boolean value of given metadata field.
1316
  // If not set, it is assumed to be true and toggled to false
1317
  // For more than one cell, the first one is used to toggle
1318
  // all cells to the inverted state
1319
  private toggle_metadata_boolean_on_cells(
1320
    cell_ids: string[],
1321
    key: string,
1322
    default_value: boolean, // default metadata value, if the metadata field is not set.
1323
    save: boolean = true,
1324
  ): void {
1325
    for (const id of cell_ids) {
1326
      this.set_cell_metadata({
1327
        id,
1328
        metadata: {
1329
          [key]: !this.store.getIn(
1330
            ["cells", id, "metadata", key],
1331
            default_value,
1332
          ),
1333
        },
1334
        merge: true,
1335
        save: false,
1336
        bypass_edit_protection: true,
1337
      });
1338
    }
1339
    if (save) {
1340
      this.save_asap();
1341
    }
1342
  }
1343

1344
  public toggle_jupyter_metadata_boolean(
1345
    id: string,
1346
    key: string,
1347
    save: boolean = true,
1348
  ): void {
1349
    const jupyter = this.store
1350
      .getIn(["cells", id, "metadata", "jupyter"], immutable.Map())
1351
      .toJS();
1352
    jupyter[key] = !jupyter[key];
1353
    this.set_cell_metadata({
1354
      id,
1355
      metadata: { jupyter },
1356
      merge: true,
1357
      save,
1358
    });
1359
  }
1360

1361
  public set_jupyter_metadata(
1362
    id: string,
1363
    key: string,
1364
    value: any,
1365
    save: boolean = true,
1366
  ): void {
1367
    const jupyter = this.store
1368
      .getIn(["cells", id, "metadata", "jupyter"], immutable.Map())
1369
      .toJS();
1370
    if (value == null && jupyter[key] == null) return; // nothing to do.
1371
    if (value != null) {
1372
      jupyter[key] = value;
1373
    } else {
1374
      delete jupyter[key];
1375
    }
1376
    this.set_cell_metadata({
1377
      id,
1378
      metadata: { jupyter },
1379
      merge: true,
1380
      save,
1381
    });
1382
  }
1383

1384
  // Paste cells from the internal clipboard; also
1385
  //   delta = 0 -- replace cell_ids cells
1386
  //   delta = 1 -- paste cells below last cell in cell_ids
1387
  //   delta = -1 -- paste cells above first cell in cell_ids.
1388
  public paste_cells_at(cell_ids: string[], delta: 0 | 1 | -1 = 1): void {
1389
    const clipboard = this.store.get_global_clipboard();
1390
    if (clipboard == null || clipboard.size === 0) {
1391
      return; // nothing to do
1392
    }
1393

1394
    if (cell_ids.length === 0) {
1395
      // There are no cells currently selected.  This can
1396
      // happen in an edge case with slow network -- see
1397
      // https://github.com/sagemathinc/cocalc/issues/3899
1398
      clipboard.forEach((cell, i) => {
1399
        cell = cell.set("id", this.new_id()); // randomize the id of the cell
1400
        cell = cell.set("pos", i);
1401
        this._set(cell, false);
1402
      });
1403
      this.ensure_positions_are_unique();
1404
      this._sync();
1405
      return;
1406
    }
1407

1408
    let cell_before_pasted_id: string;
1409
    const cells = this.store.get("cells");
1410
    if (delta === -1 || delta === 0) {
1411
      // one before first selected
1412
      cell_before_pasted_id = this.store.get_cell_id(-1, cell_ids[0]) ?? "";
1413
    } else if (delta === 1) {
1414
      // last selected
1415
      cell_before_pasted_id = cell_ids[cell_ids.length - 1];
1416
    } else {
1417
      // Typescript should prevent this, but just to be sure.
1418
      throw Error(`delta (=${delta}) must be 0, -1, or 1`);
1419
    }
1420
    try {
1421
      let after_pos: number, before_pos: number | undefined;
1422
      if (delta === 0) {
1423
        // replace, so delete cell_ids, unless just one, since
1424
        // cursor cell_ids selection is confusing with Jupyter's model.
1425
        if (cell_ids.length > 1) {
1426
          this.delete_cells(cell_ids, false);
1427
        }
1428
      }
1429
      // put the cells from the clipboard into the document, setting their positions
1430
      if (cell_before_pasted_id == null) {
1431
        // very top cell
1432
        before_pos = undefined;
1433
        after_pos = cells.getIn([cell_ids[0], "pos"]) as number;
1434
      } else {
1435
        before_pos = cells.getIn([cell_before_pasted_id, "pos"]) as
1436
          | number
1437
          | undefined;
1438
        after_pos = cells.getIn([
1439
          this.store.get_cell_id(+1, cell_before_pasted_id),
1440
          "pos",
1441
        ]) as number;
1442
      }
1443
      const positions = cell_utils.positions_between(
1444
        before_pos,
1445
        after_pos,
1446
        clipboard.size,
1447
      );
1448
      clipboard.forEach((cell, i) => {
1449
        cell = cell.set("id", this.new_id()); // randomize the id of the cell
1450
        cell = cell.set("pos", positions[i]);
1451
        this._set(cell, false);
1452
      });
1453
    } finally {
1454
      // very important that we save whatever is done above, so other viewers see it.
1455
      this._sync();
1456
    }
1457
  }
1458

1459
  // File --> Open: just show the file listing page.
1460
  file_open = (): void => {
1461
    if (this.redux == null) return;
1462
    this.redux
1463
      .getProjectActions(this.store.get("project_id"))
1464
      .set_active_tab("files");
1465
  };
1466

1467
  // File --> New: like open, but also show the create panel
1468
  file_new = (): void => {
1469
    if (this.redux == null) return;
1470
    const project_actions = this.redux.getProjectActions(
1471
      this.store.get("project_id"),
1472
    );
1473
    project_actions.set_active_tab("new");
1474
  };
1475

1476
  private _get_cell_input = (id?: string | undefined): string => {
1477
    this.deprecated("_get_cell_input", id);
1478
    return "";
1479
  };
1480

1481
  // Version of the cell's input stored in store.
1482
  // (A live codemirror editor could have a slightly
1483
  // newer version, so this is only a fallback).
1484
  get_cell_input(id: string): string {
1485
    return this.store.getIn(["cells", id, "input"], "");
1486
  }
1487

1488
  // Attempt to fetch completions for give code and cursor_pos
1489
  // If successful, the completions are put in store.get('completions') and looks like
1490
  // this (as an immutable map):
1491
  //    cursor_end   : 2
1492
  //    cursor_start : 0
1493
  //    matches      : ['the', 'completions', ...]
1494
  //    status       : "ok"
1495
  //    code         : code
1496
  //    cursor_pos   : cursor_pos
1497
  //
1498
  // If not successful, result is:
1499
  //    status       : "error"
1500
  //    code         : code
1501
  //    cursor_pos   : cursor_pos
1502
  //    error        : 'an error message'
1503
  //
1504
  // Only the most recent fetch has any impact, and calling
1505
  // clear_complete() ensures any fetch made before that
1506
  // is ignored.
1507

1508
  // Returns true if a dialog with options appears, and false otherwise.
1509
  public async complete(
1510
    code: string,
1511
    pos?: { line: number; ch: number } | number,
1512
    id?: string,
1513
    offset?: any,
1514
  ): Promise<boolean> {
1515
    let cursor_pos;
1516
    const req = (this._complete_request =
1517
      (this._complete_request != null ? this._complete_request : 0) + 1);
1518

1519
    this.setState({ complete: undefined });
1520

1521
    // pos can be either a {line:?, ch:?} object as in codemirror,
1522
    // or a number.
1523
    if (pos == null || typeof pos == "number") {
1524
      cursor_pos = pos;
1525
    } else {
1526
      cursor_pos = codemirror_to_jupyter_pos(code, pos);
1527
    }
1528
    cursor_pos = js_idx_to_char_idx(cursor_pos, code);
1529

1530
    const start = new Date();
1531
    let complete;
1532
    try {
1533
      complete = await this.api().complete({
1534
        code,
1535
        cursor_pos,
1536
      });
1537
    } catch (err) {
1538
      if (this._complete_request > req) return false;
1539
      this.setState({ complete: { error: err } });
1540
      // no op for now...
1541
      throw Error(`ignore -- ${err}`);
1542
      //return false;
1543
    }
1544

1545
    if (this.last_cursor_move_time >= start) {
1546
      // see https://github.com/sagemathinc/cocalc/issues/3611
1547
      throw Error("ignore");
1548
      //return false;
1549
    }
1550
    if (this._complete_request > req) {
1551
      // future completion or clear happened; so ignore this result.
1552
      throw Error("ignore");
1553
      //return false;
1554
    }
1555

1556
    if (complete.status !== "ok") {
1557
      this.setState({
1558
        complete: {
1559
          error: complete.error ? complete.error : "completion failed",
1560
        },
1561
      });
1562
      return false;
1563
    }
1564

1565
    if (complete.matches == 0) {
1566
      return false;
1567
    }
1568

1569
    delete complete.status;
1570
    complete.base = code;
1571
    complete.code = code;
1572
    complete.pos = char_idx_to_js_idx(cursor_pos, code);
1573
    complete.cursor_start = char_idx_to_js_idx(complete.cursor_start, code);
1574
    complete.cursor_end = char_idx_to_js_idx(complete.cursor_end, code);
1575
    complete.id = id;
1576
    // Set the result so the UI can then react to the change.
1577
    if (offset != null) {
1578
      complete.offset = offset;
1579
    }
1580
    // For some reason, sometimes complete.matches are not unique, which is annoying/confusing,
1581
    // and breaks an assumption in our react code too.
1582
    // I think the reason is e.g., a filename and a variable could be the same.   We're not
1583
    // worrying about that now.
1584
    complete.matches = Array.from(new Set(complete.matches));
1585
    // sort in a way that matches how JupyterLab sorts completions, which
1586
    // is case insensitive with % magics at the bottom
1587
    complete.matches.sort((x, y) => {
1588
      const c = misc.cmp(getCompletionGroup(x), getCompletionGroup(y));
1589
      if (c) {
1590
        return c;
1591
      }
1592
      return misc.cmp(x.toLowerCase(), y.toLowerCase());
1593
    });
1594
    const i_complete = immutable.fromJS(complete);
1595
    if (complete.matches && complete.matches.length === 1 && id != null) {
1596
      // special case -- a unique completion and we know id of cell in which completing is given.
1597
      this.select_complete(id, complete.matches[0], i_complete);
1598
      return false;
1599
    } else {
1600
      this.setState({ complete: i_complete });
1601
      return true;
1602
    }
1603
  }
1604

1605
  clear_complete = (): void => {
1606
    this._complete_request =
1607
      (this._complete_request != null ? this._complete_request : 0) + 1;
1608
    this.setState({ complete: undefined });
1609
  };
1610

1611
  public select_complete(
1612
    id: string,
1613
    item: string,
1614
    complete?: immutable.Map<string, any>,
1615
  ): void {
1616
    if (complete == null) {
1617
      complete = this.store.get("complete");
1618
    }
1619
    this.clear_complete();
1620
    if (complete == null) {
1621
      return;
1622
    }
1623
    const input = complete.get("code");
1624
    if (input != null && complete.get("error") == null) {
1625
      const starting = input.slice(0, complete.get("cursor_start"));
1626
      const ending = input.slice(complete.get("cursor_end"));
1627
      const new_input = starting + item + ending;
1628
      const base = complete.get("base");
1629
      this.complete_cell(id, base, new_input);
1630
    }
1631
  }
1632

1633
  complete_cell(id: string, base: string, new_input: string): void {
1634
    this.merge_cell_input(id, base, new_input);
1635
  }
1636

1637
  merge_cell_input(
1638
    id: string,
1639
    base: string,
1640
    input: string,
1641
    save: boolean = true,
1642
  ): void {
1643
    const remote = this.store.getIn(["cells", id, "input"]);
1644
    if (remote == null || base == null || input == null) {
1645
      return;
1646
    }
1647
    const new_input = three_way_merge({
1648
      base,
1649
      local: input,
1650
      remote,
1651
    });
1652
    this.set_cell_input(id, new_input, save);
1653
  }
1654

1655
  is_introspecting(): boolean {
1656
    const actions = this.getFrameActions() as any;
1657
    return actions?.store?.get("introspect") != null;
1658
  }
1659

1660
  introspect_close = () => {
1661
    if (this.is_introspecting()) {
1662
      this.getFrameActions()?.setState({ introspect: undefined });
1663
    }
1664
  };
1665

1666
  introspect_at_pos = async (
1667
    code: string,
1668
    level: 0 | 1 = 0,
1669
    pos: { ch: number; line: number },
1670
  ): Promise<void> => {
1671
    if (code === "") return; // no-op if there is no code (should never happen)
1672
    await this.introspect(code, level, codemirror_to_jupyter_pos(code, pos));
1673
  };
1674

1675
  introspect = async (
1676
    code: string,
1677
    level: 0 | 1,
1678
    cursor_pos?: number,
1679
  ): Promise<immutable.Map<string, any> | undefined> => {
1680
    const req = (this._introspect_request =
1681
      (this._introspect_request != null ? this._introspect_request : 0) + 1);
1682

1683
    if (cursor_pos == null) {
1684
      cursor_pos = code.length;
1685
    }
1686
    cursor_pos = js_idx_to_char_idx(cursor_pos, code);
1687

1688
    let introspect;
1689
    try {
1690
      introspect = await this.api().introspect({
1691
        code,
1692
        cursor_pos,
1693
        level,
1694
      });
1695
      if (introspect.status !== "ok") {
1696
        introspect = { error: "completion failed" };
1697
      }
1698
      delete introspect.status;
1699
    } catch (err) {
1700
      introspect = { error: err };
1701
    }
1702
    if (this._introspect_request > req) return;
1703
    const i = immutable.fromJS(introspect);
1704
    this.getFrameActions()?.setState({
1705
      introspect: i,
1706
    });
1707
    return i; // convenient / useful, e.g., for use by whiteboard.
1708
  };
1709

1710
  clear_introspect = (): void => {
1711
    this._introspect_request =
1712
      (this._introspect_request != null ? this._introspect_request : 0) + 1;
1713
    this.getFrameActions()?.setState({ introspect: undefined });
1714
  };
1715

1716
  public async signal(signal = "SIGINT"): Promise<void> {
1717
    const api = this.api({ timeout: 5000 });
1718
    try {
1719
      await api.signal(signal);
1720
    } catch (err) {
1721
      this.set_error(err);
1722
    }
1723
  }
1724

1725
  // Kill the running kernel and does NOT start it up again.
1726
  halt = reuseInFlight(async (): Promise<void> => {
1727
    if (this.restartKernelOnClose != null && this.jupyter_kernel != null) {
1728
      this.jupyter_kernel.removeListener("closed", this.restartKernelOnClose);
1729
      delete this.restartKernelOnClose;
1730
    }
1731
    this.clear_all_cell_run_state();
1732
    await this.signal("SIGKILL");
1733
    // Wait a little, since SIGKILL has to really happen on backend,
1734
    // and server has to respond and change state.
1735
    const not_running = (s): boolean => {
1736
      if (this._state === "closed") return true;
1737
      const t = s.get_one({ type: "settings" });
1738
      return t != null && t.get("backend_state") != "running";
1739
    };
1740
    try {
1741
      await this.syncdb.wait(not_running, 30);
1742
      // worked -- and also no need to show "kernel got killed" message since this was intentional.
1743
      this.set_error("");
1744
    } catch (err) {
1745
      // failed
1746
      this.set_error(err);
1747
    }
1748
  });
1749

1750
  restart = reuseInFlight(async (): Promise<void> => {
1751
    await this.halt();
1752
    if (this._state === "closed") return;
1753
    this.clear_all_cell_run_state();
1754
    // Actually start it running again (rather than waiting for
1755
    // user to do something), since this is called "restart".
1756
    try {
1757
      await this.set_backend_kernel_info(); // causes kernel to start
1758
    } catch (err) {
1759
      this.set_error(err);
1760
    }
1761
  });
1762

1763
  public shutdown = reuseInFlight(async (): Promise<void> => {
1764
    if (this._state === ("closed" as State)) {
1765
      return;
1766
    }
1767
    await this.signal("SIGKILL");
1768
    if (this._state === ("closed" as State)) {
1769
      return;
1770
    }
1771
    this.clear_all_cell_run_state();
1772
    await this.save_asap();
1773
  });
1774

1775
  set_backend_kernel_info = async (): Promise<void> => {
1776
    if (this._state === "closed" || this.syncdb.is_read_only()) {
1777
      return;
1778
    }
1779

1780
    if (this.is_project || this.is_compute_server) {
1781
      const dbg = this.dbg(`set_backend_kernel_info ${misc.uuid()}`);
1782
      if (
1783
        this.jupyter_kernel == null ||
1784
        this.jupyter_kernel.get_state() == "closed"
1785
      ) {
1786
        dbg("no Jupyter kernel defined");
1787
        return;
1788
      }
1789
      dbg("getting kernel_info...");
1790
      let backend_kernel_info: KernelInfo;
1791
      try {
1792
        backend_kernel_info = immutable.fromJS(
1793
          await this.jupyter_kernel.kernel_info(),
1794
        );
1795
      } catch (err) {
1796
        dbg(`error = ${err}`);
1797
        return;
1798
      }
1799
      this.setState({ backend_kernel_info });
1800
    } else {
1801
      await this._set_backend_kernel_info_client();
1802
    }
1803
  };
1804

1805
  _set_backend_kernel_info_client = reuseInFlight(async (): Promise<void> => {
1806
    await retry_until_success({
1807
      max_time: 120000,
1808
      start_delay: 1000,
1809
      max_delay: 10000,
1810
      f: this._fetch_backend_kernel_info_from_server,
1811
      desc: "jupyter:_set_backend_kernel_info_client",
1812
    });
1813
  });
1814

1815
  _fetch_backend_kernel_info_from_server = async (): Promise<void> => {
1816
    const f = async () => {
1817
      if (this._state === "closed") {
1818
        return;
1819
      }
1820
      const data = await this.api().kernel_info();
1821
      this.setState({
1822
        backend_kernel_info: immutable.fromJS(data),
1823
        // this is when the server for this doc started, not when kernel last started!
1824
        start_time: data.start_time,
1825
      });
1826
    };
1827
    try {
1828
      await retry_until_success({
1829
        max_time: 1000 * 60 * 30,
1830
        start_delay: 500,
1831
        max_delay: 3000,
1832
        f,
1833
        desc: "jupyter:_fetch_backend_kernel_info_from_server",
1834
      });
1835
    } catch (err) {
1836
      this.set_error(err);
1837
    }
1838
    if (this.is_closed()) return;
1839
    // Update the codemirror editor options.
1840
    this.set_cm_options();
1841
  };
1842

1843
  // Do a file action, e.g., 'compress', 'delete', 'rename', 'duplicate', 'move',
1844
  // 'copy', 'share', 'download', 'open_file', 'close_file', 'reopen_file'
1845
  // Each just shows
1846
  // the corresponding dialog in
1847
  // the file manager, so gives a step to confirm, etc.
1848
  // The path may optionally be *any* file in this project.
1849
  public async file_action(action_name: string, path?: string): Promise<void> {
1850
    if (this._state == "closed") return;
1851
    const a = this.redux.getProjectActions(this.project_id);
1852
    if (path == null) {
1853
      path = this.store.get("path");
1854
      if (path == null) {
1855
        throw Error("path must be defined in the store to use default");
1856
      }
1857
    }
1858
    if (action_name === "reopen_file") {
1859
      a.close_file(path);
1860
      // ensure the side effects from changing registered
1861
      // editors in project_file.* finish happening
1862
      await delay(0);
1863
      a.open_file({ path });
1864
      return;
1865
    }
1866
    if (action_name === "close_file") {
1867
      await this.syncdb.save();
1868
      a.close_file(path);
1869
      return;
1870
    }
1871
    if (action_name === "open_file") {
1872
      a.open_file({ path });
1873
      return;
1874
    }
1875
    if (action_name == "download") {
1876
      a.download_file({ path });
1877
      return;
1878
    }
1879
    const { head, tail } = misc.path_split(path);
1880
    a.open_directory(head);
1881
    a.set_all_files_unchecked();
1882
    a.set_file_checked(path, true);
1883
    return a.set_file_action(action_name, () => tail);
1884
  }
1885

1886
  set_max_output_length = (n) => {
1887
    return this._set({
1888
      type: "settings",
1889
      max_output_length: n,
1890
    });
1891
  };
1892

1893
  fetch_more_output = async (id: string): Promise<void> => {
1894
    const time = this._client.server_time().valueOf();
1895
    try {
1896
      const more_output = await this.api({ timeout: 60000 }).more_output(id);
1897
      if (!this.store.getIn(["cells", id, "scrolled"])) {
1898
        // make output area scrolled, since there is going to be a lot of output
1899
        this.toggle_output(id, "scrolled");
1900
      }
1901
      this.set_more_output(id, { time, mesg_list: more_output });
1902
    } catch (err) {
1903
      this.set_error(err);
1904
    }
1905
  };
1906

1907
  // NOTE: set_more_output on project-actions is different
1908
  set_more_output = (id: string, more_output: any, _?: any): void => {
1909
    if (this.store.getIn(["cells", id]) == null) {
1910
      return;
1911
    }
1912
    const x = this.store.get("more_output", immutable.Map());
1913
    this.setState({
1914
      more_output: x.set(id, immutable.fromJS(more_output)),
1915
    });
1916
  };
1917

1918
  reset_more_output = (id?: any): void => {
1919
    let left: any;
1920
    const more_output =
1921
      (left = this.store.get("more_output")) != null ? left : immutable.Map();
1922
    if (more_output.has(id)) {
1923
      this.setState({ more_output: more_output.delete(id) });
1924
    }
1925
  };
1926

1927
  protected set_cm_options(): void {
1928
    // this only does something in browser-actions.
1929
  }
1930

1931
  set_trust_notebook = (trust: any, save: boolean = true) => {
1932
    return this._set(
1933
      {
1934
        type: "settings",
1935
        trust: !!trust,
1936
      },
1937
      save,
1938
    ); // case to bool
1939
  };
1940

1941
  scroll(pos): any {
1942
    this.deprecated("scroll", pos);
1943
  }
1944

1945
  // submit input for a particular cell -- this is used by the
1946
  // Input component output message type for interactive input.
1947
  public async submit_input(id: string, value: string): Promise<void> {
1948
    const output = this.store.getIn(["cells", id, "output"]);
1949
    if (output == null) {
1950
      return;
1951
    }
1952
    const n = `${output.size - 1}`;
1953
    const mesg = output.get(n);
1954
    if (mesg == null) {
1955
      return;
1956
    }
1957

1958
    if (mesg.getIn(["opts", "password"])) {
1959
      // handle password input separately by first submitting to the backend.
1960
      try {
1961
        await this.submit_password(id, value);
1962
      } catch (err) {
1963
        this.set_error(`Error setting backend key/value store (${err})`);
1964
        return;
1965
      }
1966
      const m = value.length;
1967
      value = "";
1968
      for (let i = 0; i < m; i++) {
1969
        value == "●";
1970
      }
1971
      this.set_cell_output(id, output.set(n, mesg.set("value", value)), false);
1972
      this.save_asap();
1973
      return;
1974
    }
1975

1976
    this.set_cell_output(id, output.set(n, mesg.set("value", value)), false);
1977
    this.save_asap();
1978
  }
1979

1980
  submit_password = async (id: string, value: any): Promise<void> => {
1981
    await this.set_in_backend_key_value_store(id, value);
1982
  };
1983

1984
  set_in_backend_key_value_store = async (
1985
    key: any,
1986
    value: any,
1987
  ): Promise<void> => {
1988
    try {
1989
      await this.api().store({ key, value });
1990
    } catch (err) {
1991
      this.set_error(err);
1992
    }
1993
  };
1994

1995
  set_to_ipynb = async (
1996
    ipynb: any,
1997
    data_only: boolean = false,
1998
  ): Promise<void> => {
1999
    /*
2000
     * set_to_ipynb - set from ipynb object.  This is
2001
     * mainly meant to be run on the backend in the project,
2002
     * but is also run on the frontend too, e.g.,
2003
     * for client-side nbviewer (in which case it won't remove images, etc.).
2004
     *
2005
     * See the documentation for load_ipynb_file in project-actions.ts for
2006
     * documentation about the data_only input variable.
2007
     */
2008
    if (typeof ipynb != "object") {
2009
      throw Error("ipynb must be an object");
2010
    }
2011

2012
    this._state = "load";
2013

2014
    //dbg(misc.to_json(ipynb))
2015

2016
    // We try to parse out the kernel so we can use process_output below.
2017
    // (TODO: rewrite so process_output is not associated with a specific kernel)
2018
    let kernel: string | undefined;
2019
    const ipynb_metadata = ipynb.metadata;
2020
    if (ipynb_metadata != null) {
2021
      const kernelspec = ipynb_metadata.kernelspec;
2022
      if (kernelspec != null) {
2023
        kernel = kernelspec.name;
2024
      }
2025
    }
2026
    //dbg("kernel in ipynb: name='#{kernel}'")
2027

2028
    const existing_ids = this.store.get_cell_list().toJS();
2029

2030
    let set, trust;
2031
    if (data_only) {
2032
      trust = undefined;
2033
      set = function () {};
2034
    } else {
2035
      if (typeof this.reset_more_output === "function") {
2036
        this.reset_more_output();
2037
        // clear the more output handler (only on backend)
2038
      }
2039
      // We delete all of the cells.
2040
      // We do NOT delete everything, namely the last_loaded and
2041
      // the settings entry in the database, because that would
2042
      // throw away important information, e.g., the current kernel
2043
      // and its state.  NOTe: Some of that extra info *should* be
2044
      // moved to a different ephemeral table, but I haven't got
2045
      // around to doing so.
2046
      this.syncdb.delete({ type: "cell" });
2047
      // preserve trust state across file updates/loads
2048
      trust = this.store.get("trust");
2049
      set = (obj) => {
2050
        this.syncdb.set(obj);
2051
      };
2052
    }
2053

2054
    // Change kernel to what is in the file if necessary:
2055
    set({ type: "settings", kernel });
2056
    this.ensure_backend_kernel_setup();
2057

2058
    const importer = new IPynbImporter();
2059

2060
    // NOTE: Below we re-use any existing ids to make the patch that defines changing
2061
    // to the contents of ipynb more efficient.   In case of a very slight change
2062
    // on disk, this can be massively more efficient.
2063

2064
    importer.import({
2065
      ipynb,
2066
      existing_ids,
2067
      new_id: this.new_id.bind(this),
2068
      output_handler:
2069
        this.jupyter_kernel != null
2070
          ? this._output_handler.bind(this)
2071
          : undefined, // undefined in client; defined in project
2072
    });
2073

2074
    if (data_only) {
2075
      importer.close();
2076
      return;
2077
    }
2078

2079
    // Set all the cells
2080
    const object = importer.cells();
2081
    for (const _ in object) {
2082
      const cell = object[_];
2083
      set(cell);
2084
    }
2085

2086
    // Set the settings
2087
    set({ type: "settings", kernel: importer.kernel(), trust });
2088

2089
    // Set extra user-defined metadata
2090
    const metadata = importer.metadata();
2091
    if (metadata != null) {
2092
      set({ type: "settings", metadata });
2093
    }
2094

2095
    importer.close();
2096

2097
    this.syncdb.commit();
2098
    await this.syncdb.save();
2099
    this.ensure_backend_kernel_setup();
2100
    this._state = "ready";
2101
  };
2102

2103
  public set_cell_slide(id: string, value: any): void {
2104
    if (!value) {
2105
      value = null; // delete
2106
    }
2107
    if (this.check_edit_protection(id, "making a cell aslide")) {
2108
      return;
2109
    }
2110
    this._set({
2111
      type: "cell",
2112
      id,
2113
      slide: value,
2114
    });
2115
  }
2116

2117
  public ensure_positions_are_unique(): void {
2118
    if (this._state != "ready" || this.store == null) {
2119
      // because of debouncing, this ensure_positions_are_unique can
2120
      // be called after jupyter actions are closed.
2121
      return;
2122
    }
2123
    const changes = cell_utils.ensure_positions_are_unique(
2124
      this.store.get("cells"),
2125
    );
2126
    if (changes != null) {
2127
      for (const id in changes) {
2128
        const pos = changes[id];
2129
        this.set_cell_pos(id, pos, false);
2130
      }
2131
    }
2132
    this._sync();
2133
  }
2134

2135
  public set_default_kernel(kernel?: string): void {
2136
    if (kernel == null || kernel === "") return;
2137
    // doesn't make sense for project (right now at least)
2138
    if (this.is_project || this.is_compute_server) return;
2139
    const account_store = this.redux.getStore("account") as any;
2140
    if (account_store == null) return;
2141
    const cur: any = {};
2142
    // if available, retain existing jupyter config
2143
    const acc_jup = account_store.getIn(["editor_settings", "jupyter"]);
2144
    if (acc_jup != null) {
2145
      Object.assign(cur, acc_jup.toJS());
2146
    }
2147
    // set new kernel and save it
2148
    cur.kernel = kernel;
2149
    (this.redux.getTable("account") as any).set({
2150
      editor_settings: { jupyter: cur },
2151
    });
2152
  }
2153

2154
  edit_attachments = (id: string): void => {
2155
    this.setState({ edit_attachments: id });
2156
  };
2157

2158
  _attachment_markdown = (name: any) => {
2159
    return `![${name}](attachment:${name})`;
2160
    // Don't use this because official Jupyter tooling can't deal with it. See
2161
    //    https://github.com/sagemathinc/cocalc/issues/5055
2162
    return `<img src="attachment:${name}" style="max-width:100%">`;
2163
  };
2164

2165
  insert_input_at_cursor = (id: string, s: string, save: boolean = true) => {
2166
    // TODO: this maybe doesn't make sense anymore...
2167
    // TODO: redo this -- note that the input below is wrong, since it is
2168
    // from the store, not necessarily from what is live in the cell.
2169

2170
    if (this.store.getIn(["cells", id]) == null) {
2171
      return;
2172
    }
2173
    if (this.check_edit_protection(id, "inserting input")) {
2174
      return;
2175
    }
2176
    let input = this.store.getIn(["cells", id, "input"], "");
2177
    const cursor = this._cursor_locs != null ? this._cursor_locs[0] : undefined;
2178
    if ((cursor != null ? cursor.id : undefined) === id) {
2179
      const v = input.split("\n");
2180
      const line = v[cursor.y];
2181
      v[cursor.y] = line.slice(0, cursor.x) + s + line.slice(cursor.x);
2182
      input = v.join("\n");
2183
    } else {
2184
      input += s;
2185
    }
2186
    return this._set({ type: "cell", id, input }, save);
2187
  };
2188

2189
  // Sets attachments[name] = val
2190
  public set_cell_attachment(
2191
    id: string,
2192
    name: string,
2193
    val: any,
2194
    save: boolean = true,
2195
  ): void {
2196
    const cell = this.store.getIn(["cells", id]);
2197
    if (cell == null) {
2198
      throw Error(`no cell ${id}`);
2199
    }
2200
    if (this.check_edit_protection(id, "setting an attachment")) return;
2201
    const attachments = cell.get("attachments", immutable.Map()).toJS();
2202
    attachments[name] = val;
2203
    this._set(
2204
      {
2205
        type: "cell",
2206
        id,
2207
        attachments,
2208
      },
2209
      save,
2210
    );
2211
  }
2212

2213
  public async add_attachment_to_cell(id: string, path: string): Promise<void> {
2214
    if (this.check_edit_protection(id, "adding an attachment")) {
2215
      return;
2216
    }
2217
    let name: string = encodeURIComponent(
2218
      misc.path_split(path).tail.toLowerCase(),
2219
    );
2220
    name = name.replace(/\(/g, "%28").replace(/\)/g, "%29");
2221
    this.set_cell_attachment(id, name, { type: "load", value: path });
2222
    await callback2(this.store.wait, {
2223
      until: () =>
2224
        this.store.getIn(["cells", id, "attachments", name, "type"]) === "sha1",
2225
      timeout: 0,
2226
    });
2227
    // This has to happen in the next render loop, since changing immediately
2228
    // can update before the attachments props are updated.
2229
    await delay(10);
2230
    this.insert_input_at_cursor(id, this._attachment_markdown(name), true);
2231
  }
2232

2233
  delete_attachment_from_cell = (id: string, name: any) => {
2234
    if (this.check_edit_protection(id, "deleting an attachment")) {
2235
      return;
2236
    }
2237
    this.set_cell_attachment(id, name, null, false);
2238
    this.set_cell_input(
2239
      id,
2240
      misc.replace_all(
2241
        this._get_cell_input(id),
2242
        this._attachment_markdown(name),
2243
        "",
2244
      ),
2245
    );
2246
  };
2247

2248
  add_tag(id: string, tag: string, save: boolean = true): void {
2249
    if (this.check_edit_protection(id, "adding a tag")) {
2250
      return;
2251
    }
2252
    return this._set(
2253
      {
2254
        type: "cell",
2255
        id,
2256
        tags: { [tag]: true },
2257
      },
2258
      save,
2259
    );
2260
  }
2261

2262
  remove_tag(id: string, tag: string, save: boolean = true): void {
2263
    if (this.check_edit_protection(id, "removing a tag")) {
2264
      return;
2265
    }
2266
    return this._set(
2267
      {
2268
        type: "cell",
2269
        id,
2270
        tags: { [tag]: null },
2271
      },
2272
      save,
2273
    );
2274
  }
2275

2276
  toggle_tag(id: string, tag: string, save: boolean = true): void {
2277
    const cell = this.store.getIn(["cells", id]);
2278
    if (cell == null) {
2279
      throw Error(`no cell with id ${id}`);
2280
    }
2281
    const tags = cell.get("tags");
2282
    if (tags == null || !tags.get(tag)) {
2283
      this.add_tag(id, tag, save);
2284
    } else {
2285
      this.remove_tag(id, tag, save);
2286
    }
2287
  }
2288

2289
  edit_cell_metadata = (id: string): void => {
2290
    const metadata = this.store.getIn(
2291
      ["cells", id, "metadata"],
2292
      immutable.Map(),
2293
    );
2294
    this.setState({ edit_cell_metadata: { id, metadata } });
2295
  };
2296

2297
  public set_global_metadata(metadata: object, save: boolean = true): void {
2298
    const cur = this.syncdb.get_one({ type: "settings" })?.toJS()?.metadata;
2299
    if (cur) {
2300
      metadata = {
2301
        ...cur,
2302
        ...metadata,
2303
      };
2304
    }
2305
    this.syncdb.set({ type: "settings", metadata });
2306
    if (save) {
2307
      this.syncdb.commit();
2308
    }
2309
  }
2310

2311
  public set_cell_metadata(opts: {
2312
    id: string;
2313
    metadata?: object; // not given = delete it
2314
    save?: boolean; // defaults to true if not given
2315
    merge?: boolean; // defaults to false if not given, in which case sets metadata, rather than merge.  If true, does a SHALLOW merge.
2316
    bypass_edit_protection?: boolean;
2317
  }): void {
2318
    let { id, metadata, save, merge, bypass_edit_protection } = (opts =
2319
      defaults(opts, {
2320
        id: required,
2321
        metadata: required,
2322
        save: true,
2323
        merge: false,
2324
        bypass_edit_protection: false,
2325
      }));
2326

2327
    if (
2328
      !bypass_edit_protection &&
2329
      this.check_edit_protection(id, "editing cell metadata")
2330
    ) {
2331
      return;
2332
    }
2333
    // Special case: delete metdata (unconditionally)
2334
    if (metadata == null || misc.len(metadata) === 0) {
2335
      this._set(
2336
        {
2337
          type: "cell",
2338
          id,
2339
          metadata: null,
2340
        },
2341
        save,
2342
      );
2343
      return;
2344
    }
2345

2346
    if (merge) {
2347
      const current = this.store.getIn(
2348
        ["cells", id, "metadata"],
2349
        immutable.Map(),
2350
      );
2351
      metadata = current.merge(immutable.fromJS(metadata)).toJS();
2352
    }
2353

2354
    // special fields
2355
    // "collapsed", "scrolled", "slideshow", and "tags"
2356
    if (metadata.tags != null) {
2357
      for (const tag of metadata.tags) {
2358
        this.add_tag(id, tag, false);
2359
      }
2360
      delete metadata.tags;
2361
    }
2362
    // important to not store redundant inconsistent fields:
2363
    for (const field of ["collapsed", "scrolled", "slideshow"]) {
2364
      if (metadata[field] != null) {
2365
        delete metadata[field];
2366
      }
2367
    }
2368

2369
    if (!merge) {
2370
      // first delete -- we have to do this due to shortcomings in syncdb, but it
2371
      // can have annoying side effects on the UI
2372
      this._set(
2373
        {
2374
          type: "cell",
2375
          id,
2376
          metadata: null,
2377
        },
2378
        false,
2379
      );
2380
    }
2381
    // now set
2382
    this._set(
2383
      {
2384
        type: "cell",
2385
        id,
2386
        metadata,
2387
      },
2388
      save,
2389
    );
2390
    if (this.store.getIn(["edit_cell_metadata", "id"]) === id) {
2391
      this.edit_cell_metadata(id); // updates the state while editing
2392
    }
2393
  }
2394

2395
  set_raw_ipynb(): void {
2396
    if (this._state != "ready") {
2397
      // lies otherwise...
2398
      return;
2399
    }
2400

2401
    this.setState({
2402
      raw_ipynb: immutable.fromJS(this.store.get_ipynb()),
2403
    });
2404
  }
2405

2406
  set_mode(mode: "escape" | "edit"): void {
2407
    this.deprecated("set_mode", mode);
2408
  }
2409

2410
  public focus(wait?: boolean): void {
2411
    this.deprecated("focus", wait);
2412
  }
2413

2414
  public blur(): void {
2415
    this.deprecated("blur");
2416
  }
2417

2418
  public check_edit_protection(id: string, reason?: string): boolean {
2419
    if (!this.store.is_cell_editable(id)) {
2420
      this.show_not_editable_error(reason);
2421
      return true;
2422
    } else {
2423
      return false;
2424
    }
2425
  }
2426

2427
  public check_delete_protection(id: string): boolean {
2428
    if (!this.store.is_cell_deletable(id)) {
2429
      this.show_not_deletable_error();
2430
      return true;
2431
    } else {
2432
      return false;
2433
    }
2434
  }
2435

2436
  split_current_cell = () => {
2437
    this.deprecated("split_current_cell");
2438
  };
2439

2440
  handle_nbconvert_change(_oldVal, _newVal): void {
2441
    throw Error("define this in derived class");
2442
  }
2443

2444
  set_kernel_error = (err) => {
2445
    this._set({
2446
      type: "settings",
2447
      kernel_error: `${err}`,
2448
    });
2449
    this.save_asap();
2450
  };
2451

2452
  // Returns true if the .ipynb file was explicitly deleted.
2453
  // Returns false if it is NOT known to be explicitly deleted.
2454
  // Returns undefined if not known or implemented.
2455
  // NOTE: this is different than the file not being present on disk.
2456
  protected isDeleted = () => {
2457
    if (this.store == null || this._client == null) {
2458
      return;
2459
    }
2460
    return this._client.is_deleted?.(this.store.get("path"), this.project_id);
2461
    // [ ] TODO: we also need to do this on compute servers, but
2462
    // they don't yet have the listings table.
2463
  };
2464

2465
  processRenderedMarkdown = ({ value, id }: { value: string; id: string }) => {
2466
    value = latexEnvs(value);
2467

2468
    const labelRegExp = /\s*\\label\{.*?\}\s*/g;
2469
    const figLabelRegExp = /\s*\\figlabel\{.*?\}\s*/g;
2470
    if (this.labels == null) {
2471
      const labels = (this.labels = { math: {}, fig: {} });
2472
      // do initial full document scan
2473
      if (this.store == null) {
2474
        return;
2475
      }
2476
      const cells = this.store.get("cells");
2477
      if (cells == null) {
2478
        return;
2479
      }
2480
      let mathN = 0;
2481
      let figN = 0;
2482
      for (const id of this.store.get_cell_ids_list()) {
2483
        const cell = cells.get(id);
2484
        if (cell?.get("cell_type") == "markdown") {
2485
          const value = latexEnvs(cell.get("input") ?? "");
2486
          value.replace(labelRegExp, (labelContent) => {
2487
            const label = extractLabel(labelContent);
2488
            mathN += 1;
2489
            labels.math[label] = { tag: `${mathN}`, id };
2490
            return "";
2491
          });
2492
          value.replace(figLabelRegExp, (labelContent) => {
2493
            const label = extractLabel(labelContent);
2494
            figN += 1;
2495
            labels.fig[label] = { tag: `${figN}`, id };
2496
            return "";
2497
          });
2498
        }
2499
      }
2500
    }
2501
    const labels = this.labels;
2502
    if (labels == null) {
2503
      throw Error("bug");
2504
    }
2505
    value = value.replace(labelRegExp, (labelContent) => {
2506
      const label = extractLabel(labelContent);
2507
      if (labels.math[label] == null) {
2508
        labels.math[label] = { tag: `${misc.len(labels.math) + 1}`, id };
2509
      } else {
2510
        // in case it moved to a different cell due to cut/paste
2511
        labels.math[label].id = id;
2512
      }
2513
      return `\\tag{${labels.math[label].tag}}`;
2514
    });
2515
    value = value.replace(figLabelRegExp, (labelContent) => {
2516
      const label = extractLabel(labelContent);
2517
      if (labels.fig[label] == null) {
2518
        labels.fig[label] = { tag: `${misc.len(labels.fig) + 1}`, id };
2519
      } else {
2520
        // in case it moved to a different cell due to cut/paste
2521
        labels.fig[label].id = id;
2522
      }
2523
      return ` ${labels.fig[label].tag ?? "?"}`;
2524
    });
2525
    const refRegExp = /\\ref\{.*?\}/g;
2526
    value = value.replace(refRegExp, (refContent) => {
2527
      const label = extractLabel(refContent);
2528
      if (labels.fig[label] == null && labels.math[label] == null) {
2529
        // do not know the label
2530
        return "?";
2531
      }
2532
      const { tag, id } = labels.fig[label] ?? labels.math[label];
2533
      return `[${tag}](#id=${id})`;
2534
    });
2535

2536
    return value;
2537
  };
2538

2539
  // Update run progress, which is a number between 0 and 100,
2540
  // giving the number of runnable cells that have been run since
2541
  // the kernel was last set to the running state.
2542
  // Currently only run in the browser, but could maybe be useful
2543
  // elsewhere someday.
2544
  updateRunProgress = () => {
2545
    if (this.store == null) {
2546
      return;
2547
    }
2548
    if (this.store.get("backend_state") != "running") {
2549
      this.setState({ runProgress: 0 });
2550
      return;
2551
    }
2552
    const cells = this.store.get("cells");
2553
    if (cells == null) {
2554
      return;
2555
    }
2556
    const last = this.store.get("last_backend_state");
2557
    if (last == null) {
2558
      // not supported yet, e.g., old backend, kernel never started
2559
      return;
2560
    }
2561
    // count of number of cells that are runnable and
2562
    // have start greater than last, and end set...
2563
    // count a currently running cell as 0.5.
2564
    let total = 0;
2565
    let ran = 0;
2566
    for (const [_, cell] of cells) {
2567
      if (
2568
        cell.get("cell_type", "code") != "code" ||
2569
        !cell.get("input")?.trim()
2570
      ) {
2571
        // not runnable
2572
        continue;
2573
      }
2574
      total += 1;
2575
      if ((cell.get("start") ?? 0) >= last) {
2576
        if (cell.get("end")) {
2577
          ran += 1;
2578
        } else {
2579
          ran += 0.5;
2580
        }
2581
      }
2582
    }
2583
    this.setState({ runProgress: total > 0 ? (100 * ran) / total : 100 });
2584
  };
2585
}
2586

2587
function extractLabel(content: string): string {
2588
  const i = content.indexOf("{");
2589
  const j = content.lastIndexOf("}");
2590
  return content.slice(i + 1, j);
2591
}
2592

2593
function bounded_integer(n: any, min: any, max: any, def: any) {
2594
  if (typeof n !== "number") {
2595
    n = parseInt(n);
2596
  }
2597
  if (isNaN(n)) {
2598
    return def;
2599
  }
2600
  n = Math.round(n);
2601
  if (n < min) {
2602
    return min;
2603
  }
2604
  if (n > max) {
2605
    return max;
2606
  }
2607
  return n;
2608
}
2609

2610
function getCompletionGroup(x: string): number {
2611
  switch (x[0]) {
2612
    case "_":
2613
      return 1;
2614
    case "%":
2615
      return 2;
2616
    default:
2617
      return 0;
2618
  }
2619
}
2620

2621