1
/*
2
 *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
3
 *  License: MS-RSL – see LICENSE.md for details
4
 */
5

6
/*
7
The Redux Store For Jupyter Notebooks
8

9
This is used by everybody involved in using jupyter -- the project, the browser client, etc.
10
*/
11

12
import { List, Map, OrderedMap, Set } from "immutable";
13
import { export_to_ipynb } from "@cocalc/jupyter/ipynb/export-to-ipynb";
14
import { KernelSpec } from "@cocalc/jupyter/ipynb/parse";
15
import {
16
  Cell,
17
  CellToolbarName,
18
  KernelInfo,
19
  NotebookMode,
20
} from "@cocalc/jupyter/types";
21
import {
22
  Kernel,
23
  Kernels,
24
  get_kernel_selection,
25
} from "@cocalc/jupyter/util/misc";
26
import { Syntax } from "@cocalc/util/code-formatter";
27
import { startswith } from "@cocalc/util/misc";
28
import { Store } from "@cocalc/util/redux/Store";
29
import type { ImmutableUsageInfo } from "@cocalc/util/types/project-usage-info";
30
import { cloneDeep } from "lodash";
31

32
// Used for copy/paste.  We make a single global clipboard, so that
33
// copy/paste between different notebooks works.
34
let global_clipboard: any = undefined;
35

36
export type show_kernel_selector_reasons = "bad kernel" | "user request";
37

38
export function canonical_language(
39
  kernel?: string | null,
40
  kernel_info_lang?: string,
41
): string | undefined {
42
  let lang;
43
  // special case: sage is language "python", but the snippet dialog needs "sage"
44
  if (startswith(kernel, "sage")) {
45
    lang = "sage";
46
  } else {
47
    lang = kernel_info_lang;
48
  }
49
  return lang;
50
}
51

52
export interface JupyterStoreState {
53
  about: boolean;
54
  backend_kernel_info: KernelInfo;
55
  cell_list: List<string>; // list of id's of the cells, in order by pos.
56
  cell_toolbar?: CellToolbarName;
57
  cells: Map<string, Cell>; // map from string id to cell; the structure of a cell is complicated...
58
  check_select_kernel_init: boolean;
59
  closestKernel?: Kernel;
60
  cm_options: any;
61
  complete: any;
62
  confirm_dialog: any;
63
  connection_file?: string;
64
  contents?: List<Map<string, any>>; // optional global contents info (about sections, problems, etc.)
65
  default_kernel?: string;
66
  directory: string;
67
  edit_attachments?: string;
68
  edit_cell_metadata: any;
69
  error?: string;
70
  fatal: string;
71
  find_and_replace: any;
72
  has_uncommitted_changes?: boolean;
73
  has_unsaved_changes?: boolean;
74
  introspect: any;
75
  kernel_error?: string;
76
  kernel_info?: any;
77
  kernel_selection?: Map<string, string>;
78
  kernel_usage?: ImmutableUsageInfo;
79
  kernel?: string | ""; // "": means "no kernel"
80
  kernels_by_language?: OrderedMap<string, List<string>>;
81
  kernels_by_name?: OrderedMap<string, Map<string, string>>;
82
  kernels?: Kernels;
83
  keyboard_shortcuts: any;
84
  max_output_length: number;
85
  md_edit_ids: Set<string>;
86
  metadata: any; // documented at https://nbformat.readthedocs.io/en/latest/format_description.html#cell-metadata
87
  mode: NotebookMode;
88
  more_output: any;
89
  name: string;
90
  nbconvert_dialog: any;
91
  nbconvert: any;
92
  path: string;
93
  project_id: string;
94
  raw_ipynb: any;
95
  read_only: boolean;
96
  scroll: any;
97
  sel_ids: any;
98
  show_kernel_selector_reason?: show_kernel_selector_reasons;
99
  show_kernel_selector: boolean;
100
  start_time: any;
101
  toolbar?: boolean;
102
  widgetModelIdState: Map<string, string>; // model_id --> '' (=supported), 'loading' (definitely loading), '(widget module).(widget name)' (=if NOT supported), undefined (=not known yet)
103
  // run progress = Percent (0-100) of runnable cells that have been run since the last
104
  // kernel restart. (Thus markdown and empty cells are excluded.)
105
  runProgress?: number;
106
}
107

108
export const initial_jupyter_store_state: {
109
  [K in keyof JupyterStoreState]?: JupyterStoreState[K];
110
} = {
111
  check_select_kernel_init: false,
112
  show_kernel_selector: false,
113
  widgetModelIdState: Map(),
114
  cell_list: List(),
115
  cells: Map(),
116
};
117

118
export class JupyterStore extends Store<JupyterStoreState> {
119
  // manipulated in jupyter/project-actions.ts
120
  _more_output: { [id: string]: any } = {};
121

122
  // immutable List
123
  get_cell_list = (): List<string> => {
124
    return this.get("cell_list") ?? List();
125
  };
126

127
  // string[]
128
  get_cell_ids_list(): string[] {
129
    return this.get_cell_list().toJS();
130
  }
131

132
  get_cell_type(id: string): "markdown" | "code" | "raw" {
133
    // NOTE: default cell_type is "code", which is common, to save space.
134
    // TODO: We use unsafe_getIn because maybe the cell type isn't spelled out yet, or our typescript isn't good enough.
135
    const type = this.unsafe_getIn(["cells", id, "cell_type"], "code");
136
    if (type != "markdown" && type != "code" && type != "raw") {
137
      throw Error(`invalid cell type ${type} for cell ${id}`);
138
    }
139
    return type;
140
  }
141

142
  get_cell_index(id: string): number {
143
    const cell_list = this.get("cell_list");
144
    if (cell_list == null) {
145
      // truly fatal
146
      throw Error("ordered list of cell id's not known");
147
    }
148
    const i = cell_list.indexOf(id);
149
    if (i === -1) {
150
      throw Error(`unknown cell id ${id}`);
151
    }
152
    return i;
153
  }
154

155
  // Get the id of the cell that is delta positions from
156
  // cell with given id (second input).
157
  // Returns undefined if delta positions moves out of
158
  // the notebook (so there is no such cell) or there
159
  // is no cell with the given id; in particular,
160
  // we do NOT wrap around.
161
  get_cell_id(delta = 0, id: string): string | undefined {
162
    let i: number;
163
    try {
164
      i = this.get_cell_index(id);
165
    } catch (_) {
166
      // no such cell. This can happen, e.g., https://github.com/sagemathinc/cocalc/issues/6686
167
      return;
168
    }
169
    i += delta;
170
    const cell_list = this.get("cell_list");
171
    if (cell_list == null || i < 0 || i >= cell_list.size) {
172
      return; // .get negative for List in immutable wraps around rather than undefined (like Python)
173
    }
174
    return cell_list.get(i);
175
  }
176

177
  set_global_clipboard = (clipboard: any) => {
178
    global_clipboard = clipboard;
179
  };
180

181
  get_global_clipboard = () => {
182
    return global_clipboard;
183
  };
184

185
  get_kernel_info = (
186
    kernel: string | null | undefined,
187
  ): KernelSpec | undefined => {
188
    // slow/inefficient, but ok since this is rarely called
189
    let info: any = undefined;
190
    const kernels = this.get("kernels");
191
    if (kernels === undefined) return;
192
    if (kernels === null) {
193
      return {
194
        name: "No Kernel",
195
        language: "",
196
        display_name: "No Kernel",
197
      };
198
    }
199
    kernels.forEach((x: any) => {
200
      if (x.get("name") === kernel) {
201
        info = x.toJS() as KernelSpec;
202
        return false;
203
      }
204
    });
205
    return info;
206
  };
207

208
  // Export the Jupyer notebook to an ipynb object.
209
  get_ipynb = (blob_store?: any) => {
210
    if (this.get("cells") == null || this.get("cell_list") == null) {
211
      // not sufficiently loaded yet.
212
      return;
213
    }
214

215
    const cell_list = this.get("cell_list");
216
    const more_output: { [id: string]: any } = {};
217
    for (const id of cell_list.toJS()) {
218
      const x = this.get_more_output(id);
219
      if (x != null) {
220
        more_output[id] = x;
221
      }
222
    }
223

224
    // export_to_ipynb mutates its input... mostly not a problem, since
225
    // we're toJS'ing most of it, but be careful with more_output.
226
    return export_to_ipynb({
227
      cells: this.get("cells").toJS(),
228
      cell_list: cell_list.toJS(),
229
      metadata: this.get("metadata")?.toJS(), // custom metadata
230
      kernelspec: this.get_kernel_info(this.get("kernel")),
231
      language_info: this.get_language_info(),
232
      blob_store,
233
      more_output: cloneDeep(more_output),
234
    });
235
  };
236

237
  get_language_info(): object | undefined {
238
    for (const key of ["backend_kernel_info", "metadata"]) {
239
      const language_info = this.unsafe_getIn([key, "language_info"]);
240
      if (language_info != null) {
241
        return language_info;
242
      }
243
    }
244
  }
245

246
  get_cm_mode() {
247
    let metadata_immutable = this.get("backend_kernel_info");
248
    if (metadata_immutable == null) {
249
      metadata_immutable = this.get("metadata");
250
    }
251
    let metadata: { language_info?: any; kernelspec?: any } | undefined;
252
    if (metadata_immutable != null) {
253
      metadata = metadata_immutable.toJS();
254
    } else {
255
      metadata = undefined;
256
    }
257
    let mode: any;
258
    if (metadata != null) {
259
      if (
260
        metadata.language_info != null &&
261
        metadata.language_info.codemirror_mode != null
262
      ) {
263
        mode = metadata.language_info.codemirror_mode;
264
      } else if (
265
        metadata.language_info != null &&
266
        metadata.language_info.name != null
267
      ) {
268
        mode = metadata.language_info.name;
269
      } else if (
270
        metadata.kernelspec != null &&
271
        metadata.kernelspec.language != null
272
      ) {
273
        mode = metadata.kernelspec.language.toLowerCase();
274
      }
275
    }
276
    if (mode == null) {
277
      // As a fallback in case none of the metadata has been filled in yet by the backend,
278
      // we can guess a mode from the kernel in many cases.   Any mode is vastly better
279
      // than nothing!
280
      let kernel = this.get("kernel"); // may be better than nothing...; e.g., octave kernel has no mode.
281
      if (kernel != null) {
282
        kernel = kernel.toLowerCase();
283
        // The kernel is just a string that names the kernel, so we use heuristics.
284
        if (kernel.indexOf("python") != -1) {
285
          if (kernel.indexOf("python3") != -1) {
286
            mode = { name: "python", version: 3 };
287
          } else {
288
            mode = { name: "python", version: 2 };
289
          }
290
        } else if (kernel.indexOf("sage") != -1) {
291
          mode = { name: "python", version: 3 };
292
        } else if (kernel.indexOf("anaconda") != -1) {
293
          mode = { name: "python", version: 3 };
294
        } else if (kernel.indexOf("octave") != -1) {
295
          mode = "octave";
296
        } else if (kernel.indexOf("bash") != -1) {
297
          mode = "shell";
298
        } else if (kernel.indexOf("julia") != -1) {
299
          mode = "text/x-julia";
300
        } else if (kernel.indexOf("haskell") != -1) {
301
          mode = "text/x-haskell";
302
        } else if (kernel.indexOf("javascript") != -1) {
303
          mode = "javascript";
304
        } else if (kernel.indexOf("ir") != -1) {
305
          mode = "r";
306
        } else if (
307
          kernel.indexOf("root") != -1 ||
308
          kernel.indexOf("xeus") != -1
309
        ) {
310
          mode = "text/x-c++src";
311
        } else if (kernel.indexOf("gap") != -1) {
312
          mode = "gap";
313
        } else {
314
          // Python 3 is probably a good fallback.
315
          mode = { name: "python", version: 3 };
316
        }
317
      }
318
    }
319
    if (typeof mode === "string") {
320
      mode = { name: mode }; // some kernels send a string back for the mode; others an object
321
    }
322
    return mode;
323
  }
324

325
  get_more_output = (id: string) => {
326
    // this._more_output only gets set in project-actions in
327
    // set_more_output, for the project or compute server that
328
    // has that extra output.
329
    if (this._more_output != null) {
330
      // This is ONLY used by the backend for storing and retrieving
331
      // extra output messages.
332
      const output = this._more_output[id];
333
      if (output == null) {
334
        return;
335
      }
336
      let { messages } = output;
337

338
      for (const x of ["discarded", "truncated"]) {
339
        if (output[x]) {
340
          var text;
341
          if (x === "truncated") {
342
            text = "WARNING: some intermediate output was truncated.\n";
343
          } else {
344
            text = `WARNING: at least ${output[x]} intermediate output ${
345
              output[x] > 1 ? "messages were" : "message was"
346
            } ${x}.\n`;
347
          }
348
          const warn = [{ text, name: "stderr" }];
349
          if (messages.length > 0) {
350
            messages = warn.concat(messages).concat(warn);
351
          } else {
352
            messages = warn;
353
          }
354
        }
355
      }
356
      return messages;
357
    } else {
358
      // client  -- return what we know
359
      const msg_list = this.getIn(["more_output", id, "mesg_list"]);
360
      if (msg_list != null) {
361
        return msg_list.toJS();
362
      }
363
    }
364
  };
365

366
  get_default_kernel = (): string | undefined => {
367
    const account = this.redux.getStore("account");
368
    if (account != null) {
369
      // TODO: getIn types
370
      return account.getIn(["editor_settings", "jupyter", "kernel"]);
371
    } else {
372
      return undefined;
373
    }
374
  };
375

376
  get_kernel_selection = (kernels: Kernels): Map<string, string> => {
377
    return get_kernel_selection(kernels);
378
  };
379

380
  // NOTE: defaults for these happen to be true if not given (due to bad
381
  // choice of name by some extension author).
382
  is_cell_editable = (id: string): boolean => {
383
    return this.get_cell_metadata_flag(id, "editable", true);
384
  };
385

386
  is_cell_deletable = (id: string): boolean => {
387
    if (!this.is_cell_editable(id)) {
388
      // I've decided that if a cell is not editable, then it is
389
      // automatically not deletable.  Relevant facts:
390
      //    1. It makes sense to me.
391
      //    2. This is what Jupyter classic does.
392
      //    3. This is NOT what JupyterLab does.
393
      //    4. The spec doesn't mention deletable: https://nbformat.readthedocs.io/en/latest/format_description.html#cell-metadata
394
      // See my rant here: https://github.com/jupyter/notebook/issues/3700
395
      return false;
396
    }
397
    return this.get_cell_metadata_flag(id, "deletable", true);
398
  };
399

400
  get_cell_metadata_flag = (
401
    id: string,
402
    key: string,
403
    default_value: boolean = false,
404
  ): boolean => {
405
    return this.unsafe_getIn(["cells", id, "metadata", key], default_value);
406
  };
407

408
  // canonicalize the language of the kernel
409
  get_kernel_language = (): string | undefined => {
410
    return canonical_language(
411
      this.get("kernel"),
412
      this.getIn(["kernel_info", "language"]),
413
    );
414
  };
415

416
  // map the kernel language to the syntax of a language we know
417
  get_kernel_syntax = (): Syntax | undefined => {
418
    let lang = this.get_kernel_language();
419
    if (!lang) return undefined;
420
    lang = lang.toLowerCase();
421
    switch (lang) {
422
      case "python":
423
      case "python3":
424
        return "python3";
425
      case "r":
426
        return "R";
427
      case "c++":
428
      case "c++17":
429
        return "c++";
430
      case "javascript":
431
        return "JavaScript";
432
    }
433
  };
434

435
  jupyter_kernel_key = async (): Promise<string> => {
436
    const project_id = this.get("project_id");
437
    const projects_store = this.redux.getStore("projects");
438
    const customize = this.redux.getStore("customize");
439
    const computeServerId = await this.redux
440
      .getActions(this.name)
441
      ?.getComputeServerId();
442
    if (customize == null) {
443
      // the customize store doesn't exist, e.g., in a compute server.
444
      // In that case no need for a complicated jupyter kernel key as
445
      // there is only one image.
446
      // (??)
447
      return `${project_id}-${computeServerId}-default`;
448
    }
449
    const dflt_img = await customize.getDefaultComputeImage();
450
    const compute_image = projects_store.getIn(
451
      ["project_map", project_id, "compute_image"],
452
      dflt_img,
453
    );
454
    const key = [project_id, `${computeServerId}`, compute_image].join("::");
455
    // console.log("jupyter store / jupyter_kernel_key", key);
456
    return key;
457
  };
458
}
459

460