1
/*
2
Handle opening files in a project to save/load from disk and also enable compute capabilities.
3

4
DEVELOPMENT:
5

6
0. From the browser with the project opened, terminate the open-files api service:
7

8

9
    await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'open-files'})
10

11

12

13
Set env variables as in a project (see  api/index.ts ), then in nodejs:
14

15
DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:conat:* node
16

17
    x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x)
18

19

20
[ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers', 'cc' ]
21

22
> x.openFiles.getAll();
23

24
> Object.keys(x.openDocs)
25

26
> s = x.openDocs['z4.tasks']
27
// now you can directly work with the syncdoc for a given file,
28
// but from the perspective of the project, not the browser!
29
//
30
//
31

32
OR:
33

34
 echo "require('@cocalc/project/conat/open-files').init(); require('@cocalc/project/bug-counter').init()" | node
35

36
COMPUTE SERVER:
37

38
To simulate a compute server, do exactly as above, but also set the environment
39
variable COMPUTE_SERVER_ID to the *global* (not project specific) id of the compute
40
server:
41

42
   COMPUTE_SERVER_ID=84 node
43

44
In this case, you aso don't need to use the terminate command if the compute
45
server isn't actually running.  To terminate a compute server open files service though:
46

47
    (TODO)
48

49

50
EDITOR ACTIONS:
51

52
Stop the open-files server and define x as above in a terminal.  You can
53
then get the actions or store in a nodejs terminal for a particular document
54
as follows:
55

56
project_id = '00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'; path = '2025-03-21-100921.ipynb';
57
redux = require("@cocalc/jupyter/redux/app").redux;  a = redux.getEditorActions(project_id, path); s = redux.getEditorStore(project_id, path); 0;
58

59

60
IN A LIVE RUNNING PROJECT IN KUCALC:
61

62
Ssh in to the project itself.  You can use a terminal because that very terminal will be broken by
63
doing this!  Then:
64

65
/cocalc/github/src/packages/project$ . /cocalc/nvm/nvm.sh
66
/cocalc/github/src/packages/project$ COCALC_PROJECT_ID=... COCALC_SECRET_TOKEN="/secrets/secret-token/token"  CONAT_SERVER=hub-conat node  # not sure about CONAT_SERVER
67
Welcome to Node.js v20.19.0.
68
Type ".help" for more information.
69
> x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x)
70
[ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers' ]
71
>
72

73

74
*/
75

76
import {
77
  openFiles as createOpenFiles,
78
  type OpenFiles,
79
  type OpenFileEntry,
80
} from "@cocalc/project/conat/sync";
81
import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat";
82
import { compute_server_id, project_id } from "@cocalc/project/data";
83
import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc";
84
import { getClient } from "@cocalc/project/client";
85
import { SyncString } from "@cocalc/sync/editor/string/sync";
86
import { SyncDB } from "@cocalc/sync/editor/db/sync";
87
import getLogger from "@cocalc/backend/logger";
88
import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
89
import { delay } from "awaiting";
90
import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel";
91
import { filename_extension, original_path } from "@cocalc/util/misc";
92
import { createFormatterService } from "./formatter";
93
import { type ConatService } from "@cocalc/conat/service/service";
94
import { exists } from "@cocalc/backend/misc/async-utils-node";
95
import { map as awaitMap } from "awaiting";
96
import { unlink } from "fs/promises";
97
import { join } from "path";
98
import {
99
  computeServerManager,
100
  ComputeServerManager,
101
} from "@cocalc/conat/compute/manager";
102
import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names";
103
import { connectToConat } from "@cocalc/project/conat/connection";
104

105
// ensure conat connection stuff is initialized
106
import "@cocalc/project/conat/env";
107
import { chdir } from "node:process";
108

109
const logger = getLogger("project:conat:open-files");
110

111
// we check all files we are currently managing this frequently to
112
// see if they exist on the filesystem:
113
const FILE_DELETION_CHECK_INTERVAL = 5000;
114

115
// once we determine that a file does not exist for some reason, we
116
// wait this long and check *again* just to be sure.  If it is still missing,
117
// then we close the file in memory and set the file as deleted in the
118
// shared openfile state.
119
const FILE_DELETION_GRACE_PERIOD = 2000;
120

121
// We NEVER check a file for deletion for this long after first opening it.
122
// This is VERY important, since some documents, e.g., jupyter notebooks,
123
// can take a while to get created on disk the first time.
124
const FILE_DELETION_INITIAL_DELAY = 15000;
125

126
let openFiles: OpenFiles | null = null;
127
let formatter: any = null;
128
const openDocs: { [path: string]: SyncDoc | ConatService } = {};
129
let computeServers: ComputeServerManager | null = null;
130
const openTimes: { [path: string]: number } = {};
131

132
export function getSyncDoc(path: string): SyncDoc | undefined {
133
  const doc = openDocs[path];
134
  if (doc instanceof SyncString || doc instanceof SyncDB) {
135
    return doc;
136
  }
137
  return undefined;
138
}
139

140
export async function init() {
141
  logger.debug("init");
142

143
  if (process.env.HOME) {
144
    chdir(process.env.HOME);
145
  }
146

147
  openFiles = await createOpenFiles();
148

149
  computeServers = computeServerManager({ project_id });
150
  await computeServers.waitUntilReady();
151
  computeServers.on("change", async ({ path, id }) => {
152
    if (openFiles == null) {
153
      return;
154
    }
155
    const entry = openFiles?.get(path);
156
    if (entry != null) {
157
      await handleChange({ ...entry, id });
158
    } else {
159
      await closeDoc(path);
160
    }
161
  });
162

163
  // initialize
164
  for (const entry of openFiles.getAll()) {
165
    handleChange(entry);
166
  }
167

168
  // start loop to watch for and close files that aren't touched frequently:
169
  closeIgnoredFilesLoop();
170

171
  // periodically update timestamp on backend for files we have open
172
  touchOpenFilesLoop();
173
  // watch if any file that is currently opened on this host gets deleted,
174
  // and if so, mark it as such, and set it to closed.
175
  watchForFileDeletionLoop();
176

177
  // handle changes
178
  openFiles.on("change", (entry) => {
179
    // we ONLY actually try to open the file here if there
180
    // is a doctype set.  When it is first being created,
181
    // the doctype won't be the first field set, and we don't
182
    // want to launch this until it is set.
183
    if (entry.doctype) {
184
      handleChange(entry);
185
    }
186
  });
187

188
  formatter = await createFormatterService({ openSyncDocs: openDocs });
189

190
  // useful for development
191
  return {
192
    openFiles,
193
    openDocs,
194
    formatter,
195
    terminate,
196
    computeServers,
197
    cc: connectToConat(),
198
  };
199
}
200

201
export function terminate() {
202
  logger.debug("terminating open-files service");
203
  for (const path in openDocs) {
204
    closeDoc(path);
205
  }
206
  openFiles?.close();
207
  openFiles = null;
208

209
  formatter?.close();
210
  formatter = null;
211

212
  computeServers?.close();
213
  computeServers = null;
214
}
215

216
function getCutoff(): number {
217
  return Date.now() - 2.5 * CONAT_OPEN_FILE_TOUCH_INTERVAL;
218
}
219

220
function computeServerId(path: string): number {
221
  return computeServers?.get(path) ?? 0;
222
}
223

224
async function handleChange({
225
  path,
226
  time,
227
  deleted,
228
  backend,
229
  doctype,
230
  id,
231
}: OpenFileEntry & { id?: number }) {
232
  try {
233
    if (id == null) {
234
      id = computeServerId(path);
235
    }
236
    logger.debug("handleChange", { path, time, deleted, backend, doctype, id });
237
    const syncDoc = openDocs[path];
238
    const isOpenHere = syncDoc != null;
239

240
    if (id != compute_server_id) {
241
      if (backend?.id == compute_server_id) {
242
        // we are definitely not the backend right now.
243
        openFiles?.setNotBackend(path, compute_server_id);
244
      }
245
      // only thing we should do is close it if it is open.
246
      if (isOpenHere) {
247
        await closeDoc(path);
248
      }
249
      return;
250
    }
251

252
    if (deleted?.deleted) {
253
      if (await exists(path)) {
254
        // it's back
255
        openFiles?.setNotDeleted(path);
256
      } else {
257
        if (isOpenHere) {
258
          await closeDoc(path);
259
        }
260
        return;
261
      }
262
    }
263

264
    if (time != null && time >= getCutoff()) {
265
      if (!isOpenHere) {
266
        logger.debug("handleChange: opening", { path });
267
        // users actively care about this file being opened HERE, but it isn't
268
        await openDoc(path);
269
      }
270
      return;
271
    }
272
  } catch (err) {
273
    console.trace(err);
274
    logger.debug(`handleChange: WARNING - error opening ${path} -- ${err}`);
275
  }
276
}
277

278
function supportAutoclose(path: string): boolean {
279
  // this feels way too "hard coded"; alternatively, maybe we make the kernel or whatever
280
  // actually update the interest?  or something else...
281
  if (
282
    path.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS) ||
283
    path.endsWith(".sagews") ||
284
    path.endsWith(".term")
285
  ) {
286
    return false;
287
  }
288
  return true;
289
}
290

291
async function closeIgnoredFilesLoop() {
292
  while (openFiles?.state == "connected") {
293
    await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL);
294
    if (openFiles?.state != "connected") {
295
      return;
296
    }
297
    const paths = Object.keys(openDocs);
298
    if (paths.length == 0) {
299
      logger.debug("closeIgnoredFiles: no paths currently open");
300
      continue;
301
    }
302
    logger.debug(
303
      "closeIgnoredFiles: checking",
304
      paths.length,
305
      "currently open paths...",
306
    );
307
    const cutoff = getCutoff();
308
    for (const entry of openFiles.getAll()) {
309
      if (
310
        entry != null &&
311
        entry.time != null &&
312
        openDocs[entry.path] != null &&
313
        entry.time <= cutoff &&
314
        supportAutoclose(entry.path)
315
      ) {
316
        logger.debug("closeIgnoredFiles: closing due to inactivity", entry);
317
        closeDoc(entry.path);
318
      }
319
    }
320
  }
321
}
322

323
async function touchOpenFilesLoop() {
324
  while (openFiles?.state == "connected" && openDocs != null) {
325
    for (const path in openDocs) {
326
      openFiles.setBackend(path, compute_server_id);
327
    }
328
    await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL);
329
  }
330
}
331

332
async function checkForFileDeletion(path: string) {
333
  if (openFiles == null) {
334
    return;
335
  }
336
  if (Date.now() - (openTimes[path] ?? 0) <= FILE_DELETION_INITIAL_DELAY) {
337
    return;
338
  }
339
  const id = computeServerId(path);
340
  if (id != compute_server_id) {
341
    // not our concern
342
    return;
343
  }
344

345
  if (path.endsWith(".term")) {
346
    // term files are exempt -- we don't save data in them and often
347
    // don't actually make the hidden ones for each frame in the
348
    // filesystem at all.
349
    return;
350
  }
351
  const entry = openFiles.get(path);
352
  if (entry == null) {
353
    return;
354
  }
355
  if (entry.deleted?.deleted) {
356
    // already set as deleted -- shouldn't still be opened
357
    await closeDoc(entry.path);
358
  } else {
359
    if (!process.env.HOME) {
360
      // too dangerous
361
      return;
362
    }
363
    const fullPath = join(process.env.HOME, entry.path);
364
    // if file doesn't exist and still doesn't exist in a while,
365
    // mark deleted, which also causes a close.
366
    if (await exists(fullPath)) {
367
      return;
368
    }
369
    // still doesn't exist?
370
    // We must give things a reasonable amount of time, e.g., otherwise
371
    // creating a file (e.g., jupyter notebook) might take too long and
372
    // we randomly think it is deleted before we even make it!
373
    await delay(FILE_DELETION_GRACE_PERIOD);
374
    if (await exists(fullPath)) {
375
      return;
376
    }
377
    // still doesn't exist
378
    if (openFiles != null) {
379
      logger.debug("checkForFileDeletion: marking as deleted -- ", entry);
380
      openFiles.setDeleted(entry.path);
381
      await closeDoc(fullPath);
382
      // closing a file may cause it to try to save to disk the last version,
383
      // so we delete it if that happens.
384
      // TODO: add an option to close everywhere to not do this, and/or make
385
      // it not save on close if the file doesn't exist.
386
      try {
387
        if (await exists(fullPath)) {
388
          await unlink(fullPath);
389
        }
390
      } catch {}
391
    }
392
  }
393
}
394

395
async function watchForFileDeletionLoop() {
396
  while (openFiles != null && openFiles.state == "connected") {
397
    await delay(FILE_DELETION_CHECK_INTERVAL);
398
    if (openFiles?.state != "connected") {
399
      return;
400
    }
401
    const paths = Object.keys(openDocs);
402
    if (paths.length == 0) {
403
      // logger.debug("watchForFileDeletionLoop: no paths currently open");
404
      continue;
405
    }
406
    //     logger.debug(
407
    //       "watchForFileDeletionLoop: checking",
408
    //       paths.length,
409
    //       "currently open paths to see if any were deleted",
410
    //     );
411
    await awaitMap(paths, 20, checkForFileDeletion);
412
  }
413
}
414

415
const closeDoc = reuseInFlight(async (path: string) => {
416
  logger.debug("close", { path });
417
  try {
418
    const doc = openDocs[path];
419
    if (doc == null) {
420
      return;
421
    }
422
    delete openDocs[path];
423
    delete openTimes[path];
424
    try {
425
      await doc.close();
426
    } catch (err) {
427
      logger.debug(`WARNING -- issue closing doc -- ${err}`);
428
      openFiles?.setError(path, err);
429
    }
430
  } finally {
431
    if (openDocs[path] == null) {
432
      openFiles?.setNotBackend(path, compute_server_id);
433
    }
434
  }
435
});
436

437
const openDoc = reuseInFlight(async (path: string) => {
438
  logger.debug("openDoc", { path });
439
  try {
440
    const doc = openDocs[path];
441
    if (doc != null) {
442
      return;
443
    }
444
    openTimes[path] = Date.now();
445

446
    if (path.endsWith(".term")) {
447
      // terminals are handled directly by the project api -- also since
448
      // doctype probably not set for them, they won't end up here.
449
      // (this could change though, e.g., we might use doctype to
450
      // set the terminal command).
451
      return;
452
    }
453

454
    const client = getClient();
455
    let doctype: any = openFiles?.get(path)?.doctype;
456
    logger.debug("openDoc: open files table knows ", openFiles?.get(path), {
457
      path,
458
    });
459
    if (doctype == null) {
460
      logger.debug("openDoc: doctype must be set but isn't, so bailing", {
461
        path,
462
      });
463
    } else {
464
      logger.debug("openDoc: got doctype from openFiles table", {
465
        path,
466
        doctype,
467
      });
468
    }
469

470
    let syncdoc;
471
    if (doctype.type == "string") {
472
      syncdoc = new SyncString({
473
        ...doctype.opts,
474
        project_id,
475
        path,
476
        client,
477
      });
478
    } else {
479
      syncdoc = new SyncDB({
480
        ...doctype.opts,
481
        project_id,
482
        path,
483
        client,
484
      });
485
    }
486
    openDocs[path] = syncdoc;
487

488
    syncdoc.on("error", (err) => {
489
      closeDoc(path);
490
      openFiles?.setError(path, err);
491
      logger.debug(`syncdoc error -- ${err}`, path);
492
    });
493

494
    // Extra backend support in some cases, e.g., Jupyter, Sage, etc.
495
    const ext = filename_extension(path);
496
    switch (ext) {
497
      case JUPYTER_SYNCDB_EXTENSIONS:
498
        logger.debug("initializing Jupyter backend for ", path);
499
        await initJupyterRedux(syncdoc, client);
500
        const path1 = original_path(syncdoc.get_path());
501
        syncdoc.on("closed", async () => {
502
          logger.debug("removing Jupyter backend for ", path1);
503
          await removeJupyterRedux(path1, project_id);
504
        });
505
        break;
506
    }
507
  } finally {
508
    if (openDocs[path] != null) {
509
      openFiles?.setBackend(path, compute_server_id);
510
    }
511
  }
512
});
513

514