1
/*
2
Keep track of open files.
3

4
We use the "dko" distributed key:value store because of the potential of merge
5
conflicts, e.g,. one client changes the compute server id and another changes
6
whether a file is deleted.  By using dko, only the field that changed is sync'd
7
out, so we get last-write-wins on the level of fields.
8

9
WARNINGS:
10
An old version use dkv with merge conflict resolution, but with multiple clients
11
and the project, feedback loops or something happened and it would start getting
12
slow -- basically, merge conflicts could take a few seconds to resolve, which would
13
make opening a file start to be slow.  Instead we use DKO data type, where fields
14
are treated separately atomically by the storage system.  A *subtle issue* is
15
that when you set an object, this is NOT treated atomically.  E.g., if you 
16
set 2 fields in a set operation, then 2 distinct changes are emitted as the
17
two fields get set. 
18

19
DEVELOPMENT:
20

21
Change to packages/backend, since packages/conat doesn't have a way to connect:
22

23
~/cocalc/src/packages/backend$ node
24

25
> z = await require('@cocalc/backend/conat/sync').openFiles({project_id:cc.current().project_id})
26
> z.touch({path:'a.txt'})
27
> z.get({path:'a.txt'})
28
{ open: true, count: 1, time:2025-02-09T16:37:20.713Z }
29
> z.touch({path:'a.txt'})
30
> z.get({path:'a.txt'})
31
{ open: true, count: 2 }
32
> z.time({path:'a.txt'})
33
2025-02-09T16:36:58.510Z
34
> z.touch({path:'foo/b.md',id:0})
35
> z.getAll()
36
{
37
  'a.txt': { open: true, count: 3 },
38
  'foo/b.md': { open: true, count: 1 }
39

40
Frontend Dev in browser:
41

42
z = await cc.client.conat_client.openFiles({project_id:cc.current().project_id))
43
z.getAll()
44
}
45
*/
46

47
import { type State } from "@cocalc/conat/types";
48
import { dko, type DKO } from "@cocalc/conat/sync/dko";
49
import { EventEmitter } from "events";
50
import getTime, { getSkew } from "@cocalc/conat/time";
51

52
// info about interest in open files (and also what was explicitly deleted) older
53
// than this is automatically purged.
54
const MAX_AGE_MS = 1000 * 60 * 60 * 24;
55

56
interface Deleted {
57
  // what deleted state is
58
  deleted: boolean;
59
  // when deleted state set
60
  time: number;
61
}
62

63
interface Backend {
64
  // who has it opened -- the compute_server_id (0 for project)
65
  id: number;
66
  // when they last reported having it opened
67
  time: number;
68
}
69

70
export interface KVEntry {
71
  // a web browser has the file open at this point in time (in ms)
72
  time?: number;
73
  // if the file was removed from disk (and not immmediately written back),
74
  // then deleted gets set to the time when this happened (in ms since epoch)
75
  // and the file is closed on the backend.  It won't be re-opened until
76
  // either (1) the file is created on disk again, or (2) deleted is cleared.
77
  // Note: the actual time here isn't really important -- what matter is the number
78
  // is nonzero.  It's just used for a display to the user.
79
  // We store the deleted state *and* when this was set, so that in case
80
  // of merge conflict we can do something sensible.
81
  deleted?: Deleted;
82

83
  // if file is actively opened on a compute server/project, then it sets
84
  // this entry.  Right when it closes the file, it clears this.
85
  // If it gets killed/broken and doesn't have a chance to clear it, then
86
  // backend.time can be used to decide this isn't valid.
87
  backend?: Backend;
88

89
  // optional information
90
  doctype?;
91
}
92

93
export interface Entry extends KVEntry {
94
  // path to file relative to HOME
95
  path: string;
96
}
97

98
interface Options {
99
  project_id: string;
100
  noAutosave?: boolean;
101
  noCache?: boolean;
102
}
103

104
export async function createOpenFiles(opts: Options) {
105
  const openFiles = new OpenFiles(opts);
106
  await openFiles.init();
107
  return openFiles;
108
}
109

110
export class OpenFiles extends EventEmitter {
111
  private project_id: string;
112
  private noCache?: boolean;
113
  private noAutosave?: boolean;
114
  private kv?: DKO;
115
  public state: "disconnected" | "connected" | "closed" = "disconnected";
116

117
  constructor({ project_id, noAutosave, noCache }: Options) {
118
    super();
119
    if (!project_id) {
120
      throw Error("project_id must be specified");
121
    }
122
    this.project_id = project_id;
123
    this.noAutosave = noAutosave;
124
    this.noCache = noCache;
125
  }
126

127
  private setState = (state: State) => {
128
    this.state = state;
129
    this.emit(state);
130
  };
131

132
  private initialized = false;
133
  init = async () => {
134
    if (this.initialized) {
135
      throw Error("init can only be called once");
136
    }
137
    this.initialized = true;
138
    const d = await dko<KVEntry>({
139
      name: "open-files",
140
      project_id: this.project_id,
141
      config: {
142
        max_age: MAX_AGE_MS,
143
      },
144
      noAutosave: this.noAutosave,
145
      noCache: this.noCache,
146
      noInventory: true,
147
    });
148
    this.kv = d;
149
    d.on("change", this.handleChange);
150
    // ensure clock is synchronized
151
    await getSkew();
152
    this.setState("connected");
153
  };
154

155
  private handleChange = ({ key: path }) => {
156
    const entry = this.get(path);
157
    if (entry != null) {
158
      // not deleted and timestamp is set:
159
      this.emit("change", entry as Entry);
160
    }
161
  };
162

163
  close = () => {
164
    if (this.kv == null) {
165
      return;
166
    }
167
    this.setState("closed");
168
    this.removeAllListeners();
169
    this.kv.removeListener("change", this.handleChange);
170
    this.kv.close();
171
    delete this.kv;
172
    // @ts-ignore
173
    delete this.project_id;
174
  };
175

176
  private getKv = () => {
177
    const { kv } = this;
178
    if (kv == null) {
179
      throw Error("closed");
180
    }
181
    return kv;
182
  };
183

184
  private set = (path, entry: KVEntry) => {
185
    this.getKv().set(path, entry);
186
  };
187

188
  // When a client has a file open, they should periodically
189
  // touch it to indicate that it is open.
190
  // updates timestamp and ensures open=true.
191
  touch = (path: string, doctype?) => {
192
    if (!path) {
193
      throw Error("path must be specified");
194
    }
195
    const kv = this.getKv();
196
    const cur = kv.get(path);
197
    const time = getTime();
198
    if (doctype) {
199
      this.set(path, {
200
        ...cur,
201
        time,
202
        doctype,
203
      });
204
    } else {
205
      this.set(path, {
206
        ...cur,
207
        time,
208
      });
209
    }
210
  };
211

212
  setError = (path: string, err?: any) => {
213
    const kv = this.getKv();
214
    if (!err) {
215
      const current = { ...kv.get(path) };
216
      delete current.error;
217
      this.set(path, current);
218
    } else {
219
      const current = { ...kv.get(path) };
220
      current.error = { time: Date.now(), error: `${err}` };
221
      this.set(path, current);
222
    }
223
  };
224

225
  setDeleted = (path: string) => {
226
    const kv = this.getKv();
227
    this.set(path, {
228
      ...kv.get(path),
229
      deleted: { deleted: true, time: getTime() },
230
    });
231
  };
232

233
  isDeleted = (path: string) => {
234
    return !!this.getKv().get(path)?.deleted?.deleted;
235
  };
236

237
  setNotDeleted = (path: string) => {
238
    const kv = this.getKv();
239
    this.set(path, {
240
      ...kv.get(path),
241
      deleted: { deleted: false, time: getTime() },
242
    });
243
  };
244

245
  // set that id is the backend with the file open.
246
  // This should be called by that backend periodically
247
  // when it has the file opened.
248
  setBackend = (path: string, id: number) => {
249
    const kv = this.getKv();
250
    this.set(path, {
251
      ...kv.get(path),
252
      backend: { id, time: getTime() },
253
    });
254
  };
255

256
  // get current backend that has file opened.
257
  getBackend = (path: string): Backend | undefined => {
258
    return this.getKv().get(path)?.backend;
259
  };
260

261
  // ONLY if backend for path is currently set to id, then clear
262
  // the backend field.
263
  setNotBackend = (path: string, id: number) => {
264
    const kv = this.getKv();
265
    const cur = { ...kv.get(path) };
266
    if (cur?.backend?.id == id) {
267
      delete cur.backend;
268
      this.set(path, cur);
269
    }
270
  };
271

272
  getAll = (): Entry[] => {
273
    const x = this.getKv().getAll();
274
    return Object.keys(x).map((path) => {
275
      return { ...x[path], path };
276
    });
277
  };
278

279
  get = (path: string): Entry | undefined => {
280
    const x = this.getKv().get(path);
281
    if (x == null) {
282
      return x;
283
    }
284
    return { ...x, path };
285
  };
286

287
  delete = (path) => {
288
    this.getKv().delete(path);
289
  };
290

291
  clear = () => {
292
    this.getKv().clear();
293
  };
294

295
  save = async () => {
296
    await this.getKv().save();
297
  };
298

299
  hasUnsavedChanges = () => {
300
    return this.getKv().hasUnsavedChanges();
301
  };
302
}
303

304