1
/*
2
Testing basic ops with dkv
3

4
DEVELOPMENT:
5

6
pnpm test ./dkv.test.ts
7

8
*/
9

10
import { dkv as createDkv } from "@cocalc/backend/conat/sync";
11
import { once } from "@cocalc/util/async-utils";
12
import { delay } from "awaiting";
13
import { before, after, connect } from "@cocalc/backend/conat/test/setup";
14
import { wait } from "@cocalc/backend/conat/test/util";
15

16
beforeAll(before);
17

18
describe("create a public dkv and do basic operations", () => {
19
  let kv;
20
  const name = `test-${Math.random()}`;
21

22
  it("creates the dkv", async () => {
23
    kv = await createDkv({ name, noCache: true });
24
    expect(kv.getAll()).toEqual({});
25
  });
26

27
  it("adds a key to the dkv", () => {
28
    kv.a = 10;
29
    expect(kv.a).toEqual(10);
30
  });
31

32
  it("waits for the dkv to be longterm saved, then closing and recreates the kv and verifies that the key is there.", async () => {
33
    await kv.save();
34
    kv.close();
35
    kv = await createDkv({ name, noCache: true });
36
    expect(kv.a).toEqual(10);
37
  });
38

39
  it("closes the kv", async () => {
40
    await kv.clear();
41
    await kv.close();
42
    expect(() => kv.getAll()).toThrow("closed");
43
  });
44
});
45

46
describe("opens a dkv twice and verifies the cache works and is reference counted", () => {
47
  let kv1;
48
  let kv2;
49
  const name = `test-${Math.random()}`;
50

51
  it("creates the same dkv twice", async () => {
52
    kv1 = await createDkv({ name });
53
    kv2 = await createDkv({ name });
54
    expect(kv1.getAll()).toEqual({});
55
    expect(kv1 === kv2).toBe(true);
56
  });
57

58
  it("closes kv1 (one reference)", async () => {
59
    await kv1.close();
60
    expect(kv2.getAll).not.toThrow();
61
  });
62

63
  it("closes kv2 (another reference)", async () => {
64
    await kv2.close();
65
    await delay(1);
66
    // really closed!
67
    expect(() => kv2.getAll()).toThrow("closed");
68
  });
69

70
  it("create and see it is new now", async () => {
71
    kv1 = await createDkv({ name });
72
    expect(kv1 === kv2).toBe(false);
73
  });
74
});
75

76
describe("opens a dkv twice at once and observe sync", () => {
77
  let kv1;
78
  let kv2;
79
  const name = `test-${Math.random()}`;
80

81
  it("creates the dkv twice", async () => {
82
    kv1 = await createDkv({ name, noCache: true, noAutosave: true });
83
    kv2 = await createDkv({ name, noCache: true, noAutosave: true });
84
    expect(kv1.getAll()).toEqual({});
85
    expect(kv2.getAll()).toEqual({});
86
    expect(kv1 === kv2).toBe(false);
87
  });
88

89
  it("sets a value in one and sees that it is NOT instantly set in the other", () => {
90
    kv1.a = 25;
91
    expect(kv2.a).toBe(undefined);
92
  });
93

94
  it("awaits save and then sees the value *eventually* appears in the other", async () => {
95
    // not initially not there.
96
    expect(kv2.a).toBe(undefined);
97
    await kv1.save();
98
    if (kv2.a === undefined) {
99
      await once(kv2, "change");
100
    }
101
    expect(kv2.a).toBe(kv1.a);
102
  });
103

104
  it("close up", async () => {
105
    await kv1.clear();
106
    await kv1.close();
107
    await kv2.clear();
108
    await kv2.close();
109
  });
110
});
111

112
describe("check server assigned times", () => {
113
  let kv;
114
  const name = `test-${Math.random()}`;
115

116
  it("create a kv", async () => {
117
    kv = await createDkv({ name });
118
    expect(kv.getAll()).toEqual({});
119
    expect(kv.time()).toEqual({});
120
  });
121

122
  it("set a key, then get the time and confirm it is reasonable", async () => {
123
    kv.a = { b: 7 };
124
    // not serve assigned yet
125
    expect(kv.time("a")).toEqual(undefined);
126
    kv.save();
127
    await once(kv, "change");
128
    // now we must have it.
129
    // sanity check: within a second
130
    expect(kv.time("a").getTime()).toBeCloseTo(Date.now(), -3);
131
    // all the times
132
    expect(Object.keys(kv.time()).length).toBe(1);
133
  });
134

135
  it("setting again with a *different* value changes the time", async () => {
136
    kv.a = { b: 8 };
137
    const t0 = kv.time("a");
138
    await once(kv, "change");
139
    expect(kv.time("a").getTime()).toBeCloseTo(Date.now(), -3);
140
    expect(t0).not.toEqual(kv.time("a"));
141
  });
142

143
  it("close", async () => {
144
    await kv.clear();
145
    await kv.close();
146
  });
147
});
148

149
describe("test deleting and clearing a dkv", () => {
150
  let kv1;
151
  let kv2;
152

153
  const reset = async () => {
154
    const name = `test-${Math.random()}`;
155
    kv1 = await createDkv({ name, noCache: true });
156
    kv2 = await createDkv({ name, noCache: true });
157
  };
158

159
  it("creates the dkv twice without caching so can make sure sync works", async () => {
160
    await reset();
161
    expect(kv1.getAll()).toEqual({});
162
    expect(kv2.getAll()).toEqual({});
163
    expect(kv1 === kv2).toBe(false);
164
  });
165

166
  it("adds an entry, deletes it and confirms", async () => {
167
    kv1.foo = "bar";
168
    expect(kv1.has("foo")).toBe(true);
169
    expect(kv2.has("foo")).toBe(false);
170
    await once(kv2, "change");
171
    expect(kv2.foo).toBe(kv1.foo);
172
    expect(kv2.has("foo")).toBe(true);
173
    delete kv1.foo;
174
    await once(kv2, "change");
175
    expect(kv2.foo).toBe(undefined);
176
    expect(kv2.has("foo")).toBe(false);
177
  });
178

179
  it("adds an entry, clears it and confirms", async () => {
180
    await reset();
181

182
    kv1.foo10 = "bar";
183
    await once(kv2, "change");
184
    expect(kv2.foo10).toBe(kv1.foo10);
185
    kv2.clear();
186
    expect(kv2.has("foo10")).toBe(false);
187
    await once(kv1, "change");
188
    while (kv1.has("foo10")) {
189
      await once(kv1, "change");
190
    }
191
    expect(kv1.has("foo10")).toBe(false);
192
  });
193

194
  it("adds an entry, syncs, adds another local entry (not sync'd), clears in sync and confirms NOT everything was cleared", async () => {
195
    await reset();
196
    kv1["foo"] = Math.random();
197
    await kv1.save();
198
    if (kv2["foo"] != kv1["foo"]) {
199
      await once(kv2, "change");
200
    }
201
    expect(kv2["foo"]).toBe(kv1["foo"]);
202
    kv1["bar"] = "yyy";
203
    expect(kv2["bar"]).toBe(undefined);
204
    // this ONLY clears 'foo', not 'bar'
205
    kv2.clear();
206
    await once(kv1, "change");
207
    expect(kv1.has("bar")).toBe(true);
208
  });
209

210
  it("adds an entry, syncs, adds another local entry (not sync'd), clears in first one, and confirms everything was cleared", async () => {
211
    await reset();
212

213
    const key = Math.random();
214
    kv1[key] = Math.random();
215
    await kv1.save();
216
    if (kv2[key] != kv1[key]) {
217
      await once(kv2, "change");
218
    }
219
    const key2 = Math.random();
220
    kv1[key2] = "yyy";
221
    expect(kv2[key2]).toBe(undefined);
222
    // this ONLY clears foo, not xxx
223
    kv1.clear();
224
    expect(kv1.has(key2)).toBe(false);
225
  });
226

227
  it("close", async () => {
228
    await kv1.clear();
229
    await kv1.close();
230
    await kv2.clear();
231
    await kv2.close();
232
  });
233
});
234

235
describe("set several items, confirm write worked, save, and confirm they are still there after save", () => {
236
  const name = `test-${Math.random()}`;
237
  const count = 50;
238
  // the time thresholds should be "trivial"
239
  it(`adds ${count} entries`, async () => {
240
    const kv = await createDkv({ name });
241
    expect(kv.getAll()).toEqual({});
242
    const obj: any = {};
243
    const t0 = Date.now();
244
    for (let i = 0; i < count; i++) {
245
      obj[`${i}`] = i;
246
      kv.set(`${i}`, i);
247
    }
248
    expect(Date.now() - t0).toBeLessThan(50);
249
    expect(Object.keys(kv.getAll()).length).toEqual(count);
250
    expect(kv.getAll()).toEqual(obj);
251
    await kv.save();
252
    expect(Date.now() - t0).toBeLessThan(2000);
253
    await wait({ until: () => Object.keys(kv.getAll()).length == count });
254
    expect(Object.keys(kv.getAll()).length).toEqual(count);
255
    //     // the local state maps should also get cleared quickly,
256
    //     // but there is no event for this, so we loop:
257
    //  @ts-ignore: saved is private
258
    while (Object.keys(kv.local).length > 0) {
259
      await delay(10);
260
    }
261
    // @ts-ignore: local is private
262
    expect(kv.local).toEqual({});
263
    // @ts-ignore: saved is private
264
    expect(kv.saved).toEqual({});
265

266
    await kv.clear();
267
    await kv.close();
268
  });
269
});
270

271
describe("do an insert and clear test", () => {
272
  const name = `test-${Math.random()}`;
273
  const count = 25;
274
  it(`adds ${count} entries, saves, clears, and confirms empty`, async () => {
275
    const kv = await createDkv({ name });
276
    expect(kv.getAll()).toEqual({});
277
    for (let i = 0; i < count; i++) {
278
      kv[`${i}`] = i;
279
    }
280
    expect(Object.keys(kv.getAll()).length).toEqual(count);
281
    await kv.save();
282
    await wait({ until: () => Object.keys(kv.getAll()).length == count });
283
    expect(Object.keys(kv.getAll()).length).toEqual(count);
284
    kv.clear();
285
    expect(kv.getAll()).toEqual({});
286
    await kv.save();
287
    await wait({ until: () => Object.keys(kv.getAll()).length == 0 });
288
    expect(kv.getAll()).toEqual({});
289
  });
290
});
291

292
describe("create many distinct clients at once, write to all of them, and see that that results are merged", () => {
293
  const name = `test-${Math.random()}`;
294
  const count = 4;
295
  const clients: any[] = [];
296

297
  it(`creates the ${count} clients`, async () => {
298
    for (let i = 0; i < count; i++) {
299
      clients[i] = await createDkv({ name, noCache: true });
300
    }
301
  });
302

303
  // what the combination should be
304
  let combined: any = {};
305
  it("writes a separate key/value for each client", () => {
306
    for (let i = 0; i < count; i++) {
307
      clients[i].set(`${i}`, i);
308
      combined[`${i}`] = i;
309
      expect(clients[i].get(`${i}`)).toEqual(i);
310
    }
311
  });
312

313
  it("saves and checks that everybody has the combined values", async () => {
314
    for (const kv of clients) {
315
      await kv.save();
316
    }
317
    let done = false;
318
    let i = 0;
319
    while (!done && i < 50) {
320
      done = true;
321
      i += 1;
322
      for (const client of clients) {
323
        if (client.length != count) {
324
          done = false;
325
          await delay(10);
326
          break;
327
        }
328
      }
329
    }
330
    for (const client of clients) {
331
      expect(client.length).toEqual(count);
332
      expect(client.getAll()).toEqual(combined);
333
    }
334
  });
335

336
  it("close", async () => {
337
    for (const client of clients) {
338
      await client.clear();
339
      await client.close();
340
    }
341
  });
342
});
343

344
describe("tests involving null/undefined values and delete", () => {
345
  let kv1;
346
  let kv2;
347
  const name = `test-${Math.round(100 * Math.random())}`;
348

349
  it("creates the dkv twice", async () => {
350
    kv1 = await createDkv({ name, noAutosave: true, noCache: true });
351
    kv2 = await createDkv({ name, noAutosave: true, noCache: true });
352
    expect(kv1.getAll()).toEqual({});
353
    expect(kv1 === kv2).toBe(false);
354
  });
355

356
  //   it("sets a value to null, which is fully supported like any other value", () => {
357
  //     kv1.a = null;
358
  //     expect(kv1.a).toBe(null);
359
  //     expect(kv1.a === null).toBe(true);
360
  //     expect(kv1.length).toBe(1);
361
  //   });
362

363
  it("make sure null value sync's as expected", async () => {
364
    kv1.a = null;
365
    await kv1.save();
366
    await wait({ until: () => kv2.has("a") });
367
    expect(kv2.a).toBe(null);
368
    expect(kv2.a === null).toBe(true);
369
    expect(kv2.length).toBe(1);
370
  });
371

372
  it("sets a value to undefined, which is the same as deleting a value", () => {
373
    kv1.a = undefined;
374
    expect(kv1.has("a")).toBe(false);
375
    expect(kv1.a).toBe(undefined);
376
    expect(kv1.a === undefined).toBe(true);
377
    expect(kv1.length).toBe(0);
378
    expect(kv1.getAll()).toEqual({});
379
  });
380

381
  it("make sure undefined (i.e., delete) sync's as expected", async () => {
382
    await kv1.save();
383
    await wait({ until: () => kv2.a === undefined });
384
    expect(kv2.a).toBe(undefined);
385
    expect(kv2.a === undefined).toBe(true);
386
    expect(kv2.length).toBe(0);
387
    expect(kv1.getAll()).toEqual({});
388
  });
389

390
  it("makes sure using '' as a key works", async () => {
391
    kv1.set("", 10);
392
    expect(kv1.get("")).toEqual(10);
393
    await kv1.save();
394
    if (kv2.get("") != 10) {
395
      await once(kv2, "change");
396
    }
397
    expect(kv2.get("")).toEqual(10);
398
  });
399

400
  it("close", async () => {
401
    await kv1.clear();
402
    await kv1.close();
403
    await kv2.clear();
404
    await kv2.close();
405
  });
406
});
407

408
describe("ensure there isn't a really obvious subscription leak", () => {
409
  let client;
410

411
  it("create a client, which initially has only one subscription (the inbox)", async () => {
412
    client = connect();
413
    await client.getInbox();
414
    expect(client.numSubscriptions()).toBe(1);
415
  });
416

417
  const count = 10;
418
  it(`creates and closes ${count} streams and checks there is no leak`, async () => {
419
    const before = client.numSubscriptions();
420
    // create
421
    const a: any = [];
422
    for (let i = 0; i < count; i++) {
423
      a[i] = await createDkv({
424
        name: `${Math.random()}`,
425
        noAutosave: true,
426
        noCache: true,
427
      });
428
    }
429
    // NOTE: in fact there's very unlikely to be a subscription leak, since
430
    // dkv's don't use new subscriptions -- they all use requestMany instead
431
    // to a common inbox prefix, and there's just one subscription for an inbox.
432
    expect(client.numSubscriptions()).toEqual(before);
433
    for (let i = 0; i < count; i++) {
434
      await a[i].close();
435
    }
436
    const after = client.numSubscriptions();
437
    expect(after).toBe(before);
438

439
    // also check count on server went down.
440
    expect((await client.getSubscriptions()).size).toBe(before);
441
  });
442

443
  it("does another leak test, but with a set operation each time", async () => {
444
    const before = client.numSubscriptions();
445
    // create
446
    const a: any = [];
447
    for (let i = 0; i < count; i++) {
448
      a[i] = await createDkv({
449
        name: `${Math.random()}`,
450
        noAutosave: true,
451
        noCache: true,
452
      });
453
      a[i].set(`${i}`, i);
454
      await a[i].save();
455
    }
456
    // this isn't going to be a problem:
457
    expect(client.numSubscriptions()).toEqual(before);
458
    for (let i = 0; i < count; i++) {
459
      await a[i].close();
460
    }
461
    const after = client.numSubscriptions();
462
    expect(after).toBe(before);
463
  });
464
});
465

466
describe("test creating and closing a dkv doesn't leak subscriptions", () => {
467
  let client;
468
  let kv;
469
  let name = "sub.count";
470
  let subs;
471

472
  it("make a new client and count subscriptions", async () => {
473
    client = connect();
474
    await once(client, "connected");
475
    await client.getInbox();
476
    subs = client.numSubscriptions();
477
    expect(subs).toBe(1); // the inbox
478
  });
479

480
  it("creates dkv", async () => {
481
    kv = await createDkv({ name });
482
    kv.set("x", 5);
483
    await wait({ until: () => kv.length == 1 });
484
  });
485

486
  it("close the kv and confirm subs returns to 1", async () => {
487
    kv.close();
488
    await expect(() => {
489
      client.numSubscriptions() == 1;
490
    });
491
  });
492
});
493

494
afterAll(after);
495

496