1
/*
2
DEVELOPMENT:
3

4
This tests pull replication by setting up two separate file-servers on disk locally
5
and doing pulls from one to the other over ssh.   This involves password-less ssh 
6
to root on localhost, and creating multiple pools, so use with caution and don't
7
expect this to work unless you really know what you're doing.  
8
Also, these tests are going to take a while. 
9

10
Efficient powerful backup isn't trivial and is very valuable, so
11
its' worth the wait!
12

13
pnpm exec jest --watch pull.test.ts
14
*/
15

16
import { join } from "path";
17
import { createTestPools, deleteTestPools, init, describe } from "./util";
18
import {
19
  createFilesystem,
20
  createSnapshot,
21
  deleteSnapshot,
22
  deleteFilesystem,
23
  pull,
24
  archiveFilesystem,
25
  dearchiveFilesystem,
26
} from "@cocalc/file-server/zfs";
27
import { context, setContext } from "@cocalc/file-server/zfs/config";
28
import { filesystemMountpoint } from "@cocalc/file-server/zfs/names";
29
import { readFile, writeFile } from "fs/promises";
30
import { filesystemExists, get } from "@cocalc/file-server/zfs/db";
31
import { SYNCED_FIELDS } from "../pull";
32

33
describe("create two separate file servers, then do pulls to sync one to the other under various conditions", () => {
34
  let one: any = null,
35
    two: any = null;
36
  const prefix1 = context.PREFIX + ".1";
37
  const prefix2 = context.PREFIX + ".2";
38
  const remote = "root@localhost";
39

40
  beforeAll(async () => {
41
    one = await createTestPools({ count: 1, size: "1G", prefix: prefix1 });
42
    setContext({ prefix: prefix1 });
43
    await init();
44
    two = await createTestPools({
45
      count: 1,
46
      size: "1G",
47
      prefix: prefix2,
48
    });
49
    setContext({ prefix: prefix2 });
50
    await init();
51
  });
52

53
  afterAll(async () => {
54
    await deleteTestPools(one);
55
    await deleteTestPools(two);
56
  });
57

58
  it("creates a filesystem in pool one, writes a file and takes a snapshot", async () => {
59
    setContext({ prefix: prefix1 });
60
    const fs = await createFilesystem({
61
      project_id: "00000000-0000-0000-0000-000000000001",
62
    });
63
    await writeFile(join(filesystemMountpoint(fs), "a.txt"), "hello");
64
    await createSnapshot(fs);
65
    expect(await filesystemExists(fs)).toEqual(true);
66
  });
67

68
  it("pulls filesystem one to filesystem two, and confirms the fs and file were indeed sync'd", async () => {
69
    setContext({ prefix: prefix2 });
70
    expect(
71
      await filesystemExists({
72
        project_id: "00000000-0000-0000-0000-000000000001",
73
      }),
74
    ).toEqual(false);
75

76
    // first dryRun
77
    const { toUpdate, toDelete } = await pull({
78
      remote,
79
      prefix: prefix1,
80
      dryRun: true,
81
    });
82
    expect(toDelete.length).toBe(0);
83
    expect(toUpdate.length).toBe(1);
84
    expect(toUpdate[0].remoteFs.owner_id).toEqual(
85
      "00000000-0000-0000-0000-000000000001",
86
    );
87
    expect(toUpdate[0].localFs).toBe(undefined);
88

89
    // now for real
90
    const { toUpdate: toUpdate1, toDelete: toDelete1 } = await pull({
91
      remote,
92
      prefix: prefix1,
93
    });
94

95
    expect(toDelete1).toEqual(toDelete);
96
    expect(toUpdate1).toEqual(toUpdate);
97
    const fs = { project_id: "00000000-0000-0000-0000-000000000001" };
98
    expect(await filesystemExists(fs)).toEqual(true);
99
    expect(
100
      (await readFile(join(filesystemMountpoint(fs), "a.txt"))).toString(),
101
    ).toEqual("hello");
102

103
    // nothing if we sync again:
104
    const { toUpdate: toUpdate2, toDelete: toDelete2 } = await pull({
105
      remote,
106
      prefix: prefix1,
107
    });
108
    expect(toDelete2.length).toBe(0);
109
    expect(toUpdate2.length).toBe(0);
110
  });
111

112
  it("creates another file in our filesystem, creates another snapshot, syncs again, and sees that the sync worked", async () => {
113
    setContext({ prefix: prefix1 });
114
    const fs = { project_id: "00000000-0000-0000-0000-000000000001" };
115
    await writeFile(join(filesystemMountpoint(fs), "b.txt"), "cocalc");
116
    await createSnapshot({ ...fs, force: true });
117
    const { snapshots } = get(fs);
118
    expect(snapshots.length).toBe(2);
119

120
    setContext({ prefix: prefix2 });
121
    await pull({ remote, prefix: prefix1 });
122

123
    expect(
124
      (await readFile(join(filesystemMountpoint(fs), "b.txt"))).toString(),
125
    ).toEqual("cocalc");
126
  });
127

128
  it("archives the project, does sync, and see the other one got archived", async () => {
129
    const fs = { project_id: "00000000-0000-0000-0000-000000000001" };
130
    setContext({ prefix: prefix2 });
131
    const project2before = get(fs);
132
    expect(project2before.archived).toBe(false);
133

134
    setContext({ prefix: prefix1 });
135
    await archiveFilesystem(fs);
136
    const project1 = get(fs);
137
    expect(project1.archived).toBe(true);
138

139
    setContext({ prefix: prefix2 });
140
    await pull({ remote, prefix: prefix1 });
141
    const project2 = get(fs);
142
    expect(project2.archived).toBe(true);
143
    expect(project1.last_edited).toEqual(project2.last_edited);
144
  });
145

146
  it("dearchives, does sync, then sees the other gets dearchived; this just tests that sync de-archives, but works even if there are no new snapshots", async () => {
147
    const fs = { project_id: "00000000-0000-0000-0000-000000000001" };
148
    setContext({ prefix: prefix1 });
149
    await dearchiveFilesystem(fs);
150
    const project1 = get(fs);
151
    expect(project1.archived).toBe(false);
152

153
    setContext({ prefix: prefix2 });
154
    await pull({ remote, prefix: prefix1 });
155
    const project2 = get(fs);
156
    expect(project2.archived).toBe(false);
157
  });
158

159
  it("archives project, does sync, de-archives project, adds another snapshot, then does sync, thus testing that sync both de-archives *and* pulls latest snapshot", async () => {
160
    const fs = { project_id: "00000000-0000-0000-0000-000000000001" };
161
    setContext({ prefix: prefix1 });
162
    expect(get(fs).archived).toBe(false);
163
    await archiveFilesystem(fs);
164
    expect(get(fs).archived).toBe(true);
165
    setContext({ prefix: prefix2 });
166
    await pull({ remote, prefix: prefix1 });
167
    expect(get(fs).archived).toBe(true);
168

169
    // now dearchive
170
    setContext({ prefix: prefix1 });
171
    await dearchiveFilesystem(fs);
172
    // write content
173
    await writeFile(join(filesystemMountpoint(fs), "d.txt"), "hello");
174
    // snapshot
175
    await createSnapshot({ ...fs, force: true });
176
    const project1 = get(fs);
177

178
    setContext({ prefix: prefix2 });
179
    await pull({ remote, prefix: prefix1 });
180
    const project2 = get(fs);
181
    expect(project2.snapshots).toEqual(project1.snapshots);
182
    expect(project2.archived).toBe(false);
183
  });
184

185
  it("deletes project, does sync, then sees the other does NOT gets deleted without passing the deleteFilesystemCutoff option, and also with deleteFilesystemCutoff an hour ago, but does get deleted with it now", async () => {
186
    const fs = { project_id: "00000000-0000-0000-0000-000000000001" };
187
    setContext({ prefix: prefix1 });
188
    expect(await filesystemExists(fs)).toEqual(true);
189
    await deleteFilesystem(fs);
190
    expect(await filesystemExists(fs)).toEqual(false);
191

192
    setContext({ prefix: prefix2 });
193
    expect(await filesystemExists(fs)).toEqual(true);
194
    await pull({ remote, prefix: prefix1 });
195
    expect(await filesystemExists(fs)).toEqual(true);
196

197
    await pull({
198
      remote,
199
      prefix: prefix1,
200
      deleteFilesystemCutoff: new Date(Date.now() - 1000 * 60 * 60),
201
    });
202
    expect(await filesystemExists(fs)).toEqual(true);
203

204
    await pull({
205
      remote,
206
      prefix: prefix1,
207
      deleteFilesystemCutoff: new Date(),
208
    });
209
    expect(await filesystemExists(fs)).toEqual(false);
210
  });
211

212
  const v = [
213
    { project_id: "00000000-0000-0000-0000-000000000001", affinity: "math" },
214
    {
215
      account_id: "00000000-0000-0000-0000-000000000002",
216
      name: "cocalc",
217
      affinity: "math",
218
    },
219
    {
220
      group_id: "00000000-0000-0000-0000-000000000003",
221
      namespace: "test",
222
      name: "data",
223
      affinity: "sage",
224
    },
225
  ];
226
  it("creates 3 filesystems in 2 different namespaces, and confirms sync works", async () => {
227
    setContext({ prefix: prefix1 });
228
    for (const fs of v) {
229
      await createFilesystem(fs);
230
    }
231
    // write files to fs2 and fs3, so data will get sync'd too
232
    await writeFile(join(filesystemMountpoint(v[1]), "a.txt"), "hello");
233
    await writeFile(join(filesystemMountpoint(v[2]), "b.txt"), "cocalc");
234
    // snapshot
235
    await createSnapshot({ ...v[1], force: true });
236
    await createSnapshot({ ...v[2], force: true });
237
    const p = v.map((x) => get(x));
238

239
    // do the sync
240
    setContext({ prefix: prefix2 });
241
    await pull({ remote, prefix: prefix1 });
242

243
    // verify that we have everything
244
    for (const fs of v) {
245
      expect(await filesystemExists(fs)).toEqual(true);
246
    }
247
    const p2 = v.map((x) => get(x));
248
    for (let i = 0; i < p.length; i++) {
249
      // everything matches (even snapshots, since no trimming happened)
250
      for (const field of SYNCED_FIELDS) {
251
        expect({ i, field, value: p[i][field] }).toEqual({
252
          i,
253
          field,
254
          value: p2[i][field],
255
        });
256
      }
257
    }
258
  });
259

260
  it("edits some files on one of the above filesystems, snapshots, sync's, goes back and deletes a snapshot, edits more files, sync's, and notices that snapshots on sync target properly match snapshots on source.", async () => {
261
    // edits some files on one of the above filesystems, snapshots:
262
    setContext({ prefix: prefix1 });
263
    await writeFile(join(filesystemMountpoint(v[1]), "a2.txt"), "hello2");
264
    await createSnapshot({ ...v[1], force: true });
265

266
    // sync's
267
    setContext({ prefix: prefix2 });
268
    await pull({ remote, prefix: prefix1 });
269

270
    // delete snapshot
271
    setContext({ prefix: prefix1 });
272
    const fs1 = get(v[1]);
273
    await deleteSnapshot({ ...v[1], snapshot: fs1.snapshots[0] });
274

275
    // do more edits and make another snapshot
276
    await writeFile(join(filesystemMountpoint(v[1]), "a3.txt"), "hello3");
277
    await createSnapshot({ ...v[1], force: true });
278
    const snapshots1 = get(v[1]).snapshots;
279

280
    // sync
281
    setContext({ prefix: prefix2 });
282
    await pull({ remote, prefix: prefix1 });
283

284
    // snapshots do NOT initially match, since we didn't enable snapshot deleting!
285
    let snapshots2 = get(v[1]).snapshots;
286
    expect(snapshots1).not.toEqual(snapshots2);
287

288
    await pull({ remote, prefix: prefix1, deleteSnapshots: true });
289
    // now snapshots should match exactly!
290
    snapshots2 = get(v[1]).snapshots;
291
    expect(snapshots1).toEqual(snapshots2);
292
  });
293

294
  it("test directly pulling one filesystem, rather than doing a full sync", async () => {
295
    setContext({ prefix: prefix1 });
296
    await writeFile(join(filesystemMountpoint(v[1]), "a3.txt"), "hello2");
297
    await createSnapshot({ ...v[1], force: true });
298
    await writeFile(join(filesystemMountpoint(v[2]), "a4.txt"), "hello");
299
    await createSnapshot({ ...v[2], force: true });
300
    const p = v.map((x) => get(x));
301

302
    setContext({ prefix: prefix2 });
303
    await pull({ remote, prefix: prefix1, filesystem: v[1] });
304
    const p2 = v.map((x) => get(x));
305

306
    // now filesystem 1 should match, but not filesystem 2
307
    expect(p[1].snapshots).toEqual(p2[1].snapshots);
308
    expect(p[2].snapshots).not.toEqual(p2[2].snapshots);
309

310
    // finally a full sync will get filesystem 2
311
    await pull({ remote, prefix: prefix1 });
312
    const p2b = v.map((x) => get(x));
313
    expect(p[2].snapshots).toEqual(p2b[2].snapshots);
314
  });
315
});
316

317