1
/*
2
Manage creating and deleting rolling snapshots of a filesystem.
3

4
We keep track of all state in the sqlite database, so only have to touch
5
ZFS when we actually need to do something.  Keep this in mind though since
6
if you try to mess with snapshots directly then the sqlite database won't
7
know you did that.
8
*/
9

10
import { exec } from "./util";
11
import { get, getRecent, set } from "./db";
12
import { filesystemDataset, filesystemMountpoint } from "./names";
13
import { splitlines } from "@cocalc/util/misc";
14
import getLogger from "@cocalc/backend/logger";
15
import {
16
  SNAPSHOT_INTERVAL_MS,
17
  SNAPSHOT_INTERVALS_MS,
18
  SNAPSHOT_COUNTS,
19
} from "./config";
20
import { syncProperties } from "./properties";
21
import { primaryKey, type PrimaryKey } from "./types";
22
import { isEqual } from "lodash";
23

24
const logger = getLogger("file-server:zfs/snapshots");
25

26
export async function maintainSnapshots(cutoff?: Date) {
27
  await deleteExtraSnapshotsOfActiveFilesystems(cutoff);
28
  await snapshotActiveFilesystems(cutoff);
29
}
30

31
// If there any changes to the filesystem since the last snapshot,
32
// and there are no snapshots since SNAPSHOT_INTERVAL_MS ms ago,
33
// make a new one.  Always returns the most recent snapshot name.
34
// Error if filesystem is archived.
35
export async function createSnapshot({
36
  force,
37
  ifChanged,
38
  ...fs
39
}: PrimaryKey & {
40
  force?: boolean;
41
  // note -- ifChanged is VERY fast, but it's not instantaneous...
42
  ifChanged?: boolean;
43
}): Promise<string> {
44
  logger.debug("createSnapshot: ", fs);
45
  const pk = primaryKey(fs);
46
  const { pool, archived, snapshots } = get(pk);
47
  if (archived) {
48
    throw Error("cannot snapshot an archived filesystem");
49
  }
50
  if (!force && !ifChanged && snapshots.length > 0) {
51
    // check for sufficiently recent snapshot
52
    const last = new Date(snapshots[snapshots.length - 1]);
53
    if (Date.now() - last.valueOf() < SNAPSHOT_INTERVAL_MS) {
54
      // snapshot sufficiently recent
55
      return snapshots[snapshots.length - 1];
56
    }
57
  }
58

59
  // Check to see if nothing change on disk since last snapshot - if so, don't make a new one:
60
  if (!force && snapshots.length > 0) {
61
    const written = await getWritten(pk);
62
    if (written == 0) {
63
      // for sure definitely nothing written, so no possible
64
      // need to make a snapshot
65
      return snapshots[snapshots.length - 1];
66
    }
67
  }
68

69
  const snapshot = new Date().toISOString();
70
  await exec({
71
    verbose: true,
72
    command: "sudo",
73
    args: [
74
      "zfs",
75
      "snapshot",
76
      `${filesystemDataset({ ...pk, pool })}@${snapshot}`,
77
    ],
78
    what: { ...pk, desc: "creating snapshot of project" },
79
  });
80
  set({
81
    ...pk,
82
    snapshots: ({ snapshots }) => [...snapshots, snapshot],
83
  });
84
  syncProperties(pk);
85
  return snapshot;
86
}
87

88
async function getWritten(fs: PrimaryKey) {
89
  const pk = primaryKey(fs);
90
  const { pool } = get(pk);
91
  const { stdout } = await exec({
92
    verbose: true,
93
    command: "zfs",
94
    args: ["list", "-Hpo", "written", filesystemDataset({ ...pk, pool })],
95
    what: {
96
      ...pk,
97
      desc: "getting amount of newly written data in project since last snapshot",
98
    },
99
  });
100
  return parseInt(stdout);
101
}
102

103
export async function zfsGetSnapshots(dataset: string) {
104
  const { stdout } = await exec({
105
    command: "zfs",
106
    args: ["list", "-j", "-o", "name", "-r", "-t", "snapshot", dataset],
107
  });
108
  const snapshots = Object.keys(JSON.parse(stdout).datasets).map(
109
    (name) => name.split("@")[1],
110
  );
111
  return snapshots;
112
}
113

114
// gets snapshots from disk via zfs *and* sets the list of snapshots
115
// in the database to match (and also updates sizes)
116
export async function getSnapshots(fs: PrimaryKey) {
117
  const pk = primaryKey(fs);
118
  const filesystem = get(fs);
119
  const snapshots = await zfsGetSnapshots(filesystemDataset(filesystem));
120
  if (!isEqual(snapshots, filesystem.snapshots)) {
121
    set({ ...pk, snapshots });
122
    syncProperties(fs);
123
  }
124
  return snapshots;
125
}
126

127
export async function deleteSnapshot({
128
  snapshot,
129
  ...fs
130
}: PrimaryKey & { snapshot: string }) {
131
  const pk = primaryKey(fs);
132
  logger.debug("deleteSnapshot: ", pk, snapshot);
133
  const { pool, last_send_snapshot } = get(pk);
134
  if (snapshot == last_send_snapshot) {
135
    throw Error(
136
      "can't delete snapshot since it is the last one used for a zfs send",
137
    );
138
  }
139
  await exec({
140
    verbose: true,
141
    command: "sudo",
142
    args: [
143
      "zfs",
144
      "destroy",
145
      `${filesystemDataset({ ...pk, pool })}@${snapshot}`,
146
    ],
147
    what: { ...pk, desc: "destroying a snapshot of a project" },
148
  });
149
  set({
150
    ...pk,
151
    snapshots: ({ snapshots }) => snapshots.filter((x) => x != snapshot),
152
  });
153
  syncProperties(pk);
154
}
155

156
/*
157
Remove snapshots according to our retention policy, and
158
never delete last_stream if set.
159

160
Returns names of deleted snapshots.
161
*/
162
export async function deleteExtraSnapshots(fs: PrimaryKey): Promise<string[]> {
163
  const pk = primaryKey(fs);
164
  logger.debug("deleteExtraSnapshots: ", pk);
165
  const { last_send_snapshot, snapshots } = get(pk);
166
  if (snapshots.length == 0) {
167
    // nothing to do
168
    return [];
169
  }
170

171
  // sorted from BIGGEST to smallest
172
  const times = snapshots.map((x) => new Date(x).valueOf());
173
  times.reverse();
174
  const save = new Set<number>();
175
  if (last_send_snapshot) {
176
    save.add(new Date(last_send_snapshot).valueOf());
177
  }
178
  for (const type in SNAPSHOT_COUNTS) {
179
    const count = SNAPSHOT_COUNTS[type];
180
    const length_ms = SNAPSHOT_INTERVALS_MS[type];
181

182
    // Pick the first count newest snapshots at intervals of length
183
    // length_ms milliseconds.
184
    let n = 0,
185
      i = 0,
186
      last_tm = 0;
187
    while (n < count && i < times.length) {
188
      const tm = times[i];
189
      if (!last_tm || tm <= last_tm - length_ms) {
190
        save.add(tm);
191
        last_tm = tm;
192
        n += 1; // found one more
193
      }
194
      i += 1; // move to next snapshot
195
    }
196
  }
197
  const toDelete = snapshots.filter((x) => !save.has(new Date(x).valueOf()));
198
  for (const snapshot of toDelete) {
199
    await deleteSnapshot({ ...pk, snapshot });
200
  }
201
  return toDelete;
202
}
203

204
// Go through ALL projects with last_edited >= cutoff stored
205
// here and run trimActiveFilesystemSnapshots.
206
export async function deleteExtraSnapshotsOfActiveFilesystems(cutoff?: Date) {
207
  const v = getRecent({ cutoff });
208
  logger.debug(
209
    `deleteSnapshotsOfActiveFilesystems: considering ${v.length} filesystems`,
210
  );
211
  let i = 0;
212
  for (const fs of v) {
213
    if (fs.archived) {
214
      continue;
215
    }
216
    try {
217
      await deleteExtraSnapshots(fs);
218
    } catch (err) {
219
      logger.debug(`deleteSnapshotsOfActiveFilesystems: error -- ${err}`);
220
    }
221
    i += 1;
222
    if (i % 10 == 0) {
223
      logger.debug(`deleteSnapshotsOfActiveFilesystems: ${i}/${v.length}`);
224
    }
225
  }
226
}
227

228
// Go through ALL projects with last_edited >= cutoff and snapshot them
229
// if they are due a snapshot.
230
// cutoff = a Date (default = 1 week ago)
231
export async function snapshotActiveFilesystems(cutoff?: Date) {
232
  logger.debug("snapshotActiveFilesystems: getting...");
233
  const v = getRecent({ cutoff });
234
  logger.debug(
235
    `snapshotActiveFilesystems: considering ${v.length} projects`,
236
    cutoff,
237
  );
238
  let i = 0;
239
  for (const fs of v) {
240
    if (fs.archived) {
241
      continue;
242
    }
243
    try {
244
      await createSnapshot(fs);
245
    } catch (err) {
246
      // error is already logged in error field of database
247
      logger.debug(`snapshotActiveFilesystems: error -- ${err}`);
248
    }
249
    i += 1;
250
    if (i % 10 == 0) {
251
      logger.debug(`snapshotActiveFilesystems: ${i}/${v.length}`);
252
    }
253
  }
254
}
255

256
/*
257
Get list of files modified since given snapshot (or last snapshot if not given).
258

259
**There's probably no good reason to ever use this code!**
260

261
The reason is because it's really slow, e.g., I added the
262
cocalc src directory (5000) files and it takes about 6 seconds
263
to run this.  In contrast. "time find .", which lists EVERYTHING
264
takes less than 0.074s.  You could do that before and after, then
265
compare them, and it'll be a fraction of a second.
266
*/
267
interface Mod {
268
  time: number;
269
  change: "-" | "+" | "M" | "R"; // remove/create/modify/rename
270
  // see "man zfs diff":
271
  type: "B" | "C" | "/" | ">" | "|" | "@" | "P" | "=" | "F";
272
  path: string;
273
}
274

275
export async function getModifiedFiles({
276
  snapshot,
277
  ...fs
278
}: PrimaryKey & { snapshot: string }) {
279
  const pk = primaryKey(fs);
280
  logger.debug(`getModifiedFiles: `, pk);
281
  const { pool, snapshots } = get(pk);
282
  if (snapshots.length == 0) {
283
    return [];
284
  }
285
  if (snapshot == null) {
286
    snapshot = snapshots[snapshots.length - 1];
287
  }
288
  const { stdout } = await exec({
289
    verbose: true,
290
    command: "sudo",
291
    args: [
292
      "zfs",
293
      "diff",
294
      "-FHt",
295
      `${filesystemDataset({ ...pk, pool })}@${snapshot}`,
296
    ],
297
    what: { ...pk, desc: "getting files modified since last snapshot" },
298
  });
299
  const mnt = filesystemMountpoint(pk) + "/";
300
  const files: Mod[] = [];
301
  for (const line of splitlines(stdout)) {
302
    const x = line.split(/\t/g);
303
    let path = x[3];
304
    if (path.startsWith(mnt)) {
305
      path = path.slice(mnt.length);
306
    }
307
    files.push({
308
      time: parseFloat(x[0]) * 1000,
309
      change: x[1] as any,
310
      type: x[2] as any,
311
      path,
312
    });
313
  }
314
  return files;
315
}
316

317