1
/*
2
Make backups using bup.
3
*/
4

5
import { bupFilesystemMountpoint, filesystemSnapshotMountpoint } from "./names";
6
import { get, getRecent, set } from "./db";
7
import { exec } from "./util";
8
import getLogger from "@cocalc/backend/logger";
9
import { exists } from "@cocalc/backend/misc/async-utils-node";
10
import { join } from "path";
11
import { mountFilesystem } from "./properties";
12
import { split } from "@cocalc/util/misc";
13
import { BUP_INTERVAL_MS } from "./config";
14
import { primaryKey, type PrimaryKey } from "./types";
15
import { createSnapshot } from "./snapshots";
16

17
const logger = getLogger("file-server:zfs:backup");
18

19
const EXCLUDES = [".conda", ".npm", "cache", ".julia", ".local/share/pnpm"];
20

21
export async function createBackup(
22
  fs: PrimaryKey,
23
): Promise<{ BUP_DIR: string }> {
24
  const pk = primaryKey(fs);
25
  logger.debug("createBackup", pk);
26
  const filesystem = get(pk);
27
  await mountFilesystem(pk);
28
  const snapshot = await createSnapshot({ ...filesystem, ifChanged: true });
29
  const mountpoint = filesystemSnapshotMountpoint({ ...filesystem, snapshot });
30
  const excludes: string[] = [];
31
  for (const path of EXCLUDES) {
32
    excludes.push("--exclude");
33
    excludes.push(join(mountpoint, path));
34
  }
35
  logger.debug("createBackup: index", pk);
36
  const BUP_DIR = bupFilesystemMountpoint(filesystem);
37
  if (!(await exists(BUP_DIR))) {
38
    await exec({
39
      verbose: true,
40
      command: "sudo",
41
      args: ["mkdir", "-p", BUP_DIR],
42
      what: { ...pk, desc: "make bup repo" },
43
    });
44
    await exec({
45
      verbose: true,
46
      command: "sudo",
47
      args: ["bup", "-d", BUP_DIR, "init"],
48
      what: { ...pk, desc: "bup init" },
49
    });
50
  }
51
  await exec({
52
    verbose: true,
53
    env: { BUP_DIR },
54
    command: "sudo",
55
    args: [
56
      "--preserve-env",
57
      "bup",
58
      "index",
59
      ...excludes,
60
      "-x",
61
      mountpoint,
62
      "--no-check-device",
63
    ],
64
    what: { ...pk, desc: "creating bup index" },
65
  });
66
  logger.debug("createBackup: save", pk);
67
  await exec({
68
    verbose: true,
69
    env: { BUP_DIR },
70
    command: "sudo",
71
    args: [
72
      "--preserve-env",
73
      "bup",
74
      "save",
75
      "-q",
76
      "--strip",
77
      "-n",
78
      "master",
79
      mountpoint,
80
    ],
81
    what: { ...pk, desc: "save new bup snapshot" },
82
  });
83

84
  const { stdout } = await exec({
85
    env: { BUP_DIR },
86
    command: "sudo",
87
    args: ["--preserve-env", "bup", "ls", "master"],
88
    what: { ...pk, desc: "getting name of backup" },
89
  });
90
  const v = split(stdout);
91
  const last_bup_backup = v[v.length - 2];
92
  logger.debug("createBackup: created ", { last_bup_backup });
93
  set({ ...pk, last_bup_backup });
94

95
  // prune-older --unsafe --keep-all-for 8d --keep-dailies-for 4w --keep-monthlies-for 6m --keep-yearlies-for 10y
96
  logger.debug("createBackup: prune", pk);
97
  await exec({
98
    verbose: true,
99
    env: { BUP_DIR },
100
    command: "sudo",
101
    args: [
102
      "--preserve-env",
103
      "bup",
104
      "prune-older",
105
      "--unsafe",
106
      "--keep-all-for",
107
      "8d",
108
      "--keep-dailies-for",
109
      "4w",
110
      "--keep-monthlies-for",
111
      "6m",
112
      "--keep-yearlies-for",
113
      "5y",
114
    ],
115
    what: { ...pk, desc: "save new bup snapshot" },
116
  });
117

118
  return { BUP_DIR };
119
}
120

121
// Go through ALL filesystems with last_edited >= cutoff and make a bup
122
// backup if they are due.
123
// cutoff = a Date (default = 1 week ago)
124
export async function maintainBackups(cutoff?: Date) {
125
  logger.debug("backupActiveFilesystems: getting...");
126
  const v = getRecent({ cutoff });
127
  logger.debug(
128
    `backupActiveFilesystems: considering ${v.length} filesystems`,
129
    cutoff,
130
  );
131
  let i = 0;
132
  for (const { archived, last_edited, last_bup_backup, ...pk } of v) {
133
    if (archived || !last_edited) {
134
      continue;
135
    }
136
    const age =
137
      new Date(last_edited).valueOf() - bupToDate(last_bup_backup).valueOf();
138
    if (age < BUP_INTERVAL_MS) {
139
      // there's a new backup already
140
      continue;
141
    }
142
    try {
143
      await createBackup(pk);
144
    } catch (err) {
145
      logger.debug(`backupActiveFilesystems: error -- ${err}`);
146
    }
147
    i += 1;
148
    if (i % 10 == 0) {
149
      logger.debug(`backupActiveFilesystems: ${i}/${v.length}`);
150
    }
151
  }
152
}
153

154
function bupToDate(dateString?: string): Date {
155
  if (!dateString) {
156
    return new Date(0);
157
  }
158
  // Extract components using regular expression
159
  const match = dateString.match(
160
    /^(\d{4})-(\d{2})-(\d{2})-(\d{2})(\d{2})(\d{2})$/,
161
  );
162

163
  if (match) {
164
    const [_, year, month, day, hour, minute, second] = match; // Destructure components
165
    return new Date(
166
      parseInt(year),
167
      parseInt(month) - 1,
168
      parseInt(day),
169
      parseInt(hour),
170
      parseInt(minute),
171
      parseInt(second),
172
    );
173
  } else {
174
    throw Error("Invalid bup date format");
175
  }
176
}
177

178