1
/*
2

3
BUP Architecture:
4

5
There is a single global dedup'd backup archive stored in the btrfs filesystem.
6
Obviously, admins should rsync this regularly to a separate location as a genuine
7
backup strategy.
8

9
NOTE: we use bup instead of btrfs send/recv !
10

11
Not used.  Instead we will rely on bup (and snapshots of the underlying disk) for backups, since:
12
 - much easier to check they are valid
13
 - decoupled from any btrfs issues
14
 - not tied to any specific filesystem at all
15
 - easier to offsite via incremental rsync
16
 - much more space efficient with *global* dedup and compression
17
 - bup is really just git, which is much more proven than even btrfs
18

19
The drawback is speed, but that can be managed.
20
*/
21

22
import { type DirectoryListingEntry } from "@cocalc/util/types";
23
import { type Subvolume } from "./subvolume";
24
import { sudo, parseBupTime } from "./util";
25
import { join, normalize } from "path";
26
import getLogger from "@cocalc/backend/logger";
27

28
const BUP_SNAPSHOT = "temp-bup-snapshot";
29

30
const logger = getLogger("file-server:btrfs:subvolume-bup");
31

32
export class SubvolumeBup {
33
  constructor(private subvolume: Subvolume) {}
34

35
  // create a new bup backup
36
  save = async ({
37
    // timeout used for bup index and bup save commands
38
    timeout = 30 * 60 * 1000,
39
  }: { timeout?: number } = {}) => {
40
    if (await this.subvolume.snapshots.exists(BUP_SNAPSHOT)) {
41
      logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`);
42
      await this.subvolume.snapshots.delete(BUP_SNAPSHOT);
43
    }
44
    try {
45
      logger.debug(
46
        `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`,
47
      );
48
      await this.subvolume.snapshots.create(BUP_SNAPSHOT);
49
      const target = this.subvolume.normalize(
50
        this.subvolume.snapshots.path(BUP_SNAPSHOT),
51
      );
52

53
      logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`);
54
      await sudo({
55
        command: "bup",
56
        args: [
57
          "-d",
58
          this.subvolume.filesystem.bup,
59
          "index",
60
          "--exclude",
61
          join(target, ".snapshots"),
62
          "-x",
63
          target,
64
        ],
65
        timeout,
66
      });
67

68
      logger.debug(`createBackup: saving ${BUP_SNAPSHOT}`);
69
      await sudo({
70
        command: "bup",
71
        args: [
72
          "-d",
73
          this.subvolume.filesystem.bup,
74
          "save",
75
          "--strip",
76
          "-n",
77
          this.subvolume.name,
78
          target,
79
        ],
80
        timeout,
81
      });
82
    } finally {
83
      logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`);
84
      await this.subvolume.snapshots.delete(BUP_SNAPSHOT);
85
    }
86
  };
87

88
  restore = async (path: string) => {
89
    // path -- branch/revision/path/to/dir
90
    if (path.startsWith("/")) {
91
      path = path.slice(1);
92
    }
93
    path = normalize(path);
94
    // ... but to avoid potential data loss, we make a snapshot before deleting it.
95
    await this.subvolume.snapshots.create();
96
    const i = path.indexOf("/"); // remove the commit name
97
    // remove the target we're about to restore
98
    await this.subvolume.fs.rm(path.slice(i + 1), { recursive: true });
99
    await sudo({
100
      command: "bup",
101
      args: [
102
        "-d",
103
        this.subvolume.filesystem.bup,
104
        "restore",
105
        "-C",
106
        this.subvolume.path,
107
        join(`/${this.subvolume.name}`, path),
108
        "--quiet",
109
      ],
110
    });
111
  };
112

113
  ls = async (path: string = ""): Promise<DirectoryListingEntry[]> => {
114
    if (!path) {
115
      const { stdout } = await sudo({
116
        command: "bup",
117
        args: ["-d", this.subvolume.filesystem.bup, "ls", this.subvolume.name],
118
      });
119
      const v: DirectoryListingEntry[] = [];
120
      let newest = 0;
121
      for (const x of stdout.trim().split("\n")) {
122
        const name = x.split(" ").slice(-1)[0];
123
        if (name == "latest") {
124
          continue;
125
        }
126
        const mtime = parseBupTime(name).valueOf() / 1000;
127
        newest = Math.max(mtime, newest);
128
        v.push({ name, isdir: true, mtime });
129
      }
130
      if (v.length > 0) {
131
        v.push({ name: "latest", isdir: true, mtime: newest });
132
      }
133
      return v;
134
    }
135

136
    path = normalize(path);
137
    const { stdout } = await sudo({
138
      command: "bup",
139
      args: [
140
        "-d",
141
        this.subvolume.filesystem.bup,
142
        "ls",
143
        "--almost-all",
144
        "--file-type",
145
        "-l",
146
        join(`/${this.subvolume.name}`, path),
147
      ],
148
    });
149
    const v: DirectoryListingEntry[] = [];
150
    for (const x of stdout.split("\n")) {
151
      // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"]
152
      const w = x.split(/\s+/);
153
      if (w.length >= 6) {
154
        let isdir, name;
155
        if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) {
156
          w[5] = w[5].slice(0, -1);
157
        }
158
        if (w[5].endsWith("/")) {
159
          isdir = true;
160
          name = w[5].slice(0, -1);
161
        } else {
162
          name = w[5];
163
          isdir = false;
164
        }
165
        const size = parseInt(w[2]);
166
        const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000;
167
        v.push({ name, size, mtime, isdir });
168
      }
169
    }
170
    return v;
171
  };
172

173
  prune = async ({
174
    dailies = "1w",
175
    monthlies = "4m",
176
    all = "3d",
177
  }: { dailies?: string; monthlies?: string; all?: string } = {}) => {
178
    await sudo({
179
      command: "bup",
180
      args: [
181
        "-d",
182
        this.subvolume.filesystem.bup,
183
        "prune-older",
184
        `--keep-dailies-for=${dailies}`,
185
        `--keep-monthlies-for=${monthlies}`,
186
        `--keep-all-for=${all}`,
187
        "--unsafe",
188
        this.subvolume.name,
189
      ],
190
    });
191
  };
192
}
193

194