1
import getLogger from "@cocalc/backend/logger";
2
import type { PostgreSQL } from "./types";
3
import getPool from "@cocalc/database/pool";
4
import { callback2 } from "@cocalc/util/async-utils";
5
import { minutes_ago } from "@cocalc/util/misc";
6
import { uuidsha1 } from "@cocalc/backend/misc_node";
7
import { delete_patches } from "./delete-patches";
8

9
const logger = getLogger("database:blobs");
10

11
/*
12
archivePatches:  Offlines and archives the patches, unless the string is active very recently, in
13
which case this is a no-op.  This removes all patches from the database for this syncstring and
14
puts them in a single blob, then stores the id of that blob in the archived field of the syncstrings
15
table.
16

17
TODO: this ignores all syncstrings marked as "huge:true", because the patches are too large.
18
      come up with a better strategy (incremental?) to generate the blobs to avoid the problem.
19
*/
20

21
export async function archivePatches({
22
  db,
23
  string_id,
24
  compress = "zlib",
25
  level = -1,
26
  cutoff = minutes_ago(30),
27
}: {
28
  db: PostgreSQL;
29
  string_id: string;
30
  compress?: string;
31
  level?: number;
32
  cutoff?: Date;
33
}) {
34
  const dbg = (...args) => {
35
    logger.debug("archivePatches", { string_id }, ...args);
36
  };
37
  dbg("get syncstring info");
38
  const pool = getPool();
39
  const { rows: syncstrings } = await pool.query(
40
    "SELECT project_id, archived, last_active, huge FROM syncstrings WHERE string_id=$1",
41
    [string_id],
42
  );
43
  if (syncstrings.length == 0) {
44
    throw Error(`no syncstring with id '${string_id}`);
45
  }
46
  const { project_id, archived, last_active, huge } = syncstrings[0];
47
  if (archived) {
48
    throw Error(
49
      `string_id='#{opts.string_id}' already archived as blob id '${archived}'`,
50
    );
51
  }
52
  if (last_active && last_active >= cutoff) {
53
    dbg("excluding due to cutoff");
54
    return;
55
  }
56
  if (huge) {
57
    dbg("excluding due to being huge");
58
    return;
59
  }
60
  dbg("get patches");
61
  const patches = await exportPatches(string_id);
62
  dbg("create blob from patches");
63
  let blob;
64
  try {
65
    blob = Buffer.from(JSON.stringify({ patches, string_id }));
66
  } catch (err) {
67
    // This might happen if the total length of all patches is too big.
68
    // need to break patches up...
69
    // This is not exactly the end of the world as the entire point of all this is to
70
    // just save some space in the database.
71
    await pool.query("UPDATE syncstrings SET huge=true WHERE string_id=$1", [
72
      string_id,
73
    ]);
74
    return;
75
  }
76
  const uuid = uuidsha1(blob);
77
  dbg("save blob", uuid);
78
  await callback2(db.save_blob, {
79
    uuid,
80
    blob,
81
    project_id,
82
    compress,
83
    level,
84
  });
85
  dbg("update syncstring to indicate patches have been archived in a blob");
86
  await pool.query("UPDATE syncstrings SET archived=$1 WHERE string_id=$2", [
87
    uuid,
88
    string_id,
89
  ]);
90
  dbg("delete patches");
91
  await delete_patches({ db, string_id });
92
}
93

94
export async function exportPatches(string_id: string) {
95
  const pool = getPool();
96
  const { rows } = await pool.query(
97
    "SELECT * FROM patches WHERE string_id=$1",
98
    [string_id],
99
  );
100
  return rows;
101
}
102

103

104