1
/*
2
Where Data is Stored:
3

4
We centralize here determination of all directories on the file system
5
where data is stored for any of the components of CoCalc, run in any way.
6

7
All information here must be determinable when this module is initialized,
8
e.g., from environment variables or heuristics involving the file system.
9
In particular, nothing here can be impacted by command line flags
10
or content of a database.
11
*/
12

13
import Dict = NodeJS.Dict;
14

15
const DEFINITION = `CoCalc Environment Variables:
16
- root -- if COCALC_ROOT is set then it; otherwise use [cocalc-source]/src/.
17
- data -- if the environment variable DATA is set, use that.  Otherwise, use {root}/data
18
- pgdata -- if env var PGDATA is set, use that; otherwise, it is {data}/postgres: where data data is stored (if running locally)
19
- pghost - if env var PGHOST is set, use that; otherwise, it is {data}/postgres/socket: what database connects to
20
- projects -- If env var PROJECTS is set, use that; otherwise, it is {data}"/projects/[project_id]";
21
              This is where project home directories are (or shared files for share server), and it MUST
22
              contain the string "[project_id]".
23
- secrets -- if env var SECRETS is set, use that; otherwise, it is {data}/secrets:  where to store secrets
24
- logs -- if env var LOGS is set, use that; otherwise, {data}/logs:  directory in which to store logs
25
`;
26

27
import { join, resolve } from "path";
28
import { ConnectionOptions } from "node:tls";
29
import { existsSync, mkdirSync, readFileSync } from "fs";
30
import { isEmpty } from "lodash";
31
import basePath from "@cocalc/backend/base-path";
32
import port from "@cocalc/backend/port";
33

34
function determineRootFromPath(): string {
35
  const cur = __dirname;
36
  const search = "/src/";
37
  const i = cur.lastIndexOf(search);
38
  const root = resolve(cur.slice(0, i + search.length - 1));
39
  process.env.COCALC_ROOT = root;
40
  return root;
41
}
42

43
// Each field value in this interface is to be treated as though it originated from a raw
44
// environment variable. These environment variables are used to configure CoCalc's SSL connection
45
// to the database.
46
//
47
interface CoCalcSSLEnvConfig extends Dict<string> {
48
  SMC_DB_SSL?: string;
49
  SMC_DB_SSL_CA_FILE?: string;
50
  SMC_DB_SSL_CLIENT_CERT_FILE?: string;
51
  SMC_DB_SSL_CLIENT_KEY_FILE?: string;
52
  SMC_DB_SSL_CLIENT_KEY_PASSPHRASE?: string;
53
}
54

55
// This interface is used to specify environment variables to be passed to the "psql" command for
56
// SSL configuration.
57
//
58
// See https://www.postgresql.org/docs/current/libpq-envars.html for more information.
59
//
60
export interface PsqlSSLEnvConfig {
61
  // We could also add "verify-ca" here, but it's probably best to assume that we'd like the
62
  // most secure option out of the box. The differences between "verify-ca" and "verify-full"
63
  // can be found here: https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-CLIENTCERT
64
  //
65
  PGSSLMODE?: "verify-full" | "require";
66
  // This typing is redundant but included for clarity.
67
  //
68
  PGSSLROOTCERT?: "system" | string;
69
  PGSSLCERT?: string;
70
  PGSSLKEY?: string;
71
}
72

73
// A full list of property types and SSL config options can be found here:
74
//
75
// http://nodejs.org/api/tls.html#tls_tls_connect_options_callback
76
//
77
// We extend the existing ConnectionOptions interface to include certificate file paths, since these
78
// are used when connecting to Postgres outside of Node (e.g., for raw psql queries).
79
//
80
export type SSLConfig =
81
  | (ConnectionOptions & {
82
      caFile?: string;
83
      clientCertFile?: string;
84
      clientKeyFile?: string;
85
    })
86
  | boolean
87
  | undefined;
88

89
/**
90
 * Converts an environment-variable-driven SSLEnvConfig into a superset of the SSL context expected
91
 * by node when generating SSL connections.
92
 *
93
 * @param env
94
 */
95
export function sslConfigFromCoCalcEnv(
96
  env: CoCalcSSLEnvConfig = process.env,
97
): SSLConfig {
98
  const sslConfig: SSLConfig = {};
99

100
  if (env.SMC_DB_SSL_CA_FILE) {
101
    sslConfig.caFile = env.SMC_DB_SSL_CA_FILE;
102
    sslConfig.ca = readFileSync(env.SMC_DB_SSL_CA_FILE);
103
  }
104

105
  if (env.SMC_DB_SSL_CLIENT_CERT_FILE) {
106
    sslConfig.clientCertFile = env.SMC_DB_SSL_CLIENT_CERT_FILE;
107
    sslConfig.cert = readFileSync(env.SMC_DB_SSL_CLIENT_CERT_FILE);
108
  }
109

110
  if (env.SMC_DB_SSL_CLIENT_KEY_FILE) {
111
    sslConfig.clientKeyFile = env.SMC_DB_SSL_CLIENT_KEY_FILE;
112
    sslConfig.key = readFileSync(env.SMC_DB_SSL_CLIENT_KEY_FILE);
113
  }
114

115
  if (env.SMC_DB_SSL_CLIENT_KEY_PASSPHRASE) {
116
    sslConfig.passphrase = env.SMC_DB_SSL_CLIENT_KEY_PASSPHRASE;
117
  }
118

119
  return isEmpty(sslConfig)
120
    ? env.SMC_DB_SSL?.toLowerCase() === "true"
121
    : sslConfig;
122
}
123

124
/**
125
 * Converts a provided SSLConfig object into (a subset of) its corresponding `psql` environment
126
 * variables. See
127
 *
128
 * http://nodejs.org/api/tls.html#tls_tls_connect_options_callback
129
 *
130
 * for more information about these options.
131
 *
132
 * @param config
133
 */
134
export function sslConfigToPsqlEnv(config: SSLConfig): PsqlSSLEnvConfig {
135
  if (!config) {
136
    return {};
137
  } else if (config === true) {
138
    return {
139
      PGSSLMODE: "require",
140
    };
141
  }
142

143
  // If SSL config is anything other than a boolean, require CA validation
144
  //
145
  const psqlArgs: PsqlSSLEnvConfig = {
146
    PGSSLMODE: "verify-full",
147
  };
148

149
  // Server CA. Uses CA file when provided and system certs otherwise.
150
  //
151
  if (config.caFile) {
152
    psqlArgs.PGSSLROOTCERT = `${config.caFile}`;
153
  } else {
154
    psqlArgs.PGSSLROOTCERT = "system";
155
  }
156

157
  // Client cert
158
  //
159
  if (config.clientCertFile) {
160
    psqlArgs.PGSSLCERT = `${config.clientCertFile}`;
161
  }
162

163
  // Client key
164
  //
165
  if (config.clientKeyFile) {
166
    psqlArgs.PGSSLKEY = `${config.clientKeyFile}`;
167
  }
168

169
  return psqlArgs;
170
}
171

172
export const root: string = process.env.COCALC_ROOT ?? determineRootFromPath();
173
export const data: string = process.env.DATA ?? join(root, "data");
174
export const pguser: string = process.env.PGUSER ?? "smc";
175
export const pgdata: string = process.env.PGDATA ?? join(data, "postgres");
176
export const pghost: string = process.env.PGHOST ?? join(pgdata, "socket");
177
export const pgssl = sslConfigFromCoCalcEnv();
178
export const pgdatabase: string =
179
  process.env.SMC_DB ?? process.env.PGDATABASE ?? "smc";
180
export const projects: string =
181
  process.env.PROJECTS ?? join(data, "projects", "[project_id]");
182
export const secrets: string = process.env.SECRETS ?? join(data, "secrets");
183

184
export const syncFiles = {
185
  // Persistent local storage of streams and kv's as sqlite3 files
186
  local: process.env.COCALC_SYNC ?? join(data, "sync"),
187
  // Archived storage of streams and kv's as sqlite3 files, if set.
188
  // This could be a gcsfuse mountpoint.
189
  archive: process.env.COCALC_SYNC_ARCHIVE ?? "",
190
};
191

192
// if the directory secrets doesn't exist, create it (sync, during this load):
193
if (!existsSync(secrets)) {
194
  try {
195
    // Mode '0o700' allows read/write/execute only for the owner
196
    mkdirSync(secrets, { recursive: true, mode: 0o700 });
197
  } catch {
198
    // non-fatal, e.g., maybe user doesn't even have write access to the secrets path
199
  }
200
}
201

202
export const logs: string = process.env.LOGS ?? join(data, "logs");
203

204
// CONAT server and password
205
export let conatServer =
206
  process.env.CONAT_SERVER ??
207
  `http://localhost:${port}${basePath.length > 1 ? basePath : ""}`;
208
if (conatServer.split("//").length > 2) {
209
  // i make this mistake too much
210
  throw Error(
211
    `env variable CONAT_SERVER invalid -- too many /s' --'${process.env.CONAT_SERVER}'`,
212
  );
213
}
214

215
export function setConatServer(server: string) {
216
  conatServer = server;
217
}
218

219
// Password used by hub (not users or projects) to connect to a Conat server:
220
export let conatPassword = "";
221
export const conatPasswordPath = join(secrets, "conat-password");
222
try {
223
  conatPassword = readFileSync(conatPasswordPath).toString().trim();
224
} catch {}
225
export function setConatPassword(password: string) {
226
  conatPassword = password;
227
}
228

229
export let conatValkey: string = process.env.CONAT_VALKEY ?? "";
230
export function setConatValkey(valkey: string) {
231
  conatValkey = valkey;
232
}
233

234
export let valkeyPassword = "";
235
const valkeyPasswordPath = join(secrets, "valkey-password");
236
try {
237
  valkeyPassword = readFileSync(valkeyPasswordPath).toString().trim();
238
} catch {}
239

240
export let conatSocketioCount = parseInt(
241
  process.env.CONAT_SOCKETIO_COUNT ?? "1",
242
);
243

244
// number of persist servers (if configured to run)
245
export let conatPersistCount = parseInt(process.env.CONAT_PERSIST_COUNT ?? "1");
246

247
// number of api servers (if configured to run)
248
export let conatApiCount = parseInt(process.env.CONAT_API_COUNT ?? "1");
249

250
// if configured, will create a socketio cluster using
251
// the cluster adapter, listening on the given port.
252
// It makes no sense to use both this *and* valkey. It's
253
// one or the other.
254
export let conatClusterPort = parseInt(process.env.CONAT_CLUSTER_PORT ?? "0");
255
// if set, a simple http server will be started listening on conatClusterHealthPort
256
// which returns an error only if the socketio server is not "healthy".
257
export let conatClusterHealthPort = parseInt(
258
  process.env.CONAT_CLUSTER_HEALTH_PORT ?? "0",
259
);
260

261
// API keys
262

263
export let apiKey: string = process.env.API_KEY ?? "";
264
export let apiServer: string = process.env.API_SERVER ?? "";
265

266
// Delete API_KEY from environment to reduce chances of it leaking, e.g., to
267
// spawned terminal subprocess.
268
// Important note: It's critical that only one version of the @cocalc/backend
269
// package is being used, or some parts of the code will get the API_KEY and
270
// others will not.
271
delete process.env.API_KEY;
272

273
export function setApi({ key, server }: { key?: string; server?: string }) {
274
  if (key != null) {
275
    apiKey = key;
276
  }
277
  if (server != null) {
278
    checkApiServer(server);
279
    apiServer = server;
280
  }
281
}
282

283
function sanityChecks() {
284
  // Do a sanity check on projects:
285
  if (!projects.includes("[project_id]")) {
286
    throw Error(
287
      `${DEFINITION}\n\nenv variable PROJECTS must contain "[project_id]" but it is "${process.env.PROJECTS}"`,
288
    );
289
  }
290
  checkApiServer(apiServer);
291
}
292

293
function checkApiServer(server) {
294
  if (!server) return;
295
  if (server.endsWith("/")) {
296
    throw Error("API_SERVER must not end in /");
297
  }
298
  if (!server.startsWith("http://") && !server.startsWith("https://")) {
299
    throw Error("API_SERVER must start with http:// or https://");
300
  }
301
}
302

303
sanityChecks();
304

305