1
//########################################################################
2
// This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
3
// License: MS-RSL – see LICENSE.md for details
4
//########################################################################
5

6
// This is the CoCalc Global HUB.  It runs as a daemon, sitting in the
7
// middle of the action, connected to potentially thousands of clients,
8
// many Sage sessions, and PostgreSQL database.
9

10
import { callback } from "awaiting";
11
import blocked from "blocked";
12
import { spawn } from "child_process";
13
import { program as commander, Option } from "commander";
14
import basePath from "@cocalc/backend/base-path";
15
import {
16
  pghost as DEFAULT_DB_HOST,
17
  pgdatabase as DEFAULT_DB_NAME,
18
  pguser as DEFAULT_DB_USER,
19
} from "@cocalc/backend/data";
20
import { trimLogFileSize } from "@cocalc/backend/logger";
21
import port from "@cocalc/backend/port";
22
import { init_start_always_running_projects } from "@cocalc/database/postgres/always-running";
23
import { load_server_settings_from_env } from "@cocalc/database/settings/server-settings";
24
import { init_passport } from "@cocalc/server/hub/auth";
25
import { initialOnPremSetup } from "@cocalc/server/initial-onprem-setup";
26
import initHandleMentions from "@cocalc/server/mentions/handle";
27
import initMessageMaintenance from "@cocalc/server/messages/maintenance";
28
import initProjectControl, {
29
  COCALC_MODES,
30
} from "@cocalc/server/projects/control";
31
import initIdleTimeout from "@cocalc/server/projects/control/stop-idle-projects";
32
import initNewProjectPoolMaintenanceLoop from "@cocalc/server/projects/pool/maintain";
33
import initPurchasesMaintenanceLoop from "@cocalc/server/purchases/maintenance";
34
import initSalesloftMaintenance from "@cocalc/server/salesloft/init";
35
import { stripe_sync } from "@cocalc/server/stripe/sync";
36
import { callback2, retry_until_success } from "@cocalc/util/async-utils";
37
import { set_agent_endpoint } from "./health-checks";
38
import { start as startHubRegister } from "@cocalc/server/metrics/hub_register";
39
import { getLogger } from "./logger";
40
import initDatabase, { database } from "./servers/database";
41
import initExpressApp from "./servers/express-app";
42
import {
43
  loadConatConfiguration,
44
  initConatChangefeedServer,
45
  initConatApi,
46
  initConatPersist,
47
} from "@cocalc/server/conat";
48
import { initConatServer } from "@cocalc/server/conat/socketio";
49

50
import initHttpRedirect from "./servers/http-redirect";
51

52
import * as MetricsRecorder from "@cocalc/server/metrics/metrics-recorder";
53
import { addErrorListeners } from "@cocalc/server/metrics/error-listener";
54

55
// Logger tagged with 'hub' for this file.
56
const logger = getLogger("hub");
57

58
// program gets populated with the command line options below.
59
let program: { [option: string]: any } = {};
60
export { program };
61

62
const REGISTER_INTERVAL_S = 20;
63

64
async function reset_password(email_address: string): Promise<void> {
65
  try {
66
    await callback2(database.reset_password, { email_address });
67
    logger.info(`Password changed for ${email_address}`);
68
  } catch (err) {
69
    logger.info(`Error resetting password -- ${err}`);
70
  }
71
}
72

73
// This calculates and updates the statistics for the /stats endpoint.
74
// It's important that we call this periodically, because otherwise the /stats data is outdated.
75
async function init_update_stats(): Promise<void> {
76
  logger.info("init updating stats periodically");
77
  const update = () => callback2(database.get_stats);
78
  // Do it every minute:
79
  setInterval(() => update(), 60000);
80
  // Also do it once now:
81
  await update();
82
}
83

84
// This calculates and updates the site_license_usage_log.
85
// It's important that we call this periodically, if we want
86
// to be able to monitor site license usage. This is enabled
87
// by default only for dev mode (so for development).
88
async function init_update_site_license_usage_log() {
89
  logger.info("init updating site license usage log periodically");
90
  const update = async () => await database.update_site_license_usage_log();
91
  setInterval(update, 31000);
92
  await update();
93
}
94

95
async function initMetrics() {
96
  logger.info("Initializing Metrics Recorder...");
97
  MetricsRecorder.init();
98
  return {
99
    metric_blocked: MetricsRecorder.new_counter(
100
      "blocked_ms_total",
101
      'accumulates the "blocked" time in the hub [ms]',
102
    ),
103
  };
104
}
105

106
async function startServer(): Promise<void> {
107
  logger.info("start_server");
108

109
  logger.info(`basePath='${basePath}'`);
110
  logger.info(
111
    `database: name="${program.databaseName}" nodes="${program.databaseNodes}" user="${program.databaseUser}"`,
112
  );
113

114
  const { metric_blocked } = await initMetrics();
115

116
  // Log anything that blocks the CPU for more than ~100ms -- see https://github.com/tj/node-blocked
117
  blocked((ms: number) => {
118
    if (ms > 100) {
119
      metric_blocked.inc(ms);
120
    }
121
    // record that something blocked:
122
    if (ms > 100) {
123
      logger.debug(`BLOCKED for ${ms}ms`);
124
    }
125
  });
126

127
  // Wait for database connection to work.  Everything requires this.
128
  await retry_until_success({
129
    f: async () => await callback2(database.connect),
130
    start_delay: 1000,
131
    max_delay: 10000,
132
  });
133
  logger.info("connected to database.");
134

135
  if (program.updateDatabaseSchema) {
136
    logger.info("Update database schema");
137
    await callback2(database.update_schema);
138

139
    // in those cases where we initialize the database upon startup
140
    // (essentially only relevant for kucalc's hub-websocket)
141
    if (program.mode === "kucalc") {
142
      // and for on-prem setups, also initialize the admin account, set a registration token, etc.
143
      await initialOnPremSetup(database);
144
    }
145
  }
146

147
  // set server settings based on environment variables
148
  await load_server_settings_from_env(database);
149

150
  if (program.agentPort) {
151
    logger.info("Configure agent port");
152
    set_agent_endpoint(program.agentPort, program.hostname);
153
  }
154

155
  // Mentions
156
  if (program.mentions) {
157
    logger.info("enabling handling of mentions...");
158
    initHandleMentions();
159
    logger.info("enabling handling of messaging...");
160
    initMessageMaintenance();
161
  }
162

163
  // Project control
164
  logger.info("initializing project control...");
165
  const projectControl = initProjectControl(program.mode);
166
  // used for nextjs hot module reloading dev server
167
  process.env["COCALC_MODE"] = program.mode;
168

169
  if (program.mode != "kucalc" && program.conatServer) {
170
    // We handle idle timeout of projects.
171
    // This can be disabled via COCALC_NO_IDLE_TIMEOUT.
172
    // This only uses the admin-configurable settings field of projects
173
    // in the database and isn't aware of licenses or upgrades.
174
    initIdleTimeout(projectControl);
175
  }
176

177
  // This loads from the database credentials to use Conat.
178
  await loadConatConfiguration();
179

180
  if (program.conatRouter) {
181
    // launch standalone socketio websocket server (no http server)
182
    await initConatServer({ kucalc: program.mode == "kucalc" });
183
  }
184

185
  if (program.conatApi || program.conatServer) {
186
    await initConatApi();
187
    await initConatChangefeedServer();
188
  }
189

190
  if (program.conatPersist || program.conatServer) {
191
    await initConatPersist();
192
  }
193

194
  if (program.conatServer) {
195
    if (program.mode == "single-user" && process.env.USER == "user") {
196
      // Definitely in dev mode, probably on cocalc.com in a project, so we kill
197
      // all the running projects when starting the hub:
198
      // Whenever we start the dev server, we just assume
199
      // all projects are stopped, since assuming they are
200
      // running when they are not is bad.  Something similar
201
      // is done in cocalc-docker.
202
      logger.info("killing all projects...");
203
      await callback2(database._query, {
204
        safety_check: false,
205
        query: 'update projects set state=\'{"state":"opened"}\'',
206
      });
207
      await spawn("pkill", ["-f", "node_modules/.bin/cocalc-project"]);
208

209
      // Also, unrelated to killing projects, for purposes of developing
210
      // custom software images, we inject a couple of random nonsense entries
211
      // into the table in the DB:
212
      logger.info("inserting random nonsense compute images in database");
213
      await callback2(database.insert_random_compute_images);
214
    }
215

216
    if (program.mode != "kucalc") {
217
      await init_update_stats();
218
      await init_update_site_license_usage_log();
219
      // This is async but runs forever, so don't wait for it.
220
      logger.info("init starting always running projects");
221
      init_start_always_running_projects(database);
222
    }
223
  }
224

225
  if (
226
    program.conatServer ||
227
    program.proxyServer ||
228
    program.nextServer ||
229
    program.conatApi
230
  ) {
231
    const { router, httpServer } = await initExpressApp({
232
      isPersonal: program.personal,
233
      projectControl,
234
      conatServer: !!program.conatServer,
235
      proxyServer: true, // always
236
      nextServer: !!program.nextServer,
237
      cert: program.httpsCert,
238
      key: program.httpsKey,
239
    });
240

241
    // The express app create via initExpressApp above **assumes** that init_passport is done
242
    // or complains a lot. This is obviously not really necessary, but we leave it for now.
243
    await callback2(init_passport, {
244
      router,
245
      database,
246
      host: program.hostname,
247
    });
248

249
    logger.info(`starting webserver listening on ${program.hostname}:${port}`);
250
    await callback(httpServer.listen.bind(httpServer), port, program.hostname);
251

252
    if (port == 443 && program.httpsCert && program.httpsKey) {
253
      // also start a redirect from port 80 to port 443.
254
      await initHttpRedirect(program.hostname);
255
    }
256

257
    logger.info(
258
      "Starting registering periodically with the database and updating a health check...",
259
    );
260

261
    // register the hub with the database periodically, and
262
    // also confirms that database is working.
263
    await callback2(startHubRegister, {
264
      database,
265
      host: program.hostname,
266
      port,
267
      interval_s: REGISTER_INTERVAL_S,
268
    });
269

270
    const protocol = program.httpsKey ? "https" : "http";
271
    const target = `${protocol}://${program.hostname}:${port}${basePath}`;
272

273
    const msg = `Started HUB!\n\n-----------\n\n The following URL *might* work: ${target}\n\n\nPORT=${port}\nBASE_PATH=${basePath}\nPROTOCOL=${protocol}\n\n${
274
      basePath.length <= 1
275
        ? ""
276
        : "If you are developing cocalc inside of cocalc, take the URL of the host cocalc\nand append " +
277
          basePath +
278
          " to it."
279
    }\n\n-----------\n\n`;
280
    logger.info(msg);
281
    console.log(msg);
282
  }
283

284
  if (program.all || program.mentions) {
285
    // kucalc: for now we just have the hub-mentions servers
286
    // do the new project pool maintenance, since there is only
287
    // one hub-stats.
288
    // On non-cocalc it'll get done by *the* hub because of program.all.
289
    initNewProjectPoolMaintenanceLoop();
290
    // Starts periodic maintenance on pay-as-you-go purchases, e.g., quota
291
    // upgrades of projects.
292
    initPurchasesMaintenanceLoop();
293
    initSalesloftMaintenance();
294
    setInterval(trimLogFileSize, 1000 * 60 * 3);
295
  }
296

297
  addErrorListeners();
298
}
299

300
//############################################
301
// Process command line arguments
302
//############################################
303
async function main(): Promise<void> {
304
  commander
305
    .name("cocalc-hub-server")
306
    .usage("options")
307
    .addOption(
308
      new Option(
309
        "--mode [string]",
310
        `REQUIRED mode in which to run CoCalc (${COCALC_MODES.join(
311
          ", ",
312
        )}) - or set COCALC_MODE env var`,
313
      ).choices(COCALC_MODES as any as string[]),
314
    )
315
    .option(
316
      "--all",
317
      "runs all of the servers: websocket, proxy, next (so you don't have to pass all those opts separately), and also mentions updator and updates db schema on startup; use this in situations where there is a single hub that serves everything (instead of a microservice situation like kucalc)",
318
    )
319
    .option(
320
      "--conat-server",
321
      "run a hub that provides a single-core conat server (i.e., conat-router but integrated with the http server), api, and persistence, along with an http server. This is for dev and small deployments of cocalc (and if given, do not bother with --conat-[core|api|persist] below.)",
322
    )
323
    .option(
324
      "--conat-router",
325
      "run a hub that provides the core conat communication layer server over a websocket (but not http server).",
326
    )
327
    .option(
328
      "--conat-api",
329
      "run a hub that connect to conat-router and provides the standard conat API services, e.g., basic api, LLM's, changefeeds, http file upload/download, etc.  There must be at least one of these.   You can increase or decrease the number of these servers with no coordination needed.",
330
    )
331
    .option(
332
      "--conat-persist",
333
      "run a hub that connects to conat-router and provides persistence for streams (e.g., key for sync editing).  There must be at least one of these, and they need access to common shared disk to store sqlite files.  Only one server uses a given sqlite file at a time.  You can increase or decrease the number of these servers with no coordination needed.",
334
    )
335
    .option("--proxy-server", "run a proxy server in this process")
336
    .option(
337
      "--next-server",
338
      "run a nextjs server (landing pages, share server, etc.) in this process",
339
    )
340
    .option(
341
      "--https-key [string]",
342
      "serve over https.  argument should be a key filename (both https-key and https-cert must be specified)",
343
    )
344
    .option(
345
      "--https-cert [string]",
346
      "serve over https.  argument should be a cert filename (both https-key and https-cert must be specified)",
347
    )
348
    .option(
349
      "--agent-port <n>",
350
      "port for HAProxy agent-check (default: 0 -- do not start)",
351
      (n) => parseInt(n),
352
      0,
353
    )
354
    .option(
355
      "--hostname [string]",
356
      'host of interface to bind to (default: "127.0.0.1")',
357
      "127.0.0.1",
358
    )
359
    .option(
360
      "--database-nodes <string,string,...>",
361
      `database address (default: '${DEFAULT_DB_HOST}')`,
362
      DEFAULT_DB_HOST,
363
    )
364
    .option(
365
      "--database-name [string]",
366
      `Database name to use (default: "${DEFAULT_DB_NAME}")`,
367
      DEFAULT_DB_NAME,
368
    )
369
    .option(
370
      "--database-user [string]",
371
      `Database username to use (default: "${DEFAULT_DB_USER}")`,
372
      DEFAULT_DB_USER,
373
    )
374
    .option("--passwd [email_address]", "Reset password of given user", "")
375
    .option(
376
      "--update-database-schema",
377
      "If specified, updates database schema on startup (always happens when mode is not kucalc).",
378
    )
379
    .option(
380
      "--stripe-sync",
381
      "Sync stripe subscriptions to database for all users with stripe id",
382
      "yes",
383
    )
384
    .option(
385
      "--update-stats",
386
      "Calculates the statistics for the /stats endpoint and stores them in the database",
387
      "yes",
388
    )
389
    .option("--delete-expired", "Delete expired data from the database", "yes")
390
    .option(
391
      "--blob-maintenance",
392
      "Do blob-related maintenance (dump to tarballs, offload to gcloud)",
393
      "yes",
394
    )
395
    .option(
396
      "--mentions",
397
      "if given, periodically handle mentions; on kucalc there is only one of these.  It also managed the new project pool.  Maybe this should be renamed --singleton!",
398
    )
399
    .option(
400
      "--test",
401
      "terminate after setting up the hub -- used to test if it starts up properly",
402
    )
403
    .option(
404
      "--db-concurrent-warn <n>",
405
      "be very unhappy if number of concurrent db requests exceeds this (default: 300)",
406
      (n) => parseInt(n),
407
      300,
408
    )
409
    .option(
410
      "--personal",
411
      "run VERY UNSAFE: there is only one user and no authentication",
412
    )
413
    .parse(process.argv);
414
  // Everywhere else in our code, we just refer to program.[options] since we
415
  // wrote this code against an ancient version of commander.
416
  const opts = commander.opts();
417
  for (const name in opts) {
418
    program[name] = opts[name];
419
  }
420
  if (!program.mode) {
421
    program.mode = process.env.COCALC_MODE;
422
    if (!program.mode) {
423
      throw Error(
424
        `the --mode option must be specified or the COCALC_MODE env var set to one of ${COCALC_MODES.join(
425
          ", ",
426
        )}`,
427
      );
428
      process.exit(1);
429
    }
430
  }
431
  if (program.all) {
432
    program.conatServer =
433
      program.proxyServer =
434
      program.nextServer =
435
      program.mentions =
436
      program.updateDatabaseSchema =
437
        true;
438
  }
439
  if (process.env.COCALC_DISABLE_NEXT) {
440
    program.nextServer = false;
441
  }
442

443
  //console.log("got opts", opts);
444

445
  try {
446
    // Everything we do here requires the database to be initialized. Once
447
    // this is called, require('@cocalc/database/postgres/database').default() is a valid db
448
    // instance that can be used.
449
    initDatabase({
450
      host: program.databaseNodes,
451
      database: program.databaseName,
452
      user: program.databaseUser,
453
      concurrent_warn: program.dbConcurrentWarn,
454
    });
455

456
    if (program.passwd) {
457
      logger.debug("Resetting password");
458
      await reset_password(program.passwd);
459
      process.exit();
460
    } else if (program.stripeSync) {
461
      logger.debug("Stripe sync");
462
      await stripe_sync({ database, logger: logger });
463
      process.exit();
464
    } else if (program.deleteExpired) {
465
      await callback2(database.delete_expired, {
466
        count_only: false,
467
      });
468
      process.exit();
469
    } else if (program.blobMaintenance) {
470
      await callback2(database.blob_maintenance);
471
      process.exit();
472
    } else if (program.updateStats) {
473
      await callback2(database.get_stats);
474
      process.exit();
475
    } else {
476
      await startServer();
477
    }
478
  } catch (err) {
479
    console.log(err);
480
    logger.error("Error -- ", err);
481
    process.exit(1);
482
  }
483
}
484

485
main();
486

487