1
/*
2
 *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
3
 *  License: MS-RSL – see LICENSE.md for details
4
 */
5

6
import { getLogger } from "@cocalc/backend/logger";
7
import { secretToken } from "@cocalc/project/data";
8
import { enable_mesg } from "@cocalc/backend/misc_node";
9
import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol";
10
import { connectToLockedSocket } from "@cocalc/backend/tcp/locked-socket";
11
import * as message from "@cocalc/util/message";
12
import * as common from "./common";
13
import { forget_port, get_port } from "./port_manager";
14
import {
15
  SAGE_SERVER_MAX_STARTUP_TIME_S,
16
  restartSageServer,
17
} from "./sage_restart";
18
import { until, once } from "@cocalc/util/async-utils";
19

20
const logger = getLogger("get-sage-socket");
21

22
// Get a new connection to the Sage server.  If the server
23
// isn't running, e.g., it was killed due to running out of memory,
24
// attempt to restart it and try to connect.
25
export async function getSageSocket(): Promise<CoCalcSocket> {
26
  let socket: CoCalcSocket | undefined = undefined;
27
  await until(
28
    async () => {
29
      try {
30
        socket = await _getSageSocket();
31
        return true;
32
      } catch (err) {
33
        logger.debug(
34
          `unable to get sage socket, so restarting sage server - ${err}`,
35
        );
36
        // Failed for some reason: try to restart one time, then try again.
37
        // We do this because the Sage server can easily get killed due to out of memory conditions.
38
        // But we don't constantly try to restart the server, since it can easily fail to start if
39
        // there is something wrong with a local Sage install.
40
        // Note that restarting the sage server doesn't impact currently running worksheets (they
41
        // have their own process that isn't killed).
42
        try {
43
          // starting the sage server can also easily fail, so must be in the try
44
          await restartSageServer();
45
          // get socket immediately -- don't want to wait up to ~5s!
46
          socket = await _getSageSocket();
47
          return true;
48
        } catch (err) {
49
          logger.debug(
50
            `error restarting sage server or getting socket -- ${err}`,
51
          );
52
        }
53
        return false;
54
      }
55
    },
56
    {
57
      start: 1000,
58
      max: 5000,
59
      decay: 1.5,
60
      timeout: SAGE_SERVER_MAX_STARTUP_TIME_S * 1000,
61
    },
62
  );
63
  if (socket === undefined) {
64
    throw Error("bug");
65
  }
66
  return socket;
67
}
68

69
async function _getSageSocket(): Promise<CoCalcSocket> {
70
  logger.debug("get sage server port");
71
  const port = await get_port("sage");
72
  logger.debug(`get and unlock socket on port ${port}`);
73
  if (!port) {
74
    throw new Error("port is not set");
75
  }
76
  try {
77
    const sage_socket: CoCalcSocket = await connectToLockedSocket({
78
      port,
79
      token: secretToken,
80
    });
81
    logger.debug("Successfully unlocked a sage session connection.");
82

83
    logger.debug("request sage session from server.");
84
    enable_mesg(sage_socket);
85
    sage_socket.write_mesg("json", message.start_session({ type: "sage" }));
86
    logger.debug(
87
      "Waiting to read one JSON message back, which will describe the session....",
88
    );
89
    const [_type, desc] = await once(sage_socket, "mesg", 30000);
90
    logger.debug(`Got message back from Sage server: ${common.json(desc)}`);
91
    sage_socket.pid = desc.pid;
92
    return sage_socket;
93
  } catch (err) {
94
    forget_port("sage");
95
    const msg = `_new_session: sage session denied connection: ${err}`;
96
    logger.debug(`Failed to connect -- ${msg}`);
97
    throw Error(msg);
98
  }
99
}
100

101