1
/*
2
The main hub express app.
3
*/
4

5
import cookieParser from "cookie-parser";
6
import express from "express";
7
import ms from "ms";
8
import { join } from "path";
9
import { parse as parseURL } from "url";
10
import webpackDevMiddleware from "webpack-dev-middleware";
11
import webpackHotMiddleware from "webpack-hot-middleware";
12
import { path as WEBAPP_PATH } from "@cocalc/assets";
13
import { path as CDN_PATH } from "@cocalc/cdn";
14
import vhostShare from "@cocalc/next/lib/share/virtual-hosts";
15
import { path as STATIC_PATH } from "@cocalc/static";
16
import { initAnalytics } from "../analytics";
17
import { setup_health_checks as setupHealthChecks } from "../health-checks";
18
import { getLogger } from "../logger";
19
import initProxy from "../proxy";
20
import initAppRedirect from "./app/app-redirect";
21
import initBlobUpload from "./app/blob-upload";
22
import initUpload from "./app/upload";
23
import initBlobs from "./app/blobs";
24
import initCustomize from "./app/customize";
25
import { initMetricsEndpoint, setupInstrumentation } from "./app/metrics";
26
import initNext from "./app/next";
27
import initStats from "./app/stats";
28
import { database } from "./database";
29
import initHttpServer from "./http";
30
import initRobots from "./robots";
31
import basePath from "@cocalc/backend/base-path";
32
import { initConatServer } from "@cocalc/server/conat/socketio";
33
import { conatSocketioCount } from "@cocalc/backend/data";
34

35
// NOTE: we are not using compression because that interferes with streaming file download,
36
// and could be generally confusing.
37

38
// Used for longterm caching of files. This should be in units of seconds.
39
const MAX_AGE = Math.round(ms("10 days") / 1000);
40
const SHORT_AGE = Math.round(ms("10 seconds") / 1000);
41

42
interface Options {
43
  projectControl;
44
  isPersonal: boolean;
45
  nextServer: boolean;
46
  proxyServer: boolean;
47
  conatServer: boolean;
48
  cert?: string;
49
  key?: string;
50
}
51

52
export default async function init(opts: Options): Promise<{
53
  httpServer;
54
  router: express.Router;
55
}> {
56
  const winston = getLogger("express-app");
57
  winston.info("creating express app");
58

59
  // Create an express application
60
  const app = express();
61
  app.disable("x-powered-by"); // https://github.com/sagemathinc/cocalc/issues/6101
62

63
  // makes JSON (e.g. the /customize endpoint) pretty-printed
64
  app.set("json spaces", 2);
65

66
  // healthchecks are for internal use, no basePath prefix
67
  // they also have to come first, since e.g. the vhost depends
68
  // on the DB, which could be down
69
  const basicEndpoints = express.Router();
70
  await setupHealthChecks({ router: basicEndpoints, db: database });
71
  app.use(basicEndpoints);
72

73
  // also, for the same reasons as above, setup the /metrics endpoint
74
  initMetricsEndpoint(basicEndpoints);
75

76
  // now, we build the router for some other endpoints
77
  const router = express.Router();
78

79
  // This must go very early - we handle virtual hosts, like wstein.org
80
  // before any other routes or middleware interfere.
81
  if (opts.nextServer) {
82
    app.use(vhostShare());
83
  }
84

85
  app.use(cookieParser());
86

87
  // Install custom middleware to track response time metrics via prometheus
88
  setupInstrumentation(router);
89

90
  // see http://stackoverflow.com/questions/10849687/express-js-how-to-get-remote-client-address
91
  app.enable("trust proxy");
92

93
  router.use("/robots.txt", initRobots());
94

95
  // setup the analytics.js endpoint
96
  await initAnalytics(router, database);
97

98
  // The /static content, used by docker, development, etc.
99
  // This is the stuff that's packaged up via webpack in packages/static.
100
  await initStatic(router);
101

102
  // Static assets that are used by the webapp, the landing page, etc.
103
  router.use(
104
    "/webapp",
105
    express.static(WEBAPP_PATH, { setHeaders: cacheLongTerm }),
106
  );
107

108
  // This is @cocalc/cdn – cocalc serves everything it might get from a CDN on its own.
109
  // This is defined in the @cocalc/cdn package.  See the comments in packages/cdn.
110
  router.use("/cdn", express.static(CDN_PATH, { setHeaders: cacheLongTerm }));
111

112
  // Redirect requests to /app to /static/app.html.
113
  // TODO: this will likely go away when rewrite the landing pages to not
114
  // redirect users to /app in the first place.
115
  router.get("/app", (req, res) => {
116
    // query is exactly "?key=value,key=..."
117
    const query = parseURL(req.url, true).search || "";
118
    res.redirect(join(basePath, "static/app.html") + query);
119
  });
120

121
  initBlobs(router);
122
  initBlobUpload(router);
123
  initUpload(router);
124
  initCustomize(router, opts.isPersonal);
125
  initStats(router);
126
  initAppRedirect(router);
127

128
  if (basePath !== "/") {
129
    app.use(basePath, router);
130
  } else {
131
    app.use(router);
132
  }
133

134
  const httpServer = initHttpServer({
135
    cert: opts.cert,
136
    key: opts.key,
137
    app,
138
  });
139

140
  if (opts.conatServer) {
141
    winston.info(`initializing the Conat Server`);
142
    initConatServer({
143
      httpServer,
144
      ssl: !!opts.cert,
145
    });
146
  }
147

148
  // This must be second to the last, since it will prevent any
149
  // other upgrade handlers from being added to httpServer.
150
  if (opts.proxyServer) {
151
    winston.info(`initializing the http proxy server`, {
152
      conatSocketioCount,
153
      conatServer: !!opts.conatServer,
154
      isPersonal: opts.isPersonal,
155
    });
156
    initProxy({
157
      projectControl: opts.projectControl,
158
      isPersonal: opts.isPersonal,
159
      httpServer,
160
      app,
161
      // enable proxy server for /conat if:
162
      //  (1) we are not running conat at all from here, or
163
      //  (2) we are running socketio in cluster mode, hence
164
      //      on a different port
165
      proxyConat: !opts.conatServer || (conatSocketioCount ?? 1) >= 2,
166
    });
167
  }
168

169
  // IMPORTANT:
170
  // The nextjs server must be **LAST** (!), since it takes
171
  // all routes not otherwise handled above.
172
  if (opts.nextServer) {
173
    // The Next.js server
174
    await initNext(app);
175
  }
176
  return { httpServer, router };
177
}
178

179
function cacheShortTerm(res) {
180
  res.setHeader(
181
    "Cache-Control",
182
    `public, max-age=${SHORT_AGE}, must-revalidate`,
183
  );
184
  res.setHeader(
185
    "Expires",
186
    new Date(Date.now().valueOf() + SHORT_AGE).toUTCString(),
187
  );
188
}
189

190
// Various files such as the webpack static content should be cached long-term,
191
// and we use this function to set appropriate headers at various points below.
192
function cacheLongTerm(res) {
193
  res.setHeader(
194
    "Cache-Control",
195
    `public, max-age=${MAX_AGE}, must-revalidate'`,
196
  );
197
  res.setHeader(
198
    "Expires",
199
    new Date(Date.now().valueOf() + MAX_AGE).toUTCString(),
200
  );
201
}
202

203
async function initStatic(router) {
204
  let compiler: any = null;
205
  if (
206
    process.env.NODE_ENV != "production" &&
207
    !process.env.NO_RSPACK_DEV_SERVER
208
  ) {
209
    // Try to use the integrated rspack dev server, if it is installed.
210
    // It might not be installed at all, e.g., in production, and there
211
    // @cocalc/static can't even be imported.
212
    try {
213
      const { rspackCompiler } = require("@cocalc/static/rspack-compiler");
214
      compiler = rspackCompiler();
215
    } catch (err) {
216
      console.warn("rspack is not available", err);
217
    }
218
  }
219

220
  if (compiler != null) {
221
    console.warn(
222
      "\n-----------\n| RSPACK: Running rspack dev server for frontend /static app.\n| Set env variable NO_RSPACK_DEV_SERVER to disable.\n-----------\n",
223
    );
224
    router.use("/static", webpackDevMiddleware(compiler, {}));
225
    router.use("/static", webpackHotMiddleware(compiler, {}));
226
  } else {
227
    router.use(
228
      join("/static", STATIC_PATH, "app.html"),
229
      express.static(join(STATIC_PATH, "app.html"), {
230
        setHeaders: cacheShortTerm,
231
      }),
232
    );
233
    router.use(
234
      "/static",
235
      express.static(STATIC_PATH, { setHeaders: cacheLongTerm }),
236
    );
237
  }
238

239
  // Also, immediately 404 if anything else under static is requested
240
  // which isn't handled above, rather than passing this on to the next app
241
  router.use("/static", (_, res) => res.status(404).end());
242
}
243

244