1
/*
2
 *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
3
 *  License: MS-RSL – see LICENSE.md for details
4
 */
5

6
// This unifies the entire webapp configuration – endpoint /customize
7
// The main goal is to optimize this, to use as little DB interactions
8
// as necessary, use caching, etc.
9
// This manages the webapp's configuration based on the hostname
10
// (allows whitelabeling).
11

12
import { delay } from "awaiting";
13
import debug from "debug";
14
import { isEmpty } from "lodash";
15
import LRU from "lru-cache";
16
import type { PostgreSQL } from "@cocalc/database/postgres/types";
17
import { get_passport_manager, PassportManager } from "@cocalc/server/hub/auth";
18
import { getSoftwareEnvironments } from "@cocalc/server/software-envs";
19
import { callback2 as cb2 } from "@cocalc/util/async-utils";
20
import { EXTRAS as SERVER_SETTINGS_EXTRAS } from "@cocalc/util/db-schema/site-settings-extras";
21
import { SoftwareEnvConfig } from "@cocalc/util/sanitize-software-envs";
22
import { site_settings_conf as SITE_SETTINGS_CONF } from "@cocalc/util/schema";
23
import { CustomLLMPublic } from "@cocalc/util/types/llm";
24
import { parseDomain, ParseResultType } from "parse-domain";
25
import getServerSettings, {
26
  ServerSettingsDynamic,
27
} from "./servers/server-settings";
28
import { have_active_registration_tokens } from "./utils";
29

30
const L = debug("hub:webapp-config");
31

32
const CACHE = new LRU({ max: 1000, ttl: 30 * 1000 }); // 1 minutes
33

34
export function clear_cache(): void {
35
  CACHE.clear();
36
}
37

38
type Theme = { [key: string]: string | boolean };
39

40
interface Config {
41
  // todo
42
  configuration: any;
43
  registration: any;
44
  strategies: object;
45
  software: SoftwareEnvConfig | null;
46
  ollama: { [key: string]: CustomLLMPublic };
47
  custom_openai: { [key: string]: CustomLLMPublic };
48
}
49

50
async function get_passport_manager_async(): Promise<PassportManager> {
51
  // the only issue here is, that the http server already starts up before the
52
  // passport manager is configured – but, the passport manager depends on the http server
53
  // we just retry during that initial period of uncertainty…
54
  let ms = 100;
55
  while (true) {
56
    const pp_manager = get_passport_manager();
57
    if (pp_manager != null) {
58
      return pp_manager;
59
    } else {
60
      L(
61
        `WARNING: Passport Manager not available yet -- trying again in ${ms}ms`,
62
      );
63
      await delay(ms);
64
      ms = Math.min(10000, 1.3 * ms);
65
    }
66
  }
67
}
68

69
export class WebappConfiguration {
70
  private readonly db: PostgreSQL;
71
  private data?: ServerSettingsDynamic;
72

73
  constructor({ db }) {
74
    this.db = db;
75
    this.init();
76
  }
77

78
  private async init(): Promise<void> {
79
    // this.data.pub updates automatically – do not modify it!
80
    this.data = await getServerSettings();
81
    await get_passport_manager_async();
82
  }
83

84
  // server settings with whitelabeling settings
85
  // TODO post-process all values
86
  public async settings(vID: string) {
87
    const res = await cb2(this.db._query, {
88
      query: "SELECT id, settings FROM whitelabeling",
89
      cache: true,
90
      where: { "id = $::TEXT": vID },
91
    });
92
    if (this.data == null) {
93
      // settings not yet initialized
94
      return {};
95
    }
96
    const data = res.rows[0];
97
    if (data != null) {
98
      return { ...this.data.all, ...data.settings };
99
    } else {
100
      return this.data.all;
101
    }
102
  }
103

104
  // derive the vanity ID from the host string
105
  private get_vanity_id(host: string): string | undefined {
106
    const host_parsed = parseDomain(host);
107
    if (host_parsed.type === ParseResultType.Listed) {
108
      // vanity for vanity.cocalc.com or foo.p for foo.p.cocalc.com
109
      return host_parsed.subDomains.join(".");
110
    }
111
    return undefined;
112
  }
113

114
  private async theme(vID: string): Promise<Theme> {
115
    const res = await cb2(this.db._query, {
116
      query: "SELECT id, theme FROM whitelabeling",
117
      cache: true,
118
      where: { "id = $::TEXT": vID },
119
    });
120
    const data = res.rows[0];
121
    if (data != null) {
122
      // post-process data, but do not set default values…
123
      const theme: Theme = {};
124
      for (const [key, value] of Object.entries(data.theme)) {
125
        const config = SITE_SETTINGS_CONF[key] ?? SERVER_SETTINGS_EXTRAS[key];
126
        if (typeof config?.to_val == "function") {
127
          theme[key] = config.to_val(value, data.theme);
128
        } else {
129
          if (typeof value == "string" || typeof value == "boolean") {
130
            theme[key] = value;
131
          }
132
        }
133
      }
134
      L(`vanity theme=${JSON.stringify(theme)}`);
135
      return theme;
136
    } else {
137
      L(`theme id=${vID} not found`);
138
      return {};
139
    }
140
  }
141

142
  private async get_vanity(vID): Promise<object> {
143
    if (vID != null && vID !== "") {
144
      L(`vanity ID = "${vID}"`);
145
      return await this.theme(vID);
146
    } else {
147
      return {};
148
    }
149
  }
150

151
  // returns the global configuration + eventually vanity specific site config settings
152
  private async get_configuration({ host, country }) {
153
    if (this.data == null) {
154
      // settings not yet initialized
155
      return {};
156
    }
157
    const vID = this.get_vanity_id(host);
158
    const config = this.data.pub;
159
    const vanity = await this.get_vanity(vID);
160
    return { ...config, ...vanity, ...{ country, dns: host } };
161
  }
162

163
  private async get_strategies(): Promise<object> {
164
    const key = "strategies";
165
    let strategies = CACHE.get(key);
166
    if (strategies == null) {
167
      // wait until this.passport_manager is initialized.
168
      // this could happen right at the start of the server
169
      const passport_manager = await get_passport_manager_async();
170
      strategies = passport_manager.get_strategies_v2();
171
      CACHE.set(key, strategies);
172
    }
173
    return strategies as object;
174
  }
175

176
  // derives the public ollama model configuration from the private one
177
  private get_ollama_public(): { [key: string]: CustomLLMPublic } {
178
    if (this.data == null) {
179
      throw new Error("server settings not yet initialized");
180
    }
181
    const ollama = this.data.all.ollama_configuration;
182
    return processCustomLLM(ollama, "Ollama");
183
  }
184

185
  private get_custom_openai_public(): { [key: string]: CustomLLMPublic } {
186
    if (this.data == null) {
187
      throw new Error("server settings not yet initialized");
188
    }
189
    const custom_openai = this.data.all.custom_openai_configuration;
190
    return processCustomLLM(custom_openai, "OpenAI (custom)");
191
  }
192

193
  private async get_config({ country, host }): Promise<Config> {
194
    while (this.data == null) {
195
      L.debug("waiting for server settings to be initialized");
196
      await delay(100);
197
    }
198

199
    const [configuration, registration, software, ollama, custom_openai] =
200
      await Promise.all([
201
        this.get_configuration({ host, country }),
202
        have_active_registration_tokens(this.db),
203
        getSoftwareEnvironments("webapp"),
204
        this.get_ollama_public(),
205
        this.get_custom_openai_public(),
206
      ]);
207
    const strategies = await this.get_strategies();
208
    return {
209
      configuration,
210
      registration,
211
      strategies,
212
      software,
213
      ollama,
214
      custom_openai,
215
    };
216
  }
217

218
  // it returns a shallow copy, hence you can modify/add keys in the returned map!
219
  public async get({ country, host }): Promise<Config> {
220
    const key = `config::${country}::${host}`;
221
    let config = CACHE.get(key);
222
    if (config == null) {
223
      config = await this.get_config({ country, host });
224
      CACHE.set(key, config);
225
    } else {
226
      L(`cache hit -- '${key}'`);
227
    }
228
    return config as Config;
229
  }
230
}
231

232
// for Ollama or Custom OpenAI
233
function processCustomLLM(
234
  data: any,
235
  displayFallback,
236
): { [key: string]: CustomLLMPublic } {
237
  if (isEmpty(data)) return {};
238

239
  const ret: { [key: string]: CustomLLMPublic } = {};
240
  for (const key in data) {
241
    const conf = data[key];
242
    const cocalc = conf.cocalc ?? {};
243
    if (cocalc.disabled) continue;
244
    const model = conf.model ?? key;
245
    ret[key] = {
246
      model,
247
      display: cocalc.display ?? `${displayFallback} ${model}`,
248
      icon: cocalc.icon, // fallback is the Ollama or OpenAI icon, frontend does that
249
      desc: cocalc.desc ?? "",
250
    };
251
  }
252
  return ret;
253
}
254

255