1
/*
2
 *  This file is part of CoCalc: Copyright © 2021 Sagemath, Inc.
3
 *  License: MS-RSL – see LICENSE.md for details
4
 */
5

6
import { keys, map, sortBy, zipObject } from "lodash";
7
import { promises } from "node:fs";
8
import { basename } from "node:path";
9

10
import {
11
  SOFTWARE_ENV_NAMES,
12
  SoftwareEnvNames,
13
} from "@cocalc/util/consts/software-envs";
14
import { hours_ago } from "@cocalc/util/relative-time";
15
import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
16
import withCustomize from "lib/with-customize";
17
import { SOFTWARE_FALLBACK, SOFTWARE_URLS } from "./software-data";
18
import {
19
  ComputeComponents,
20
  ComputeInventory,
21
  EnvData,
22
  LanguageName,
23
  SoftwareSpec,
24
} from "./types";
25

26
const { readFile } = promises;
27

28
async function makeObject(keys, fn) {
29
  return zipObject(keys, await Promise.all(map(keys, fn)));
30
}
31

32
type SoftwareEnvironments = { [key in SoftwareEnvNames]: EnvData };
33

34
let SoftwareEnvSpecs: SoftwareEnvironments | null = null;
35
let SoftwareEnvDownloadedTimestamp: number = 0;
36

37
async function file2json(path: string): Promise<any> {
38
  const data = await readFile(path, "utf8");
39
  return JSON.parse(data);
40
}
41

42
async function downloadInventoryJson(name: SoftwareEnvNames): Promise<EnvData> {
43
  try {
44
    const raw = await fetch(SOFTWARE_URLS[name]);
45
    if (!raw.ok) {
46
      console.log(`Problem downloading: ${raw.status}: ${raw.statusText}`);
47
    } else {
48
      const data = await raw.json();
49
      console.log(`Downloaded software inventory ${name} successfully`);
50
      return data;
51
    }
52
  } catch (err) {
53
    console.log(`Problem downloading: ${err}`);
54
  }
55
  return SOFTWARE_FALLBACK[name] as EnvData;
56
}
57

58
// load the current version of the software specs – if there is a problem, use the locally stored files as fallback.
59
async function fetchInventory(): Promise<SoftwareEnvironments> {
60
  // for development, set the env variable to directory, where this files are
61
  const localSpec = process.env.COCALC_SOFTWARE_ENVIRONMENTS;
62
  if (localSpec != null) {
63
    // read compute-inventory.json and compute-components.json from the local filesystem
64
    console.log(`Reading inventory information from directory ${localSpec}`);
65
    return await makeObject(
66
      SOFTWARE_ENV_NAMES,
67
      async (name) =>
68
        await file2json(`${localSpec}/software-inventory-${name}.json`),
69
    );
70
  }
71
  try {
72
    // download the files for the newest information from the server
73
    const ret = await makeObject(
74
      SOFTWARE_ENV_NAMES,
75
      async (name) => await downloadInventoryJson(name),
76
    );
77
    return ret;
78
  } catch (err) {
79
    console.error(`Problem fetching software inventory: ${err}`);
80
    return SOFTWARE_FALLBACK;
81
  }
82
}
83

84
const fetchSoftwareSpec = reuseInFlight(async function () {
85
  SoftwareEnvSpecs = await fetchInventory();
86
  SoftwareEnvDownloadedTimestamp = Date.now();
87
  return SoftwareEnvSpecs;
88
});
89

90
/**
91
 * get a cached copy of the software specs
92
 */
93
async function getSoftwareInfo(name: SoftwareEnvNames): Promise<EnvData> {
94
  // if SoftwareEnvSpecs is not set or not older than one hour, fetch it
95
  if (SoftwareEnvSpecs != null) {
96
    if (SoftwareEnvDownloadedTimestamp > hours_ago(1).getTime()) {
97
      // fresh enough, just return it
98
      return SoftwareEnvSpecs[name];
99
    } else {
100
      // we asynchroneously fetch to refresh, but return the data immediately to the client
101
      fetchSoftwareSpec();
102
      return SoftwareEnvSpecs[name];
103
    }
104
  } else {
105
    const specs = await fetchSoftwareSpec();
106
    return specs[name];
107
  }
108
}
109

110
async function getSoftwareInfoLang(
111
  name: SoftwareEnvNames,
112
  lang: LanguageName,
113
): Promise<{
114
  inventory: ComputeInventory[LanguageName];
115
  components: ComputeComponents[LanguageName];
116
  timestamp: string;
117
}> {
118
  const { inventory, data, timestamp } = await getSoftwareInfo(name);
119
  return { inventory: inventory[lang], components: data[lang], timestamp };
120
}
121

122
// during startup, we fetch getSoftwareSpec() once to warm up the cache…
123
(async function () {
124
  fetchSoftwareSpec(); // not blocking
125
})();
126

127
// cached processed software specs
128
let SPEC: Record<SoftwareEnvNames, Readonly<SoftwareSpec> | null> = {} as any;
129

130
async function getSoftwareSpec(name: SoftwareEnvNames): Promise<SoftwareSpec> {
131
  const cached = SPEC[name];
132
  if (cached != null) return cached;
133
  const nextSpec: Partial<SoftwareSpec> = {};
134
  const { inventory } = await getSoftwareInfo(name);
135
  for (const cmd in inventory.language_exes) {
136
    const info = inventory.language_exes[cmd];
137
    if (nextSpec[info.lang] == null) {
138
      nextSpec[info.lang] = {};
139
    }
140
    // the basename of the cmd path
141
    const base = cmd.indexOf(" ") > 0 ? cmd : basename(cmd);
142
    nextSpec[info.lang][base] = {
143
      cmd,
144
      name: info.name,
145
      doc: info.doc,
146
      url: info.url,
147
      path: info.path,
148
    };
149
  }
150
  SPEC[name] = nextSpec as SoftwareSpec;
151
  return nextSpec as SoftwareSpec;
152
}
153

154
/**
155
 * This determines the order of columns when there is more than on executable for a language.
156
 */
157
function getLanguageExecutables({ lang, inventory }): string[] {
158
  if (inventory == null) return [];
159
  return sortBy(keys(inventory[lang]), (x: string) => {
160
    if (lang === "python") {
161
      if (x.endsWith("python3")) return ["0", x];
162
      if (x.indexOf("sage") >= 0) return ["2", x];
163
      if (x.endsWith("python2")) return ["3", x];
164
      return ["1", x]; // anaconda envs and others
165
    } else {
166
      return x.toLowerCase();
167
    }
168
  });
169
}
170

171
// this is for the server side getServerSideProps function
172
export async function withCustomizedAndSoftwareSpec(
173
  context,
174
  lang: LanguageName | "executables",
175
) {
176
  const { name } = context.params;
177

178
  // if name is not in SOFTWARE_ENV_NAMES, return {notFound : true}
179
  if (!SOFTWARE_ENV_NAMES.includes(name)) {
180
    return { notFound: true };
181
  }
182

183
  const [customize, spec] = await Promise.all([
184
    withCustomize({ context }),
185
    getSoftwareSpec(name),
186
  ]);
187

188
  customize.props.name = name;
189

190
  if (lang === "executables") {
191
    // this is instant because specs are already in the cache
192
    const softwareInfo = await getSoftwareInfo(name);
193
    const { inventory, timestamp } = softwareInfo;
194
    customize.props.executablesSpec = inventory.executables;
195
    customize.props.timestamp = timestamp;
196
    return customize;
197
  } else {
198
    customize.props.spec = spec[lang];
199
    // this is instant because specs are already in the cache
200
    const { inventory, components, timestamp } = await getSoftwareInfoLang(
201
      name,
202
      lang,
203
    );
204
    customize.props.inventory = inventory;
205
    customize.props.components = components;
206
    customize.props.timestamp = timestamp;
207
  }
208

209
  // at this point, lang != "executables"
210
  // we gather the list of interpreters (executables) for the given language
211
  const { inventory } = await getSoftwareInfo(name);
212
  customize.props.execInfo = {};
213
  for (const cmd of getLanguageExecutables({ inventory, lang })) {
214
    const path = inventory.language_exes[cmd]?.path ?? cmd;
215
    customize.props.execInfo[path] = inventory.executables?.[path] ?? null;
216
  }
217

218
  return customize;
219
}
220

221