1
/*
2
 *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
3
 *  License: MS-RSL – see LICENSE.md for details
4
 */
5

6
import { TypedMap, redux } from "@cocalc/frontend/app-framework";
7
import { webapp_client } from "@cocalc/frontend/webapp-client";
8
import { Listing } from "@cocalc/util/db-schema/listings";
9
import type { DirectoryListingEntry } from "@cocalc/util/types";
10
import { EventEmitter } from "events";
11
import { fromJS, List } from "immutable";
12
import {
13
  listingsClient,
14
  type ListingsClient,
15
  createListingsApiClient,
16
  type ListingsApi,
17
  MIN_INTEREST_INTERVAL_MS,
18
} from "@cocalc/conat/service/listings";
19
import { delay } from "awaiting";
20
import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
21
import { getLogger } from "@cocalc/conat/client";
22

23
const logger = getLogger("listings");
24

25
export const WATCH_THROTTLE_MS = MIN_INTEREST_INTERVAL_MS;
26

27
type ImmutablePathEntry = TypedMap<DirectoryListingEntry>;
28

29
type State = "init" | "ready" | "closed";
30

31
export type ImmutableListing = TypedMap<Listing>;
32

33
export class Listings extends EventEmitter {
34
  private project_id: string;
35
  private compute_server_id: number;
36
  private state: State = "init";
37
  private listingsClient?: ListingsClient;
38
  private api: ListingsApi;
39

40
  constructor(project_id: string, compute_server_id: number = 0) {
41
    super();
42
    this.project_id = project_id;
43
    this.compute_server_id = compute_server_id;
44
    this.api = createListingsApiClient({ project_id, compute_server_id });
45
    this.init();
46
  }
47

48
  private createClient = async () => {
49
    let d = 3000;
50
    const MAX_DELAY_MS = 15000;
51
    while (this.state != "closed") {
52
      try {
53
        this.listingsClient = await listingsClient({
54
          project_id: this.project_id,
55
          compute_server_id: this.compute_server_id,
56
        });
57
        // success!
58
        return;
59
      } catch (err) {
60
        logger.debug(
61
          `WARNING: temporary issue connecting to project listings service -- ${err}`,
62
        );
63
      }
64
      if (this.state == ("closed" as State)) return;
65
      d = Math.min(MAX_DELAY_MS, d * 1.3);
66
      await delay(d);
67
    }
68
  };
69

70
  private init = reuseInFlight(async () => {
71
    //let start = Date.now();
72
    await this.createClient();
73
    // console.log("createClient finished in ", Date.now() - start, "ms");
74
    if (this.state == "closed") return;
75
    if (this.listingsClient == null) {
76
      throw Error("bug");
77
    }
78
    this.listingsClient.on("change", (path) => {
79
      this.emit("change", [path]);
80
    });
81
    // cause load of all cached data into redux
82
    this.emit("change", Object.keys(this.listingsClient.getAll()));
83
    // [ ] TODO: delete event for deleted paths
84
    this.setState("ready");
85
  });
86

87
  // Watch directory for changes.
88
  watch = reuseInFlight(async (path: string, force?): Promise<void> => {
89
    if (this.state == "closed") {
90
      return;
91
    }
92
    if (this.state != "ready") {
93
      await this.init();
94
    }
95
    if (this.state != "ready") {
96
      // failed forever or closed explicitly so don't care
97
      return;
98
    }
99
    if (this.listingsClient == null) {
100
      throw Error("listings not ready");
101
    }
102
    if (this.listingsClient == null) return;
103
    while (this.state != ("closed" as any) && this.listingsClient != null) {
104
      try {
105
        await this.listingsClient.watch(path, force);
106
        return;
107
      } catch (err) {
108
        if (this.listingsClient == null) {
109
          return;
110
        }
111
        force = true;
112
        logger.debug(
113
          `WARNING: not yet able to watch '${path}' in ${this.project_id} -- ${err}`,
114
        );
115
        try {
116
          await this.listingsClient.api.conat.waitFor({
117
            maxWait: 7.5 * 1000 * 60,
118
          });
119
        } catch (err) {
120
          console.log(`WARNING -- waiting for directory listings -- ${err}`);
121
          await delay(3000);
122
        }
123
      }
124
    }
125
  });
126

127
  get = async (
128
    path: string,
129
    trigger_start_project?: boolean,
130
  ): Promise<DirectoryListingEntry[] | undefined> => {
131
    if (this.listingsClient == null) {
132
      throw Error("listings not ready");
133
    }
134
    const x = this.listingsClient?.get(path);
135
    if (x != null) {
136
      if (x.error) {
137
        throw Error(x.error);
138
      }
139
      if (!x.exists) {
140
        throw Error(`ENOENT: no such directory '${path}'`);
141
      }
142
      return x.files;
143
    }
144
    if (trigger_start_project) {
145
      if (
146
        !(await redux.getActions("projects").start_project(this.project_id))
147
      ) {
148
        return;
149
      }
150
    }
151
    return await this.api.getListing({ path, hidden: true });
152
  };
153

154
  // Does a call to the project to directly determine whether or
155
  // not the given path exists.  This doesn't depend on the table.
156
  // Can throw an exception if it can't contact the project.
157
  exists = async (path: string): Promise<boolean> => {
158
    return (
159
      (
160
        await webapp_client.exec({
161
          project_id: this.project_id,
162
          command: "test",
163
          args: ["-e", path],
164
          err_on_exit: false,
165
        })
166
      ).exit_code == 0
167
    );
168
  };
169

170
  // Returns:
171
  //  - List<ImmutablePathEntry> in case of a proper directory listing
172
  //  - string in case of an error
173
  //  - undefined if directory listing not known (and error not known either).
174
  getForStore = async (
175
    path: string,
176
  ): Promise<List<ImmutablePathEntry> | undefined | string> => {
177
    try {
178
      const x = await this.get(path);
179
      return fromJS(x) as unknown as List<ImmutablePathEntry>;
180
    } catch (err) {
181
      return `${err}`;
182
    }
183
  };
184

185
  getUsingDatabase = async (
186
    path: string,
187
  ): Promise<DirectoryListingEntry[] | undefined> => {
188
    if (this.listingsClient == null) {
189
      return;
190
    }
191
    return this.listingsClient.get(path)?.files;
192
  };
193

194
  // TODO: we now only know there are more, not how many
195
  getMissingUsingDatabase = async (
196
    path: string,
197
  ): Promise<number | undefined> => {
198
    if (this.listingsClient == null) {
199
      // throw Error("listings not ready");
200
      return;
201
    }
202
    return this.listingsClient.get(path)?.more ? 1 : 0;
203
  };
204

205
  getMissing = (path: string): number | undefined => {
206
    if (this.listingsClient == null) {
207
      return;
208
    }
209
    return this.listingsClient.get(path)?.more ? 1 : 0;
210
  };
211

212
  getListingDirectly = async (
213
    path: string,
214
    trigger_start_project?: boolean,
215
  ): Promise<DirectoryListingEntry[]> => {
216
    // console.trace("getListingDirectly", { path });
217
    if (trigger_start_project) {
218
      if (
219
        !(await redux.getActions("projects").start_project(this.project_id))
220
      ) {
221
        throw Error("project not running");
222
      }
223
    }
224
    return await this.api.getListing({ path, hidden: true });
225
  };
226

227
  close = (): void => {
228
    if (this.state == "closed") {
229
      return;
230
    }
231
    this.setState("closed");
232
    this.listingsClient?.close();
233
    delete this.listingsClient;
234
  };
235

236
  isReady = (): boolean => {
237
    return this.state == ("ready" as State);
238
  };
239

240
  setState = (state: State) => {
241
    this.state = state;
242
    this.emit(state);
243
  };
244
}
245

246
export function listings(
247
  project_id: string,
248
  compute_server_id: number = 0,
249
): Listings {
250
  return new Listings(project_id, compute_server_id);
251
}
252

253