1
/*
2
Service for watching directory listings in a project or compute server.
3
*/
4

5
import { createServiceClient, createServiceHandler } from "./typed";
6
import type { DirectoryListingEntry } from "@cocalc/util/types";
7
import { dkv, type DKV } from "@cocalc/conat/sync/dkv";
8
import { EventEmitter } from "events";
9
import refCache from "@cocalc/util/refcache";
10

11
// record info about at most this many files in a given directory
12
//export const MAX_FILES_PER_DIRECTORY = 10;
13
export const MAX_FILES_PER_DIRECTORY = 500;
14

15
// cache listing info about at most this many directories
16
//export const MAX_DIRECTORIES = 3;
17
export const MAX_DIRECTORIES = 50;
18

19
// watch directories with interest that are this recent
20
//export const INTEREST_CUTOFF_MS = 1000 * 30;
21
export const INTEREST_CUTOFF_MS = 1000 * 60 * 10;
22

23
export const MIN_INTEREST_INTERVAL_MS = 15 * 1000;
24

25
export interface ListingsApi {
26
  // cause the directory listing key:value store to watch path
27
  watch: (path: string) => Promise<void>;
28

29
  // just directly get the listing info now for this path
30
  getListing: (opts: {
31
    path: string;
32
    hidden?: boolean;
33
  }) => Promise<DirectoryListingEntry[]>;
34
}
35

36
interface ListingsOptions {
37
  project_id: string;
38
  compute_server_id?: number;
39
}
40

41
export function createListingsApiClient({
42
  project_id,
43
  compute_server_id = 0,
44
}: ListingsOptions) {
45
  return createServiceClient<ListingsApi>({
46
    project_id,
47
    compute_server_id,
48
    service: "listings",
49
  });
50
}
51

52
export type ListingsServiceApi = ReturnType<typeof createListingsApiClient>;
53

54
export async function createListingsService({
55
  project_id,
56
  compute_server_id = 0,
57
  impl,
58
}: ListingsOptions & { impl }) {
59
  const c = compute_server_id ? ` (compute server: ${compute_server_id})` : "";
60
  return await createServiceHandler<ListingsApi>({
61
    project_id,
62
    compute_server_id,
63
    service: "listings",
64
    description: `Directory listing service: ${c}`,
65
    impl,
66
  });
67
}
68

69
const config = {
70
  max_msgs: MAX_DIRECTORIES,
71
};
72

73
export interface Listing {
74
  files?: DirectoryListingEntry[];
75
  exists?: boolean;
76
  error?: string;
77
  time: number;
78
  more?: boolean;
79
  deleted?: string[];
80
}
81

82
export async function getListingsKV(
83
  opts: ListingsOptions,
84
): Promise<DKV<Listing>> {
85
  return await dkv<Listing>({
86
    name: "listings",
87
    config,
88
    ...opts,
89
  });
90
}
91

92
export interface Times {
93
  // time last files for a given directory were attempted to be updated
94
  updated?: number;
95
  // time user requested to watch a given directory
96
  interest?: number;
97
}
98

99
export async function getListingsTimesKV(
100
  opts: ListingsOptions,
101
): Promise<DKV<Times>> {
102
  return await dkv<Times>({
103
    name: "listings-times",
104
    config,
105
    ...opts,
106
  });
107
}
108

109
/* Unified interface to the above components for clients */
110

111
export class ListingsClient extends EventEmitter {
112
  options: { project_id: string; compute_server_id: number };
113
  api: Awaited<ReturnType<typeof createListingsApiClient>>;
114
  times?: DKV<Times>;
115
  listings?: DKV<Listing>;
116

117
  constructor({
118
    project_id,
119
    compute_server_id = 0,
120
  }: {
121
    project_id: string;
122
    compute_server_id?: number;
123
  }) {
124
    super();
125
    this.options = { project_id, compute_server_id };
126
  }
127

128
  init = async () => {
129
    try {
130
      this.api = createListingsApiClient(this.options);
131
      this.times = await getListingsTimesKV(this.options);
132
      this.listings = await getListingsKV(this.options);
133
      this.listings.on("change", this.handleListingsChange);
134
    } catch (err) {
135
      this.close();
136
      throw err;
137
    }
138
  };
139

140
  handleListingsChange = ({ key: path }) => {
141
    this.emit("change", path);
142
  };
143

144
  get = (path: string): Listing | undefined => {
145
    if (this.listings == null) {
146
      throw Error("not ready");
147
    }
148
    return this.listings.get(path);
149
  };
150

151
  getAll = () => {
152
    if (this.listings == null) {
153
      throw Error("not ready");
154
    }
155
    return this.listings.getAll();
156
  };
157

158
  close = () => {
159
    this.removeAllListeners();
160
    this.times?.close();
161
    delete this.times;
162
    if (this.listings != null) {
163
      this.listings.removeListener("change", this.handleListingsChange);
164
      this.listings.close();
165
      delete this.listings;
166
    }
167
  };
168

169
  watch = async (path, force = false) => {
170
    if (this.times == null) {
171
      throw Error("not ready");
172
    }
173
    if (!force) {
174
      const last = this.times.get(path)?.interest ?? 0;
175
      if (Math.abs(Date.now() - last) < MIN_INTEREST_INTERVAL_MS) {
176
        // somebody already expressed interest very recently
177
        return;
178
      }
179
    }
180
    await this.api.watch(path);
181
  };
182

183
  getListing = async (opts) => {
184
    return await this.api.getListing(opts);
185
  };
186
}
187

188
export const listingsClient = refCache<
189
  ListingsOptions & { noCache?: boolean },
190
  ListingsClient
191
>({
192
  name: "listings",
193
  createObject: async (options: ListingsOptions) => {
194
    const C = new ListingsClient(options);
195
    await C.init();
196
    return C;
197
  },
198
});
199

200