1
/* Directory Listings
2

3
- A service "listings" in each project and compute server that users call to express
4
  interest in a directory.  When there is recent interest in a
5
  directory, we watch it for changes.
6

7
- A DKV store keys paths in the filesystem and values the first
8
  few hundred (ordered by recent) files in that directory, all relative
9
  to the home directory.
10

11

12
DEVELOPMENT:
13

14
1. Stop listings service running in the project by running this in your browser:
15

16
   await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'listings'})
17

18
    {status: 'terminated', service: 'listings'}
19

20

21
2. Setup project environment variables as usual (see README.md)
22

23
3. Start your own server
24

25
.../src/packages/project/conat$ node
26

27

28
    await require('@cocalc/project/conat/listings').init()
29

30
*/
31

32
import getListing from "@cocalc/backend/get-listing";
33
import {
34
  createListingsService,
35
  getListingsKV,
36
  getListingsTimesKV,
37
  MAX_FILES_PER_DIRECTORY,
38
  INTEREST_CUTOFF_MS,
39
  type Listing,
40
  type Times,
41
} from "@cocalc/conat/service/listings";
42
import { compute_server_id, project_id } from "@cocalc/project/data";
43
import { init as initClient } from "@cocalc/project/client";
44
import { delay } from "awaiting";
45
import { type DKV } from "./sync";
46
import { type ConatService } from "@cocalc/conat/service";
47
import { MultipathWatcher } from "@cocalc/backend/path-watcher";
48
import getLogger from "@cocalc/backend/logger";
49
import { path_split } from "@cocalc/util/misc";
50

51
const logger = getLogger("project:conat:listings");
52

53
let service: ConatService | null;
54
export async function init() {
55
  logger.debug("init: initializing");
56
  initClient();
57

58
  service = await createListingsService({
59
    project_id,
60
    compute_server_id,
61
    impl,
62
  });
63
  const L = new Listings();
64
  await L.init();
65
  listings = L;
66
  logger.debug("init: fully ready");
67
}
68

69
export async function close() {
70
  service?.close();
71
  listings?.close();
72
}
73

74
let listings: Listings | null;
75

76
const impl = {
77
  // cause the directory listing key:value store to watch path
78
  watch: async (path: string) => {
79
    while (listings == null) {
80
      await delay(3000);
81
    }
82
    listings.watch(path);
83
  },
84

85
  getListing: async ({ path, hidden }) => {
86
    return await getListing(path, hidden);
87
  },
88
};
89

90
export function isDeleted(filename: string) {
91
  return listings?.isDeleted(filename);
92
}
93

94
class Listings {
95
  private listings: DKV<Listing>;
96

97
  private times: DKV<Times>;
98

99
  private watcher: MultipathWatcher;
100

101
  private state: "init" | "ready" | "closed" = "init";
102

103
  constructor() {
104
    this.watcher = new MultipathWatcher();
105
    this.watcher.on("change", this.updateListing);
106
  }
107

108
  init = async () => {
109
    logger.debug("Listings.init: start");
110
    this.listings = await getListingsKV({ project_id, compute_server_id });
111
    this.times = await getListingsTimesKV({ project_id, compute_server_id });
112
    // start watching paths with recent interest
113
    const cutoff = Date.now() - INTEREST_CUTOFF_MS;
114
    const times = this.times.getAll();
115
    for (const path in times) {
116
      if ((times[path].interest ?? 0) >= cutoff) {
117
        await this.updateListing(path);
118
      }
119
    }
120
    this.monitorInterestLoop();
121
    this.state = "ready";
122
    logger.debug("Listings.init: done");
123
  };
124

125
  private monitorInterestLoop = async () => {
126
    while (this.state != "closed") {
127
      const cutoff = Date.now() - INTEREST_CUTOFF_MS;
128
      const times = this.times.getAll();
129
      for (const path in times) {
130
        if ((times[path].interest ?? 0) <= cutoff) {
131
          if (this.watcher.has(path)) {
132
            logger.debug("monitorInterestLoop: stop watching", { path });
133
            this.watcher.delete(path);
134
          }
135
        }
136
      }
137
      await delay(30 * 1000);
138
    }
139
  };
140

141
  close = () => {
142
    this.state = "closed";
143
    this.watcher.close();
144
    this.listings?.close();
145
    this.times?.close();
146
  };
147

148
  updateListing = async (path: string) => {
149
    logger.debug("updateListing", { path });
150
    path = canonicalPath(path);
151
    this.watcher.add(canonicalPath(path));
152
    const start = Date.now();
153
    try {
154
      let files = await getListing(path, true, {
155
        limit: MAX_FILES_PER_DIRECTORY + 1,
156
      });
157
      const more = files.length == MAX_FILES_PER_DIRECTORY + 1;
158
      if (more) {
159
        files = files.slice(0, MAX_FILES_PER_DIRECTORY);
160
      }
161
      this.listings.set(path, {
162
        files,
163
        exists: true,
164
        time: Date.now(),
165
        more,
166
      });
167
      logger.debug("updateListing: success", {
168
        path,
169
        ms: Date.now() - start,
170
        count: files.length,
171
        more,
172
      });
173
    } catch (err) {
174
      let error = `${err}`;
175
      if (error.startsWith("Error: ")) {
176
        error = error.slice("Error: ".length);
177
      }
178
      this.listings.set(path, {
179
        error,
180
        time: Date.now(),
181
        exists: error.includes("ENOENT") ? false : undefined,
182
      });
183
      logger.debug("updateListing: error", {
184
        path,
185
        ms: Date.now() - start,
186
        error,
187
      });
188
    }
189
  };
190

191
  watch = async (path: string) => {
192
    logger.debug("watch", { path });
193
    path = canonicalPath(path);
194
    this.times.set(path, { ...this.times.get(path), interest: Date.now() });
195
    this.updateListing(path);
196
  };
197

198
  isDeleted = (filename: string): boolean | undefined => {
199
    if (this.listings == null) {
200
      return undefined;
201
    }
202
    const { head: path, tail } = path_split(filename);
203
    const listing = this.listings.get(path);
204
    if (listing == null) {
205
      return undefined;
206
    }
207
    if (listing.deleted?.includes(tail)) {
208
      return true;
209
    }
210
    return false;
211
  };
212
}
213

214
// this does a tiny amount to make paths more canonical.
215
function canonicalPath(path: string): string {
216
  if (path == "." || path == "~") {
217
    return "";
218
  }
219
  return path;
220
}
221

222