1
/*
2
 *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
3
 *  License: MS-RSL – see LICENSE.md for details
4
 */
5

6
/*
7
Watch A DIRECTORY for changes of the files in *that* directory only (not recursive).
8
Use ./watcher.ts for a single file.
9

10
Slightly generalized fs.watch that works even when the directory doesn't exist,
11
but also doesn't provide any information about what changed.
12

13
NOTE: We could maintain the directory listing and just try to update info about the filename,
14
taking into account the type.  That's probably really hard to get right, and just
15
debouncing and computing the whole listing is going to be vastly easier and good
16
enough at least for first round of this.
17

18
We assume path is relative to HOME and contained inside of HOME.
19

20
The code below deals with two very different cases:
21
 - when that path doesn't exist: use fs.watch on the parent directory.
22
        NOTE: this case can't happen when path='', which exists, so we can assume to have read perms on parent.
23
 - when the path does exist: use fs.watch (hence inotify) on the path itself to report when it changes
24

25
NOTE: if you are running on a file system like NFS, inotify won't work well or not at all.
26
In that case, set the env variable COCALC_FS_WATCHER=poll to use polling instead.
27
You can configure the poll interval by setting COCALC_FS_WATCHER_POLL_INTERVAL_MS.
28

29
UPDATE: We are using polling in ALL cases.  We have subtle bugs
30
with adding and removing directories otherwise, and also
31
we are only ever watching a relatively small number of directories
32
with a long interval, so polling is not so bad.
33
*/
34

35
import { watch, WatchOptions } from "chokidar";
36
import { FSWatcher } from "fs";
37
import { join } from "path";
38
import { EventEmitter } from "events";
39
import { debounce } from "lodash";
40
import { exists } from "@cocalc/backend/misc/async-utils-node";
41
import { close, path_split } from "@cocalc/util/misc";
42
import { getLogger } from "./logger";
43

44
const logger = getLogger("backend:path-watcher");
45

46
// const COCALC_FS_WATCHER = process.env.COCALC_FS_WATCHER ?? "inotify";
47
// if (!["inotify", "poll"].includes(COCALC_FS_WATCHER)) {
48
//   throw new Error(
49
//     `$COCALC_FS_WATCHER=${COCALC_FS_WATCHER} -- must be "inotify" or "poll"`,
50
//   );
51
// }
52
// const POLLING = COCALC_FS_WATCHER === "poll";
53

54
const POLLING = true;
55

56
const DEFAULT_POLL_MS = parseInt(
57
  process.env.COCALC_FS_WATCHER_POLL_INTERVAL_MS ?? "2000",
58
);
59

60
const ChokidarOpts: WatchOptions = {
61
  persistent: true, // otherwise won't work
62
  followSymlinks: false, // don't wander about
63
  disableGlobbing: true, // watch the path as it is, that's it
64
  usePolling: POLLING,
65
  interval: DEFAULT_POLL_MS,
66
  binaryInterval: DEFAULT_POLL_MS,
67
  depth: 0, // we only care about the explicitly mentioned path – there could be a lot of files and sub-dirs!
68
  // maybe some day we want this:
69
  // awaitWriteFinish: {
70
  //   stabilityThreshold: 100,
71
  //   pollInterval: 50,
72
  // },
73
  ignorePermissionErrors: true,
74
  alwaysStat: false,
75
} as const;
76

77
export class Watcher extends EventEmitter {
78
  private path: string;
79
  private exists: boolean;
80
  private watchContents?: FSWatcher;
81
  private watchExistence?: FSWatcher;
82
  private debounce_ms: number;
83
  private debouncedChange: any;
84
  private log: Function;
85

86
  constructor(
87
    path: string,
88
    { debounce: debounce_ms = DEFAULT_POLL_MS }: { debounce?: number } = {},
89
  ) {
90
    super();
91
    this.log = logger.extend(path).debug;
92
    this.log(`initializing: poll=${POLLING}`);
93
    if (process.env.HOME == null) {
94
      throw Error("bug -- HOME must be defined");
95
    }
96
    this.path = path.startsWith("/") ? path : join(process.env.HOME, path);
97
    this.debounce_ms = debounce_ms;
98
    this.debouncedChange = this.debounce_ms
99
      ? debounce(this.change, this.debounce_ms, {
100
          leading: true,
101
          trailing: true,
102
        }).bind(this)
103
      : this.change;
104
    this.init();
105
  }
106

107
  private async init(): Promise<void> {
108
    this.log("init watching", this.path);
109
    this.exists = await exists(this.path);
110
    if (this.path != "") {
111
      this.log("init watching", this.path, " for existence");
112
      this.initWatchExistence();
113
    }
114
    if (this.exists) {
115
      this.log("init watching", this.path, " contents");
116
      this.initWatchContents();
117
    }
118
  }
119

120
  private initWatchContents(): void {
121
    this.watchContents = watch(this.path, ChokidarOpts);
122
    this.watchContents.on("all", this.debouncedChange);
123
    this.watchContents.on("error", (err) => {
124
      this.log(`error watching listings -- ${err}`);
125
    });
126
  }
127

128
  private async initWatchExistence(): Promise<void> {
129
    const containing_path = path_split(this.path).head;
130
    this.watchExistence = watch(containing_path, ChokidarOpts);
131
    this.watchExistence.on("all", this.watchExistenceChange);
132
    this.watchExistence.on("error", (err) => {
133
      this.log(`error watching for existence of ${this.path} -- ${err}`);
134
    });
135
  }
136

137
  private watchExistenceChange = async (_, path) => {
138
    if (path != this.path) return;
139
    const e = await exists(this.path);
140
    if (!this.exists && e) {
141
      // it sprung into existence
142
      this.exists = e;
143
      this.initWatchContents();
144
      this.change();
145
    } else if (this.exists && !e) {
146
      // it got deleted
147
      this.exists = e;
148
      if (this.watchContents != null) {
149
        this.watchContents.close();
150
        delete this.watchContents;
151
      }
152

153
      this.change();
154
    }
155
  };
156

157
  private change = (): void => {
158
    this.emit("change");
159
  };
160

161
  public close(): void {
162
    this.watchExistence?.close();
163
    this.watchContents?.close();
164
    close(this);
165
  }
166
}
167

168
export class MultipathWatcher extends EventEmitter {
169
  private paths: { [path: string]: Watcher } = {};
170
  private options;
171

172
  constructor(options?) {
173
    super();
174
    this.options = options;
175
  }
176

177
  has = (path: string) => {
178
    return this.paths[path] != null;
179
  };
180

181
  add = (path: string) => {
182
    if (this.has(path)) {
183
      // already watching
184
      return;
185
    }
186
    this.paths[path] = new Watcher(path, this.options);
187
    this.paths[path].on("change", () => this.emit("change", path));
188
  };
189

190
  delete = (path: string) => {
191
    if (!this.has(path)) {
192
      return;
193
    }
194
    this.paths[path].close();
195
    delete this.paths[path];
196
  };
197

198
  close = () => {
199
    for (const path in this.paths) {
200
      this.delete(path);
201
    }
202
  };
203
}
204

205