1
/*
2
Test async streaming read of files from a compute servers using NATS.
3

4

5
DEVELOPMENT:
6

7
pnpm test ./read.test.ts
8

9
*/
10

11
import { before, after } from "@cocalc/backend/conat/test/setup";
12

13
beforeAll(before);
14

15
import { close, createServer, readFile } from "@cocalc/conat/files/read";
16
import { createReadStream } from "fs";
17
import { file as tempFile } from "tmp-promise";
18
import { writeFile as fsWriteFile } from "fs/promises";
19
import { sha1 } from "@cocalc/backend/sha1";
20

21
describe("do a basic test that the file read service works", () => {
22
  const project_id = "00000000-0000-4000-8000-000000000000";
23
  const compute_server_id = 0;
24
  it("create the read server", async () => {
25
    await createServer({
26
      project_id,
27
      compute_server_id,
28
      createReadStream,
29
    });
30
  });
31

32
  let cleanups: any[] = [];
33
  const CONTENT = "cocalc";
34
  let source;
35
  it("creates the file we will read", async () => {
36
    const { path, cleanup } = await tempFile();
37
    source = path;
38
    await fsWriteFile(path, CONTENT);
39
    cleanups.push(cleanup);
40
  });
41

42
  it("reads the file into memory", async () => {
43
    const r = await readFile({ project_id, compute_server_id, path: source });
44
    // will get just one chunk
45
    for await (const chunk of r) {
46
      expect(chunk.toString()).toEqual(CONTENT);
47
    }
48
  });
49

50
  it("closes the write server", async () => {
51
    close({ project_id, compute_server_id });
52
    for (const f of cleanups) {
53
      f();
54
    }
55
  });
56
});
57

58
describe("do a larger test that involves multiple chunks and a different name", () => {
59
  const project_id = "00000000-0000-4000-8000-000000000000";
60
  const compute_server_id = 0;
61
  const name = "b";
62
  it("create the read server", async () => {
63
    await createServer({
64
      project_id,
65
      compute_server_id,
66
      createReadStream,
67
      name,
68
    });
69
  });
70

71
  let cleanups: any[] = [];
72
  let CONTENT = "";
73
  for (let i = 0; i < 1000000; i++) {
74
    CONTENT += `${i}`;
75
  }
76
  let source;
77
  it("creates the file we will read", async () => {
78
    const { path, cleanup } = await tempFile();
79
    source = path;
80
    await fsWriteFile(path, CONTENT);
81
    cleanups.push(cleanup);
82
  });
83

84
  it("reads the file into memory", async () => {
85
    const r = await readFile({
86
      project_id,
87
      compute_server_id,
88
      path: source,
89
      name,
90
    });
91
    // will get many chunks.
92
    let chunks: any[] = [];
93
    for await (const chunk of r) {
94
      chunks.push(chunk);
95
    }
96
    expect(chunks.length).toBeGreaterThan(1);
97
    const s = Buffer.concat(chunks).toString();
98
    expect(s.length).toBe(CONTENT.length);
99
    expect(sha1(s)).toEqual(sha1(CONTENT));
100
  });
101

102
  it("closes the write server", async () => {
103
    close({ project_id, compute_server_id, name });
104
    for (const f of cleanups) {
105
      f();
106
    }
107
  });
108
});
109

110

111
afterAll(after);
112

113