1
/*
2
 * smoke-all.test.ts
3
 *
4
 * Copyright (C) 2022 Posit Software, PBC
5
 */
6

7
import { expandGlobSync } from "../../src/core/deno/expand-glob.ts";
8
import { testQuartoCmd, Verify } from "../test.ts";
9
import { initYamlIntelligenceResourcesFromFilesystem } from "../../src/core/schema/utils.ts";
10
import {
11
  initState,
12
  setInitializer,
13
} from "../../src/core/lib/yaml-validation/state.ts";
14
import { os } from "../../src/deno_ral/platform.ts";
15
import { asArray } from "../../src/core/array.ts";
16

17
import { breakQuartoMd } from "../../src/core/lib/break-quarto-md.ts";
18
import { parse } from "../../src/core/yaml.ts";
19
import { cleanoutput } from "./render/render.ts";
20
import {
21
  ensureCssRegexMatches,
22
  ensureEpubFileRegexMatches,
23
  ensureDocxRegexMatches,
24
  ensureDocxXpath,
25
  ensureFileRegexMatches,
26
  ensureHtmlElements,
27
  ensureIpynbCellMatches,
28
  ensurePdfRegexMatches,
29
  ensurePdfTextPositions,
30
  ensurePdfMetadata,
31
  ensureJatsXpath,
32
  ensureOdtXpath,
33
  ensurePptxRegexMatches,
34
  ensureTypstFileRegexMatches,
35
  ensureSnapshotMatches,
36
  fileExists,
37
  noErrors,
38
  noErrorsOrWarnings,
39
  ensurePptxXpath,
40
  ensurePptxLayout,
41
  ensurePptxMaxSlides,
42
  ensureLatexFileRegexMatches,
43
  printsMessage,
44
  shouldError,
45
  ensureHtmlElementContents,
46
  ensureHtmlElementCount,
47
  ensureLlmsMdRegexMatches,
48
  ensureLlmsMdExists,
49
  ensureLlmsMdDoesNotExist,
50
  ensureLlmsTxtRegexMatches,
51
  ensureLlmsTxtExists,
52
  ensureLlmsTxtDoesNotExist,
53
} from "../verify.ts";
54
import { readYamlFromMarkdown } from "../../src/core/yaml.ts";
55
import { findProjectDir, findProjectOutputDir, outputForInput } from "../utils.ts";
56
import { jupyterNotebookToMarkdown } from "../../src/command/convert/jupyter.ts";
57
import { basename, dirname, join, relative } from "../../src/deno_ral/path.ts";
58
import { WalkEntry } from "../../src/deno_ral/fs.ts";
59
import { quarto } from "../../src/quarto.ts";
60
import { safeExistsSync, safeRemoveSync } from "../../src/core/path.ts";
61
import { runningInCI } from "../../src/core/ci-info.ts";
62

63
async function fullInit() {
64
  await initYamlIntelligenceResourcesFromFilesystem();
65
}
66

67
async function guessFormat(fileName: string): Promise<string[]> {
68
  const { cells } = await breakQuartoMd(Deno.readTextFileSync(fileName));
69

70
  const formats: Set<string> = new Set();
71

72
  for (const cell of cells) {
73
    if (cell.cell_type === "raw") {
74
      const src = cell.source.value.replaceAll(/^---$/mg, "");
75
      let yaml;
76
      try {
77
        yaml = parse(src);
78
      } catch (e) {
79
        if (!(e instanceof Error)) throw e;
80
        if (e.message.includes("unknown tag")) {
81
          // assume it's not necessary to guess the format
82
          continue;
83
        }
84
      }
85
      if (yaml && typeof yaml === "object") {
86
        // deno-lint-ignore no-explicit-any
87
        const format = (yaml as Record<string, any>).format;
88
        if (typeof format === "object") {
89
          for (
90
            const [k, _] of Object.entries(
91
              // deno-lint-ignore no-explicit-any
92
              (yaml as Record<string, any>).format || {},
93
            )
94
          ) {
95
            formats.add(k);
96
          }
97
        } else if (typeof format === "string") {
98
          formats.add(format);
99
        }
100
      }
101
    }
102
  }
103
  return Array.from(formats);
104
}
105

106
function skipTest(metadata: Record<string, any>): string | undefined {
107
  // deno-lint-ignore no-explicit-any
108
  const quartoMeta = metadata["_quarto"] as any;
109
  const runConfig = quartoMeta?.tests?.run;
110

111
  // No run config means run everywhere
112
  if (!runConfig) {
113
    return undefined;
114
  }
115

116
  // Check explicit skip with message
117
  if (runConfig.skip) {
118
    return typeof runConfig.skip === "string" ? runConfig.skip : "tests.run.skip is true";
119
  }
120

121
  // Check CI
122
  if (runningInCI() && runConfig.ci === false) {
123
    return "tests.run.ci is false";
124
  }
125

126
  // Check OS blacklist (not_os)
127
  const notOs = runConfig.not_os;
128
  if (notOs !== undefined && asArray(notOs).includes(os)) {
129
    return `tests.run.not_os includes ${os}`;
130
  }
131

132
  // Check OS whitelist (os) - if specified, must match
133
  const onlyOs = runConfig.os;
134
  if (onlyOs !== undefined && !asArray(onlyOs).includes(os)) {
135
    return `tests.run.os does not include ${os}`;
136
  }
137

138
  return undefined;
139
}
140

141
//deno-lint-ignore no-explicit-any
142
function hasTestSpecs(metadata: any, input: string): boolean {
143
  const tests = metadata?.["_quarto"]?.["tests"];
144
  if (!tests && metadata?.["_quarto"]?.["test"] != undefined) {
145
    throw new Error(`Test is ${input} is using 'test' in metadata instead of 'tests'. This is probably a typo.`);
146
  }
147
  // Check if tests has any format specs (keys other than 'run')
148
  if (tests && typeof tests === "object") {
149
    const formatKeys = Object.keys(tests).filter(key => key !== "run");
150
    return formatKeys.length > 0;
151
  }
152
  return false;
153
}
154

155
interface QuartoInlineTestSpec {
156
  format: string;
157
  verifyFns: Verify[];
158
}
159

160
// Functions to cleanup leftover testing
161
const postRenderCleanupFiles: string[] = [];
162
function registerPostRenderCleanupFile(file: string): void {
163
  postRenderCleanupFiles.push(file);
164
}
165
const postRenderCleanup = () => {
166
  if (Deno.env.get("QUARTO_TEST_KEEP_OUTPUTS")) {
167
    return;
168
  }
169
  for (const file of postRenderCleanupFiles) {
170
    console.log(`Cleaning up ${file} in ${Deno.cwd()}`);
171
    if (safeExistsSync(file)) {
172
      Deno.removeSync(file);
173
    }
174
  }
175
}
176

177
function resolveTestSpecs(
178
  input: string,
179
  // deno-lint-ignore no-explicit-any
180
  metadata: Record<string, any>,
181
): QuartoInlineTestSpec[] {
182
  const specs = metadata["_quarto"]["tests"];
183

184
  const result = [];
185
  // deno-lint-ignore no-explicit-any
186
  const verifyMap: Record<string, any> = {
187
    ensureCssRegexMatches,
188
    ensureEpubFileRegexMatches,
189
    ensureHtmlElements,
190
    ensureHtmlElementContents,
191
    ensureHtmlElementCount,
192
    ensureFileRegexMatches,
193
    ensureIpynbCellMatches,
194
    ensureLatexFileRegexMatches,
195
    ensureTypstFileRegexMatches,
196
    ensureDocxRegexMatches,
197
    ensureDocxXpath,
198
    ensureOdtXpath,
199
    ensureJatsXpath,
200
    ensurePdfRegexMatches,
201
    ensurePdfTextPositions,
202
    ensurePdfMetadata,
203
    ensurePptxRegexMatches,
204
    ensurePptxXpath,
205
    ensurePptxLayout,
206
    ensurePptxMaxSlides,
207
    ensureSnapshotMatches,
208
    printsMessage,
209
    ensureLlmsMdRegexMatches,
210
    ensureLlmsMdExists,
211
    ensureLlmsMdDoesNotExist,
212
    ensureLlmsTxtRegexMatches,
213
    ensureLlmsTxtExists,
214
    ensureLlmsTxtDoesNotExist,
215
  };
216

217
  for (const [format, testObj] of Object.entries(specs)) {
218
    // Skip the 'run' key - it's not a format
219
    if (format === "run") {
220
      continue;
221
    }
222
    let checkWarnings = true;
223
    const verifyFns: Verify[] = [];
224
    if (testObj && typeof testObj === "object") {
225
      for (
226
        // deno-lint-ignore no-explicit-any
227
        const [key, value] of Object.entries(testObj as Record<string, any>)
228
      ) {
229
        if (key == "postRenderCleanup") {
230
          // This is a special key to register cleanup operations
231
          // each entry is a file to cleanup relative to the input file
232
          for (let file of value) {
233
            // if value has `${input_stem}` in the string, replace by input_stem value (input file name without extension)
234
            if (file.includes("${input_stem}")) {
235
              const extension = input.endsWith('.qmd') ? '.qmd' : '.ipynb';
236
              const inputStem = basename(input, extension);
237
              file = file.replace("${input_stem}", inputStem);
238
            }
239
            // file is registered for cleanup in testQuartoCmd teardown step
240
            registerPostRenderCleanupFile(join(dirname(input), file));
241
          }
242
        } else if (key == "shouldError") {
243
          checkWarnings = false;
244
          verifyFns.push(shouldError);
245
        } else if (key === "noErrors") {
246
          checkWarnings = false;
247
          verifyFns.push(noErrors);
248
        } else if (key === "noErrorsOrWarnings") {
249
          checkWarnings = false;
250
          verifyFns.push(noErrorsOrWarnings);
251
        } else {
252
          // See if there is a project and grab it's type
253
          const projectPath = findRootTestsProjectDir(input)
254
          const projectOutDir = findProjectOutputDir(projectPath);
255
          const outputFile = outputForInput(input, format, projectOutDir, projectPath, metadata);
256
          if (key === "fileExists") {
257
            for (
258
              const [path, file] of Object.entries(
259
                value as Record<string, string>,
260
              )
261
            ) {
262
              if (path === "outputPath") {
263
                verifyFns.push(
264
                  fileExists(join(dirname(outputFile.outputPath), file)),
265
                );
266
              } else if (path === "supportPath") {
267
                verifyFns.push(
268
                  fileExists(join(outputFile.supportPath, file)),
269
                );
270
              }
271
            }
272
          } else if (["ensurePptxLayout", "ensurePptxXpath"].includes(key)) {
273
            if (Array.isArray(value) && Array.isArray(value[0])) {
274
              // several slides to check
275
              value.forEach((slide: any) => {
276
                verifyFns.push(verifyMap[key](outputFile.outputPath, ...slide));
277
              });
278
            } else {
279
              verifyFns.push(verifyMap[key](outputFile.outputPath, ...value));
280
            }
281
          } else if (key === "printsMessage") {
282
            // Support both single object and array of printsMessage checks
283
            const messages = Array.isArray(value) ? value : [value];
284
            for (const msg of messages) {
285
              verifyFns.push(verifyMap[key](msg));
286
            }
287
          } else if (key === "ensureEpubFileRegexMatches") {
288
            // this ensure function is special because it takes an array of path + regex specifiers,
289
            // so we should never use the spread operator
290
            verifyFns.push(verifyMap[key](outputFile.outputPath, value));
291
          } else if (verifyMap[key]) {
292
            // FIXME: We should find another way that having this requirement of keep-* in the metadata
293
            if (key === "ensureTypstFileRegexMatches") {
294
              if (!metadata.format?.typst?.['keep-typ'] && !metadata['keep-typ'] && metadata.format?.typst?.['output-ext'] !== 'typ' && metadata['output-ext'] !== 'typ') {
295
                throw new Error(`Using ensureTypstFileRegexMatches requires setting 'keep-typ: true' in file ${input}`);
296
              }
297
            } else if (key === "ensureLatexFileRegexMatches") {
298
              if (!metadata.format?.pdf?.['keep-tex'] && !metadata['keep-tex']) {
299
                throw new Error(`Using ensureLatexFileRegexMatches requires setting 'keep-tex: true' in file ${input}`);
300
              }
301
            }
302

303
            // keep-typ/keep-tex files are alongside source, so pass input path
304
            // But output-ext: typ puts files in output directory, so don't pass input path
305
            const usesKeepTyp = key === "ensureTypstFileRegexMatches" &&
306
              (metadata.format?.typst?.['keep-typ'] || metadata['keep-typ']) &&
307
              !(metadata.format?.typst?.['output-ext'] === 'typ' || metadata['output-ext'] === 'typ');
308
            const usesKeepTex = key === "ensureLatexFileRegexMatches" &&
309
              (metadata.format?.pdf?.['keep-tex'] || metadata['keep-tex']);
310
            const needsInputPath = usesKeepTyp || usesKeepTex;
311

312
            // For book projects, use intermediateTypstPath (index.typ at project root)
313
            // instead of the output path (which would be _book/BookTitle.typ)
314
            let targetPath = outputFile.outputPath;
315
            if (key === "ensureTypstFileRegexMatches" && outputFile.intermediateTypstPath) {
316
              targetPath = outputFile.intermediateTypstPath;
317
            }
318

319
            if (typeof value === "object" && Array.isArray(value)) {
320
              // value is [matches, noMatches?] - ensure inputFile goes in the right position
321
              const matches = value[0];
322
              const noMatches = value[1];
323
              const inputFile = needsInputPath ? input : undefined;
324
              verifyFns.push(verifyMap[key](targetPath, matches, noMatches, inputFile));
325
            } else {
326
              verifyFns.push(verifyMap[key](targetPath, value, undefined, needsInputPath ? input : undefined));
327
            }
328
          } else {
329
            throw new Error(`Unknown verify function used: ${key} in file ${input} for format ${format}`) ;
330
          }
331
        }
332
      }
333
    }
334
    if (checkWarnings) {
335
      verifyFns.push(noErrorsOrWarnings);
336
    }
337

338
    result.push({
339
      format,
340
      verifyFns,
341
    });
342
  }
343
  return result;
344
}
345

346
await initYamlIntelligenceResourcesFromFilesystem();
347

348
// Ideally we'd just walk the one single glob here,
349
// but because smoke-all.test.ts ends up being called
350
// from a number of different places (including different shell
351
// scripts run under a variety of shells), it's
352
// actually non-trivial to guarantee that we'll see a single
353
// unexpanded glob pattern. So we assume that a pattern
354
// might have already been expanded here, and we also
355
// accommodate cases where it hasn't been expanded.
356
//
357
// (Do note that this means that files that don't exist will
358
// be silently ignored.)
359
const files: WalkEntry[] = [];
360
if (Deno.args.length === 0) {
361
  // ignore file starting with `_`
362
  files.push(...[...expandGlobSync("docs/smoke-all/**/*.{md,qmd,ipynb}")].filter((entry) => /^[^_]/.test(basename(entry.path))));
363
} else {
364
  for (const arg of Deno.args) {
365
    files.push(...expandGlobSync(arg));
366
  }
367
}
368

369
// To store project path we render before testing file testSpecs
370
const renderedProjects: Set<string> = new Set();
371
// To store information of all the project we render so that we can cleanup after testing
372
const testedProjects: Set<string> = new Set();
373

374
// Create an array to hold all the promises for the tests of files
375
let testFilesPromises = [];
376

377
for (const { path: fileName } of files) {
378
  const input = relative(Deno.cwd(), fileName);
379
  
380
  const metadata = input.endsWith("md") // qmd or md
381
    ? readYamlFromMarkdown(Deno.readTextFileSync(input))
382
    : readYamlFromMarkdown(await jupyterNotebookToMarkdown(input, false));
383

384
  const skipReason = skipTest(metadata);
385
  if (skipReason !== undefined) {
386
    console.log(`Skipping tests for ${input}: ${skipReason}`);
387
    continue;
388
  }
389

390
  const testSpecs: QuartoInlineTestSpec[] = [];
391

392
  if (hasTestSpecs(metadata, input)) {
393
    testSpecs.push(...resolveTestSpecs(input, metadata));
394
  } else {
395
    const formats = await guessFormat(input);
396

397
    if (formats.length == 0) {
398
      formats.push("html");
399
    }
400
    for (const format of formats) {
401
      testSpecs.push({ format: format, verifyFns: [noErrorsOrWarnings] });
402
    }
403
  }
404

405
  // Get project path for this input and store it if this is a project (used for cleaning)
406
  const projectPath = findRootTestsProjectDir(input);
407
  if (projectPath) testedProjects.add(projectPath);
408

409
  // Render project before testing individual document if required
410
  if (
411
    (metadata["_quarto"] as any)?.["render-project"] && 
412
    projectPath && 
413
    !renderedProjects.has(projectPath)
414
  ) {
415
      await quarto(["render", projectPath]);
416
      renderedProjects.add(projectPath);
417
    }
418

419
  testFilesPromises.push(new Promise<void>(async (resolve, reject) => {
420
    try {
421

422
      // Create an array to hold all the promises for the testSpecs
423
      let testSpecPromises = [];
424
      
425
      for (const testSpec of testSpecs) {
426
        const {
427
          format,
428
          verifyFns,
429
          //deno-lint-ignore no-explicit-any
430
        } = testSpec as any;
431
        testSpecPromises.push(new Promise<void>((testSpecResolve, testSpecReject) => {
432
          try {
433
            if (format === "editor-support-crossref") {
434
              const tempFile = Deno.makeTempFileSync();
435
              testQuartoCmd("editor-support", ["crossref", "--input", input, "--output", tempFile], verifyFns, {
436
                teardown: () => {
437
                  Deno.removeSync(tempFile);
438
                  testSpecResolve(); // Resolve the promise for the testSpec
439
                  return Promise.resolve();
440
                }
441
              }, `quarto editor-support crossref < ${input}`);
442
            } else {
443
              testQuartoCmd("render", [input, "--to", format], verifyFns, {
444
                prereq: async () => {
445
                  setInitializer(fullInit);
446
                  await initState();
447
                  return Promise.resolve(true);
448
                },
449
                teardown: () => {
450
                  cleanoutput(input, format, undefined, undefined, metadata);
451
                  postRenderCleanup()
452
                  testSpecResolve(); // Resolve the promise for the testSpec
453
                  return Promise.resolve();
454
                },
455
              });
456
            }
457
          } catch (error) {
458
            testSpecReject(error);
459
          }
460
        }));
461
          
462
      }
463

464
      // Wait for all the promises to resolve
465
      await Promise.all(testSpecPromises);
466

467
      // Resolve the promise for the file
468
      resolve();
469

470
    } catch (error) {
471
      reject(error);
472
    }
473
  }));
474
}
475

476
// Wait for all the promises to resolve
477
// Meaning all the files have been tested and we can clean
478
Promise.all(testFilesPromises).then(() => {
479
  if (Deno.env.get("QUARTO_TEST_KEEP_OUTPUTS")) {
480
    return;
481
  }
482
  // Clean up any projects that were tested
483
  for (const project of testedProjects) {
484
    // Clean project output directory
485
    const projectOutDir = join(project, findProjectOutputDir(project));
486
    if (projectOutDir !== project && safeExistsSync(projectOutDir)) {
487
      safeRemoveSync(projectOutDir, { recursive: true });
488
    }
489
    // Clean hidden .quarto directory
490
    const hiddenQuarto = join(project, ".quarto");
491
    if (safeExistsSync(hiddenQuarto)) {
492
      safeRemoveSync(hiddenQuarto, { recursive: true });
493
    }
494
  }
495
}).catch((_error) => {});
496

497
function findRootTestsProjectDir(input: string) {
498
  const smokeAllRootDir = 'smoke-all$'
499
  const ffMatrixRootDir = 'feature-format-matrix[/]qmd-files$'
500

501
  const RootTestsRegex = new RegExp(`${smokeAllRootDir}|${ffMatrixRootDir}`);
502
  
503
  return findProjectDir(input, RootTestsRegex);
504
}
505