1
/*
2
 * render.ts
3
 *
4
 * Copyright (C) 2020-2022 Posit Software, PBC
5
 */
6

7
import { ensureDirSync, existsSync } from "../../deno_ral/fs.ts";
8

9
import { dirname, isAbsolute, join, relative } from "../../deno_ral/path.ts";
10

11
import { Document, parseHtml } from "../../core/deno-dom.ts";
12

13
import { mergeConfigs } from "../../core/config.ts";
14
import { resourcePath } from "../../core/resources.ts";
15
import { inputFilesDir } from "../../core/render.ts";
16
import {
17
  normalizePath,
18
  pathWithForwardSlashes,
19
  safeExistsSync,
20
} from "../../core/path.ts";
21

22
import { FormatPandoc } from "../../config/types.ts";
23
import {
24
  executionEngine,
25
  executionEngineKeepMd,
26
} from "../../execute/engine.ts";
27
import { engineProjectContext } from "../../project/engine-project-context.ts";
28

29
import {
30
  HtmlPostProcessor,
31
  HtmlPostProcessResult,
32
  PandocInputTraits,
33
  PandocOptions,
34
  PandocRenderCompletion,
35
  RenderedFormat,
36
} from "./types.ts";
37
import { runPandoc } from "./pandoc.ts";
38
import { renderCleanup } from "./cleanup.ts";
39
import { projectOffset } from "../../project/project-shared.ts";
40

41
import { ExecutedFile, RenderedFile, RenderResult } from "./types.ts";
42
import { PandocIncludes } from "../../execute/types.ts";
43
import { Metadata } from "../../config/types.ts";
44
import { isHtmlFileOutput } from "../../config/format.ts";
45

46
import { isSelfContainedOutput } from "./render-info.ts";
47
import {
48
  pop as popTiming,
49
  push as pushTiming,
50
  withTiming,
51
  withTimingAsync,
52
} from "../../core/timing.ts";
53
import { filesDirMediabagDir } from "./render-paths.ts";
54
import { replaceNotebookPlaceholders } from "../../core/jupyter/jupyter-embed.ts";
55
import {
56
  kIncludeAfterBody,
57
  kIncludeBeforeBody,
58
  kIncludeInHeader,
59
  kInlineIncludes,
60
  kResourcePath,
61
} from "../../config/constants.ts";
62
import { pandocIngestSelfContainedContent } from "../../core/pandoc/self-contained.ts";
63
import { existsSync1 } from "../../core/file.ts";
64
import { projectType } from "../../project/types/project-types.ts";
65

66
export async function renderPandoc(
67
  file: ExecutedFile,
68
  quiet: boolean,
69
): Promise<PandocRenderCompletion> {
70
  // alias options
71
  const { context, recipe, executeResult, resourceFiles } = file;
72

73
  // alias format
74
  const format = recipe.format;
75

76
  // merge any pandoc options provided by the computation
77
  if (executeResult.includes) {
78
    format.pandoc = mergePandocIncludes(
79
      format.pandoc || {},
80
      executeResult.includes,
81
    );
82
  }
83
  if (executeResult.pandoc) {
84
    format.pandoc = mergeConfigs(
85
      format.pandoc || {},
86
      executeResult.pandoc,
87
    );
88
  }
89

90
  // run the dependencies step if we didn't do it during execute
91
  if (executeResult.engineDependencies) {
92
    for (const engineName of Object.keys(executeResult.engineDependencies)) {
93
      const engine = executionEngine(engineName)!;
94
      const engineInstance = engine.launch(engineProjectContext(context.project));
95
      const dependenciesResult = await engineInstance.dependencies({
96
        target: context.target,
97
        format,
98
        output: recipe.output,
99
        resourceDir: resourcePath(),
100
        tempDir: context.options.services.temp.createDir(),
101
        projectDir: context.project?.dir,
102
        libDir: context.libDir,
103
        dependencies: executeResult.engineDependencies[engineName],
104
        quiet: context.options.flags?.quiet,
105
      });
106
      format.pandoc = mergePandocIncludes(
107
        format.pandoc,
108
        dependenciesResult.includes,
109
      );
110
    }
111
  }
112

113
  // the mediabag dir should be created here based on the context
114
  // (it could be in the _files dir). if its a single file book
115
  // though it can even be in a temp dir
116
  const mediabagDir = filesDirMediabagDir(context.target.source);
117
  ensureDirSync(join(dirname(context.target.source), mediabagDir));
118

119
  // Process any placeholder for notebooks that have been injected
120
  const notebookResult = await replaceNotebookPlaceholders(
121
    format.pandoc.to || "html",
122
    context,
123
    context.options.flags || {},
124
    executeResult.markdown,
125
    context.options.services,
126
  );
127

128
  const embedSupporting: string[] = [];
129
  if (notebookResult.supporting.length) {
130
    embedSupporting.push(...notebookResult.supporting);
131
  }
132

133
  // Map notebook includes to pandoc includes
134
  const pandocIncludes: PandocIncludes = {
135
    [kIncludeAfterBody]: notebookResult.includes?.afterBody
136
      ? [notebookResult.includes?.afterBody]
137
      : undefined,
138
    [kIncludeInHeader]: notebookResult.includes?.inHeader
139
      ? [notebookResult.includes?.inHeader]
140
      : undefined,
141
  };
142

143
  // Inject dependencies
144
  format.pandoc = mergePandocIncludes(
145
    format.pandoc,
146
    pandocIncludes,
147
  );
148

149
  // resolve markdown. for [  ] output type we collect up
150
  // the includes so they can be proccessed by Lua
151
  let markdownInput = notebookResult.markdown
152
    ? notebookResult.markdown
153
    : executeResult.markdown;
154
  if (format.render[kInlineIncludes]) {
155
    const collectIncludes = (
156
      location:
157
        | "include-in-header"
158
        | "include-before-body"
159
        | "include-after-body",
160
    ) => {
161
      const includes = format.pandoc[location];
162
      if (includes) {
163
        const append = location === "include-after-body";
164
        for (const include of includes) {
165
          const includeMd = Deno.readTextFileSync(include);
166
          if (append) {
167
            markdownInput = `${markdownInput}\n\n${includeMd}`;
168
          } else {
169
            markdownInput = `${includeMd}\n\n${markdownInput}`;
170
          }
171
        }
172
        delete format.pandoc[location];
173
      }
174
    };
175
    collectIncludes(kIncludeInHeader);
176
    collectIncludes(kIncludeBeforeBody);
177
    collectIncludes(kIncludeAfterBody);
178
  }
179

180
  // pandoc options
181
  const pandocOptions: PandocOptions = {
182
    markdown: markdownInput,
183
    source: context.target.source,
184
    output: recipe.output,
185
    keepYaml: recipe.keepYaml,
186
    mediabagDir,
187
    libDir: context.libDir,
188
    format,
189
    executionEngine: executeResult.engine,
190
    project: context.project,
191
    args: recipe.args,
192
    services: context.options.services,
193
    metadata: executeResult.metadata,
194
    quiet,
195
    flags: context.options.flags,
196
  };
197

198
  // add offset if we are in a project
199
  if (context.project) {
200
    pandocOptions.offset = projectOffset(context.project, context.target.input);
201
  }
202

203
  // run pandoc conversion (exit on failure)
204
  const pandocResult = await runPandoc(pandocOptions, executeResult.filters);
205
  if (!pandocResult) {
206
    return Promise.reject();
207
  }
208

209
  return {
210
    complete: async (renderedFormats: RenderedFormat[], cleanup?: boolean) => {
211
      pushTiming("render-postprocessor");
212
      // run optional post-processor (e.g. to restore html-preserve regions)
213
      if (executeResult.postProcess) {
214
        await withTimingAsync("engine-postprocess", async () => {
215
          return await context.engine.postprocess({
216
            engine: context.engine,
217
            target: context.target,
218
            format,
219
            output: recipe.output,
220
            tempDir: context.options.services.temp.createDir(),
221
            projectDir: context.project?.dir,
222
            preserve: executeResult.preserve,
223
            quiet: context.options.flags?.quiet,
224
          });
225
        });
226
      }
227

228
      // run html postprocessors if we have them
229
      const canHtmlPostProcess = isHtmlFileOutput(format.pandoc);
230
      if (!canHtmlPostProcess && pandocResult.htmlPostprocessors.length > 0) {
231
        const postProcessorNames = pandocResult.htmlPostprocessors.map((p) =>
232
          p.name
233
        ).join(", ");
234
        const msg =
235
          `Attempt to HTML post process non HTML output using: ${postProcessorNames}`;
236
        throw new Error(msg);
237
      }
238
      const htmlPostProcessors = canHtmlPostProcess
239
        ? pandocResult.htmlPostprocessors
240
        : [];
241
      const htmlFinalizers = canHtmlPostProcess
242
        ? pandocResult.htmlFinalizers || []
243
        : [];
244

245
      const htmlPostProcessResult = await runHtmlPostprocessors(
246
        pandocResult.inputMetadata,
247
        pandocResult.inputTraits,
248
        pandocOptions,
249
        htmlPostProcessors,
250
        htmlFinalizers,
251
        renderedFormats,
252
        quiet,
253
      );
254

255
      // Compute the path to the output file
256
      const outputFile = isAbsolute(pandocOptions.output)
257
        ? pandocOptions.output
258
        : join(dirname(pandocOptions.source), pandocOptions.output);
259

260
      // run generic postprocessors
261
      const postProcessSupporting: string[] = [];
262
      const postProcessResources: string[] = [];
263
      if (pandocResult.postprocessors) {
264
        for (const postprocessor of pandocResult.postprocessors) {
265
          const result = await postprocessor(outputFile);
266
          if (result && result.supporting) {
267
            postProcessSupporting.push(...result.supporting);
268
          }
269
          if (result && result.resources) {
270
            postProcessResources.push(...result.resources);
271
          }
272
        }
273
      }
274

275
      let finalOutput: string;
276
      let selfContained: boolean;
277

278
      await withTimingAsync("postprocess-selfcontained", async () => {
279
        // ensure flags
280
        const flags = context.options.flags || {};
281
        // determine whether this is self-contained output
282
        finalOutput = recipe.output;
283

284
        // note that we intentionally call isSelfContainedOutput twice
285
        // the first needs to happen before recipe completion
286
        // because ingestion of self-contained output needs
287
        // to happen before recipe completion (which cleans up some files)
288
        selfContained = isSelfContainedOutput(
289
          flags,
290
          format,
291
          finalOutput,
292
        );
293

294
        if (selfContained && isHtmlFileOutput(format.pandoc)) {
295
          await pandocIngestSelfContainedContent(
296
            outputFile,
297
            format.pandoc[kResourcePath],
298
          );
299
        }
300

301
        // call complete handler (might e.g. run latexmk to complete the render)
302
        finalOutput = (await recipe.complete(pandocOptions)) || recipe.output;
303

304
        // note that we intentionally call isSelfContainedOutput twice
305
        // the second call happens because some recipes change
306
        // their output extension on completion (notably, .pdf files)
307
        // and become self-contained for purposes of cleanup
308
        selfContained = isSelfContainedOutput(
309
          flags,
310
          format,
311
          finalOutput,
312
        );
313
      });
314

315
      // compute the relative path to the files dir
316
      let filesDir: string | undefined = inputFilesDir(context.target.source);
317
      // undefine it if it doesn't exist
318
      filesDir = existsSync(join(dirname(context.target.source), filesDir))
319
        ? filesDir
320
        : undefined;
321

322
      // add any injected libs to supporting
323
      let supporting = filesDir ? executeResult.supporting : undefined;
324
      if (filesDir && isHtmlFileOutput(format.pandoc)) {
325
        const filesDirAbsolute = join(dirname(context.target.source), filesDir);
326
        if (
327
          existsSync(filesDirAbsolute) &&
328
          (!supporting || !supporting.includes(filesDirAbsolute))
329
        ) {
330
          const filesLibs = join(
331
            dirname(context.target.source),
332
            context.libDir,
333
          );
334
          if (
335
            existsSync(filesLibs) &&
336
            (!supporting || !supporting.includes(filesLibs))
337
          ) {
338
            supporting = supporting || [];
339
            supporting.push(filesLibs);
340
          }
341
        }
342
      }
343
      if (
344
        htmlPostProcessResult.supporting &&
345
        htmlPostProcessResult.supporting.length > 0
346
      ) {
347
        supporting = supporting || [];
348
        supporting.push(...htmlPostProcessResult.supporting);
349
      }
350
      if (embedSupporting && embedSupporting.length > 0) {
351
        supporting = supporting || [];
352
        supporting.push(...embedSupporting);
353
      }
354
      if (postProcessSupporting && postProcessSupporting.length > 0) {
355
        supporting = supporting || [];
356
        supporting.push(...postProcessSupporting);
357
      }
358

359
      // Deal with self contained by passing them to be cleaned up
360
      // but if this is a project, instead make sure that we're not
361
      // including the lib dir
362
      let cleanupSelfContained: string[] | undefined = undefined;
363
      if (selfContained! && supporting) {
364
        cleanupSelfContained = [...supporting];
365
        if (context.project!) {
366
          const libDir = context.project?.config?.project["lib-dir"];
367
          if (libDir) {
368
            const absLibDir = join(context.project.dir, libDir);
369
            cleanupSelfContained = cleanupSelfContained.filter((file) =>
370
              !file.startsWith(absLibDir)
371
            );
372
          }
373
        }
374
      }
375

376
      if (cleanup !== false) {
377
        withTiming("render-cleanup", () =>
378
          renderCleanup(
379
            context.target.input,
380
            finalOutput!,
381
            format,
382
            file.context.project,
383
            cleanupSelfContained,
384
            executionEngineKeepMd(context),
385
          ));
386
      }
387

388
      // Compute the project-relative path for the input source file.
389
      // Uses normalizePath to handle both relative and absolute source paths.
390
      const projectRelativeInput = (sourcePath: string) => {
391
        if (context.project) {
392
          return relative(
393
            normalizePath(context.project.dir),
394
            normalizePath(sourcePath),
395
          );
396
        }
397
        return sourcePath;
398
      };
399

400
      // Resolve an output file path to a project-relative path.
401
      // Output paths (like "page.html") are relative to the source file's
402
      // directory, so we join with dirname(target.source) before computing
403
      // the project-relative result. Absolute output paths pass through
404
      // normalizePath directly.
405
      const projectOutputPath = (path: string) => {
406
        if (context.project) {
407
          if (isAbsolute(path)) {
408
            return relative(
409
              normalizePath(context.project.dir),
410
              normalizePath(path),
411
            );
412
          } else {
413
            return relative(
414
              normalizePath(context.project.dir),
415
              normalizePath(join(dirname(context.target.source), path)),
416
            );
417
          }
418
        } else {
419
          return path;
420
        }
421
      };
422
      popTiming();
423

424
      // Forward along any specific resources
425
      const files = resourceFiles.concat(htmlPostProcessResult.resources)
426
        .concat(postProcessResources);
427

428
      const result: RenderedFile = {
429
        isTransient: recipe.isOutputTransient,
430
        input: projectRelativeInput(context.target.source),
431
        markdown: executeResult.markdown,
432
        format,
433
        supporting: supporting
434
          ? supporting.filter(existsSync1).map((file: string) =>
435
            context.project ? relative(context.project.dir, file) : file
436
          )
437
          : undefined,
438
        file: recipe.isOutputTransient
439
          ? finalOutput!
440
          : projectOutputPath(finalOutput!),
441
        resourceFiles: {
442
          globs: pandocResult.resources,
443
          files,
444
        },
445
        selfContained: selfContained!,
446
      };
447
      return result;
448
    },
449
  };
450
}
451

452
export function renderResultFinalOutput(
453
  renderResults: RenderResult,
454
  relativeToInputDir?: string,
455
) {
456
  // final output defaults to the first output of the first result
457
  // that isn't a supplemental render file (a file that wasn't explicitly
458
  // rendered but that was a side effect of rendering some other file)
459
  let result = renderResults.files.find((file) => {
460
    return !file.supplemental;
461
  });
462
  if (!result) {
463
    return undefined;
464
  }
465

466
  // see if we can find an index.html instead
467
  for (const fileResult of renderResults.files) {
468
    if (fileResult.file === "index.html" && !fileResult.supplemental) {
469
      result = fileResult;
470
      break;
471
    }
472
  }
473

474
  // Allow project types to provide this
475
  if (renderResults.context) {
476
    const projType = projectType(renderResults.context.config?.project.type);
477
    if (projType && projType.renderResultFinalOutput) {
478
      const projectResult = projType.renderResultFinalOutput(
479
        renderResults,
480
        relativeToInputDir,
481
      );
482
      if (projectResult) {
483
        result = projectResult;
484
      }
485
    }
486
  }
487

488
  // determine final output
489
  let finalInput = result.input;
490
  let finalOutput = result.file;
491

492
  if (renderResults.baseDir) {
493
    finalInput = join(renderResults.baseDir, finalInput);
494
    if (renderResults.outputDir) {
495
      finalOutput = join(
496
        renderResults.baseDir,
497
        renderResults.outputDir,
498
        finalOutput,
499
      );
500
    } else {
501
      finalOutput = join(renderResults.baseDir, finalOutput);
502
    }
503
  } else {
504
    finalOutput = join(dirname(finalInput), finalOutput);
505
  }
506

507
  // if the final output doesn't exist then we must have been targetin stdout,
508
  // so return undefined
509
  if (!safeExistsSync(finalOutput)) {
510
    return undefined;
511
  }
512

513
  // return a path relative to the input file
514
  if (relativeToInputDir) {
515
    const inputRealPath = normalizePath(relativeToInputDir);
516
    const outputRealPath = normalizePath(finalOutput);
517
    return relative(inputRealPath, outputRealPath);
518
  } else {
519
    return finalOutput;
520
  }
521
}
522

523
export function renderResultUrlPath(
524
  renderResult: RenderResult,
525
) {
526
  if (renderResult.baseDir && renderResult.outputDir) {
527
    const finalOutput = renderResultFinalOutput(
528
      renderResult,
529
    );
530
    if (finalOutput) {
531
      const targetPath = pathWithForwardSlashes(relative(
532
        join(renderResult.baseDir, renderResult.outputDir),
533
        finalOutput,
534
      ));
535
      return targetPath;
536
    }
537
  }
538
  return undefined;
539
}
540

541
function mergePandocIncludes(
542
  format: FormatPandoc,
543
  pandocIncludes: PandocIncludes,
544
) {
545
  return mergeConfigs(format, pandocIncludes);
546
}
547

548
async function runHtmlPostprocessors(
549
  inputMetadata: Metadata,
550
  inputTraits: PandocInputTraits,
551
  options: PandocOptions,
552
  htmlPostprocessors: Array<HtmlPostProcessor>,
553
  htmlFinalizers: Array<(doc: Document) => Promise<void>>,
554
  renderedFormats: RenderedFormat[],
555
  quiet?: boolean,
556
): Promise<HtmlPostProcessResult> {
557
  const postProcessResult: HtmlPostProcessResult = {
558
    resources: [],
559
    supporting: [],
560
  };
561
  if (htmlPostprocessors.length > 0 || htmlFinalizers.length > 0) {
562
    await withTimingAsync("htmlPostprocessors", async () => {
563
      const outputFile = isAbsolute(options.output)
564
        ? options.output
565
        : join(dirname(options.source), options.output);
566
      const htmlInput = Deno.readTextFileSync(outputFile);
567
      const doctypeMatch = htmlInput.match(/^<!DOCTYPE.*?>/);
568
      const doc = await parseHtml(htmlInput);
569
      for (let i = 0; i < htmlPostprocessors.length; i++) {
570
        const postprocessor = htmlPostprocessors[i];
571
        const result = await postprocessor(
572
          doc,
573
          {
574
            inputMetadata,
575
            inputTraits,
576
            renderedFormats,
577
            quiet,
578
          },
579
        );
580

581
        postProcessResult.resources.push(...result.resources);
582
        postProcessResult.supporting.push(...result.supporting);
583
      }
584

585
      // After the post processing is complete, allow any finalizers
586
      // an opportunity at the document
587
      for (let i = 0; i < htmlFinalizers.length; i++) {
588
        const finalizer = htmlFinalizers[i];
589
        await finalizer(doc);
590
      }
591

592
      const htmlOutput = (doctypeMatch ? doctypeMatch[0] + "\n" : "") +
593
        doc.documentElement?.outerHTML!;
594
      Deno.writeTextFileSync(outputFile, htmlOutput);
595
    });
596
  }
597
  return postProcessResult;
598
}
599

600