1
/*
2
 * pandoc.ts
3
 *
4
 * Copyright (C) 2020-2022 Posit Software, PBC
5
 */
6

7
import {
8
  basename,
9
  dirname,
10
  isAbsolute,
11
  join,
12
  resolve,
13
} from "../../deno_ral/path.ts";
14

15
import { info, warning } from "../../deno_ral/log.ts";
16

17
import { ensureDir, existsSync, expandGlobSync } from "../../deno_ral/fs.ts";
18

19
import { parse as parseYml, stringify } from "../../core/yaml.ts";
20
import { copyTo } from "../../core/copy.ts";
21
import { decodeBase64, encodeBase64 } from "encoding/base64";
22

23
import * as ld from "../../core/lodash.ts";
24

25
import { Document } from "../../core/deno-dom.ts";
26

27
import { execProcess } from "../../core/process.ts";
28
import { dirAndStem, normalizePath } from "../../core/path.ts";
29
import { mergeConfigs } from "../../core/config.ts";
30
import { isExternalPath } from "../../core/url.ts";
31

32
import {
33
  Format,
34
  FormatExtras,
35
  FormatPandoc,
36
  kBodyEnvelope,
37
  kDependencies,
38
  kHtmlFinalizers,
39
  kHtmlPostprocessors,
40
  kMarkdownAfterBody,
41
  kTextHighlightingMode,
42
} from "../../config/types.ts";
43
import {
44
  isAstOutput,
45
  isBeamerOutput,
46
  isEpubOutput,
47
  isHtmlDocOutput,
48
  isHtmlFileOutput,
49
  isHtmlOutput,
50
  isIpynbOutput,
51
  isLatexOutput,
52
  isMarkdownOutput,
53
  isRevealjsOutput,
54
  isTypstOutput,
55
} from "../../config/format.ts";
56
import {
57
  isIncludeMetadata,
58
  isQuartoMetadata,
59
  metadataGetDeep,
60
} from "../../config/metadata.ts";
61
import { pandocBinaryPath, resourcePath } from "../../core/resources.ts";
62
import { pandocAutoIdentifier } from "../../core/pandoc/pandoc-id.ts";
63
import {
64
  partitionYamlFrontMatter,
65
  readYamlFromMarkdown,
66
} from "../../core/yaml.ts";
67

68
import { ProjectContext } from "../../project/types.ts";
69

70
import {
71
  deleteProjectMetadata,
72
  projectIsBook,
73
  projectIsWebsite,
74
} from "../../project/project-shared.ts";
75
import { deleteCrossrefMetadata } from "../../project/project-crossrefs.ts";
76
import { migrateProjectScratchPath } from "../../project/project-scratch.ts";
77

78
import {
79
  getPandocArg,
80
  havePandocArg,
81
  kQuartoForwardedMetadataFields,
82
  removePandocArgs,
83
} from "./flags.ts";
84
import {
85
  generateDefaults,
86
  pandocDefaultsMessage,
87
  writeDefaultsFile,
88
} from "./defaults.ts";
89
import { filterParamsJson, removeFilterParams } from "./filters.ts";
90
import {
91
  kAbstract,
92
  kAbstractTitle,
93
  kAuthor,
94
  kAuthors,
95
  kClassOption,
96
  kColorLinks,
97
  kColumns,
98
  kDate,
99
  kDateFormat,
100
  kDateModified,
101
  kDocumentClass,
102
  kEmbedResources,
103
  kFigResponsive,
104
  kFilterParams,
105
  kFontPaths,
106
  kFormatResources,
107
  kFrom,
108
  kHighlightStyle,
109
  kHtmlMathMethod,
110
  kIncludeAfterBody,
111
  kIncludeBeforeBody,
112
  kIncludeInHeader,
113
  kInstitute,
114
  kInstitutes,
115
  kKeepSource,
116
  kLatexAutoMk,
117
  kLinkColor,
118
  kMath,
119
  kMetadataFormat,
120
  kNotebooks,
121
  kNotebookView,
122
  kNumberOffset,
123
  kNumberSections,
124
  kPageTitle,
125
  kQuartoInternal,
126
  kQuartoTemplateParams,
127
  kQuartoVarsKey,
128
  kQuartoVersion,
129
  kResources,
130
  kRevealJsScripts,
131
  kSectionTitleAbstract,
132
  kSelfContained,
133
  kSyntaxDefinitions,
134
  kSyntaxHighlighting,
135
  kTemplate,
136
  kTheme,
137
  kTitle,
138
  kTitlePrefix,
139
  kTocLocation,
140
  kTocTitle,
141
  kTocTitleDocument,
142
  kTocTitleWebsite,
143
  kVariables,
144
} from "../../config/constants.ts";
145
import { TempContext } from "../../core/temp.ts";
146
import { discoverResourceRefs, fixEmptyHrefs } from "../../core/html.ts";
147

148
import { kDefaultHighlightStyle } from "./constants.ts";
149
import {
150
  HtmlPostProcessor,
151
  HtmlPostProcessResult,
152
  PandocOptions,
153
  RunPandocResult,
154
} from "./types.ts";
155
import { crossrefFilterActive } from "./crossref.ts";
156
import { overflowXPostprocessor } from "./layout.ts";
157
import {
158
  codeToolsPostprocessor,
159
  formatHasCodeTools,
160
  keepSourceBlock,
161
} from "./codetools.ts";
162
import { pandocMetadataPath } from "./render-paths.ts";
163
import { Metadata } from "../../config/types.ts";
164
import { resourcesFromMetadata } from "./resources.ts";
165
import { resolveSassBundles } from "./pandoc-html.ts";
166
import {
167
  cleanTemplatePartialMetadata,
168
  kTemplatePartials,
169
  readPartials,
170
  resolveTemplatePartialPaths,
171
  stageTemplate,
172
} from "./template.ts";
173
import {
174
  kYamlMetadataBlock,
175
  pandocFormatWith,
176
  parseFormatString,
177
  splitPandocFormatString,
178
} from "../../core/pandoc/pandoc-formats.ts";
179
import { cslNameToString, parseAuthor } from "../../core/author.ts";
180
import { logLevel } from "../../core/log.ts";
181

182
import { cacheCodePage, clearCodePageCache } from "../../core/windows.ts";
183
import { textHighlightThemePath } from "../../quarto-core/text-highlighting.ts";
184
import { resolveAndFormatDate, resolveDate } from "../../core/date.ts";
185
import { katexPostProcessor } from "../../format/html/format-html-math.ts";
186
import {
187
  readAndInjectDependencies,
188
  writeDependencies,
189
} from "./pandoc-dependencies-html.ts";
190
import {
191
  processFormatResources,
192
  writeFormatResources,
193
} from "./pandoc-dependencies-resources.ts";
194
import { withTiming } from "../../core/timing.ts";
195

196
import {
197
  requiresShortcodeUnescapePostprocessor,
198
  shortcodeUnescapePostprocessor,
199
} from "../../format/markdown/format-markdown.ts";
200

201
import { kRevealJSPlugins } from "../../extension/constants.ts";
202
import { kCitation } from "../../format/html/format-html-shared.ts";
203
import { cslDate } from "../../core/csl.ts";
204
import {
205
  createMarkdownPipeline,
206
  MarkdownPipelineHandler,
207
} from "../../core/markdown-pipeline.ts";
208
import { getenv } from "../../core/env.ts";
209
import { Zod } from "../../resources/types/zod/schema-types.ts";
210
import { kFieldCategories } from "../../project/types/website/listing/website-listing-shared.ts";
211
import { isWindows } from "../../deno_ral/platform.ts";
212
import { appendToCombinedLuaProfile } from "../../core/performance/perfetto-utils.ts";
213
import { makeTimedFunctionAsync } from "../../core/performance/function-times.ts";
214
import { walkJson } from "../../core/json.ts";
215
import { safeCloneDeep } from "../../core/safe-clone-deep.ts";
216
import { assert } from "testing/asserts";
217
import { call } from "../../deno_ral/process.ts";
218

219
// in case we are running multiple pandoc processes
220
// we need to make sure we capture all of the trace files
221
let traceCount = 0;
222

223
const handleCombinedLuaProfiles = (
224
  source: string,
225
  paramsJson: Record<string, unknown>,
226
  temp: TempContext,
227
) => {
228
  const beforePandocHooks: (() => unknown)[] = [];
229
  const afterPandocHooks: (() => unknown)[] = [];
230
  const tmp = temp.createFile();
231

232
  const combinedProfile = Deno.env.get("QUARTO_COMBINED_LUA_PROFILE");
233
  if (combinedProfile) {
234
    beforePandocHooks.push(() => {
235
      paramsJson["lua-profiler-output"] = tmp;
236
    });
237
    afterPandocHooks.push(() => {
238
      appendToCombinedLuaProfile(
239
        source,
240
        tmp,
241
        combinedProfile,
242
      );
243
    });
244
  }
245
  return {
246
    before: beforePandocHooks,
247
    after: afterPandocHooks,
248
  };
249
};
250

251
function captureRenderCommand(
252
  args: Deno.CommandOptions,
253
  temp: TempContext,
254
  outputDir: string,
255
) {
256
  Deno.mkdirSync(outputDir, { recursive: true });
257
  const newArgs: typeof args.args = (args.args ?? []).map((_arg) => {
258
    const arg = _arg as string; // we know it's a string, TypeScript doesn't somehow
259
    if (!arg.startsWith(temp.baseDir)) {
260
      return arg;
261
    }
262
    const newArg = join(outputDir, basename(arg));
263
    if (arg.match(/^.*quarto\-defaults.*.yml$/)) {
264
      // we need to correct the defaults YML because it contains a reference to a template in a temp directory
265
      const ymlDefaults = Deno.readTextFileSync(arg);
266
      const defaults = parseYml(ymlDefaults);
267
      const templateDirectory = dirname(defaults.template);
268
      const newTemplateDirectory = join(
269
        outputDir,
270
        basename(templateDirectory),
271
      );
272
      copyTo(templateDirectory, newTemplateDirectory);
273
      defaults.template = join(
274
        newTemplateDirectory,
275
        basename(defaults.template),
276
      );
277
      const defaultsOutputFile = join(outputDir, basename(arg));
278
      Deno.writeTextFileSync(defaultsOutputFile, stringify(defaults));
279
      return defaultsOutputFile;
280
    }
281
    Deno.copyFileSync(arg, newArg);
282
    return newArg;
283
  });
284

285
  // now we need to correct entries in filterParams
286
  const filterParams = JSON.parse(
287
    new TextDecoder().decode(decodeBase64(args.env!["QUARTO_FILTER_PARAMS"])),
288
  );
289
  walkJson(
290
    filterParams,
291
    (v: unknown) => typeof v === "string" && v.startsWith(temp.baseDir),
292
    (_v: unknown) => {
293
      const v = _v as string;
294
      const newV = join(outputDir, basename(v));
295
      Deno.copyFileSync(v, newV);
296
      return newV;
297
    },
298
  );
299

300
  Deno.writeTextFileSync(
301
    join(outputDir, "render-command.json"),
302
    JSON.stringify(
303
      {
304
        ...args,
305
        args: newArgs,
306
        env: {
307
          ...args.env,
308
          "QUARTO_FILTER_PARAMS": encodeBase64(JSON.stringify(filterParams)),
309
        },
310
      },
311
      undefined,
312
      2,
313
    ),
314
  );
315
}
316

317
export async function runPandoc(
318
  options: PandocOptions,
319
  sysFilters: string[],
320
): Promise<RunPandocResult | null> {
321
  const beforePandocHooks: (() => unknown)[] = [];
322
  const afterPandocHooks: (() => unknown)[] = [];
323
  const setupPandocHooks = (
324
    hooks: { before: (() => unknown)[]; after: (() => unknown)[] },
325
  ) => {
326
    beforePandocHooks.push(...hooks.before);
327
    afterPandocHooks.push(...hooks.after);
328
  };
329

330
  const pandocEnv: { [key: string]: string } = {};
331

332
  const setupPandocEnv = () => {
333
    pandocEnv["QUARTO_FILTER_PARAMS"] = encodeBase64(
334
      JSON.stringify(paramsJson),
335
    );
336

337
    const traceFilters =
338
      // deno-lint-ignore no-explicit-any
339
      (pandocMetadata as any)?.["_quarto"]?.["trace-filters"] ||
340
      Deno.env.get("QUARTO_TRACE_FILTERS");
341

342
    if (traceFilters) {
343
      // in case we are running multiple pandoc processes
344
      // we need to make sure we capture all of the trace files
345
      let traceCountSuffix = "";
346
      if (traceCount > 0) {
347
        traceCountSuffix = `-${traceCount}`;
348
      }
349
      ++traceCount;
350
      if (traceFilters === true) {
351
        pandocEnv["QUARTO_TRACE_FILTERS"] = "quarto-filter-trace.json" +
352
          traceCountSuffix;
353
      } else {
354
        pandocEnv["QUARTO_TRACE_FILTERS"] = traceFilters + traceCountSuffix;
355
      }
356
    }
357

358
    // https://github.com/quarto-dev/quarto-cli/issues/8274
359
    // do not use the default LUA_CPATH, as it will cause pandoc to
360
    // load the system lua libraries, which may not be compatible with
361
    // the lua version we are using
362
    if (Deno.env.get("QUARTO_LUA_CPATH") !== undefined) {
363
      pandocEnv["LUA_CPATH"] = getenv("QUARTO_LUA_CPATH");
364
    } else {
365
      pandocEnv["LUA_CPATH"] = "";
366
    }
367
  };
368

369
  // compute cwd for render
370
  const cwd = dirname(options.source);
371

372
  // build the pandoc command (we'll feed it the input on stdin)
373
  const cmd = [pandocBinaryPath(), "+RTS", "-K512m", "-RTS"];
374

375
  // build command line args
376
  const args = [...options.args];
377

378
  // propagate debug
379
  if (logLevel() === "DEBUG") {
380
    args.push("--verbose");
381
    args.push("--trace");
382
  }
383

384
  // propagate quiet
385
  if (options.flags?.quiet || logLevel() === "ERROR") {
386
    args.push("--quiet");
387
  }
388

389
  // merge in any extra metadata
390
  if (options.metadata) {
391
    options.format.metadata = mergeConfigs(
392
      options.format.metadata,
393
      options.metadata,
394
    );
395
  }
396

397
  // save args and metadata so we can print them (we may subsequently edit them)
398
  const printArgs = [...args];
399
  let printMetadata = {
400
    ...options.format.metadata,
401
    crossref: {
402
      ...(options.format.metadata.crossref || {}),
403
    },
404
    ...options.flags?.metadata,
405
  } as Metadata;
406

407
  const cleanQuartoTestsMetadata = (metadata: Metadata) => {
408
    // remove any metadata that is only used for testing
409
    if (metadata["_quarto"] && typeof metadata["_quarto"] === "object") {
410
      delete (metadata._quarto as { [key: string]: unknown })?.tests;
411
      if (Object.keys(metadata._quarto).length === 0) {
412
        delete metadata._quarto;
413
      }
414
    }
415
  };
416

417
  // remove some metadata that are used as parameters to our lua filters
418
  const cleanMetadataForPrinting = (metadata: Metadata) => {
419
    delete metadata.params;
420
    delete metadata[kQuartoInternal];
421
    delete metadata[kQuartoVarsKey];
422
    delete metadata[kQuartoVersion];
423
    delete metadata[kFigResponsive];
424
    delete metadata[kQuartoTemplateParams];
425
    delete metadata[kRevealJsScripts];
426
    deleteProjectMetadata(metadata);
427
    deleteCrossrefMetadata(metadata);
428
    removeFilterParams(metadata);
429

430
    // Don't print empty reveal-js plugins
431
    if (
432
      metadata[kRevealJSPlugins] &&
433
      (metadata[kRevealJSPlugins] as Array<unknown>).length === 0
434
    ) {
435
      delete metadata[kRevealJSPlugins];
436
    }
437

438
    // Don't print _quarto.tests
439
    // This can cause issue on regex test for printed output
440
    cleanQuartoTestsMetadata(metadata);
441

442
    // Filter out bundled engines from the engines array
443
    if (Array.isArray(metadata.engines)) {
444
      const filteredEngines = metadata.engines.filter((engine) => {
445
        const enginePath = typeof engine === "string" ? engine : engine.path;
446
        // Keep user engines, filter out bundled ones
447
        return !enginePath?.replace(/\\/g, "/").includes(
448
          "resources/extension-subtrees/",
449
        );
450
      });
451

452
      // Remove the engines key entirely if empty, otherwise assign filtered array
453
      if (filteredEngines.length === 0) {
454
        delete metadata.engines;
455
      } else {
456
        metadata.engines = filteredEngines;
457
      }
458
    }
459
  };
460

461
  cleanMetadataForPrinting(printMetadata);
462

463
  // Forward flags metadata into the format
464
  kQuartoForwardedMetadataFields.forEach((field) => {
465
    if (options.flags?.pandocMetadata?.[field]) {
466
      options.format.metadata[field] = options.flags.pandocMetadata[field];
467
    }
468
  });
469

470
  // generate defaults and capture defaults to be printed
471
  let allDefaults = (await generateDefaults(options)) || {};
472
  let printAllDefaults = safeCloneDeep(allDefaults);
473

474
  // capture any filterParams in the FormatExtras
475
  const formatFilterParams = {} as Record<string, unknown>;
476

477
  // Note whether we should be forcing math on for this render
478

479
  const forceMath = options.format.metadata[kMath];
480
  delete options.format.metadata[kMath];
481

482
  // the "ojs" filter is a special value that results in us
483
  // just signaling our standard filter chain that the ojs
484
  // filter should be active
485
  const kOJSFilter = "ojs";
486
  if (sysFilters.includes(kOJSFilter)) {
487
    formatFilterParams[kOJSFilter] = true;
488
    sysFilters = sysFilters.filter((filter) => filter !== kOJSFilter);
489
  }
490

491
  // pass the format language along to filter params
492
  formatFilterParams["language"] = options.format.language;
493

494
  // if there is no toc title then provide the appropirate default
495
  if (
496
    !options.format.metadata[kTocTitle] && !isAstOutput(options.format.pandoc)
497
  ) {
498
    options.format.metadata[kTocTitle] = options.format.language[
499
      (projectIsWebsite(options.project) && !projectIsBook(options.project) &&
500
          isHtmlOutput(options.format.pandoc, true))
501
        ? kTocTitleWebsite
502
        : kTocTitleDocument
503
    ];
504
  }
505

506
  // if toc-location is set, enable the TOC as well
507
  if (
508
    options.format.metadata[kTocLocation] &&
509
    options.format.pandoc.toc === undefined
510
  ) {
511
    options.format.pandoc.toc = true;
512
  }
513

514
  // if there is an abtract then forward abtract-title
515
  if (
516
    options.format.metadata[kAbstract] &&
517
    (isHtmlDocOutput(options.format.pandoc) ||
518
      isEpubOutput(options.format.pandoc))
519
  ) {
520
    options.format.metadata[kAbstractTitle] =
521
      options.format.metadata[kAbstractTitle] ||
522
      options.format.language[kSectionTitleAbstract];
523
  }
524

525
  // see if there are extras
526
  const postprocessors: Array<
527
    (
528
      output: string,
529
    ) => Promise<{ supporting?: string[]; resources?: string[] } | void>
530
  > = [];
531
  const htmlPostprocessors: Array<HtmlPostProcessor> = [];
532
  const htmlFinalizers: Array<(doc: Document) => Promise<void>> = [];
533
  const htmlRenderAfterBody: string[] = [];
534
  const dependenciesFile = options.services.temp.createFile();
535

536
  if (
537
    sysFilters.length > 0 || options.format.formatExtras ||
538
    options.project?.formatExtras
539
  ) {
540
    const projectExtras = options.project?.formatExtras
541
      ? (await options.project.formatExtras(
542
        options.source,
543
        options.flags || {},
544
        options.format,
545
        options.services,
546
      ))
547
      : {};
548

549
    const formatExtras = options.format.formatExtras
550
      ? (await options.format.formatExtras(
551
        options.source,
552
        options.markdown,
553
        options.flags || {},
554
        options.format,
555
        options.libDir,
556
        options.services,
557
        options.offset,
558
        options.project,
559
        options.quiet,
560
      ))
561
      : {};
562

563
    // start with the merge
564
    const inputExtras = mergeConfigs(
565
      projectExtras,
566
      formatExtras,
567
      {
568
        metadata: projectExtras.metadata?.[kDocumentClass]
569
          ? {
570
            [kDocumentClass]: projectExtras.metadata?.[kDocumentClass],
571
          }
572
          : undefined,
573
      },
574
    );
575

576
    const extras = await resolveExtras(
577
      options.source,
578
      inputExtras,
579
      options.format,
580
      cwd,
581
      options.libDir,
582
      dependenciesFile,
583
      options.project,
584
    );
585

586
    // record postprocessors
587
    postprocessors.push(...(extras.postprocessors || []));
588

589
    // add a keep-source post processor if we need one
590
    if (
591
      options.format?.render[kKeepSource] || formatHasCodeTools(options.format)
592
    ) {
593
      htmlPostprocessors.push(codeToolsPostprocessor(options.format));
594
    }
595

596
    // save post-processors
597
    htmlPostprocessors.push(...(extras.html?.[kHtmlPostprocessors] || []));
598

599
    // Save finalizers
600
    htmlFinalizers.push(...(extras.html?.[kHtmlFinalizers] || []));
601

602
    if (isHtmlFileOutput(options.format.pandoc)) {
603
      // add a post-processor for fixing overflow-x in cell output display
604
      htmlPostprocessors.push(overflowXPostprocessor);
605

606
      // katex post-processor
607
      if (
608
        options.flags?.katex ||
609
        options.format.pandoc[kHtmlMathMethod] === "katex"
610
      ) {
611
        htmlPostprocessors.push(katexPostProcessor());
612
      }
613

614
      if (!projectIsWebsite(options.project)) {
615
        // add a resource discovery postProcessor if we are not in a website project
616
        htmlPostprocessors.push(discoverResourceRefs);
617

618
        // in order for tabsets etc to show the right mouse cursor,
619
        // we need hrefs in anchor elements to be "empty" instead of missing.
620
        // Existing href attributes trigger the any-link pseudo-selector that
621
        // browsers set to `cursor: pointer`.
622
        //
623
        // In project websites, quarto-nav.js does the same thing so this step
624
        // isn't necessary.
625

626
        htmlPostprocessors.push(fixEmptyHrefs);
627
      }
628

629
      // Include Math, if explicitly requested (this will result
630
      // in math dependencies being injected into the page)
631
      if (forceMath) {
632
        const htmlMarkdownHandlers: MarkdownPipelineHandler[] = [];
633
        htmlMarkdownHandlers.push({
634
          getUnrendered: () => {
635
            return {
636
              inlines: {
637
                "quarto-enable-math-inline": "$e = mC^2$",
638
              },
639
            };
640
          },
641
          processRendered: (
642
            _rendered: unknown,
643
            _doc: Document,
644
          ) => {
645
          },
646
        });
647

648
        const htmlMarkdownPipeline = createMarkdownPipeline(
649
          "quarto-book-math",
650
          htmlMarkdownHandlers,
651
        );
652

653
        const htmlPipelinePostProcessor = (
654
          doc: Document,
655
        ): Promise<HtmlPostProcessResult> => {
656
          htmlMarkdownPipeline.processRenderedMarkdown(doc);
657
          return Promise.resolve({
658
            resources: [],
659
            supporting: [],
660
          });
661
        };
662

663
        htmlRenderAfterBody.push(htmlMarkdownPipeline.markdownAfterBody());
664
        htmlPostprocessors.push(htmlPipelinePostProcessor);
665
      }
666
    }
667

668
    // Capture markdown that should be appended post body
669
    htmlRenderAfterBody.push(...(extras.html?.[kMarkdownAfterBody] || []));
670

671
    // merge sysFilters if we have them
672
    if (sysFilters.length > 0) {
673
      extras.filters = extras.filters || {};
674
      extras.filters.post = extras.filters.post || [];
675
      extras.filters.post.unshift(
676
        ...(sysFilters.map((filter) => resourcePath(join("filters", filter)))),
677
      );
678
    }
679

680
    // merge args
681
    if (extras.args) {
682
      args.push(...extras.args);
683
      printArgs.push(...extras.args);
684
    }
685

686
    // merge pandoc
687
    if (extras.pandoc) {
688
      // Special case - we need to more intelligently merge pandoc from
689
      // by breaking apart the from string
690
      if (
691
        typeof (allDefaults[kFrom]) === "string" &&
692
        typeof (extras.pandoc[kFrom]) === "string"
693
      ) {
694
        const userFrom = splitPandocFormatString(allDefaults[kFrom] as string);
695
        const extrasFrom = splitPandocFormatString(
696
          extras.pandoc[kFrom] as string,
697
        );
698
        allDefaults[kFrom] = pandocFormatWith(
699
          userFrom.format,
700
          "",
701
          extrasFrom.options + userFrom.options,
702
        );
703
        printAllDefaults[kFrom] = allDefaults[kFrom];
704
      }
705

706
      allDefaults = mergeConfigs(extras.pandoc, allDefaults);
707
      printAllDefaults = mergeConfigs(extras.pandoc, printAllDefaults);
708

709
      // Special case - theme is resolved on extras and should override allDefaults
710
      // Clean up deprecated kHighlightStyle if user used old name
711
      delete printAllDefaults[kHighlightStyle];
712
      delete allDefaults[kHighlightStyle];
713
      if (extras.pandoc[kSyntaxHighlighting] === null) {
714
        delete printAllDefaults[kSyntaxHighlighting];
715
        allDefaults[kSyntaxHighlighting] = null;
716
      } else if (extras.pandoc[kSyntaxHighlighting]) {
717
        delete printAllDefaults[kSyntaxHighlighting];
718
        allDefaults[kSyntaxHighlighting] = extras.pandoc[kSyntaxHighlighting];
719
      } else {
720
        delete printAllDefaults[kSyntaxHighlighting];
721
        delete allDefaults[kSyntaxHighlighting];
722
      }
723
    }
724

725
    // merge metadata
726
    if (extras.metadata || extras.metadataOverride) {
727
      // before we merge metadata, ensure that partials are proper paths
728
      resolveTemplatePartialPaths(
729
        options.format.metadata,
730
        cwd,
731
        options.project,
732
      );
733
      options.format.metadata = {
734
        ...mergeConfigs(
735
          extras.metadata || {},
736
          options.format.metadata,
737
        ),
738
        ...extras.metadataOverride || {},
739
      };
740
      printMetadata = mergeConfigs(extras.metadata || {}, printMetadata);
741
      cleanMetadataForPrinting(printMetadata);
742
    }
743

744
    // merge notebooks that have been provided by the document / user
745
    // or by the project as format extras
746
    if (extras[kNotebooks]) {
747
      const documentNotebooks = options.format.render[kNotebookView];
748
      // False means that the user has explicitely disabled notebooks
749
      if (documentNotebooks !== false) {
750
        const userNotebooks = documentNotebooks === true
751
          ? []
752
          : Array.isArray(documentNotebooks)
753
          ? documentNotebooks
754
          : documentNotebooks !== undefined
755
          ? [documentNotebooks]
756
          : [];
757

758
        // Only add notebooks that aren't already present
759
        const uniqExtraNotebooks = extras[kNotebooks].filter((nb) => {
760
          return !userNotebooks.find((userNb) => {
761
            return userNb.notebook === nb.notebook;
762
          });
763
        });
764

765
        options.format.render[kNotebookView] = [
766
          ...userNotebooks,
767
          ...uniqExtraNotebooks,
768
        ];
769
      }
770
    }
771

772
    // clean 'columns' from pandoc defaults to typst
773
    if (isTypstOutput(options.format.pandoc)) {
774
      delete allDefaults[kColumns];
775
      delete printAllDefaults[kColumns];
776
    }
777

778
    // The user template (if any)
779
    const userTemplate = getPandocArg(args, "--template") ||
780
      allDefaults[kTemplate];
781

782
    // The user partials (if any)
783
    const userPartials = readPartials(options.format.metadata, cwd);
784
    const inputDir = normalizePath(cwd);
785
    const resolvePath = (path: string) => {
786
      if (isAbsolute(path)) {
787
        return path;
788
      } else {
789
        return join(inputDir, path);
790
      }
791
    };
792

793
    const templateContext = extras.templateContext;
794
    if (templateContext) {
795
      // Clean the template partial output
796
      cleanTemplatePartialMetadata(
797
        printMetadata,
798
        templateContext.partials || [],
799
      );
800

801
      // The format is providing a more robust local template
802
      // to use, stage the template and pass it on to pandoc
803
      const template = userTemplate
804
        ? resolvePath(userTemplate)
805
        : templateContext.template;
806

807
      // Validate any user partials
808
      if (!userTemplate && userPartials.length > 0) {
809
        const templateNames = templateContext.partials?.map((temp) =>
810
          basename(temp)
811
        );
812

813
        if (templateNames) {
814
          const userPartialNames = userPartials.map((userPartial) =>
815
            basename(userPartial)
816
          );
817

818
          const hasAtLeastOnePartial = userPartialNames.find((userPartial) => {
819
            return templateNames.includes(userPartial);
820
          });
821

822
          if (!hasAtLeastOnePartial) {
823
            const errorMsg =
824
              `The format '${allDefaults.to}' only supports the following partials:\n${
825
                templateNames.join("\n")
826
              }\n\nPlease provide one or more of these partials.`;
827
            throw new Error(errorMsg);
828
          }
829
        } else {
830
          throw new Error(
831
            `The format ${allDefaults.to} does not support providing any template partials.`,
832
          );
833
        }
834
      }
835

836
      // Place any user partials at the end of the list of partials
837
      const partials: string[] = templateContext.partials || [];
838
      partials.push(...userPartials);
839

840
      // Stage the template and partials
841
      const stagedTemplate = await stageTemplate(
842
        options,
843
        extras,
844
        {
845
          template,
846
          partials,
847
        },
848
      );
849

850
      // Clean out partials from metadata, they are not needed downstream
851
      delete options.format.metadata[kTemplatePartials];
852

853
      allDefaults[kTemplate] = stagedTemplate;
854
    } else {
855
      // ipynb is allowed to have templates without warning
856
      if (userPartials.length > 0 && !isIpynbOutput(options.format.pandoc)) {
857
        // The user passed partials to a format that doesn't support
858
        // staging and partials.
859
        throw new Error(
860
          `The format ${allDefaults.to} does not support providing any template partials.`,
861
        );
862
      } else if (userTemplate) {
863
        // Use the template provided by the user
864
        allDefaults[kTemplate] = userTemplate;
865
      }
866
    }
867

868
    // more cleanup
869
    options.format.metadata = cleanupPandocMetadata({
870
      ...options.format.metadata,
871
    });
872
    printMetadata = cleanupPandocMetadata(printMetadata);
873

874
    if (extras[kIncludeInHeader]) {
875
      if (
876
        allDefaults[kIncludeInHeader] !== undefined &&
877
        !ld.isArray(allDefaults[kIncludeInHeader])
878
      ) {
879
        // FIXME we need to fix the type up in FormatExtras..
880
        allDefaults[kIncludeInHeader] = [
881
          allDefaults[kIncludeInHeader],
882
        ] as unknown as string[];
883
      }
884
      allDefaults = {
885
        ...allDefaults,
886
        [kIncludeInHeader]: [
887
          ...extras[kIncludeInHeader] || [],
888
          ...allDefaults[kIncludeInHeader] || [],
889
        ],
890
      };
891
    }
892
    if (
893
      extras[kIncludeBeforeBody]
894
    ) {
895
      if (
896
        allDefaults[kIncludeBeforeBody] !== undefined &&
897
        !ld.isArray(allDefaults[kIncludeBeforeBody])
898
      ) {
899
        // FIXME we need to fix the type up in FormatExtras..
900
        allDefaults[kIncludeBeforeBody] = [
901
          allDefaults[kIncludeBeforeBody],
902
        ] as unknown as string[];
903
      }
904
      allDefaults = {
905
        ...allDefaults,
906
        [kIncludeBeforeBody]: [
907
          ...extras[kIncludeBeforeBody] || [],
908
          ...allDefaults[kIncludeBeforeBody] || [],
909
        ],
910
      };
911
    }
912
    if (extras[kIncludeAfterBody]) {
913
      if (
914
        allDefaults[kIncludeAfterBody] !== undefined &&
915
        !ld.isArray(allDefaults[kIncludeAfterBody])
916
      ) {
917
        // FIXME we need to fix the type up in FormatExtras..
918
        allDefaults[kIncludeAfterBody] = [
919
          allDefaults[kIncludeAfterBody],
920
        ] as unknown as string[];
921
      }
922
      allDefaults = {
923
        ...allDefaults,
924
        [kIncludeAfterBody]: [
925
          ...allDefaults[kIncludeAfterBody] || [],
926
          ...extras[kIncludeAfterBody] || [],
927
        ],
928
      };
929
    }
930

931
    // Resolve the body envelope here
932
    // body envelope to includes (project body envelope always wins)
933
    if (extras.html?.[kBodyEnvelope] && projectExtras.html?.[kBodyEnvelope]) {
934
      extras.html[kBodyEnvelope] = projectExtras.html[kBodyEnvelope];
935
    }
936
    resolveBodyEnvelope(allDefaults, extras, options.services.temp);
937

938
    // add any filters
939
    allDefaults.filters = [
940
      ...extras.filters?.pre || [],
941
      ...allDefaults.filters || [],
942
      ...extras.filters?.post || [],
943
    ];
944

945
    // make the filter paths windows safe
946
    allDefaults.filters = allDefaults.filters.map((filter) => {
947
      if (typeof filter === "string") {
948
        return pandocMetadataPath(filter);
949
      } else {
950
        return {
951
          type: filter.type,
952
          path: pandocMetadataPath(filter.path),
953
        };
954
      }
955
    });
956

957
    // Capture any format filter params
958
    const filterParams = extras[kFilterParams];
959
    if (filterParams) {
960
      Object.keys(filterParams).forEach((key) => {
961
        formatFilterParams[key] = filterParams[key];
962
      });
963
    }
964
  }
965

966
  // add a shortcode escaping post-processor if we need one
967
  if (
968
    isMarkdownOutput(options.format) &&
969
    requiresShortcodeUnescapePostprocessor(options.markdown)
970
  ) {
971
    postprocessors.push(shortcodeUnescapePostprocessor);
972
  }
973

974
  // resolve some title variables
975
  const title = allDefaults?.[kVariables]?.[kTitle] ||
976
    options.format.metadata[kTitle];
977
  const pageTitle = allDefaults?.[kVariables]?.[kPageTitle] ||
978
    options.format.metadata[kPageTitle];
979
  const titlePrefix = allDefaults?.[kTitlePrefix];
980

981
  // provide default page title if necessary
982
  if (!title && !pageTitle && isHtmlFileOutput(options.format.pandoc)) {
983
    const [_dir, stem] = dirAndStem(options.source);
984
    args.push(
985
      "--metadata",
986
      `pagetitle:${pandocAutoIdentifier(stem, false)}`,
987
    );
988
  }
989

990
  // don't ever duplicate pagetite/title and title-prefix
991
  if (
992
    (pageTitle !== undefined && pageTitle === titlePrefix) ||
993
    (pageTitle === undefined && title === titlePrefix)
994
  ) {
995
    delete allDefaults[kTitlePrefix];
996
  }
997

998
  // if we are doing keepYaml then remove it from pandoc 'to'
999
  if (options.keepYaml && allDefaults.to) {
1000
    allDefaults.to = allDefaults.to.replaceAll(`+${kYamlMetadataBlock}`, "");
1001
  }
1002

1003
  // Attempt to cache the code page, if this windows.
1004
  // We cache the code page to prevent looking it up
1005
  // in the registry repeatedly (which triggers MS Defender)
1006
  if (isWindows) {
1007
    await cacheCodePage();
1008
  }
1009

1010
  // filter results json file
1011
  const filterResultsFile = options.services.temp.createFile();
1012

1013
  const writerKeys: ("to" | "writer")[] = ["to", "writer"];
1014
  for (const key of writerKeys) {
1015
    if (allDefaults[key]?.match(/[.]lua$/)) {
1016
      formatFilterParams["custom-writer"] = allDefaults[key];
1017
      allDefaults[key] = resourcePath("filters/customwriter/customwriter.lua");
1018
    }
1019
  }
1020

1021
  // set up the custom .qmd reader
1022
  if (allDefaults.from) {
1023
    formatFilterParams["user-defined-from"] = allDefaults.from;
1024
  }
1025
  allDefaults.from = resourcePath("filters/qmd-reader.lua");
1026

1027
  // set parameters required for filters (possibily mutating all of it's arguments
1028
  // to pull includes out into quarto parameters so they can be merged)
1029
  let pandocArgs = args;
1030
  const paramsJson = await filterParamsJson(
1031
    pandocArgs,
1032
    options,
1033
    allDefaults,
1034
    formatFilterParams,
1035
    filterResultsFile,
1036
    dependenciesFile,
1037
  );
1038

1039
  setupPandocHooks(
1040
    handleCombinedLuaProfiles(
1041
      options.source,
1042
      paramsJson,
1043
      options.services.temp,
1044
    ),
1045
  );
1046

1047
  // remove selected args and defaults if we are handling some things on behalf of pandoc
1048
  // (e.g. handling section numbering). note that section numbering is handled by the
1049
  // crossref filter so we only do this if the user hasn't disabled the crossref filter
1050
  if (
1051
    !isLatexOutput(options.format.pandoc) &&
1052
    !isTypstOutput(options.format.pandoc) &&
1053
    !isMarkdownOutput(options.format) && crossrefFilterActive(options)
1054
  ) {
1055
    delete allDefaults[kNumberSections];
1056
    delete allDefaults[kNumberOffset];
1057
    const removeArgs = new Map<string, boolean>();
1058
    removeArgs.set("--number-sections", false);
1059
    removeArgs.set("--number-offset", true);
1060
    pandocArgs = removePandocArgs(pandocArgs, removeArgs);
1061
  }
1062

1063
  // https://github.com/quarto-dev/quarto-cli/issues/3126
1064
  // it seems that we still need to coerce number-offset to be an number list,
1065
  // otherwise pandoc fails.
1066
  if (typeof allDefaults[kNumberOffset] === "number") {
1067
    allDefaults[kNumberOffset] = [allDefaults[kNumberOffset]];
1068
  }
1069

1070
  // We always use our own pandoc data-dir, so tear off the user
1071
  // data-dir and use ours.
1072
  const dataDirArgs = new Map<string, boolean>();
1073
  dataDirArgs.set("--data-dir", true);
1074
  pandocArgs = removePandocArgs(
1075
    pandocArgs,
1076
    dataDirArgs,
1077
  );
1078
  pandocArgs.push("--data-dir", resourcePath("pandoc/datadir"));
1079

1080
  // add any built-in syntax definition files
1081
  allDefaults[kSyntaxDefinitions] = allDefaults[kSyntaxDefinitions] || [];
1082
  const syntaxDefinitions = expandGlobSync(
1083
    join(resourcePath(join("pandoc", "syntax-definitions")), "*.xml"),
1084
  );
1085
  for (const syntax of syntaxDefinitions) {
1086
    allDefaults[kSyntaxDefinitions]?.push(syntax.path);
1087
  }
1088

1089
  // provide default webtex url
1090
  if (allDefaults[kHtmlMathMethod] === "webtex") {
1091
    allDefaults[kHtmlMathMethod] = {
1092
      method: "webtex",
1093
      url: "https://latex.codecogs.com/svg.latex?",
1094
    };
1095
  }
1096

1097
  // provide alternate markdown template that actually prints the title block
1098
  if (
1099
    !allDefaults[kTemplate] && !havePandocArg(args, "--template") &&
1100
    !options.keepYaml &&
1101
    allDefaults.to
1102
  ) {
1103
    const formatDesc = parseFormatString(allDefaults.to);
1104
    const lookupTo = formatDesc.baseFormat;
1105
    if (
1106
      [
1107
        "gfm",
1108
        "commonmark",
1109
        "commonmark_x",
1110
        "markdown_strict",
1111
        "markdown_phpextra",
1112
        "markdown_github",
1113
        "markua",
1114
      ].includes(
1115
        lookupTo,
1116
      )
1117
    ) {
1118
      allDefaults[kTemplate] = resourcePath(
1119
        join("pandoc", "templates", "default.markdown"),
1120
      );
1121
    }
1122
  }
1123

1124
  // "Hide" self contained from pandoc. Since we inject dependencies
1125
  // during post processing, we need to implement self-contained ourselves
1126
  // so don't allow pandoc to see this flag (but still print it)
1127
  if (isHtmlFileOutput(options.format.pandoc)) {
1128
    // Hide self-contained arguments
1129
    pandocArgs = pandocArgs.filter((
1130
      arg,
1131
    ) => (arg !== "--self-contained" && arg !== "--embed-resources"));
1132

1133
    // Remove from defaults
1134
    delete allDefaults[kSelfContained];
1135
    delete allDefaults[kEmbedResources];
1136
  }
1137

1138
  // write the defaults file
1139
  if (allDefaults) {
1140
    const defaultsFile = await writeDefaultsFile(
1141
      allDefaults,
1142
      options.services.temp,
1143
    );
1144
    cmd.push("--defaults", defaultsFile);
1145
  }
1146

1147
  // remove front matter from markdown (we've got it all incorporated into options.format.metadata)
1148
  // also save the engine metadata as that will have the result of e.g. resolved inline expressions,
1149
  // (which we will use immediately below)
1150
  const paritioned = partitionYamlFrontMatter(options.markdown);
1151
  const engineMetadata =
1152
    (paritioned?.yaml ? readYamlFromMarkdown(paritioned.yaml) : {}) as Metadata;
1153
  const markdown = paritioned?.markdown || options.markdown;
1154

1155
  // selectively overwrite some resolved metadata (e.g. ensure that metadata
1156
  // computed from inline r expressions gets included @ the bottom).
1157
  const pandocMetadata = safeCloneDeep(options.format.metadata || {});
1158
  for (const key of Object.keys(engineMetadata)) {
1159
    const isChapterTitle = key === kTitle && projectIsBook(options.project);
1160

1161
    if (!isQuartoMetadata(key) && !isChapterTitle && !isIncludeMetadata(key)) {
1162
      // if it's standard pandoc metadata and NOT contained in a format specific
1163
      // override then use the engine metadata value
1164

1165
      // don't do if they've overridden the value in a format
1166
      const formats = engineMetadata[kMetadataFormat] as Metadata;
1167
      if (ld.isObject(formats) && metadataGetDeep(formats, key).length > 0) {
1168
        continue;
1169
      }
1170

1171
      // don't process some format specific metadata that may have been processed already
1172
      // - theme is handled specifically already for revealjs with a metadata override and should not be overridden by user input
1173
      if (key === kTheme && isRevealjsOutput(options.format.pandoc)) {
1174
        continue;
1175
      }
1176
      // - categories are handled specifically already for website projects with a metadata override and should not be overridden by user input
1177
      if (key === kFieldCategories && projectIsWebsite(options.project)) {
1178
        continue;
1179
      }
1180
      // perform the override
1181
      pandocMetadata[key] = engineMetadata[key];
1182
    }
1183
  }
1184

1185
  // Resolve any date fields
1186
  const dateRaw = pandocMetadata[kDate];
1187
  const dateFields = [kDate, kDateModified];
1188
  dateFields.forEach((dateField) => {
1189
    const date = pandocMetadata[dateField];
1190
    const format = pandocMetadata[kDateFormat];
1191
    assert(format === undefined || typeof format === "string");
1192
    pandocMetadata[dateField] = resolveAndFormatDate(
1193
      options.source,
1194
      date,
1195
      format,
1196
    );
1197
  });
1198

1199
  // Ensure that citationMetadata is expanded into
1200
  // and object for downstream use
1201
  if (
1202
    typeof (pandocMetadata[kCitation]) === "boolean" &&
1203
    pandocMetadata[kCitation] === true
1204
  ) {
1205
    pandocMetadata[kCitation] = {};
1206
  }
1207

1208
  // Expand citation dates into CSL dates
1209
  const citationMetadata = pandocMetadata[kCitation];
1210
  if (citationMetadata) {
1211
    assert(typeof citationMetadata === "object");
1212
    // ideally we should be asserting non-arrayness here but that's not very fast.
1213
    // assert(!Array.isArray(citationMetadata));
1214
    const citationMetadataObj = citationMetadata as Record<string, unknown>;
1215
    const docCSLDate = dateRaw
1216
      ? cslDate(resolveDate(options.source, dateRaw))
1217
      : undefined;
1218
    const fields = ["issued", "available-date"];
1219
    fields.forEach((field) => {
1220
      if (citationMetadataObj[field]) {
1221
        citationMetadataObj[field] = cslDate(citationMetadataObj[field]);
1222
      } else if (docCSLDate) {
1223
        citationMetadataObj[field] = docCSLDate;
1224
      }
1225
    });
1226
  }
1227

1228
  // Resolve the author metadata into a form that Pandoc will recognize
1229
  const authorsRaw = pandocMetadata[kAuthors] || pandocMetadata[kAuthor];
1230
  if (authorsRaw) {
1231
    const authors = parseAuthor(pandocMetadata[kAuthor], true);
1232
    if (authors) {
1233
      pandocMetadata[kAuthor] = authors.map((author) =>
1234
        cslNameToString(author.name)
1235
      );
1236
      pandocMetadata[kAuthors] = Array.isArray(authorsRaw)
1237
        ? authorsRaw
1238
        : [authorsRaw];
1239
    }
1240
  }
1241

1242
  // Ensure that there are institutes around for use when resolving authors
1243
  // and affilations
1244
  const instituteRaw = pandocMetadata[kInstitute];
1245
  if (instituteRaw) {
1246
    pandocMetadata[kInstitutes] = Array.isArray(instituteRaw)
1247
      ? instituteRaw
1248
      : [instituteRaw];
1249
  }
1250

1251
  // If the user provides only `zh` as a lang, disambiguate to 'simplified'
1252
  if (pandocMetadata.lang === "zh") {
1253
    pandocMetadata.lang = "zh-Hans";
1254
  }
1255

1256
  // If there are no specified options for link coloring in PDF, set them
1257
  // do not color links for obviously printed book output or beamer presentations
1258
  if (
1259
    isLatexOutput(options.format.pandoc) &&
1260
    !isBeamerOutput(options.format.pandoc)
1261
  ) {
1262
    const docClass = pandocMetadata[kDocumentClass];
1263
    assert(!docClass || typeof docClass === "string");
1264
    const isPrintDocumentClass = docClass &&
1265
      ["book", "scrbook"].includes(docClass as string);
1266

1267
    if (!isPrintDocumentClass) {
1268
      if (pandocMetadata[kColorLinks] === undefined) {
1269
        pandocMetadata[kColorLinks] = true;
1270
      }
1271

1272
      if (pandocMetadata[kLinkColor] === undefined) {
1273
        pandocMetadata[kLinkColor] = "blue";
1274
      }
1275
    }
1276
  }
1277

1278
  // If the format provides any additional markdown to render after the body
1279
  // then append that before rendering
1280
  const markdownWithRenderAfter =
1281
    isHtmlOutput(options.format.pandoc) && htmlRenderAfterBody.length > 0
1282
      ? markdown + "\n\n\n" + htmlRenderAfterBody.join("\n") + "\n\n"
1283
      : markdown;
1284

1285
  // append render after + keep-source if requested
1286
  const input = markdownWithRenderAfter +
1287
    keepSourceBlock(options.format, options.source);
1288

1289
  // write input to temp file and pass it to pandoc
1290
  const inputTemp = options.services.temp.createFile({
1291
    prefix: "quarto-input",
1292
    suffix: ".md",
1293
  });
1294
  Deno.writeTextFileSync(inputTemp, input);
1295
  cmd.push(inputTemp);
1296

1297
  // Pass metadata to Pandoc. This metadata reflects all of our merged project and format
1298
  // metadata + the user's original metadata from the top of the document. Note that we
1299
  // used to append this to the end of the file (so it would always 'win' over the front-matter
1300
  // at the top) however we ran into problems w/ the pandoc parser seeing an hr (------)
1301
  // followed by text on the next line as the beginning of a table that was terminated
1302
  // with our yaml block! Note that subsequent to the original implementation we started
1303
  // stripping the yaml from the top, see:
1304
  //   https://github.com/quarto-dev/quarto-cli/commit/35f4729defb20ceb8b45e08d0a97c079e7a3bab6
1305
  // The way this yaml is now processed relative to other yaml sources is described in
1306
  // the docs for --metadata-file:
1307
  //   Values in files specified later on the command line will be preferred over those
1308
  //   specified in earlier files. Metadata values specified inside the document, or by
1309
  //   using -M, overwrite values specified with this option.
1310
  // This gives the semantics we want, as our metadata is 'logically' at the top of the
1311
  // file and subsequent blocks within the file should indeed override it (as should
1312
  // user invocations of --metadata-file or -M, which are included below in pandocArgs)
1313
  const metadataTemp = options.services.temp.createFile({
1314
    prefix: "quarto-metadata",
1315
    suffix: ".yml",
1316
  });
1317
  const pandocPassedMetadata = safeCloneDeep(pandocMetadata);
1318
  delete pandocPassedMetadata.format;
1319
  delete pandocPassedMetadata.project;
1320
  delete pandocPassedMetadata.website;
1321
  delete pandocPassedMetadata.about;
1322
  // these shouldn't be visible because they are emitted on markdown output
1323
  // and it breaks ensureFileRegexMatches
1324
  cleanQuartoTestsMetadata(pandocPassedMetadata);
1325

1326
  // Filter out bundled engines from metadata passed to Pandoc
1327
  if (Array.isArray(pandocPassedMetadata.engines)) {
1328
    const filteredEngines = pandocPassedMetadata.engines.filter((engine) => {
1329
      const enginePath = typeof engine === "string" ? engine : engine.path;
1330
      if (!enginePath) return true;
1331
      return !enginePath.replace(/\\/g, "/").includes(
1332
        "resources/extension-subtrees/",
1333
      );
1334
    });
1335

1336
    if (filteredEngines.length === 0) {
1337
      delete pandocPassedMetadata.engines;
1338
    } else {
1339
      pandocPassedMetadata.engines = filteredEngines;
1340
    }
1341
  }
1342

1343
  // Escape @ in book metadata to prevent false citeproc warnings (#12136).
1344
  // Book metadata can't be deleted like website/about because {{< meta book.* >}}
1345
  // shortcodes depend on it. Pandoc resolves &#64; back to @ in the AST.
1346
  if (pandocPassedMetadata.book) {
1347
    pandocPassedMetadata.book = escapeAtInMetadata(pandocPassedMetadata.book);
1348
  }
1349

1350
  Deno.writeTextFileSync(
1351
    metadataTemp,
1352
    stringify(pandocPassedMetadata, {
1353
      indent: 2,
1354
      lineWidth: -1,
1355
      sortKeys: false,
1356
      skipInvalid: true,
1357
    }),
1358
  );
1359
  cmd.push("--metadata-file", metadataTemp);
1360

1361
  // add user command line args
1362
  cmd.push(...pandocArgs);
1363

1364
  // print full resolved input to pandoc
1365
  if (!options.quiet && !options.flags?.quiet) {
1366
    runPandocMessage(
1367
      printArgs,
1368
      printAllDefaults,
1369
      sysFilters,
1370
      printMetadata,
1371
    );
1372
  }
1373

1374
  // run beforePandoc hooks
1375
  for (const hook of beforePandocHooks) {
1376
    await hook();
1377
  }
1378

1379
  setupPandocEnv();
1380

1381
  const params = {
1382
    cmd: cmd[0],
1383
    args: cmd.slice(1),
1384
    cwd,
1385
    env: pandocEnv,
1386
    ourEnv: Deno.env.toObject(),
1387
  };
1388
  const captureCommand = Deno.env.get("QUARTO_CAPTURE_RENDER_COMMAND");
1389
  if (captureCommand) {
1390
    captureRenderCommand(params, options.services.temp, captureCommand);
1391
  }
1392
  const pandocRender = makeTimedFunctionAsync("pandoc-render", async () => {
1393
    return await execProcess(params);
1394
  });
1395

1396
  // run pandoc
1397
  const result = await pandocRender();
1398

1399
  // run afterPandoc hooks
1400
  for (const hook of afterPandocHooks) {
1401
    await hook();
1402
  }
1403

1404
  // resolve resource files from metadata
1405
  const resources: string[] = resourcesFromMetadata(
1406
    options.format.metadata[kResources],
1407
  );
1408

1409
  // read any resourceFiles generated by filters
1410
  let inputTraits = {};
1411
  if (existsSync(filterResultsFile)) {
1412
    const filterResultsJSON = Deno.readTextFileSync(filterResultsFile);
1413
    if (filterResultsJSON.trim().length > 0) {
1414
      const filterResults = JSON.parse(filterResultsJSON);
1415

1416
      // Read any input traits
1417
      inputTraits = filterResults.inputTraits;
1418

1419
      // Read any resource files
1420
      const resourceFiles = filterResults.resourceFiles || [];
1421
      resources.push(...resourceFiles);
1422
    }
1423
  }
1424

1425
  if (result.success) {
1426
    return {
1427
      inputMetadata: pandocMetadata,
1428
      inputTraits,
1429
      resources,
1430
      postprocessors,
1431
      htmlPostprocessors: isHtmlOutput(options.format.pandoc)
1432
        ? htmlPostprocessors
1433
        : [],
1434
      htmlFinalizers: isHtmlDocOutput(options.format.pandoc)
1435
        ? htmlFinalizers
1436
        : [],
1437
    };
1438
  } else {
1439
    // Since this render wasn't successful, clear the code page cache
1440
    // (since the code page could've changed and we could be caching the
1441
    // wrong value)
1442
    if (isWindows) {
1443
      clearCodePageCache();
1444
    }
1445

1446
    return null;
1447
  }
1448
}
1449

1450
// this mutates metadata[kClassOption]
1451
function cleanupPandocMetadata(metadata: Metadata) {
1452
  // pdf classoption can end up with duplicated options
1453
  const classoption = metadata[kClassOption];
1454
  if (Array.isArray(classoption)) {
1455
    metadata[kClassOption] = ld.uniqBy(
1456
      classoption.reverse(),
1457
      (option: string) => {
1458
        return option.replace(/=.+$/, "");
1459
      },
1460
    ).reverse();
1461
  }
1462

1463
  return metadata;
1464
}
1465

1466
async function resolveExtras(
1467
  input: string,
1468
  extras: FormatExtras, // input format extras (project, format, brand)
1469
  format: Format,
1470
  inputDir: string,
1471
  libDir: string,
1472
  dependenciesFile: string,
1473
  project: ProjectContext,
1474
) {
1475
  // resolve format resources
1476
  await writeFormatResources(
1477
    inputDir,
1478
    dependenciesFile,
1479
    format.render[kFormatResources],
1480
  );
1481

1482
  // perform html-specific merging
1483
  if (isHtmlOutput(format.pandoc)) {
1484
    // resolve sass bundles
1485
    extras = await resolveSassBundles(
1486
      inputDir,
1487
      extras,
1488
      format,
1489
      project,
1490
    );
1491

1492
    // resolve dependencies
1493
    await writeDependencies(dependenciesFile, extras);
1494

1495
    const htmlDependenciesPostProcesor = (
1496
      doc: Document,
1497
      _inputMedata: Metadata,
1498
    ): Promise<HtmlPostProcessResult> => {
1499
      return withTiming(
1500
        "pandocDependenciesPostProcessor",
1501
        async () =>
1502
          await readAndInjectDependencies(
1503
            dependenciesFile,
1504
            inputDir,
1505
            libDir,
1506
            doc,
1507
            project,
1508
          ),
1509
      );
1510
    };
1511

1512
    // Add a post processor to resolve dependencies
1513
    extras.html = extras.html || {};
1514
    extras.html[kHtmlPostprocessors] = extras.html?.[kHtmlPostprocessors] || [];
1515
    if (isHtmlFileOutput(format.pandoc)) {
1516
      extras.html[kHtmlPostprocessors]!.unshift(htmlDependenciesPostProcesor);
1517
    }
1518

1519
    // Remove the dependencies which will now process in the post
1520
    // processor
1521
    delete extras.html?.[kDependencies];
1522
  } else {
1523
    delete extras.html;
1524
  }
1525

1526
  // perform typst-specific merging
1527
  if (isTypstOutput(format.pandoc)) {
1528
    const brand = (await project.resolveBrand(input))?.light;
1529
    const fontdirs: Set<string> = new Set();
1530
    const base_urls = {
1531
      google: "https://fonts.googleapis.com/css",
1532
      bunny: "https://fonts.bunny.net/css",
1533
    };
1534
    const ttf_urls = [], woff_urls: Array<string> = [];
1535
    if (brand?.data.typography) {
1536
      const fonts = brand.data.typography.fonts || [];
1537
      for (const _font of fonts) {
1538
        // if font lacks a source, we assume google in typst output
1539

1540
        // deno-lint-ignore no-explicit-any
1541
        const source: string = (_font as any).source ?? "google";
1542
        if (source === "file") {
1543
          const font = Zod.BrandFontFile.parse(_font);
1544
          for (const file of font.files || []) {
1545
            const path = typeof file === "object" ? file.path : file;
1546
            fontdirs.add(resolve(dirname(join(brand.brandDir, path))));
1547
          }
1548
        } else if (source === "bunny") {
1549
          const font = Zod.BrandFontBunny.parse(_font);
1550
          console.log(
1551
            "Font bunny is not yet supported for Typst, skipping",
1552
            font.family,
1553
          );
1554
        } else if (source === "google" /* || font.source === "bunny" */) {
1555
          const font = Zod.BrandFontGoogle.parse(_font);
1556
          let { family, style, weight } = font;
1557
          const parts = [family!];
1558
          if (style) {
1559
            style = Array.isArray(style) ? style : [style];
1560
            parts.push(style.join(","));
1561
          }
1562
          if (weight) {
1563
            weight = Array.isArray(weight) ? weight : [weight];
1564
            parts.push(weight.join(","));
1565
          }
1566
          const response = await fetch(
1567
            `${base_urls[source]}?family=${parts.join(":")}`,
1568
          );
1569
          const lines = (await response.text()).split("\n");
1570
          for (const line of lines) {
1571
            const sourcelist = line.match(/^ *src: (.*); *$/);
1572
            if (sourcelist) {
1573
              const sources = sourcelist[1].split(",").map((s) => s.trim());
1574
              let found = false;
1575
              const failed_formats = [];
1576
              for (const source of sources) {
1577
                const match = source.match(
1578
                  /url\(([^)]*)\) *format\('([^)]*)'\)/,
1579
                );
1580
                if (match) {
1581
                  const [_, url, format] = match;
1582
                  if (["truetype", "opentype"].includes(format)) {
1583
                    ttf_urls.push(url);
1584
                    found = true;
1585
                    break;
1586
                  }
1587
                  //  else if (["woff", "woff2"].includes(format)) {
1588
                  //   woff_urls.push(url);
1589
                  //   break;
1590
                  // }
1591
                  failed_formats.push(format);
1592
                }
1593
              }
1594
              if (!found) {
1595
                console.log(
1596
                  "skipping",
1597
                  family,
1598
                  "\nnot currently able to use formats",
1599
                  failed_formats.join(", "),
1600
                );
1601
              }
1602
            }
1603
          }
1604
        }
1605
      }
1606
    }
1607
    if (ttf_urls.length || woff_urls.length) {
1608
      const font_cache = migrateProjectScratchPath(
1609
        brand!.projectDir,
1610
        "typst-font-cache",
1611
        "typst/fonts",
1612
      );
1613
      const url_to_path = (url: string) => url.replace(/^https?:\/\//, "");
1614
      const cached = async (url: string) => {
1615
        const path = url_to_path(url);
1616
        try {
1617
          await Deno.lstat(join(font_cache, path));
1618
          return true;
1619
        } catch (err) {
1620
          if (!(err instanceof Deno.errors.NotFound)) {
1621
            throw err;
1622
          }
1623
          return false;
1624
        }
1625
      };
1626
      const download = async (url: string) => {
1627
        const path = url_to_path(url);
1628
        await ensureDir(
1629
          join(font_cache, dirname(path)),
1630
        );
1631

1632
        const response = await fetch(url);
1633
        const blob = await response.blob();
1634
        const buffer = await blob.arrayBuffer();
1635
        const bytes = new Uint8Array(buffer);
1636
        await Deno.writeFile(join(font_cache, path), bytes);
1637
      };
1638
      const woff2ttf = async (url: string) => {
1639
        const path = url_to_path(url);
1640
        await call("ttx", { args: [join(font_cache, path)] });
1641
        await call("ttx", {
1642
          args: [join(font_cache, path.replace(/woff2?$/, "ttx"))],
1643
        });
1644
      };
1645
      const ttf_urls2: Array<string> = [], woff_urls2: Array<string> = [];
1646
      await Promise.all(ttf_urls.map(async (url) => {
1647
        if (!await cached(url)) {
1648
          ttf_urls2.push(url);
1649
        }
1650
      }));
1651

1652
      await woff_urls.reduce((cur, next) => {
1653
        return cur.then(() => woff2ttf(next));
1654
      }, Promise.resolve());
1655
      // await Promise.all(woff_urls.map(async (url) => {
1656
      //   if (!await cached(url)) {
1657
      //     woff_urls2.push(url);
1658
      //   }
1659
      // }));
1660
      await Promise.all(ttf_urls2.concat(woff_urls2).map(download));
1661
      if (woff_urls2.length) {
1662
        await Promise.all(woff_urls2.map(woff2ttf));
1663
      }
1664
      fontdirs.add(font_cache);
1665
    }
1666
    let fontPaths = format.metadata[kFontPaths] as Array<string> || [];
1667
    if (typeof fontPaths === "string") {
1668
      fontPaths = [fontPaths];
1669
    }
1670
    fontPaths = fontPaths.map((path) =>
1671
      path[0] === "/" ? join(project.dir, path) : path
1672
    );
1673
    fontPaths.push(...fontdirs);
1674
    format.metadata[kFontPaths] = fontPaths;
1675
  }
1676

1677
  // Process format resources
1678

1679
  // If we're generating the PDF, we can move the format resources once the pandoc
1680
  // render has completed.
1681
  if (format.render[kLatexAutoMk] === false) {
1682
    // Process the format resouces right here on the spot
1683
    await processFormatResources(inputDir, dependenciesFile);
1684
  } else {
1685
    const resourceDependenciesPostProcessor = async (_output: string) => {
1686
      return await processFormatResources(inputDir, dependenciesFile);
1687
    };
1688
    extras.postprocessors = extras.postprocessors || [];
1689
    extras.postprocessors.push(resourceDependenciesPostProcessor);
1690
  }
1691

1692
  // Resolve the highlighting theme (if any)
1693
  extras = resolveTextHighlightStyle(
1694
    inputDir,
1695
    extras,
1696
    format.pandoc,
1697
  );
1698

1699
  return extras;
1700
}
1701

1702
function resolveBodyEnvelope(
1703
  pandoc: FormatPandoc,
1704
  extras: FormatExtras,
1705
  temp: TempContext,
1706
) {
1707
  const envelope = extras.html?.[kBodyEnvelope];
1708
  if (envelope) {
1709
    const writeBodyFile = (
1710
      type: "include-in-header" | "include-before-body" | "include-after-body",
1711
      prepend: boolean, // should we prepend or append this element
1712
      content?: string,
1713
    ) => {
1714
      if (content) {
1715
        const file = temp.createFile({ suffix: ".html" });
1716
        Deno.writeTextFileSync(file, content);
1717
        if (!prepend) {
1718
          pandoc[type] = (pandoc[type] || []).concat(file);
1719
        } else {
1720
          pandoc[type] = [file].concat(pandoc[type] || []);
1721
        }
1722
      }
1723
    };
1724
    writeBodyFile(kIncludeInHeader, true, envelope.header);
1725
    writeBodyFile(kIncludeBeforeBody, true, envelope.before);
1726

1727
    // Process the after body preamble and postamble (include-after-body appears between these)
1728
    writeBodyFile(kIncludeAfterBody, true, envelope.afterPreamble);
1729
    writeBodyFile(kIncludeAfterBody, false, envelope.afterPostamble);
1730
  }
1731
}
1732

1733
function runPandocMessage(
1734
  args: string[],
1735
  pandoc: FormatPandoc | undefined,
1736
  sysFilters: string[],
1737
  metadata: Metadata,
1738
  debug?: boolean,
1739
) {
1740
  info(`pandoc ${args.join(" ")}`, { bold: true });
1741
  if (pandoc) {
1742
    info(pandocDefaultsMessage(pandoc, sysFilters, debug), { indent: 2 });
1743
  }
1744

1745
  const keys = Object.keys(metadata);
1746
  if (keys.length > 0) {
1747
    const printMetadata = safeCloneDeep(metadata);
1748
    delete printMetadata.format;
1749

1750
    // print message
1751
    if (Object.keys(printMetadata).length > 0) {
1752
      info("metadata", { bold: true });
1753
      info(
1754
        stringify(printMetadata, {
1755
          indent: 2,
1756
          lineWidth: -1,
1757
          sortKeys: false,
1758
          skipInvalid: true,
1759
        }),
1760
        { indent: 2 },
1761
      );
1762
    }
1763
  }
1764
}
1765

1766
function resolveTextHighlightStyle(
1767
  inputDir: string,
1768
  extras: FormatExtras,
1769
  pandoc: FormatPandoc,
1770
): FormatExtras {
1771
  extras = {
1772
    ...extras,
1773
    pandoc: extras.pandoc ? { ...extras.pandoc } : {},
1774
  } as FormatExtras;
1775

1776
  // Get the user selected theme or choose a default
1777
  // Check both syntax-highlighting (new) and highlight-style (deprecated alias)
1778
  const highlightTheme = pandoc[kSyntaxHighlighting] ||
1779
    pandoc[kHighlightStyle] ||
1780
    kDefaultHighlightStyle;
1781
  const textHighlightingMode = extras.html?.[kTextHighlightingMode];
1782

1783
  if (highlightTheme === "none") {
1784
    // Disable highlighting - pass "none" string (not null, which Pandoc 3.8+ rejects)
1785
    extras.pandoc = extras.pandoc || {};
1786
    extras.pandoc[kSyntaxHighlighting] = "none";
1787
    return extras;
1788
  }
1789

1790
  if (highlightTheme === "idiomatic") {
1791
    if (isRevealjsOutput(pandoc)) {
1792
      // reveal.js idiomatic mode doesn't produce working highlighting
1793
      // Fall through to default skylighting instead
1794
      warning(
1795
        "syntax-highlighting: idiomatic is not supported for reveal.js. Using default highlighting.",
1796
      );
1797
    } else {
1798
      // Use native format highlighting (typst native, LaTeX listings)
1799
      // Pass through to Pandoc 3.8+ which handles this natively
1800
      extras.pandoc = extras.pandoc || {};
1801
      extras.pandoc[kSyntaxHighlighting] = "idiomatic";
1802
      return extras;
1803
    }
1804
  }
1805

1806
  // create the possible name matches based upon the dark vs. light
1807
  // and find a matching theme file
1808
  // Themes from
1809
  // https://invent.kde.org/frameworks/syntax-highlighting/-/tree/master/data/themes
1810
  switch (textHighlightingMode) {
1811
    case "light":
1812
    case "dark":
1813
      // Set light or dark mode as appropriate
1814
      extras.pandoc = extras.pandoc || {};
1815
      extras.pandoc[kSyntaxHighlighting] = textHighlightThemePath(
1816
        inputDir,
1817
        highlightTheme,
1818
        textHighlightingMode,
1819
      ) ||
1820
        highlightTheme;
1821

1822
      break;
1823
    case "none":
1824
      // Clear the highlighting
1825
      if (extras.pandoc) {
1826
        extras.pandoc = extras.pandoc || {};
1827
        extras.pandoc[kSyntaxHighlighting] = textHighlightThemePath(
1828
          inputDir,
1829
          "none",
1830
        );
1831
      }
1832
      break;
1833
    case undefined:
1834
    default:
1835
      // Set the the light (default) highlighting mode
1836
      extras.pandoc = extras.pandoc || {};
1837
      extras.pandoc[kSyntaxHighlighting] =
1838
        textHighlightThemePath(inputDir, highlightTheme, "light") ||
1839
        highlightTheme;
1840
      break;
1841
  }
1842
  return extras;
1843
}
1844

1845
// deno-lint-ignore no-explicit-any
1846
function escapeAtInMetadata(value: any): any {
1847
  if (typeof value === "string") {
1848
    return isExternalPath(value) ? value.replaceAll("@", "&#64;") : value;
1849
  }
1850
  if (Array.isArray(value)) {
1851
    return value.map(escapeAtInMetadata);
1852
  }
1853
  if (value !== null && typeof value === "object") {
1854
    const result: Record<string, unknown> = {};
1855
    for (const [k, v] of Object.entries(value)) {
1856
      result[k] = escapeAtInMetadata(v);
1857
    }
1858
    return result;
1859
  }
1860
  return value;
1861
}
1862

1863