1
/*
2
 * project-shared.ts
3
 *
4
 * Copyright (C) 2020-2022 Posit Software, PBC
5
 */
6

7
import { existsSync, safeRemoveSync } from "../deno_ral/fs.ts";
8
import {
9
  dirname,
10
  isAbsolute,
11
  join,
12
  relative,
13
  SEP_PATTERN,
14
} from "../deno_ral/path.ts";
15
import { kHtmlMathMethod } from "../config/constants.ts";
16
import { Format, Metadata } from "../config/types.ts";
17
import { mergeConfigs } from "../core/config.ts";
18
import { getFrontMatterSchema } from "../core/lib/yaml-schema/front-matter.ts";
19

20
import { normalizePath, pathWithForwardSlashes } from "../core/path.ts";
21
import { readAndValidateYamlFromFile } from "../core/schema/validated-yaml.ts";
22
import {
23
  FileInclusion,
24
  FileInformation,
25
  FileInformationCache,
26
  kProjectOutputDir,
27
  kProjectType,
28
  ProjectConfig,
29
  ProjectContext,
30
} from "./types.ts";
31
import { projectType } from "./types/project-types.ts";
32
import { ProjectType } from "./types/types.ts";
33
import { kWebsite } from "./types/website/website-constants.ts";
34
import { existsSync1 } from "../core/file.ts";
35
import { kManuscriptType } from "./types/manuscript/manuscript-types.ts";
36
import { expandIncludes } from "../core/handlers/base.ts";
37
import { MappedString, mappedStringFromFile } from "../core/mapped-text.ts";
38
import { createTempContext } from "../core/temp.ts";
39
import { RenderContext, RenderFlags } from "../command/render/types.ts";
40
import { LanguageCellHandlerOptions } from "../core/handlers/types.ts";
41
import { ExecutionEngineInstance } from "../execute/types.ts";
42
import { InspectedMdCell } from "../inspect/inspect-types.ts";
43
import { breakQuartoMd, QuartoMdCell } from "../core/lib/break-quarto-md.ts";
44
import { partitionCellOptionsText } from "../core/lib/partition-cell-options.ts";
45
import { parse } from "../core/yaml.ts";
46
import { mappedIndexToLineCol } from "../core/lib/mapped-text.ts";
47
import { normalizeNewlines } from "../core/lib/text.ts";
48
import { DirectiveCell } from "../core/lib/break-quarto-md-types.ts";
49
import { QuartoJSONSchema, readYamlFromMarkdown } from "../core/yaml.ts";
50
import { refSchema } from "../core/lib/yaml-schema/common.ts";
51
import { Zod } from "../resources/types/zod/schema-types.ts";
52
import {
53
  Brand,
54
  LightDarkBrand,
55
  LightDarkBrandDarkFlag,
56
  splitUnifiedBrand,
57
} from "../core/brand/brand.ts";
58
import { assert } from "testing/asserts";
59
import { Cloneable, safeCloneDeep } from "../core/safe-clone-deep.ts";
60

61
export function projectExcludeDirs(context: ProjectContext): string[] {
62
  const outputDir = projectOutputDir(context);
63
  if (outputDir) {
64
    return [outputDir];
65
  } else {
66
    return [];
67
  }
68
}
69

70
export function projectFormatOutputDir(
71
  format: Format,
72
  context: ProjectContext,
73
  type: ProjectType,
74
) {
75
  const projOutputDir = projectOutputDir(context);
76
  if (type.formatOutputDirectory) {
77
    const formatOutputDir = type.formatOutputDirectory(format);
78
    if (formatOutputDir) {
79
      return join(projOutputDir, formatOutputDir);
80
    } else {
81
      return projOutputDir;
82
    }
83
  } else {
84
    return projOutputDir;
85
  }
86
}
87

88
export function projectOutputDir(context: ProjectContext): string {
89
  let outputDir = context.config?.project[kProjectOutputDir];
90
  if (outputDir) {
91
    if (!isAbsolute(outputDir)) {
92
      outputDir = join(context.dir, outputDir);
93
    }
94
  } else {
95
    outputDir = context.dir;
96
  }
97
  if (existsSync(outputDir!)) {
98
    return normalizePath(outputDir!);
99
  } else {
100
    return outputDir!;
101
  }
102
}
103

104
export function hasProjectOutputDir(context: ProjectContext): boolean {
105
  return !!context.config?.project[kProjectOutputDir];
106
}
107

108
export function isProjectInputFile(path: string, context: ProjectContext) {
109
  if (existsSync(path)) {
110
    const renderPath = normalizePath(path);
111
    return context.files.input.map((file) => normalizePath(file)).includes(
112
      renderPath,
113
    );
114
  } else {
115
    return false;
116
  }
117
}
118

119
export function projectConfigFile(dir: string): string | undefined {
120
  return ["_quarto.yml", "_quarto.yaml"]
121
    .map((file) => join(dir, file))
122
    .find(existsSync1);
123
}
124

125
export function projectVarsFile(dir: string): string | undefined {
126
  return ["_variables.yml", "_variables.yaml"]
127
    .map((file) => join(dir, file))
128
    .find(existsSync1);
129
}
130

131
export function projectOffset(context: ProjectContext, input: string) {
132
  const projDir = normalizePath(context.dir);
133
  const inputDir = normalizePath(dirname(input));
134
  const offset = relative(inputDir, projDir) || ".";
135
  return pathWithForwardSlashes(offset);
136
}
137

138
export function toInputRelativePaths(
139
  type: ProjectType,
140
  baseDir: string,
141
  inputDir: string,
142
  collection: Array<unknown> | Record<string, unknown>,
143
  ignoreResources?: string[],
144
) {
145
  const existsCache = new Map<string, string>();
146
  const resourceIgnoreFields = ignoreResources ||
147
    ignoreFieldsForProjectType(type) || [];
148
  const offset = relative(inputDir, baseDir);
149

150
  const fixup = (value: string) => {
151
    // if this is a valid file, then transform it to be relative to the input path
152
    if (!existsCache.has(value)) {
153
      const projectPath = join(baseDir, value);
154
      try {
155
        if (existsSync(projectPath)) {
156
          existsCache.set(
157
            value,
158
            pathWithForwardSlashes(join(offset!, value)),
159
          );
160
        } else {
161
          existsCache.set(value, value);
162
        }
163
      } catch {
164
        existsCache.set(value, value);
165
      }
166
    }
167
    return existsCache.get(value);
168
  };
169

170
  const inner = (
171
    collection: Array<unknown> | Record<string, unknown>,
172
    parentKey?: unknown,
173
  ) => {
174
    if (Array.isArray(collection)) {
175
      for (let index = 0; index < collection.length; ++index) {
176
        const value = collection[index];
177
        if (Array.isArray(value) || value instanceof Object) {
178
          inner(value as Array<unknown>);
179
        } else if (typeof value === "string") {
180
          if (value.length > 0 && !isAbsolute(value)) {
181
            collection[index] = fixup(value);
182
          }
183
        }
184
      }
185
    } else {
186
      for (const index of Object.keys(collection)) {
187
        const value = collection[index];
188
        if (
189
          (parentKey === kHtmlMathMethod && index === "method") ||
190
          resourceIgnoreFields!.includes(index)
191
        ) {
192
          // don't fixup html-math-method
193
        } else if (Array.isArray(value) || value instanceof Object) {
194
          // deno-lint-ignore no-explicit-any
195
          inner(value as any, index);
196
        } else if (typeof value === "string") {
197
          if (value.length > 0 && !isAbsolute(value)) {
198
            collection[index] = fixup(value);
199
          }
200
        }
201
      }
202
    }
203
  };
204

205
  inner(collection);
206
  return collection;
207
}
208

209
export function ignoreFieldsForProjectType(type?: ProjectType) {
210
  const resourceIgnoreFields = type
211
    ? ["project"].concat(
212
      type.resourceIgnoreFields ? type.resourceIgnoreFields() : [],
213
    )
214
    : [] as string[];
215
  return resourceIgnoreFields;
216
}
217

218
export function projectIsWebsite(context?: ProjectContext): boolean {
219
  if (context) {
220
    const projType = projectType(context.config?.project?.[kProjectType]);
221
    return projectTypeIsWebsite(projType);
222
  } else {
223
    return false;
224
  }
225
}
226

227
export function projectIsManuscript(context?: ProjectContext): boolean {
228
  if (context) {
229
    const projType = projectType(context.config?.project?.[kProjectType]);
230
    return projType.type === kManuscriptType;
231
  } else {
232
    return false;
233
  }
234
}
235

236
export function projectPreviewServe(context?: ProjectContext) {
237
  return context?.config?.project?.preview?.serve;
238
}
239

240
export function projectIsServeable(context?: ProjectContext): boolean {
241
  return projectIsWebsite(context) || projectIsManuscript(context) ||
242
    !!projectPreviewServe(context);
243
}
244

245
export function projectTypeIsWebsite(projType: ProjectType): boolean {
246
  return projType.type === kWebsite || projType.inheritsType === kWebsite;
247
}
248

249
export function projectIsBook(context?: ProjectContext): boolean {
250
  if (context) {
251
    const projType = projectType(context.config?.project?.[kProjectType]);
252
    return projType.type === "book";
253
  } else {
254
    return false;
255
  }
256
}
257

258
export function deleteProjectMetadata(metadata: Metadata) {
259
  // see if the active project type wants to filter the config printed
260
  const projType = projectType(
261
    (metadata as ProjectConfig).project?.[kProjectType],
262
  );
263
  if (projType.metadataFields) {
264
    for (const field of projType.metadataFields().concat("project")) {
265
      if (typeof field === "string") {
266
        delete metadata[field];
267
      } else {
268
        for (const key of Object.keys(metadata)) {
269
          if (field.test(key)) {
270
            delete metadata[key];
271
          }
272
        }
273
      }
274
    }
275
  }
276

277
  // remove project config
278
  delete metadata.project;
279
}
280

281
export function normalizeFormatYaml(yamlFormat: unknown) {
282
  if (yamlFormat) {
283
    if (typeof yamlFormat === "string") {
284
      yamlFormat = {
285
        [yamlFormat]: {},
286
      };
287
    } else if (typeof yamlFormat === "object") {
288
      const formats = Object.keys(yamlFormat);
289
      for (const format of formats) {
290
        if (
291
          (yamlFormat as Record<string, unknown>)[format] === "default"
292
        ) {
293
          (yamlFormat as Record<string, unknown>)[format] = {};
294
        }
295
      }
296
    }
297
  }
298
  return (yamlFormat || {}) as Record<string, unknown>;
299
}
300
export async function directoryMetadataForInputFile(
301
  project: ProjectContext,
302
  inputDir: string,
303
) {
304
  const projectDir = project.dir;
305
  // Finds a metadata file in a directory
306
  const metadataFile = (dir: string) => {
307
    return ["_metadata.yml", "_metadata.yaml"]
308
      .map((file) => join(dir, file))
309
      .find(existsSync1);
310
  };
311

312
  // The path from the project dir to the input dir
313
  const relativePath = relative(projectDir, inputDir);
314
  const dirs = relativePath.split(SEP_PATTERN);
315

316
  // The config we'll ultimately return
317
  let config = {};
318

319
  // Walk through each directory (starting from the project and
320
  // walking deeper to the input)
321
  let currentDir = projectDir;
322
  const frontMatterSchema = await getFrontMatterSchema();
323
  for (let i = 0; i < dirs.length; i++) {
324
    const dir = dirs[i];
325
    currentDir = join(currentDir, dir);
326
    const file = metadataFile(currentDir);
327
    if (file) {
328
      // There is a metadata file, read it and merge it
329
      // Note that we need to convert paths that are relative
330
      // to the metadata file to be relative to input
331
      const errMsg = "Directory metadata validation failed for " + file + ".";
332
      const yaml = ((await readAndValidateYamlFromFile(
333
        file,
334
        frontMatterSchema,
335
        errMsg,
336
      )) || {}) as Record<string, unknown>;
337

338
      // resolve format into expected structure
339
      if (yaml.format) {
340
        yaml.format = normalizeFormatYaml(yaml.format);
341
      }
342

343
      config = mergeConfigs(
344
        config,
345
        toInputRelativePaths(
346
          projectType(project?.config?.project?.[kProjectType]),
347
          currentDir,
348
          inputDir,
349
          yaml as Record<string, unknown>,
350
        ),
351
      );
352
    }
353
  }
354

355
  return config;
356
}
357

358
const mdForFile = async (
359
  _project: ProjectContext,
360
  engine: ExecutionEngineInstance | undefined,
361
  file: string,
362
): Promise<MappedString> => {
363
  if (engine) {
364
    return await engine.markdownForFile(file);
365
  } else {
366
    // Last resort, just read the file
367
    return Promise.resolve(mappedStringFromFile(file));
368
  }
369
};
370

371
export async function projectResolveCodeCellsForFile(
372
  project: ProjectContext,
373
  engine: ExecutionEngineInstance | undefined,
374
  file: string,
375
  markdown?: MappedString,
376
  force?: boolean,
377
): Promise<InspectedMdCell[]> {
378
  const cache = ensureFileInformationCache(project, file);
379
  if (!force && cache.codeCells) {
380
    return cache.codeCells || [];
381
  }
382
  if (!markdown) {
383
    markdown = await mdForFile(project, engine, file);
384
  }
385

386
  const result: InspectedMdCell[] = [];
387
  const fileStack: string[] = [];
388

389
  const inner = async (file: string, cells: QuartoMdCell[]) => {
390
    if (fileStack.includes(file)) {
391
      throw new Error(
392
        "Circular include detected:\n  " + fileStack.join(" ->\n  "),
393
      );
394
    }
395
    fileStack.push(file);
396
    for (const cell of cells) {
397
      if (typeof cell.cell_type === "string") {
398
        continue;
399
      }
400
      if (cell.cell_type.language === "_directive") {
401
        const directiveCell = cell.cell_type as DirectiveCell;
402
        if (directiveCell.name !== "include") {
403
          continue;
404
        }
405
        const arg = directiveCell.shortcode.params[0];
406
        const paths = arg.startsWith("/")
407
          ? [project.dir, arg]
408
          : [project.dir, relative(project.dir, dirname(file)), arg];
409
        const innerFile = join(...paths);
410
        await inner(
411
          innerFile,
412
          (await breakQuartoMd(
413
            await mdForFile(project, engine, innerFile),
414
          )).cells,
415
        );
416
      }
417
      if (
418
        cell.cell_type.language !== "_directive"
419
      ) {
420
        const cellOptions = partitionCellOptionsText(
421
          cell.cell_type.language,
422
          cell.sourceWithYaml ?? cell.source,
423
        );
424
        const metadata = cellOptions.yaml
425
          ? parse(cellOptions.yaml.value, {
426
            schema: QuartoJSONSchema,
427
          }) as Record<string, unknown>
428
          : {};
429
        const lineLocator = mappedIndexToLineCol(cell.sourceVerbatim);
430
        result.push({
431
          start: lineLocator(0).line,
432
          end: lineLocator(cell.sourceVerbatim.value.length - 1).line,
433
          file: file,
434
          source: normalizeNewlines(cell.source.value),
435
          language: cell.cell_type.language,
436
          metadata,
437
        });
438
      }
439
    }
440
    fileStack.pop();
441
  };
442
  await inner(file, (await breakQuartoMd(markdown)).cells);
443
  cache.codeCells = result;
444
  return result;
445
}
446

447
export async function projectFileMetadata(
448
  project: ProjectContext,
449
  file: string,
450
  force?: boolean,
451
): Promise<Metadata> {
452
  const cache = ensureFileInformationCache(project, file);
453
  if (!force && cache.metadata) {
454
    return cache.metadata;
455
  }
456
  const { engine } = await project.fileExecutionEngineAndTarget(file);
457
  const markdown = await mdForFile(project, engine, file);
458
  const metadata = readYamlFromMarkdown(markdown.value);
459
  cache.metadata = metadata;
460
  return metadata;
461
}
462

463
export async function projectResolveFullMarkdownForFile(
464
  project: ProjectContext,
465
  engine: ExecutionEngineInstance | undefined,
466
  file: string,
467
  markdown?: MappedString,
468
  force?: boolean,
469
): Promise<MappedString> {
470
  const cache = ensureFileInformationCache(project, file);
471
  if (!force && cache.fullMarkdown) {
472
    return cache.fullMarkdown;
473
  }
474

475
  const temp = createTempContext();
476

477
  if (!markdown) {
478
    markdown = await mdForFile(project, engine, file);
479
  }
480

481
  const options: LanguageCellHandlerOptions = {
482
    name: "",
483
    temp,
484
    stage: "pre-engine",
485
    format: undefined as unknown as Format,
486
    markdown,
487
    context: {
488
      project,
489
      target: {
490
        source: file,
491
      },
492
    } as unknown as RenderContext,
493
    flags: {} as RenderFlags,
494
  };
495
  try {
496
    const result = await expandIncludes(markdown, options, file);
497
    cache.fullMarkdown = result;
498
    cache.includeMap = options.state?.include.includes as FileInclusion[];
499
    return result;
500
  } finally {
501
    temp.cleanup();
502
  }
503
}
504

505
export const ensureFileInformationCache = (
506
  project: ProjectContext,
507
  file: string,
508
) => {
509
  if (!project.fileInformationCache) {
510
    project.fileInformationCache = new FileInformationCacheMap();
511
  }
512
  assert(
513
    project.fileInformationCache instanceof Map,
514
    JSON.stringify(project.fileInformationCache),
515
  );
516
  if (!project.fileInformationCache.has(file)) {
517
    project.fileInformationCache.set(file, {} as FileInformation);
518
  }
519
  return project.fileInformationCache.get(file)!;
520
};
521

522
export async function projectResolveBrand(
523
  project: ProjectContext,
524
  fileName?: string,
525
): Promise<LightDarkBrandDarkFlag | undefined> {
526
  async function loadSingleBrand(brandPath: string): Promise<Brand> {
527
    const brand = await readAndValidateYamlFromFile(
528
      brandPath,
529
      refSchema("brand-single", "Format-independent brand configuration."),
530
      "Brand validation failed for " + brandPath + ".",
531
    );
532
    return new Brand(brand, dirname(brandPath), project.dir);
533
  }
534
  async function loadUnifiedBrand(
535
    brandPath: string,
536
  ): Promise<LightDarkBrandDarkFlag> {
537
    const brand = await readAndValidateYamlFromFile(
538
      brandPath,
539
      refSchema("brand-unified", "Format-independent brand configuration."),
540
      "Brand validation failed for " + brandPath + ".",
541
    );
542
    return splitUnifiedBrand(brand, dirname(brandPath), project.dir);
543
  }
544
  function resolveBrandPath(
545
    brandPath: string,
546
    dir: string = dirname(fileName!),
547
  ): string {
548
    let resolved: string = "";
549
    if (brandPath.startsWith("/")) {
550
      resolved = join(project.dir, brandPath);
551
    } else {
552
      resolved = join(dir, brandPath);
553
    }
554
    return resolved;
555
  }
556
  if (fileName === undefined) {
557
    if (project.brandCache) {
558
      return project.brandCache.brand;
559
    }
560
    project.brandCache = {};
561
    let fileNames = [
562
      "_brand.yml",
563
      "_brand.yaml",
564
      "_brand/_brand.yml",
565
      "_brand/_brand.yaml",
566
    ].map((file) => join(project.dir, file));
567
    const brand = (project?.config?.brand ??
568
      project?.config?.project.brand) as
569
        | boolean
570
        | string
571
        | {
572
          light?: string;
573
          dark?: string;
574
        };
575
    if (brand === false) {
576
      project.brandCache.brand = undefined;
577
      return project.brandCache.brand;
578
    }
579
    if (
580
      typeof brand === "object" && brand &&
581
      ("light" in brand || "dark" in brand)
582
    ) {
583
      project.brandCache.brand = {
584
        light: brand.light
585
          ? await loadSingleBrand(resolveBrandPath(brand.light, project.dir))
586
          : undefined,
587
        dark: brand.dark
588
          ? await loadSingleBrand(resolveBrandPath(brand.dark, project.dir))
589
          : undefined,
590
        enablesDarkMode: !!brand.dark,
591
      };
592
      return project.brandCache.brand;
593
    }
594
    if (typeof brand === "string") {
595
      fileNames = [join(project.dir, brand)];
596
    }
597

598
    for (const brandPath of fileNames) {
599
      if (!existsSync(brandPath)) {
600
        continue;
601
      }
602
      project.brandCache.brand = await loadUnifiedBrand(brandPath);
603
    }
604
    return project.brandCache.brand;
605
  } else {
606
    const metadata = await project.fileMetadata(fileName);
607
    if (metadata.brand === undefined) {
608
      return project.resolveBrand();
609
    }
610
    const brand = Zod.BrandPathBoolLightDark.parse(metadata.brand);
611
    if (brand === false) {
612
      return undefined;
613
    }
614
    if (brand === true) {
615
      return project.resolveBrand();
616
    }
617
    const fileInformation = ensureFileInformationCache(project, fileName);
618
    if (fileInformation.brand) {
619
      return fileInformation.brand;
620
    }
621
    if (typeof brand === "string") {
622
      fileInformation.brand = await loadUnifiedBrand(resolveBrandPath(brand));
623
      return fileInformation.brand;
624
    } else {
625
      assert(typeof brand === "object");
626
      if ("light" in brand || "dark" in brand) {
627
        let light, dark;
628
        if (typeof brand.light === "string") {
629
          light = await loadSingleBrand(resolveBrandPath(brand.light));
630
        } else if (brand.light) {
631
          light = new Brand(
632
            brand.light,
633
            dirname(fileName),
634
            project.dir,
635
          );
636
        }
637
        if (typeof brand.dark === "string") {
638
          dark = await loadSingleBrand(resolveBrandPath(brand.dark));
639
        } else if (brand.dark) {
640
          dark = new Brand(
641
            brand.dark,
642
            dirname(fileName),
643
            project.dir,
644
          );
645
        }
646
        fileInformation.brand = { light, dark, enablesDarkMode: !!dark };
647
      } else {
648
        fileInformation.brand = splitUnifiedBrand(
649
          brand,
650
          dirname(fileName),
651
          project.dir,
652
        );
653
      }
654
      return fileInformation.brand;
655
    }
656
  }
657
}
658

659
// A Map that normalizes path keys for cross-platform consistency.
660
// All path operations normalize keys (forward slashes, lowercase on Windows).
661
// Implements Cloneable but shares state intentionally - in preview mode,
662
// the project context is reused across renders and cache state must persist.
663
export class FileInformationCacheMap extends Map<string, FileInformation>
664
  implements FileInformationCache, Cloneable<Map<string, FileInformation>> {
665
  override get(key: string): FileInformation | undefined {
666
    return super.get(normalizePath(key));
667
  }
668

669
  override has(key: string): boolean {
670
    return super.has(normalizePath(key));
671
  }
672

673
  override set(key: string, value: FileInformation): this {
674
    return super.set(normalizePath(key), value);
675
  }
676

677
  override delete(key: string): boolean {
678
    return super.delete(normalizePath(key));
679
  }
680

681
  // Note: Iterator methods (keys(), entries(), forEach(), [Symbol.iterator])
682
  // return normalized keys as stored. Code iterating over the cache sees
683
  // normalized paths, which is consistent with how keys are stored.
684

685
  // Removes a cache entry and cleans up any associated transient files.
686
  // In preview mode, this should be used instead of delete() to ensure
687
  // transient notebooks (.quarto_ipynb) are removed from disk before the
688
  // cache entry is dropped. Without this, the collision-avoidance logic
689
  // in jupyter.ts target() creates numbered variants on each re-render.
690
  invalidateForFile(key: string): void {
691
    const existing = this.get(key);
692
    if (existing?.target?.data) {
693
      const data = existing.target.data as { transient?: boolean };
694
      if (data.transient && existing.target.input) {
695
        safeRemoveSync(existing.target.input);
696
      }
697
    }
698
    this.delete(key);
699
  }
700

701
  // Returns this instance (shared reference) rather than a copy.
702
  // This is intentional: in preview mode, project context is cloned for
703
  // each render but the cache must be shared so invalidations persist.
704
  clone(): Map<string, FileInformation> {
705
    return this;
706
  }
707
}
708

709
export function cleanupFileInformationCache(project: ProjectContext) {
710
  for (const key of [...project.fileInformationCache.keys()]) {
711
    project.fileInformationCache.invalidateForFile(key);
712
  }
713
}
714

715
export async function withProjectCleanup<T>(
716
  project: ProjectContext,
717
  fn: (project: ProjectContext) => Promise<T>,
718
): Promise<T> {
719
  try {
720
    return await fn(project);
721
  } finally {
722
    project.cleanup();
723
  }
724
}
725

726