1
#!/usr/bin/env python3
2
# pylint: disable=R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0917,C0302
3
# Copyright(c) 2025: Mauro Carvalho Chehab <[email protected]>.
4
# SPDX-License-Identifier: GPL-2.0
5

6
"""
7
Parse ABI documentation and produce results from it.
8
"""
9

10
from argparse import Namespace
11
import logging
12
import os
13
import re
14

15
from pprint import pformat
16
from random import randrange, seed
17

18
# Import Python modules
19

20
from abi.helpers import AbiDebug, ABI_DIR
21

22

23
class AbiParser:
24
    """Main class to parse ABI files."""
25

26
    #: Valid tags at Documentation/ABI.
27
    TAGS = r"(what|where|date|kernelversion|contact|description|users)"
28

29
    #: ABI elements that will auto-generate cross-references.
30
    XREF = r"(?:^|\s|\()(\/(?:sys|config|proc|dev|kvd)\/[^,.:;\)\s]+)(?:[,.:;\)\s]|\Z)"
31

32
    def __init__(self, directory, logger=None,
33
                 enable_lineno=False, show_warnings=True, debug=0):
34
        """Stores arguments for the class and initialize class vars."""
35

36
        self.directory = directory
37
        self.enable_lineno = enable_lineno
38
        self.show_warnings = show_warnings
39
        self.debug = debug
40

41
        if not logger:
42
            self.log = logging.getLogger("get_abi")
43
        else:
44
            self.log = logger
45

46
        self.data = {}
47
        self.what_symbols = {}
48
        self.file_refs = {}
49
        self.what_refs = {}
50

51
        # Ignore files that contain such suffixes
52
        self.ignore_suffixes = (".rej", ".org", ".orig", ".bak", "~")
53

54
        # Regular expressions used on parser
55
        self.re_abi_dir = re.compile(r"(.*)" + ABI_DIR)
56
        self.re_tag = re.compile(r"(\S+)(:\s*)(.*)", re.I)
57
        self.re_valid = re.compile(self.TAGS)
58
        self.re_start_spc = re.compile(r"(\s*)(\S.*)")
59
        self.re_whitespace = re.compile(r"^\s+")
60

61
        # Regular used on print
62
        self.re_what = re.compile(r"(\/?(?:[\w\-]+\/?){1,2})")
63
        self.re_escape = re.compile(r"([\.\x01-\x08\x0e-\x1f\x21-\x2f\x3a-\x40\x7b-\xff])")
64
        self.re_unprintable = re.compile(r"([\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\xff]+)")
65
        self.re_title_mark = re.compile(r"\n[\-\*\=\^\~]+\n")
66
        self.re_doc = re.compile(r"Documentation/(?!devicetree)(\S+)\.rst")
67
        self.re_abi = re.compile(r"(Documentation/ABI/)([\w\/\-]+)")
68
        self.re_xref_node = re.compile(self.XREF)
69

70
    def warn(self, fdata, msg, extra=None):
71
        """Displays a parse error if warning is enabled."""
72

73
        if not self.show_warnings:
74
            return
75

76
        msg = f"{fdata.fname}:{fdata.ln}: {msg}"
77
        if extra:
78
            msg += "\n\t\t" + extra
79

80
        self.log.warning(msg)
81

82
    def add_symbol(self, what, fname, ln=None, xref=None):
83
        """Create a reference table describing where each 'what' is located."""
84

85
        if what not in self.what_symbols:
86
            self.what_symbols[what] = {"file": {}}
87

88
        if fname not in self.what_symbols[what]["file"]:
89
            self.what_symbols[what]["file"][fname] = []
90

91
        if ln and ln not in self.what_symbols[what]["file"][fname]:
92
            self.what_symbols[what]["file"][fname].append(ln)
93

94
        if xref:
95
            self.what_symbols[what]["xref"] = xref
96

97
    def _parse_line(self, fdata, line):
98
        """Parse a single line of an ABI file."""
99

100
        new_what = False
101
        new_tag = False
102
        content = None
103

104
        match = self.re_tag.match(line)
105
        if match:
106
            new = match.group(1).lower()
107
            sep = match.group(2)
108
            content = match.group(3)
109

110
            match = self.re_valid.search(new)
111
            if match:
112
                new_tag = match.group(1)
113
            else:
114
                if fdata.tag == "description":
115
                    # New "tag" is actually part of description.
116
                    # Don't consider it a tag
117
                    new_tag = False
118
                elif fdata.tag != "":
119
                    self.warn(fdata, f"tag '{fdata.tag}' is invalid", line)
120

121
        if new_tag:
122
            # "where" is Invalid, but was a common mistake. Warn if found
123
            if new_tag == "where":
124
                self.warn(fdata, "tag 'Where' is invalid. Should be 'What:' instead")
125
                new_tag = "what"
126

127
            if new_tag == "what":
128
                fdata.space = None
129

130
                if content not in self.what_symbols:
131
                    self.add_symbol(what=content, fname=fdata.fname, ln=fdata.ln)
132

133
                if fdata.tag == "what":
134
                    fdata.what.append(content.strip("\n"))
135
                else:
136
                    if fdata.key:
137
                        if "description" not in self.data.get(fdata.key, {}):
138
                            self.warn(fdata, f"{fdata.key} doesn't have a description")
139

140
                        for w in fdata.what:
141
                            self.add_symbol(what=w, fname=fdata.fname,
142
                                            ln=fdata.what_ln, xref=fdata.key)
143

144
                    fdata.label = content
145
                    new_what = True
146

147
                    key = "abi_" + content.lower()
148
                    fdata.key = self.re_unprintable.sub("_", key).strip("_")
149

150
                    # Avoid duplicated keys but using a defined seed, to make
151
                    # the namespace identical if there aren't changes at the
152
                    # ABI symbols
153
                    seed(42)
154

155
                    while fdata.key in self.data:
156
                        char = randrange(0, 51) + ord("A")
157
                        if char > ord("Z"):
158
                            char += ord("a") - ord("Z") - 1
159

160
                        fdata.key += chr(char)
161

162
                    if fdata.key and fdata.key not in self.data:
163
                        self.data[fdata.key] = {
164
                            "what": [content],
165
                            "file": [fdata.file_ref],
166
                            "path": fdata.ftype,
167
                            "line_no": fdata.ln,
168
                        }
169

170
                    fdata.what = self.data[fdata.key]["what"]
171

172
                self.what_refs[content] = fdata.key
173
                fdata.tag = new_tag
174
                fdata.what_ln = fdata.ln
175

176
                if fdata.nametag["what"]:
177
                    t = (content, fdata.key)
178
                    if t not in fdata.nametag["symbols"]:
179
                        fdata.nametag["symbols"].append(t)
180

181
                return
182

183
            if fdata.tag and new_tag:
184
                fdata.tag = new_tag
185

186
                if new_what:
187
                    fdata.label = ""
188

189
                    if "description" in self.data[fdata.key]:
190
                        self.data[fdata.key]["description"] += "\n\n"
191

192
                    if fdata.file_ref not in self.data[fdata.key]["file"]:
193
                        self.data[fdata.key]["file"].append(fdata.file_ref)
194

195
                    if self.debug == AbiDebug.WHAT_PARSING:
196
                        self.log.debug("what: %s", fdata.what)
197

198
                if not fdata.what:
199
                    self.warn(fdata, "'What:' should come first:", line)
200
                    return
201

202
                if new_tag == "description":
203
                    fdata.space = None
204

205
                    if content:
206
                        sep = sep.replace(":", " ")
207

208
                        c = " " * len(new_tag) + sep + content
209
                        c = c.expandtabs()
210

211
                        match = self.re_start_spc.match(c)
212
                        if match:
213
                            # Preserve initial spaces for the first line
214
                            fdata.space = match.group(1)
215
                            content = match.group(2) + "\n"
216

217
                self.data[fdata.key][fdata.tag] = content
218

219
            return
220

221
        # Store any contents before tags at the database
222
        if not fdata.tag and "what" in fdata.nametag:
223
            fdata.nametag["description"] += line
224
            return
225

226
        if fdata.tag == "description":
227
            content = line.expandtabs()
228

229
            if self.re_whitespace.sub("", content) == "":
230
                self.data[fdata.key][fdata.tag] += "\n"
231
                return
232

233
            if fdata.space is None:
234
                match = self.re_start_spc.match(content)
235
                if match:
236
                    # Preserve initial spaces for the first line
237
                    fdata.space = match.group(1)
238

239
                    content = match.group(2) + "\n"
240
            else:
241
                if content.startswith(fdata.space):
242
                    content = content[len(fdata.space):]
243

244
                else:
245
                    fdata.space = ""
246

247
            if fdata.tag == "what":
248
                w = content.strip("\n")
249
                if w:
250
                    self.data[fdata.key][fdata.tag].append(w)
251
            else:
252
                self.data[fdata.key][fdata.tag] += content
253
            return
254

255
        content = line.strip()
256
        if fdata.tag:
257
            if fdata.tag == "what":
258
                w = content.strip("\n")
259
                if w:
260
                    self.data[fdata.key][fdata.tag].append(w)
261
            else:
262
                self.data[fdata.key][fdata.tag] += "\n" + content.rstrip("\n")
263
            return
264

265
        # Everything else is error
266
        if content:
267
            self.warn(fdata, "Unexpected content", line)
268

269
    def parse_readme(self, nametag, fname):
270
        """Parse ABI README file."""
271

272
        nametag["what"] = ["Introduction"]
273
        nametag["path"] = "README"
274
        with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp:
275
            for line in fp:
276
                match = self.re_tag.match(line)
277
                if match:
278
                    new = match.group(1).lower()
279

280
                    match = self.re_valid.search(new)
281
                    if match:
282
                        nametag["description"] += "\n:" + line
283
                        continue
284

285
                nametag["description"] += line
286

287
    def parse_file(self, fname, path, basename):
288
        """Parse a single file."""
289

290
        ref = f"abi_file_{path}_{basename}"
291
        ref = self.re_unprintable.sub("_", ref).strip("_")
292

293
        # Store per-file state into a namespace variable. This will be used
294
        # by the per-line parser state machine and by the warning function.
295
        fdata = Namespace
296

297
        fdata.fname = fname
298
        fdata.name = basename
299

300
        pos = fname.find(ABI_DIR)
301
        if pos > 0:
302
            f = fname[pos:]
303
        else:
304
            f = fname
305

306
        fdata.file_ref = (f, ref)
307
        self.file_refs[f] = ref
308

309
        fdata.ln = 0
310
        fdata.what_ln = 0
311
        fdata.tag = ""
312
        fdata.label = ""
313
        fdata.what = []
314
        fdata.key = None
315
        fdata.xrefs = None
316
        fdata.space = None
317
        fdata.ftype = path.split("/")[0]
318

319
        fdata.nametag = {}
320
        fdata.nametag["what"] = [f"ABI file {path}/{basename}"]
321
        fdata.nametag["type"] = "File"
322
        fdata.nametag["path"] = fdata.ftype
323
        fdata.nametag["file"] = [fdata.file_ref]
324
        fdata.nametag["line_no"] = 1
325
        fdata.nametag["description"] = ""
326
        fdata.nametag["symbols"] = []
327

328
        self.data[ref] = fdata.nametag
329

330
        if self.debug & AbiDebug.WHAT_OPEN:
331
            self.log.debug("Opening file %s", fname)
332

333
        if basename == "README":
334
            self.parse_readme(fdata.nametag, fname)
335
            return
336

337
        with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp:
338
            for line in fp:
339
                fdata.ln += 1
340

341
                self._parse_line(fdata, line)
342

343
            if "description" in fdata.nametag:
344
                fdata.nametag["description"] = fdata.nametag["description"].lstrip("\n")
345

346
            if fdata.key:
347
                if "description" not in self.data.get(fdata.key, {}):
348
                    self.warn(fdata, f"{fdata.key} doesn't have a description")
349

350
                for w in fdata.what:
351
                    self.add_symbol(what=w, fname=fname, xref=fdata.key)
352

353
    def _parse_abi(self, root=None):
354
        """Internal function to parse documentation ABI recursively."""
355

356
        if not root:
357
            root = self.directory
358

359
        with os.scandir(root) as obj:
360
            for entry in obj:
361
                name = os.path.join(root, entry.name)
362

363
                if entry.is_dir():
364
                    self._parse_abi(name)
365
                    continue
366

367
                if not entry.is_file():
368
                    continue
369

370
                basename = os.path.basename(name)
371

372
                if basename.startswith("."):
373
                    continue
374

375
                if basename.endswith(self.ignore_suffixes):
376
                    continue
377

378
                path = self.re_abi_dir.sub("", os.path.dirname(name))
379

380
                self.parse_file(name, path, basename)
381

382
    def parse_abi(self, root=None):
383
        """Parse documentation ABI."""
384

385
        self._parse_abi(root)
386

387
        if self.debug & AbiDebug.DUMP_ABI_STRUCTS:
388
            self.log.debug(pformat(self.data))
389

390
    def desc_txt(self, desc):
391
        """Print description as found inside ABI files."""
392

393
        desc = desc.strip(" \t\n")
394

395
        return desc + "\n\n"
396

397
    def xref(self, fname):
398
        """
399
        Converts a Documentation/ABI + basename into a ReST cross-reference.
400
        """
401

402
        xref = self.file_refs.get(fname)
403
        if not xref:
404
            return None
405
        else:
406
            return xref
407

408
    def desc_rst(self, desc):
409
        """Enrich ReST output by creating cross-references."""
410

411
        # Remove title markups from the description
412
        # Having titles inside ABI files will only work if extra
413
        # care would be taken in order to strictly follow the same
414
        # level order for each markup.
415
        desc = self.re_title_mark.sub("\n\n", "\n" + desc)
416
        desc = desc.rstrip(" \t\n").lstrip("\n")
417

418
        # Python's regex performance for non-compiled expressions is a lot
419
        # than Perl, as Perl automatically caches them at their
420
        # first usage. Here, we'll need to do the same, as otherwise the
421
        # performance penalty is be high
422

423
        new_desc = ""
424
        for d in desc.split("\n"):
425
            if d == "":
426
                new_desc += "\n"
427
                continue
428

429
            # Use cross-references for doc files where needed
430
            d = self.re_doc.sub(r":doc:`/\1`", d)
431

432
            # Use cross-references for ABI generated docs where needed
433
            matches = self.re_abi.findall(d)
434
            for m in matches:
435
                abi = m[0] + m[1]
436

437
                xref = self.file_refs.get(abi)
438
                if not xref:
439
                    # This may happen if ABI is on a separate directory,
440
                    # like parsing ABI testing and symbol is at stable.
441
                    # The proper solution is to move this part of the code
442
                    # for it to be inside sphinx/kernel_abi.py
443
                    self.log.info("Didn't find ABI reference for '%s'", abi)
444
                else:
445
                    new = self.re_escape.sub(r"\\\1", m[1])
446
                    d = re.sub(fr"\b{abi}\b", f":ref:`{new} <{xref}>`", d)
447

448
            # Seek for cross reference symbols like /sys/...
449
            # Need to be careful to avoid doing it on a code block
450
            if d[0] not in [" ", "\t"]:
451
                matches = self.re_xref_node.findall(d)
452
                for m in matches:
453
                    # Finding ABI here is more complex due to wildcards
454
                    xref = self.what_refs.get(m)
455
                    if xref:
456
                        new = self.re_escape.sub(r"\\\1", m)
457
                        d = re.sub(fr"\b{m}\b", f":ref:`{new} <{xref}>`", d)
458

459
            new_desc += d + "\n"
460

461
        return new_desc + "\n\n"
462

463
    def doc(self, output_in_txt=False, show_symbols=True, show_file=True,
464
            filter_path=None):
465
        """Print ABI at stdout."""
466

467
        part = None
468
        for key, v in sorted(self.data.items(),
469
                             key=lambda x: (x[1].get("type", ""),
470
                                            x[1].get("what"))):
471

472
            wtype = v.get("type", "Symbol")
473
            file_ref = v.get("file")
474
            names = v.get("what", [""])
475

476
            if wtype == "File":
477
                if not show_file:
478
                    continue
479
            else:
480
                if not show_symbols:
481
                    continue
482

483
            if filter_path:
484
                if v.get("path") != filter_path:
485
                    continue
486

487
            msg = ""
488

489
            if wtype != "File":
490
                cur_part = names[0]
491
                if cur_part.find("/") >= 0:
492
                    match = self.re_what.match(cur_part)
493
                    if match:
494
                        symbol = match.group(1).rstrip("/")
495
                        cur_part = "Symbols under " + symbol
496

497
                if cur_part and cur_part != part:
498
                    part = cur_part
499
                    msg += part + "\n"+ "-" * len(part) +"\n\n"
500

501
                msg += f".. _{key}:\n\n"
502

503
                max_len = 0
504
                for i in range(0, len(names)):           # pylint: disable=C0200
505
                    names[i] = "**" + self.re_escape.sub(r"\\\1", names[i]) + "**"
506

507
                    max_len = max(max_len, len(names[i]))
508

509
                msg += "+-" + "-" * max_len + "-+\n"
510
                for name in names:
511
                    msg += f"| {name}" + " " * (max_len - len(name)) + " |\n"
512
                    msg += "+-" + "-" * max_len + "-+\n"
513
                msg += "\n"
514

515
            for ref in file_ref:
516
                if wtype == "File":
517
                    msg += f".. _{ref[1]}:\n\n"
518
                else:
519
                    base = os.path.basename(ref[0])
520
                    msg += f"Defined on file :ref:`{base} <{ref[1]}>`\n\n"
521

522
            if wtype == "File":
523
                msg += names[0] +"\n" + "-" * len(names[0]) +"\n\n"
524

525
            desc = v.get("description")
526
            if not desc and wtype != "File":
527
                msg += f"DESCRIPTION MISSING for {names[0]}\n\n"
528

529
            if desc:
530
                if output_in_txt:
531
                    msg += self.desc_txt(desc)
532
                else:
533
                    msg += self.desc_rst(desc)
534

535
            symbols = v.get("symbols")
536
            if symbols:
537
                msg += "Has the following ABI:\n\n"
538

539
                for w, label in symbols:
540
                    # Escape special chars from content
541
                    content = self.re_escape.sub(r"\\\1", w)
542

543
                    msg += f"- :ref:`{content} <{label}>`\n\n"
544

545
            users = v.get("users")
546
            if users and users.strip(" \t\n"):
547
                users = users.strip("\n").replace('\n', '\n\t')
548
                msg += f"Users:\n\t{users}\n\n"
549

550
            ln = v.get("line_no", 1)
551

552
            yield (msg, file_ref[0][0], ln)
553

554
    def check_issues(self):
555
        """Warn about duplicated ABI entries."""
556

557
        for what, v in self.what_symbols.items():
558
            files = v.get("file")
559
            if not files:
560
                # Should never happen if the parser works properly
561
                self.log.warning("%s doesn't have a file associated", what)
562
                continue
563

564
            if len(files) == 1:
565
                continue
566

567
            f = []
568
            for fname, lines in sorted(files.items()):
569
                if not lines:
570
                    f.append(f"{fname}")
571
                elif len(lines) == 1:
572
                    f.append(f"{fname}:{lines[0]}")
573
                else:
574
                    m = fname + "lines "
575
                    m += ", ".join(str(x) for x in lines)
576
                    f.append(m)
577

578
            self.log.warning("%s is defined %d times: %s", what, len(f), "; ".join(f))
579

580
    def search_symbols(self, expr):
581
        """ Searches for ABI symbols."""
582

583
        regex = re.compile(expr, re.I)
584

585
        found_keys = 0
586
        for t in sorted(self.data.items(), key=lambda x: [0]):
587
            v = t[1]
588

589
            wtype = v.get("type", "")
590
            if wtype == "File":
591
                continue
592

593
            for what in v.get("what", [""]):
594
                if regex.search(what):
595
                    found_keys += 1
596

597
                    kernelversion = v.get("kernelversion", "").strip(" \t\n")
598
                    date = v.get("date", "").strip(" \t\n")
599
                    contact = v.get("contact", "").strip(" \t\n")
600
                    users = v.get("users", "").strip(" \t\n")
601
                    desc = v.get("description", "").strip(" \t\n")
602

603
                    files = []
604
                    for f in v.get("file", ()):
605
                        files.append(f[0])
606

607
                    what = str(found_keys) + ". " + what
608
                    title_tag = "-" * len(what)
609

610
                    print(f"\n{what}\n{title_tag}\n")
611

612
                    if kernelversion:
613
                        print(f"Kernel version:\t\t{kernelversion}")
614

615
                    if date:
616
                        print(f"Date:\t\t\t{date}")
617

618
                    if contact:
619
                        print(f"Contact:\t\t{contact}")
620

621
                    if users:
622
                        print(f"Users:\t\t\t{users}")
623

624
                    print("Defined on file(s):\t" + ", ".join(files))
625

626
                    if desc:
627
                        desc = desc.strip("\n")
628
                        print(f"\n{desc}\n")
629

630
        if not found_keys:
631
            print(f"Regular expression /{expr}/ not found.")
632

633