1
#!/usr/bin/env python3
2
# SPDX-License-Identifier: GPL-2.0+
3
#
4
# pylint: disable=C0103,C0209
5
#
6
#
7

8
"""
9
Interacts with the POSIX jobserver during the Kernel build time.
10

11
A "normal" jobserver task, like the one initiated by a make subrocess would do:
12

13
    - open read/write file descriptors to communicate with the job server;
14
    - ask for one slot by calling::
15

16
        claim = os.read(reader, 1)
17

18
    - when the job finshes, call::
19

20
        os.write(writer, b"+")  # os.write(writer, claim)
21

22
Here, the goal is different: This script aims to get the remaining number
23
of slots available, using all of them to run a command which handle tasks in
24
parallel. To to that, it has a loop that ends only after there are no
25
slots left. It then increments the number by one, in order to allow a
26
call equivalent to ``make -j$((claim+1))``, e.g. having a parent make creating
27
$claim child to do the actual work.
28

29
The end goal here is to keep the total number of build tasks under the
30
limit established by the initial ``make -j$n_proc`` call.
31

32
See:
33
    https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
34
"""
35

36
import errno
37
import os
38
import subprocess
39
import sys
40

41
def warn(text, *args):
42
    print(f'WARNING: {text}', *args, file = sys.stderr)
43

44
class JobserverExec:
45
    """
46
    Claim all slots from make using POSIX Jobserver.
47

48
    The main methods here are:
49

50
    - open(): reserves all slots;
51
    - close(): method returns all used slots back to make;
52
    - run(): executes a command setting PARALLELISM=<available slots jobs + 1>.
53
    """
54

55
    def __init__(self):
56
        """Initialize internal vars."""
57
        self.claim = 0
58
        self.jobs = b""
59
        self.reader = None
60
        self.writer = None
61
        self.is_open = False
62

63
    def open(self):
64
        """Reserve all available slots to be claimed later on."""
65

66
        if self.is_open:
67
            return
68
        self.is_open = True  # We only try once
69
        self.claim = None
70
        #
71
        # Check the make flags for "--jobserver=R,W"
72
        # Note that GNU Make has used --jobserver-fds and --jobserver-auth
73
        # so this handles all of them.
74
        #
75
        flags = os.environ.get('MAKEFLAGS', '')
76
        opts = [x for x in flags.split(" ") if x.startswith("--jobserver")]
77
        if not opts:
78
            return
79
        #
80
        # Separate out the provided file descriptors
81
        #
82
        split_opt = opts[-1].split('=', 1)
83
        if len(split_opt) != 2:
84
            warn('unparseable option:', opts[-1])
85
            return
86
        fds = split_opt[1]
87
        #
88
        # As of GNU Make 4.4, we'll be looking for a named pipe
89
        # identified as fifo:path
90
        #
91
        if fds.startswith('fifo:'):
92
            path = fds[len('fifo:'):]
93
            try:
94
                self.reader = os.open(path, os.O_RDONLY | os.O_NONBLOCK)
95
                self.writer = os.open(path, os.O_WRONLY)
96
            except (OSError, IOError):
97
                warn('unable to open jobserver pipe', path)
98
                return
99
        #
100
        # Otherwise look for integer file-descriptor numbers.
101
        #
102
        else:
103
            split_fds = fds.split(',')
104
            if len(split_fds) != 2:
105
                warn('malformed jobserver file descriptors:', fds)
106
                return
107
            try:
108
                self.reader = int(split_fds[0])
109
                self.writer = int(split_fds[1])
110
            except ValueError:
111
                warn('non-integer jobserver file-descriptors:', fds)
112
                return
113
            try:
114
                #
115
                # Open a private copy of reader to avoid setting nonblocking
116
                # on an unexpecting process with the same reader fd.
117
                #
118
                self.reader = os.open(f"/proc/self/fd/{self.reader}",
119
                                      os.O_RDONLY | os.O_NONBLOCK)
120
            except (IOError, OSError) as e:
121
                warn('Unable to reopen jobserver read-side pipe:', repr(e))
122
                return
123
        #
124
        # OK, we have the channel to the job server; read out as many jobserver
125
        # slots as possible.
126
        #
127
        while True:
128
            try:
129
                slot = os.read(self.reader, 8)
130
                if not slot:
131
                    #
132
                    # Something went wrong.  Clear self.jobs to avoid writing
133
                    # weirdness back to the jobserver and give up.
134
                    self.jobs = b""
135
                    warn("unexpected empty token from jobserver;"
136
                         " possible invalid '--jobserver-auth=' setting")
137
                    self.claim = None
138
                    return
139
            except (OSError, IOError) as e:
140
                #
141
                # If there is nothing more to read then we are done.
142
                #
143
                if e.errno == errno.EWOULDBLOCK:
144
                    break
145
                #
146
                # Anything else says that something went weird; give back
147
                # the jobs and give up.
148
                #
149
                if self.jobs:
150
                    os.write(self.writer, self.jobs)
151
                    self.claim = None
152
                    warn('error reading from jobserver pipe', repr(e))
153
                    return
154
            self.jobs += slot
155
        #
156
        # Add a bump for our caller's reserveration, since we're just going
157
        # to sit here blocked on our child.
158
        #
159
        self.claim = len(self.jobs) + 1
160

161
    def close(self):
162
        """Return all reserved slots to Jobserver."""
163

164
        if not self.is_open:
165
            return
166

167
        # Return all the reserved slots.
168
        if len(self.jobs):
169
            os.write(self.writer, self.jobs)
170

171
        self.is_open = False
172

173
    def __enter__(self):
174
        self.open()
175
        return self
176

177
    def __exit__(self, exc_type, exc_value, exc_traceback):
178
        self.close()
179

180
    def run(self, cmd, *args, **pwargs):
181
        """
182
        Run a command setting PARALLELISM env variable to the number of
183
        available job slots (claim) + 1, e.g. it will reserve claim slots
184
        to do the actual build work, plus one to monitor its children.
185
        """
186
        self.open()             # Ensure that self.claim is set
187

188
        # We can only claim parallelism if there was a jobserver (i.e. a
189
        # top-level "-jN" argument) and there were no other failures. Otherwise
190
        # leave out the environment variable and let the child figure out what
191
        # is best.
192
        if self.claim:
193
            os.environ["PARALLELISM"] = str(self.claim)
194

195
        return subprocess.call(cmd, *args, **pwargs)
196

197