1
#!/usr/bin/env python3
2
"""
3
PURPOSE: Automate building, installing, and publishing our modules.
4
This is like a little clone of "lerna" for our purposes.
5

6
NOTE: I wrote this initially using npm and with the goal of publishing
7
to npmjs.com.   Now I don't care at all about publishing to npmjs.com,
8
and we're using pnpm.  So this is being turned into a package just
9
for cleaning/installing/building.
10

11
TEST:
12
 - This should always work:  "mypy workspaces.py"
13
"""
14

15
import argparse, json, os, platform, shutil, subprocess, sys, time
16

17
from typing import Any, Optional, Callable, List
18

19
MAX_PACKAGE_LOCK_SIZE_MB = 5
20

21

22
def newest_file(path: str) -> str:
23
    if platform.system() != 'Darwin':
24
        # See https://gist.github.com/brwyatt/c21a888d79927cb476a4 for this Linux
25
        # version:
26
        cmd = 'find . -type f -printf "%C@ %p\n" | sort -rn | head -n 1 | cut -d" " -f2'
27
    else:
28
        # but we had to rewrite this as suggested at
29
        # https://unix.stackexchange.com/questions/272491/bash-error-find-printf-unknown-primary-or-operator
30
        # etc to work on MacOS.
31
        cmd = 'find . -type f -print0 | xargs -0r stat -f "%Fc %N" | sort -rn | head -n 1 | cut -d" " -f2'
32
    return os.popen(f'cd "{path}" && {cmd}').read().strip()
33

34

35
SUCCESSFUL_BUILD = ".successful-build"
36

37

38
def needs_build(package: str) -> bool:
39
    # Code below was hopelessly naive, e.g, a failed build would not get retried.
40
    # We only need to do a build if the newest file in the tree is not
41
    # in the dist directory.
42
    path = os.path.join(os.path.dirname(__file__), package)
43
    if not os.path.exists(os.path.join(path, 'dist')):
44
        return True
45
    newest = newest_file(path)
46
    return not newest.startswith('./' + SUCCESSFUL_BUILD)
47

48

49
def handle_path(s: str,
50
                path: Optional[str] = None,
51
                verbose: bool = True) -> None:
52
    desc = s
53
    if path is not None:
54
        os.chdir(path)
55
        desc += " # in '%s'" % path
56
    if verbose:
57
        print(desc)
58

59

60
def cmd(s: str,
61
        path: Optional[str] = None,
62
        verbose: bool = True,
63
        noerr=False) -> None:
64
    home: str = os.path.abspath(os.curdir)
65
    try:
66
        handle_path(s, path, verbose)
67
        if os.system(s):
68
            msg = f"Error executing '{s}'"
69
            if noerr:
70
                print(msg)
71
            else:
72
                raise RuntimeError(msg)
73
    finally:
74
        os.chdir(home)
75

76

77
def run(s: str, path: Optional[str] = None, verbose: bool = True) -> str:
78
    home = os.path.abspath(os.curdir)
79
    try:
80
        handle_path(s, path, verbose)
81
        a = subprocess.run(s, shell=True, stdout=subprocess.PIPE)
82
        out = a.stdout.decode('utf8')
83
        if a.returncode:
84
            raise RuntimeError("Error executing '%s'" % s)
85
        return out
86
    finally:
87
        os.chdir(home)
88

89

90
def thread_map(callable: Callable,
91
               inputs: List[Any],
92
               nb_threads: int = 10) -> List:
93
    if len(inputs) == 0:
94
        return []
95
    if nb_threads == 1:
96
        return [callable(x) for x in inputs]
97
    from multiprocessing.pool import ThreadPool
98
    tp = ThreadPool(nb_threads)
99
    return tp.map(callable, inputs)
100

101

102
def all_packages() -> List[str]:
103
    # Compute all the packages.  Explicit order in some cases *does* matter as noted in comments,
104
    # but we use "tsc --build", which automatically builds deps if not built.
105
    v = [
106
        'packages/',  # top level workspace, e.g., typescript
107
        'packages/cdn',  # packages/hub assumes this is built
108
        'packages/util',
109
        'packages/sync',
110
        'packages/sync-client',
111
        'packages/sync-fs',
112
        'packages/conat',
113
        'packages/backend',
114
        'packages/api-client',
115
        'packages/jupyter',
116
        'packages/comm',
117
        'packages/project',
118
        'packages/assets',
119
        'packages/frontend',  # static depends on frontend; frontend depends on assets
120
        'packages/static',  # packages/hub assumes this is built (for webpack dev server)
121
        'packages/server',  # packages/next assumes this is built
122
        'packages/database',  # packages/next also assumes database is built (or at least the coffeescript in it is)
123
        'packages/file-server',
124
        'packages/next',
125
        'packages/hub',  # hub won't build if next isn't already built
126
    ]
127
    for x in os.listdir('packages'):
128
        path = os.path.join("packages", x)
129
        if path not in v and os.path.isdir(path) and os.path.exists(
130
                os.path.join(path, 'package.json')):
131
            v.append(path)
132
    return v
133

134

135
def packages(args) -> List[str]:
136
    v = all_packages()
137
    # Filter to only the ones in packages (if given)
138
    if args.packages:
139
        packages = set(args.packages.split(','))
140
        v = [x for x in v if x.split('/')[-1] in packages]
141

142
    # Only take things not in exclude
143
    if args.exclude:
144
        exclude = set(args.exclude.split(','))
145
        v = [x for x in v if x.split('/')[-1] not in exclude]
146

147
    print("Packages: ", ', '.join(v))
148
    return v
149

150

151
def package_json(package: str) -> dict:
152
    return json.loads(open(f'{package}/package.json').read())
153

154

155
def write_package_json(package: str, x: dict) -> None:
156
    open(f'{package}/package.json', 'w').write(json.dumps(x, indent=2))
157

158

159
def dependent_packages(package: str) -> List[str]:
160
    # Get a list of the packages
161
    # it depends on by reading package.json
162
    x = package_json(package)
163
    if "workspaces" not in x:
164
        # no workspaces
165
        return []
166
    v: List[str] = []
167
    for path in x["workspaces"]:
168
        # path is a relative path
169
        npath = os.path.normpath(os.path.join(package, path))
170
        if npath != package:
171
            v.append(npath)
172
    return v
173

174

175
def get_package_version(package: str) -> str:
176
    return package_json(package)["version"]
177

178

179
def get_package_npm_name(package: str) -> str:
180
    return package_json(package)["name"]
181

182

183
def update_dependent_versions(package: str) -> None:
184
    """
185
    Update the versions of all of the workspaces that this
186
    package depends on.  The versions are set to whatever the
187
    current version is in the dependent packages package.json.
188

189
    There is a problem here, if you are publishing two
190
    packages A and B with versions vA and vB.  If you first publish
191
    A, then you set it as depending on B@vB.  However, when you then
192
    publish B you set its new version as vB+1, so A got published
193
    with the wrong version.  It's thus important to first
194
    update all the versions of the packages that will be published
195
    in a single phase, then update the dependent version numbers, and
196
    finally actually publish the packages to npm.  There will unavoidably
197
    be an interval of time when some of the packages are impossible to
198
    install (e.g., because A got published and depends on B@vB+1, but B
199
    isn't yet published).
200
    """
201
    x = package_json(package)
202
    changed = False
203
    for dependent in dependent_packages(package):
204
        print(f"Considering '{dependent}'")
205
        try:
206
            package_version = '^' + get_package_version(dependent)
207
        except:
208
            print(f"Skipping '{dependent}' since package not available")
209
            continue
210
        npm_name = get_package_npm_name(dependent)
211
        dev = npm_name in x.get("devDependencies", {})
212
        if dev:
213
            current_version = x.get("devDependencies", {}).get(npm_name, '')
214
        else:
215
            current_version = x.get("dependencies", {}).get(npm_name, '')
216
        # print(dependent, npm_name, current_version, package_version)
217
        if current_version != package_version:
218
            print(
219
                f"{package}: {dependent} changed from '{current_version}' to '{package_version}'"
220
            )
221
            x['devDependencies' if dev else 'dependencies'][
222
                npm_name] = package_version
223
            changed = True
224
    if changed:
225
        write_package_json(package, x)
226

227

228
def update_all_dependent_versions() -> None:
229
    for package in all_packages():
230
        update_dependent_versions(package)
231

232

233
def banner(s: str) -> None:
234
    print("\n" + "=" * 70)
235
    print("|| " + s)
236
    print("=" * 70 + "\n")
237

238

239
def install(args) -> None:
240
    v = packages(args)
241

242
    # The trick we use to build only a subset of the packages in a pnpm workspace
243
    # is to temporarily modify packages/pnpm-workspace.yaml to explicitly remove
244
    # the packages that we do NOT want to build.  This should be supported by
245
    # pnpm via the --filter option but I can't figure that out in a way that doesn't
246
    # break the global lockfile, so this is the hack we have for now.
247
    ws = "packages/pnpm-workspace.yaml"
248
    tmp = ws + ".tmp"
249
    allp = all_packages()
250
    try:
251
        if v != allp:
252
            shutil.copy(ws, tmp)
253
            s = open(ws,'r').read() + '\n'
254
            for package in allp:
255
                if package not in v:
256
                     s += '  - "!%s"\n'%package.split('/')[-1]
257

258
            open(ws,'w').write(s)
259

260
        print("install packages")
261
        # much faster special case
262
        # see https://github.com/pnpm/pnpm/issues/6778 for why we put that confirm option in
263
        # for the package-import-method, needed on zfs!, see https://github.com/pnpm/pnpm/issues/7024
264
        c = "cd packages && pnpm install --config.confirmModulesPurge=false --package-import-method=hardlink"
265
        if args.prod:
266
            args.dist_only = False
267
            args.node_modules_only = True
268
            args.parallel = True
269
            clean(args)
270
            c += " --prod"
271
        cmd(c)
272
    finally:
273
        if os.path.exists(tmp):
274
            shutil.move(tmp, ws)
275

276

277
# Build all the packages that need to be built.
278
def build(args) -> None:
279
    v = [package for package in packages(args) if needs_build(package)]
280
    CUR = os.path.abspath('.')
281

282
    def f(path: str) -> None:
283
        if not args.parallel and path != 'packages/static':
284
            # NOTE: in parallel mode we don't delete or there is no
285
            # hope of this working.
286
            dist = os.path.join(CUR, path, 'dist')
287
            if os.path.exists(dist):
288
                # clear dist/ dir
289
                shutil.rmtree(dist, ignore_errors=True)
290
        package_path = os.path.join(CUR, path)
291
        if args.dev and '"build-dev"' in open(
292
                os.path.join(CUR, path, 'package.json')).read():
293
            cmd("pnpm run build-dev", package_path)
294
        else:
295
            cmd("pnpm run build", package_path)
296
        # The build succeeded, so touch a file
297
        # to indicate this, so we won't build again
298
        # until something is newer than this file
299
        cmd("touch " + SUCCESSFUL_BUILD, package_path)
300

301
    if args.parallel:
302
        thread_map(f, v)
303
    else:
304
        thread_map(f, v, 1)
305

306

307
def clean(args) -> None:
308
    v = packages(args)
309

310
    if args.dist_only:
311
        folders = ['dist']
312
    elif args.node_modules_only:
313
        folders = ['node_modules']
314
    else:
315
        folders = ['node_modules', 'dist', SUCCESSFUL_BUILD]
316

317
    paths = []
318
    for path in v:
319
        for x in folders:
320
            y = os.path.abspath(os.path.join(path, x))
321
            if os.path.exists(y):
322
                paths.append(y)
323

324
    def f(path):
325
        print("rm -rf '%s'" % path)
326
        if not os.path.exists(path):
327
            return
328
        if os.path.isfile(path):
329
            os.unlink(path)
330
            return
331
        shutil.rmtree(path, ignore_errors=True)
332
        if os.path.exists(path):
333
            shutil.rmtree(path, ignore_errors=True)
334
        if os.path.exists(path):
335
            raise RuntimeError(f'failed to delete {path}')
336

337
    if (len(paths) == 0):
338
        banner("No node_modules or dist directories")
339
    else:
340
        banner("Deleting " + ', '.join(paths))
341
        thread_map(f, paths + ['packages/node_modules'], nb_threads=10)
342

343
    if not args.node_modules_only:
344
        banner("Running 'pnpm run clean' if it exists...")
345

346
        def g(path):
347
            # can only use --if-present with npm, but should be fine since clean is
348
            # usually just "rm".
349
            cmd("npm run clean --if-present", path)
350

351
        thread_map(g, [os.path.abspath(path) for path in v],
352
                   nb_threads=3 if args.parallel else 1)
353

354

355
def delete_package_lock(args) -> None:
356

357
    def f(path: str) -> None:
358
        p = os.path.join(path, 'package-lock.json')
359
        if os.path.exists(p):
360
            os.unlink(p)
361
        # See https://github.com/sagemathinc/cocalc/issues/6123
362
        # If we don't delete node_modules, then package-lock.json may blow up in size.
363
        node_modules = os.path.join(path, 'node_modules')
364
        if os.path.exists(node_modules):
365
            shutil.rmtree(node_modules, ignore_errors=True)
366

367
    thread_map(f, [os.path.abspath(path) for path in packages(args)],
368
               nb_threads=10)
369

370

371
def pnpm(args, noerr=False) -> None:
372
    v = packages(args)
373
    inputs: List[List[str]] = []
374
    for path in v:
375
        s = 'pnpm ' + ' '.join(['%s' % x for x in args.args])
376
        inputs.append([s, os.path.abspath(path)])
377

378
    def f(args) -> None:
379
        # kwds to make mypy happy
380
        kwds = {"noerr": noerr}
381
        cmd(*args, **kwds)
382

383
    if args.parallel:
384
        thread_map(f, inputs, 3)
385
    else:
386
        thread_map(f, inputs, 1)
387

388

389
def pnpm_noerror(args) -> None:
390
    pnpm(args, noerr=True)
391

392

393
def version_check(args):
394
    cmd("scripts/check_npm_packages.py")
395
    cmd("pnpm check-deps", './packages')
396

397

398
def node_version_check() -> None:
399
    version = int(os.popen('node --version').read().split('.')[0][1:])
400
    if version < 14:
401
        err = f"CoCalc requires node.js v14, but you're using node v{version}."
402
        if os.environ.get("COCALC_USERNAME",
403
                          '') == 'user' and 'COCALC_PROJECT_ID' in os.environ:
404
            err += '\nIf you are using https://cocalc.com, put ". /cocalc/nvm/nvm.sh" in ~/.bashrc\nto get an appropriate version of node.'
405
        raise RuntimeError(err)
406

407

408
def pnpm_version_check() -> None:
409
    """
410
    Check if the pnpm utility is new enough
411
    """
412
    version = os.popen('pnpm --version').read()
413
    if int(version.split('.')[0]) < 7:
414
        raise RuntimeError(
415
            f"CoCalc requires pnpm version 7, but you're using pnpm v{version}."
416
        )
417

418

419
def main() -> None:
420
    node_version_check()
421
    pnpm_version_check()
422

423
    def packages_arg(parser):
424
        parser.add_argument(
425
            '--packages',
426
            type=str,
427
            default='',
428
            help=
429
            '(default: ""=everything) "foo,bar" means only the packages named foo and bar'
430
        )
431
        parser.add_argument(
432
            '--exclude',
433
            type=str,
434
            default='',
435
            help=
436
            '(default: ""=exclude nothing) "foo,bar" means exclude foo and bar'
437
        )
438
        parser.add_argument(
439
            '--parallel',
440
            action="store_const",
441
            const=True,
442
            help=
443
            'if given, do all in parallel; this will not work in some cases and may be ignored in others'
444
        )
445

446
    parser = argparse.ArgumentParser(prog='workspaces')
447
    subparsers = parser.add_subparsers(help='sub-command help')
448

449
    subparser = subparsers.add_parser(
450
        'install', help='install node_modules deps for all packages')
451
    subparser.add_argument('--prod',
452
                           action="store_const",
453
                           const=True,
454
                           help='only install prod deps (not dev ones)')
455
    packages_arg(subparser)
456
    subparser.set_defaults(func=install)
457

458
    subparser = subparsers.add_parser(
459
        'build', help='build all packages for which something has changed')
460
    subparser.add_argument(
461
        '--dev',
462
        action="store_const",
463
        const=True,
464
        help="only build enough for development (saves time and space)")
465
    packages_arg(subparser)
466
    subparser.set_defaults(func=build)
467

468
    subparser = subparsers.add_parser(
469
        'clean', help='delete dist and node_modules folders')
470
    packages_arg(subparser)
471
    subparser.add_argument('--dist-only',
472
                           action="store_const",
473
                           const=True,
474
                           help="only delete dist directory")
475
    subparser.add_argument('--node-modules-only',
476
                           action="store_const",
477
                           const=True,
478
                           help="only delete node_modules directory")
479
    subparser.set_defaults(func=clean)
480

481
    subparser = subparsers.add_parser('pnpm',
482
                                      help='do "pnpm ..." in each package;')
483
    packages_arg(subparser)
484
    subparser.add_argument('args',
485
                           type=str,
486
                           nargs='*',
487
                           default='',
488
                           help='arguments to npm')
489
    subparser.set_defaults(func=pnpm)
490

491
    subparser = subparsers.add_parser(
492
        'pnpm-noerr',
493
        help=
494
        'like "pnpm" but suppresses errors; e.g., use for "pnpm-noerr audit fix"'
495
    )
496
    packages_arg(subparser)
497
    subparser.add_argument('args',
498
                           type=str,
499
                           nargs='*',
500
                           default='',
501
                           help='arguments to pnpm')
502
    subparser.set_defaults(func=pnpm_noerror)
503

504
    subparser = subparsers.add_parser(
505
        'version-check', help='version consistency checks across packages')
506
    subparser.set_defaults(func=version_check)
507

508
    args = parser.parse_args()
509
    if hasattr(args, 'func'):
510
        args.func(args)
511

512

513
if __name__ == '__main__':
514
    main()
515

516