1
#!/usr/bin/env ruby
2
# -*- coding: binary -*-
3

4
#
5
# by h00die
6
#
7
# Fetches all stable GitLab EE and CE Docker tags newer than the highest version
8
# already present in version.json, then pulls only the application layer blob
9
# directly from the Docker Registry API (no Docker daemon required) and streams
10
# it through gzip+tar to extract the application-HASH.css filename.
11
#
12
# Requirements: none beyond Ruby stdlib
13
#
14
# Usage:
15
#   ruby tools/dev/update_gitlab_versions.rb [options]
16
#
17

18
require 'optparse'
19
require 'net/http'
20
require 'uri'
21
require 'json'
22
require 'set'
23
require 'zlib'
24

25
# -- paths / constants ---------------------------------------------------------
26

27
JSON_FILE = File.expand_path('../../data/gitlab_versions.json', __dir__)
28
REGISTRY = 'https://registry-1.docker.io/v2'.freeze
29
AUTH_URL = 'https://auth.docker.io/token?service=registry.docker.io'.freeze
30
MAX_CONCURRENT = 4
31

32
EE_TAG_RE = /\A(\d+)\.(\d+)\.(\d+)-ee\.(\d+)\z/
33
CE_TAG_RE = /\A(\d+)\.(\d+)\.(\d+)-ce\.(\d+)\z/
34

35
EDITIONS = [
36
  {
37
    repo: 'gitlab/gitlab-ee', tag_re: EE_TAG_RE, label: 'EE',
38
    version_fn: ->(tag) { tag.sub(/-ee\.\d+\z/, '-ee') }
39
  },
40
  {
41
    repo: 'gitlab/gitlab-ce', tag_re: CE_TAG_RE, label: 'CE',
42
    version_fn: ->(tag) { tag.sub(/-ce\.\d+\z/, '-ce') }
43
  }
44
].freeze
45

46
# Prefer Docker v2 manifest types - OCI manifests may use zstd-compressed layers
47
# which we cannot decompress. Docker v2 layers are always gzip.
48
MANIFEST_ACCEPT = [
49
  'application/vnd.docker.distribution.manifest.v2+json',
50
  'application/vnd.docker.distribution.manifest.list.v2+json',
51
  'application/vnd.oci.image.index.v1+json',
52
  'application/vnd.oci.image.manifest.v1+json'
53
].join(', ').freeze
54

55
# -- colours -------------------------------------------------------------------
56

57
class String
58
  def red
59
    "\e[1;31;40m#{self}\e[0m"
60
  end
61

62
  def yellow
63
    "\e[1;33;40m#{self}\e[0m"
64
  end
65

66
  def green
67
    "\e[1;32;40m#{self}\e[0m"
68
  end
69

70
  def cyan
71
    "\e[1;36;40m#{self}\e[0m"
72
  end
73
end
74

75
# -- helpers -------------------------------------------------------------------
76

77
def tag_semver(tag, re)
78
  m = re.match(tag)
79
  return nil unless m
80

81
  [m[1].to_i, m[2].to_i, m[3].to_i]
82
end
83

84
def parse_semver(str)
85
  m = str.match(/\A(\d+)\.(\d+)\.(\d+)/)
86
  m ? [m[1].to_i, m[2].to_i, m[3].to_i] : nil
87
end
88

89
def http_get(url)
90
  uri = URI(url)
91
  Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https',
92
                                      open_timeout: 15, read_timeout: 30) do |http|
93
    http.request(Net::HTTP::Get.new(uri))
94
  end
95
end
96

97
# -- Docker Hub tag enumeration -------------------------------------------------
98

99
def fetch_all_tags(repo)
100
  tags = []
101
  url = "https://hub.docker.com/v2/repositories/#{repo}/tags?page_size=100&ordering=last_updated"
102

103
  loop do
104
    resp = http_get(url)
105
    unless resp.is_a?(Net::HTTPSuccess)
106
      warn "  Docker Hub request failed (#{resp.code}): #{url}".red
107
      break
108
    end
109

110
    data = JSON.parse(resp.body)
111
    tags.concat(data.fetch('results', []).map { |t| t['name'] })
112

113
    url = data['next']
114
    break unless url
115
  end
116

117
  tags
118
end
119

120
# -- Streaming tar scanner ------------------------------------------------------
121
#
122
# Feeds decompressed gzip data into this object chunk by chunk. It walks tar
123
# headers (512 bytes each), skips file data blocks, and stops as soon as a
124
# path matching /assets/application-HASH.css is found. The caller can abort
125
# the HTTP download immediately after `found` is set, avoiding downloading the
126
# rest of the layer.
127

128
# Handles three tar long-name extensions:
129
#   PAX ('x'/'X' typeflag) - extended header with `path=<full>` key
130
#   GNU  ('L' typeflag)    - next data block is the full filename
131
#   USTAR prefix field     - 155-byte prefix at offset 345 prepended to name
132
#
133
# When sample_re is set, collects all matching filenames into #samples instead
134
# of stopping at the first CSS_HASH_RE match. Used for --sample diagnostics.
135
class TarCssScanner
136
  HEADER_SIZE = 512
137
  CSS_HASH_RE = %r{/assets/application-([a-f0-9]+)\.css\z}
138

139
  attr_reader :found, :samples
140

141
  def initialize
142
    @buf = ''.b
143
    @skip = 0
144
    @found = nil
145
    @pax_path = nil   # path extracted from PAX extended header
146
    @gnu_name = nil   # name extracted from GNU ././@LongLink block
147
    @collect_type = nil   # :pax or :gnu - currently collecting payload
148
    @collect_need = 0     # bytes still needed
149
    @collect_buf = ''.b
150
  end
151

152
  def <<(data)
153
    @buf << data.b
154
    step while @found.nil? && enough?
155
    self
156
  end
157

158
  private
159

160
  def enough?
161
    return !@buf.empty? if @collect_type
162

163
    @skip > 0 ? !@buf.empty? : @buf.size >= HEADER_SIZE
164
  end
165

166
  def step
167
    # -- collect mode: gather PAX / GNU payload ----------------------------
168
    if @collect_type
169
      take = [@collect_need, @buf.size].min
170
      @collect_buf << @buf.byteslice(0, take)
171
      @buf = @buf.byteslice(take..) || ''.b
172
      @collect_need -= take
173

174
      return unless @collect_need <= 0
175

176
      case @collect_type
177
      when :pax
178
        # PAX lines: "<decimal-len> <key>=<value>\n" - keep binary to match binary literals
179
        @collect_buf.scan(/\d+ path=([^\n]+)/) { |m| @pax_path = m[0] }
180
      when :gnu
181
        @gnu_name = @collect_buf.delete("\x00")
182
      end
183

184
      @collect_type = nil
185
      @collect_buf = ''.b
186
      return
187
    end
188

189
    # -- skip mode: discard data / padding blocks ---------------------------
190
    if @skip > 0
191
      take = [@skip, @buf.size].min
192
      @buf = @buf.byteslice(take..)
193
      @skip -= take
194
      return
195
    end
196

197
    # -- header mode --------------------------------------------------------
198
    header = @buf.byteslice(0, HEADER_SIZE)
199
    @buf = @buf.byteslice(HEADER_SIZE..) || ''.b
200
    typeflag = header.byteslice(156, 1) || "\x00"
201
    name = header.byteslice(0, 100).delete("\x00")
202
    size = header.byteslice(124, 12).strip.to_i(8)
203

204
    case typeflag
205
    when 'x', 'X' # PAX extended header
206
      padded = ((size + 511) / 512) * 512
207
      @collect_type = :pax
208
      @collect_need = padded
209
      @collect_buf = ''.b
210
    when 'L' # GNU long filename follows in data block
211
      padded = ((size + 511) / 512) * 512
212
      @collect_type = :gnu
213
      @collect_need = padded
214
      @collect_buf = ''.b
215
    else
216
      return if name.empty? # end-of-archive zero block
217

218
      # Reconstruct full path: PAX > GNU > USTAR-prefix+name
219
      effective = if @pax_path
220
                    @pax_path
221
                  elsif @gnu_name
222
                    @gnu_name
223
                  else
224
                    prefix = header.byteslice(345, 155).delete("\x00")
225
                    prefix.empty? ? name : "#{prefix}/#{name}"
226
                  end
227

228
      @pax_path = nil
229
      @gnu_name = nil
230

231
      if (m = effective.match(CSS_HASH_RE))
232
        @found = m[1]
233
        return
234
      end
235

236
      @skip = ((size + 511) / 512) * 512
237
    end
238
  end
239
end
240

241
# Collects every filename in a gzip-compressed tar layer that matches a regex.
242
# Never stops early - reads the whole stream. Used for --sample diagnostics.
243
class TarFilenameCollector
244
  HEADER_SIZE = 512
245

246
  attr_reader :filenames
247

248
  def initialize(pattern = /\.css\z/)
249
    @pattern = pattern
250
    @filenames = []
251
    @buf = ''.b
252
    @skip = 0
253
    @collect_type = nil
254
    @collect_need = 0
255
    @collect_buf = ''.b
256
    @pax_path = nil
257
    @gnu_name = nil
258
  end
259

260
  def <<(data)
261
    @buf << data.b
262
    step while enough?
263
    self
264
  end
265

266
  private
267

268
  def enough?
269
    return !@buf.empty? if @collect_type
270

271
    @skip > 0 ? !@buf.empty? : @buf.size >= HEADER_SIZE
272
  end
273

274
  def step
275
    if @collect_type
276
      take = [@collect_need, @buf.size].min
277
      @collect_buf << @buf.byteslice(0, take)
278
      @buf = @buf.byteslice(take..) || ''.b
279
      @collect_need -= take
280
      return unless @collect_need <= 0
281

282
      case @collect_type
283
      when :pax
284
        @collect_buf.scan(/\d+ path=([^\n]+)/) { |m| @pax_path = m[0] }
285
      when :gnu
286
        @gnu_name = @collect_buf.delete("\x00")
287
      end
288
      @collect_type = nil
289
      @collect_buf = ''.b
290
      return
291
    end
292

293
    if @skip > 0
294
      take = [@skip, @buf.size].min
295
      @buf = @buf.byteslice(take..)
296
      @skip -= take
297
      return
298
    end
299

300
    return unless @buf.size >= HEADER_SIZE
301

302
    header = @buf.byteslice(0, HEADER_SIZE)
303
    @buf = @buf.byteslice(HEADER_SIZE..) || ''.b
304
    typeflag = header.byteslice(156, 1) || "\x00"
305
    name = header.byteslice(0, 100).delete("\x00")
306
    size = header.byteslice(124, 12).strip.to_i(8)
307

308
    case typeflag
309
    when 'x', 'X'
310
      padded = ((size + 511) / 512) * 512
311
      @collect_type = :pax
312
      @collect_need = padded
313
      @collect_buf = ''.b
314
    when 'L'
315
      padded = ((size + 511) / 512) * 512
316
      @collect_type = :gnu
317
      @collect_need = padded
318
      @collect_buf = ''.b
319
    else
320
      return if name.empty?
321

322
      effective = if @pax_path then @pax_path
323
                  elsif @gnu_name then @gnu_name
324
                  else
325
                    prefix = header.byteslice(345, 155).delete("\x00")
326
                    prefix.empty? ? name : "#{prefix}/#{name}"
327
                  end
328
      @pax_path = nil
329
      @gnu_name = nil
330

331
      @filenames << effective if effective.match?(@pattern)
332
      @skip = ((size + 511) / 512) * 512
333
    end
334
  end
335
end
336

337
# -- Docker Registry API --------------------------------------------------------
338

339
def registry_token(repo)
340
  resp = http_get("#{AUTH_URL}&scope=repository:#{repo}:pull")
341
  JSON.parse(resp.body)['token']
342
end
343

344
def registry_get(uri, token)
345
  Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: 15, read_timeout: 30) do |http|
346
    req = Net::HTTP::Get.new(uri)
347
    req['Authorization'] = "Bearer #{token}"
348
    req['Accept'] = MANIFEST_ACCEPT
349
    http.request(req)
350
  end
351
end
352

353
# Wraps registry_get with automatic retry on 429 (rate limit).
354
# Sleeps for Retry-After seconds (or exponential backoff) before retrying.
355
def registry_get_with_retry(uri, token, max_retries: 3)
356
  (max_retries + 1).times do |attempt|
357
    resp = registry_get(uri, token)
358
    return resp unless resp.code == '429'
359

360
    wait = [(resp['retry-after'].to_i.nonzero? || 2**attempt * 15), 120].min
361
    warn "  [rate limited (429), waiting #{wait}s before retry #{attempt + 1}/#{max_retries}]".yellow
362
    sleep wait
363
  end
364
  registry_get(uri, token)
365
end
366

367
# Returns the v2 image manifest for a given tag, resolving manifest lists to
368
# the linux/amd64 platform entry automatically.
369
# Returns nil on error, :expired on 401 (token needs refresh).
370
def fetch_manifest(repo, tag, token, verbose: false)
371
  uri = URI("#{REGISTRY}/#{repo}/manifests/#{tag}")
372
  resp = registry_get_with_retry(uri, token)
373
  return :expired if resp.code == '401'
374

375
  if !resp.is_a?(Net::HTTPSuccess)
376
    warn "  manifest #{tag}: HTTP #{resp.code}".yellow if verbose
377
    return nil
378
  end
379

380
  manifest = JSON.parse(resp.body)
381
  ct = resp['content-type'].to_s
382

383
  # Multi-platform manifest list - drill down to linux/amd64
384
  if ct.include?('manifest.list') || ct.include?('image.index')
385
    entry = manifest['manifests']&.find do |m|
386
      m.dig('platform', 'os') == 'linux' && m.dig('platform', 'architecture') == 'amd64'
387
    end
388
    return nil unless entry
389

390
    resp = registry_get_with_retry(URI("#{REGISTRY}/#{repo}/manifests/#{entry['digest']}"), token)
391
    return :expired if resp.code == '401'
392

393
    if !resp.is_a?(Net::HTTPSuccess)
394
      warn "  manifest #{tag} (amd64): HTTP #{resp.code}".yellow if verbose
395
      return nil
396
    end
397

398
    manifest = JSON.parse(resp.body)
399
  end
400

401
  manifest
402
end
403

404
# Streams a single gzip-compressed layer blob and scans tar headers for the
405
# application-HASH.css filename. Stops the download as soon as the hash is
406
# found - no need to consume the full layer.
407
#
408
# Registry blobs usually redirect (302) to cloud storage (S3/GCS/CDN); the
409
# redirect is followed manually so we never send the registry Bearer token to
410
# a third-party host.
411
def scan_layer(repo, digest, token, mediatype: nil, verbose: false)
412
  if mediatype&.include?('zstd')
413
    warn "    [skip zstd] #{digest[7, 16]}...".yellow if verbose
414
    return nil
415
  end
416
  warn "    [scan] #{digest[7, 16]}... (#{mediatype || 'unknown'})".cyan if verbose
417

418
  blob_uri = URI("#{REGISTRY}/#{repo}/blobs/#{digest}")
419
  redirect_uri = nil
420
  inline = nil
421

422
  # First request: registry endpoint - expect a 302 to cloud storage
423
  Net::HTTP.start(blob_uri.host, blob_uri.port, use_ssl: true,
424
                                                open_timeout: 15, read_timeout: 30) do |http|
425
    req = Net::HTTP::Get.new(blob_uri)
426
    req['Authorization'] = "Bearer #{token}"
427
    http.request(req) do |resp|
428
      case resp
429
      when Net::HTTPRedirection
430
        redirect_uri = URI(resp['location'])
431
      when Net::HTTPSuccess
432
        inline = scan_blob_stream(resp)
433
      end
434
    end
435
  end
436

437
  return inline unless redirect_uri
438

439
  # Second request: cloud storage - stream and scan, no auth header
440
  result = nil
441
  Net::HTTP.start(redirect_uri.host, redirect_uri.port, use_ssl: true,
442
                                                        open_timeout: 15, read_timeout: 300) do |http|
443
    req = Net::HTTP::Get.new(redirect_uri)
444
    http.request(req) do |resp|
445
      result = scan_blob_stream(resp) if resp.is_a?(Net::HTTPSuccess)
446
    end
447
  end
448
  result
449
rescue Zlib::Error, Errno::ECONNRESET, Net::ReadTimeout, OpenSSL::SSL::SSLError => e
450
  warn "    layer #{digest[7, 16]}... #{e.class}: #{e.message}".yellow
451
  nil
452
end
453

454
# Streams resp body through gzip decompression and a TarCssScanner.
455
# Throws :done as soon as the CSS hash is found so the caller's read_body
456
# loop exits early and the rest of the blob is not downloaded.
457
def scan_blob_stream(resp)
458
  return nil unless resp.is_a?(Net::HTTPSuccess)
459

460
  scanner = TarCssScanner.new
461
  inflater = Zlib::Inflate.new(Zlib::MAX_WBITS | 16) # gzip mode
462

463
  begin
464
    catch(:done) do
465
      resp.read_body do |chunk|
466
        scanner << inflater.inflate(chunk)
467
        throw :done if scanner.found
468
      end
469
    end
470
    scanner.found
471
  ensure
472
    begin
473
      inflater.close
474
    rescue StandardError
475
      nil
476
    end
477
  end
478
end
479

480
# Streams the full layer and collects every filename matching pattern.
481
def collect_layer_filenames(repo, digest, token, pattern: /\.css\z/)
482
  blob_uri = URI("#{REGISTRY}/#{repo}/blobs/#{digest}")
483
  redirect_uri = nil
484

485
  Net::HTTP.start(blob_uri.host, blob_uri.port, use_ssl: true,
486
                                                open_timeout: 15, read_timeout: 30) do |http|
487
    req = Net::HTTP::Get.new(blob_uri)
488
    req['Authorization'] = "Bearer #{token}"
489
    http.request(req) do |resp|
490
      return stream_filenames(resp, pattern) if resp.is_a?(Net::HTTPSuccess)
491

492
      redirect_uri = URI(resp['location']) if resp.is_a?(Net::HTTPRedirection)
493
    end
494
  end
495

496
  return [] unless redirect_uri
497

498
  Net::HTTP.start(redirect_uri.host, redirect_uri.port, use_ssl: true,
499
                                                        open_timeout: 15, read_timeout: 300) do |http|
500
    req = Net::HTTP::Get.new(redirect_uri)
501
    http.request(req) do |resp|
502
      return stream_filenames(resp, pattern) if resp.is_a?(Net::HTTPSuccess)
503
    end
504
  end
505
  []
506
rescue StandardError => e
507
  warn "  collect error: #{e.class}: #{e.message}".red
508
  []
509
end
510

511
def stream_filenames(resp, pattern)
512
  collector = TarFilenameCollector.new(pattern)
513
  inflater = Zlib::Inflate.new(Zlib::MAX_WBITS | 16)
514
  begin
515
    resp.read_body { |chunk| collector << inflater.inflate(chunk) }
516
    collector.filenames
517
  ensure
518
    begin
519
      inflater.close
520
    rescue StandardError
521
      nil
522
    end
523
  end
524
end
525

526
# Prints all filenames matching pattern across all layers of repo:tag.
527
def sample_tag(repo, tag, pattern: /\.css\z/)
528
  puts "\nSampling #{repo}:#{tag} for filenames matching #{pattern}...".cyan
529
  token = registry_token(repo)
530
  manifest = fetch_manifest(repo, tag, token)
531
  unless manifest
532
    warn '  Could not fetch manifest'.red
533
    return
534
  end
535

536
  layers = manifest['layers']&.reverse || []
537
  puts "  #{layers.size} layer(s), scanning newest-first...".cyan
538

539
  layers.each_with_index do |layer, idx|
540
    next if layer['mediaType']&.include?('zstd')
541

542
    print "  Layer #{idx + 1}/#{layers.size} #{layer['digest'][7, 16]}... "
543
    $stdout.flush
544
    files = collect_layer_filenames(repo, layer['digest'], token, pattern: pattern)
545
    puts "(#{files.size} match#{files.size == 1 ? '' : 'es'})"
546
    files.each { |f| puts "    #{f}" }
547
  end
548
end
549

550
# Fetches the manifest for repo:tag and scans layers newest-first.
551
# token_box is a single-element array [token] shared across threads; mutex
552
# protects refreshes so only one thread re-fetches when the token expires.
553
def get_css_hash(repo, tag, token_box, token_mutex, verbose: false)
554
  token = token_mutex.synchronize { token_box[0] }
555
  manifest = fetch_manifest(repo, tag, token, verbose: verbose)
556

557
  if manifest == :expired
558
    token_mutex.synchronize do
559
      # Only refresh if another thread hasn't already done it
560
      if token_box[0] == token
561
        token_box[0] = registry_token(repo)
562
        warn "  [token refreshed for #{repo}]".yellow if verbose
563
      end
564
    end
565
    token = token_mutex.synchronize { token_box[0] }
566
    manifest = fetch_manifest(repo, tag, token, verbose: verbose)
567
    return nil if manifest == :expired
568
  end
569

570
  return nil unless manifest
571

572
  layers = manifest['layers']&.reverse
573
  return nil if layers.nil? || layers.empty?
574

575
  warn "  #{layers.size} layer(s) found, scanning newest-first...".cyan if verbose
576
  layers.each do |layer|
577
    result = scan_layer(repo, layer['digest'], token, mediatype: layer['mediaType'], verbose: verbose)
578
    return result if result
579
  end
580
  nil
581
end
582

583
# -- JSON file helpers ----------------------------------------------------------
584

585
def load_json_map
586
  JSON.parse(File.read(JSON_FILE))
587
end
588

589
def max_version_in_map(data)
590
  data.values.flatten.filter_map { |v| parse_semver(v) }.max
591
end
592

593
def collapse_ranges(version_hashes)
594
  entries = []
595
  version_hashes.each do |ver, hash|
596
    if entries.last && entries.last[:hash] == hash
597
      entries.last[:high] = ver
598
    else
599
      entries << { hash: hash, low: ver, high: ver }
600
    end
601
  end
602
  entries
603
end
604

605
def write_json_map(data)
606
  lines = data.map { |k, v| "  #{k.to_json}: #{v.to_json}" }
607
  File.write(JSON_FILE, "{\n#{lines.join(",\n")}\n}\n")
608
end
609

610
def update_version_file(new_entries, dry_run:)
611
  data = load_json_map
612
  added = []
613
  updated = []
614

615
  new_entries.each do |e|
616
    if data.key?(e[:hash])
617
      # Hash already known - extend the high end of the range if the new version is higher.
618
      # When semvers are equal but suffixes differ, prefer -ee over -ce.
619
      existing_high_str = data[e[:hash]][1]
620
      existing_high = parse_semver(existing_high_str)
621
      new_high = parse_semver(e[:high])
622
      next unless new_high && existing_high
623

624
      cmp = new_high <=> existing_high
625
      next if cmp < 0
626
      # Same semver: only replace if we're upgrading from -ce to -ee
627
      next if cmp == 0 && !(existing_high_str.end_with?('-ce') && e[:high].end_with?('-ee'))
628

629
      data[e[:hash]][1] = e[:high] unless dry_run
630
      updated << e
631
    else
632
      data[e[:hash]] = [e[:low], e[:high]] unless dry_run
633
      added << e
634
    end
635
  end
636

637
  if added.empty? && updated.empty?
638
    puts 'No new entries to add - already up to date.'.green
639
    return
640
  end
641

642
  tag = dry_run ? ' [dry-run]' : ''
643
  unless added.empty?
644
    puts "\n#{added.size} new entr#{added.size == 1 ? 'y' : 'ies'} added#{tag}:".green
645
    added.each { |e| puts "  #{e[:hash].to_json}: #{[e[:low], e[:high]].to_json}" }
646
  end
647
  unless updated.empty?
648
    puts "\n#{updated.size} existing entr#{updated.size == 1 ? 'y' : 'ies'} range-extended#{tag}:".cyan
649
    updated.each { |e| puts "  #{e[:hash][0, 16]}... high -> #{e[:high]}" }
650
  end
651

652
  write_json_map(data) unless dry_run
653
end
654

655
def process_edition(edition, current_max, opts)
656
  repo = edition[:repo]
657
  tag_re = edition[:tag_re]
658
  label = edition[:label]
659
  version_fn = edition[:version_fn]
660

661
  puts "\nFetching GitLab #{label} tags from Docker Hub..."
662
  all_tags = fetch_all_tags(repo)
663
  puts "  #{all_tags.size} total tags fetched."
664

665
  candidates = all_tags.select { |t| tag_re.match?(t) }.select do |t|
666
    sv = tag_semver(t, tag_re)
667
    sv && (sv <=> current_max) > 0
668
  end.sort_by { |t| tag_semver(t, tag_re) }
669

670
  if candidates.empty?
671
    puts "  No new #{label} versions found.".green
672
    return []
673
  end
674

675
  puts "  Found #{candidates.size} new #{label} tag(s):".cyan
676
  candidates.each { |t| puts "    #{t}" }
677

678
  if opts[:dry_run]
679
    puts '[dry-run] skipping registry layer fetch'.cyan
680
    return candidates.map do |t|
681
      { hash: "dryrun#{'0' * 57}", low: version_fn.call(t), high: version_fn.call(t) }
682
    end
683
  end
684

685
  token_box = [registry_token(repo)]
686
  token_mutex = Mutex.new
687
  lock = Mutex.new
688
  results = {}
689
  work = Queue.new
690
  candidates.each { |t| work << t }
691

692
  puts "  Fetching CSS hashes (#{[MAX_CONCURRENT, candidates.size].min} parallel workers)...".cyan
693

694
  workers = [MAX_CONCURRENT, candidates.size].min.times.map do
695
    Thread.new do
696
      loop do
697
        tag = begin; work.pop(true); rescue ThreadError; break; end
698
        begin
699
          ver = version_fn.call(tag)
700
          hash = get_css_hash(repo, tag, token_box, token_mutex, verbose: opts[:verbose])
701
          lock.synchronize do
702
            if hash
703
              puts "  #{tag} ... #{hash[0, 16].green}..."
704
            else
705
              puts "  #{tag} ... #{'no CSS hash found'.yellow}"
706
            end
707
            results[tag] = [ver, hash] if hash
708
          end
709
        rescue StandardError => e
710
          lock.synchronize { warn "  #{tag} ... #{e.class}: #{e.message}".red }
711
        end
712
      end
713
    end
714
  end
715

716
  workers.each(&:join)
717

718
  ordered = candidates.filter_map { |t| results[t] }
719
  collapse_ranges(ordered)
720
end
721

722
# -- CLI -----------------------------------------------------------------------
723

724
options = { dry_run: false, verbose: false, sample: nil }
725

726
OptionParser.new do |opts|
727
  opts.banner = 'Usage: ruby tools/dev/update_gitlab_versions.rb [options]'
728
  opts.separator ''
729
  opts.separator 'Fetches GitLab EE/CE tags from Docker Hub, streams only the'
730
  opts.separator 'application layer from the Docker Registry API (no Docker daemon'
731
  opts.separator 'required), and updates version.json directly.'
732
  opts.separator ''
733

734
  opts.on('-n', '--dry-run', 'Show what would be added without modifying any files') do
735
    options[:dry_run] = true
736
  end
737

738
  opts.on('-v', '--verbose', 'Print layer mediatypes and scan progress') do
739
    options[:verbose] = true
740
  end
741

742
  opts.on('-s', '--sample REPO:TAG',
743
          'Dump all .css filenames from all layers of REPO:TAG (e.g. gitlab/gitlab-ce:17.0.0-ce.0)') do |val|
744
    options[:sample] = val
745
  end
746

747
  opts.on('-h', '--help', 'Display this help') do
748
    puts opts
749
    exit
750
  end
751
end.parse!
752

753
# -- sample mode ---------------------------------------------------------------
754

755
if options[:sample]
756
  repo, tag = options[:sample].split(':', 2)
757
  abort 'Usage: --sample REPO:TAG  (e.g. gitlab/gitlab-ce:17.0.0-ce.0)'.red unless repo && tag
758
  sample_tag(repo, tag)
759
  exit
760
end
761

762
# -- main ----------------------------------------------------------------------
763

764
data = load_json_map
765
current_max = max_version_in_map(data)
766
abort 'Could not determine current max version from version.json'.red unless current_max
767

768
puts "Current max version in GITLAB_CSS_MAP: #{current_max.join('.')}".cyan
769

770
all_entries = EDITIONS.flat_map { |ed| process_edition(ed, current_max, options) }
771

772
update_version_file(all_entries, dry_run: options[:dry_run])
773

774