Real-time collaboration for Jupyter Notebooks, Linux Terminals, LaTeX, VS Code, R IDE, and more,
all in one place.

1
#!/usr/bin/env ruby
2
# -*- coding: binary -*-
3

4
#
5
# Check for data scripts to ensure they are up to date
6
#
7
# by h00die
8
#
9

10
require 'digest'
11
require 'open-uri'
12
require 'optparse'
13
require 'tempfile'
14

15
options = {}
16
optparse = OptionParser.new do |opts|
17
  opts.banner = 'Usage: check_external_scripts.rb [options]'
18
  opts.on('-u', '--update', 'Overwrite old scripts with newer ones.') do
19
    options[:update] = true
20
  end
21
  opts.on('-h', '--help', 'Display this screen.') do
22
    puts opts
23
    exit
24
  end
25
end
26
optparse.parse!
27

28
# colors and puts templates from msftidy.rb
29

30
class String
31
  def red
32
    "\e[1;31;40m#{self}\e[0m"
33
  end
34

35
  def yellow
36
    "\e[1;33;40m#{self}\e[0m"
37
  end
38

39
  def green
40
    "\e[1;32;40m#{self}\e[0m"
41
  end
42

43
  def cyan
44
    "\e[1;36;40m#{self}\e[0m"
45
  end
46
end
47

48
#
49
# Display an error message, given some text
50
#
51
def error(txt)
52
  puts "[#{'ERROR'.red}] #{cleanup_text(txt)}"
53
end
54

55
#
56
# Display a warning message, given some text
57
#
58
def warning(txt)
59
  puts "[#{'WARNING'.yellow}] #{cleanup_text(txt)}"
60
end
61

62
#
63
# Display a info message, given some text
64
#
65
def info(txt)
66
  puts "[#{'INFO'.cyan}] #{cleanup_text(txt)}"
67
end
68

69
def cleanup_text(txt)
70
  # remove line breaks
71
  txt = txt.gsub(/[\r\n]/, ' ')
72
  # replace multiple spaces by one space
73
  txt.gsub(/\s{2,}/, ' ')
74
end
75

76
def cleanup_sqlmap_decloak_dir
77
  unless system('rm -rf /tmp/sqlmap_decloak')
78
    error 'Could not remove existing /tmp/sqlmap_decloak directory'
79
  end
80
end
81

82
def clone_sqlmap_decloak
83
  cleanup_sqlmap_decloak_dir
84
  unless system('git clone -q --depth=1 https://github.com/sqlmapproject/sqlmap.git /tmp/sqlmap_decloak')
85
    error "Either 'git' is not installed, your internet is not connected, or /tmp/sqlmap_decloak could not be removed."
86
  end
87
end
88

89
# https://github.com/rapid7/metasploit-framework/pull/13191#issuecomment-626584689
90
def decloak(file)
91
  unless system("python /tmp/sqlmap_decloak/extra/cloak/cloak.py -d -i #{file.path} -o #{file.path}_decloak")
92
    unless system("python3 /tmp/sqlmap_decloak/extra/cloak/cloak.py -d -i #{file.path} -o #{file.path}_decloak")
93
      error "Either python is not installed, or the file at #{file.path} could not be found! Please double check your computer's setup and check that the #{file.path} file exists!"
94
    end
95
  end
96
  File.binread("#{file.path}_decloak")
97
end
98

99
#
100
#
101
#  Main
102
#
103
#
104

105
scripts = []
106

107
###
108
# Bloodhound/Sharphound files
109
###
110

111
# https://github.com/BloodHoundAD/BloodHound/commit/b6ab5cd369c70219c6376d9f5c4fcd63f34fb4a0
112
scripts << {
113
  name: 'Sharphound (Bloodhound) ps1',
114
  addr: 'https://raw.githubusercontent.com/BloodHoundAD/BloodHound/master/Collectors/SharpHound.ps1',
115
  dest: '/data/post/powershell/SharpHound.ps1',
116
  subs: [
117
    ["\t", '    '], # tabs to spaces
118
    [/\s+$/, ''] # trailing whitespace
119
  ]
120
}
121
scripts << {
122
  name: 'Sharphound (Bloodhound) exe',
123
  addr: 'https://raw.githubusercontent.com/BloodHoundAD/BloodHound/master/Collectors/SharpHound.exe',
124
  dest: '/data/post/SharpHound.exe',
125
  subs: []
126
}
127
###
128
# JTR files
129
###
130
scripts << {
131
  name: 'JTR - dumb16.conf',
132
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/dumb16.conf',
133
  dest: '/data/jtr/dumb16.conf',
134
  subs: []
135
}
136
scripts << {
137
  name: 'JTR - alnumspace.chr',
138
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/alnumspace.chr',
139
  dest: '/data/jtr/alnumspace.chr',
140
  subs: []
141
}
142
scripts << {
143
  name: 'JTR - regex_alphabets.conf',
144
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/regex_alphabets.conf',
145
  dest: '/data/jtr/regex_alphabets.conf',
146
  subs: []
147
}
148
scripts << {
149
  name: 'JTR - latin1.chr',
150
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/latin1.chr',
151
  dest: '/data/jtr/latin1.chr',
152
  subs: []
153
}
154
scripts << {
155
  name: 'JTR - lowerspace.chr',
156
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/lowerspace.chr',
157
  dest: '/data/jtr/lowerspace.chr',
158
  subs: []
159
}
160
scripts << {
161
  name: 'JTR - utf8.chr',
162
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/utf8.chr',
163
  dest: '/data/jtr/utf8.chr',
164
  subs: []
165
}
166
scripts << {
167
  name: 'JTR - john.conf',
168
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/john.conf',
169
  dest: '/data/jtr/john.conf',
170
  subs: []
171
}
172
scripts << {
173
  name: 'JTR - dumb32.conf',
174
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/dumb32.conf',
175
  dest: '/data/jtr/dumb32.conf',
176
  subs: []
177
}
178
scripts << {
179
  name: 'JTR - alpha.chr',
180
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/alpha.chr',
181
  dest: '/data/jtr/alpha.chr',
182
  subs: []
183
}
184
scripts << {
185
  name: 'JTR - dynamic.conf',
186
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/dynamic.conf',
187
  dest: '/data/jtr/dynamic.conf',
188
  subs: []
189
}
190
scripts << {
191
  name: 'JTR - repeats32.conf',
192
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/repeats32.conf',
193
  dest: '/data/jtr/repeats32.conf',
194
  subs: []
195
}
196
scripts << {
197
  name: 'JTR - lm_ascii.chr',
198
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/lm_ascii.chr',
199
  dest: '/data/jtr/lm_ascii.chr',
200
  subs: []
201
}
202
scripts << {
203
  name: 'JTR - upper.chr',
204
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/upper.chr',
205
  dest: '/data/jtr/upper.chr',
206
  subs: []
207
}
208
scripts << {
209
  name: 'JTR - lowernum.chr',
210
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/lowernum.chr',
211
  dest: '/data/jtr/lowernum.chr',
212
  subs: []
213
}
214
scripts << {
215
  name: 'JTR - ascii.chr',
216
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/ascii.chr',
217
  dest: '/data/jtr/ascii.chr',
218
  subs: []
219
}
220
scripts << {
221
  name: 'JTR - dynamic_disabled.conf',
222
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/dynamic_disabled.conf',
223
  dest: '/data/jtr/dynamic_disabled.conf',
224
  subs: []
225
}
226
scripts << {
227
  name: 'JTR - hybrid.conf',
228
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/hybrid.conf',
229
  dest: '/data/jtr/hybrid.conf',
230
  subs: []
231
}
232
scripts << {
233
  name: 'JTR - repeats16.conf',
234
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/repeats16.conf',
235
  dest: '/data/jtr/repeats16.conf',
236
  subs: []
237
}
238
scripts << {
239
  name: 'JTR - digits.chr',
240
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/digits.chr',
241
  dest: '/data/jtr/digits.chr',
242
  subs: []
243
}
244
scripts << {
245
  name: 'JTR - uppernum.chr',
246
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/uppernum.chr',
247
  dest: '/data/jtr/uppernum.chr',
248
  subs: []
249
}
250
scripts << {
251
  name: 'JTR - lanman.chr',
252
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/lanman.chr',
253
  dest: '/data/jtr/lanman.chr',
254
  subs: []
255
}
256
scripts << {
257
  name: 'JTR - dynamic_flat_sse_formats.conf',
258
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/dynamic_flat_sse_formats.conf',
259
  dest: '/data/jtr/dynamic_flat_sse_formats.conf',
260
  subs: []
261
}
262
scripts << {
263
  name: 'JTR - alnum.chr',
264
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/alnum.chr',
265
  dest: '/data/jtr/alnum.chr',
266
  subs: []
267
}
268
scripts << {
269
  name: 'JTR - lower.chr',
270
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/lower.chr',
271
  dest: '/data/jtr/lower.chr',
272
  subs: []
273
}
274
scripts << {
275
  name: 'JTR - korelogic.conf',
276
  addr: 'https://raw.githubusercontent.com/magnumripper/JohnTheRipper/bleeding-jumbo/run/korelogic.conf',
277
  dest: '/data/jtr/korelogic.conf',
278
  subs: []
279
}
280

281
###
282
# SQLMap UDF files
283
###
284
scripts << {
285
  name: 'SQLMap UDF - lib_mysqludf_sys_32.so',
286
  addr: 'https://github.com/sqlmapproject/sqlmap/blob/master/data/udf/mysql/linux/32/lib_mysqludf_sys.so_?raw=true',
287
  dest: '/data/exploits/mysql/lib_mysqludf_sys_32.so',
288
  subs: []
289
}
290
scripts << {
291
  name: 'SQLMap UDF - lib_mysqludf_sys_64.so',
292
  addr: 'https://github.com/sqlmapproject/sqlmap/blob/master/data/udf/mysql/linux/64/lib_mysqludf_sys.so_?raw=true',
293
  dest: '/data/exploits/mysql/lib_mysqludf_sys_64.so',
294
  subs: []
295
}
296
scripts << {
297
  name: 'SQLMap UDF - lib_mysqludf_sys_32.dll',
298
  addr: 'https://github.com/sqlmapproject/sqlmap/blob/master/data/udf/mysql/windows/32/lib_mysqludf_sys.dll_?raw=true',
299
  dest: '/data/exploits/mysql/lib_mysqludf_sys_32.dll',
300
  subs: []
301
}
302
scripts << {
303
  name: 'SQLMap UDF - lib_mysqludf_sys_64.dll',
304
  addr: 'https://github.com/sqlmapproject/sqlmap/blob/master/data/udf/mysql/windows/64/lib_mysqludf_sys.dll_?raw=true',
305
  dest: '/data/exploits/mysql/lib_mysqludf_sys_64.dll',
306
  subs: []
307
}
308

309
###
310
# CMS Files
311
###
312

313
# https://github.com/rapid7/metasploit-framework/pull/11862#issuecomment-496578367
314
scripts << {
315
  name: 'WordPress - Plugins List',
316
  addr: 'https://plugins.svn.wordpress.org',
317
  dest: '/data/wordlists/wp-plugins.txt',
318
  subs: [
319
    [/^((?!  <li>).)*/, ''], # remove all non-plugin lines
320
    [/  <li><a href="[^"]+">/, ''], # remove beginning
321
    [/\/<\/a><\/li>/,''], # remove end
322
    [/^\s*/,''] # remove empty lines
323
  ]
324
}
325

326
scripts << {
327
  name: 'WordPress - Themes List',
328
  addr: 'https://themes.svn.wordpress.org',
329
  dest: '/data/wordlists/wp-themes.txt',
330
  subs: [
331
    [/^((?!  <li>).)*/, ''], # remove all non-plugin lines
332
    [/  <li><a href="[^"]+">/, ''], # remove beginning
333
    [/\/<\/a><\/li>/,''], # remove end
334
    [/^\s*/,''] # remove empty lines
335
  ]
336
}
337

338
# Joomla's is more complicated. It looks for more than
339
# just components.  Because of that, if you want the
340
# file updated, see:
341
# https://github.com/rapid7/metasploit-framework/pull/11199#issue-242415518
342
# python3 tools/dev/update_joomla_components.py
343

344
path = File.expand_path('../../', File.dirname(__FILE__))
345

346
clone_sqlmap_decloak
347

348
scripts.each do |script|
349
  puts "Downloading: #{script[:name]}"
350
  begin
351
    old_content = File.binread(path + script[:dest])
352
    old_hash = Digest::SHA1.hexdigest old_content
353
    info "Old Hash: #{old_hash}"
354

355
    new_content = URI.open(script[:addr]).read
356
    if script.key?(:subs)
357
      script[:subs].each do |sub|
358
        new_content.gsub!(sub[0], sub[1])
359
      end
360
    end
361

362
    if script[:name].start_with?('SQLMap UDF')
363
      info('Performing decloaking')
364
      f = Tempfile.new('sqlmap_udf')
365
      f.write(new_content)
366
      f.close
367
      new_content = decloak(f)
368
    end
369

370
    new_hash = Digest::SHA1.hexdigest new_content
371
    info "New Hash: #{new_hash}"
372

373
    unless old_hash == new_hash
374
      warning '  New version identified!'
375
      if options[:update] == true
376
        warning "    Updating MSF copy of #{script[:dest]}"
377
        File.binwrite(path + script[:dest], new_content)
378
      end
379
    end
380
  rescue OpenURI::HTTPError
381
    error "Unable to download, check URL: #{script[:addr]}"
382
  rescue Errno::ENOENT
383
    error "Destination not found, check path: #{path + script[:dest]}"
384
  end
385
end
386

387
cleanup_sqlmap_decloak_dir
388

389