CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!

1
# -*- coding: binary -*-
2
require "rex/parser/nokogiri_doc_mixin"
3
require 'rex'
4
require 'uri'
5

6
module Rex
7
  module Parser
8

9
    # If Nokogiri is available, define the Acunetix document class.
10
    load_nokogiri && class AcunetixDocument < Nokogiri::XML::SAX::Document
11

12
    include NokogiriDocMixin
13

14
    # The resolver prefers your local /etc/hosts (or windows equiv), but will
15
    # fall back to regular DNS. It retains a cache for the import to avoid
16
    # spamming your network with DNS requests.
17
    attr_reader :resolv_cache
18

19
    # If name resolution of the host fails out completely, you will not be
20
    # able to import that Scan task. Other scan tasks in the same report
21
    # should be unaffected.
22
    attr_reader :parse_warnings
23

24
    def start_document
25
      @parse_warnings = []
26
      @resolv_cache = {}
27
      @host_object = nil
28
    end
29

30
    def start_element(name=nil,attrs=[])
31
      attrs = normalize_attrs(attrs)
32
      block = @block
33
      @state[:current_tag][name] = true
34
      case name
35
      when "Scan" # Start of the thing.
36
        @state[:report_item] = {}
37
      when "Name", "StartURL", "StartTime", "Banner", "Os", "Text", "Severity", "CWE", "URL", "Parameter"
38
        @state[:has_text] = true
39
      when "LoginSequence" # Skipping for now
40
      when "ReportItem"
41
        @state[:report_item] = {}
42
      when "Crawler"
43
        record_crawler(attrs)
44
      when "FullURL"
45
        @state[:has_text] = true
46
      when "Variable"
47
        record_variable(attrs)
48
      when "Request", "Response"
49
        @state[:has_text] = true
50
      end
51
    end
52

53
    def end_element(name=nil)
54
      block = @block
55
      case name
56
      when "Scan"
57
        # Clears most of the @state out, we're done with this web site.
58
        @state.delete_if {|k| k != :current_tag}
59
      when "Name"
60
        @state[:has_text] = false
61
        collect_scan_name
62
        collect_report_item_name
63
        @text = nil
64
      when "StartURL" # Populates @state[:starturl_uri], we use this a lot
65
        @state[:has_text] = false
66
        # StartURL does not always include the scheme
67
        @text.prepend("http://") unless URI.parse(@text).scheme
68
        collect_host
69
        collect_service_from_url
70
        @text = nil
71
        handle_parse_warnings &block
72
        @host_object = report_host &block
73
        if @host_object
74
          report_starturl_service(&block)
75
          db.report_import_note(@args[:workspace],@host_object)
76
        end
77
      when "StartTime"
78
        @state[:has_text] = false
79
        @state[:timestamp] = @text.to_time
80
        @text = nil
81
      when "Text"
82
        @state[:has_text] = false
83
        service = collect_service_from_kbitem_text
84
        @text = nil
85
        return unless service
86
        handle_parse_warnings &block
87
        if @host_object
88
          report_kbitem_service(service,&block)
89
        end
90
      when "Severity"
91
        @state[:has_text] = false
92
        collect_report_item_severity
93
        @text = nil
94
      when "CWE"
95
        @state[:has_text] = false
96
        collect_report_item_cwe
97
        @text = nil
98
      when "URL"
99
        @state[:has_text] = false
100
        collect_report_item_reference_url
101
        @text = nil
102
      when "Parameter"
103
        @state[:has_text] = false
104
        collect_report_item_parameter
105
        @text = nil
106
      when "ReportItem"
107
        vuln = collect_vuln_from_report_item
108
        if vuln.nil?
109
          @state[:page_request] = @state[:page_response] = nil
110
          return
111
        end
112
        handle_parse_warnings &block
113
        if @state[:vuln_info][:refs].nil?
114
          report_web_vuln(&block)
115
        else
116
          report_other_vuln(&block)
117
        end
118
        @state[:page_request] = @state[:page_response] = nil
119
      when "Banner"
120
        @state[:has_text] = false
121
        collect_and_report_banner
122
      when "Os"
123
        @state[:has_text] = false
124
        report_os_fingerprint
125
      when "LoginSequence" # This comes up later in the report anyway
126
      when "Crawler"
127
        report_starturl_web_site(&block)
128
      when "FullURL"
129
        @state[:has_text] = false
130
        report_web_site(@text,&block)
131
        @text = nil
132
      when "Inputs"
133
        report_web_form(&block)
134
      when "Request"
135
        @state[:has_text] = false
136
        collect_page_request
137
        @text = nil
138
      when "Response"
139
        @state[:has_text] = false
140
        collect_page_response
141
        @text = nil
142
        report_web_page(&block)
143
      end
144
      @state[:current_tag].delete name
145
    end
146

147
    def collect_page_response
148
      return unless in_tag("TechnicalDetails")
149
      return unless in_tag("ReportItem")
150
      return unless @text
151
      return if @text.to_s.empty?
152
      @state[:page_response] = @text
153
    end
154

155
    def collect_page_request
156
      return unless in_tag("TechnicalDetails")
157
      return unless in_tag("ReportItem")
158
      return unless @text
159
      return if @text.to_s.empty?
160
      @state[:page_request] = @text
161
    end
162

163
    def collect_scan_name
164
      return unless in_tag("Scan")
165
      return if in_tag("ReportItems")
166
      return if in_tag("Crawler")
167
      return unless @text
168
      return if @text.strip.empty?
169
      @state[:scan_name] = @text.strip
170
    end
171

172
    def collect_host
173
      return unless in_tag("Scan")
174
      return unless @text
175
      return if @text.strip.empty?
176
      uri = URI.parse(@text) rescue nil
177
      return unless uri
178
      address = resolve_scan_starturl_address(uri)
179
      @report_data[:host] = address
180
      @report_data[:state] = Msf::HostState::Alive
181
    end
182

183
    def collect_service_from_url
184
      return unless @report_data[:host]
185
      return unless in_tag("Scan")
186
      return unless @text
187
      return if @text.strip.empty?
188
      uri = URI.parse(@text) rescue nil
189
      return unless uri
190
      @state[:starturl_uri] = uri
191
      @report_data[:ports] ||= []
192
      @report_data[:ports] << @state[:starturl_port]
193
    end
194

195
    def collect_service_from_kbitem_text
196
      return unless @host_object
197
      return unless in_tag("Scan")
198
      return unless in_tag("KBase")
199
      return unless in_tag("KBItem")
200
      return unless @text
201
      return if @text.strip.empty?
202
      return unless @text =~ /server is running/
203
      matched = / (?<name>\w+) server is running on (?<proto>\w+) port (?<portnum>\d+)\./.match(@text)
204
      @report_data[:ports] ||= []
205
      @report_data[:ports] << matched[:portnum]
206
      return matched
207
    end
208

209
    def collect_vuln_from_report_item
210
      @state[:vuln_info] = nil
211
      return unless @host_object
212
      return unless in_tag("Scan")
213
      return unless in_tag("ReportItems")
214
      return unless in_tag("ReportItem")
215
      return unless @state[:report_item][:name]
216
      return unless @state[:report_item][:severity]
217
      return unless @state[:report_item][:severity].downcase == "high"
218

219
      @state[:vuln_info] = {}
220
      @state[:vuln_info][:name] = @state[:report_item][:name]
221
      if @state[:page_request_verb].nil? && @state[:report_item][:name] =~ /deprecated/
222
        # Treating this as a regular vuln, not web-specific
223
        @state[:vuln_info][:refs] = ["ACX-#{@state[:report_item][:reference_url]}"]
224
        unless @state[:report_item_cwe].nil?
225
          @state[:vuln_info][:refs][0] << ",#{@state[:report_item][:cwe]}"
226
        end
227
      end
228
      @state[:vuln_info][:severity] = @state[:report_item][:severity].downcase
229
      @state[:vuln_info][:cwe] = @state[:report_item][:cwe]
230
      return @state[:vuln_info]
231
    end
232

233
    def collect_and_report_banner
234
      return unless (svc = @state[:starturl_service_object]) # Yes i want assignment
235
      return unless @text
236
      return if @text.strip.empty?
237
      return unless in_tag("Scan")
238
      svc_info = {
239
        :host => svc.host,
240
        :port => svc.port,
241
        :proto => svc.proto,
242
        :info => @text.strip
243
      }
244
      db_report(:service, svc_info)
245
      @text = nil
246
    end
247

248
    def collect_report_item_name
249
      return unless in_tag("ReportItem")
250
      return unless @text
251
      return if @text.strip.empty?
252
      @state[:report_item][:name] = @text
253
    end
254

255
    def collect_report_item_severity
256
      return unless in_tag("ReportItem")
257
      return unless @text
258
      return if @text.strip.empty?
259
      @state[:report_item][:severity] = @text
260
    end
261

262
    def collect_report_item_cwe
263
      return unless in_tag("ReportItem")
264
      return unless @text
265
      return if @text.strip.empty?
266
      @state[:report_item][:cwe] = @text
267
    end
268

269
    def collect_report_item_reference_url
270
      return unless in_tag("ReportItem")
271
      return unless in_tag("References")
272
      return unless in_tag("Reference")
273
      return unless @text
274
      return if @text.strip.empty?
275
      @state[:report_item][:reference_url] = @text
276
    end
277

278
    def collect_report_item_parameter
279
      return unless in_tag("ReportItem")
280
      return unless @text
281
      return if @text.strip.empty?
282
      @state[:report_item][:parameter] = @text
283
    end
284

285
    # @state[:fullurl] is set by report_web_site
286
    def record_variable(attrs)
287
      return unless in_tag("Inputs")
288
      return unless @state[:fullurl].kind_of? URI
289
      method = attr_hash(attrs)["Type"]
290
      return unless method
291
      return if method.strip.empty?
292
      @state[:form_variables] ||= []
293
      @state[:form_variables] << [attr_hash(attrs)["Name"],method]
294
    end
295

296
    def record_crawler(attrs)
297
      return unless in_tag("Scan")
298
      return unless @state[:starturl_service_object]
299
      starturl = attr_hash(attrs)["StartUrl"]
300
      return unless starturl
301
      @state[:crawler_starturl] = starturl
302
    end
303

304
    def report_web_form(&block)
305
      return unless in_tag("SiteFiles")
306
      return unless @state[:web_site]
307
      return unless @state[:fullurl].kind_of? URI
308
      return unless @state[:form_variables].kind_of? Array
309
      return if @state[:form_variables].empty?
310
      method = parse_method(@state[:form_variables].first[1])
311
      vars = @state[:form_variables].map {|x| x[0]}
312
      form_info = {}
313
      form_info[:web_site] = @state[:web_site]
314
      form_info[:path] = @state[:fullurl].path
315
      form_info[:query] = @state[:fullurl].query
316
      form_info[:method] = method
317
      form_info[:params] = vars
318
      url = @state[:fullurl].to_s
319
      db.emit(:web_form,url,&block) if block
320
      db_report(:web_form,form_info)
321
      @state[:fullurl] = nil
322
      @state[:form_variables] = nil
323
    end
324

325
    def report_web_page(&block)
326
      return if should_skip_this_page
327
      return unless @state[:web_site]
328
      @state[:page_request_verb] = nil
329
      return unless @state[:page_request]
330
      return if @state[:page_request].strip.empty?
331
      verb,path,query_string = parse_request(@state[:page_request])
332
      return unless path
333
      @state[:page_request_verb] = verb
334
      web_page_info = {}
335
      if @state[:page_response].strip.blank?
336
        web_page_info[:code] = ""
337
        web_page_info[:headers] = {}
338
        web_page_info[:body] = ""
339
      else
340
        parsed_response = parse_response(@state[:page_response])
341
        return unless parsed_response
342
        web_page_info[:code] = parsed_response[:code].to_i
343
        web_page_info[:headers] = parsed_response[:headers]
344
        web_page_info[:body] = parsed_response[:body]
345
      end
346
      web_page_info[:web_site] = @state[:web_site]
347
      web_page_info[:path] = path
348
      web_page_info[:query] = query_string || ""
349
      url = ""
350
      url << @state[:web_site].service.name.to_s << "://"
351
      url << @state[:web_site].vhost.to_s << ":"
352
      url << path
353
      uri = URI.parse(url) rescue nil
354
      return unless uri # Sanity checker
355
      db.emit(:web_page, url, &block) if block
356
      web_page_object = db_report(:web_page,web_page_info)
357
      @state[:web_page] = web_page_object
358
    end
359

360
    def report_web_vuln(&block)
361
      return if should_skip_this_page
362
      return unless @state[:web_page]
363
      return unless @state[:web_site]
364
      return unless @state[:vuln_info]
365

366
      web_vuln_info = {}
367
      web_vuln_info[:web_site] = @state[:web_site]
368
      web_vuln_info[:path] = @state[:web_page][:path]
369
      web_vuln_info[:query] = @state[:web_page][:query]
370
      web_vuln_info[:method] = @state[:page_request_verb]
371
      web_vuln_info[:pname] = ""
372
      if @state[:page_response].blank?
373
        web_vuln_info[:proof] = "<empty response>"
374
      else
375
        web_vuln_info[:proof] = @state[:page_response]
376
      end
377
      web_vuln_info[:risk] = 5
378
      web_vuln_info[:params] = []
379
      unless @state[:report_item][:parameter].blank?
380
        # Acunetix only lists a single parameter...
381
        web_vuln_info[:params] << [ @state[:report_item][:parameter].to_s, "" ]
382
      end
383
      web_vuln_info[:category] = "imported"
384
      web_vuln_info[:confidence] = 100
385
      web_vuln_info[:name] = @state[:vuln_info][:name]
386

387
      db.emit(:web_vuln, web_vuln_info[:name], &block) if block
388
      vuln = db_report(:web_vuln, web_vuln_info)
389
    end
390

391
    def report_other_vuln(&block)
392
      return if should_skip_this_page
393
      return unless @state[:vuln_info]
394

395
      db.emit(:vuln, @state[:vuln_info][:name], &block) if block
396
      db_report(:vuln, @state[:vuln_info].merge(:host => @host_object))
397
    end
398

399
    # Reasons why we shouldn't collect a particular web page.
400
    def should_skip_this_page
401
      if @state[:report_item][:name] =~ /Unrestricted File Upload/
402
        # This means that the page being collected is something the
403
        # auditor put there, so it's not useful to report on.
404
        return true
405
      end
406
      return false
407
    end
408

409
    # XXX Rex::Proto::Http::Packet seems broken for
410
    # actually parsing requests and responses, but all I
411
    # need are the headers anyway
412
    def parse_request(request)
413
      headers = Rex::Proto::Http::Packet::Header.new
414
      headers.from_s request.dup # It's destructive.
415
      return unless headers.cmd_string
416
      verb,req = headers.cmd_string.split(/\s+/)
417
      return unless verb
418
      return unless req
419
      path,query_string = req.split(/\?/)[0,2]
420
      return verb,path,query_string
421
    end
422

423
    def parse_response(response)
424
      headers = Rex::Proto::Http::Packet::Header.new
425
      headers.from_s response.dup # It's destructive.
426
      return unless headers.cmd_string
427
      http,code,msg = headers.cmd_string.split(/\s+/)
428
      return unless code
429
      return unless code.to_i.to_s == code
430
      parsed = {}
431
      parsed[:code] = code
432
      parsed[:headers] = {}
433
      headers.each do |k,v|
434
        parsed[:headers][k.to_s.downcase] = []
435
        parsed[:headers][k.to_s.downcase] << v
436
      end
437
      parsed[:body] = "" # We never seem to get this from Acunetix
438
      parsed
439
    end
440

441
    # Don't cause the web report to die just because we can't tell
442
    # what method was used -- default to GET. Sometimes it's just "POST," and
443
    # sometimes it's "URL encoded POST," and sometimes it might be something
444
    # else.
445
    def parse_method(meth)
446
      verbs = "(GET|POST|PATH)"
447
      real_method = meth.match(/^\s*#{verbs}/)
448
      real_method ||= meth.match(/\s*#{verbs}\s*$/)
449
      ( real_method && real_method[1] ) ? real_method[1] : "GET"
450
    end
451

452
    def report_host(&block)
453
      return unless @report_data[:host]
454
      return unless in_tag("Scan")
455
      if host_is_okay
456
        db.emit(:address,@report_data[:host],&block) if block
457
        host_info = @report_data.merge(:workspace => @args[:workspace])
458
        db_report(:host,host_info)
459
      end
460
    end
461

462
    # The service is super important, so we hang on to it for the
463
    # rest of the scan.
464
    def report_starturl_service(&block)
465
      return unless @host_object
466
      return unless @state[:starturl_uri]
467
      name = @state[:starturl_uri].scheme
468
      port = @state[:starturl_uri].port
469
      addr = @host_object.address
470
      svc = {
471
        :host => @host_object,
472
        :port => port,
473
        :name => name.dup,
474
        :proto => "tcp"
475
      }
476
      if name and port
477
        db.emit(:service,[addr,port].join(":"),&block) if block
478
        @state[:starturl_service_object] = db_report(:service,svc)
479
      end
480
    end
481

482
    def report_kbitem_service(service,&block)
483
      return unless @host_object
484
      return unless @state[:starturl_uri]
485
      addr = @host_object.address
486
      svc = {
487
        :host => @host_object,
488
        :port => service[:portnum].to_i,
489
        :name => service[:name].dup.downcase,
490
        :proto => service[:proto].dup.downcase
491
      }
492
      if service[:name] and service[:portnum]
493
        db.emit(:service,[addr,service[:portnum]].join(":"),&block) if block
494
        db_report(:service,svc)
495
      end
496
    end
497

498
    def report_web_site(url,&block)
499
      return unless in_tag("Crawler")
500
      return unless url
501
      return if url.strip.empty?
502
      uri = URI.parse(url) rescue nil
503
      return unless uri
504
      host = uri.host
505
      port = uri.port
506
      scheme = uri.scheme
507
      return unless scheme[/^https?/]
508
      return unless (host && port && scheme)
509
      address = resolve_address(host)
510
      return unless address
511
      # If we didn't create the service, we don't care about the site
512
      service_object = db.services(:workspace => @args[:workspace],
513
                                   :hosts => {address: address},
514
                                   :proto => 'tcp',
515
                                   :port => port).first
516
      return unless service_object
517
      web_site_info = {
518
        :workspace => @args[:workspace],
519
        :service => service_object,
520
        :vhost => host,
521
        :ssl => (scheme == "https")
522
      }
523
      @state[:web_site] = db_report(:web_site,web_site_info)
524
      @state[:fullurl] = uri
525
    end
526

527
    def report_starturl_web_site(&block)
528
      return unless @state[:crawler_starturl]
529
      starturl = @state[:crawler_starturl].dup
530
      report_web_site(starturl,&block)
531
    end
532

533
    def report_os_fingerprint
534
      return unless @state[:starturl_service_object]
535
      return unless @text
536
      return if @text.strip.empty?
537
      return unless in_tag("Scan")
538
      host = @state[:starturl_service_object].host
539
      fp_note = {
540
        :workspace => host.workspace,
541
        :host => host,
542
        :type => 'host.os.acunetix_fingerprint',
543
        :data => {:os => @text}
544
      }
545
      db_report(:note, fp_note)
546
      @text = nil
547
    end
548

549
    def resolve_port(uri)
550
      @state[:port] = uri.port
551
      unless @state[:port]
552
        @parse_warnings << "Could not determine a port for '#{@state[:scan_name]}'"
553
      end
554
      @state[:port] = uri.port
555
    end
556

557
    def resolve_address(host)
558
      return @resolv_cache[host] if @resolv_cache[host]
559
      address = Rex::Socket.resolv_to_dotted(host) rescue nil
560
      @resolv_cache[host] = address
561
      return address
562
    end
563

564
    def resolve_scan_starturl_address(uri)
565
      if uri.host
566
        address = resolve_address(uri.host)
567
        unless address
568
          @parse_warnings << "Could not resolve address for '#{uri.host}', skipping '#{@state[:scan_name]}'"
569
        end
570
      else
571
        @parse_warnings << "Could not determine a host for '#{@state[:scan_name]}'"
572
      end
573
      address
574
    end
575

576
    def handle_parse_warnings(&block)
577
      return if @parse_warnings.empty?
578
      @parse_warnings.each do |pwarn|
579
        db.emit(:warning, pwarn, &block) if block
580
      end
581
    end
582

583
  end
584
  end
585
end
586

587

588