1
# -*- coding: binary -*-
2
require "rex/parser/nokogiri_doc_mixin"
3
require 'rex'
4
require 'uri'
5

6
module Rex
7
  module Parser
8

9
    # If Nokogiri is available, define the Acunetix document class.
10
    load_nokogiri && class AcunetixDocument < Nokogiri::XML::SAX::Document
11

12
    include NokogiriDocMixin
13

14
    # The resolver prefers your local /etc/hosts (or windows equiv), but will
15
    # fall back to regular DNS. It retains a cache for the import to avoid
16
    # spamming your network with DNS requests.
17
    attr_reader :resolv_cache
18

19
    # If name resolution of the host fails out completely, you will not be
20
    # able to import that Scan task. Other scan tasks in the same report
21
    # should be unaffected.
22
    attr_reader :parse_warnings
23

24
    def start_document
25
      @parse_warnings = []
26
      @resolv_cache = {}
27
      @host_object = nil
28
    end
29

30
    def start_element(name=nil,attrs=[])
31
      attrs = normalize_attrs(attrs)
32
      block = @block
33
      @state[:current_tag][name] = true
34
      case name
35
      when "Scan" # Start of the thing.
36
        @state[:report_item] = {}
37
      when "Name", "StartURL", "StartTime", "Banner", "Os", "Text", "Severity", "CWE", "URL", "Parameter"
38
        @state[:has_text] = true
39
      when "LoginSequence" # Skipping for now
40
      when "ReportItem"
41
        @state[:report_item] = {}
42
      when "Crawler"
43
        record_crawler(attrs)
44
      when "FullURL"
45
        @state[:has_text] = true
46
      when "Variable"
47
        record_variable(attrs)
48
      when "Request", "Response"
49
        @state[:has_text] = true
50
      end
51
    end
52

53
    def end_element(name=nil)
54
      block = @block
55
      case name
56
      when "Scan"
57
        # Clears most of the @state out, we're done with this web site.
58
        @state.delete_if {|k| k != :current_tag}
59
      when "Name"
60
        @state[:has_text] = false
61
        collect_scan_name
62
        collect_report_item_name
63
        @text = nil
64
      when "StartURL" # Populates @state[:starturl_uri], we use this a lot
65
        @state[:has_text] = false
66
        # StartURL does not always include the scheme
67
        @text.prepend("http://") unless URI.parse(@text).scheme
68
        collect_host
69
        collect_service_from_url
70
        @text = nil
71
        handle_parse_warnings &block
72
        @host_object = report_host &block
73
        if @host_object
74
          report_starturl_service(&block)
75
          db.report_import_note(@args[:workspace],@host_object)
76
        end
77
      when "StartTime"
78
        @state[:has_text] = false
79
        @state[:timestamp] = @text.to_time
80
        @text = nil
81
      when "Text"
82
        @state[:has_text] = false
83
        service = collect_service_from_kbitem_text
84
        @text = nil
85
        return unless service
86
        handle_parse_warnings &block
87
        if @host_object
88
          report_kbitem_service(service,&block)
89
        end
90
      when "Severity"
91
        @state[:has_text] = false
92
        collect_report_item_severity
93
        @text = nil
94
      when "CWE"
95
        @state[:has_text] = false
96
        collect_report_item_cwe
97
        @text = nil
98
      when "URL"
99
        @state[:has_text] = false
100
        collect_report_item_reference_url
101
        @text = nil
102
      when "Parameter"
103
        @state[:has_text] = false
104
        collect_report_item_parameter
105
        @text = nil
106
      when "ReportItem"
107
        vuln = collect_vuln_from_report_item
108
        if vuln.nil?
109
          @state[:page_request] = @state[:page_response] = nil
110
          return
111
        end
112
        handle_parse_warnings &block
113
        if @state[:vuln_info][:refs].nil?
114
          report_web_vuln(&block)
115
        else
116
          report_other_vuln(&block)
117
        end
118
        @state[:page_request] = @state[:page_response] = nil
119
      when "Banner"
120
        @state[:has_text] = false
121
        collect_and_report_banner
122
      when "Os"
123
        @state[:has_text] = false
124
        report_os_fingerprint
125
      when "LoginSequence" # This comes up later in the report anyway
126
      when "Crawler"
127
        report_starturl_web_site(&block)
128
      when "FullURL"
129
        @state[:has_text] = false
130
        report_web_site(@text,&block)
131
        @text = nil
132
      when "Inputs"
133
        report_web_form(&block)
134
      when "Request"
135
        @state[:has_text] = false
136
        collect_page_request
137
        @text = nil
138
      when "Response"
139
        @state[:has_text] = false
140
        collect_page_response
141
        @text = nil
142
        report_web_page(&block)
143
      end
144
      @state[:current_tag].delete name
145
    end
146

147
    def collect_page_response
148
      return unless in_tag("TechnicalDetails")
149
      return unless in_tag("ReportItem")
150
      return unless @text
151
      return if @text.to_s.empty?
152
      @state[:page_response] = @text
153
    end
154

155
    def collect_page_request
156
      return unless in_tag("TechnicalDetails")
157
      return unless in_tag("ReportItem")
158
      return unless @text
159
      return if @text.to_s.empty?
160
      @state[:page_request] = @text
161
    end
162

163
    def collect_scan_name
164
      return unless in_tag("Scan")
165
      return if in_tag("ReportItems")
166
      return if in_tag("Crawler")
167
      return unless @text
168
      return if @text.strip.empty?
169
      @state[:scan_name] = @text.strip
170
    end
171

172
    def collect_host
173
      return unless in_tag("Scan")
174
      return unless @text
175
      return if @text.strip.empty?
176
      uri = URI.parse(@text) rescue nil
177
      return unless uri
178
      address = resolve_scan_starturl_address(uri)
179
      @report_data[:host] = address
180
      @report_data[:state] = Msf::HostState::Alive
181
    end
182

183
    def collect_service_from_url
184
      return unless @report_data[:host]
185
      return unless in_tag("Scan")
186
      return unless @text
187
      return if @text.strip.empty?
188
      uri = URI.parse(@text) rescue nil
189
      return unless uri
190
      @state[:starturl_uri] = uri
191
      @report_data[:ports] ||= []
192
      @report_data[:ports] << @state[:starturl_port]
193
    end
194

195
    def collect_service_from_kbitem_text
196
      return unless @host_object
197
      return unless in_tag("Scan")
198
      return unless in_tag("KBase")
199
      return unless in_tag("KBItem")
200
      return unless @text
201
      return if @text.strip.empty?
202
      return unless @text =~ /server is running/
203
      matched = / (?<name>\w+) server is running on (?<proto>\w+) port (?<portnum>\d+)\./.match(@text)
204
      @report_data[:ports] ||= []
205
      @report_data[:ports] << matched[:portnum]
206
      return matched
207
    end
208

209
    def collect_vuln_from_report_item
210
      @state[:vuln_info] = nil
211
      return unless @host_object
212
      return unless in_tag("Scan")
213
      return unless in_tag("ReportItems")
214
      return unless in_tag("ReportItem")
215
      return unless @state[:report_item][:name]
216
      return unless @state[:report_item][:severity]
217
      return unless @state[:report_item][:severity].downcase == "high"
218

219
      @state[:vuln_info] = {}
220
      @state[:vuln_info][:name] = @state[:report_item][:name]
221
      if @state[:page_request_verb].nil? && @state[:report_item][:name] =~ /deprecated/
222
        # Treating this as a regular vuln, not web-specific
223
        @state[:vuln_info][:refs] = ["ACX-#{@state[:report_item][:reference_url]}"]
224
        unless @state[:report_item_cwe].nil?
225
          @state[:vuln_info][:refs][0] << ",#{@state[:report_item][:cwe]}"
226
        end
227
      end
228
      @state[:vuln_info][:severity] = @state[:report_item][:severity].downcase
229
      @state[:vuln_info][:cwe] = @state[:report_item][:cwe]
230
      return @state[:vuln_info]
231
    end
232

233
    def collect_and_report_banner
234
      return unless (svc = @state[:starturl_service_object]) # Yes i want assignment
235
      return unless @text
236
      return if @text.strip.empty?
237
      return unless in_tag("Scan")
238
      svc_info = {
239
        :host => svc.host,
240
        :port => svc.port,
241
        :proto => svc.proto,
242
        :info => @text.strip
243
      }
244
      db_report(:service, svc_info)
245
      @text = nil
246
    end
247

248
    def collect_report_item_name
249
      return unless in_tag("ReportItem")
250
      return unless @text
251
      return if @text.strip.empty?
252
      @state[:report_item][:name] = @text
253
    end
254

255
    def collect_report_item_severity
256
      return unless in_tag("ReportItem")
257
      return unless @text
258
      return if @text.strip.empty?
259
      @state[:report_item][:severity] = @text
260
    end
261

262
    def collect_report_item_cwe
263
      return unless in_tag("ReportItem")
264
      return unless @text
265
      return if @text.strip.empty?
266
      @state[:report_item][:cwe] = @text
267
    end
268

269
    def collect_report_item_reference_url
270
      return unless in_tag("ReportItem")
271
      return unless in_tag("References")
272
      return unless in_tag("Reference")
273
      return unless @text
274
      return if @text.strip.empty?
275
      @state[:report_item][:reference_url] = @text
276
    end
277

278
    def collect_report_item_parameter
279
      return unless in_tag("ReportItem")
280
      return unless @text
281
      return if @text.strip.empty?
282
      @state[:report_item][:parameter] = @text
283
    end
284

285
    # @state[:fullurl] is set by report_web_site
286
    def record_variable(attrs)
287
      return unless in_tag("Inputs")
288
      return unless @state[:fullurl].kind_of? URI
289
      method = attr_hash(attrs)["Type"]
290
      return unless method
291
      return if method.strip.empty?
292
      @state[:form_variables] ||= []
293
      @state[:form_variables] << [attr_hash(attrs)["Name"],method]
294
    end
295

296
    def record_crawler(attrs)
297
      return unless in_tag("Scan")
298
      return unless @state[:starturl_service_object]
299
      starturl = attr_hash(attrs)["StartUrl"]
300
      return unless starturl
301
      @state[:crawler_starturl] = starturl
302
    end
303

304
    def report_web_form(&block)
305
      return unless in_tag("SiteFiles")
306
      return unless @state[:web_site]
307
      return unless @state[:fullurl].kind_of? URI
308
      return unless @state[:form_variables].kind_of? Array
309
      return if @state[:form_variables].empty?
310
      method = parse_method(@state[:form_variables].first[1])
311
      vars = @state[:form_variables].map {|x| x[0]}
312
      form_info = {}
313
      form_info[:web_site] = @state[:web_site]
314
      form_info[:path] = @state[:fullurl].path
315
      form_info[:query] = @state[:fullurl].query
316
      form_info[:method] = method
317
      form_info[:params] = vars
318
      url = @state[:fullurl].to_s
319
      db.emit(:web_form,url,&block) if block
320
      db_report(:web_form,form_info)
321
      @state[:fullurl] = nil
322
      @state[:form_variables] = nil
323
    end
324

325
    def report_web_page(&block)
326
      return if should_skip_this_page
327
      return unless @state[:web_site]
328
      @state[:page_request_verb] = nil
329
      return unless @state[:page_request]
330
      return if @state[:page_request].strip.empty?
331
      verb,path,query_string = parse_request(@state[:page_request])
332
      return unless path
333
      @state[:page_request_verb] = verb
334
      web_page_info = {}
335
      if @state[:page_response].strip.blank?
336
        web_page_info[:code] = ""
337
        web_page_info[:headers] = {}
338
        web_page_info[:body] = ""
339
      else
340
        parsed_response = parse_response(@state[:page_response])
341
        return unless parsed_response
342
        web_page_info[:code] = parsed_response[:code].to_i
343
        web_page_info[:headers] = parsed_response[:headers]
344
        web_page_info[:body] = parsed_response[:body]
345
      end
346
      web_page_info[:web_site] = @state[:web_site]
347
      web_page_info[:path] = path
348
      web_page_info[:query] = query_string || ""
349
      url = ""
350
      url << @state[:web_site].service.name.to_s << "://"
351
      url << @state[:web_site].vhost.to_s << ":"
352
      url << path
353
      uri = URI.parse(url) rescue nil
354
      return unless uri # Sanity checker
355
      db.emit(:web_page, url, &block) if block
356
      web_page_object = db_report(:web_page,web_page_info)
357
      @state[:web_page] = web_page_object
358
    end
359

360
    def report_web_vuln(&block)
361
      return if should_skip_this_page
362
      return unless @state[:web_site]
363
      return unless @state[:vuln_info]
364

365
      # If we have a web_page, report as a web vulnerability (detailed)
366
      if @state[:web_page]
367
        web_vuln_info = {}
368
        web_vuln_info[:web_site] = @state[:web_site]
369
        web_vuln_info[:path] = @state[:web_page][:path]
370
        web_vuln_info[:query] = @state[:web_page][:query]
371
        web_vuln_info[:method] = @state[:page_request_verb]
372
        web_vuln_info[:pname] = ""
373
        if @state[:page_response].blank?
374
          web_vuln_info[:proof] = "<empty response>"
375
        else
376
          web_vuln_info[:proof] = @state[:page_response]
377
        end
378
        web_vuln_info[:risk] = 5
379
        web_vuln_info[:params] = []
380
        unless @state[:report_item][:parameter].blank?
381
          # Acunetix only lists a single parameter...
382
          web_vuln_info[:params] << [ @state[:report_item][:parameter].to_s, "" ]
383
        end
384
        web_vuln_info[:category] = "imported"
385
        web_vuln_info[:confidence] = 100
386
        web_vuln_info[:name] = @state[:vuln_info][:name]
387

388
        db.emit(:web_vuln, web_vuln_info[:name], &block) if block
389
        vuln = db_report(:web_vuln, web_vuln_info)
390
      else
391
        # If web_page is not available, report as a regular vulnerability
392
        # This allows vulnerabilities to be imported even without complete request/response data
393
        report_other_vuln(&block)
394
      end
395
    end
396

397
    def report_other_vuln(&block)
398
      return if should_skip_this_page
399
      return unless @state[:vuln_info]
400

401
      db.emit(:vuln, @state[:vuln_info][:name], &block) if block
402
      db_report(:vuln, @state[:vuln_info].merge(:host => @host_object))
403
    end
404

405
    # Reasons why we shouldn't collect a particular web page.
406
    def should_skip_this_page
407
      if @state[:report_item][:name] =~ /Unrestricted File Upload/
408
        # This means that the page being collected is something the
409
        # auditor put there, so it's not useful to report on.
410
        return true
411
      end
412
      return false
413
    end
414

415
    # XXX Rex::Proto::Http::Packet seems broken for
416
    # actually parsing requests and responses, but all I
417
    # need are the headers anyway
418
    def parse_request(request)
419
      headers = Rex::Proto::Http::Packet::Header.new
420
      headers.from_s request.dup # It's destructive.
421
      return unless headers.cmd_string
422
      verb,req = headers.cmd_string.split(/\s+/)
423
      return unless verb
424
      return unless req
425
      path,query_string = req.split('?')[0,2]
426
      return verb,path,query_string
427
    end
428

429
    def parse_response(response)
430
      headers = Rex::Proto::Http::Packet::Header.new
431
      headers.from_s response.dup # It's destructive.
432
      return unless headers.cmd_string
433
      http,code,msg = headers.cmd_string.split(/\s+/)
434
      return unless code
435
      return unless code.to_i.to_s == code
436
      parsed = {}
437
      parsed[:code] = code
438
      parsed[:headers] = {}
439
      headers.each do |k,v|
440
        parsed[:headers][k.to_s.downcase] = []
441
        parsed[:headers][k.to_s.downcase] << v
442
      end
443
      parsed[:body] = "" # We never seem to get this from Acunetix
444
      parsed
445
    end
446

447
    # Don't cause the web report to die just because we can't tell
448
    # what method was used -- default to GET. Sometimes it's just "POST," and
449
    # sometimes it's "URL encoded POST," and sometimes it might be something
450
    # else.
451
    def parse_method(meth)
452
      verbs = "(GET|POST|PATH)"
453
      real_method = meth.match(/^\s*#{verbs}/)
454
      real_method ||= meth.match(/\s*#{verbs}\s*$/)
455
      ( real_method && real_method[1] ) ? real_method[1] : "GET"
456
    end
457

458
    def report_host(&block)
459
      return unless @report_data[:host]
460
      return unless in_tag("Scan")
461
      if host_is_okay
462
        db.emit(:address,@report_data[:host],&block) if block
463
        host_info = @report_data.merge(:workspace => @args[:workspace])
464
        db_report(:host,host_info)
465
      end
466
    end
467

468
    # The service is super important, so we hang on to it for the
469
    # rest of the scan.
470
    def report_starturl_service(&block)
471
      return unless @host_object
472
      return unless @state[:starturl_uri]
473
      name = @state[:starturl_uri].scheme
474
      port = @state[:starturl_uri].port
475
      addr = @host_object.address
476
      svc = {
477
        :host => @host_object,
478
        :port => port,
479
        :name => name.dup,
480
        :proto => "tcp"
481
      }
482
      if name and port
483
        db.emit(:service,[addr,port].join(":"),&block) if block
484
        @state[:starturl_service_object] = db_report(:service,svc)
485
      end
486
    end
487

488
    def report_kbitem_service(service,&block)
489
      return unless @host_object
490
      return unless @state[:starturl_uri]
491
      addr = @host_object.address
492
      svc = {
493
        :host => @host_object,
494
        :port => service[:portnum].to_i,
495
        :name => service[:name].dup.downcase,
496
        :proto => service[:proto].dup.downcase
497
      }
498
      if service[:name] and service[:portnum]
499
        db.emit(:service,[addr,service[:portnum]].join(":"),&block) if block
500
        db_report(:service,svc)
501
      end
502
    end
503

504
    def report_web_site(url,&block)
505
      return unless in_tag("Crawler")
506
      return unless url
507
      return if url.strip.empty?
508
      uri = URI.parse(url) rescue nil
509
      return unless uri
510
      host = uri.host
511
      port = uri.port
512
      scheme = uri.scheme
513
      return unless scheme[/^https?/]
514
      return unless (host && port && scheme)
515
      address = resolve_address(host)
516
      return unless address
517
      # If we didn't create the service, we don't care about the site
518
      service_object = db.services(:workspace => @args[:workspace],
519
                                   :hosts => {address: address},
520
                                   :proto => 'tcp',
521
                                   :port => port).first
522
      return unless service_object
523
      web_site_info = {
524
        :workspace => @args[:workspace],
525
        :service => service_object,
526
        :vhost => host,
527
        :ssl => (scheme == "https")
528
      }
529
      @state[:web_site] = db_report(:web_site,web_site_info)
530
      @state[:fullurl] = uri
531
    end
532

533
    def report_starturl_web_site(&block)
534
      return unless @state[:crawler_starturl]
535
      starturl = @state[:crawler_starturl].dup
536
      report_web_site(starturl,&block)
537
    end
538

539
    def report_os_fingerprint
540
      return unless @state[:starturl_service_object]
541
      return unless @text
542
      return if @text.strip.empty?
543
      return unless in_tag("Scan")
544
      host = @state[:starturl_service_object].host
545
      fp_note = {
546
        :workspace => host.workspace,
547
        :host => host,
548
        :type => 'host.os.acunetix_fingerprint',
549
        :data => {:os => @text}
550
      }
551
      db_report(:note, fp_note)
552
      @text = nil
553
    end
554

555
    def resolve_port(uri)
556
      @state[:port] = uri.port
557
      unless @state[:port]
558
        @parse_warnings << "Could not determine a port for '#{@state[:scan_name]}'"
559
      end
560
      @state[:port] = uri.port
561
    end
562

563
    def resolve_address(host)
564
      return @resolv_cache[host] if @resolv_cache[host]
565
      address = Rex::Socket.resolv_to_dotted(host) rescue nil
566
      @resolv_cache[host] = address
567
      return address
568
    end
569

570
    def resolve_scan_starturl_address(uri)
571
      if uri.host
572
        address = resolve_address(uri.host)
573
        unless address
574
          @parse_warnings << "Could not resolve address for '#{uri.host}', skipping '#{@state[:scan_name]}'"
575
        end
576
      else
577
        @parse_warnings << "Could not determine a host for '#{@state[:scan_name]}'"
578
      end
579
      address
580
    end
581

582
    def handle_parse_warnings(&block)
583
      return if @parse_warnings.empty?
584
      @parse_warnings.each do |pwarn|
585
        db.emit(:warning, pwarn, &block) if block
586
      end
587
    end
588

589
  end
590
  end
591
end
592

593