CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!

1
# -*- coding: binary -*-
2
require "rex/parser/nokogiri_doc_mixin"
3

4
module Rex
5
  module Parser
6

7
    # If Nokogiri is available, define AppScan document class.
8
    load_nokogiri && class AppscanDocument < Nokogiri::XML::SAX::Document
9

10
    include NokogiriDocMixin
11

12
    # The resolver prefers your local /etc/hosts (or windows equiv), but will
13
    # fall back to regular DNS. It retains a cache for the import to avoid
14
    # spamming your network with DNS requests.
15
    attr_reader :resolv_cache
16

17
    # If name resolution of the host fails out completely, you will not be
18
    # able to import that Scan task. Other scan tasks in the same report
19
    # should be unaffected.
20
    attr_reader :parse_warning
21

22
    def start_document
23
      @parse_warnings = []
24
      @resolv_cache = {}
25
    end
26

27
    def start_element(name=nil,attrs=[])
28
      attrs = normalize_attrs(attrs)
29
      block = @block
30
      @state[:current_tag][name] = true
31
      case name
32
      when "Issue" # Start of the stuff we want
33
        collect_issue(attrs)
34
      when "Entity" # Start of the stuff we want
35
        collect_entity(attrs)
36
      when "Severity", "Url", "OriginalHttpTraffic"
37
        @state[:has_text] = true
38
      end
39
    end
40

41
    def end_element(name=nil)
42
      block = @block
43
      case name
44
      when "Issue" # Wrap it up
45
        record_issue
46
        # Reset the state once we close an issue
47
        @state = @state.select do
48
          |k| [:current_tag, :web_sites].include? k
49
        end
50
      when "Url" # Populates @state[:web_site]
51
        @state[:has_text] = false
52
        record_url
53
        @text = nil
54
        report_web_site(&block)
55
        handle_parse_warnings(&block)
56
      when "Severity"
57
        @state[:has_text] = false
58
        record_risk
59
        @text = nil
60
      when "OriginalHttpTraffic" # Request and response
61
        @state[:has_text] = false
62
        record_request_and_response
63
        report_service_info
64
        page_info = report_web_page(&block)
65
        if page_info
66
          form_info = report_web_form(page_info,&block)
67
          if form_info
68
            report_web_vuln(form_info,&block)
69
          end
70
        end
71
        @text = nil
72
      end
73
      @state[:current_tag].delete name
74
    end
75

76
    def report_web_vuln(form_info,&block)
77
      return unless(in_issue && has_text)
78
      return unless form_info.kind_of? Hash
79
      return unless @state[:issue]
80
      return unless @state[:issue]["Noise"]
81
      return unless @state[:issue]["Noise"].to_s.downcase == "false"
82
      return unless @state[:issue][:vuln_param]
83
      web_vuln_info = {}
84
      web_vuln_info[:web_site] = form_info[:web_site]
85
      web_vuln_info[:path] = form_info[:path]
86
      web_vuln_info[:query] = form_info[:query]
87
      web_vuln_info[:method] = form_info[:method]
88
      web_vuln_info[:params] = form_info[:params]
89
      web_vuln_info[:pname] = @state[:issue][:vuln_param]
90
      web_vuln_info[:proof] = "unknown" # TODO: pick this up from <Difference> maybe?
91
      web_vuln_info[:risk] = @state[:issue][:risk]
92
      web_vuln_info[:name] = @state[:issue]["IssueTypeID"]
93
      web_vuln_info[:category] = "imported"
94
      web_vuln_info[:confidence] = 100 # Seems pretty binary, noise or not
95
      db.emit(:web_vuln, web_vuln_info[:name], &block) if block
96
      web_vuln = db_report(:web_vuln, web_vuln_info)
97
    end
98

99
    def collect_entity(attrs)
100
      return unless in_issue
101
      return unless @state[:issue].kind_of? Hash
102
      ent_hash = attr_hash(attrs)
103
      return unless ent_hash
104
      return unless ent_hash["Type"].to_s.downcase == "parameter"
105
      @state[:issue][:vuln_param] = ent_hash["Name"]
106
    end
107

108
    def report_web_form(page_info,&block)
109
      return unless(in_issue && has_text)
110
      return unless page_info.kind_of? Hash
111
      return unless @state[:request_body]
112
      return if @state[:request_body].strip.empty?
113
      web_form_info = {}
114
      web_form_info[:web_site] = page_info[:web_site]
115
      web_form_info[:path] = page_info[:path]
116
      web_form_info[:query] = page_info[:query]
117
      web_form_info[:method] = @state[:request_headers].cmd_string.split(/\s+/)[0]
118
      parsed_params = parse_params(@state[:request_body])
119
      return unless parsed_params
120
      return if parsed_params.empty?
121
      web_form_info[:params] = parsed_params
122
      web_form = db_report(:web_form, web_form_info)
123
      @state[:web_forms] ||= []
124
      unless @state[:web_forms].include? web_form
125
        db.emit(:web_form, @state[:uri].to_s, &block) if block
126
        @state[:web_forms] << web_form
127
      end
128
      web_form_info
129
    end
130

131
    def parse_params(request_body)
132
      return unless request_body
133
      pairs = request_body.split(/&/)
134
      params = []
135
      pairs.each do |pair|
136
        param,value = pair.split("=",2)
137
        params << [param,""] # Can't tell what's default
138
      end
139
      params
140
    end
141

142
    def report_web_page(&block)
143
      return unless(in_issue && has_text)
144
      return unless @state[:web_site].present?
145
      return unless @state[:response_headers].present?
146
      return unless @state[:uri].present?
147
      web_page_info = {}
148
      web_page_info[:web_site] = @state[:web_site]
149
      web_page_info[:path] = @state[:uri].path
150
      web_page_info[:body] = @state[:response_body].to_s
151
      web_page_info[:query] = @state[:uri].query
152
      code = @state[:response_headers].cmd_string.split(/\s+/)[1]
153
      return unless code
154
      web_page_info[:code] = code
155
      parsed_headers = {}
156
      @state[:response_headers].each do |k,v|
157
        parsed_headers[k.to_s.downcase] ||= []
158
        parsed_headers[k.to_s.downcase] << v
159
      end
160
      return if parsed_headers.empty?
161
      web_page_info[:headers] = parsed_headers
162
      web_page = db_report(:web_page, web_page_info)
163
      @state[:web_pages] ||= []
164
      unless @state[:web_pages].include? web_page
165
        db.emit(:web_page, @state[:uri].to_s, &block) if block
166
        @state[:web_pages] << web_page
167
      end
168
      web_page_info
169
    end
170

171
    def report_service_info
172
      return unless(in_issue && has_text)
173
      return unless @state[:web_site]
174
      return unless @state[:response_headers]
175
      banner = @state[:response_headers]["server"]
176
      return unless banner
177
      service = @state[:web_site].service
178
      return unless service.info.to_s.empty?
179
      service_info = {
180
        :host => service.host,
181
        :port => service.port,
182
        :proto => service.proto,
183
        :info => banner
184
      }
185
      db_report(:service, service_info)
186
    end
187

188
    def record_request_and_response
189
      return unless(in_issue && has_text)
190
      return unless @state[:web_site].present?
191
      really_original_traffic = unindent_and_crlf(@text)
192
      request_headers, request_body, response_headers, response_body = really_original_traffic.split(/\r\n\r\n/)
193
      return unless(request_headers && response_headers)
194
      req_header = Rex::Proto::Http::Packet::Header.new
195
      res_header = Rex::Proto::Http::Packet::Header.new
196
      req_header.from_s request_headers.lstrip
197
      res_header.from_s response_headers.lstrip
198
      if response_body.to_s.empty?
199
        response_body = ''
200
      end
201
      @state[:request_headers] = req_header
202
      @state[:request_body] = request_body.lstrip
203
      @state[:response_headers] = res_header
204
      @state[:response_body] = response_body.lstrip
205
    end
206

207
    # Appscan tab-indents which makes parsing a little difficult. They
208
    # also don't record CRLFs, just LFs.
209
    def unindent_and_crlf(text)
210
      second_line = text.split(/\r*\n/)[1]
211
      indent_level = second_line.size - second_line.lstrip.size
212
      unindented_text_lines = []
213
      text.split(/\r*\n/).each do |line|
214
        if line =~ /^\t{#{indent_level}}/
215
          unindented_line = line[indent_level,line.size]
216
          unindented_text_lines << unindented_line
217
        else
218
          unindented_text_lines << line
219
        end
220
      end
221
      unindented_text_lines.join("\r\n")
222
    end
223

224
    def record_risk
225
      return unless(in_issue && has_text)
226
      @state[:issue] ||= {}
227
      @state[:issue][:risk] = map_severity_to_risk
228
    end
229

230
    def map_severity_to_risk
231
      case @text.to_s.downcase
232
      when "high"   ; 5
233
      when "medium" ; 3
234
      when "low"    ; 1
235
      else          ; 0
236
      end
237
    end
238

239
    # TODO
240
    def record_issue
241
      return unless in_issue
242
      return unless @report_data[:issue].kind_of? Hash
243
      return unless @state[:web_site]
244
      return if @state[:issue]["Noise"].to_s.downcase == "true"
245
    end
246

247
    def collect_issue(attrs)
248
      return unless in_issue
249
      @state[:issue] = {}
250
      @state[:issue].merge! attr_hash(attrs)
251
    end
252

253
    def report_web_site(&block)
254
      return unless @state[:uri]
255
      uri = @state[:uri]
256
      hostname = uri.host # Assume the first one is the real hostname
257
      address = resolve_issue_url_address(uri)
258
      return unless address
259
      unless @resolv_cache.values.include? address
260
        db.emit(:address, address, &block) if block
261
      end
262
      port = resolve_port(uri)
263
      return unless port
264
      scheme = uri.scheme
265
      return unless scheme
266
      web_site_info = {:workspace => @args[:workspace]}
267
      web_site_info[:vhost] = hostname
268
      service_obj = check_for_existing_service(address,port)
269
      if service_obj
270
        web_site_info[:service] = service_obj
271
      else
272
        web_site_info[:host] = address
273
        web_site_info[:port] = port
274
        web_site_info[:ssl] = scheme == "https"
275
      end
276
      web_site_obj = db_report(:web_site, web_site_info)
277
      @state[:web_sites] ||= []
278
      unless @state[:web_sites].include? web_site_obj
279
        url = "#{uri.scheme}://#{uri.host}:#{uri.port}"
280
        db.emit(:web_site, url, &block) if block
281
        db.report_import_note(@args[:workspace], web_site_obj.service.host)
282
        @state[:web_sites] << web_site_obj
283
      end
284
      @state[:service] = service_obj || web_site_obj.service
285
      @state[:host] = (service_obj || web_site_obj.service).host
286
      @state[:web_site] = web_site_obj
287
    end
288

289
    def check_for_existing_service(address,port)
290
      # only one result can be returned, as the +port+ field restricts potential results to a single service
291
      db.services(:workspace => @args[:workspace],
292
                  :hosts => {address: address},
293
                  :proto => 'tcp',
294
                  :port => port).first
295
    end
296

297
    def resolve_port(uri)
298
      @state[:port] = uri.port
299
      unless @state[:port]
300
        @parse_warnings << "Could not determine a port for '#{@state[:scan_name]}'"
301
      end
302
      return @state[:port]
303
    end
304

305
    def resolve_address(host)
306
      return @resolv_cache[host] if @resolv_cache[host]
307
      address = Rex::Socket.resolv_to_dotted(host) rescue nil
308
      @resolv_cache[host] = address
309
      if address
310
        block = @block
311
        db.emit(:address, address, &block) if block
312
      end
313
      return address
314
    end
315

316
    # Alias this
317
    def resolve_issue_url_address(uri)
318
      if uri.host
319
        address = resolve_address(uri.host)
320
        unless address
321
          @parse_warnings << "Could not resolve address for '#{uri.host}', skipping."
322
        end
323
      else
324
        @parse_warnings << "Could not determine a host for this import."
325
      end
326
      address
327
    end
328

329
    def handle_parse_warnings(&block)
330
      return if @parse_warnings.empty?
331
      @parse_warnings.each do |pwarn|
332
        db.emit(:warning, pwarn, &block) if block
333
      end
334
    end
335

336
    def record_url
337
      return unless in_issue
338
      return unless has_text
339
      uri = URI.parse(@text) rescue nil
340
      return unless uri
341
      @state[:uri] = uri
342
    end
343

344
    def in_issue
345
      return false unless in_tag("Issue")
346
      return false unless in_tag("Issues")
347
      return false unless in_tag("XmlReport")
348
      return true
349
    end
350

351
    def has_text
352
      return false unless @text
353
      return false if @text.strip.empty?
354
      @text = @text.strip
355
    end
356

357
  end
358

359
end
360
end
361

362

363