Real-time collaboration for Jupyter Notebooks, Linux Terminals, LaTeX, VS Code, R IDE, and more,
all in one place.
Real-time collaboration for Jupyter Notebooks, Linux Terminals, LaTeX, VS Code, R IDE, and more,
all in one place.
Path: blob/master/lib/rex/parser/nexpose_raw_document.rb
Views: 11777
# -*- coding: binary -*-1require "rex/parser/nokogiri_doc_mixin"2require "date"34module Rex5module Parser67# If Nokogiri is available, define Template document class.8load_nokogiri && class NexposeRawDocument < Nokogiri::XML::SAX::Document910include NokogiriDocMixin1112attr_reader :tests1314NEXPOSE_HOST_DETAIL_FIELDS = %W{ nx_device_id nx_site_name nx_site_importance nx_scan_template nx_risk_score }15NEXPOSE_VULN_DETAIL_FIELDS = %W{16nx_scan_id17nx_vulnerable_since18nx_pci_compliance_status19}2021# Triggered every time a new element is encountered. We keep state22# ourselves with the @state variable, turning things on when we23# get here (and turning things off when we exit in end_element()).24def start_element(name=nil,attrs=[])25attrs = normalize_attrs(attrs)26block = @block27@state[:current_tag][name] = true28case name29when "nodes" # There are two main sections, nodes and VulnerabilityDefinitions30@tests = {}31when "node"32record_host(attrs)33when "name"34@state[:has_text] = true35when "endpoint"36@state.delete(:cached_service_object)37record_service(attrs)38when "service"39record_service_info(attrs)40when "fingerprint"41record_service_fingerprint(attrs)42when "os"43record_os_fingerprint(attrs)44when "test" # All the vulns tested for45@state[:has_text] = true46record_host_test(attrs)47record_service_test(attrs)48when "vulnerability"49record_vuln(attrs)50when "reference"51@state[:has_text] = true52record_reference(attrs)53when "description"54@state[:has_text] = true55record_vuln_description(attrs)56when "solution"57@state[:has_text] = true58record_vuln_solution(attrs)59when "tag"60@state[:has_text] = true61when "tags"62@state[:tags] = []63#64# These are markup tags only present within description/solutions65#66when "ContainerBlockElement", # Overall container, no formatting67"Paragraph", # <Paragraph preformat="true">68"UnorderedList", # List container (bulleted)69"ListItem", # List item70"URLLink" # <URLLink LinkURL="http://support.microsoft.com/kb/887429" LinkTitle="http://support.microsoft.com/kb/887429" href="http://support.microsoft.com/kb/887429">KB 887429</URLLink>7172record_formatted_content(name, attrs)7374end75end7677# When we exit a tag, this is triggered.78def end_element(name=nil)79block = @block80case name81when "node" # Wrap it up82collect_host_data83host_object = report_host &block84report_services(host_object)85report_fingerprint(host_object)86# Reset the state once we close a host87@state.delete_if {|k| k.to_s !~ /^(current_tag|in_nodes)$/}88@report_data = {:workspace => @args[:workspace]}89when "name"90collect_hostname91@state[:has_text] = false92@text = nil93when "endpoint"94collect_service_data95@state.delete(:cached_service_object)96when "os"97collect_os_fingerprints98when "test"99report_test(&block)100@state[:has_text] = false101@text = nil102when "vulnerability"103collect_vuln_info104report_vuln(&block)105@state.delete_if {|k| k.to_s !~ /^(current_tag|in_vulndefs)$/}106when "reference"107@state[:has_text] = false108collect_reference109@text = nil110when "description"111@state[:has_text] = false112collect_vuln_description113@text = nil114when "solution"115@state[:has_text] = false116collect_vuln_solution117@text = nil118when "tag"119@state[:has_text] = false120collect_tag121@text = nil122when "tags"123@report_data[:vuln_tags] = @state[:tags]124@state.delete(:tags)125#126# These are markup tags only present within description/solutions127#128when "ContainerBlockElement", # Overall container, no formatting129"Paragraph", # <Paragraph preformat="true">130"UnorderedList", # List container (bulleted)131"ListItem", # List item132"URLLink" # <URLLink LinkURL="http://support.microsoft.com/kb/887429" LinkTitle="http://support.microsoft.com/kb/887429" href="http://support.microsoft.com/kb/887429">KB 887429</URLLink>133134collect_formatted_content(name)135end136@state[:current_tag].delete name137end138139def collect_reference140return unless in_tag("references")141return unless in_tag("vulnerability")142return unless @state[:vuln]143@state[:ref][:value] = @text.to_s.strip144@report_data[:refs] ||= []145@report_data[:refs] << @state[:ref]146@state[:ref] = nil147end148149def collect_vuln_description150return unless in_tag("description")151return unless in_tag("vulnerability")152return unless @state[:vuln]153@report_data[:vuln_description] = clean_formatted_text( @report_data[:vuln_description_stack].join.strip )154end155156def collect_vuln_solution157return unless in_tag("solution")158return unless in_tag("vulnerability")159return unless @state[:vuln]160@report_data[:vuln_solution] = clean_formatted_text( @report_data[:vuln_solution_stack].join.strip )161end162163def collect_tag164return unless in_tag("tag")165return unless in_tag("tags")166return unless in_tag("vulnerability")167return unless @state[:vuln]168@state[:tags] ||= []169@state[:tags] << @text.to_s.strip170end171172def collect_vuln_info173return unless in_tag("VulnerabilityDefinitions")174return unless in_tag("vulnerability")175return unless @state[:vuln]176vuln = @state[:vuln]177vuln[:refs] = @report_data[:refs]178@report_data[:vuln] = vuln179@state[:vuln] = nil180@report_data[:refs] = nil181end182183def report_vuln(&block)184return unless in_tag("VulnerabilityDefinitions")185return unless @report_data[:vuln]186return unless @report_data[:vuln][:matches].kind_of? Array187188::ApplicationRecord.connection_pool.with_connection {189190refs = normalize_references(@report_data[:vuln][:refs])191refs << "NEXPOSE-#{report_data[:vuln]["id"]}"192vuln_instances = @report_data[:vuln][:matches].size193db.emit(:vuln, [refs.last,vuln_instances], &block) if block194195# TODO: potential remove the size limit on this field, might require196# some additional UX197if @report_data[:vuln]['title'].length > 255198db.emit :warning, 'Vulnerability name longer than 255 characters, truncating.', &block if block199@report_data[:vuln]['title'] = @report_data[:vuln]['title'][0..254]200end201202vuln_ids = @report_data[:vuln][:matches].map{ |v| v[0] }203vdet_ids = @report_data[:vuln][:matches].map{ |v| v[1] }204205refs = refs.uniq.map{|x| db.find_or_create_ref(:name => x) }206207# Assign title and references to all vuln_ids208# Mass update fails due to the join table || ::Mdm::Vuln.where(:id => vuln_ids).update_all({ :name => @report_data[:vuln]["title"], :refs => refs } )209vuln_ids.each do |vid|210vuln = ::Mdm::Vuln.find(vid)211next unless vuln212vuln.name = @report_data[:vuln]["title"]213214if refs.length > 0215vuln.refs += refs216end217218if vuln.changed?219vuln.save!220end221end222223# Mass update vulnerability details across the database based on conditions224vdet_info = { :title => @report_data[:vuln]["title"] }225vdet_info[:description] = @report_data[:vuln_description] unless @report_data[:vuln_description].to_s.empty?226vdet_info[:solution] = @report_data[:vuln_solution] unless @report_data[:vuln_solution].to_s.empty?227vdet_info[:nx_tags] = @report_data[:vuln_tags].sort.uniq.join(", ") if ( @report_data[:vuln_tags].kind_of?(::Array) and @report_data[:vuln_tags].length > 0 )228vdet_info[:nx_severity] = @report_data[:vuln]["severity"].to_f if @report_data[:vuln]["severity"]229vdet_info[:nx_pci_severity] = @report_data[:vuln]["pciSeverity"].to_f if @report_data[:vuln]["pciSeverity"]230vdet_info[:cvss_score] = @report_data[:vuln]["cvssScore"].to_f if @report_data[:vuln]["cvssScore"]231vdet_info[:cvss_vector] = @report_data[:vuln]["cvssVector"] if @report_data[:vuln]["cvssVector"]232233%W{ published added modified }.each do |tf|234next if not @report_data[:vuln][tf]235ts = DateTime.parse(@report_data[:vuln][tf]) rescue nil236next if not ts237vdet_info[ "nx_#{tf}".to_sym ] = ts238end239240::Mdm::VulnDetail.where(:id => vdet_ids).update_all(vdet_info)241242@report_data[:vuln] = nil243244}245end246247def record_reference(attrs)248return unless in_tag("VulnerabilityDefinitions")249return unless in_tag("vulnerability")250@state[:ref] = attr_hash(attrs)251end252253def record_vuln(attrs)254return unless in_tag("VulnerabilityDefinitions")255vuln = attr_hash(attrs)256matching_tests = @tests[ vuln["id"].downcase ]257return unless matching_tests258return if matching_tests.empty?259@state[:vuln] = vuln260@state[:vuln][:matches] = matching_tests261end262263def record_vuln_description(attrs)264@report_data[:vuln_description_stack] = []265end266267def record_vuln_solution(attrs)268@report_data[:vuln_solution_stack] = []269end270271272def record_formatted_content(name, eattrs)273attrs = attr_hash(eattrs)274stack = nil275276if in_tag("solution")277stack = @report_data[:vuln_solution_stack]278end279280if in_tag("description")281stack = @report_data[:vuln_description_stack]282end283284if in_tag("test")285stack = @report_data[:vuln_proof_stack]286end287288return if not stack289290@report_data[:formatted_indent] ||= 0291292data = @text.to_s.strip.split(/\n+/).map{|t| t.strip}.join(" ")293@text = ""294295case name296when 'ListItem'297@report_data[:formatted_indent] = 1298# data = "\n* " + data299when 'URLLink'300@report_data[:formatted_link] = attrs["LinkURL"]301else302303if @report_data[:formatted_indent] > 1304data = (" " * (@report_data[:formatted_indent])) + data305end306307if @report_data[:formatted_indent] == 1308@report_data[:formatted_indent] = 6309end310end311312if data.length > 0313stack << data314end315end316317def collect_formatted_content(name)318stack = nil319prefix = ""320321if in_tag("solution")322stack = @report_data[:vuln_solution_stack]323end324325if in_tag("description")326stack = @report_data[:vuln_description_stack]327end328329if in_tag("test")330stack = @report_data[:vuln_proof_stack]331end332333return if not stack334335data = @text.to_s.strip.split(/\n+/).map{|t| t.strip}.join(" ")336@text = ""337338case name339when 'URLLink'340if @report_data[:formatted_link]341if data != @report_data[:formatted_link]342if data.empty?343data << (" " + @report_data[:formatted_link])344else345data = " " + data + " ( " + @report_data[:formatted_link] + " )"346end347end348end349when 'Paragraph'350data << "\n\n"351when 'ListItem'352@report_data[:formatted_indent] = 0353data << "\n"354end355356if data.length > 0357stack << data358end359end360361# XML Export 2.0 includes additional test keys:362# <test id="unix-unowned-files-or-dirs" status="vulnerable-exploited" scan-id="6381" vulnerable-since="20120322T124352665" pci-compliance-status="pass">363364def report_test365return unless in_tag("nodes")366return unless in_tag("node")367return unless @state[:test]368369vuln_info = {370:workspace => @args[:workspace],371# This name will be overwritten during the vuln definition372# parsing via mass-update.373:name => "NEXPOSE-" + @state[:test][:id].downcase,374:host => @state[:cached_host_object] || @state[:address]375}376377if in_tag("endpoint") and @state[:test][:port]378# Verify this port actually has some relation to our tracked state379# since it may not due to greedy vulnerability matching380if @state[:cached_service_object] and @state[:cached_service_object].port.to_i == @state[:test][:port].to_i381vuln_info[:service] = @state[:cached_service_object]382else383vuln_info[:port] = @state[:test][:port]384vuln_info[:proto] = @state[:test][:protocol] if @state[:test][:protocol]385end386end387388# This hash feeds a vuln_details row for this vulnerability389vdet = { :src => 'nexpose', :nx_vuln_id => @state[:test][:id] }390391# This hash defines the matching criteria to overwrite an existing entry392vkey = { :src => 'nexpose', :nx_vuln_id => @state[:test][:id] }393394if @state[:nx_device_id]395vdet[:nx_device_id] = @state[:nx_device_id]396vkey[:nx_device_id] = @state[:nx_device_id]397end398399if @state[:test][:key]400vdet[:nx_proof_key] = @state[:test][:key]401vkey[:nx_proof_key] = @state[:test][:key]402end403404vdet[:nx_console_id] = @nx_console_id if @nx_console_id405vdet[:nx_vuln_status] = @state[:test][:status] if @state[:test][:status]406407vdet[:nx_scan_id] = @state[:test][:nx_scan_id] if @state[:test][:nx_scan_id]408vdet[:nx_pci_compliance_status] = @state[:test][:nx_pci_compliance_status] if @state[:test][:nx_pci_compliance_status]409410if @state[:test][:nx_vulnerable_since]411ts = ::DateTime.parse(@state[:test][:nx_vulnerable_since]) rescue nil412vdet[:nx_vulnerable_since] = ts if ts413end414415proof = clean_formatted_text(@report_data[:vuln_proof_stack].join.strip)416@report_data[:vuln_proof_stack] = []417418vuln_info[:info] = proof419vdet[:proof] = proof420421# Configure the find key for vuln_details422vdet[:key] = vkey423424# Pass this key to the vuln hash to find existing entries425# that may have been renamed (re-import nexpose vulns)426vuln_info[:details_match] = vkey427428::ApplicationRecord.connection_pool.with_connection {429430# Report the vulnerability431vuln = db.report_vuln(vuln_info)432433if vuln434# Report the vulnerability details435detail = db.report_vuln_details(vuln, vdet)436437# Cache returned host and service objects if necessary438@state[:cached_host_object] ||= vuln.host439440# The vuln.service may be found via greedy matching441if in_tag("endpoint") and vuln.service442@state[:cached_service_object] ||= vuln.service443end444445# Record the ID of this vuln for a future mass update that446# brings in title, risk, description, solution, etc447@tests[ @state[:test][:id].downcase ] ||= []448@tests[ @state[:test][:id].downcase ] << [ vuln.id, detail.id ]449end450451}452@state[:test] = nil453end454455def record_os_fingerprint(attrs)456return unless in_tag("nodes")457return unless in_tag("fingerprints")458return unless in_tag("node")459return if in_tag("service")460@state[:os] = attr_hash(attrs)461end462463# Just keep the highest scoring, which is usually the most vague. :(464def collect_os_fingerprints465@report_data[:os] ||= {}466return unless @state[:os]["certainty"].to_f > 0467return if @report_data[:os]["os_certainty"].to_f > @state[:os]["certainty"].to_f468@report_data[:os] = {} # Zero it out if we're replacing it.469@report_data[:os]["os_certainty"] = @state[:os]["certainty"]470@report_data[:os]["os_vendor"] = @state[:os]["vendor"]471@report_data[:os]["os_family"] = @state[:os]["family"]472@report_data[:os]["os_product"] = @state[:os]["product"]473@report_data[:os]["os_version"] = @state[:os]["version"]474@report_data[:os]["os_arch"] = @state[:os]["arch"]475end476477# Just taking the first one.478def collect_hostname479if in_tag("node")480@state[:hostname] ||= @text.to_s.strip if @text481@text = nil482end483end484485def record_service_fingerprint(attrs)486return unless in_tag("nodes")487return unless in_tag("node")488return unless in_tag("service")489return unless in_tag("fingerprint")490@state[:service_fingerprint] = attr_hash(attrs)491end492493def record_service_info(attrs)494return unless in_tag("nodes")495return unless in_tag("node")496return unless in_tag("service")497@state[:service].merge! attr_hash(attrs)498end499500def report_fingerprint(host_object)501return unless host_object.kind_of? ::Mdm::Host502return unless @report_data[:os].kind_of? Hash503note = {504:workspace => host_object.workspace,505:host => host_object,506:type => "host.os.nexpose_fingerprint",507:data => {508:family => @report_data[:os]["os_family"],509:certainty => @report_data[:os]["os_certainty"]510}511}512note[:data][:vendor] = @report_data[:os]["os_vendor"] if @report_data[:os]["os_vendor"]513note[:data][:product] = @report_data[:os]["os_product"] if @report_data[:os]["os_product"]514note[:data][:version] = @report_data[:os]["os_version"] if @report_data[:os]["os_version"]515note[:data][:arch] = @report_data[:os]["os_arch"] if @report_data[:os]["os_arch"]516db_report(:note, note)517end518519def report_services(host_object)520return unless host_object.kind_of? ::Mdm::Host521return unless @report_data[:ports]522return if @report_data[:ports].empty?523reported = []524@report_data[:ports].each do |svc|525reported << db_report(:service, svc.merge(:host => host_object))526end527reported528end529530def record_service(attrs)531return unless in_tag("nodes")532return unless in_tag("node")533return unless in_tag("endpoint")534@state[:service] = attr_hash(attrs)535end536537def collect_service_data538return unless in_tag("node")539return unless in_tag("endpoint")540port_hash = {}541@report_data[:ports] ||= []542@state[:service].each do |k,v|543case k544when "protocol"545port_hash[:proto] = v546when "port"547port_hash[:port] = v548when "status"549port_hash[:status] = (v == "open" ? Msf::ServiceState::Open : Msf::ServiceState::Closed)550end551end552if @state[:service]553if state[:service]["name"] == "<unknown>"554sname = nil555else556sname = db.service_name_map(@state[:service]["name"])557end558port_hash[:name] = sname559end560if @state[:service_fingerprint]561info = []562info << @state[:service_fingerprint]["product"] if @state[:service_fingerprint]["product"]563info << @state[:service_fingerprint]["version"] if @state[:service_fingerprint]["version"]564port_hash[:info] = info.join(" ") if info[0]565end566@report_data[:ports] << port_hash.clone567@state.delete :service_fingerprint568@state.delete :service569@report_data[:ports]570end571572def actually_vulnerable(test)573return false unless test.has_key? "status"574return false unless test.has_key? "id"575['vulnerable-exploited', 'vulnerable-version', 'potential'].include? test["status"]576end577578def record_host_test(attrs)579return unless in_tag("nodes")580return unless in_tag("node")581return if in_tag("service")582return unless in_tag("tests")583584test = attr_hash(attrs)585return unless actually_vulnerable(test)586@state[:test] = {:id => test["id"].downcase}587@state[:test][:key] = test["key"] if test["key"]588@state[:test][:nx_scan_id] = test["scan-id"] if test["scan-id"]589@state[:test][:nx_vulnerable_since] = test["vulnerable-since"] if test["vulnerable-since"]590@state[:test][:nx_pci_compliance_status] = test["pci-compliance-status"] if test["pci-compliance-status"]591592@report_data[:vuln_proof_stack] = []593end594595def record_service_test(attrs)596return unless in_tag("nodes")597return unless in_tag("node")598return unless in_tag("service")599return unless in_tag("tests")600test = attr_hash(attrs)601return unless actually_vulnerable(test)602@state[:test] = {603:id => test["id"].downcase,604:port => @state[:service]["port"],605:protocol => @state[:service]["protocol"],606}607@state[:test][:key] = test["key"] if test["key"]608@state[:test][:status] = test["status"] if test["status"]609@state[:test][:nx_scan_id] = test["scan-id"] if test["scan-id"]610@state[:test][:nx_vulnerable_since] = test["vulnerable-since"] if test["vulnerable-since"]611@state[:test][:nx_pci_compliance_status] = test["pci-compliance-status"] if test["pci-compliance-status"]612@report_data[:vuln_proof_stack] = []613end614615def record_host(attrs)616return unless in_tag("nodes")617host_attrs = attr_hash(attrs)618if host_attrs["status"] == "alive"619@state[:host_is_alive] = true620@state[:address] = host_attrs["address"]621@state[:mac] = host_attrs["hardware-address"] if host_attrs["hardware-address"]622623NEXPOSE_HOST_DETAIL_FIELDS.each do |f|624fs = f.to_sym625fk = f.sub(/^nx_/, '').gsub('_', '-')626if host_attrs[fk]627@state[fs] = host_attrs[fk]628end629end630end631end632633def collect_host_data634return unless in_tag("node")635@report_data[:host] = @state[:address]636@report_data[:state] = Msf::HostState::Alive637@report_data[:name] = @state[:hostname] if @state[:hostname]638if @state[:mac]639if @state[:mac] =~ /[0-9a-fA-f]{12}/640@report_data[:mac] = @state[:mac].scan(/.{2}/).join(":")641else642@report_data[:mac] = @state[:mac]643end644end645646NEXPOSE_HOST_DETAIL_FIELDS.each do |f|647v = @state[f.to_sym]648@report_data[f.to_sym] = v if v649end650end651652def report_host(&block)653if host_is_okay654db.emit(:address,@report_data[:host],&block) if block655device_id = @report_data[:nx_device_id]656657host_object = db_report(:host, @report_data.merge(:workspace => @args[:workspace] ) )658if host_object659db.report_import_note(host_object.workspace, host_object)660if device_id661detail = {662:key => { :src => 'nexpose' },663:src => 'nexpose',664:nx_device_id => device_id665}666detail[:nx_console_id] = @nx_console_id if @nx_console_id667668NEXPOSE_HOST_DETAIL_FIELDS.each do |f|669v = @report_data.delete(f.to_sym)670detail[f.to_sym] = v if v671end672673674db.report_host_details(host_object, detail)675end676end677host_object678end679end680681def clean_formatted_text(txt)682txt.split(/\n/).map{ |t|683t.sub(/^\s+$/, '').684sub(/^(\s{6,20})/, ' ')685}.join("\n").gsub(/\n{4,10}/, "\n\n\n")686end687688end689690end691end692693694695