Path: blob/master/lib/rex/proto/http/client.rb
19812 views
# -*- coding: binary -*-12require 'rex/socket'34require 'rex/text'5require 'digest'67module Rex8module Proto9module Http10###11#12# Acts as a client to an HTTP server, sending requests and receiving responses.13#14# See the RFC: http://www.w3.org/Protocols/rfc2616/rfc2616.html15#16###17class Client1819#20# Creates a new client instance21#22# @param [Rex::Proto::Http::HttpSubscriber] subscriber A subscriber to Http requests/responses23def initialize(host, port = 80, context = {}, ssl = nil, ssl_version = nil, proxies = nil, username = '', password = '', kerberos_authenticator: nil, comm: nil, subscriber: nil, sslkeylogfile: nil)24self.hostname = host25self.port = port.to_i26self.context = context27self.ssl = ssl28self.ssl_version = ssl_version29self.proxies = proxies30self.username = username31self.password = password32self.kerberos_authenticator = kerberos_authenticator33self.comm = comm34self.subscriber = subscriber || HttpSubscriber.new35self.sslkeylogfile = sslkeylogfile3637# Take ClientRequest's defaults, but override with our own38self.config = Http::ClientRequest::DefaultConfig.merge({39'read_max_data' => (1024 * 1024 * 1),40'vhost' => hostname,41'ssl_server_name_indication' => hostname42})43config['agent'] ||= Rex::UserAgent.session_agent4445# XXX: This info should all be controlled by ClientRequest46self.config_types = {47'uri_encode_mode' => ['hex-normal', 'hex-all', 'hex-random', 'hex-noslashes', 'u-normal', 'u-random', 'u-all'],48'uri_encode_count' => 'integer',49'uri_full_url' => 'bool',50'pad_method_uri_count' => 'integer',51'pad_uri_version_count' => 'integer',52'pad_method_uri_type' => ['space', 'tab', 'apache'],53'pad_uri_version_type' => ['space', 'tab', 'apache'],54'method_random_valid' => 'bool',55'method_random_invalid' => 'bool',56'method_random_case' => 'bool',57'version_random_valid' => 'bool',58'version_random_invalid' => 'bool',59'uri_dir_self_reference' => 'bool',60'uri_dir_fake_relative' => 'bool',61'uri_use_backslashes' => 'bool',62'pad_fake_headers' => 'bool',63'pad_fake_headers_count' => 'integer',64'pad_get_params' => 'bool',65'pad_get_params_count' => 'integer',66'pad_post_params' => 'bool',67'pad_post_params_count' => 'integer',68'shuffle_get_params' => 'bool',69'shuffle_post_params' => 'bool',70'uri_fake_end' => 'bool',71'uri_fake_params_start' => 'bool',72'header_folding' => 'bool',73'chunked_size' => 'integer',74'partial' => 'bool'75}76end7778#79# Set configuration options80#81def set_config(opts = {})82opts.each_pair do |var, val|83# Default type is string84typ = config_types[var] || 'string'8586# These are enum types87if typ.is_a?(Array) && !typ.include?(val)88raise "The specified value for #{var} is not one of the valid choices"89end9091# The caller should have converted these to proper ruby types, but92# take care of the case where they didn't before setting the93# config.9495if (typ == 'bool')96val = val == true || val.to_s =~ /^(t|y|1)/i97end9899if (typ == 'integer')100val = val.to_i101end102103config[var] = val104end105end106107#108# Create an arbitrary HTTP request109#110# @param opts [Hash]111# @option opts 'agent' [String] User-Agent header value112# @option opts 'connection' [String] Connection header value113# @option opts 'cookie' [String] Cookie header value114# @option opts 'data' [String] HTTP data (only useful with some methods, see rfc2616)115# @option opts 'encode' [Bool] URI encode the supplied URI, default: false116# @option opts 'headers' [Hash] HTTP headers, e.g. <code>{ "X-MyHeader" => "value" }</code>117# @option opts 'method' [String] HTTP method to use in the request, not limited to standard methods defined by rfc2616, default: GET118# @option opts 'proto' [String] protocol, default: HTTP119# @option opts 'query' [String] raw query string120# @option opts 'raw_headers' [String] Raw HTTP headers121# @option opts 'uri' [String] the URI to request122# @option opts 'version' [String] version of the protocol, default: 1.1123# @option opts 'vhost' [String] Host header value124#125# @return [ClientRequest]126def request_raw(opts = {})127opts = config.merge(opts)128129opts['cgi'] = false130opts['port'] = port131opts['ssl'] = ssl132133ClientRequest.new(opts)134end135136#137# Create a CGI compatible request138#139# @param (see #request_raw)140# @option opts (see #request_raw)141# @option opts 'ctype' [String] Content-Type header value, default for POST requests: +application/x-www-form-urlencoded+142# @option opts 'encode_params' [Bool] URI encode the GET or POST variables (names and values), default: true143# @option opts 'vars_get' [Hash] GET variables as a hash to be translated into a query string144# @option opts 'vars_post' [Hash] POST variables as a hash to be translated into POST data145# @option opts 'vars_form_data' [Hash] POST form_data variables as a hash to be translated into multi-part POST form data146#147# @return [ClientRequest]148def request_cgi(opts = {})149opts = config.merge(opts)150151opts['cgi'] = true152opts['port'] = port153opts['ssl'] = ssl154155ClientRequest.new(opts)156end157158#159# Connects to the remote server if possible.160#161# @param t [Integer] Timeout162# @see Rex::Socket::Tcp.create163# @return [Rex::Socket::Tcp]164def connect(t = -1)165# If we already have a connection and we aren't pipelining, close it.166if conn167if !pipelining?168close169else170return conn171end172end173174timeout = (t.nil? or t == -1) ? 0 : t175176self.conn = Rex::Socket::Tcp.create(177'PeerHost' => hostname,178'PeerHostname' => config['ssl_server_name_indication'] || config['vhost'],179'PeerPort' => port.to_i,180'LocalHost' => local_host,181'LocalPort' => local_port,182'Context' => context,183'SSL' => ssl,184'SSLVersion' => ssl_version,185'SSLKeyLogFile' => sslkeylogfile,186'Proxies' => proxies,187'Timeout' => timeout,188'Comm' => comm189)190end191192#193# Closes the connection to the remote server.194#195def close196if conn && !conn.closed?197conn.shutdown198conn.close199end200201self.conn = nil202self.ntlm_client = nil203end204205#206# Sends a request and gets a response back207#208# If the request is a 401, and we have creds, it will attempt to complete209# authentication and return the final response210#211# @return (see #_send_recv)212def send_recv(req, t = -1, persist = false)213res = _send_recv(req, t, persist)214if res and res.code == 401 and res.headers['WWW-Authenticate']215res = send_auth(res, req.opts, t, persist)216end217res218end219220#221# Transmit an HTTP request and receive the response222#223# If persist is set, then the request will attempt to reuse an existing224# connection.225#226# Call this directly instead of {#send_recv} if you don't want automatic227# authentication handling.228#229# @return (see #read_response)230def _send_recv(req, t = -1, persist = false)231@pipeline = persist232subscriber.on_request(req)233if req.respond_to?(:opts) && req.opts['ntlm_transform_request'] && ntlm_client234req = req.opts['ntlm_transform_request'].call(ntlm_client, req)235elsif req.respond_to?(:opts) && req.opts['krb_transform_request'] && krb_encryptor236req = req.opts['krb_transform_request'].call(krb_encryptor, req)237end238239send_request(req, t)240241res = read_response(t, original_request: req)242if req.respond_to?(:opts) && req.opts['ntlm_transform_response'] && ntlm_client243req.opts['ntlm_transform_response'].call(ntlm_client, res)244elsif req.respond_to?(:opts) && req.opts['krb_transform_response'] && krb_encryptor245req = req.opts['krb_transform_response'].call(krb_encryptor, res)246end247res.request = req.to_s if res248res.peerinfo = peerinfo if res249subscriber.on_response(res)250res251end252253#254# Send an HTTP request to the server255#256# @param req [Request,ClientRequest,#to_s] The request to send257# @param t (see #connect)258#259# @return [void]260def send_request(req, t = -1)261connect(t)262conn.put(req.to_s)263end264265# Resends an HTTP Request with the proper authentication headers266# set. If we do not support the authentication type the server requires267# we return the original response object268#269# @param res [Response] the HTTP Response object270# @param opts [Hash] the options used to generate the original HTTP request271# @param t [Integer] the timeout for the request in seconds272# @param persist [Boolean] whether or not to persist the TCP connection (pipelining)273#274# @return [Response] the last valid HTTP response object we received275def send_auth(res, opts, t, persist)276if opts['username'].nil? or opts['username'] == ''277if username and !(username == '')278opts['username'] = username279opts['password'] = password280else281opts['username'] = nil282opts['password'] = nil283end284end285286if opts[:kerberos_authenticator].nil?287opts[:kerberos_authenticator] = kerberos_authenticator288end289290return res if (opts['username'].nil? or opts['username'] == '') and opts[:kerberos_authenticator].nil?291292supported_auths = res.headers['WWW-Authenticate']293294# if several providers are available, the client may want one in particular295preferred_auth = opts['preferred_auth']296297if supported_auths.include?('Basic') && (preferred_auth.nil? || preferred_auth == 'Basic')298opts['headers'] ||= {}299opts['headers']['Authorization'] = basic_auth_header(opts['username'], opts['password'])300req = request_cgi(opts)301res = _send_recv(req, t, persist)302return res303elsif supported_auths.include?('Digest') && (preferred_auth.nil? || preferred_auth == 'Digest')304temp_response = digest_auth(opts)305if temp_response.is_a? Rex::Proto::Http::Response306res = temp_response307end308return res309elsif supported_auths.include?('NTLM') && (preferred_auth.nil? || preferred_auth == 'NTLM')310opts['provider'] = 'NTLM'311temp_response = negotiate_auth(opts)312if temp_response.is_a? Rex::Proto::Http::Response313res = temp_response314end315return res316elsif supported_auths.include?('Negotiate') && (preferred_auth.nil? || preferred_auth == 'Negotiate')317opts['provider'] = 'Negotiate'318temp_response = negotiate_auth(opts)319if temp_response.is_a? Rex::Proto::Http::Response320res = temp_response321end322return res323elsif supported_auths.include?('Negotiate') && (preferred_auth.nil? || preferred_auth == 'Kerberos')324opts['provider'] = 'Negotiate'325temp_response = kerberos_auth(opts)326if temp_response.is_a? Rex::Proto::Http::Response327res = temp_response328end329return res330end331return res332end333334# Converts username and password into the HTTP Basic authorization335# string.336#337# @return [String] A value suitable for use as an Authorization header338def basic_auth_header(username, password)339auth_str = username.to_s + ':' + password.to_s340'Basic ' + Rex::Text.encode_base64(auth_str)341end342# Send a series of requests to complete Digest Authentication343#344# @param opts [Hash] the options used to build an HTTP request345# @return [Response] the last valid HTTP response we received346def digest_auth(opts = {})347to = opts['timeout'] || 20348349digest_user = opts['username'] || ''350digest_password = opts['password'] || ''351352method = opts['method']353path = opts['uri']354iis = true355if (opts['DigestAuthIIS'] == false or config['DigestAuthIIS'] == false)356iis = false357end358359begin360resp = opts['response']361362if !resp363# Get authentication-challenge from server, and read out parameters required364r = request_cgi(opts.merge({365'uri' => path,366'method' => method367}))368resp = _send_recv(r, to)369unless resp.is_a? Rex::Proto::Http::Response370return nil371end372373if resp.code != 401374return resp375end376return resp unless resp.headers['WWW-Authenticate']377end378379# Don't anchor this regex to the beginning of string because header380# folding makes it appear later when the server presents multiple381# WWW-Authentication options (such as is the case with IIS configured382# for Digest or NTLM).383resp['www-authenticate'] =~ /Digest (.*)/384385parameters = {}386::Regexp.last_match(1).split(/,[[:space:]]*/).each do |p|387k, v = p.split('=', 2)388parameters[k] = v.gsub('"', '')389end390391auth_digest = Rex::Proto::Http::AuthDigest.new392auth = auth_digest.digest(digest_user, digest_password, method, path, parameters, iis)393394headers = { 'Authorization' => auth.join(', ') }395headers.merge!(opts['headers']) if opts['headers']396397# Send main request with authentication398r = request_cgi(opts.merge({399'uri' => path,400'method' => method,401'headers' => headers402}))403resp = _send_recv(r, to, true)404unless resp.is_a? Rex::Proto::Http::Response405return nil406end407408return resp409rescue ::Errno::EPIPE, ::Timeout::Error410end411end412413def kerberos_auth(opts = {})414to = opts['timeout'] || 20415auth_result = kerberos_authenticator.authenticate(mechanism: Rex::Proto::Gss::Mechanism::KERBEROS)416gss_data = auth_result[:security_blob]417gss_data_b64 = Rex::Text.encode_base64(gss_data)418419# Separate options for the auth requests420auth_opts = opts.clone421auth_opts['headers'] = opts['headers'].clone422auth_opts['headers']['Authorization'] = "Kerberos #{gss_data_b64}"423424if auth_opts['no_body_for_auth']425auth_opts.delete('data')426auth_opts.delete('krb_transform_request')427auth_opts.delete('krb_transform_response')428end429430begin431# Send the auth request432r = request_cgi(auth_opts)433resp = _send_recv(r, to)434unless resp.is_a? Rex::Proto::Http::Response435return nil436end437438# Get the challenge and craft the response439response = resp.headers['WWW-Authenticate'].scan(/Kerberos ([A-Z0-9\x2b\x2f=]+)/ni).flatten[0]440return resp unless response441442decoded = Rex::Text.decode_base64(response)443mutual_auth_result = kerberos_authenticator.parse_gss_init_response(decoded, auth_result[:session_key])444self.krb_encryptor = kerberos_authenticator.get_message_encryptor(mutual_auth_result[:ap_rep_subkey],445auth_result[:client_sequence_number],446mutual_auth_result[:server_sequence_number])447448if opts['no_body_for_auth']449# If the body wasn't sent in the authentication, now do the actual request450r = request_cgi(opts)451resp = _send_recv(r, to, true)452end453return resp454rescue ::Errno::EPIPE, ::Timeout::Error455return nil456end457end458459#460# Builds a series of requests to complete Negotiate Auth. Works essentially461# the same way as Digest auth. Same pipelining concerns exist.462#463# @option opts (see #send_request_cgi)464# @option opts provider ["Negotiate","NTLM"] What Negotiate provider to use465#466# @return [Response] the last valid HTTP response we received467def negotiate_auth(opts = {})468to = opts['timeout'] || 20469opts['username'] ||= ''470opts['password'] ||= ''471472if opts['provider'] and opts['provider'].include? 'Negotiate'473provider = 'Negotiate '474else475provider = 'NTLM '476end477478opts['method'] ||= 'GET'479opts['headers'] ||= {}480481workstation_name = Rex::Text.rand_text_alpha(rand(6..13))482domain_name = config['domain']483484ntlm_client = ::Net::NTLM::Client.new(485opts['username'],486opts['password'],487workstation: workstation_name,488domain: domain_name489)490type1 = ntlm_client.init_context491492begin493# Separate options for the auth requests494auth_opts = opts.clone495auth_opts['headers'] = opts['headers'].clone496auth_opts['headers']['Authorization'] = provider + type1.encode64497498if auth_opts['no_body_for_auth']499auth_opts.delete('data')500auth_opts.delete('ntlm_transform_request')501auth_opts.delete('ntlm_transform_response')502end503504# First request to get the challenge505r = request_cgi(auth_opts)506resp = _send_recv(r, to)507unless resp.is_a? Rex::Proto::Http::Response508return nil509end510511return resp unless resp.code == 401 && resp.headers['WWW-Authenticate']512513# Get the challenge and craft the response514ntlm_challenge = resp.headers['WWW-Authenticate'].scan(/#{provider}([A-Z0-9\x2b\x2f=]+)/ni).flatten[0]515return resp unless ntlm_challenge516517ntlm_message_3 = ntlm_client.init_context(ntlm_challenge, channel_binding)518519self.ntlm_client = ntlm_client520# Send the response521auth_opts['headers']['Authorization'] = "#{provider}#{ntlm_message_3.encode64}"522r = request_cgi(auth_opts)523resp = _send_recv(r, to, true)524525unless resp.is_a? Rex::Proto::Http::Response526return nil527end528529if opts['no_body_for_auth']530# If the body wasn't sent in the authentication, now do the actual request531r = request_cgi(opts)532resp = _send_recv(r, to, true)533end534return resp535rescue ::Errno::EPIPE, ::Timeout::Error536return nil537end538end539540def channel_binding541if !conn.respond_to?(:peer_cert) or conn.peer_cert.nil?542nil543else544Net::NTLM::ChannelBinding.create(OpenSSL::X509::Certificate.new(conn.peer_cert))545end546end547548# Read a response from the server549#550# Wait at most t seconds for the full response to be read in.551# If t is specified as a negative value, it indicates an indefinite wait cycle.552# If t is specified as nil or 0, it indicates no response parsing is required.553#554# @return [Response]555def read_response(t = -1, opts = {})556# Return a nil response if timeout is nil or 0557return if t.nil? || t == 0558559resp = Response.new560resp.max_data = config['read_max_data']561562original_request = opts.fetch(:original_request) { nil }563parse_opts = {}564unless original_request.nil?565parse_opts = { orig_method: original_request.opts['method'] }566end567568Timeout.timeout((t < 0) ? nil : t) do569rv = nil570while (571!conn.closed? and572rv != Packet::ParseCode::Completed and573rv != Packet::ParseCode::Error574)575576begin577buff = conn.get_once(resp.max_data, 1)578rv = resp.parse(buff || '', parse_opts)579580# Handle unexpected disconnects581rescue ::Errno::EPIPE, ::EOFError, ::IOError582case resp.state583when Packet::ParseState::ProcessingHeader584resp = nil585when Packet::ParseState::ProcessingBody586# truncated request, good enough587resp.error = :truncated588end589break590end591592# This is a dirty hack for broken HTTP servers593next unless rv == Packet::ParseCode::Completed594595rbody = resp.body596rbufq = resp.bufq597598rblob = rbody.to_s + rbufq.to_s599tries = 0600begin601# XXX: This doesn't deal with chunked encoding602while tries < 1000 and resp.headers['Content-Type'] and resp.headers['Content-Type'].start_with?('text/html') and rblob !~ %r{</html>}i603buff = conn.get_once(-1, 0.05)604break if !buff605606rblob += buff607tries += 1608end609rescue ::Errno::EPIPE, ::EOFError, ::IOError610end611612resp.bufq = ''613resp.body = rblob614end615end616617return resp if !resp618619# As a last minute hack, we check to see if we're dealing with a 100 Continue here.620# Most of the time this is handled by the parser via check_100()621if resp.proto == '1.1' and resp.code == 100 and !(opts[:skip_100])622# Read the real response from the body if we found one623# If so, our real response became the body, so we re-parse it.624if resp.body.to_s =~ /^HTTP/625body = resp.body626resp = Response.new627resp.max_data = config['read_max_data']628resp.parse(body, parse_opts)629# We found a 100 Continue but didn't read the real reply yet630# Otherwise reread the reply, but don't try this hack again631else632resp = read_response(t, skip_100: true)633end634end635636resp637rescue Timeout::Error638# Allow partial response due to timeout639resp if config['partial']640end641642#643# Cleans up any outstanding connections and other resources.644#645def stop646close647end648649#650# Returns whether or not the conn is valid.651#652def conn?653conn != nil654end655656#657# Whether or not connections should be pipelined.658#659def pipelining?660pipeline661end662663#664# Target host addr and port for this connection665#666def peerinfo667if conn668pi = conn.peerinfo || nil669if pi670return {671'addr' => pi.split(':')[0],672'port' => pi.split(':')[1].to_i673}674end675end676nil677end678679#680# An optional comm to use for creating the underlying socket.681#682attr_accessor :comm683#684# The client request configuration685#686attr_accessor :config687#688# The client request configuration classes689#690attr_accessor :config_types691#692# Whether or not pipelining is in use.693#694attr_accessor :pipeline695#696# The local host of the client.697#698attr_accessor :local_host699#700# The local port of the client.701#702attr_accessor :local_port703#704# The underlying connection.705#706attr_accessor :conn707#708# The calling context to pass to the socket709#710attr_accessor :context711#712# The proxy list713#714attr_accessor :proxies715716# Auth717attr_accessor :username, :password, :kerberos_authenticator718719# When parsing the request, thunk off the first response from the server, since junk720attr_accessor :junk_pipeline721722# @return [Rex::Proto::Http::HttpSubscriber] The HTTP subscriber723attr_accessor :subscriber724725protected726727# https728attr_accessor :ssl, :ssl_version # :nodoc:729730attr_accessor :hostname, :port # :nodoc:731732#733# The SSL key log file for the connected socket.734#735# @return [String]736attr_accessor :sslkeylogfile737738#739# The established NTLM connection info740#741attr_accessor :ntlm_client742743#744# The established kerberos connection info745#746attr_accessor :krb_encryptor747end748end749end750end751752753