Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
rapid7
GitHub Repository: rapid7/metasploit-framework
Path: blob/master/lib/rex/proto/http/client.rb
19812 views
1
# -*- coding: binary -*-
2
3
require 'rex/socket'
4
5
require 'rex/text'
6
require 'digest'
7
8
module Rex
9
module Proto
10
module Http
11
###
12
#
13
# Acts as a client to an HTTP server, sending requests and receiving responses.
14
#
15
# See the RFC: http://www.w3.org/Protocols/rfc2616/rfc2616.html
16
#
17
###
18
class Client
19
20
#
21
# Creates a new client instance
22
#
23
# @param [Rex::Proto::Http::HttpSubscriber] subscriber A subscriber to Http requests/responses
24
def initialize(host, port = 80, context = {}, ssl = nil, ssl_version = nil, proxies = nil, username = '', password = '', kerberos_authenticator: nil, comm: nil, subscriber: nil, sslkeylogfile: nil)
25
self.hostname = host
26
self.port = port.to_i
27
self.context = context
28
self.ssl = ssl
29
self.ssl_version = ssl_version
30
self.proxies = proxies
31
self.username = username
32
self.password = password
33
self.kerberos_authenticator = kerberos_authenticator
34
self.comm = comm
35
self.subscriber = subscriber || HttpSubscriber.new
36
self.sslkeylogfile = sslkeylogfile
37
38
# Take ClientRequest's defaults, but override with our own
39
self.config = Http::ClientRequest::DefaultConfig.merge({
40
'read_max_data' => (1024 * 1024 * 1),
41
'vhost' => hostname,
42
'ssl_server_name_indication' => hostname
43
})
44
config['agent'] ||= Rex::UserAgent.session_agent
45
46
# XXX: This info should all be controlled by ClientRequest
47
self.config_types = {
48
'uri_encode_mode' => ['hex-normal', 'hex-all', 'hex-random', 'hex-noslashes', 'u-normal', 'u-random', 'u-all'],
49
'uri_encode_count' => 'integer',
50
'uri_full_url' => 'bool',
51
'pad_method_uri_count' => 'integer',
52
'pad_uri_version_count' => 'integer',
53
'pad_method_uri_type' => ['space', 'tab', 'apache'],
54
'pad_uri_version_type' => ['space', 'tab', 'apache'],
55
'method_random_valid' => 'bool',
56
'method_random_invalid' => 'bool',
57
'method_random_case' => 'bool',
58
'version_random_valid' => 'bool',
59
'version_random_invalid' => 'bool',
60
'uri_dir_self_reference' => 'bool',
61
'uri_dir_fake_relative' => 'bool',
62
'uri_use_backslashes' => 'bool',
63
'pad_fake_headers' => 'bool',
64
'pad_fake_headers_count' => 'integer',
65
'pad_get_params' => 'bool',
66
'pad_get_params_count' => 'integer',
67
'pad_post_params' => 'bool',
68
'pad_post_params_count' => 'integer',
69
'shuffle_get_params' => 'bool',
70
'shuffle_post_params' => 'bool',
71
'uri_fake_end' => 'bool',
72
'uri_fake_params_start' => 'bool',
73
'header_folding' => 'bool',
74
'chunked_size' => 'integer',
75
'partial' => 'bool'
76
}
77
end
78
79
#
80
# Set configuration options
81
#
82
def set_config(opts = {})
83
opts.each_pair do |var, val|
84
# Default type is string
85
typ = config_types[var] || 'string'
86
87
# These are enum types
88
if typ.is_a?(Array) && !typ.include?(val)
89
raise "The specified value for #{var} is not one of the valid choices"
90
end
91
92
# The caller should have converted these to proper ruby types, but
93
# take care of the case where they didn't before setting the
94
# config.
95
96
if (typ == 'bool')
97
val = val == true || val.to_s =~ /^(t|y|1)/i
98
end
99
100
if (typ == 'integer')
101
val = val.to_i
102
end
103
104
config[var] = val
105
end
106
end
107
108
#
109
# Create an arbitrary HTTP request
110
#
111
# @param opts [Hash]
112
# @option opts 'agent' [String] User-Agent header value
113
# @option opts 'connection' [String] Connection header value
114
# @option opts 'cookie' [String] Cookie header value
115
# @option opts 'data' [String] HTTP data (only useful with some methods, see rfc2616)
116
# @option opts 'encode' [Bool] URI encode the supplied URI, default: false
117
# @option opts 'headers' [Hash] HTTP headers, e.g. <code>{ "X-MyHeader" => "value" }</code>
118
# @option opts 'method' [String] HTTP method to use in the request, not limited to standard methods defined by rfc2616, default: GET
119
# @option opts 'proto' [String] protocol, default: HTTP
120
# @option opts 'query' [String] raw query string
121
# @option opts 'raw_headers' [String] Raw HTTP headers
122
# @option opts 'uri' [String] the URI to request
123
# @option opts 'version' [String] version of the protocol, default: 1.1
124
# @option opts 'vhost' [String] Host header value
125
#
126
# @return [ClientRequest]
127
def request_raw(opts = {})
128
opts = config.merge(opts)
129
130
opts['cgi'] = false
131
opts['port'] = port
132
opts['ssl'] = ssl
133
134
ClientRequest.new(opts)
135
end
136
137
#
138
# Create a CGI compatible request
139
#
140
# @param (see #request_raw)
141
# @option opts (see #request_raw)
142
# @option opts 'ctype' [String] Content-Type header value, default for POST requests: +application/x-www-form-urlencoded+
143
# @option opts 'encode_params' [Bool] URI encode the GET or POST variables (names and values), default: true
144
# @option opts 'vars_get' [Hash] GET variables as a hash to be translated into a query string
145
# @option opts 'vars_post' [Hash] POST variables as a hash to be translated into POST data
146
# @option opts 'vars_form_data' [Hash] POST form_data variables as a hash to be translated into multi-part POST form data
147
#
148
# @return [ClientRequest]
149
def request_cgi(opts = {})
150
opts = config.merge(opts)
151
152
opts['cgi'] = true
153
opts['port'] = port
154
opts['ssl'] = ssl
155
156
ClientRequest.new(opts)
157
end
158
159
#
160
# Connects to the remote server if possible.
161
#
162
# @param t [Integer] Timeout
163
# @see Rex::Socket::Tcp.create
164
# @return [Rex::Socket::Tcp]
165
def connect(t = -1)
166
# If we already have a connection and we aren't pipelining, close it.
167
if conn
168
if !pipelining?
169
close
170
else
171
return conn
172
end
173
end
174
175
timeout = (t.nil? or t == -1) ? 0 : t
176
177
self.conn = Rex::Socket::Tcp.create(
178
'PeerHost' => hostname,
179
'PeerHostname' => config['ssl_server_name_indication'] || config['vhost'],
180
'PeerPort' => port.to_i,
181
'LocalHost' => local_host,
182
'LocalPort' => local_port,
183
'Context' => context,
184
'SSL' => ssl,
185
'SSLVersion' => ssl_version,
186
'SSLKeyLogFile' => sslkeylogfile,
187
'Proxies' => proxies,
188
'Timeout' => timeout,
189
'Comm' => comm
190
)
191
end
192
193
#
194
# Closes the connection to the remote server.
195
#
196
def close
197
if conn && !conn.closed?
198
conn.shutdown
199
conn.close
200
end
201
202
self.conn = nil
203
self.ntlm_client = nil
204
end
205
206
#
207
# Sends a request and gets a response back
208
#
209
# If the request is a 401, and we have creds, it will attempt to complete
210
# authentication and return the final response
211
#
212
# @return (see #_send_recv)
213
def send_recv(req, t = -1, persist = false)
214
res = _send_recv(req, t, persist)
215
if res and res.code == 401 and res.headers['WWW-Authenticate']
216
res = send_auth(res, req.opts, t, persist)
217
end
218
res
219
end
220
221
#
222
# Transmit an HTTP request and receive the response
223
#
224
# If persist is set, then the request will attempt to reuse an existing
225
# connection.
226
#
227
# Call this directly instead of {#send_recv} if you don't want automatic
228
# authentication handling.
229
#
230
# @return (see #read_response)
231
def _send_recv(req, t = -1, persist = false)
232
@pipeline = persist
233
subscriber.on_request(req)
234
if req.respond_to?(:opts) && req.opts['ntlm_transform_request'] && ntlm_client
235
req = req.opts['ntlm_transform_request'].call(ntlm_client, req)
236
elsif req.respond_to?(:opts) && req.opts['krb_transform_request'] && krb_encryptor
237
req = req.opts['krb_transform_request'].call(krb_encryptor, req)
238
end
239
240
send_request(req, t)
241
242
res = read_response(t, original_request: req)
243
if req.respond_to?(:opts) && req.opts['ntlm_transform_response'] && ntlm_client
244
req.opts['ntlm_transform_response'].call(ntlm_client, res)
245
elsif req.respond_to?(:opts) && req.opts['krb_transform_response'] && krb_encryptor
246
req = req.opts['krb_transform_response'].call(krb_encryptor, res)
247
end
248
res.request = req.to_s if res
249
res.peerinfo = peerinfo if res
250
subscriber.on_response(res)
251
res
252
end
253
254
#
255
# Send an HTTP request to the server
256
#
257
# @param req [Request,ClientRequest,#to_s] The request to send
258
# @param t (see #connect)
259
#
260
# @return [void]
261
def send_request(req, t = -1)
262
connect(t)
263
conn.put(req.to_s)
264
end
265
266
# Resends an HTTP Request with the proper authentication headers
267
# set. If we do not support the authentication type the server requires
268
# we return the original response object
269
#
270
# @param res [Response] the HTTP Response object
271
# @param opts [Hash] the options used to generate the original HTTP request
272
# @param t [Integer] the timeout for the request in seconds
273
# @param persist [Boolean] whether or not to persist the TCP connection (pipelining)
274
#
275
# @return [Response] the last valid HTTP response object we received
276
def send_auth(res, opts, t, persist)
277
if opts['username'].nil? or opts['username'] == ''
278
if username and !(username == '')
279
opts['username'] = username
280
opts['password'] = password
281
else
282
opts['username'] = nil
283
opts['password'] = nil
284
end
285
end
286
287
if opts[:kerberos_authenticator].nil?
288
opts[:kerberos_authenticator] = kerberos_authenticator
289
end
290
291
return res if (opts['username'].nil? or opts['username'] == '') and opts[:kerberos_authenticator].nil?
292
293
supported_auths = res.headers['WWW-Authenticate']
294
295
# if several providers are available, the client may want one in particular
296
preferred_auth = opts['preferred_auth']
297
298
if supported_auths.include?('Basic') && (preferred_auth.nil? || preferred_auth == 'Basic')
299
opts['headers'] ||= {}
300
opts['headers']['Authorization'] = basic_auth_header(opts['username'], opts['password'])
301
req = request_cgi(opts)
302
res = _send_recv(req, t, persist)
303
return res
304
elsif supported_auths.include?('Digest') && (preferred_auth.nil? || preferred_auth == 'Digest')
305
temp_response = digest_auth(opts)
306
if temp_response.is_a? Rex::Proto::Http::Response
307
res = temp_response
308
end
309
return res
310
elsif supported_auths.include?('NTLM') && (preferred_auth.nil? || preferred_auth == 'NTLM')
311
opts['provider'] = 'NTLM'
312
temp_response = negotiate_auth(opts)
313
if temp_response.is_a? Rex::Proto::Http::Response
314
res = temp_response
315
end
316
return res
317
elsif supported_auths.include?('Negotiate') && (preferred_auth.nil? || preferred_auth == 'Negotiate')
318
opts['provider'] = 'Negotiate'
319
temp_response = negotiate_auth(opts)
320
if temp_response.is_a? Rex::Proto::Http::Response
321
res = temp_response
322
end
323
return res
324
elsif supported_auths.include?('Negotiate') && (preferred_auth.nil? || preferred_auth == 'Kerberos')
325
opts['provider'] = 'Negotiate'
326
temp_response = kerberos_auth(opts)
327
if temp_response.is_a? Rex::Proto::Http::Response
328
res = temp_response
329
end
330
return res
331
end
332
return res
333
end
334
335
# Converts username and password into the HTTP Basic authorization
336
# string.
337
#
338
# @return [String] A value suitable for use as an Authorization header
339
def basic_auth_header(username, password)
340
auth_str = username.to_s + ':' + password.to_s
341
'Basic ' + Rex::Text.encode_base64(auth_str)
342
end
343
# Send a series of requests to complete Digest Authentication
344
#
345
# @param opts [Hash] the options used to build an HTTP request
346
# @return [Response] the last valid HTTP response we received
347
def digest_auth(opts = {})
348
to = opts['timeout'] || 20
349
350
digest_user = opts['username'] || ''
351
digest_password = opts['password'] || ''
352
353
method = opts['method']
354
path = opts['uri']
355
iis = true
356
if (opts['DigestAuthIIS'] == false or config['DigestAuthIIS'] == false)
357
iis = false
358
end
359
360
begin
361
resp = opts['response']
362
363
if !resp
364
# Get authentication-challenge from server, and read out parameters required
365
r = request_cgi(opts.merge({
366
'uri' => path,
367
'method' => method
368
}))
369
resp = _send_recv(r, to)
370
unless resp.is_a? Rex::Proto::Http::Response
371
return nil
372
end
373
374
if resp.code != 401
375
return resp
376
end
377
return resp unless resp.headers['WWW-Authenticate']
378
end
379
380
# Don't anchor this regex to the beginning of string because header
381
# folding makes it appear later when the server presents multiple
382
# WWW-Authentication options (such as is the case with IIS configured
383
# for Digest or NTLM).
384
resp['www-authenticate'] =~ /Digest (.*)/
385
386
parameters = {}
387
::Regexp.last_match(1).split(/,[[:space:]]*/).each do |p|
388
k, v = p.split('=', 2)
389
parameters[k] = v.gsub('"', '')
390
end
391
392
auth_digest = Rex::Proto::Http::AuthDigest.new
393
auth = auth_digest.digest(digest_user, digest_password, method, path, parameters, iis)
394
395
headers = { 'Authorization' => auth.join(', ') }
396
headers.merge!(opts['headers']) if opts['headers']
397
398
# Send main request with authentication
399
r = request_cgi(opts.merge({
400
'uri' => path,
401
'method' => method,
402
'headers' => headers
403
}))
404
resp = _send_recv(r, to, true)
405
unless resp.is_a? Rex::Proto::Http::Response
406
return nil
407
end
408
409
return resp
410
rescue ::Errno::EPIPE, ::Timeout::Error
411
end
412
end
413
414
def kerberos_auth(opts = {})
415
to = opts['timeout'] || 20
416
auth_result = kerberos_authenticator.authenticate(mechanism: Rex::Proto::Gss::Mechanism::KERBEROS)
417
gss_data = auth_result[:security_blob]
418
gss_data_b64 = Rex::Text.encode_base64(gss_data)
419
420
# Separate options for the auth requests
421
auth_opts = opts.clone
422
auth_opts['headers'] = opts['headers'].clone
423
auth_opts['headers']['Authorization'] = "Kerberos #{gss_data_b64}"
424
425
if auth_opts['no_body_for_auth']
426
auth_opts.delete('data')
427
auth_opts.delete('krb_transform_request')
428
auth_opts.delete('krb_transform_response')
429
end
430
431
begin
432
# Send the auth request
433
r = request_cgi(auth_opts)
434
resp = _send_recv(r, to)
435
unless resp.is_a? Rex::Proto::Http::Response
436
return nil
437
end
438
439
# Get the challenge and craft the response
440
response = resp.headers['WWW-Authenticate'].scan(/Kerberos ([A-Z0-9\x2b\x2f=]+)/ni).flatten[0]
441
return resp unless response
442
443
decoded = Rex::Text.decode_base64(response)
444
mutual_auth_result = kerberos_authenticator.parse_gss_init_response(decoded, auth_result[:session_key])
445
self.krb_encryptor = kerberos_authenticator.get_message_encryptor(mutual_auth_result[:ap_rep_subkey],
446
auth_result[:client_sequence_number],
447
mutual_auth_result[:server_sequence_number])
448
449
if opts['no_body_for_auth']
450
# If the body wasn't sent in the authentication, now do the actual request
451
r = request_cgi(opts)
452
resp = _send_recv(r, to, true)
453
end
454
return resp
455
rescue ::Errno::EPIPE, ::Timeout::Error
456
return nil
457
end
458
end
459
460
#
461
# Builds a series of requests to complete Negotiate Auth. Works essentially
462
# the same way as Digest auth. Same pipelining concerns exist.
463
#
464
# @option opts (see #send_request_cgi)
465
# @option opts provider ["Negotiate","NTLM"] What Negotiate provider to use
466
#
467
# @return [Response] the last valid HTTP response we received
468
def negotiate_auth(opts = {})
469
to = opts['timeout'] || 20
470
opts['username'] ||= ''
471
opts['password'] ||= ''
472
473
if opts['provider'] and opts['provider'].include? 'Negotiate'
474
provider = 'Negotiate '
475
else
476
provider = 'NTLM '
477
end
478
479
opts['method'] ||= 'GET'
480
opts['headers'] ||= {}
481
482
workstation_name = Rex::Text.rand_text_alpha(rand(6..13))
483
domain_name = config['domain']
484
485
ntlm_client = ::Net::NTLM::Client.new(
486
opts['username'],
487
opts['password'],
488
workstation: workstation_name,
489
domain: domain_name
490
)
491
type1 = ntlm_client.init_context
492
493
begin
494
# Separate options for the auth requests
495
auth_opts = opts.clone
496
auth_opts['headers'] = opts['headers'].clone
497
auth_opts['headers']['Authorization'] = provider + type1.encode64
498
499
if auth_opts['no_body_for_auth']
500
auth_opts.delete('data')
501
auth_opts.delete('ntlm_transform_request')
502
auth_opts.delete('ntlm_transform_response')
503
end
504
505
# First request to get the challenge
506
r = request_cgi(auth_opts)
507
resp = _send_recv(r, to)
508
unless resp.is_a? Rex::Proto::Http::Response
509
return nil
510
end
511
512
return resp unless resp.code == 401 && resp.headers['WWW-Authenticate']
513
514
# Get the challenge and craft the response
515
ntlm_challenge = resp.headers['WWW-Authenticate'].scan(/#{provider}([A-Z0-9\x2b\x2f=]+)/ni).flatten[0]
516
return resp unless ntlm_challenge
517
518
ntlm_message_3 = ntlm_client.init_context(ntlm_challenge, channel_binding)
519
520
self.ntlm_client = ntlm_client
521
# Send the response
522
auth_opts['headers']['Authorization'] = "#{provider}#{ntlm_message_3.encode64}"
523
r = request_cgi(auth_opts)
524
resp = _send_recv(r, to, true)
525
526
unless resp.is_a? Rex::Proto::Http::Response
527
return nil
528
end
529
530
if opts['no_body_for_auth']
531
# If the body wasn't sent in the authentication, now do the actual request
532
r = request_cgi(opts)
533
resp = _send_recv(r, to, true)
534
end
535
return resp
536
rescue ::Errno::EPIPE, ::Timeout::Error
537
return nil
538
end
539
end
540
541
def channel_binding
542
if !conn.respond_to?(:peer_cert) or conn.peer_cert.nil?
543
nil
544
else
545
Net::NTLM::ChannelBinding.create(OpenSSL::X509::Certificate.new(conn.peer_cert))
546
end
547
end
548
549
# Read a response from the server
550
#
551
# Wait at most t seconds for the full response to be read in.
552
# If t is specified as a negative value, it indicates an indefinite wait cycle.
553
# If t is specified as nil or 0, it indicates no response parsing is required.
554
#
555
# @return [Response]
556
def read_response(t = -1, opts = {})
557
# Return a nil response if timeout is nil or 0
558
return if t.nil? || t == 0
559
560
resp = Response.new
561
resp.max_data = config['read_max_data']
562
563
original_request = opts.fetch(:original_request) { nil }
564
parse_opts = {}
565
unless original_request.nil?
566
parse_opts = { orig_method: original_request.opts['method'] }
567
end
568
569
Timeout.timeout((t < 0) ? nil : t) do
570
rv = nil
571
while (
572
!conn.closed? and
573
rv != Packet::ParseCode::Completed and
574
rv != Packet::ParseCode::Error
575
)
576
577
begin
578
buff = conn.get_once(resp.max_data, 1)
579
rv = resp.parse(buff || '', parse_opts)
580
581
# Handle unexpected disconnects
582
rescue ::Errno::EPIPE, ::EOFError, ::IOError
583
case resp.state
584
when Packet::ParseState::ProcessingHeader
585
resp = nil
586
when Packet::ParseState::ProcessingBody
587
# truncated request, good enough
588
resp.error = :truncated
589
end
590
break
591
end
592
593
# This is a dirty hack for broken HTTP servers
594
next unless rv == Packet::ParseCode::Completed
595
596
rbody = resp.body
597
rbufq = resp.bufq
598
599
rblob = rbody.to_s + rbufq.to_s
600
tries = 0
601
begin
602
# XXX: This doesn't deal with chunked encoding
603
while tries < 1000 and resp.headers['Content-Type'] and resp.headers['Content-Type'].start_with?('text/html') and rblob !~ %r{</html>}i
604
buff = conn.get_once(-1, 0.05)
605
break if !buff
606
607
rblob += buff
608
tries += 1
609
end
610
rescue ::Errno::EPIPE, ::EOFError, ::IOError
611
end
612
613
resp.bufq = ''
614
resp.body = rblob
615
end
616
end
617
618
return resp if !resp
619
620
# As a last minute hack, we check to see if we're dealing with a 100 Continue here.
621
# Most of the time this is handled by the parser via check_100()
622
if resp.proto == '1.1' and resp.code == 100 and !(opts[:skip_100])
623
# Read the real response from the body if we found one
624
# If so, our real response became the body, so we re-parse it.
625
if resp.body.to_s =~ /^HTTP/
626
body = resp.body
627
resp = Response.new
628
resp.max_data = config['read_max_data']
629
resp.parse(body, parse_opts)
630
# We found a 100 Continue but didn't read the real reply yet
631
# Otherwise reread the reply, but don't try this hack again
632
else
633
resp = read_response(t, skip_100: true)
634
end
635
end
636
637
resp
638
rescue Timeout::Error
639
# Allow partial response due to timeout
640
resp if config['partial']
641
end
642
643
#
644
# Cleans up any outstanding connections and other resources.
645
#
646
def stop
647
close
648
end
649
650
#
651
# Returns whether or not the conn is valid.
652
#
653
def conn?
654
conn != nil
655
end
656
657
#
658
# Whether or not connections should be pipelined.
659
#
660
def pipelining?
661
pipeline
662
end
663
664
#
665
# Target host addr and port for this connection
666
#
667
def peerinfo
668
if conn
669
pi = conn.peerinfo || nil
670
if pi
671
return {
672
'addr' => pi.split(':')[0],
673
'port' => pi.split(':')[1].to_i
674
}
675
end
676
end
677
nil
678
end
679
680
#
681
# An optional comm to use for creating the underlying socket.
682
#
683
attr_accessor :comm
684
#
685
# The client request configuration
686
#
687
attr_accessor :config
688
#
689
# The client request configuration classes
690
#
691
attr_accessor :config_types
692
#
693
# Whether or not pipelining is in use.
694
#
695
attr_accessor :pipeline
696
#
697
# The local host of the client.
698
#
699
attr_accessor :local_host
700
#
701
# The local port of the client.
702
#
703
attr_accessor :local_port
704
#
705
# The underlying connection.
706
#
707
attr_accessor :conn
708
#
709
# The calling context to pass to the socket
710
#
711
attr_accessor :context
712
#
713
# The proxy list
714
#
715
attr_accessor :proxies
716
717
# Auth
718
attr_accessor :username, :password, :kerberos_authenticator
719
720
# When parsing the request, thunk off the first response from the server, since junk
721
attr_accessor :junk_pipeline
722
723
# @return [Rex::Proto::Http::HttpSubscriber] The HTTP subscriber
724
attr_accessor :subscriber
725
726
protected
727
728
# https
729
attr_accessor :ssl, :ssl_version # :nodoc:
730
731
attr_accessor :hostname, :port # :nodoc:
732
733
#
734
# The SSL key log file for the connected socket.
735
#
736
# @return [String]
737
attr_accessor :sslkeylogfile
738
739
#
740
# The established NTLM connection info
741
#
742
attr_accessor :ntlm_client
743
744
#
745
# The established kerberos connection info
746
#
747
attr_accessor :krb_encryptor
748
end
749
end
750
end
751
end
752
753