CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!

1
require 'rex'
2
require 'anemone/page'
3
require 'anemone/cookie_store'
4

5

6
#
7
# This is an alternate Anemone::HTTP implementation that uses the Metasploit Rex
8
# library and the Rex::Proto::Http protocol stack.
9
#
10

11
module Anemone
12
  class HTTP
13
    # Maximum number of redirects to follow on each get_response
14
    # REDIRECT_LIMIT = 5
15

16
    # CookieStore for this HTTP client
17
    attr_reader :cookie_store
18

19
    def initialize(opts = {})
20
      @connections = {}
21
      @opts = opts
22
      @cookie_store = CookieStore.new(@opts[:cookies])
23
    end
24

25
    #
26
    # Fetch a single Page from the response of an HTTP request to *url*.
27
    # Just gets the final destination page.
28
    #
29
    def fetch_page(url, referer = nil, depth = nil)
30
      fetch_pages(url, referer, depth).last
31
    end
32

33
    #
34
    # Create new Pages from the response of an HTTP request to *url*,
35
    # including redirects
36
    #
37
    def fetch_pages(url, referer = nil, depth = nil)
38
      begin
39
        url = URI(url) unless url.is_a?(URI)
40
        pages = []
41
        get(url, referer) do |response, code, location, redirect_to, response_time|
42

43
          page = Page.new(location, :body => response.body.dup,
44
                                      :code => code,
45
                                      :headers => response.headers,
46
                                      :referer => referer,
47
                                      :depth => depth,
48
                                      :redirect_to => redirect_to,
49
                                      :response_time => response_time,
50
                                      :dirbust => @opts[:dirbust]
51
          )
52
          # Store the associated raw HTTP request
53
          page.request = response.request
54
      pages << page
55
        end
56

57
        return pages
58
      rescue => e
59
        if verbose?
60
          puts e.inspect
61
          puts e.backtrace
62
        end
63
        return [Page.new(url, :error => e)]
64
      end
65
    end
66

67
    #
68
    # The maximum number of redirects to follow
69
    #
70
    def redirect_limit
71
      @opts[:redirect_limit] || REDIRECT_LIMIT
72
    end
73

74
    #
75
    # The user-agent string which will be sent with each request,
76
    # or nil if no such option is set
77
    #
78
    def user_agent
79
      @opts[:user_agent]
80
    end
81

82
    #
83
    # The virtual host to override the host header with, per url
84
    # TODO: implement
85
    #
86
    def virtual_host(url)
87
      url.host
88
    end
89

90
    #
91
    # Does this HTTP client accept cookies from the server?
92
    #
93
    def accept_cookies?
94
      @opts[:accept_cookies]
95
    end
96

97
    private
98

99
    #
100
    # Retrieve HTTP responses for *url*, including redirects.
101
    # Yields the response object, response code, and URI location
102
    # for each response.
103
    #
104
    def get(url, referer = nil)
105
      limit = redirect_limit
106
      loc = url
107
      begin
108
          # if redirected to a relative url, merge it with the host of the original
109
          # request url
110
          loc = url.merge(loc) if loc.relative?
111

112
          response, response_time = get_response(loc, referer)
113
          code = response.code.to_i
114

115
          redirect_to = nil
116
          if code >= 300 and code <= 310
117
          	redirect_to = URI(response['location']).normalize
118
          end
119

120
          yield response, code, loc, redirect_to, response_time
121

122

123
          limit -= 1
124
      end while (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
125
    end
126

127
    #
128
    # Get an HTTPResponse for *url*, sending the appropriate User-Agent string
129
    #
130
    # MODIFIED: Change get_response to allow fine tuning of the HTTP request before
131
    #           it is sent to the remote system.
132
    #
133
    def get_response(url, referer = nil)
134
      opts = {
135
      	'uri'   => url.path,
136
        'query' => url.query
137
      }
138

139
      opts['agent']   = user_agent if user_agent
140
      opts['cookie']  = @cookie_store.to_s unless @cookie_store.empty? || (!accept_cookies? && @opts[:cookies].nil?)
141

142
      head = {}
143
      if referer
144
      	head['Referer'] = referer.to_s
145
      end
146

147
      if @opts[:http_basic_auth]
148
      	head['Authorization'] = "Basic " + @opts[:http_basic_auth]
149
      end
150

151
      @opts[:inject_headers].each do |hdr|
152
      	k,v = hdr.split(':', 2)
153
      	head[k] = v
154
      end
155

156
      opts['headers'] = head
157

158
      retries = 0
159
      begin
160
        start = Time.now()
161

162
        response = nil
163
        request  = nil
164
        begin
165
      conn     = connection(url)
166
      request  = conn.request_raw(opts)
167
      response = conn.send_recv(request, @opts[:timeout] || 10 )
168
    rescue ::Errno::EPIPE, ::Timeout::Error
169
    end
170

171
        finish = Time.now()
172

173
        response_time = ((finish - start) * 1000).round
174
        @cookie_store.merge!(response['Set-Cookie']) if accept_cookies?
175
        return response, response_time
176
      rescue EOFError
177
        retries += 1
178
        retry unless retries > (@opts[:retry_limit] || 3)
179
      end
180
    end
181

182
    def connection(url)
183
    context =  { }
184
    context['Msf']        = @opts[:framework] if @opts[:framework]
185
    context['MsfExploit'] = @opts[:module] if @opts[:module]
186

187
    conn = Rex::Proto::Http::Client.new(
188
      url.host,
189
      url.port.to_i,
190
      context,
191
      url.scheme == "https",
192
      @opts[:ssl_version],
193
      @opts[:proxies],
194
                    @opts[:username],
195
                    @opts[:password]
196
    )
197

198
    conn.set_config(
199
      'vhost'      => virtual_host(url),
200
      'agent'      => user_agent,
201
      'domain'     => @opts[:domain]
202
    )
203

204
    conn
205
    end
206

207
    def verbose?
208
      @opts[:verbose]
209
    end
210

211
    #
212
    # Allowed to connect to the requested url?
213
    #
214
    def allowed?(to_url, from_url)
215
      to_url.host.nil? || (to_url.host == from_url.host)
216
    end
217

218
  end
219
end
220

221