CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!

1
require 'net/https'
2
require 'anemone/page'
3
require 'anemone/cookie_store'
4

5
module Anemone
6
  class HTTP
7
    # Maximum number of redirects to follow on each get_response
8
    REDIRECT_LIMIT = 5
9

10
    # CookieStore for this HTTP client
11
    attr_reader :cookie_store
12

13
    def initialize(opts = {})
14
      @connections = {}
15
      @opts = opts
16
      @cookie_store = CookieStore.new(@opts[:cookies])
17
    end
18

19
    #
20
    # Fetch a single Page from the response of an HTTP request to *url*.
21
    # Just gets the final destination page.
22
    #
23
    def fetch_page(url, referer = nil, depth = nil)
24
      fetch_pages(url, referer, depth).last
25
    end
26

27
    #
28
    # Create new Pages from the response of an HTTP request to *url*,
29
    # including redirects
30
    #
31
    def fetch_pages(url, referer = nil, depth = nil)
32
      begin
33
        url = URI(url) unless url.is_a?(URI)
34
        pages = []
35
        get(url, referer) do |response, code, location, redirect_to, response_time|
36
          pages << Page.new(location, :body => response.body.dup,
37
                                      :code => code,
38
                                      :headers => response.to_hash,
39
                                      :referer => referer,
40
                                      :depth => depth,
41
                                      :redirect_to => redirect_to,
42
                                      :response_time => response_time)
43
        end
44

45
        return pages
46
      rescue => e
47
        if verbose?
48
          puts e.inspect
49
          puts e.backtrace
50
        end
51
        return [Page.new(url, :error => e)]
52
      end
53
    end
54

55
    #
56
    # The maximum number of redirects to follow
57
    #
58
    def redirect_limit
59
      @opts[:redirect_limit] || REDIRECT_LIMIT
60
    end
61

62
    #
63
    # The user-agent string which will be sent with each request,
64
    # or nil if no such option is set
65
    #
66
    def user_agent
67
      @opts[:user_agent]
68
    end
69

70
    #
71
    # Does this HTTP client accept cookies from the server?
72
    #
73
    def accept_cookies?
74
      @opts[:accept_cookies]
75
    end
76

77
    private
78

79
    #
80
    # Retrieve HTTP responses for *url*, including redirects.
81
    # Yields the response object, response code, and URI location
82
    # for each response.
83
    #
84
    def get(url, referer = nil)
85
      limit = redirect_limit
86
      loc = url
87
      begin
88
          # if redirected to a relative url, merge it with the host of the original
89
          # request url
90
          loc = url.merge(loc) if loc.relative?
91

92
          response, response_time = get_response(loc, referer)
93
          code = Integer(response.code)
94
          redirect_to = response.is_a?(Net::HTTPRedirection) ?  URI(response['location']).normalize : nil
95
          yield response, code, loc, redirect_to, response_time
96
          limit -= 1
97
      end while (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
98
    end
99

100
    #
101
    # Get an HTTPResponse for *url*, sending the appropriate User-Agent string
102
    #
103
    # MODIFIED: Change get_response to allow fine tuning of the HTTP request before
104
    #           it is sent to the remote system.
105
    #
106
    def get_response(url, referer = nil)
107
      full_path = url.query.nil? ? url.path : "#{url.path}?#{url.query}"
108

109
      opts = {}
110
      opts['User-Agent'] = user_agent if user_agent
111
      opts['Referer'] = referer.to_s if referer
112
      opts['Cookie'] = @cookie_store.to_s unless @cookie_store.empty? || (!accept_cookies? && @opts[:cookies].nil?)
113
      
114
      if @opts[:http_basic_auth]
115
      	opts['Authorization'] = "Basic " + @opts[:http_basic_auth]
116
      end
117

118
      if not @opts[:inject_headers].nil? 
119
      	@opts[:inject_headers].each do |hdr|
120
      	  k,v = hdr.split(':', 2)
121
      	  opts[k] = v
122
        end
123
      end
124
      
125
      retries = 0
126
      begin
127
        start = Time.now()
128
        response = nil
129
        if @opts[:request_factory]
130
          response = @opts[:request_factory].call(connection(url), full_path, opts)
131
        else
132
          response = connection(url).get(full_path, opts)
133
        end
134
        finish = Time.now()
135
        response_time = ((finish - start) * 1000).round
136
        @cookie_store.merge!(response['Set-Cookie']) if accept_cookies?
137
        return response, response_time
138
      rescue EOFError
139
        refresh_connection(url)
140
        retries += 1
141
        retry unless retries > (@opts[:retry_limit] || 3)
142
      end
143
    end
144

145
    def connection(url)
146
      @connections[url.host] ||= {}
147

148
      if conn = @connections[url.host][url.port]
149
        return conn
150
      end
151

152
      refresh_connection url
153
    end
154

155
    #
156
    # MODIFIED: Change refresh_connection to allow a HTTP factory to be used to
157
    #           create the Net::HTTP object. This allows much more granular
158
    #           control over the requests.
159
    #
160
    def refresh_connection(url)
161
      http = nil
162
      if @opts[:http_factory]
163
        http = @opts[:http_factory].call(url)
164
      else
165
        http = Net::HTTP.new(url.host, url.port)
166
        if url.scheme == 'https'
167
          http.use_ssl = true
168
          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
169
        end
170
      end
171
      @connections[url.host][url.port] = http.start
172
    end
173

174
    def verbose?
175
      @opts[:verbose]
176
    end
177

178
    #
179
    # Allowed to connect to the requested url?
180
    #
181
    def allowed?(to_url, from_url)
182
      to_url.host.nil? || (to_url.host == from_url.host)
183
    end
184

185
  end
186
end
187

188