CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!

1
require 'anemone/http'
2

3
module Anemone
4
  class Tentacle
5

6
    #
7
    # Create a new Tentacle
8
    #
9
    def initialize(link_queue, page_queue, opts = {})
10
      @link_queue = link_queue
11
      @page_queue = page_queue
12
      @http = Anemone::HTTP.new(opts)
13
      @opts = opts
14
    end
15

16
    #
17
    # Gets links from @link_queue, and returns the fetched
18
    # Page objects into @page_queue
19
    #
20
    def run
21
      loop do
22
        link, referer, depth = @link_queue.deq
23

24
        break if link == :END
25

26
        @http.fetch_pages(link, referer, depth).each { |page| @page_queue << page }
27

28
        delay
29
      end
30
    end
31

32
    private
33

34
    def delay
35
      sleep @opts[:delay] if @opts[:delay] > 0
36
    end
37

38
  end
39
end
40

41