CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!

1
require 'anemone'
2
require 'optparse'
3
require 'ostruct'
4

5
begin
6
  # make sure that the first option is a URL we can crawl
7
  root = URI(ARGV[0])
8
rescue
9
  puts <<-INFO
10
Usage:
11
  anemone serialize [options] <url>
12

13
Synopsis:
14
  Crawls a site starting at the given URL and saves the resulting
15
  PageStore object to a file using Marshal serialization.
16

17
Options:
18
  -o, --output filename      Filename to save PageStore to. Defaults to crawl.{Time.now}
19
INFO
20
  exit(0)
21
end
22

23
options = OpenStruct.new
24
options.output_file = "crawl.#{Time.now.to_i}"
25

26
# parse command-line options
27
opts = OptionParser.new
28
opts.on('-o', '--output filename') {|o| options.output_file = o }
29
opts.parse!(ARGV)
30

31
Anemone.crawl(root) do |anemone|
32
  anemone.after_crawl do |pages|
33
    open(options.output_file, 'w') {|f| Marshal.dump(pages, f)}
34
  end
35
end
36

37