我是使用Ruby和Mechanize的新手，我正在尝试在搜索结果列表中收集唯一链接。一切似乎运行良好，直到脚本运行96次，然后停止，即使queue.length持有96个元素？什么是96号，我错过了什么？
require 'mechanize'
a = Mechanize.new { |agent|
agent.user_agent = 'Linux Mozilla'
agent.user_agent_alias = 'Linux Mozilla'
agent.open_timeout = 3
agent.read_timeout = 4
agent.keep_alive = false
agent.max_history = 0
}
queue = Array.new()
history = Array.new()
collection = Array.new()
domain = 'http://domain.com';
queue.push('http://domain.com/firstpage');
queue.each_with_index {|url, i|
page = a.get(url)
puts "#{i} Visiting " + url
history.push(url)
queue.delete(url)
listings = page.search("a.result")
listings.each {|listing|
collection.push(listing.attributes['href'])
}
steps = page.search("#pagination a")
steps.each {|step|
step = URI.join( domain, step.attributes['href'] ).to_s
unless history.include? step or queue.include? step
queue.push(step)
#puts "New url to visit "+ step
end
}
puts "In queue " + queue.length.to_s
puts "Harvested " + collection.length.to_s
}
0 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Added url to queue
Queue size is now 3
1 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 4
2 Visited url from queue, so removing it from queue
Added url to queue
Queue size is now 4
3 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 5
4 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 6
5 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 7
6 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 8
7 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 9
8 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 10
9 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 11
10 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 12
11 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 13
12 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 14
13 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 15
14 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 16
15 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 17
16 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 18
17 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 19
18 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 20
19 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 21
20 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 22
21 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 23
22 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 24
23 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 25
24 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 26
25 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 27
26 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 28
27 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 29
28 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 30
29 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 31
30 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 32
31 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 33
32 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 34
33 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 35
34 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 36
35 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 37
36 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 38
37 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 39
38 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 40
39 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 41
40 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 42
41 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 43
42 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 44
43 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 45
44 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 46
45 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 47
46 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 48
47 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 49
48 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 50
49 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 51
50 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 52
51 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 53
52 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 54
53 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 55
54 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 56
55 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 57
56 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 58
57 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 59
58 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 60
59 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 61
60 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 62
61 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 63
62 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 64
63 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 65
64 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 66
65 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 67
66 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 68
67 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 69
68 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 70
69 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 71
70 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 72
71 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 73
72 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 74
73 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 75
74 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 76
75 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 77
76 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 78
77 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 79
78 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 80
79 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 81
80 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 82
81 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 83
82 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 84
83 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 85
84 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 86
85 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 87
86 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 88
87 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 89
88 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 90
89 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 91
90 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 92
91 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 93
92 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 94
93 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 95
94 Visited url from queue, so removing it from queue
Added url to queue
Added url to queue
Queue size is now 96
95 Visited url from queue, so removing it from queue
Added url to queue
Queue size is now 96