crawlinx.rb

   1 #!/usr/bin/ruby1.8
   2
   3 require 'net/http'
   4 require 'uri'
   5 require 'rexml/document'
   6
   7
   8 initial_url = URI.parse ARGV[0]
   9 limit_url = URI.parse ARGV[1]
  10
  11 urls = Array.new
  12 urls << initial_url
  13
  14 begin
  15         current_url = urls.shift
  16         current_url.route_from(limit_url)
  17
  18         begin
  19         page = REXML::Document.new(Net::HTTP.get(current_url))
  20
  21         rescue REXML::ParseException
  22                 puts "erreur XML dans " + current_url
  23                 next
  24         end
  25
  26         # extract urls
  27         page.elements.each('//a[@href]') do |anchor|
  28                 url = URI.parse(anchor.attributes['href'])
  29                 if url.relative?
  30                 then
  31                         url = current_url + url
  32                 end
  33
  34                 urls << url
  35         end
  36
  37         urls.uniq!
  38
  39         # print urls
  40         urls.each do |url|
  41                 puts url
  42         end
  43
  44 end while not urls.empty?