def self.get(uri, options={})
headers = options.clone
headers = {:follow_limit => headers} if Numeric === headers
limit=headers[:follow_limit] ? headers.delete(:follow_limit).to_i : 10
require 'net/http'
uri = URI(uri) unless URI === uri
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true if uri.scheme == 'https'
options.each do |key, value|
http.send "#{key}=", headers.delete(key) if http.respond_to? "#{key}="
end
request = Net::HTTP::Get.new(uri.request_uri)
auth = headers.delete(:basic_auth)
auth ||= [uri.user, uri.password] if uri.user and uri.password
request.basic_auth auth.first, auth.last if auth
headers.each {|key, value| request[key.to_s] = value.to_s}
response = http.request(request)
case response
when Net::HTTPSuccess
doc = parse(reencode(response.body, response['content-type']), options)
doc.instance_variable_set('@response', response)
doc.class.send(:attr_reader, :response)
doc
when Net::HTTPRedirection
response.value if limit <= 1
location = URI.join(uri, response['location'])
get(location, options.merge(:follow_limit => limit-1))
else
response.value
end
end