def self.fetch_raw(urls, options = {})
url_queue = [*urls]
multi = Curl::Multi.new
responses = {}
url_queue.each do |url|
easy = Curl::Easy.new(url) do |curl|
curl.headers["User-Agent"] = (options[:user_agent] || USER_AGENT)
curl.headers["If-Modified-Since"] = options[:if_modified_since].httpdate if options.has_key?(:if_modified_since)
curl.headers["If-None-Match"] = options[:if_none_match] if options.has_key?(:if_none_match)
curl.headers["Accept-encoding"] = 'gzip, deflate' if options.has_key?(:compress)
curl.follow_location = true
curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
curl.max_redirects = options[:max_redirects] if options[:max_redirects]
curl.timeout = options[:timeout] if options[:timeout]
curl.on_success do |c|
responses[url] = decode_content(c)
end
curl.on_failure do |c, err|
responses[url] = c.response_code
end
end
multi.add(easy)
end
multi.perform
urls.is_a?(String) ? responses.values.first : responses
end