# File lib/robots.rb, line 101 def self.get_robots_txt(uri) begin Timeout::timeout(Robots.timeout) do io = URI.join(uri.to_s, "/robots.txt").open("User-Agent" => user_agent) rescue nil end rescue Timeout::Error STDERR.puts "robots.txt request timed out" end end