def self.fetch_title_data(uri)
return unless uri
key = %w{ plugins appendtitle title-data}.push(Digest::SHA1.hexdigest(uri)).join('-')
if v = memory_cache.get(key)
logger.debug "appendtitle: cache hit for #{uri}"
return v
end
memory_cache.set(key, {}, config.plugins.appendtitle.cache_expire)
logger.debug "appendtitle: fetching title for #{uri}"
data = {}
uri_fetch = uri
begin
io = URI.parse(uri_fetch).read
base_uri = io.base_uri.to_s
base_uri = uri_fetch if base_uri.length > 1000
data[:uri] = base_uri
charset = io.scan(/charset="?([^\s"]*)/i).flatten.inject(Hash.new{0}){|a, b| a[b]+=1; a}.to_a.sort_by{|a|a[1]}.reverse.first[0]
begin
source = Nokogiri(io, base_uri, charset)
title = source.at('title').text rescue nil
title ||= source.at('h1').text rescue nil
title ||= source.at('h2').text rescue nil
title = title.gsub(/\n/, '').gsub(/\s+/, ' ') if title
data[:title] = title if title
rescue
end
memory_cache.set(key, data, config.plugins.appendtitle.cache_expire)
data
rescue RuntimeError => error
if error.message =~ /^redirection forbidden/
logger.debug "appendtitle: #{error.message}"
uri_fetch = error.message.split(/\s+/).last
retry
end
rescue Timeout::Error, StandardError => error
logger.debug "appendtitle: error #{uri}, #{error.class.to_s}: #{error.message}"
nil
end
end