Class | Yapra::Plugin::Publish::OnMemoryDownload |
In: |
lib-plugins/yapra/plugin/publish/on_memory_download.rb
|
Parent: | Yapra::Plugin::MechanizeBase |
download web resource and set to item attribute.
example:
- module: Publish::OnMemoryDownload config: regexp: http://www\.yahoo\.co\.jp/* attribute: binary_image before_hook: "agent.get('http://www.yahoo.co.jp/'); sleep 1" url: attribute: link replace: index(\d*?)\.html to: file\1.zip referrer: attribute: link replace: 'zip' to: 'html'
# File lib-plugins/yapra/plugin/publish/on_memory_download.rb, line 25 25: def run(data) 26: regexp = nil 27: if config['regexp'] 28: regexp = Regexp.new(config['regexp']) 29: else 30: regexp = /^(https?|ftp)(:\/\/[-_.!~*\'()a-zA-Z0-9;\/?:\@&=+\$,%#]+)$/ 31: end 32: 33: wait = config['wait'] || 3 34: 35: data.each do |item| 36: url = construct_data(config['url'], item, item.respond_to?('link') ? item.link : item) 37: 38: if regexp =~ url 39: logger.debug "Download start: #{url}" 40: referrer = construct_data(config['referrer'], item) 41: download(item, url, referrer) 42: sleep wait 43: end 44: end 45: 46: data 47: end
# File lib-plugins/yapra/plugin/publish/on_memory_download.rb, line 50 50: def construct_data(config, item, original=nil) 51: if config && config['attribute'] 52: if item.respond_to?(config['attribute']) 53: original = item.__send__(config['attribute']) 54: end 55: elsif config 56: original = item 57: end 58: 59: if original && config && config['replace'] 60: original = original.gsub(config['replace'], config['to'] || Time.now.to_s) 61: end 62: 63: original 64: end
# File lib-plugins/yapra/plugin/publish/on_memory_download.rb, line 70 70: def download(item, url, referrer) 71: if config['before_hook'] 72: eval(config['before_hook']) 73: end 74: 75: dir = config['dir'] 76: 77: page = agent.get(url, referrer) 78: 79: save(config, item, page) 80: 81: if config['after_hook'] 82: eval(config['after_hook']) 83: end 84: end