--proxy

Author: gurgeous

README.md

@@ -175,8 +175,9 @@ noko.css("table")[4].css("td").select { |i| i[:width].to_i > 80 }.map(&:text)
 
 #### (unreleased)
 
-* Added support for `--limit` and `xml`
+* Support for `--limit`, `--proxy` and the `xml` variable
 * Dedup - warn and ignore if row[:url] has already been emitted
+* Auto gunzip if contents are compressed
 
 #### 2.0.1 (May 2018)
 

bin/sinew

@@ -15,6 +15,7 @@ options = Slop.parse do |o|
   o.bool '-q', '--quiet', 'suppress some output'
   o.integer '-l', '--limit', 'quit after emitting this many rows'
   o.string '-c', '--cache', 'set custom cache directory', default: "#{ENV['HOME']}/.sinew"
+  o.string '--proxy', 'use host[:port] as HTTP proxy'
   o.bool '--version', 'show version and exit'
   o.on('--help', 'show this help') do
     puts o

lib/sinew/main.rb

@@ -15,6 +15,12 @@ def initialize(options)
       @runtime_options = RuntimeOptions.new
       @request_tm = Time.at(0)
       @request_count = 0
+
+      if options[:proxy]
+        addr, port = options[:proxy].split(':')
+        runtime_options.httparty_options[:http_proxyaddr] = addr
+        runtime_options.httparty_options[:http_proxyport] = port || 80
+      end
     end
 
     def run

lib/sinew/request.rb

@@ -28,7 +28,10 @@ def initialize(sinew, method, url, options = {})
     def perform
       validate!
 
-      # merge global/options headers
+      # merge optons
+      options = self.options.merge(sinew.runtime_options.httparty_options)
+
+      # merge headers
       headers = sinew.runtime_options.headers
       headers = headers.merge(options[:headers]) if options[:headers]
       options[:headers] = headers

lib/sinew/runtime_options.rb

@@ -7,6 +7,7 @@ class RuntimeOptions
     attr_accessor :retries
     attr_accessor :rate_limit
     attr_accessor :headers
+    attr_accessor :httparty_options
     attr_accessor :before_generate_cache_key
 
     def initialize
@@ -15,6 +16,7 @@ def initialize
       self.headers = {
         'User-Agent' => "sinew/#{VERSION}",
       }
+      self.httparty_options = {}
       self.before_generate_cache_key = ->(i) { i }
 
       # for testing

sample.sinew

@@ -1 +1,9 @@
-get 'https://postmates.com/sitemap/sitemap_all_merchants_latest.xml.gz'
+get 'http://httpbin.org'
+noko.css('ul li a').each do |a|
+  row = {}
+  row[:url] = a[:href]
+  row[:title] = a.text
+  csv_emit(row)
+end
+
+get 'http://httpbin.org/redirect/2'