- if r.is_redirect:
- return None, r.status_code, r.next.url, None, None
- return None, None, None, r.headers, r.content
+ headers = request_headers
+
+ if url_host in self.proxy_ok:
+ self.log(' Immediately trying with the proxy')
+ error, r = request_get(url, headers, self.timeout, self.proxy)
+ else:
+ error, r = request_get(url, headers, self.timeout, None)
+ if error is not None:
+ self.log(' Error : %s' % error)
+ if self.proxy and error != '404 not_found':
+ self.log(' Retrying with the proxy...')
+ error, r = request_get(url, headers,
+ self.timeout, self.proxy)
+ if error is None:
+ self.proxy_ok.add(url_host)
+ if error is not None:
+ if self.proxy and error != '404 not_found':
+ self.log(' Proxy error : %s' % error)
+ if url_host not in self.proxy_ok:
+ self.proxy_error.add(url_host)
+ return error, None, None, None, None
+ if r.is_redirect:
+ return None, r.status_code, r.next.url, None, None
+ return None, None, None, r.headers, r.content