- error, r = request_get(url, self.timeout, None)
- if error is not None:
- self.log(' Error: %s' % error)
- if self.proxy:
- self.log(' Retrying with the proxy...')
- error, r = request_get(url, self.timeout, self.proxy)
+ split_results = urlsplit(url)
+ url_host = split_results.hostname
+
+ if url_host in self.proxy_error:
+ return 'proxy error', None, None, None, None
+
+ if url_host in self.proxy_ok:
+ self.log(' Immediately trying with the proxy')
+ error, r = request_get(url, self.timeout, self.proxy)
+ else:
+ error, r = request_get(url, self.timeout, None)
+ if error is not None:
+ self.log(' Error: %s' % error)
+ if self.proxy:
+ self.log(' Retrying with the proxy...')
+ error, r = request_get(url, self.timeout, self.proxy)
+ if error is None:
+ self.proxy_ok.add(url_host)