import aiohttp
import aiohttp.client_exceptions
-from Robots.base import robot_base
from Robots.multi_mixin import multi_mixin
current_href = contextvars.ContextVar('current_href')
-class robot_aio(multi_mixin, robot_base):
+class robot_aio(multi_mixin):
def __init__(self, *args, **kw):
- robot_base.__init__(self, *args, **kw)
multi_mixin.__init__(self, *args, **kw)
# We need one event loop for the entire application
cpu_count = os.cpu_count()
-class concurrent_futures(multi_mixin, robot_base):
+class concurrent_futures(multi_mixin):
# We're I/O-bound, not CPU-bound
max_urls = 2*cpu_count if cpu_count else 10
def __init__(self, *args, **kw):
self.concurrent_class_name = self.concurrent_class.__name__
multi_mixin.__init__(self, *args, **kw)
- robot_base.__init__(self, *args, **kw)
self.executor = self.concurrent_class(max_workers=self.max_urls)
def version_str(self):
__all__ = ['multi_mixin']
-class multi_mixin:
+from Robots.base import robot_base
+
+
+class multi_mixin(robot_base):
max_urls = 10 # Small default
report_checked = False
# maps {URL: [bookmark, saved parent, task]}
self.bookmarks = {}
self.pending = set() # pending tasks
+ robot_base.__init__(self, *args, **kw)
def check_bookmark(self, bookmark):
href = bookmark.href