Fix(bkmk_rmultirequests): Limit number of URLs to load into workers

diff --git a/Robots/bkmk_rmultirequests.py b/Robots/bkmk_rmultirequests.py
index b31c57a2b8f9c8736b877e53b83ee49dc26cc008..16a3c669da77daa63c2de770156341ff31710fcc 100644 (file)
--- a/Robots/bkmk_rmultirequests.py
+++ b/Robots/bkmk_rmultirequests.py
@@ -61,8 +61,8 @@ class robot_multirequests(robot_requests):
         bookmarks = self.bookmarks
         pending = self.pending
 
-        process = self.max_workers - len(pending)
-        if bookmarks and (process > 0):  # there's job and free workers
+        free_workers = self.max_workers - len(pending)
+        if bookmarks and (free_workers > 0):  # there's job and free workers,
             for href in bookmarks:
                 bookmark, parent, ft = bookmarks[href]
                 if ft is not None:  # it's already pending
@@ -74,6 +74,10 @@ class robot_multirequests(robot_requests):
                 bookmarks[href] = [bookmark, parent, ft]
                 pending.add(ft)
 
+                free_workers -= 1
+                if free_workers == 0:
+                    break
+
         if pending:
             done, pending = concurrent.futures.wait(
                 pending, self.timeout+1,