From 6b3189c2d8c206c2de7c12e15b1a9039f80cfa5c Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Fri, 16 Aug 2024 16:53:31 +0300 Subject: [PATCH] Fix(bkmk_rmultirequests): Limit number of URLs to load into workers --- Robots/bkmk_rmultirequests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Robots/bkmk_rmultirequests.py b/Robots/bkmk_rmultirequests.py index b31c57a..16a3c66 100644 --- a/Robots/bkmk_rmultirequests.py +++ b/Robots/bkmk_rmultirequests.py @@ -61,8 +61,8 @@ class robot_multirequests(robot_requests): bookmarks = self.bookmarks pending = self.pending - process = self.max_workers - len(pending) - if bookmarks and (process > 0): # there's job and free workers + free_workers = self.max_workers - len(pending) + if bookmarks and (free_workers > 0): # there's job and free workers, for href in bookmarks: bookmark, parent, ft = bookmarks[href] if ft is not None: # it's already pending @@ -74,6 +74,10 @@ class robot_multirequests(robot_requests): bookmarks[href] = [bookmark, parent, ft] pending.add(ft) + free_workers -= 1 + if free_workers == 0: + break + if pending: done, pending = concurrent.futures.wait( pending, self.timeout+1, -- 2.39.5