public/wallabicher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactoring

Browse Source
This commit is contained in:
danielhuici
2024-10-29 20:42:56 +01:00
parent ab8b506ea2
commit d366222c27
2 changed files with 15 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
worker.py → managers/worker.py
View File

@@ -5,9 +5,9 @@ from datalayer.wallapop_article import WallapopArticle
from managers.telegram_manager import TelegramManager from managers.telegram_manager import TelegramManager
import traceback import traceback
REQUEST_SLEEP_TIME = 10 REQUEST_SLEEP_TIME = 15
REQUEST_RETRY_TIME = 3 REQUEST_RETRY_TIME = 3
ERROR_SLEEP_TIME = 10 ERROR_SLEEP_TIME = 30
NOTIFIED_ARTICLES_LIMIT = 300 NOTIFIED_ARTICLES_LIMIT = 300
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36' USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36'
@@ -15,7 +15,7 @@ class Worker:
def __init__(self, item_to_monitor): def __init__(self, item_to_monitor):
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self._item_monitoring = item_to_monitor self._item_monitoring = item_to_monitor
self._notified_articles = self._request_articles()[1:] self._notified_articles = self._request_articles()
self.telegram_manager = TelegramManager() self.telegram_manager = TelegramManager()
def _request_articles(self): def _request_articles(self):
@@ -103,25 +103,28 @@ class Worker:
while True: while True:
start_time = time.time() start_time = time.time()
articles = self._request_articles() articles = self._request_articles()
new_articles = 0
for article in articles: for article in articles:
if self._meets_item_conditions(article): if self._meets_item_conditions(article):
try: try:
self.telegram_manager.send_telegram_article(article) self.telegram_manager.send_telegram_article(article)
new_articles += 1
except Exception as e: except Exception as e:
self.logger.error(f"{self._item_monitoring.get_search_query()} worker crashed: {e}") self.logger.error(f"{self._item_monitoring.get_search_query()} worker crashed: {e}")
self._notified_articles.insert(0, article) self._notified_articles.insert(0, article)
self._notified_articles = self._notified_articles[:NOTIFIED_ARTICLES_LIMIT] self._notified_articles = self._notified_articles[:NOTIFIED_ARTICLES_LIMIT]
time.sleep(REQUEST_SLEEP_TIME) time.sleep(REQUEST_SLEEP_TIME)
exec_times.append(time.time() - start_time) exec_times.append(time.time() - start_time)
self.logger.debug(f"\'{self._item_monitoring.get_search_query()}\' node-> last: {exec_times[-1]}" self.logger.info(
f" max: {max(exec_times)} avg: {sum(exec_times) / len(exec_times)}") f"Worker '{self._item_monitoring.get_search_query()}': {new_articles} new articles found. "
f"Execution time stats - Last: {exec_times[-1]:.2f}s, Max: {max(exec_times):.2f}s, "
f"Average: {sum(exec_times) / len(exec_times):.2f}s."
)
def run(self): def run(self):
while True: while True:
try: try:
self.logger.info(f"Wallapop monitor worker started. Checking for " self.logger.info(f"Wallapop monitor worker started - {self._item_monitoring.get_search_query()}")
f"new items containing '{self._item_monitoring.get_search_query()}' "
f"with given parameters periodically")
self.work() self.work()
except Exception as e: except Exception as e:
self.logger.error(f"{''.join(traceback.format_exception(None, e, e.__traceback__))}") self.logger.error(f"{''.join(traceback.format_exception(None, e, e.__traceback__))}")

7
main.py → wallamonitor.py
View File

@@ -4,13 +4,14 @@ from logging.handlers import RotatingFileHandler
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from datalayer.item_monitor import ItemMonitor from datalayer.item_monitor import ItemMonitor
from worker import Worker from managers.worker import Worker
from managers.telegram_manager import TelegramManager
def configure_logger(): def configure_logger():
logging.getLogger("httpx").setLevel(logging.WARNING)
console_handler = logging.StreamHandler() console_handler = logging.StreamHandler()
console_handler.setLevel(logging.INFO) console_handler.setLevel(logging.INFO)
console_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) console_handler.setFormatter(logging.Formatter('%(levelname)s [%(asctime)s] %(name)s - %(message)s'))
file_handler = RotatingFileHandler('monitor.log', maxBytes=10e6) file_handler = RotatingFileHandler('monitor.log', maxBytes=10e6)
file_handler.setLevel(logging.DEBUG) file_handler.setLevel(logging.DEBUG)