From 9f28b791b4f1bbba95e6b5ce4e7d430308a40170 Mon Sep 17 00:00:00 2001 From: danielhuici Date: Tue, 29 Oct 2024 19:54:17 +0100 Subject: [PATCH] Fix: bot not notifying new articles + minor refactor --- item_monitor.py => datalayer/item_monitor.py | 1 - article.py => datalayer/wallapop_article.py | 9 +++---- main.py | 25 +++++++++---------- .../telegram_manager.py | 5 +--- worker.py | 19 ++++++++------ 5 files changed, 28 insertions(+), 31 deletions(-) rename item_monitor.py => datalayer/item_monitor.py (99%) rename article.py => datalayer/wallapop_article.py (86%) rename telegram_handler.py => managers/telegram_manager.py (96%) diff --git a/item_monitor.py b/datalayer/item_monitor.py similarity index 99% rename from item_monitor.py rename to datalayer/item_monitor.py index 5ddb563..a3ddd87 100644 --- a/item_monitor.py +++ b/datalayer/item_monitor.py @@ -1,4 +1,3 @@ -import string class ItemMonitor: def __init__(self, search_query, latitude, longitude, max_distance, diff --git a/article.py b/datalayer/wallapop_article.py similarity index 86% rename from article.py rename to datalayer/wallapop_article.py index 27f53a0..8fac892 100644 --- a/article.py +++ b/datalayer/wallapop_article.py @@ -1,6 +1,5 @@ - -class Article: +class WallapopArticle: def __init__(self, id, title, description, price, currency, location, allows_shipping, url): self._id = id self._title = title @@ -48,9 +47,9 @@ class Article: def get_url(self): return self._url - def __eq__(self, article2): - return self.get_id() == article2.get_id() + def __eq__(self, article): + return self.get_id() == article.get_id() def __str__(self): - return f"Article(id={self._id}, title='{self._title}', description='{self._description}', " \ + return f"Article(id={self._id}, title='{self._title}', " \ f"price={self._price} {self._currency}, url='{self._url}')" \ No newline at end of file diff --git a/main.py b/main.py index 24eeef0..b827553 100644 --- a/main.py +++ b/main.py @@ -2,22 +2,21 @@ import json import threading import logging from logging.handlers import RotatingFileHandler -from item_monitor import ItemMonitor +from datalayer.item_monitor import ItemMonitor from worker import Worker -# Configure the console logger -console_handler = logging.StreamHandler() -console_handler.setLevel(logging.INFO) -console_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) +def configure_logger(): + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) -# Configure the file logger -file_handler = RotatingFileHandler('monitor.log', maxBytes=10e6) -file_handler.setLevel(logging.DEBUG) -file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) + file_handler = RotatingFileHandler('monitor.log', maxBytes=10e6) + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) -# Configure the root logger with both handlers -logging.basicConfig(level=logging.NOTSET, - handlers=[console_handler, file_handler]) + # Configure the root logger with both handlers + logging.basicConfig(level=logging.NOTSET, + handlers=[console_handler, file_handler]) def parse_items_to_monitor(): with open("args.json") as f: @@ -26,7 +25,7 @@ def parse_items_to_monitor(): return items if __name__ == "__main__": - logger = logging.getLogger(__name__) + configure_logger() items = parse_items_to_monitor() for item in items: diff --git a/telegram_handler.py b/managers/telegram_manager.py similarity index 96% rename from telegram_handler.py rename to managers/telegram_manager.py index 5143a4a..6d7a2df 100644 --- a/telegram_handler.py +++ b/managers/telegram_manager.py @@ -1,11 +1,8 @@ -from article import Article import asyncio -import threading import yaml import telegram import re - ITEM_TEXT = "- *Artículo*: {}\n" \ "- *Descripción*: {}\n" \ "- *Localidad*: {}\n" \ @@ -14,7 +11,7 @@ ITEM_TEXT = "- *Artículo*: {}\n" \ "[Ir al anuncio](https://es.wallapop.com/item/{})" -class TelegramHandler: +class TelegramManager: def __init__(self): token, channel = self.get_config() self._channel = channel diff --git a/worker.py b/worker.py index 79da299..30d4809 100644 --- a/worker.py +++ b/worker.py @@ -1,10 +1,9 @@ import time import requests import logging -from article import Article -from telegram_handler import TelegramHandler +from datalayer.wallapop_article import WallapopArticle +from managers.telegram_manager import TelegramManager import traceback -import asyncio REQUEST_SLEEP_TIME = 10 REQUEST_RETRY_TIME = 3 @@ -16,8 +15,8 @@ class Worker: def __init__(self, item_to_monitor): self.logger = logging.getLogger(__name__) self._item_monitoring = item_to_monitor - self._notified_articles = self._request_articles() - self._telegram_handler = TelegramHandler() + self._notified_articles = self._request_articles()[1:] + self.telegram_manager = TelegramManager() def _request_articles(self): url = ( @@ -35,7 +34,11 @@ class Worker: while True: try: - response = requests.get(url, headers={'User-Agent': USER_AGENT}) + headers = { + 'X-DeviceOS': '0', + 'User-Agent': USER_AGENT + } + response = requests.get(url, headers=headers) response.raise_for_status() break except requests.exceptions.RequestException as err: @@ -49,7 +52,7 @@ class Worker: def _parse_json_response(self, json_response): articles = [] for json_article in json_response: - articles.append(Article.load_from_json(json_article)) + articles.append(WallapopArticle.load_from_json(json_article)) return articles def _has_words(self, text, word_list): @@ -103,7 +106,7 @@ class Worker: for article in articles: if self._meets_item_conditions(article): try: - self._telegram_handler.send_telegram_article(article) + self.telegram_manager.send_telegram_article(article) except Exception as e: self.logger.error(f"{self._item_monitoring.get_search_query()} worker crashed: {e}") self._notified_articles.insert(0, article)