public/wallabicher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix: bot not notifying new articles + minor refactor

Browse Source
This commit is contained in:
danielhuici
2024-10-29 19:54:17 +01:00
parent eba5f90920
commit 9f28b791b4
5 changed files with 28 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
item_monitor.py → datalayer/item_monitor.py
View File

@@ -1,4 +1,3 @@
import string
class ItemMonitor: class ItemMonitor:
def __init__(self, search_query, latitude, longitude, max_distance, def __init__(self, search_query, latitude, longitude, max_distance,

9
article.py → datalayer/wallapop_article.py
View File

@@ -1,6 +1,5 @@
class WallapopArticle:
class Article:
def __init__(self, id, title, description, price, currency, location, allows_shipping, url): def __init__(self, id, title, description, price, currency, location, allows_shipping, url):
self._id = id self._id = id
self._title = title self._title = title
@@ -48,9 +47,9 @@ class Article:
def get_url(self): def get_url(self):
return self._url return self._url
def __eq__(self, article2): def __eq__(self, article):
return self.get_id() == article2.get_id() return self.get_id() == article.get_id()
def __str__(self): def __str__(self):
return f"Article(id={self._id}, title='{self._title}', description='{self._description}', " \ return f"Article(id={self._id}, title='{self._title}', " \
f"price={self._price} {self._currency}, url='{self._url}')" f"price={self._price} {self._currency}, url='{self._url}')"

25
main.py
View File

@@ -2,22 +2,21 @@ import json
import threading import threading
import logging import logging
from logging.handlers import RotatingFileHandler from logging.handlers import RotatingFileHandler
from item_monitor import ItemMonitor from datalayer.item_monitor import ItemMonitor
from worker import Worker from worker import Worker
# Configure the console logger def configure_logger():
console_handler = logging.StreamHandler() console_handler = logging.StreamHandler()
console_handler.setLevel(logging.INFO) console_handler.setLevel(logging.INFO)
console_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) console_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
# Configure the file logger file_handler = RotatingFileHandler('monitor.log', maxBytes=10e6)
file_handler = RotatingFileHandler('monitor.log', maxBytes=10e6) file_handler.setLevel(logging.DEBUG)
file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
# Configure the root logger with both handlers # Configure the root logger with both handlers
logging.basicConfig(level=logging.NOTSET, logging.basicConfig(level=logging.NOTSET,
handlers=[console_handler, file_handler]) handlers=[console_handler, file_handler])
def parse_items_to_monitor(): def parse_items_to_monitor():
with open("args.json") as f: with open("args.json") as f:
@@ -26,7 +25,7 @@ def parse_items_to_monitor():
return items return items
if __name__ == "__main__": if __name__ == "__main__":
logger = logging.getLogger(__name__) configure_logger()
items = parse_items_to_monitor() items = parse_items_to_monitor()
for item in items: for item in items:

5
telegram_handler.py → managers/telegram_manager.py
View File

@@ -1,11 +1,8 @@
from article import Article
import asyncio import asyncio
import threading
import yaml import yaml
import telegram import telegram
import re import re
ITEM_TEXT = "- *Artículo*: {}\n" \ ITEM_TEXT = "- *Artículo*: {}\n" \
"- *Descripción*: {}\n" \ "- *Descripción*: {}\n" \
"- *Localidad*: {}\n" \ "- *Localidad*: {}\n" \
@@ -14,7 +11,7 @@ ITEM_TEXT = "- *Artículo*: {}\n" \
"[Ir al anuncio](https://es.wallapop.com/item/{})" "[Ir al anuncio](https://es.wallapop.com/item/{})"
class TelegramHandler: class TelegramManager:
def __init__(self): def __init__(self):
token, channel = self.get_config() token, channel = self.get_config()
self._channel = channel self._channel = channel

19
worker.py
View File

@@ -1,10 +1,9 @@
import time import time
import requests import requests
import logging import logging
from article import Article from datalayer.wallapop_article import WallapopArticle
from telegram_handler import TelegramHandler from managers.telegram_manager import TelegramManager
import traceback import traceback
import asyncio
REQUEST_SLEEP_TIME = 10 REQUEST_SLEEP_TIME = 10
REQUEST_RETRY_TIME = 3 REQUEST_RETRY_TIME = 3
@@ -16,8 +15,8 @@ class Worker:
def __init__(self, item_to_monitor): def __init__(self, item_to_monitor):
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self._item_monitoring = item_to_monitor self._item_monitoring = item_to_monitor
self._notified_articles = self._request_articles() self._notified_articles = self._request_articles()[1:]
self._telegram_handler = TelegramHandler() self.telegram_manager = TelegramManager()
def _request_articles(self): def _request_articles(self):
url = ( url = (
@@ -35,7 +34,11 @@ class Worker:
while True: while True:
try: try:
response = requests.get(url, headers={'User-Agent': USER_AGENT}) headers = {
'X-DeviceOS': '0',
'User-Agent': USER_AGENT
}
response = requests.get(url, headers=headers)
response.raise_for_status() response.raise_for_status()
break break
except requests.exceptions.RequestException as err: except requests.exceptions.RequestException as err:
@@ -49,7 +52,7 @@ class Worker:
def _parse_json_response(self, json_response): def _parse_json_response(self, json_response):
articles = [] articles = []
for json_article in json_response: for json_article in json_response:
articles.append(Article.load_from_json(json_article)) articles.append(WallapopArticle.load_from_json(json_article))
return articles return articles
def _has_words(self, text, word_list): def _has_words(self, text, word_list):
@@ -103,7 +106,7 @@ class Worker:
for article in articles: for article in articles:
if self._meets_item_conditions(article): if self._meets_item_conditions(article):
try: try:
self._telegram_handler.send_telegram_article(article) self.telegram_manager.send_telegram_article(article)
except Exception as e: except Exception as e:
self.logger.error(f"{self._item_monitoring.get_search_query()} worker crashed: {e}") self.logger.error(f"{self._item_monitoring.get_search_query()} worker crashed: {e}")
self._notified_articles.insert(0, article) self._notified_articles.insert(0, article)