Major refactoring
This commit is contained in:
Kifixo
2
.env
2
.env
@@ -1,2 +0,0 @@
|
||||
TELEGRAM_CHANNEL_ID = "YOUR_CHANNEL_ID"
|
||||
TELEGRAM_TOKEN = "YOUR_TELEGRAM_TOKEN"
|
||||
28
alert.py
28
alert.py
@@ -1,28 +0,0 @@
|
||||
# Wallamonitor
|
||||
# 10/02/2021
|
||||
|
||||
import time
|
||||
import requests
|
||||
import json
|
||||
import telegram
|
||||
import argparse
|
||||
from dotenv import load_dotenv
|
||||
import os
|
||||
load_dotenv()
|
||||
import threading
|
||||
|
||||
from worker import Worker
|
||||
|
||||
def parse_json_file():
|
||||
f = open("args.json")
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_json_file()
|
||||
|
||||
for argument in args:
|
||||
p = threading.Thread(target=Worker.run, args=(argument, ))
|
||||
p.start()
|
||||
|
||||
main()
|
||||
117
args.json
117
args.json
@@ -1,57 +1,106 @@
|
||||
[
|
||||
{
|
||||
"product_name": "placa base",
|
||||
"distance": "0",
|
||||
"search_query": "ps4",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"max_distance":"0",
|
||||
"condition": "all",
|
||||
"min_price": "20",
|
||||
"max_price": "75",
|
||||
"title_keyword_exclude" : [],
|
||||
"exclude": []
|
||||
"max_price": "80",
|
||||
"title_exclude" : ["DualShock", "Volante"],
|
||||
"description_exclude": [],
|
||||
"title_must_include" : [],
|
||||
"description_must_include" : []
|
||||
},
|
||||
{
|
||||
"product_name": "ram",
|
||||
"distance": "0",
|
||||
{
|
||||
"search_query": "3ds",
|
||||
"max_distance":"0",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "10",
|
||||
"max_price": "40",
|
||||
"title_keyword_exclude" : [],
|
||||
"exclude": []
|
||||
"min_price": "15",
|
||||
"max_price": "70",
|
||||
"title_exclude" : ["pokemon", "Pokemon"],
|
||||
"description_exclude": [],
|
||||
"title_must_include" : [],
|
||||
"description_must_include" : []
|
||||
},
|
||||
{
|
||||
"product_name": "grafica",
|
||||
"distance": "0",
|
||||
{
|
||||
"search_query": "2ds",
|
||||
"max_distance":"0",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "90",
|
||||
"max_price": "200",
|
||||
"title_keyword_exclude" : [],
|
||||
"exclude": ["1050", "960"]
|
||||
"min_price": "15",
|
||||
"max_price": "60",
|
||||
"title_keyword_exclude" : ["pokemon", "Pokemon"],
|
||||
"exclude": [],
|
||||
"title_must_include" : [],
|
||||
"description_must_include" : []
|
||||
},
|
||||
{
|
||||
"product_name": "nvidia",
|
||||
"distance": "0",
|
||||
{
|
||||
"search_query": "nvidia",
|
||||
"max_distance":"0",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "90",
|
||||
"max_price": "200",
|
||||
"title_keyword_exclude" : [],
|
||||
"exclude": ["1050", "960"]
|
||||
"min_price": "80",
|
||||
"max_price": "160",
|
||||
"title_exclude" : ["ordenador"],
|
||||
"description_exclude": [],
|
||||
"title_must_include" : [],
|
||||
"description_must_include" : []
|
||||
},
|
||||
{
|
||||
"product_name": "gtx",
|
||||
"distance": "0",
|
||||
{
|
||||
"search_query": "gtx",
|
||||
"max_distance":"0",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "90",
|
||||
"max_price": "200",
|
||||
"title_keyword_exclude" : [],
|
||||
"exclude": ["1050", "960"]
|
||||
}
|
||||
"min_price": "80",
|
||||
"max_price": "160",
|
||||
"title_exclude" : ["ordenador"],
|
||||
"description_exclude": ["1050", "950", "960"],
|
||||
"title_must_include" : [],
|
||||
"description_must_include" : []
|
||||
},
|
||||
{
|
||||
"search_query": "grafica",
|
||||
"max_distance":"0",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "80",
|
||||
"max_price": "160",
|
||||
"title_exclude" : ["ordenador"],
|
||||
"description_exclude": ["1050", "950", "960"],
|
||||
"title_must_include" : [],
|
||||
"description_must_include" : []
|
||||
},
|
||||
{
|
||||
"search_query": "ps5",
|
||||
"max_distance":"0",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "100",
|
||||
"max_price": "300",
|
||||
"title_exclude" : [],
|
||||
"description_exclude": [],
|
||||
"title_must_include" : [],
|
||||
"description_must_include" : []
|
||||
},
|
||||
{
|
||||
"search_query": "switch",
|
||||
"max_distance":"0",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "50",
|
||||
"max_price": "200",
|
||||
"title_exclude" : [],
|
||||
"description_exclude": [],
|
||||
"title_must_include" : [],
|
||||
"description_must_include" : []
|
||||
}
|
||||
]
|
||||
|
||||
42
article.py
Normal file
42
article.py
Normal file
@@ -0,0 +1,42 @@
|
||||
|
||||
|
||||
class Article:
|
||||
def __init__(self, id, title, description, price, currency, url):
|
||||
self._id = id
|
||||
self._title = title
|
||||
self._description = description
|
||||
self._price = price
|
||||
self._currency = currency
|
||||
self._url = url
|
||||
|
||||
@classmethod
|
||||
def load_from_json(cls, json_data):
|
||||
return cls(
|
||||
json_data['id'],
|
||||
json_data['title'],
|
||||
json_data['description'],
|
||||
json_data['price'],
|
||||
json_data['currency'],
|
||||
json_data['web_slug']
|
||||
)
|
||||
|
||||
def get_id(self):
|
||||
return self._id
|
||||
|
||||
def get_title(self):
|
||||
return self._title
|
||||
|
||||
def get_description(self):
|
||||
return self._description
|
||||
|
||||
def get_price(self):
|
||||
return self._price
|
||||
|
||||
def get_currency(self):
|
||||
return self._currency
|
||||
|
||||
def get_url(self):
|
||||
return self._url
|
||||
|
||||
def __eq__(self, article2):
|
||||
return self.get_id() == article2.get_id()
|
||||
@@ -1,92 +0,0 @@
|
||||
[
|
||||
{
|
||||
"product_name": "ps4",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "40",
|
||||
"max_price": "80",
|
||||
"title_key_word_exclude" : ["juego", "juegos", "Juego", "mando", "Mando", "DualShock"],
|
||||
"exclude": []
|
||||
},
|
||||
{
|
||||
"product_name": "ps4",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "has_given_it_all",
|
||||
"min_price": "20",
|
||||
"max_price": "50",
|
||||
"title_key_word_exclude" : [],
|
||||
"exclude": []
|
||||
},
|
||||
{
|
||||
"product_name": "3ds",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "15",
|
||||
"max_price": "60",
|
||||
"title_key_word_exclude" : ["juego", "juegos", "Juego", "Juegos", "pokemon", "Pokemon"],
|
||||
"exclude": []
|
||||
},
|
||||
{
|
||||
"product_name": "nvidia",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "80",
|
||||
"max_price": "160",
|
||||
"title_key_word_exclude" : [],
|
||||
"exclude": []
|
||||
},
|
||||
{
|
||||
"product_name": "gtx",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "80",
|
||||
"max_price": "160",
|
||||
"title_key_word_exclude" : [],
|
||||
"exclude": ["1050", "950", "960"]
|
||||
},
|
||||
{
|
||||
"product_name": "grafica",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "80",
|
||||
"max_price": "160",
|
||||
"title_key_word_exclude" : [],
|
||||
"exclude": ["1050", "950", "960"]
|
||||
},
|
||||
{
|
||||
"product_name": "iphone",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "90",
|
||||
"max_price": "200",
|
||||
"title_key_word_exclude" : [],
|
||||
"exclude": ["iphone 6", "iphone 7", "iPhone 7", "iPhone 8", "Iphone 6", "Iphone 7"]
|
||||
},
|
||||
{
|
||||
"product_name": "mac",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "100",
|
||||
"max_price": "200",
|
||||
"title_key_word_exclude" : [],
|
||||
"exclude": []
|
||||
},
|
||||
{
|
||||
"product_name": "surface",
|
||||
"latitude": "40.4165",
|
||||
"longitude": "-3.70256",
|
||||
"condition": "all",
|
||||
"min_price": "100",
|
||||
"max_price": "300",
|
||||
"title_key_word_exclude" : [],
|
||||
"exclude": []
|
||||
}
|
||||
]
|
||||
@@ -1 +0,0 @@
|
||||
grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse 9jd5lyeq726k: portatil toshiba satelite pro i3 r50 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pzp1p4nql9z3: Memoria ram kingston hyperx ddr2 4 gb a 1.066 MH .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8z8gxdxyol63: MÓVILES HUAWEI P8 LITE .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse 9jd5lyeq726k: portatil toshiba satelite pro i3 r50 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse mznv5n09ok6n: torre ordenador i5 8GB SSD 240GB HDMI .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse nzxyk71xg1j2: ASUS PH-GT1030-O2G GT 1030 2GB GDDR5 .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse wzvlmpw7k46l: Ordenador portatil HP Probook (560) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse nzxyk71xg1j2: ASUS PH-GT1030-O2G GT 1030 2GB GDDR5 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .
|
||||
66
item_monitor.py
Normal file
66
item_monitor.py
Normal file
@@ -0,0 +1,66 @@
|
||||
import string
|
||||
|
||||
class ItemMonitor:
|
||||
def __init__(self, search_query, latitude, longitude, max_distance,
|
||||
condition, min_price, max_price, title_exclude,
|
||||
description_exclude, title_must_include, description_must_include):
|
||||
self._search_query = search_query
|
||||
self._latitude = latitude
|
||||
self._longitude = longitude
|
||||
self._max_distance = max_distance
|
||||
self._condition = condition
|
||||
self._min_price = min_price
|
||||
self._max_price = max_price
|
||||
self._title_exclude = title_exclude
|
||||
self._description_exclude = description_exclude
|
||||
self._title_must_include = title_must_include
|
||||
self._description_must_include = description_must_include
|
||||
|
||||
@classmethod
|
||||
def load_from_json(cls, json_data):
|
||||
return cls(
|
||||
json_data['search_query'],
|
||||
json_data['latitude'],
|
||||
json_data['longitude'],
|
||||
json_data['max_distance'],
|
||||
json_data['condition'],
|
||||
json_data['min_price'],
|
||||
json_data['max_price'],
|
||||
json_data['title_exclude'],
|
||||
json_data['description_exclude'],
|
||||
json_data['title_must_include'],
|
||||
json_data['description_must_include']
|
||||
)
|
||||
|
||||
def get_search_query(self):
|
||||
return self._search_query
|
||||
|
||||
def get_latitude(self):
|
||||
return self._latitude
|
||||
|
||||
def get_longitude(self):
|
||||
return self._longitude
|
||||
|
||||
def get_max_distance(self):
|
||||
return self._max_distance
|
||||
|
||||
def get_condition(self):
|
||||
return self._condition
|
||||
|
||||
def get_min_price(self):
|
||||
return self._min_price
|
||||
|
||||
def get_max_price(self):
|
||||
return self._max_price
|
||||
|
||||
def get_title_exclude(self):
|
||||
return self._title_exclude
|
||||
|
||||
def get_description_exclude(self):
|
||||
return self._description_exclude
|
||||
|
||||
def get_title_must_include(self):
|
||||
return self._title_must_include
|
||||
|
||||
def get_description_must_include(self):
|
||||
return self._description_must_include
|
||||
24
main.py
Normal file
24
main.py
Normal file
@@ -0,0 +1,24 @@
|
||||
import json
|
||||
import threading
|
||||
import logging
|
||||
from item_monitor import ItemMonitor
|
||||
from worker import Worker
|
||||
|
||||
logging.basicConfig(level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[logging.FileHandler('main_log.txt'), logging.StreamHandler()])
|
||||
|
||||
def parse_items_to_monitor():
|
||||
with open("args.json") as f:
|
||||
args = json.load(f)
|
||||
items = [ItemMonitor.load_from_json(item) for item in args]
|
||||
return items
|
||||
|
||||
if __name__ == "__main__":
|
||||
logger = logging.getLogger(__name__)
|
||||
items = parse_items_to_monitor()
|
||||
|
||||
for item in items:
|
||||
worker = Worker(item)
|
||||
thread = threading.Thread(target=worker.run)
|
||||
thread.start()
|
||||
36
telegram_handler.py
Normal file
36
telegram_handler.py
Normal file
@@ -0,0 +1,36 @@
|
||||
from article import Article
|
||||
import asyncio
|
||||
import threading
|
||||
import yaml
|
||||
import telegram
|
||||
|
||||
ITEM_TEXT = "*Artículo*: {}\n" \
|
||||
"*Descripción*: {}\n" \
|
||||
"*Precio*: {} {}\n" \
|
||||
"[Ir al anuncio](https://es.wallapop.com/item/{})"
|
||||
|
||||
class TelegramHandler:
|
||||
def __init__(self):
|
||||
token, channel = self.get_config()
|
||||
self._channel = channel
|
||||
self._bot = telegram.Bot(token=token)
|
||||
self._loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self._loop)
|
||||
|
||||
def get_config(self):
|
||||
config_file = 'config.yaml'
|
||||
with open(config_file, 'r') as file:
|
||||
config = yaml.safe_load(file)
|
||||
print(config)
|
||||
token = config['telegram_token']
|
||||
telegram_channel = config['telegram_channel']
|
||||
return token, telegram_channel
|
||||
|
||||
def send_telegram_article(self, article):
|
||||
self._loop.run_until_complete(self.send_telegram_article_async(article))
|
||||
|
||||
async def send_telegram_article_async(self, article):
|
||||
message = ITEM_TEXT.format(article.get_title(), article.get_description(),
|
||||
article.get_price(), article.get_currency(),
|
||||
article.get_url())
|
||||
await self._bot.send_message(self._channel, text=message, parse_mode="MARKDOWN")
|
||||
192
worker.py
192
worker.py
@@ -1,127 +1,115 @@
|
||||
import time
|
||||
import requests
|
||||
import telegram
|
||||
from dotenv import load_dotenv
|
||||
import os
|
||||
load_dotenv()
|
||||
import logging
|
||||
from article import Article
|
||||
from telegram_handler import TelegramHandler
|
||||
import traceback
|
||||
import asyncio
|
||||
|
||||
TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID")
|
||||
TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
|
||||
SLEEP_TIME = 5
|
||||
REQUEST_SLEEP_TIME = 5
|
||||
REQUEST_RETRY_TIME = 3
|
||||
ERROR_SLEEP_TIME = 10
|
||||
|
||||
worker_logger = logging.getLogger(__name__)
|
||||
worker_logger.setLevel(logging.INFO) # Set the level as needed
|
||||
worker_logger.addHandler(logging.StreamHandler())
|
||||
|
||||
class Worker:
|
||||
def __init__(self, item_to_monitor):
|
||||
self.logger = logging.getLogger(__name__)
|
||||
self._item_monitoring = item_to_monitor
|
||||
self._notified_articles = self._request_articles()
|
||||
self._telegram_handler = TelegramHandler()
|
||||
|
||||
def request(self, product_name, n_articles, latitude='40.4165', longitude='-3.70256', distance='0', condition='all', min_price=0, max_price=10000000):
|
||||
url = (f"http://api.wallapop.com/api/v3/general/search?keywords={product_name}"
|
||||
f"&order_by=newest&latitude={latitude}"
|
||||
f"&longitude={longitude}"
|
||||
f"&distance={distance}"
|
||||
f"&min_sale_price={min_price}"
|
||||
f"&max_sale_price={max_price}"
|
||||
f"&filters_source=quick_filters&language=es_ES")
|
||||
def _request_articles(self):
|
||||
url = (
|
||||
f"http://api.wallapop.com/api/v3/general/search?keywords={self._item_monitoring.get_search_query()}"
|
||||
f"&order_by=newest&latitude={self._item_monitoring.get_latitude()}"
|
||||
f"&longitude={self._item_monitoring.get_longitude()}"
|
||||
f"&distance={self._item_monitoring.get_max_distance()}"
|
||||
f"&min_sale_price={self._item_monitoring.get_min_price()}"
|
||||
f"&max_sale_price={self._item_monitoring.get_max_price()}"
|
||||
f"&filters_source=quick_filters&language=es_ES"
|
||||
)
|
||||
|
||||
if condition != "all":
|
||||
url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all
|
||||
if self._item_monitoring.get_condition() != "all":
|
||||
url += f"&condition={self._item_monitoring.get_condition()}" # new, as_good_as_new, good, fair, has_given_it_all
|
||||
|
||||
while True:
|
||||
response = requests.get(url)
|
||||
try:
|
||||
if response.status_code == 200:
|
||||
break
|
||||
else:
|
||||
print(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...")
|
||||
except Exception as e:
|
||||
print("Exception: " + e)
|
||||
time.sleep(3)
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
break
|
||||
except requests.exceptions.RequestException as err:
|
||||
self.logger.error(f"Request Exception: {err}")
|
||||
time.sleep(REQUEST_RETRY_TIME)
|
||||
|
||||
json_data = response.json()
|
||||
return json_data['search_objects']
|
||||
json_response = response.json()
|
||||
articles = self._parse_json_response(json_response['search_objects'])
|
||||
return articles
|
||||
|
||||
def first_run(self, args):
|
||||
list = []
|
||||
articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['distance'], args['condition'], args['min_price'], args['max_price'])
|
||||
for article in articles:
|
||||
list.insert(0, article['id'])
|
||||
def _parse_json_response(self, json_response):
|
||||
articles = []
|
||||
for json_article in json_response:
|
||||
articles.append(Article.load_from_json(json_article))
|
||||
return articles
|
||||
|
||||
return list
|
||||
def _has_words(self, text, word_list):
|
||||
return any(word in text for word in word_list)
|
||||
|
||||
def work(self, args, list):
|
||||
def _title_has_excluded_words(self, article):
|
||||
return self._has_words(article.get_title(),
|
||||
self._item_monitoring.get_title_exclude())
|
||||
|
||||
def _description_has_excluded_words(self, article):
|
||||
return self._has_words(article.get_description(),
|
||||
self._item_monitoring.get_description_exclude())
|
||||
|
||||
def _title_has_required_words(self, article):
|
||||
return not self._item_monitoring.get_title_must_include() \
|
||||
or self._has_words(article.get_title(),
|
||||
self._item_monitoring.get_title_must_include())
|
||||
|
||||
def _description_has_required_words(self, article):
|
||||
return not self._item_monitoring.get_description_must_include() \
|
||||
or self._has_words(article.get_description(),
|
||||
self._item_monitoring.get_description_must_include())
|
||||
|
||||
def _meets_item_conditions(self, article):
|
||||
return (
|
||||
self._title_has_required_words(article) and
|
||||
self._description_has_required_words(article) and
|
||||
not self._title_has_excluded_words(article) and
|
||||
not self._description_has_excluded_words(article) and
|
||||
article not in self._notified_articles
|
||||
)
|
||||
|
||||
def work(self):
|
||||
exec_times = []
|
||||
bot = telegram.Bot(token = TELEGRAM_TOKEN)
|
||||
|
||||
while True:
|
||||
start_time = time.time()
|
||||
articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['distance'], args['condition'], args['min_price'], args['max_price'])
|
||||
for article in articles:
|
||||
if not article['id'] in list:
|
||||
articles = self._request_articles()
|
||||
for article in articles[0:1]:
|
||||
if self._meets_item_conditions(article):
|
||||
try:
|
||||
|
||||
if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), args['exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), args['title_keyword_exclude']):
|
||||
try:
|
||||
bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n"
|
||||
f"*Descripción*: {article['description']}\n"
|
||||
f"*Precio*: {article['price']} {article['currency']}\n"
|
||||
f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})"
|
||||
, "MARKDOWN")
|
||||
except:
|
||||
bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n"
|
||||
f"*Descripción*: Descripción inválida\n"
|
||||
f"*Precio*: {article['price']} {article['currency']}\n"
|
||||
f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})"
|
||||
, "MARKDOWN")
|
||||
time.sleep(1) # Avoid Telegram flood restriction
|
||||
list.insert(0, article['id'])
|
||||
self._telegram_handler.send_telegram_article(article)
|
||||
self._notified_articles.insert(0, article)
|
||||
except Exception as e:
|
||||
print("---------- EXCEPTION -----------")
|
||||
f = open("error_log.txt", "a")
|
||||
f.write(f"{args['product_name']} worker crashed. {e}")
|
||||
f.write(f"{args['product_name']}: Trying to parse {article['id']}: {article['title']} .\n")
|
||||
f.close()
|
||||
|
||||
|
||||
time.sleep(SLEEP_TIME)
|
||||
self.logger.error(f"{self._item_monitoring.get_search_query()} worker crashed: {e}")
|
||||
time.sleep(REQUEST_SLEEP_TIME)
|
||||
exec_times.append(time.time() - start_time)
|
||||
print(f"\'{args['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}")
|
||||
self.logger.info(f"\'{self._item_monitoring.get_search_query()}\' node-> last: {exec_times[-1]}"
|
||||
f" max: {max(exec_times)} avg: {sum(exec_times) / len(exec_times)}")
|
||||
|
||||
def has_excluded_words(self, title, description, excluded_words):
|
||||
for word in excluded_words:
|
||||
print("EXCLUDER: Checking '" + word + "' for title: '" + title)
|
||||
if word in title or word in description:
|
||||
print("EXCLUDE!")
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_title_key_word_excluded(self, title, excluded_words):
|
||||
for word in excluded_words:
|
||||
print("Checking '" + word + "' for title: '" + title)
|
||||
if word in title:
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_average_time(self, exec_times):
|
||||
sum = 0
|
||||
for i in exec_times:
|
||||
sum = sum + i
|
||||
|
||||
return sum / len(exec_times)
|
||||
|
||||
def get_max_time(self, exec_times):
|
||||
largest = 0
|
||||
for i in exec_times:
|
||||
if i > largest:
|
||||
largest = i
|
||||
return largest
|
||||
|
||||
|
||||
def run(args):
|
||||
worker = Worker()
|
||||
list = worker.first_run(args)
|
||||
def run(self):
|
||||
while True:
|
||||
try:
|
||||
print(f"Wallapop monitor worker started. Checking for new items containing: \'{args['product_name']}\' with given parameters periodically")
|
||||
worker.work(args, list)
|
||||
self.logger.info(f"Wallapop monitor worker started. Checking for "
|
||||
f"new items containing '{self._item_monitoring.get_search_query()}' "
|
||||
f"with given parameters periodically")
|
||||
self.work()
|
||||
except Exception as e:
|
||||
print(f"Exception: {e}")
|
||||
print(f"{args['product_name']} worker crashed. Restarting worker...")
|
||||
time.sleep(10)
|
||||
|
||||
self.logger.error(f"{''.join(traceback.format_exception(None, e, e.__traceback__))}")
|
||||
self.logger.error(f"{self._item_monitoring.get_search_query()} worker crashed. Restarting worker...")
|
||||
time.sleep(ERROR_SLEEP_TIME)
|
||||
|
||||
Reference in New Issue
Block a user