public/wallabicher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Major refactoring

Browse Source
This commit is contained in:
Kifixo
2024-02-12 23:59:16 +01:00
parent cefa2aec20
commit 685518bbcd
10 changed files with 343 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.env
View File

@@ -1,2 +0,0 @@
TELEGRAM_CHANNEL_ID = "YOUR_CHANNEL_ID"
TELEGRAM_TOKEN = "YOUR_TELEGRAM_TOKEN"

28
alert.py
View File

@@ -1,28 +0,0 @@
# Wallamonitor
# 10/02/2021
import time
import requests
import json
import telegram
import argparse
from dotenv import load_dotenv
import os
load_dotenv()
import threading
from worker import Worker
def parse_json_file():
f = open("args.json")
return json.load(f)
def main():
args = parse_json_file()
for argument in args:
p = threading.Thread(target=Worker.run, args=(argument, ))
p.start()
main()

125
args.json
View File

@@ -1,57 +1,106 @@
[
{
"product_name": "placa base",
"distance": "0",
"search_query": "ps4",
"latitude": "40.4165",
"longitude": "-3.70256",
"max_distance":"0",
"condition": "all",
"min_price": "20",
"max_price": "75",
"title_keyword_exclude" : [],
"exclude": []
"max_price": "80",
"title_exclude" : ["DualShock", "Volante"],
"description_exclude": [],
"title_must_include" : [],
"description_must_include" : []
},
{
"product_name": "ram",
"distance": "0",
"search_query": "3ds",
"max_distance":"0",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "10",
"max_price": "40",
"title_keyword_exclude" : [],
"exclude": []
"min_price": "15",
"max_price": "70",
"title_exclude" : ["pokemon", "Pokemon"],
"description_exclude": [],
"title_must_include" : [],
"description_must_include" : []
},
{
"product_name": "grafica",
"distance": "0",
"search_query": "2ds",
"max_distance":"0",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "90",
"min_price": "15",
"max_price": "60",
"title_keyword_exclude" : ["pokemon", "Pokemon"],
"exclude": [],
"title_must_include" : [],
"description_must_include" : []
},
{
"search_query": "nvidia",
"max_distance":"0",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "80",
"max_price": "160",
"title_exclude" : ["ordenador"],
"description_exclude": [],
"title_must_include" : [],
"description_must_include" : []
},
{
"search_query": "gtx",
"max_distance":"0",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "80",
"max_price": "160",
"title_exclude" : ["ordenador"],
"description_exclude": ["1050", "950", "960"],
"title_must_include" : [],
"description_must_include" : []
},
{
"search_query": "grafica",
"max_distance":"0",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "80",
"max_price": "160",
"title_exclude" : ["ordenador"],
"description_exclude": ["1050", "950", "960"],
"title_must_include" : [],
"description_must_include" : []
},
{
"search_query": "ps5",
"max_distance":"0",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "100",
"max_price": "300",
"title_exclude" : [],
"description_exclude": [],
"title_must_include" : [],
"description_must_include" : []
},
{
"search_query": "switch",
"max_distance":"0",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "50",
"max_price": "200",
"title_keyword_exclude" : [],
"exclude": ["1050", "960"]
},
{
"product_name": "nvidia",
"distance": "0",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "90",
"max_price": "200",
"title_keyword_exclude" : [],
"exclude": ["1050", "960"]
},
{
"product_name": "gtx",
"distance": "0",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "90",
"max_price": "200",
"title_keyword_exclude" : [],
"exclude": ["1050", "960"]
"title_exclude" : [],
"description_exclude": [],
"title_must_include" : [],
"description_must_include" : []
}
]

42
article.py Normal file
View File

@@ -0,0 +1,42 @@
class Article:
def __init__(self, id, title, description, price, currency, url):
self._id = id
self._title = title
self._description = description
self._price = price
self._currency = currency
self._url = url
@classmethod
def load_from_json(cls, json_data):
return cls(
json_data['id'],
json_data['title'],
json_data['description'],
json_data['price'],
json_data['currency'],
json_data['web_slug']
)
def get_id(self):
return self._id
def get_title(self):
return self._title
def get_description(self):
return self._description
def get_price(self):
return self._price
def get_currency(self):
return self._currency
def get_url(self):
return self._url
def __eq__(self, article2):
return self.get_id() == article2.get_id()

92
back_args.json
View File

@@ -1,92 +0,0 @@
[
{
"product_name": "ps4",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "40",
"max_price": "80",
"title_key_word_exclude" : ["juego", "juegos", "Juego", "mando", "Mando", "DualShock"],
"exclude": []
},
{
"product_name": "ps4",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "has_given_it_all",
"min_price": "20",
"max_price": "50",
"title_key_word_exclude" : [],
"exclude": []
},
{
"product_name": "3ds",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "15",
"max_price": "60",
"title_key_word_exclude" : ["juego", "juegos", "Juego", "Juegos", "pokemon", "Pokemon"],
"exclude": []
},
{
"product_name": "nvidia",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "80",
"max_price": "160",
"title_key_word_exclude" : [],
"exclude": []
},
{
"product_name": "gtx",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "80",
"max_price": "160",
"title_key_word_exclude" : [],
"exclude": ["1050", "950", "960"]
},
{
"product_name": "grafica",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "80",
"max_price": "160",
"title_key_word_exclude" : [],
"exclude": ["1050", "950", "960"]
},
{
"product_name": "iphone",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "90",
"max_price": "200",
"title_key_word_exclude" : [],
"exclude": ["iphone 6", "iphone 7", "iPhone 7", "iPhone 8", "Iphone 6", "Iphone 7"]
},
{
"product_name": "mac",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "100",
"max_price": "200",
"title_key_word_exclude" : [],
"exclude": []
},
{
"product_name": "surface",
"latitude": "40.4165",
"longitude": "-3.70256",
"condition": "all",
"min_price": "100",
"max_price": "300",
"title_key_word_exclude" : [],
"exclude": []
}
]

1
error_log.txt
View File

@@ -1 +0,0 @@
grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse 9jd5lyeq726k: portatil toshiba satelite pro i3 r50 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pzp1p4nql9z3: Memoria ram kingston hyperx ddr2 4 gb a 1.066 MH .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8z8gxdxyol63: MÓVILES HUAWEI P8 LITE .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse 9jd5lyeq726k: portatil toshiba satelite pro i3 r50 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse mznv5n09ok6n: torre ordenador i5 8GB SSD 240GB HDMI .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse nzxyk71xg1j2: ASUS PH-GT1030-O2G GT 1030 2GB GDDR5 .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse wzvlmpw7k46l: Ordenador portatil HP Probook (560) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse nzxyk71xg1j2: ASUS PH-GT1030-O2G GT 1030 2GB GDDR5 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .

66
item_monitor.py Normal file
View File

@@ -0,0 +1,66 @@
import string
class ItemMonitor:
def __init__(self, search_query, latitude, longitude, max_distance,
condition, min_price, max_price, title_exclude,
description_exclude, title_must_include, description_must_include):
self._search_query = search_query
self._latitude = latitude
self._longitude = longitude
self._max_distance = max_distance
self._condition = condition
self._min_price = min_price
self._max_price = max_price
self._title_exclude = title_exclude
self._description_exclude = description_exclude
self._title_must_include = title_must_include
self._description_must_include = description_must_include
@classmethod
def load_from_json(cls, json_data):
return cls(
json_data['search_query'],
json_data['latitude'],
json_data['longitude'],
json_data['max_distance'],
json_data['condition'],
json_data['min_price'],
json_data['max_price'],
json_data['title_exclude'],
json_data['description_exclude'],
json_data['title_must_include'],
json_data['description_must_include']
)
def get_search_query(self):
return self._search_query
def get_latitude(self):
return self._latitude
def get_longitude(self):
return self._longitude
def get_max_distance(self):
return self._max_distance
def get_condition(self):
return self._condition
def get_min_price(self):
return self._min_price
def get_max_price(self):
return self._max_price
def get_title_exclude(self):
return self._title_exclude
def get_description_exclude(self):
return self._description_exclude
def get_title_must_include(self):
return self._title_must_include
def get_description_must_include(self):
return self._description_must_include

24
main.py Normal file
View File

@@ -0,0 +1,24 @@
import json
import threading
import logging
from item_monitor import ItemMonitor
from worker import Worker
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[logging.FileHandler('main_log.txt'), logging.StreamHandler()])
def parse_items_to_monitor():
with open("args.json") as f:
args = json.load(f)
items = [ItemMonitor.load_from_json(item) for item in args]
return items
if __name__ == "__main__":
logger = logging.getLogger(__name__)
items = parse_items_to_monitor()
for item in items:
worker = Worker(item)
thread = threading.Thread(target=worker.run)
thread.start()

36
telegram_handler.py Normal file
View File

@@ -0,0 +1,36 @@
from article import Article
import asyncio
import threading
import yaml
import telegram
ITEM_TEXT = "*Artículo*: {}\n" \
"*Descripción*: {}\n" \
"*Precio*: {} {}\n" \
"[Ir al anuncio](https://es.wallapop.com/item/{})"
class TelegramHandler:
def __init__(self):
token, channel = self.get_config()
self._channel = channel
self._bot = telegram.Bot(token=token)
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
def get_config(self):
config_file = 'config.yaml'
with open(config_file, 'r') as file:
config = yaml.safe_load(file)
print(config)
token = config['telegram_token']
telegram_channel = config['telegram_channel']
return token, telegram_channel
def send_telegram_article(self, article):
self._loop.run_until_complete(self.send_telegram_article_async(article))
async def send_telegram_article_async(self, article):
message = ITEM_TEXT.format(article.get_title(), article.get_description(),
article.get_price(), article.get_currency(),
article.get_url())
await self._bot.send_message(self._channel, text=message, parse_mode="MARKDOWN")

190
worker.py
View File

@@ -1,127 +1,115 @@
import time
import requests
import telegram
from dotenv import load_dotenv
import os
load_dotenv()
import logging
from article import Article
from telegram_handler import TelegramHandler
import traceback
import asyncio
TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID")
TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
SLEEP_TIME = 5
REQUEST_SLEEP_TIME = 5
REQUEST_RETRY_TIME = 3
ERROR_SLEEP_TIME = 10
worker_logger = logging.getLogger(__name__)
worker_logger.setLevel(logging.INFO) # Set the level as needed
worker_logger.addHandler(logging.StreamHandler())
class Worker:
def __init__(self, item_to_monitor):
self.logger = logging.getLogger(__name__)
self._item_monitoring = item_to_monitor
self._notified_articles = self._request_articles()
self._telegram_handler = TelegramHandler()
def request(self, product_name, n_articles, latitude='40.4165', longitude='-3.70256', distance='0', condition='all', min_price=0, max_price=10000000):
url = (f"http://api.wallapop.com/api/v3/general/search?keywords={product_name}"
f"&order_by=newest&latitude={latitude}"
f"&longitude={longitude}"
f"&distance={distance}"
f"&min_sale_price={min_price}"
f"&max_sale_price={max_price}"
f"&filters_source=quick_filters&language=es_ES")
def _request_articles(self):
url = (
f"http://api.wallapop.com/api/v3/general/search?keywords={self._item_monitoring.get_search_query()}"
f"&order_by=newest&latitude={self._item_monitoring.get_latitude()}"
f"&longitude={self._item_monitoring.get_longitude()}"
f"&distance={self._item_monitoring.get_max_distance()}"
f"&min_sale_price={self._item_monitoring.get_min_price()}"
f"&max_sale_price={self._item_monitoring.get_max_price()}"
f"&filters_source=quick_filters&language=es_ES"
)
if condition != "all":
url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all
if self._item_monitoring.get_condition() != "all":
url += f"&condition={self._item_monitoring.get_condition()}" # new, as_good_as_new, good, fair, has_given_it_all
while True:
response = requests.get(url)
try:
if response.status_code == 200:
response = requests.get(url)
response.raise_for_status()
break
else:
print(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...")
except Exception as e:
print("Exception: " + e)
time.sleep(3)
except requests.exceptions.RequestException as err:
self.logger.error(f"Request Exception: {err}")
time.sleep(REQUEST_RETRY_TIME)
json_data = response.json()
return json_data['search_objects']
json_response = response.json()
articles = self._parse_json_response(json_response['search_objects'])
return articles
def first_run(self, args):
list = []
articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['distance'], args['condition'], args['min_price'], args['max_price'])
for article in articles:
list.insert(0, article['id'])
def _parse_json_response(self, json_response):
articles = []
for json_article in json_response:
articles.append(Article.load_from_json(json_article))
return articles
return list
def _has_words(self, text, word_list):
return any(word in text for word in word_list)
def work(self, args, list):
def _title_has_excluded_words(self, article):
return self._has_words(article.get_title(),
self._item_monitoring.get_title_exclude())
def _description_has_excluded_words(self, article):
return self._has_words(article.get_description(),
self._item_monitoring.get_description_exclude())
def _title_has_required_words(self, article):
return not self._item_monitoring.get_title_must_include() \
or self._has_words(article.get_title(),
self._item_monitoring.get_title_must_include())
def _description_has_required_words(self, article):
return not self._item_monitoring.get_description_must_include() \
or self._has_words(article.get_description(),
self._item_monitoring.get_description_must_include())
def _meets_item_conditions(self, article):
return (
self._title_has_required_words(article) and
self._description_has_required_words(article) and
not self._title_has_excluded_words(article) and
not self._description_has_excluded_words(article) and
article not in self._notified_articles
)
def work(self):
exec_times = []
bot = telegram.Bot(token = TELEGRAM_TOKEN)
while True:
start_time = time.time()
articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['distance'], args['condition'], args['min_price'], args['max_price'])
for article in articles:
if not article['id'] in list:
articles = self._request_articles()
for article in articles[0:1]:
if self._meets_item_conditions(article):
try:
if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), args['exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), args['title_keyword_exclude']):
try:
bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n"
f"*Descripción*: {article['description']}\n"
f"*Precio*: {article['price']} {article['currency']}\n"
f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})"
, "MARKDOWN")
except:
bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n"
f"*Descripción*: Descripción inválida\n"
f"*Precio*: {article['price']} {article['currency']}\n"
f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})"
, "MARKDOWN")
time.sleep(1) # Avoid Telegram flood restriction
list.insert(0, article['id'])
self._telegram_handler.send_telegram_article(article)
self._notified_articles.insert(0, article)
except Exception as e:
print("---------- EXCEPTION -----------")
f = open("error_log.txt", "a")
f.write(f"{args['product_name']} worker crashed. {e}")
f.write(f"{args['product_name']}: Trying to parse {article['id']}: {article['title']} .\n")
f.close()
time.sleep(SLEEP_TIME)
self.logger.error(f"{self._item_monitoring.get_search_query()} worker crashed: {e}")
time.sleep(REQUEST_SLEEP_TIME)
exec_times.append(time.time() - start_time)
print(f"\'{args['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}")
self.logger.info(f"\'{self._item_monitoring.get_search_query()}\' node-> last: {exec_times[-1]}"
f" max: {max(exec_times)} avg: {sum(exec_times) / len(exec_times)}")
def has_excluded_words(self, title, description, excluded_words):
for word in excluded_words:
print("EXCLUDER: Checking '" + word + "' for title: '" + title)
if word in title or word in description:
print("EXCLUDE!")
return True
return False
def is_title_key_word_excluded(self, title, excluded_words):
for word in excluded_words:
print("Checking '" + word + "' for title: '" + title)
if word in title:
return True
return False
def get_average_time(self, exec_times):
sum = 0
for i in exec_times:
sum = sum + i
return sum / len(exec_times)
def get_max_time(self, exec_times):
largest = 0
for i in exec_times:
if i > largest:
largest = i
return largest
def run(args):
worker = Worker()
list = worker.first_run(args)
def run(self):
while True:
try:
print(f"Wallapop monitor worker started. Checking for new items containing: \'{args['product_name']}\' with given parameters periodically")
worker.work(args, list)
self.logger.info(f"Wallapop monitor worker started. Checking for "
f"new items containing '{self._item_monitoring.get_search_query()}' "
f"with given parameters periodically")
self.work()
except Exception as e:
print(f"Exception: {e}")
print(f"{args['product_name']} worker crashed. Restarting worker...")
time.sleep(10)
self.logger.error(f"{''.join(traceback.format_exception(None, e, e.__traceback__))}")
self.logger.error(f"{self._item_monitoring.get_search_query()} worker crashed. Restarting worker...")
time.sleep(ERROR_SLEEP_TIME)