diff --git a/README.md b/README.md index fea01b7..3c5ee5a 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,40 @@ -# Wallamonitor -Periodically checks Wallapop for new articles based on specified parameters and notify through Telegram channel. - -### Setup 馃敡 -``` -pip3 install -U python-dotenv -pip3 install python-telegram-bot -``` - -You will also need to change .env parameters: -``` -TELEGRAM_CHANNEL_ID=@Your_Telegram_Channel_ID -TELEGRAM_TOKEN=Your Telegram Token -``` - -### Usage - -``` -$ python3 alert.py -h -usage: alert.py [-h] --name NAME [--latitude LATITUDE] [--longitude LONGITUDE] [--condition CONDITION] - [--min MIN_PRICE] [--max MAX_PRICE] - -Arguments - -optional arguments: - -h, --help show this help message and exit - --name NAME Article name - --latitude LATITUDE Latitude - --longitude LONGITUDE Longitude - --condition CONDITION Item condition: all, new, as_good_as_new, good, fair, has_given_it_all - --min MIN_PRICE Min price - --max MAX_PRICE Max price -``` - -Example: -``` -$ python3 alert.py --name ps5 --condition new --min 400 --max 600 -``` - - +# Wallamonitor +Periodically checks Wallapop for new articles based on specified parameters and notify through Telegram channel. + +### Setup 馃敡 +``` +pip3 install -U python-dotenv +pip3 install python-telegram-bot +``` + +You will also need to change .env parameters: +``` +TELEGRAM_CHANNEL_ID=@Your_Telegram_Channel_ID +TELEGRAM_TOKEN=Your Telegram Token +``` + +### Usage + +``` +$ python3 alert.py -h +usage: alert.py [-h] --name NAME [--latitude LATITUDE] [--longitude LONGITUDE] [--condition CONDITION] + [--min MIN_PRICE] [--max MAX_PRICE] + +Arguments + +optional arguments: + -h, --help show this help message and exit + --name NAME Article name + --latitude LATITUDE Latitude + --longitude LONGITUDE Longitude + --condition CONDITION Item condition: all, new, as_good_as_new, good, fair, has_given_it_all + --min MIN_PRICE Min price + --max MAX_PRICE Max price +``` + +Example: +``` +$ python3 alert.py --name ps5 --condition new --min 400 --max 600 +``` + + diff --git a/__pycache__/worker.cpython-38.pyc b/__pycache__/worker.cpython-38.pyc deleted file mode 100644 index a7eb499..0000000 Binary files a/__pycache__/worker.cpython-38.pyc and /dev/null differ diff --git a/alert.py b/alert.py index d7444a2..7b05218 100644 --- a/alert.py +++ b/alert.py @@ -1,28 +1,28 @@ -# Wallamonitor -# 10/02/2021 - -import time -import requests -import json -import telegram -import argparse -from dotenv import load_dotenv -import os -load_dotenv() -import threading - -from worker import Worker - -def parse_json_file(): - f = open("args.json") - return json.load(f) - - -def main(): - args = parse_json_file() - - for argument in args: - p = threading.Thread(target=Worker.run, args=(argument, )) - p.start() - +# Wallamonitor +# 10/02/2021 + +import time +import requests +import json +import telegram +import argparse +from dotenv import load_dotenv +import os +load_dotenv() +import threading + +from worker import Worker + +def parse_json_file(): + f = open("args.json") + return json.load(f) + + +def main(): + args = parse_json_file() + + for argument in args: + p = threading.Thread(target=Worker.run, args=(argument, )) + p.start() + main() \ No newline at end of file diff --git a/args.json b/args.json index 096ebe7..aa78113 100644 --- a/args.json +++ b/args.json @@ -1,92 +1,52 @@ -[ - { - "product_name": "ps4", - "latitude": "40.4165", - "longitude": "-3.70256", - "condition": "all", - "min_price": "40", - "max_price": "80", - "title_key_word_exclude" : ["juego", "juegos", "Juego", "mando", "Mando", "DualShock"], - "exclude": [] - }, - { - "product_name": "ps4", - "latitude": "40.4165", - "longitude": "-3.70256", - "condition": "has_given_it_all", - "min_price": "20", - "max_price": "50", - "title_key_word_exclude" : [], - "exclude": [] - }, - { - "product_name": "3ds", - "latitude": "40.4165", - "longitude": "-3.70256", - "condition": "all", - "min_price": "15", - "max_price": "60", - "title_key_word_exclude" : ["juego", "juegos", "Juego", "Juegos", "pokemon", "Pokemon"], - "exclude": [] - }, - { - "product_name": "nvidia", - "latitude": "40.4165", - "longitude": "-3.70256", - "condition": "all", - "min_price": "80", - "max_price": "160", - "title_key_word_exclude" : [], - "exclude": [] - }, - { - "product_name": "gtx", - "latitude": "40.4165", - "longitude": "-3.70256", - "condition": "all", - "min_price": "80", - "max_price": "160", - "title_key_word_exclude" : [], - "exclude": ["1050", "950", "960"] - }, - { - "product_name": "grafica", - "latitude": "40.4165", - "longitude": "-3.70256", - "condition": "all", - "min_price": "80", - "max_price": "160", - "title_key_word_exclude" : [], - "exclude": ["1050", "950", "960"] - }, - { - "product_name": "iphone", - "latitude": "40.4165", - "longitude": "-3.70256", - "condition": "all", - "min_price": "90", - "max_price": "200", - "title_key_word_exclude" : [], - "exclude": ["iphone 6", "iphone 7", "iPhone 7", "iPhone 8", "Iphone 6", "Iphone 7"] - }, - { - "product_name": "mac", - "latitude": "40.4165", - "longitude": "-3.70256", - "condition": "all", - "min_price": "100", - "max_price": "200", - "title_key_word_exclude" : [], - "exclude": [] - }, - { - "product_name": "surface", - "latitude": "40.4165", - "longitude": "-3.70256", - "condition": "all", - "min_price": "100", - "max_price": "300", - "title_key_word_exclude" : [], - "exclude": [""] - } -] +[ + { + "product_name": "placa base", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "20", + "max_price": "75", + "title_keyword_exclude" : [], + "exclude": [] + }, + { + "product_name": "ram", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "10", + "max_price": "40", + "title_keyword_exclude" : [], + "exclude": [] + }, +{ + "product_name": "grafica", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "90", + "max_price": "200", + "title_keyword_exclude" : [], + "exclude": ["1050", "960"] + }, +{ + "product_name": "nvidia", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "90", + "max_price": "200", + "title_keyword_exclude" : [], + "exclude": ["1050", "960"] + }, +{ + "product_name": "gtx", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "90", + "max_price": "200", + "title_keyword_exclude" : [], + "exclude": ["1050", "960"] + } +] diff --git a/back_args.json b/back_args.json new file mode 100644 index 0000000..e54fb75 --- /dev/null +++ b/back_args.json @@ -0,0 +1,92 @@ +[ + { + "product_name": "ps4", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "40", + "max_price": "80", + "title_key_word_exclude" : ["juego", "juegos", "Juego", "mando", "Mando", "DualShock"], + "exclude": [] + }, + { + "product_name": "ps4", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "has_given_it_all", + "min_price": "20", + "max_price": "50", + "title_key_word_exclude" : [], + "exclude": [] + }, + { + "product_name": "3ds", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "15", + "max_price": "60", + "title_key_word_exclude" : ["juego", "juegos", "Juego", "Juegos", "pokemon", "Pokemon"], + "exclude": [] + }, + { + "product_name": "nvidia", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "80", + "max_price": "160", + "title_key_word_exclude" : [], + "exclude": [] + }, + { + "product_name": "gtx", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "80", + "max_price": "160", + "title_key_word_exclude" : [], + "exclude": ["1050", "950", "960"] + }, + { + "product_name": "grafica", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "80", + "max_price": "160", + "title_key_word_exclude" : [], + "exclude": ["1050", "950", "960"] + }, + { + "product_name": "iphone", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "90", + "max_price": "200", + "title_key_word_exclude" : [], + "exclude": ["iphone 6", "iphone 7", "iPhone 7", "iPhone 8", "Iphone 6", "Iphone 7"] + }, + { + "product_name": "mac", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "100", + "max_price": "200", + "title_key_word_exclude" : [], + "exclude": [] + }, + { + "product_name": "surface", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "100", + "max_price": "300", + "title_key_word_exclude" : [], + "exclude": [] + } +] diff --git a/error_log.txt b/error_log.txt new file mode 100644 index 0000000..60b44f1 --- /dev/null +++ b/error_log.txt @@ -0,0 +1 @@ +grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse 9jd5lyeq726k: portatil toshiba satelite pro i3 r50 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pzp1p4nql9z3: Memoria ram kingston hyperx ddr2 4 gb a 1.066 MH .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8z8gxdxyol63: M脫VILES HUAWEI P8 LITE .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse 9jd5lyeq726k: portatil toshiba satelite pro i3 r50 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos m骴ulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio T閏nico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 m骴ulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse mznv5n09ok6n: torre ordenador i5 8GB SSD 240GB HDMI .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse nzxyk71xg1j2: ASUS PH-GT1030-O2G GT 1030 2GB GDDR5 .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse wzvlmpw7k46l: Ordenador portatil HP Probook (560) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos m骴ulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio T閏nico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 m骴ulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse nzxyk71xg1j2: ASUS PH-GT1030-O2G GT 1030 2GB GDDR5 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos m骴ulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio T閏nico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 m骴ulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos m骴ulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz . \ No newline at end of file diff --git a/worker.py b/worker.py index 9394ec5..5d5d8f5 100644 --- a/worker.py +++ b/worker.py @@ -1,120 +1,129 @@ -import time -import requests -import json -import telegram -import argparse -from dotenv import load_dotenv -import os -load_dotenv() -import threading -from proxy_requests import ProxyRequests - - -TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") -TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN") -SLEEP_TIME=30 - -class Worker: - - def request(self, product_name, n_articles, latitude='40.4165', longitude='-3.70256', condition='all', min_price=0, max_price=10000000): - url = (f"http://api.wallapop.com/api/v3/general/search?keywords={product_name}" - f"&order_by=newest&latitude={latitude}" - f"&longitude={longitude}" - f"&min_sale_price={min_price}" - f"&max_sale_price={max_price}" - f"&filters_source=quick_filters&language=es_ES") - - if condition != "all": - url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all - - while True: - response = ProxyRequests(url) - try: - response.get() - if response.get_status_code() == 200: - break - else: - print(f"\'{product_name}\' -> Wallapop returned status {response.get_status_code() }. Illegal parameters or Wallapop service is down. Retrying...") - except: - time.sleep(3) - - json_data=json.loads(response.get_raw()) - return json_data['search_objects'] - - def first_run(self, args): - list = [] - articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['condition'], args['min_price'], args['max_price']) - for article in articles: - list.insert(0, article['id']) - - return list - - def work(self, args, list): - exec_times = [] - bot = telegram.Bot(token = TELEGRAM_TOKEN) - - while True: - start_time = time.time() - articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['condition'], args['min_price'], args['max_price']) - for article in articles: - if not article['id'] in list: - if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), args['exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), args['title_key_word_exclude']): - try: - bot.send_message(TELEGRAM_CHANNEL_ID, f"*Art铆culo*: {article['title']}\n" - f"*Descripci贸n*: {article['description']}\n" - f"*Precio*: {article['price']} {article['currency']}\n" - f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})" - , "MARKDOWN") - except: - bot.send_message(TELEGRAM_CHANNEL_ID, f"*Art铆culo*: {article['title']}\n" - f"*Descripci贸n*: Descripci贸n inv谩lida\n" - f"*Precio*: {article['price']} {article['currency']}\n" - f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})" - , "MARKDOWN") - time.sleep(3) # Avoid Telegram flood restriction - list.insert(0, article['id']) - exec_times.append(time.time() - start_time) - print(f"\'{args['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}") - - def has_excluded_words(self, title, description, excluded_words): - for word in excluded_words: - print("EXCLUDER: Checking '" + word + "' for title: '" + title) - if word in title or word in description: - print("EXCLUDE!") - return True - return False - - def is_title_key_word_excluded(self, title, excluded_words): - for word in excluded_words: - print("Checking '" + word + "' for title: '" + title) - if word in title.split()[0]: - return True - return False - - def get_average_time(self, exec_times): - sum = 0 - for i in exec_times: - sum = sum + i - - return sum / len(exec_times) - - def get_max_time(self, exec_times): - largest = 0 - for i in exec_times: - if i > largest: - largest = i - return largest - - - def run(args): - worker = Worker() - list = worker.first_run(args) - while True: - try: - print(f"Wallapop monitor worker started. Checking for new items containing: \'{args['product_name']}\' with given parameters periodically") - worker.work(args, list) - except Exception as e: - print(f"Exception: {e}") - print(f"{args['product_name']} worker crashed. Restarting worker...") - time.sleep(15) - +import time +import requests +import json +import telegram +import argparse +from dotenv import load_dotenv +import os +load_dotenv() +import threading + + +TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") +TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN") +SLEEP_TIME=30 + +class Worker: + + def request(self, product_name, n_articles, latitude='40.4165', longitude='-3.70256', condition='all', min_price=0, max_price=10000000): + url = (f"http://api.wallapop.com/api/v3/general/search?keywords={product_name}" + f"&order_by=newest&latitude={latitude}" + f"&longitude={longitude}" + f"&min_sale_price={min_price}" + f"&max_sale_price={max_price}" + f"&filters_source=quick_filters&language=es_ES") + + if condition != "all": + url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all + + while True: + response = requests.get(url) + try: + if response.status_code == 200: + break + else: + print(f"\'{product_name}\' -> Wallapop returned status {response.get_status_code() }. Illegal parameters or Wallapop service is down. Retrying...") + except Exception as e: + print("Exception: "+e) + time.sleep(3) + + json_data=response.json() + return json_data['search_objects'] + + def first_run(self, args): + list = [] + articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['condition'], args['min_price'], args['max_price']) + for article in articles: + list.insert(0, article['id']) + + return list + + def work(self, args, list): + exec_times = [] + bot = telegram.Bot(token = TELEGRAM_TOKEN) + + while True: + start_time = time.time() + articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['condition'], args['min_price'], args['max_price']) + for article in articles: + if not article['id'] in list: + try: + if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), args['exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), args['title_keyword_exclude']): + try: + bot.send_message(TELEGRAM_CHANNEL_ID, f"*Art铆culo*: {article['title']}\n" + f"*Descripci贸n*: {article['description']}\n" + f"*Precio*: {article['price']} {article['currency']}\n" + f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})" + , "MARKDOWN") + except: + bot.send_message(TELEGRAM_CHANNEL_ID, f"*Art铆culo*: {article['title']}\n" + f"*Descripci贸n*: Descripci贸n inv谩lida\n" + f"*Precio*: {article['price']} {article['currency']}\n" + f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})" + , "MARKDOWN") + time.sleep(1) # Avoid Telegram flood restriction + list.insert(0, article['id']) + except Exception as e: + print("---------- EXCEPTION -----------") + f = open("error_log.txt", "a") + f.write(f"{args['product_name']} worker crashed. {e}") + f.write(f"{args['product_name']}: Trying to parse {article['id']}: {article['title']} .") + f.close() + + + time.sleep(5) + exec_times.append(time.time() - start_time) + print(f"\'{args['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}") + + def has_excluded_words(self, title, description, excluded_words): + for word in excluded_words: + print("EXCLUDER: Checking '" + word + "' for title: '" + title) + if word in title or word in description: + print("EXCLUDE!") + return True + return False + + def is_title_key_word_excluded(self, title, excluded_words): + for word in excluded_words: + print("Checking '" + word + "' for title: '" + title) + if word in title.split()[0]: + return True + return False + + def get_average_time(self, exec_times): + sum = 0 + for i in exec_times: + sum = sum + i + + return sum / len(exec_times) + + def get_max_time(self, exec_times): + largest = 0 + for i in exec_times: + if i > largest: + largest = i + return largest + + + def run(args): + worker = Worker() + list = worker.first_run(args) + while True: + #try: + print(f"Wallapop monitor worker started. Checking for new items containing: \'{args['product_name']}\' with given parameters periodically") + worker.work(args, list) + #except Exception as e: + # print(f"Exception: {e}") + # print(f"{args['product_name']} worker crashed. Restarting worker...") + # time.sleep(10) +