From 6c37062b131aded8ea87e0c511ebc60b6301f53c Mon Sep 17 00:00:00 2001 From: Kifixo Date: Sat, 23 Oct 2021 17:27:46 +0200 Subject: [PATCH] Major updates --- .env | 4 +- __init__.py | 0 __pycache__/worker.cpython-38.pyc | Bin 0 -> 4179 bytes alert.py | 81 +++----------------- args.json | 92 +++++++++++++++++++++++ worker.py | 120 ++++++++++++++++++++++++++++++ 6 files changed, 225 insertions(+), 72 deletions(-) create mode 100644 __init__.py create mode 100644 __pycache__/worker.cpython-38.pyc create mode 100644 args.json create mode 100644 worker.py diff --git a/.env b/.env index 5148f7c..bc34e73 100644 --- a/.env +++ b/.env @@ -1,2 +1,2 @@ -TELEGRAM_CHANNEL_ID=@Your_Telegram_Channel -TELEGRAM_TOKEN=Your Telegram Token +TELEGRAM_CHANNEL_ID=@wallapopalertas +TELEGRAM_TOKEN=1640428544:AAF2RxKqp4j4_8zKmWIaHsszE4anRwG4tGY diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/__pycache__/worker.cpython-38.pyc b/__pycache__/worker.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a7eb4995e68742d3f9686b38a118f1f1c4a31a31 GIT binary patch literal 4179 zcma)9&2QYs73Yv#E|4BM%lEmDH!j8}{( zay8`YW5F!~IENk@pogL;5D1VCJ)}TS{S$KO-@t26zV~DlXn$|GE6YxTmJ7al^T~O? znfH5dKb@L#G(3O#=smyptfu{wDn}m!mCJb3FCns?)yM%PfK*)>t> zQDwvGTC5yf?N){LKpWTWu8k2!RNHX64%0++zs}7A)@>-Q0&R+0-R1`xuX6h4dSy7J?pL;yXM7)LS)D4;J?A{cc1CeHFaqRZW^_359UisL&`pYY; zt1H*NYs*FRu<`aMAFZrDJcFeITJPvnLFF>ubQMBqWFzdGs`i2+nf$4KU6PNFz5P&D?^loR*r+yqJFo4t1Dps-{XpZjbzT1Ej<)&7_*740KF|)>$jKevoYP?Q^y=`f zwJhr|E-d){u(J&-{eIH#1jz<9ENs28&=awcezcGZUj}RMu8ZC6MDlcLc)yB#E62To-xEtB^;T|4ateGSNMaslVG=Jr{0CYdvH9VddH0n|?wzq6t`ylo z#)7+P=4XS{9WHg;Yf&V6e&qIj>2C=7rklveQ&J(fut_&eU7l>m9rvcliNh?B3pi>xM5n)7 zRAuSCGnkQ{g3#F6s?C~)&2(n72D9`t%wjWolMS1tA9co40-c7BAM#*2KIJVpp!Bqn zo@@JB&N!PRuVLJnqnfE?MM+D^>YR2^9T~Z?tbKlbWaj!l!+4Y1XxEvxc`?_Cg;wPW ztgF0L&XRVL#2g9QxI7I}=pk?G@_ACN5V**!U6G(ztuTGrOA9j!)2y%$v9FFb1z9Ov z8eO{|$`l7Uh<}c1s_2@TtOHDD^kHj^#NnWTuFB*g7HQ#CSo|LB0xXP7`+{XUXJ50C znHl@Wfi|iD9}|e^z{mJ8WYA7HR7X~B0iS=3s(F>yfSbNvl^D=HyQM>vnY;ESo0rB)1=+mXX2lmB1YH{Mm=O~a>vSeMv|Hp-rKf^&{ zc$h_^sB@79GVBvKA-3)W5m>Q%LUnpzP>(3o8@CUpF!THsnSJ-SU=Ste7u~@#XmQRj zmx~19cYlx3I;=bAZ%7e@G^9(s5{O64E$m<*rHF&wVaNIPHR*!d{CE&!+~+S5x2GU? zkv_@oVJ0>fy7soX=cUo0_lUI%`{OG&KU)6e&g$UxANu;y&A4H__1!@fa{tJVWE0aQ z*)p7-cVloLh&b}oY|(xAA3Q_L1t-A_%k{Tz{US|p#EuA5Us@RVlC0~1>1C!`z1T32DKy8GjC~|Ca>V0TdTw4D|fG6|8#lf=AwIb zO$6(RcJBP6=Dd3!kx>D?=*|!A@u=r-zu#sG!ZBsa^Vo+Z{wtc}OFcO-GM+z$h$Jv5 zflC_@8h|*?T97sy&W+9f{?R`{oCLrj2ydg3>!7?2ktaYfo&o`sNy!w26$0UDyuuk`B!h_!L_g$}$QAWXJJ*pfvIp!m*y>w-RqY#;Wke#LYJYEaUl=aZ6?2 z;ueLRJK6>8d}RFAxJ&7G3+Zpw^G(f&}o+09R1Bgo@{$6H}<^3@w^Qr%lK#E z6%EhZ9QYAs&7P-1NV$31x1t>7o|3M;x?S=^(jI%T(p+jI!^z~7gsnSswr!urTgA^- zz3rAVpVI2`Kdn4Q6JLX%oUw=h7D%08yi%UW5A3(F-zh=3M1G)>b8?pQ*HCVQSfe}4VW$3H D6uSyw literal 0 HcmV?d00001 diff --git a/alert.py b/alert.py index 7447068..d7444a2 100644 --- a/alert.py +++ b/alert.py @@ -9,79 +9,20 @@ import argparse from dotenv import load_dotenv import os load_dotenv() +import threading -TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") -TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN") -SLEEP_TIME=120 +from worker import Worker -def request(product_name, n_articles, latitude, longitude, condition, min_price, max_price): - url = (f"https://api.wallapop.com/api/v3/general/search?keywords={product_name}" - f"&order_by=newest&latitude={latitude}" - f"&longitude={longitude}" - f"&min_sale_price={min_price}" - f"&max_sale_price={max_price}" - f"&filters_source=quick_filters&language=es_ES") - - if condition != "all": - url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all +def parse_json_file(): + f = open("args.json") + return json.load(f) - response = requests.get(url) - while response.status_code != 200: - print(f"Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Sleep...") - time.sleep(SLEEP_TIME) - response = requests.get(url) - - json_data=json.loads(response.text) - return json_data['search_objects'] - - -# We'll insert ignore current items in Wallapop, -# Only alert new articles published from NOW! -def first_run(): - list = [] - articles = request(args.name, 0, args.latitude, args.longitude, args.condition, args.min_price, args.max_price) - for article in articles: - list.insert(0, article['id']) - - return list -def main(args): - bot = telegram.Bot(token = TELEGRAM_TOKEN) - list = first_run() +def main(): + args = parse_json_file() - while True: - articles = request(args.name, 0, args.latitude, args.longitude, args.condition, args.min_price, args.max_price) - for article in articles: - if not article['id'] in list: - bot.send_photo(TELEGRAM_CHANNEL_ID, article['images'][0]['original']) - try: - bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n" - f"*Descripción*: {article['description']}\n" - f"*Precio*: {article['price']} {article['currency']}\n" - f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})" - , "MARKDOWN") - except: - bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n" - f"*Descripción*: Descripción inválida\n" - f"*Precio*: {article['price']} {article['currency']}\n" - f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})" - , "MARKDOWN") - list.insert(0, article['id']) - time.sleep(5) # Avoid Telegram flood restrictions - time.sleep(SLEEP_TIME) - -def argument_handler(): - parser = argparse.ArgumentParser(description='Arguments') - parser.add_argument('--name', dest='name', type=str, required=True, help='Article name') - parser.add_argument('--latitude', dest='latitude', type=str, default='40.4165', help='Latitude') - parser.add_argument('--longitude', dest='longitude', type=str, default='-3.70256', help='Longitude') - parser.add_argument('--condition', dest='condition', type=str, default='all', help='Item condition: all, new, as_good_as_new, good, fair, has_given_it_all') - parser.add_argument('--min', dest='min_price', type=str, default=0, help='Min price') - parser.add_argument('--max', dest='max_price', type=str, default=10000000, help='Max price') + for argument in args: + p = threading.Thread(target=Worker.run, args=(argument, )) + p.start() - args = parser.parse_args() - print(f"Wallapop monitor running. Checking for new items containing: \'{args.name}\' with given parameters periodically") - return args - -args = argument_handler() -main(args) \ No newline at end of file +main() \ No newline at end of file diff --git a/args.json b/args.json new file mode 100644 index 0000000..096ebe7 --- /dev/null +++ b/args.json @@ -0,0 +1,92 @@ +[ + { + "product_name": "ps4", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "40", + "max_price": "80", + "title_key_word_exclude" : ["juego", "juegos", "Juego", "mando", "Mando", "DualShock"], + "exclude": [] + }, + { + "product_name": "ps4", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "has_given_it_all", + "min_price": "20", + "max_price": "50", + "title_key_word_exclude" : [], + "exclude": [] + }, + { + "product_name": "3ds", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "15", + "max_price": "60", + "title_key_word_exclude" : ["juego", "juegos", "Juego", "Juegos", "pokemon", "Pokemon"], + "exclude": [] + }, + { + "product_name": "nvidia", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "80", + "max_price": "160", + "title_key_word_exclude" : [], + "exclude": [] + }, + { + "product_name": "gtx", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "80", + "max_price": "160", + "title_key_word_exclude" : [], + "exclude": ["1050", "950", "960"] + }, + { + "product_name": "grafica", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "80", + "max_price": "160", + "title_key_word_exclude" : [], + "exclude": ["1050", "950", "960"] + }, + { + "product_name": "iphone", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "90", + "max_price": "200", + "title_key_word_exclude" : [], + "exclude": ["iphone 6", "iphone 7", "iPhone 7", "iPhone 8", "Iphone 6", "Iphone 7"] + }, + { + "product_name": "mac", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "100", + "max_price": "200", + "title_key_word_exclude" : [], + "exclude": [] + }, + { + "product_name": "surface", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "100", + "max_price": "300", + "title_key_word_exclude" : [], + "exclude": [""] + } +] diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..9394ec5 --- /dev/null +++ b/worker.py @@ -0,0 +1,120 @@ +import time +import requests +import json +import telegram +import argparse +from dotenv import load_dotenv +import os +load_dotenv() +import threading +from proxy_requests import ProxyRequests + + +TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") +TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN") +SLEEP_TIME=30 + +class Worker: + + def request(self, product_name, n_articles, latitude='40.4165', longitude='-3.70256', condition='all', min_price=0, max_price=10000000): + url = (f"http://api.wallapop.com/api/v3/general/search?keywords={product_name}" + f"&order_by=newest&latitude={latitude}" + f"&longitude={longitude}" + f"&min_sale_price={min_price}" + f"&max_sale_price={max_price}" + f"&filters_source=quick_filters&language=es_ES") + + if condition != "all": + url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all + + while True: + response = ProxyRequests(url) + try: + response.get() + if response.get_status_code() == 200: + break + else: + print(f"\'{product_name}\' -> Wallapop returned status {response.get_status_code() }. Illegal parameters or Wallapop service is down. Retrying...") + except: + time.sleep(3) + + json_data=json.loads(response.get_raw()) + return json_data['search_objects'] + + def first_run(self, args): + list = [] + articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['condition'], args['min_price'], args['max_price']) + for article in articles: + list.insert(0, article['id']) + + return list + + def work(self, args, list): + exec_times = [] + bot = telegram.Bot(token = TELEGRAM_TOKEN) + + while True: + start_time = time.time() + articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['condition'], args['min_price'], args['max_price']) + for article in articles: + if not article['id'] in list: + if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), args['exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), args['title_key_word_exclude']): + try: + bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n" + f"*Descripción*: {article['description']}\n" + f"*Precio*: {article['price']} {article['currency']}\n" + f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})" + , "MARKDOWN") + except: + bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n" + f"*Descripción*: Descripción inválida\n" + f"*Precio*: {article['price']} {article['currency']}\n" + f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})" + , "MARKDOWN") + time.sleep(3) # Avoid Telegram flood restriction + list.insert(0, article['id']) + exec_times.append(time.time() - start_time) + print(f"\'{args['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}") + + def has_excluded_words(self, title, description, excluded_words): + for word in excluded_words: + print("EXCLUDER: Checking '" + word + "' for title: '" + title) + if word in title or word in description: + print("EXCLUDE!") + return True + return False + + def is_title_key_word_excluded(self, title, excluded_words): + for word in excluded_words: + print("Checking '" + word + "' for title: '" + title) + if word in title.split()[0]: + return True + return False + + def get_average_time(self, exec_times): + sum = 0 + for i in exec_times: + sum = sum + i + + return sum / len(exec_times) + + def get_max_time(self, exec_times): + largest = 0 + for i in exec_times: + if i > largest: + largest = i + return largest + + + def run(args): + worker = Worker() + list = worker.first_run(args) + while True: + try: + print(f"Wallapop monitor worker started. Checking for new items containing: \'{args['product_name']}\' with given parameters periodically") + worker.work(args, list) + except Exception as e: + print(f"Exception: {e}") + print(f"{args['product_name']} worker crashed. Restarting worker...") + time.sleep(15) +