Major updates

.env

@@ -1,2 +1,2 @@
-TELEGRAM_CHANNEL_ID=@Your_Telegram_Channel
+TELEGRAM_CHANNEL_ID=@wallapopalertas
-TELEGRAM_TOKEN=Your Telegram Token
+TELEGRAM_TOKEN=1640428544:AAF2RxKqp4j4_8zKmWIaHsszE4anRwG4tGY

alert.py

@@ -9,79 +9,20 @@ import argparse
 from dotenv import load_dotenv
 import os
 load_dotenv()
+import threading
 
-TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") 
+from worker import Worker
-TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
-SLEEP_TIME=120
 
-def request(product_name, n_articles, latitude, longitude, condition, min_price, max_price):
+def parse_json_file():
-    url = (f"https://api.wallapop.com/api/v3/general/search?keywords={product_name}"
+    f = open("args.json")
-                f"&order_by=newest&latitude={latitude}"
+    return json.load(f)
-                f"&longitude={longitude}"
-                f"&min_sale_price={min_price}" 
-                f"&max_sale_price={max_price}" 
-                f"&filters_source=quick_filters&language=es_ES")
-                
-    if condition != "all":
-        url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all
 
-    response = requests.get(url)
-    while response.status_code != 200:
-        print(f"Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Sleep...")
-        time.sleep(SLEEP_TIME)
-        response = requests.get(url)
-
-    json_data=json.loads(response.text)
-    return json_data['search_objects']
-
- 
-# We'll insert ignore current items in Wallapop,
-# Only alert new articles published from NOW!
-def first_run():
-    list = []
-    articles = request(args.name, 0, args.latitude, args.longitude, args.condition, args.min_price, args.max_price)
-    for article in articles: 
-        list.insert(0, article['id'])
-
-    return list
     
-def main(args):
+def main():
-    bot = telegram.Bot(token = TELEGRAM_TOKEN)
+    args = parse_json_file()
-    list = first_run()
 
-    while True:
+    for argument in args:
-        articles = request(args.name, 0, args.latitude, args.longitude, args.condition, args.min_price, args.max_price)
+        p = threading.Thread(target=Worker.run, args=(argument, ))
-        for article in articles:
+        p.start()
-            if not article['id'] in list:
-                bot.send_photo(TELEGRAM_CHANNEL_ID, article['images'][0]['original'])
-                try:
-                    bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n"
-                                                        f"*Descripción*: {article['description']}\n"
-                                                        f"*Precio*: {article['price']} {article['currency']}\n"
-                                                        f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})"
-                                                    , "MARKDOWN")
-                except:
-                    bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n"
-                                                        f"*Descripción*: Descripción inválida\n"
-                                                        f"*Precio*: {article['price']} {article['currency']}\n"
-                                                        f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})"
-                                                    , "MARKDOWN")
-                list.insert(0, article['id'])
-                time.sleep(5) # Avoid Telegram flood restrictions
-        time.sleep(SLEEP_TIME) 
-    
-def argument_handler():
-    parser = argparse.ArgumentParser(description='Arguments')
-    parser.add_argument('--name', dest='name', type=str, required=True, help='Article name')
-    parser.add_argument('--latitude', dest='latitude', type=str, default='40.4165', help='Latitude')
-    parser.add_argument('--longitude', dest='longitude', type=str, default='-3.70256', help='Longitude')
-    parser.add_argument('--condition', dest='condition', type=str, default='all', help='Item condition: all, new, as_good_as_new, good, fair, has_given_it_all')
-    parser.add_argument('--min', dest='min_price', type=str, default=0, help='Min price')
-    parser.add_argument('--max', dest='max_price', type=str, default=10000000, help='Max price')
 
-    args = parser.parse_args()
+main()
-    print(f"Wallapop monitor running. Checking for new items containing: \'{args.name}\' with given parameters periodically")
-    return args
-
-args = argument_handler()
-main(args)

args.json

@@ -0,0 +1,92 @@
+[
+  {
+    "product_name": "ps4",
+    "latitude": "40.4165",
+    "longitude": "-3.70256",
+    "condition": "all",
+    "min_price": "40",
+    "max_price": "80",
+	"title_key_word_exclude" : ["juego", "juegos", "Juego", "mando", "Mando", "DualShock"],
+	"exclude": []
+  },
+  {
+    "product_name": "ps4",
+    "latitude": "40.4165",
+    "longitude": "-3.70256",
+    "condition": "has_given_it_all",
+    "min_price": "20",
+    "max_price": "50",
+	"title_key_word_exclude" : [],
+	"exclude": []
+  },
+  {
+    "product_name": "3ds",
+    "latitude": "40.4165",
+    "longitude": "-3.70256",
+    "condition": "all",
+    "min_price": "15",
+    "max_price": "60",
+	"title_key_word_exclude" : ["juego", "juegos", "Juego", "Juegos", "pokemon", "Pokemon"],
+	"exclude": []
+  },
+  {
+    "product_name": "nvidia",
+    "latitude": "40.4165",
+    "longitude": "-3.70256",
+    "condition": "all",
+    "min_price": "80",
+    "max_price": "160",
+	"title_key_word_exclude" : [],
+	"exclude": []
+  },
+  {
+    "product_name": "gtx",
+    "latitude": "40.4165",
+    "longitude": "-3.70256",
+    "condition": "all",
+    "min_price": "80",
+    "max_price": "160",
+	"title_key_word_exclude" : [],
+	"exclude": ["1050", "950", "960"]
+  },
+  {
+    "product_name": "grafica",
+    "latitude": "40.4165",
+    "longitude": "-3.70256",
+    "condition": "all",
+    "min_price": "80",
+    "max_price": "160",
+	"title_key_word_exclude" : [],
+	"exclude": ["1050", "950", "960"]
+  },
+  {
+    "product_name": "iphone",
+    "latitude": "40.4165",
+    "longitude": "-3.70256",
+    "condition": "all",
+    "min_price": "90",
+    "max_price": "200",
+	"title_key_word_exclude" : [],
+	"exclude": ["iphone 6", "iphone 7", "iPhone 7", "iPhone 8", "Iphone 6", "Iphone 7"]
+  },
+  {
+    "product_name": "mac",
+    "latitude": "40.4165",
+    "longitude": "-3.70256",
+    "condition": "all",
+    "min_price": "100",
+    "max_price": "200",
+	"title_key_word_exclude" : [],
+	"exclude": []
+  },
+  {
+    "product_name": "surface",
+    "latitude": "40.4165",
+    "longitude": "-3.70256",
+    "condition": "all",
+    "min_price": "100",
+    "max_price": "300",
+	"title_key_word_exclude" : [],
+	"exclude": [""]
+  }
+]

worker.py

@@ -0,0 +1,120 @@
+import time
+import requests
+import json
+import telegram
+import argparse
+from dotenv import load_dotenv
+import os
+load_dotenv()
+import threading
+from proxy_requests import ProxyRequests
+
+
+TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") 
+TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
+SLEEP_TIME=30
+
+class Worker:
+
+    def request(self, product_name, n_articles, latitude='40.4165', longitude='-3.70256', condition='all', min_price=0, max_price=10000000):
+        url = (f"http://api.wallapop.com/api/v3/general/search?keywords={product_name}"
+                    f"&order_by=newest&latitude={latitude}"
+                    f"&longitude={longitude}"
+                    f"&min_sale_price={min_price}" 
+                    f"&max_sale_price={max_price}" 
+                    f"&filters_source=quick_filters&language=es_ES")
+                    
+        if condition != "all":
+            url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all
+
+        while True:
+            response = ProxyRequests(url)
+            try:
+                response.get()
+                if response.get_status_code() == 200:
+                    break
+                else:
+                    print(f"\'{product_name}\' -> Wallapop returned status {response.get_status_code() }. Illegal parameters or Wallapop service is down. Retrying...")
+            except:
+                time.sleep(3)
+
+        json_data=json.loads(response.get_raw())
+        return json_data['search_objects']
+
+    def first_run(self, args):
+        list = []
+        articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['condition'], args['min_price'], args['max_price'])
+        for article in articles: 
+            list.insert(0, article['id'])
+        
+        return list
+
+    def work(self, args, list):
+        exec_times = []
+        bot = telegram.Bot(token = TELEGRAM_TOKEN)
+        
+        while True:
+            start_time = time.time()
+            articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['condition'], args['min_price'], args['max_price'])
+            for article in articles:
+                if not article['id'] in list:
+                    if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), args['exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), args['title_key_word_exclude']):
+                        try:
+                            bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n"
+                                                                f"*Descripción*: {article['description']}\n"
+                                                                f"*Precio*: {article['price']} {article['currency']}\n"
+                                                                f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})"
+                                                            , "MARKDOWN")
+                        except:
+                            bot.send_message(TELEGRAM_CHANNEL_ID, f"*Artículo*: {article['title']}\n"
+                                                                f"*Descripción*: Descripción inválida\n"
+                                                                f"*Precio*: {article['price']} {article['currency']}\n"
+                                                                f"[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})"
+                                                            , "MARKDOWN")
+                        time.sleep(3) # Avoid Telegram flood restriction
+                    list.insert(0, article['id'])   
+            exec_times.append(time.time() - start_time)
+            print(f"\'{args['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}")
+
+    def has_excluded_words(self, title, description, excluded_words):
+        for word in excluded_words:
+            print("EXCLUDER: Checking '" + word + "' for title: '" + title)
+            if word in title or word in description:
+                print("EXCLUDE!")
+                return True
+        return False
+    
+    def is_title_key_word_excluded(self, title, excluded_words):
+        for word in excluded_words:
+            print("Checking '" + word + "' for title: '" + title)
+            if word in title.split()[0]:
+                return True
+        return False
+
+    def get_average_time(self, exec_times):
+        sum = 0
+        for i in exec_times:
+            sum = sum + i
+        
+        return sum / len(exec_times)
+
+    def get_max_time(self, exec_times):
+        largest = 0
+        for i in exec_times:
+            if i > largest:
+                largest = i
+        return largest
+        
+
+    def run(args):
+        worker = Worker()
+        list = worker.first_run(args)
+        while True:
+            try:
+                print(f"Wallapop monitor worker started. Checking for new items containing: \'{args['product_name']}\' with given parameters periodically")
+                worker.work(args, list)
+            except Exception as e:
+                print(f"Exception: {e}")
+                print(f"{args['product_name']} worker crashed. Restarting worker...")
+                time.sleep(15)
+