import json import logging from logging.handlers import RotatingFileHandler from concurrent.futures import ThreadPoolExecutor import os import shutil from datalayer.item_monitor import ItemMonitor from datalayer.general_monitor import GeneralMonitor from managers.worker import Worker from managers.queue_manager import QueueManager def initialize_config_files(): """ Inicializa los archivos de configuración si no existen, copiando los archivos .sample correspondientes. """ base_dir = os.path.dirname(os.path.abspath(__file__)) config_files = [ ('config.yaml', 'config.sample.yaml'), ('workers.json', 'workers.sample.json') ] for config_file, sample_file in config_files: config_path = os.path.join(base_dir, config_file) sample_path = os.path.join(base_dir, sample_file) if not os.path.exists(config_path): if os.path.exists(sample_path): shutil.copy2(sample_path, config_path) print(f"✓ Archivo '{config_file}' creado desde '{sample_file}'") print(f" Por favor, edita '{config_file}' con tu configuración antes de continuar.") else: raise FileNotFoundError( f"No se encontró ni '{config_file}' ni '{sample_file}'. " f"Por favor, crea uno de estos archivos." ) def configure_logger(): logging.getLogger("httpx").setLevel(logging.WARNING) console_handler = logging.StreamHandler() console_handler.setLevel(logging.INFO) console_handler.setFormatter(logging.Formatter('%(levelname)s [%(asctime)s] %(name)s - %(message)s')) file_handler = RotatingFileHandler('monitor.log', maxBytes=10e6) file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) # Configure the root logger with both handlers logging.basicConfig(level=logging.NOTSET, handlers=[console_handler, file_handler]) def parse_items_to_monitor(): import os base_dir = os.path.dirname(os.path.abspath(__file__)) workers_path = os.path.join(base_dir, "workers.json") with open(workers_path) as f: args = json.load(f) if 'items' not in args: raise ValueError("Missing mandatory field: items") items = [ItemMonitor.load_from_json(item) for item in args['items']] general_args = GeneralMonitor.load_from_json(args['general']) return items, general_args if __name__ == "__main__": initialize_config_files() configure_logger() items, general_args = parse_items_to_monitor() queue_manager = QueueManager() with ThreadPoolExecutor(max_workers=1000) as executor: for item in items: worker = Worker(item, general_args, queue_manager) executor.submit(worker.run)