79 lines
2.9 KiB
Python
79 lines
2.9 KiB
Python
import json
|
|
import logging
|
|
from logging.handlers import RotatingFileHandler
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import os
|
|
import shutil
|
|
|
|
from datalayer.item_monitor import ItemMonitor
|
|
from datalayer.general_monitor import GeneralMonitor
|
|
from managers.worker import Worker
|
|
from managers.queue_manager import QueueManager
|
|
|
|
def initialize_config_files():
|
|
"""
|
|
Inicializa los archivos de configuración si no existen,
|
|
copiando los archivos .sample correspondientes.
|
|
"""
|
|
base_dir = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
config_files = [
|
|
('config.yaml', 'config.sample.yaml'),
|
|
('workers.json', 'workers.sample.json')
|
|
]
|
|
|
|
for config_file, sample_file in config_files:
|
|
config_path = os.path.join(base_dir, config_file)
|
|
sample_path = os.path.join(base_dir, sample_file)
|
|
|
|
if not os.path.exists(config_path):
|
|
if os.path.exists(sample_path):
|
|
shutil.copy2(sample_path, config_path)
|
|
print(f"✓ Archivo '{config_file}' creado desde '{sample_file}'")
|
|
print(f" Por favor, edita '{config_file}' con tu configuración antes de continuar.")
|
|
else:
|
|
raise FileNotFoundError(
|
|
f"No se encontró ni '{config_file}' ni '{sample_file}'. "
|
|
f"Por favor, crea uno de estos archivos."
|
|
)
|
|
|
|
def configure_logger():
|
|
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
|
|
console_handler = logging.StreamHandler()
|
|
console_handler.setLevel(logging.INFO)
|
|
console_handler.setFormatter(logging.Formatter('%(levelname)s [%(asctime)s] %(name)s - %(message)s'))
|
|
|
|
file_handler = RotatingFileHandler('monitor.log', maxBytes=10e6)
|
|
file_handler.setLevel(logging.DEBUG)
|
|
file_handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
|
|
|
|
# Configure the root logger with both handlers
|
|
logging.basicConfig(level=logging.NOTSET,
|
|
handlers=[console_handler, file_handler])
|
|
|
|
def parse_items_to_monitor():
|
|
import os
|
|
base_dir = os.path.dirname(os.path.abspath(__file__))
|
|
workers_path = os.path.join(base_dir, "workers.json")
|
|
with open(workers_path) as f:
|
|
args = json.load(f)
|
|
if 'items' not in args:
|
|
raise ValueError("Missing mandatory field: items")
|
|
items = [ItemMonitor.load_from_json(item) for item in args['items']]
|
|
general_args = GeneralMonitor.load_from_json(args['general'])
|
|
return items, general_args
|
|
|
|
if __name__ == "__main__":
|
|
initialize_config_files()
|
|
configure_logger()
|
|
items, general_args = parse_items_to_monitor()
|
|
|
|
queue_manager = QueueManager()
|
|
|
|
with ThreadPoolExecutor(max_workers=1000) as executor:
|
|
for item in items:
|
|
worker = Worker(item, general_args, queue_manager)
|
|
executor.submit(worker.run)
|
|
|