Enhance caching mechanism and logging configuration

- Updated .gitignore to include additional IDE and OS files, as well as log and web build directories. - Expanded config.sample.yaml to include cache configuration options for memory and Redis. - Modified wallamonitor.py to load cache configuration and initialize ArticleCache. - Refactored QueueManager to utilize ArticleCache for tracking notified articles. - Improved logging setup to dynamically determine log file path based on environment.
2026-01-19 19:42:12 +01:00
parent b32b0b2e09
commit 9939c4d9ed
41 changed files with 6742 additions and 28 deletions
--- a/web/backend/.dockerignore
+++ b/web/backend/.dockerignore
@@ -0,0 +1,6 @@
+node_modules/
+npm-debug.log*
+.env
+.env.local
+*.log
+
--- a/web/backend/Dockerfile
+++ b/web/backend/Dockerfile
@@ -0,0 +1,26 @@
+FROM node:18-alpine
+
+# Instalar wget para healthcheck
+RUN apk add --no-cache wget
+
+WORKDIR /app
+
+# Copiar archivos de dependencias
+COPY package.json package-lock.json* ./
+
+# Instalar dependencias
+RUN npm ci --only=production
+
+# Copiar código de la aplicación
+COPY server.js .
+
+# Exponer puerto
+EXPOSE 3001
+
+# Healthcheck
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+  CMD wget --quiet --tries=1 --spider http://localhost:3001/api/stats || exit 1
+
+# Comando por defecto
+CMD ["node", "server.js"]
+
--- a/web/backend/package-lock.json
+++ b/web/backend/package-lock.json
--- a/web/backend/package.json
+++ b/web/backend/package.json
@@ -0,0 +1,23 @@
+{
+  "name": "wallamonitor-backend",
+  "version": "1.0.0",
+  "description": "Backend API para Wallamonitor Dashboard",
+  "main": "server.js",
+  "type": "module",
+  "scripts": {
+    "start": "node server.js",
+    "dev": "node --watch server.js"
+  },
+  "keywords": ["wallamonitor", "api", "express"],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "express": "^4.18.2",
+    "cors": "^2.8.5",
+    "ws": "^8.14.2",
+    "redis": "^4.6.10",
+    "yaml": "^2.3.4",
+    "chokidar": "^3.5.3"
+  }
+}
+
--- a/web/backend/server.js
+++ b/web/backend/server.js
@@ -0,0 +1,390 @@
+import express from 'express';
+import cors from 'cors';
+import { WebSocketServer } from 'ws';
+import { createServer } from 'http';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+import { readFileSync, writeFileSync, existsSync, statSync } from 'fs';
+import { watch } from 'chokidar';
+import yaml from 'yaml';
+import redis from 'redis';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+// En Docker, usar PROJECT_ROOT de env, sino usar ruta relativa
+const PROJECT_ROOT = process.env.PROJECT_ROOT || join(__dirname, '../..');
+
+const app = express();
+const server = createServer(app);
+const wss = new WebSocketServer({ server });
+
+app.use(cors());
+app.use(express.json());
+
+// Configuración
+const CONFIG_PATH = join(PROJECT_ROOT, 'config.yaml');
+const WORKERS_PATH = join(PROJECT_ROOT, 'workers.json');
+const FAVORITES_PATH = join(PROJECT_ROOT, 'favorites.json');
+
+// Función para obtener la ruta del log (en Docker puede estar en /data/logs)
+function getLogPath() {
+  const logsDirPath = join(PROJECT_ROOT, 'logs', 'monitor.log');
+  const rootLogPath = join(PROJECT_ROOT, 'monitor.log');
+  
+  if (existsSync(logsDirPath)) {
+    return logsDirPath;
+  }
+  return rootLogPath;
+}
+
+const LOG_PATH = getLogPath();
+
+let redisClient = null;
+let config = null;
+
+// Inicializar Redis si está configurado
+async function initRedis() {
+  try {
+    config = yaml.parse(readFileSync(CONFIG_PATH, 'utf8'));
+    const cacheConfig = config?.cache;
+    
+    if (cacheConfig?.type === 'redis') {
+      const redisConfig = cacheConfig.redis;
+      // En Docker, usar el nombre del servicio si no se especifica host
+      const redisHost = process.env.REDIS_HOST || redisConfig.host || 'localhost';
+      redisClient = redis.createClient({
+        socket: {
+          host: redisHost,
+          port: redisConfig.port || 6379,
+        },
+        password: redisConfig.password || undefined,
+        database: redisConfig.db || 0,
+      });
+      
+      redisClient.on('error', (err) => console.error('Redis Client Error', err));
+      await redisClient.connect();
+      console.log('✅ Conectado a Redis');
+    } else {
+      console.log('ℹ️  Redis no configurado, usando modo memoria');
+    }
+  } catch (error) {
+    console.error('Error inicializando Redis:', error.message);
+  }
+}
+
+// Broadcast a todos los clientes WebSocket
+function broadcast(data) {
+  wss.clients.forEach((client) => {
+    if (client.readyState === 1) { // WebSocket.OPEN
+      client.send(JSON.stringify(data));
+    }
+  });
+}
+
+// Leer archivo JSON de forma segura
+function readJSON(path, defaultValue = {}) {
+  try {
+    if (existsSync(path)) {
+      return JSON.parse(readFileSync(path, 'utf8'));
+    }
+    return defaultValue;
+  } catch (error) {
+    console.error(`Error leyendo ${path}:`, error.message);
+    return defaultValue;
+  }
+}
+
+// Escribir archivo JSON
+function writeJSON(path, data) {
+  try {
+    writeFileSync(path, JSON.stringify(data, null, 2), 'utf8');
+    return true;
+  } catch (error) {
+    console.error(`Error escribiendo ${path}:`, error.message);
+    return false;
+  }
+}
+
+// Obtener artículos notificados desde Redis
+async function getNotifiedArticles() {
+  if (!redisClient) {
+    return [];
+  }
+  
+  try {
+    const keys = await redisClient.keys('notified:*');
+    const articles = [];
+    
+    for (const key of keys) {
+      const parts = key.split(':');
+      if (parts.length >= 3) {
+        const platform = parts[1];
+        const id = parts.slice(2).join(':');
+        const ttl = await redisClient.ttl(key);
+        const value = await redisClient.get(key);
+        
+        // Intentar parsear como JSON (nuevo formato con toda la info)
+        let articleData = {};
+        try {
+          if (value && value !== '1') {
+            articleData = JSON.parse(value);
+          }
+        } catch (e) {
+          // Si no es JSON válido, usar valor por defecto
+        }
+        
+        articles.push({
+          platform: articleData.platform || platform,
+          id: articleData.id || id,
+          title: articleData.title || null,
+          description: articleData.description || null,
+          price: articleData.price || null,
+          currency: articleData.currency || null,
+          location: articleData.location || null,
+          allows_shipping: articleData.allows_shipping !== undefined ? articleData.allows_shipping : null,
+          url: articleData.url || null,
+          images: articleData.images || [],
+          modified_at: articleData.modified_at || null,
+          notifiedAt: Date.now() - (7 * 24 * 60 * 60 - ttl) * 1000,
+          expiresAt: Date.now() + ttl * 1000,
+        });
+      }
+    }
+    
+    return articles;
+  } catch (error) {
+    console.error('Error obteniendo artículos de Redis:', error.message);
+    return [];
+  }
+}
+
+// API Routes
+
+// Obtener estadísticas
+app.get('/api/stats', async (req, res) => {
+  try {
+    const workers = readJSON(WORKERS_PATH, { items: [] });
+    const favorites = readJSON(FAVORITES_PATH, []);
+    const notifiedArticles = await getNotifiedArticles();
+    
+    const stats = {
+      totalWorkers: workers.items?.length || 0,
+      activeWorkers: (workers.items || []).filter(w => !workers.disabled?.includes(w.name)).length,
+      totalFavorites: favorites.length,
+      totalNotified: notifiedArticles.length,
+      platforms: {
+        wallapop: notifiedArticles.filter(a => a.platform === 'wallapop').length,
+        vinted: notifiedArticles.filter(a => a.platform === 'vinted').length,
+      },
+    };
+    
+    res.json(stats);
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Obtener workers
+app.get('/api/workers', (req, res) => {
+  try {
+    const workers = readJSON(WORKERS_PATH, { items: [], general: {}, disabled: [] });
+    res.json(workers);
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Actualizar workers
+app.put('/api/workers', (req, res) => {
+  try {
+    const workers = req.body;
+    if (writeJSON(WORKERS_PATH, workers)) {
+      broadcast({ type: 'workers_updated', data: workers });
+      res.json({ success: true });
+    } else {
+      res.status(500).json({ error: 'Error guardando workers' });
+    }
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Obtener favoritos
+app.get('/api/favorites', (req, res) => {
+  try {
+    const favorites = readJSON(FAVORITES_PATH, []);
+    res.json(favorites);
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Añadir favorito
+app.post('/api/favorites', (req, res) => {
+  try {
+    const favorite = req.body;
+    const favorites = readJSON(FAVORITES_PATH, []);
+    
+    // Evitar duplicados
+    if (!favorites.find(f => f.id === favorite.id && f.platform === favorite.platform)) {
+      favorites.push({
+        ...favorite,
+        addedAt: new Date().toISOString(),
+      });
+      writeJSON(FAVORITES_PATH, favorites);
+      broadcast({ type: 'favorites_updated', data: favorites });
+      res.json({ success: true, favorites });
+    } else {
+      res.json({ success: false, message: 'Ya existe en favoritos' });
+    }
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Eliminar favorito
+app.delete('/api/favorites/:platform/:id', (req, res) => {
+  try {
+    const { platform, id } = req.params;
+    const favorites = readJSON(FAVORITES_PATH, []);
+    const filtered = favorites.filter(
+      f => !(f.platform === platform && f.id === id)
+    );
+    
+    writeJSON(FAVORITES_PATH, filtered);
+    broadcast({ type: 'favorites_updated', data: filtered });
+    res.json({ success: true, favorites: filtered });
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Obtener artículos notificados
+app.get('/api/articles', async (req, res) => {
+  try {
+    const articles = await getNotifiedArticles();
+    const limit = parseInt(req.query.limit) || 100;
+    const offset = parseInt(req.query.offset) || 0;
+    
+    const sorted = articles.sort((a, b) => b.notifiedAt - a.notifiedAt);
+    const paginated = sorted.slice(offset, offset + limit);
+    
+    res.json({
+      articles: paginated,
+      total: articles.length,
+      limit,
+      offset,
+    });
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Obtener logs (últimas líneas)
+app.get('/api/logs', (req, res) => {
+  try {
+    // Intentar múltiples ubicaciones posibles
+    let logFile = LOG_PATH;
+    if (!existsSync(logFile)) {
+      // Intentar en el directorio de logs
+      const altPath = join(PROJECT_ROOT, 'logs', 'monitor.log');
+      if (existsSync(altPath)) {
+        logFile = altPath;
+      } else {
+        return res.json({ logs: [] });
+      }
+    }
+    
+    // Verificar que no sea un directorio
+    try {
+      const stats = statSync(logFile);
+      if (stats.isDirectory()) {
+        return res.json({ logs: ['Error: monitor.log es un directorio. Por favor, elimínalo y reinicia.'] });
+      }
+    } catch (e) {
+      return res.json({ logs: [] });
+    }
+    
+    const logs = readFileSync(logFile, 'utf8');
+    const lines = logs.split('\n').filter(l => l.trim());
+    const limit = parseInt(req.query.limit) || 100;
+    const lastLines = lines.slice(-limit);
+    
+    res.json({ logs: lastLines });
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// Obtener configuración
+app.get('/api/config', (req, res) => {
+  try {
+    if (!config) {
+      config = yaml.parse(readFileSync(CONFIG_PATH, 'utf8'));
+    }
+    // No enviar token por seguridad
+    const safeConfig = { ...config };
+    if (safeConfig.telegram_token) {
+      safeConfig.telegram_token = '***';
+    }
+    res.json(safeConfig);
+  } catch (error) {
+    res.status(500).json({ error: error.message });
+  }
+});
+
+// WebSocket connection
+wss.on('connection', (ws) => {
+  console.log('Cliente WebSocket conectado');
+  
+  ws.on('close', () => {
+    console.log('Cliente WebSocket desconectado');
+  });
+  
+  ws.on('error', (error) => {
+    console.error('Error WebSocket:', error);
+  });
+});
+
+// Determinar la ruta del log para el watcher
+let watchLogPath = LOG_PATH;
+if (!existsSync(watchLogPath)) {
+  const altPath = join(PROJECT_ROOT, 'logs', 'monitor.log');
+  if (existsSync(altPath)) {
+    watchLogPath = altPath;
+  }
+}
+
+// Watch files for changes
+const watcher = watch([WORKERS_PATH, FAVORITES_PATH, watchLogPath].filter(p => existsSync(p)), {
+  persistent: true,
+  ignoreInitial: true,
+});
+
+watcher.on('change', (path) => {
+  console.log(`Archivo cambiado: ${path}`);
+  if (path === WORKERS_PATH) {
+    const workers = readJSON(WORKERS_PATH);
+    broadcast({ type: 'workers_updated', data: workers });
+  } else if (path === FAVORITES_PATH) {
+    const favorites = readJSON(FAVORITES_PATH);
+    broadcast({ type: 'favorites_updated', data: favorites });
+  } else if (path === LOG_PATH) {
+    broadcast({ type: 'logs_updated' });
+  }
+});
+
+// Inicializar servidor
+const PORT = process.env.PORT || 3001;
+
+async function startServer() {
+  await initRedis();
+  
+  server.listen(PORT, () => {
+    console.log(`🚀 Servidor backend ejecutándose en http://localhost:${PORT}`);
+    console.log(`📡 WebSocket disponible en ws://localhost:${PORT}`);
+  });
+}
+
+startServer().catch(console.error);
+