// apps/api/src/services/scrape.service.js

const axios = require('axios');
const cheerio = require('cheerio');
const { DatabaseClient } = require('../../../../packages/db/client');
const Helpers = require('../../../../packages/shared/helpers');
const extractService = require('./extract.service');
const scoreService = require('./score.service');
const antiBlockScraper = require('./antiblockscraper.service');
const { SOURCE_TYPES } = require('../../../../packages/shared/constants');

// Centralize intent and filtering patterns via the unified engine package
const UnifiedIntentEngine = require('../../../../packages/shared/UnifiedIntentEngine');

const db = new DatabaseClient();

class ScrapeService {
    constructor() {
        this.isScraping = false;
        this.queue = [];
        this.dailyScrapeCount = 0;
        this.lastScrapeDate = new Date().toDateString();
    }

    async searchAndScrape(query, location = null, source = null) {
        const today = new Date().toDateString();
        if (today !== this.lastScrapeDate) {
            this.dailyScrapeCount = 0;
            this.lastScrapeDate = today;
        }
        
        const maxDaily = parseInt(process.env.MAX_DAILY_SCRAPES) || 500;
        if (this.dailyScrapeCount >= maxDaily) {
            console.log(`[Engine] Daily operational limits reached (${maxDaily}). Halting thread operations.`);
            return { status: 'limit_reached', message: 'Daily limit reached' };
        }
        
        if (this.isScraping) {
            this.queue.push({ query, location, source });
            return { status: 'queued', position: this.queue.length };
        }
        
        this.isScraping = true;
        
        try {
            console.log(`\n🚀 Ingesting Search Footprint Pattern: "${query}"`);
            const searchResults = await antiBlockScraper.comprehensiveSearch(query, location);
            let savedCount = 0;
            
            for (const result of searchResults) {
                const combinedBody = `${result.title || ''} ${result.snippet || ''}`;
                
                // STAGE 1: Drop competitor/supplier pitches instantly using Unified Engine
                const intentResult = UnifiedIntentEngine.analyze(combinedBody);
                
                if (intentResult.isIntent && intentResult.confidence >= 30) {
                    const lead = await this.processResult(result, query, intentResult);
                    if (lead) {
                        const isNew = await this.saveLead(lead);
                        if (isNew) savedCount++;
                    }
                } else {
                    console.log(`[FILTERED] Skipping item via footprint pass: ${intentResult.reason || 'Low confidence / Competitor pitch'}`);
                }
                await Helpers.delay(400); // Prevents rapid execution flags on the local server
            }
            
            this.dailyScrapeCount += savedCount;
            return { status: 'completed', count: savedCount };
            
        } catch (error) {
            console.error(`❌ Thread exception error for [${query}]:`, error.message);
            return { status: 'failed', error: error.message };
        } finally {
            this.isScraping = false;
            if (this.queue.length > 0) {
                const next = this.queue.shift();
                setTimeout(() => {
                    this.searchAndScrape(next.query, next.location, next.source);
                }, 4000);
            }
        }
    }

    // Dynamic relational execution loops for scaling over 500+ channels
    async executeFeedIngestion() {
        console.log('🔄 Querying database for active scheduled channel footprints...');
        
        // Fetch up to 15 pending operational targets per batch execution sequence
        const targetChannels = await db.query(
            `SELECT * FROM pure_demand_channels 
             WHERE is_active = 1 AND next_run_at <= CURRENT_TIMESTAMP 
             ORDER BY next_run_at ASC LIMIT 15`
        );

        if (!targetChannels || targetChannels.length === 0) {
            console.log('💡 All target channels are resting within baseline delay windows.');
            return;
        }

        for (const channel of targetChannels) {
            const startTime = Date.now();
            let savedLeadsCount = 0;
            let statusText = 'success';
            let failureMsg = null;

            console.log(`⚡ Channel Ingestion Fire: [${channel.name}] using [${channel.engine_type}]`);

            try {
                // Instantly lock channel execution time block out to prevent thread race conditions
                await db.run(
                    `UPDATE pure_demand_channels SET last_run_at = NOW(), 
                     next_run_at = DATE_ADD(NOW(), INTERVAL ? MINUTE) WHERE id = ?`,
                    [channel.interval_minutes, channel.id]
                );

                const config = typeof channel.extraction_config === 'string' 
                    ? JSON.parse(channel.extraction_config) 
                    : (channel.extraction_config || {});

                const requestHeaders = {
                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
                    'Accept': 'application/json, text/xml, application/xml, */*',
                    ...config.headers
                };

                const response = await axios.get(channel.target_url, {
                    headers: requestHeaders,
                    timeout: 20000,
                    validateStatus: (status) => status === 200
                });

                let structuredItems = [];

                // Router matrix mapping payload shape directly from column definitions
                if (channel.engine_type === 'rest_json') {
                    const dataRoot = config.root_key ? response.data[config.root_key] : response.data;
                    const itemsArray = Array.isArray(dataRoot) ? dataRoot : (dataRoot.jobs || dataRoot.leads || []);
                    
                    structuredItems = itemsArray.map(item => ({
                        title: item[config.title_key || 'title'],
                        description: item[config.desc_key || 'description'],
                        url: item[config.url_key || 'url'] || item['link'],
                        timestamp: item[config.date_key || 'created_at'] ? new Date(item[config.date_key || 'created_at']) : new Date()
                    }));
                } 
                else if (channel.engine_type === 'rss_xml') {
                    const $ = cheerio.load(response.data, { xmlMode: true });
                    $('item, entry').each((_, el) => {
                        const node = $(el);
                        structuredItems.push({
                            title: node.find('title').text(),
                            description: node.find('description').text() || node.find('content\\:encoded').text() || node.find('summary').text(),
                            url: node.find('link').text() || node.find('link').attr('href'),
                            timestamp: node.find('pubDate').text() ? new Date(node.find('pubDate').text()) : new Date()
                        });
                    });
                }
                else if (channel.engine_type === 'reddit_sub' || channel.engine_type === 'google_footprint') {
                    // Route to search and scrape fallback patterns managed over proxies
                    const fallbackQuery = config.search_query || channel.name;
                    const fallbackRes = await this.searchAndScrape(fallbackQuery, config.location || null, channel.name);
                    savedLeadsCount = fallbackRes.count || 0;
                    statusText = fallbackRes.status === 'failed' ? 'failed' : 'success';
                }

                // Clean data ingestion loops
                if (structuredItems.length > 0) {
                    for (const rawItem of structuredItems) {
                        const normalizedBlob = `${rawItem.title} ${rawItem.description}`;
                        
                        // Apply identical shared filter verification logic across feeds
                        const intent = UnifiedIntentEngine.analyze(normalizedBlob);

                        if (!intent.isIntent || intent.confidence < 30) {
                            continue;
                        }

                        const processed = await this.processResult({
                            title: rawItem.title,
                            snippet: rawItem.description ? rawItem.description.replace(/<[^>]*>/g, '').substring(0, 400) : '',
                            url: rawItem.url,
                            source: channel.name,
                            timestamp: rawItem.timestamp
                        }, config.fallback_category || 'technology', intent);

                        if (processed) {
                            const inserted = await this.saveLead(processed);
                            if (inserted) savedLeadsCount++;
                        }
                    }
                }

                // Reset channel failure tracking markers upon clean execution pass
                await db.run('UPDATE pure_demand_channels SET failure_count = 0 WHERE id = ?', [channel.id]);

            } catch (err) {
                statusText = 'failed';
                failureMsg = err.message;
                console.error(`❌ Execution failure at channel tracking block [${channel.name}]:`, err.message);
                
                // Track failure metrics and automatically disable channels after 5 consecutive crashes
                await db.run(
                    `UPDATE pure_demand_channels 
                     SET failure_count = failure_count + 1, 
                         is_active = CASE WHEN failure_count >= 5 THEN 0 ELSE 1 END 
                     WHERE id = ?`, 
                    [channel.id]
                );
            } finally {
                const duration = Date.now() - startTime;
                await db.run(
                    `INSERT INTO channel_execution_logs 
                     (channel_id, execution_status, leads_found, error_message, execution_duration_ms) 
                     VALUES (?, ?, ?, ?, ?)`,
                    [channel.id, statusText, savedLeadsCount, failureMsg, duration]
                );
            }
        }
    }
    
    async processResult(result, targetService, preCalculatedIntent = null) {
        const text = `${result.title} ${result.snippet || ''}`.toLowerCase();
        
        // Optimization: Use precalculated metrics passed down from primary loops if available
        const intentResult = preCalculatedIntent || UnifiedIntentEngine.analyze(text);
        
        if (!intentResult.isIntent || intentResult.confidence < 30) {
            return null;
        }
        
        // Utilize the shared engine to safely harvest clean contact matches
        const metadata = UnifiedIntentEngine.extractMetadata(text);
        
        // Extract rest of operational metrics
        const serviceNeeded = targetService; // Keeps query baseline alignment intact
        const location = extractService.extractLocation(text);
        const urgency = extractService.extractUrgency(text);
        
        const scoreData = scoreService.calculateScore({
            text,
            hasContact: metadata.phones.length > 0 || metadata.emails.length > 0,
            freshness: result.timestamp || new Date(),
            clarity: text.length > 100 ? 'high' : 'medium',
            urgency: urgency.level
        });
        
        return {
            title: result.title.substring(0, 500),
            description: (result.snippet || '').substring(0, 1000),
            service_needed: serviceNeeded,
            location: location,
            contact_phone: metadata.phones[0] || extractService.extractAllContacts(text).phone || null,
            contact_email: metadata.emails[0] || extractService.extractAllContacts(text).email || null,
            contact_whatsapp: extractService.extractAllContacts(text).whatsapp || null,
            source_url: result.url,
            source_type: result.source || SOURCE_TYPES.GOOGLE,
            urgency_level: urgency.level,
            urgency_keywords: urgency.keywords.join(','),
            confidence_score: intentResult.confidence,
            score: scoreData.score,
            score_category: scoreData.category,
            posted_date: result.timestamp || new Date(),
            discovered_at: new Date().toISOString() 
        };
    }
    
    async saveLead(leadData) {
        try {
            // High speed check utilizing standard primary uniqueness criteria constraints
            const existing = await db.get(
                'SELECT id FROM leads WHERE source_url = ?',
                [leadData.source_url]
            );
            
            if (existing) return null;
            
            const result = await db.run(
                `INSERT INTO leads (
                    title, description, service_needed, location, contact_phone, contact_email,
                    contact_whatsapp, source_url, source_type, urgency_level, urgency_keywords,
                    confidence_score, score, score_category, posted_date, discovered_at, is_active
                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)`,
                [
                    leadData.title, leadData.description, leadData.service_needed, leadData.location,
                    leadData.contact_phone, leadData.contact_email, leadData.contact_whatsapp,
                    leadData.source_url, leadData.source_type, leadData.urgency_level,
                    leadData.urgency_keywords, leadData.confidence_score, leadData.score,
                    leadData.score_category, leadData.posted_date, leadData.discovered_at
                ]
            );
            return result.insertId;
        } catch (error) {
            if (error.message.includes('Duplicate entry')) return null; // Catch silent background racing records
            console.error('Save lead execution database error:', error.message);
            return null;
        }
    }
    
    getScrapeStats() {
        return {
            isScraping: this.isScraping,
            queueLength: this.queue.length,
            dailyScrapeCount: this.dailyScrapeCount,
            lastScrapeDate: this.lastScrapeDate
        };
    }
}

module.exports = new ScrapeService();