MarketAlly.AIPlugin.Extensions/MarketAlly.AIPlugin.Security/Telemetry/SecurityAnalyticsTelemetry.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;

namespace MarketAlly.AIPlugin.Security.Telemetry
{
    /// <summary>
    /// Telemetry and analytics for security analysis operations
    /// </summary>
    public class SecurityAnalyticsTelemetry : IDisposable
    {
        private readonly ConcurrentQueue<TelemetryEvent> _eventQueue;
        private readonly Timer _flushTimer;
        private readonly string _telemetryPath;
        private readonly bool _isEnabled;
        private bool _disposed;

        public SecurityAnalyticsTelemetry(string telemetryPath = null, bool enabled = true)
        {
            _telemetryPath = telemetryPath ?? Path.Combine(Path.GetTempPath(), "MarketAlly.Security.Telemetry");
            _isEnabled = enabled;
            _eventQueue = new ConcurrentQueue<TelemetryEvent>();
            
            if (_isEnabled)
            {
                EnsureTelemetryDirectory();
                _flushTimer = new Timer(FlushEvents, null, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1));
            }
        }

        /// <summary>
        /// Tracks vulnerability detection metrics
        /// </summary>
        public void TrackVulnerabilityDetection(VulnerabilityMetrics metrics)
        {
            if (!_isEnabled || metrics == null) return;

            var telemetryEvent = new TelemetryEvent
            {
                EventType = "VulnerabilityDetection",
                Timestamp = DateTime.UtcNow,
                Data = new Dictionary<string, object>
                {
                    ["vulnerabilityType"] = metrics.VulnerabilityType,
                    ["severity"] = metrics.Severity,
                    ["fileName"] = Path.GetFileName(metrics.FilePath),
                    ["fileExtension"] = Path.GetExtension(metrics.FilePath),
                    ["lineNumber"] = metrics.LineNumber,
                    ["patternName"] = metrics.PatternName,
                    ["isConfirmed"] = metrics.IsConfirmed,
                    ["falsePositive"] = metrics.IsFalsePositive
                }
            };

            _eventQueue.Enqueue(telemetryEvent);
        }

        /// <summary>
        /// Tracks performance metrics for analysis operations
        /// </summary>
        public void TrackPerformanceMetrics(AnalysisPerformance performance)
        {
            if (!_isEnabled || performance == null) return;

            var telemetryEvent = new TelemetryEvent
            {
                EventType = "PerformanceMetrics",
                Timestamp = DateTime.UtcNow,
                Data = new Dictionary<string, object>
                {
                    ["operationType"] = performance.OperationType,
                    ["durationMs"] = performance.Duration.TotalMilliseconds,
                    ["filesAnalyzed"] = performance.FilesAnalyzed,
                    ["totalFileSize"] = performance.TotalFileSize,
                    ["memoryUsedMB"] = performance.MemoryUsedMB,
                    ["parallelism"] = performance.ParallelismLevel,
                    ["cacheHitRatio"] = performance.CacheHitRatio,
                    ["success"] = performance.Success,
                    ["errorMessage"] = performance.ErrorMessage
                }
            };

            _eventQueue.Enqueue(telemetryEvent);
        }

        /// <summary>
        /// Tracks plugin usage statistics
        /// </summary>
        public void TrackPluginUsage(PluginUsageMetrics usage)
        {
            if (!_isEnabled || usage == null) return;

            var telemetryEvent = new TelemetryEvent
            {
                EventType = "PluginUsage",
                Timestamp = DateTime.UtcNow,
                Data = new Dictionary<string, object>
                {
                    ["pluginName"] = usage.PluginName,
                    ["version"] = usage.Version,
                    ["executionTimeMs"] = usage.ExecutionTime.TotalMilliseconds,
                    ["parametersUsed"] = usage.ParametersUsed,
                    ["resultCount"] = usage.ResultCount,
                    ["success"] = usage.Success,
                    ["errorType"] = usage.ErrorType
                }
            };

            _eventQueue.Enqueue(telemetryEvent);
        }

        /// <summary>
        /// Tracks security scan session metrics
        /// </summary>
        public void TrackScanSession(ScanSessionMetrics session)
        {
            if (!_isEnabled || session == null) return;

            var telemetryEvent = new TelemetryEvent
            {
                EventType = "ScanSession",
                Timestamp = DateTime.UtcNow,
                Data = new Dictionary<string, object>
                {
                    ["sessionId"] = session.SessionId,
                    ["targetPath"] = session.TargetPath,
                    ["scanType"] = session.ScanType,
                    ["totalDurationMs"] = session.TotalDuration.TotalMilliseconds,
                    ["totalIssues"] = session.TotalIssues,
                    ["criticalIssues"] = session.CriticalIssues,
                    ["highIssues"] = session.HighIssues,
                    ["mediumIssues"] = session.MediumIssues,
                    ["lowIssues"] = session.LowIssues,
                    ["filesScanned"] = session.FilesScanned,
                    ["totalFileSize"] = session.TotalFileSize,
                    ["pluginsUsed"] = session.PluginsUsed,
                    ["configurationUsed"] = session.ConfigurationUsed
                }
            };

            _eventQueue.Enqueue(telemetryEvent);
        }

        /// <summary>
        /// Tracks false positive feedback
        /// </summary>
        public void TrackFalsePositiveFeedback(FalsePositiveFeedback feedback)
        {
            if (!_isEnabled || feedback == null) return;

            var telemetryEvent = new TelemetryEvent
            {
                EventType = "FalsePositiveFeedback",
                Timestamp = DateTime.UtcNow,
                Data = new Dictionary<string, object>
                {
                    ["patternName"] = feedback.PatternName,
                    ["vulnerabilityType"] = feedback.VulnerabilityType,
                    ["isFalsePositive"] = feedback.IsFalsePositive,
                    ["confidence"] = feedback.Confidence,
                    ["userFeedback"] = feedback.UserFeedback,
                    ["context"] = feedback.Context
                }
            };

            _eventQueue.Enqueue(telemetryEvent);
        }

        /// <summary>
        /// Generates analytics report from collected telemetry
        /// </summary>
        public async Task<SecurityAnalyticsReport> GenerateAnalyticsReportAsync(DateTime? startDate = null, DateTime? endDate = null)
        {
            if (!_isEnabled) return new SecurityAnalyticsReport();

            var start = startDate ?? DateTime.UtcNow.AddDays(-30);
            var end = endDate ?? DateTime.UtcNow;

            var events = await LoadTelemetryEventsAsync(start, end);
            
            return new SecurityAnalyticsReport
            {
                ReportPeriod = new DateRange { Start = start, End = end },
                VulnerabilityTrends = AnalyzeVulnerabilityTrends(events),
                PerformanceMetrics = AnalyzePerformance(events),
                PluginUsageStats = AnalyzePluginUsage(events),
                FalsePositiveRates = AnalyzeFalsePositives(events),
                RecommendationsForImprovement = GenerateRecommendations(events)
            };
        }

        /// <summary>
        /// Gets real-time analytics dashboard data
        /// </summary>
        public async Task<DashboardMetrics> GetDashboardMetricsAsync()
        {
            if (!_isEnabled) return new DashboardMetrics();

            // Get recent events for dashboard
            var recentEvents = await LoadTelemetryEventsAsync(DateTime.UtcNow.AddHours(-24), DateTime.UtcNow);

            return new DashboardMetrics
            {
                TotalScansToday = recentEvents.FindAll(e => e.EventType == "ScanSession").Count,
                VulnerabilitiesDetectedToday = recentEvents.FindAll(e => e.EventType == "VulnerabilityDetection").Count,
                AverageAnalysisTime = CalculateAverageAnalysisTime(recentEvents),
                TopVulnerabilityTypes = GetTopVulnerabilityTypes(recentEvents),
                SystemHealth = CalculateSystemHealth(recentEvents),
                LastUpdated = DateTime.UtcNow
            };
        }

        private void EnsureTelemetryDirectory()
        {
            if (!Directory.Exists(_telemetryPath))
            {
                Directory.CreateDirectory(_telemetryPath);
            }
        }

        private void FlushEvents(object state)
        {
            if (_eventQueue.IsEmpty) return;

            try
            {
                var events = new List<TelemetryEvent>();
                while (_eventQueue.TryDequeue(out var telemetryEvent) && events.Count < 1000)
                {
                    events.Add(telemetryEvent);
                }

                if (events.Count > 0)
                {
                    var fileName = $"telemetry_{DateTime.UtcNow:yyyyMMdd_HHmmss}.json";
                    var filePath = Path.Combine(_telemetryPath, fileName);
                    
                    var json = JsonSerializer.Serialize(events, new JsonSerializerOptions 
                    { 
                        WriteIndented = true,
                        PropertyNamingPolicy = JsonNamingPolicy.CamelCase 
                    });
                    
                    File.WriteAllText(filePath, json);
                }
            }
            catch (Exception ex)
            {
                // Log error but don't throw to avoid breaking the application
                Debug.WriteLine($"Failed to flush telemetry events: {ex.Message}");
            }
        }

        private async Task<List<TelemetryEvent>> LoadTelemetryEventsAsync(DateTime start, DateTime end)
        {
            var events = new List<TelemetryEvent>();

            if (!Directory.Exists(_telemetryPath)) return events;

            var files = Directory.GetFiles(_telemetryPath, "telemetry_*.json");
            
            foreach (var file in files)
            {
                try
                {
                    var json = await File.ReadAllTextAsync(file);
                    var fileEvents = JsonSerializer.Deserialize<List<TelemetryEvent>>(json, new JsonSerializerOptions
                    {
                        PropertyNameCaseInsensitive = true
                    });

                    if (fileEvents != null)
                    {
                        events.AddRange(fileEvents.FindAll(e => e.Timestamp >= start && e.Timestamp <= end));
                    }
                }
                catch (Exception ex)
                {
                    Debug.WriteLine($"Failed to load telemetry file {file}: {ex.Message}");
                }
            }

            return events;
        }

        private VulnerabilityTrends AnalyzeVulnerabilityTrends(List<TelemetryEvent> events)
        {
            var vulnEvents = events.FindAll(e => e.EventType == "VulnerabilityDetection");
            
            var trends = new VulnerabilityTrends();
            
            foreach (var evt in vulnEvents)
            {
                if (evt.Data.TryGetValue("vulnerabilityType", out var typeObj) && typeObj is string type)
                {
                    trends.VulnerabilityTypeDistribution[type] = trends.VulnerabilityTypeDistribution.GetValueOrDefault(type, 0) + 1;
                }

                if (evt.Data.TryGetValue("severity", out var severityObj) && severityObj is string severity)
                {
                    trends.SeverityDistribution[severity] = trends.SeverityDistribution.GetValueOrDefault(severity, 0) + 1;
                }
            }

            return trends;
        }

        private PerformanceAnalytics AnalyzePerformance(List<TelemetryEvent> events)
        {
            var perfEvents = events.FindAll(e => e.EventType == "PerformanceMetrics");
            
            var analytics = new PerformanceAnalytics();
            
            if (perfEvents.Count > 0)
            {
                var durations = new List<double>();
                var fileCounts = new List<int>();
                
                foreach (var evt in perfEvents)
                {
                    if (evt.Data.TryGetValue("durationMs", out var durationObj) && durationObj is double duration)
                        durations.Add(duration);
                    
                    if (evt.Data.TryGetValue("filesAnalyzed", out var filesObj) && filesObj is int files)
                        fileCounts.Add(files);
                }

                analytics.AverageAnalysisTime = durations.Count > 0 ? durations.Sum() / durations.Count : 0;
                analytics.MedianAnalysisTime = CalculateMedian(durations);
                analytics.AverageFilesPerScan = fileCounts.Count > 0 ? fileCounts.Sum() / fileCounts.Count : 0;
            }

            return analytics;
        }

        private PluginUsageAnalytics AnalyzePluginUsage(List<TelemetryEvent> events)
        {
            var pluginEvents = events.FindAll(e => e.EventType == "PluginUsage");
            
            var analytics = new PluginUsageAnalytics();
            
            foreach (var evt in pluginEvents)
            {
                if (evt.Data.TryGetValue("pluginName", out var nameObj) && nameObj is string name)
                {
                    analytics.PluginUsageCount[name] = analytics.PluginUsageCount.GetValueOrDefault(name, 0) + 1;
                }
            }

            return analytics;
        }

        private FalsePositiveAnalytics AnalyzeFalsePositives(List<TelemetryEvent> events)
        {
            var fpEvents = events.FindAll(e => e.EventType == "FalsePositiveFeedback");
            
            var analytics = new FalsePositiveAnalytics();
            
            foreach (var evt in fpEvents)
            {
                if (evt.Data.TryGetValue("patternName", out var patternObj) && patternObj is string pattern &&
                    evt.Data.TryGetValue("isFalsePositive", out var isFpObj) && isFpObj is bool isFp)
                {
                    if (!analytics.PatternFalsePositiveRates.ContainsKey(pattern))
                        analytics.PatternFalsePositiveRates[pattern] = new FalsePositiveRate();
                    
                    analytics.PatternFalsePositiveRates[pattern].TotalReports++;
                    if (isFp) analytics.PatternFalsePositiveRates[pattern].FalsePositives++;
                }
            }

            // Calculate rates
            foreach (var kvp in analytics.PatternFalsePositiveRates)
            {
                kvp.Value.Rate = kvp.Value.TotalReports > 0 ? 
                    (double)kvp.Value.FalsePositives / kvp.Value.TotalReports : 0;
            }

            return analytics;
        }

        private List<string> GenerateRecommendations(List<TelemetryEvent> events)
        {
            var recommendations = new List<string>();

            // Analyze performance issues
            var perfEvents = events.FindAll(e => e.EventType == "PerformanceMetrics");
            var slowScans = perfEvents.FindAll(e => 
                e.Data.TryGetValue("durationMs", out var durationObj) && 
                durationObj is double duration && duration > 60000); // Over 1 minute

            if (slowScans.Count > perfEvents.Count * 0.2) // More than 20% of scans are slow
            {
                recommendations.Add("Consider optimizing scan performance - many scans are taking over 1 minute");
            }

            // Analyze false positive rates
            var fpEvents = events.FindAll(e => e.EventType == "FalsePositiveFeedback");
            var highFpPatterns = fpEvents.GroupBy(e => e.Data.GetValueOrDefault("patternName", "unknown"))
                .Where(g => g.Count(e => e.Data.GetValueOrDefault("isFalsePositive", false) is bool fp && fp) > 3);

            foreach (var pattern in highFpPatterns)
            {
                recommendations.Add($"Review pattern '{pattern.Key}' - high false positive rate detected");
            }

            return recommendations;
        }

        private double CalculateMedian(List<double> values)
        {
            if (values.Count == 0) return 0;
            
            values.Sort();
            int middle = values.Count / 2;
            
            if (values.Count % 2 == 0)
                return (values[middle - 1] + values[middle]) / 2.0;
            else
                return values[middle];
        }

        private double CalculateAverageAnalysisTime(List<TelemetryEvent> events)
        {
            var perfEvents = events.FindAll(e => e.EventType == "PerformanceMetrics");
            if (perfEvents.Count == 0) return 0;

            var totalTime = 0.0;
            var count = 0;

            foreach (var evt in perfEvents)
            {
                if (evt.Data.TryGetValue("durationMs", out var durationObj) && durationObj is double duration)
                {
                    totalTime += duration;
                    count++;
                }
            }

            return count > 0 ? totalTime / count : 0;
        }

        private Dictionary<string, int> GetTopVulnerabilityTypes(List<TelemetryEvent> events)
        {
            var vulnEvents = events.FindAll(e => e.EventType == "VulnerabilityDetection");
            var typeCount = new Dictionary<string, int>();

            foreach (var evt in vulnEvents)
            {
                if (evt.Data.TryGetValue("vulnerabilityType", out var typeObj) && typeObj is string type)
                {
                    typeCount[type] = typeCount.GetValueOrDefault(type, 0) + 1;
                }
            }

            return typeCount.OrderByDescending(kvp => kvp.Value)
                          .Take(5)
                          .ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
        }

        private string CalculateSystemHealth(List<TelemetryEvent> events)
        {
            var perfEvents = events.FindAll(e => e.EventType == "PerformanceMetrics");
            if (perfEvents.Count == 0) return "Unknown";

            var successCount = perfEvents.Count(e => 
                e.Data.TryGetValue("success", out var successObj) && 
                successObj is bool success && success);

            var successRate = (double)successCount / perfEvents.Count;

            if (successRate >= 0.95) return "Excellent";
            if (successRate >= 0.85) return "Good";
            if (successRate >= 0.70) return "Fair";
            return "Poor";
        }

        public void Dispose()
        {
            if (_disposed) return;

            _flushTimer?.Dispose();
            FlushEvents(null); // Final flush
            _disposed = true;
        }
    }

    // Telemetry data models
    public class TelemetryEvent
    {
        public string EventType { get; set; }
        public DateTime Timestamp { get; set; }
        public Dictionary<string, object> Data { get; set; } = new Dictionary<string, object>();
    }

    public class VulnerabilityMetrics
    {
        public string VulnerabilityType { get; set; }
        public string Severity { get; set; }
        public string FilePath { get; set; }
        public int LineNumber { get; set; }
        public string PatternName { get; set; }
        public bool IsConfirmed { get; set; }
        public bool IsFalsePositive { get; set; }
    }

    public class AnalysisPerformance
    {
        public string OperationType { get; set; }
        public TimeSpan Duration { get; set; }
        public int FilesAnalyzed { get; set; }
        public long TotalFileSize { get; set; }
        public double MemoryUsedMB { get; set; }
        public int ParallelismLevel { get; set; }
        public double CacheHitRatio { get; set; }
        public bool Success { get; set; }
        public string ErrorMessage { get; set; }
    }

    public class PluginUsageMetrics
    {
        public string PluginName { get; set; }
        public string Version { get; set; }
        public TimeSpan ExecutionTime { get; set; }
        public Dictionary<string, object> ParametersUsed { get; set; }
        public int ResultCount { get; set; }
        public bool Success { get; set; }
        public string ErrorType { get; set; }
    }

    public class ScanSessionMetrics
    {
        public string SessionId { get; set; }
        public string TargetPath { get; set; }
        public string ScanType { get; set; }
        public TimeSpan TotalDuration { get; set; }
        public int TotalIssues { get; set; }
        public int CriticalIssues { get; set; }
        public int HighIssues { get; set; }
        public int MediumIssues { get; set; }
        public int LowIssues { get; set; }
        public int FilesScanned { get; set; }
        public long TotalFileSize { get; set; }
        public List<string> PluginsUsed { get; set; }
        public string ConfigurationUsed { get; set; }
    }

    public class FalsePositiveFeedback
    {
        public string PatternName { get; set; }
        public string VulnerabilityType { get; set; }
        public bool IsFalsePositive { get; set; }
        public double Confidence { get; set; }
        public string UserFeedback { get; set; }
        public string Context { get; set; }
    }

    // Analytics report models
    public class SecurityAnalyticsReport
    {
        public DateRange ReportPeriod { get; set; }
        public VulnerabilityTrends VulnerabilityTrends { get; set; }
        public PerformanceAnalytics PerformanceMetrics { get; set; }
        public PluginUsageAnalytics PluginUsageStats { get; set; }
        public FalsePositiveAnalytics FalsePositiveRates { get; set; }
        public List<string> RecommendationsForImprovement { get; set; }
    }

    public class DateRange
    {
        public DateTime Start { get; set; }
        public DateTime End { get; set; }
    }

    public class VulnerabilityTrends
    {
        public Dictionary<string, int> VulnerabilityTypeDistribution { get; set; } = new Dictionary<string, int>();
        public Dictionary<string, int> SeverityDistribution { get; set; } = new Dictionary<string, int>();
    }

    public class PerformanceAnalytics
    {
        public double AverageAnalysisTime { get; set; }
        public double MedianAnalysisTime { get; set; }
        public double AverageFilesPerScan { get; set; }
    }

    public class PluginUsageAnalytics
    {
        public Dictionary<string, int> PluginUsageCount { get; set; } = new Dictionary<string, int>();
    }

    public class FalsePositiveAnalytics
    {
        public Dictionary<string, FalsePositiveRate> PatternFalsePositiveRates { get; set; } = new Dictionary<string, FalsePositiveRate>();
    }

    public class FalsePositiveRate
    {
        public int TotalReports { get; set; }
        public int FalsePositives { get; set; }
        public double Rate { get; set; }
    }

    public class DashboardMetrics
    {
        public int TotalScansToday { get; set; }
        public int VulnerabilitiesDetectedToday { get; set; }
        public double AverageAnalysisTime { get; set; }
        public Dictionary<string, int> TopVulnerabilityTypes { get; set; }
        public string SystemHealth { get; set; }
        public DateTime LastUpdated { get; set; }
    }
}