MarketAlly.AIPlugin.Extensions/MarketAlly.AIPlugin.Analysis/Infrastructure/PerformanceOptimization.cs

using Microsoft.Extensions.Logging;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Runtime.Caching;
using System.Threading;
using System.Threading.Tasks;

namespace MarketAlly.AIPlugin.Analysis.Infrastructure
{
    /// <summary>
    /// Performance optimization utilities including caching and parallel processing
    /// </summary>
    public class PerformanceOptimization
    {
        private readonly MemoryCache _cache;
        private readonly ILogger? _logger;
        private readonly SemaphoreSlim _cacheLock = new(1, 1);

        public PerformanceOptimization(ILogger? logger = null)
        {
            _logger = logger;
            _cache = new MemoryCache("AnalysisCache");
        }

        /// <summary>
        /// Executes operations in parallel with controlled concurrency
        /// </summary>
        public async Task<IEnumerable<TResult>> ExecuteInParallelAsync<TInput, TResult>(
            IEnumerable<TInput> inputs,
            Func<TInput, Task<TResult>> operation,
            int maxConcurrency = 0,
            CancellationToken cancellationToken = default)
        {
            if (maxConcurrency <= 0)
                maxConcurrency = Environment.ProcessorCount;

            var semaphore = new SemaphoreSlim(maxConcurrency, maxConcurrency);
            var results = new ConcurrentBag<TResult>();
            var tasks = new List<Task>();

            _logger?.LogDebug("Starting parallel execution with max concurrency: {MaxConcurrency}", maxConcurrency);

            foreach (var input in inputs)
            {
                tasks.Add(ProcessItemAsync(input, operation, semaphore, results, cancellationToken));
            }

            await Task.WhenAll(tasks);

            _logger?.LogDebug("Completed parallel execution of {TaskCount} tasks", tasks.Count);
            return results;
        }

        /// <summary>
        /// Gets or sets a cached value with automatic invalidation
        /// </summary>
        public async Task<T> GetOrSetCacheAsync<T>(
            string key,
            Func<Task<T>> factory,
            TimeSpan? expiration = null,
            CancellationToken cancellationToken = default)
        {
            var actualExpiration = expiration ?? TimeSpan.FromMinutes(30);
            
            // Try to get from cache first
            if (_cache.Get(key) is T cachedValue)
            {
                _logger?.LogDebug("Cache hit for key: {CacheKey}", key);
                return cachedValue;
            }

            await _cacheLock.WaitAsync(cancellationToken);
            try
            {
                // Double-check after acquiring lock
                if (_cache.Get(key) is T doubleCheckedValue)
                {
                    _logger?.LogDebug("Cache hit after lock for key: {CacheKey}", key);
                    return doubleCheckedValue;
                }

                _logger?.LogDebug("Cache miss for key: {CacheKey}, executing factory", key);
                var value = await factory();

                var policy = new CacheItemPolicy
                {
                    AbsoluteExpiration = DateTimeOffset.UtcNow.Add(actualExpiration),
                    Priority = CacheItemPriority.Default,
                    RemovedCallback = (args) =>
                    {
                        _logger?.LogDebug("Cache item removed: {CacheKey}, Reason: {Reason}", 
                            args.CacheItem.Key, args.RemovedReason);
                    }
                };

                _cache.Set(key, value, policy);
                _logger?.LogDebug("Cached value for key: {CacheKey} with expiration: {Expiration}", 
                    key, actualExpiration);

                return value;
            }
            finally
            {
                _cacheLock.Release();
            }
        }

        /// <summary>
        /// Invalidates cache entries by pattern
        /// </summary>
        public async Task InvalidateCacheAsync(string keyPattern)
        {
            await _cacheLock.WaitAsync();
            try
            {
                var keysToRemove = new List<string>();
                
                foreach (var item in _cache)
                {
                    if (item.Key.Contains(keyPattern))
                    {
                        keysToRemove.Add(item.Key);
                    }
                }

                foreach (var key in keysToRemove)
                {
                    _cache.Remove(key);
                    _logger?.LogDebug("Removed cache key: {CacheKey}", key);
                }

                _logger?.LogInformation("Invalidated {Count} cache entries matching pattern: {Pattern}", 
                    keysToRemove.Count, keyPattern);
            }
            finally
            {
                _cacheLock.Release();
            }
        }

        /// <summary>
        /// Batches operations for more efficient processing
        /// </summary>
        public async Task<IEnumerable<TResult>> ExecuteInBatchesAsync<TInput, TResult>(
            IEnumerable<TInput> inputs,
            Func<IEnumerable<TInput>, Task<IEnumerable<TResult>>> batchOperation,
            int batchSize = 100,
            CancellationToken cancellationToken = default)
        {
            var results = new List<TResult>();
            var batch = new List<TInput>(batchSize);

            _logger?.LogDebug("Starting batch processing with batch size: {BatchSize}", batchSize);

            foreach (var input in inputs)
            {
                batch.Add(input);

                if (batch.Count >= batchSize)
                {
                    var batchResults = await batchOperation(batch);
                    results.AddRange(batchResults);
                    
                    _logger?.LogDebug("Processed batch of {BatchSize} items", batch.Count);
                    batch.Clear();
                    
                    cancellationToken.ThrowIfCancellationRequested();
                }
            }

            // Process remaining items
            if (batch.Count > 0)
            {
                var batchResults = await batchOperation(batch);
                results.AddRange(batchResults);
                _logger?.LogDebug("Processed final batch of {BatchSize} items", batch.Count);
            }

            _logger?.LogInformation("Completed batch processing of {TotalCount} items", results.Count);
            return results;
        }

        /// <summary>
        /// Manages object pooling for expensive-to-create objects
        /// </summary>
        public ObjectPool<T> CreateObjectPool<T>(
            Func<T> factory,
            Action<T>? resetAction = null,
            int maxSize = 10) where T : class
        {
            return new ObjectPool<T>(factory, resetAction, maxSize, _logger);
        }

        /// <summary>
        /// Optimizes memory usage by implementing weak references for large objects
        /// </summary>
        public WeakReferenceCache<T> CreateWeakReferenceCache<T>() where T : class
        {
            return new WeakReferenceCache<T>(_logger);
        }

        /// <summary>
        /// Gets cache statistics for monitoring
        /// </summary>
        public CacheStatistics GetCacheStatistics()
        {
            var stats = new CacheStatistics();
            
            foreach (var item in _cache)
            {
                stats.TotalItems++;
                
                if (item.Value != null)
                {
                    stats.EstimatedSize += EstimateObjectSize(item.Value);
                }
            }

            return stats;
        }

        private async Task ProcessItemAsync<TInput, TResult>(
            TInput input,
            Func<TInput, Task<TResult>> operation,
            SemaphoreSlim semaphore,
            ConcurrentBag<TResult> results,
            CancellationToken cancellationToken)
        {
            await semaphore.WaitAsync(cancellationToken);
            try
            {
                var result = await operation(input);
                results.Add(result);
            }
            finally
            {
                semaphore.Release();
            }
        }

        private static long EstimateObjectSize(object obj)
        {
            // Simple size estimation - in practice, you might want to use more sophisticated methods
            return obj switch
            {
                string str => str.Length * 2, // Unicode characters are 2 bytes
                byte[] bytes => bytes.Length,
                _ => 64 // Default estimate for other objects
            };
        }

        public void Dispose()
        {
            _cache?.Dispose();
            _cacheLock?.Dispose();
        }
    }

    /// <summary>
    /// Object pool for managing expensive-to-create objects
    /// </summary>
    public class ObjectPool<T> where T : class
    {
        private readonly ConcurrentQueue<T> _objects = new();
        private readonly Func<T> _factory;
        private readonly Action<T>? _resetAction;
        private readonly int _maxSize;
        private readonly ILogger? _logger;
        private int _currentSize;

        public ObjectPool(Func<T> factory, Action<T>? resetAction, int maxSize, ILogger? logger)
        {
            _factory = factory ?? throw new ArgumentNullException(nameof(factory));
            _resetAction = resetAction;
            _maxSize = maxSize;
            _logger = logger;
        }

        public T Get()
        {
            if (_objects.TryDequeue(out var obj))
            {
                Interlocked.Decrement(ref _currentSize);
                _logger?.LogDebug("Retrieved object from pool, current size: {CurrentSize}", _currentSize);
                return obj;
            }

            _logger?.LogDebug("Creating new object, pool was empty");
            return _factory();
        }

        public void Return(T obj)
        {
            if (obj == null) return;

            if (_currentSize < _maxSize)
            {
                _resetAction?.Invoke(obj);
                _objects.Enqueue(obj);
                Interlocked.Increment(ref _currentSize);
                _logger?.LogDebug("Returned object to pool, current size: {CurrentSize}", _currentSize);
            }
            else
            {
                _logger?.LogDebug("Pool is full, discarding object");
            }
        }

        public int Count => _currentSize;
    }

    /// <summary>
    /// Weak reference cache for memory-efficient caching of large objects
    /// </summary>
    public class WeakReferenceCache<T> where T : class
    {
        private readonly ConcurrentDictionary<string, WeakReference> _cache = new();
        private readonly ILogger? _logger;

        public WeakReferenceCache(ILogger? logger)
        {
            _logger = logger;
        }

        public void Set(string key, T value)
        {
            _cache[key] = new WeakReference(value);
            _logger?.LogDebug("Added weak reference for key: {Key}", key);
        }

        public T? Get(string key)
        {
            if (_cache.TryGetValue(key, out var weakRef) && weakRef.Target is T value)
            {
                _logger?.LogDebug("Weak reference cache hit for key: {Key}", key);
                return value;
            }

            // Clean up dead reference
            if (weakRef?.Target == null)
            {
                _cache.TryRemove(key, out _);
                _logger?.LogDebug("Cleaned up dead weak reference for key: {Key}", key);
            }

            return null;
        }

        public void Remove(string key)
        {
            _cache.TryRemove(key, out _);
            _logger?.LogDebug("Removed weak reference for key: {Key}", key);
        }

        public int Count => _cache.Count;
    }

    /// <summary>
    /// Cache statistics for monitoring performance
    /// </summary>
    public class CacheStatistics
    {
        public int TotalItems { get; set; }
        public long EstimatedSize { get; set; }
        public DateTime LastUpdated { get; set; } = DateTime.UtcNow;
    }
}