using MarketAlly.ProjectDetector; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using RefactorIQ.Core; using RefactorIQ.Core.Models; using RefactorIQ.Domain.Models; using RefactorIQ.Domain.Models.Aggregates; using RefactorIQ.Persistence.Models; using RefactorIQ.Services; using RefactorIQ.Services.Configuration; using RefactorIQ.Services.Interfaces; using RefactorIQ.Services.Models; using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; namespace MarketAlly.AIPlugin.Learning { public class RefactorIQIntegration { private readonly string _configPath; private readonly IConfiguration _configuration; private IServiceProvider _serviceProvider; private IRefactorIQClient _client; public RefactorIQIntegration(string configPath) { _configPath = configPath ?? Directory.GetCurrentDirectory(); _configuration = BuildConfiguration(); _serviceProvider = BuildServiceProvider(); _client = _serviceProvider.GetRequiredService(); } /// /// Create a RefactorIQIntegration instance with project-specific database /// public static RefactorIQIntegration ForProject(Guid projectId, Guid tenantId, string? databaseDirectory = null) { var services = new ServiceCollection(); services.AddLogging(builder => builder.AddConsole()); // Build configuration var configuration = new ConfigurationBuilder() .AddJsonFile("appsettings.json", optional: true) .AddEnvironmentVariables() .Build(); // Configure RefactorIQ with project-specific database services.AddRefactorIQServices(options => { // Set project-specific database path with tenant organization // Use configured path from appsettings or environment variable var baseRefactorIQPath = configuration["RefactorIQ:DatabaseBasePath"] ?? Environment.GetEnvironmentVariable("REFACTORIQ_DATABASE_PATH") ?? "/home/repositories"; // Default to repositories base path // Organize by tenant: /home/repositories/{tenantId}/refactoriq/{projectId}.refactoriq.sqlite var dbDirectory = databaseDirectory ?? Path.Combine(baseRefactorIQPath, tenantId.ToString(), "refactoriq"); Directory.CreateDirectory(dbDirectory); var dbPath = Path.Combine(dbDirectory, $"{projectId}.refactoriq.sqlite"); options.ConnectionString = $"Data Source={dbPath}"; // Configure database options options.Database.UseSolutionSpecificDatabase = false; // We're handling it manually // Configure OpenAI settings from configuration var openAIApiKey = configuration["RefactorIQ:OpenAI:ApiKey"] ?? Environment.GetEnvironmentVariable("OPENAI_API_KEY"); if (!string.IsNullOrEmpty(openAIApiKey)) { options.OpenAI.ApiKey = openAIApiKey; options.OpenAI.Model = configuration["RefactorIQ:OpenAI:Model"] ?? "text-embedding-3-small"; options.OpenAI.MaxRetries = int.Parse(configuration["RefactorIQ:OpenAI:MaxRetries"] ?? "3"); } // Configure embedding settings options.Embedding.BatchSize = int.Parse(configuration["RefactorIQ:Embedding:BatchSize"] ?? "10"); options.Embedding.EnableProgressSaving = bool.Parse(configuration["RefactorIQ:Embedding:EnableProgressSaving"] ?? "true"); options.Embedding.ProgressSaveInterval = int.Parse(configuration["RefactorIQ:Embedding:ProgressSaveInterval"] ?? "10"); }); // Add performance optimizations (caching, parallel processing) services.AddPerformanceOptimization(configuration); var serviceProvider = services.BuildServiceProvider(); // Ensure database schema is created by running migrations using (var scope = serviceProvider.CreateScope()) { var dbContext = scope.ServiceProvider.GetRequiredService(); try { // Run migrations to create the database schema (including Solutions table) dbContext.Database.Migrate(); } catch (Exception migrationEx) { // If migrations fail, try EnsureCreated as fallback var logger = serviceProvider.GetRequiredService>(); logger.LogWarning(migrationEx, "Failed to run migrations, falling back to EnsureCreated"); dbContext.Database.EnsureCreated(); } } var client = serviceProvider.GetRequiredService(); // Log database creation var logger2 = serviceProvider.GetRequiredService>(); logger2.LogInformation("Created and initialized RefactorIQ database for project {ProjectId} in directory: {DatabaseDirectory}", projectId, databaseDirectory ?? Directory.GetCurrentDirectory()); return new RefactorIQIntegration(databaseDirectory ?? Directory.GetCurrentDirectory()) { _serviceProvider = serviceProvider, _client = client }; } private IConfiguration BuildConfiguration() { var configDir = Directory.Exists(_configPath) ? _configPath : Path.GetDirectoryName(_configPath); return new ConfigurationBuilder() .SetBasePath(configDir) .AddJsonFile("appsettings.json", optional: true) .AddEnvironmentVariables() .Build(); } private IServiceProvider BuildServiceProvider() { var services = new ServiceCollection(); // Add logging services.AddLogging(builder => builder.AddConsole()); // Add RefactorIQ services with correct configuration services.AddRefactorIQServices(options => { // Set connection string var connectionString = _configuration.GetConnectionString("RefactorIQ"); if (string.IsNullOrEmpty(connectionString)) { var dbPath = Path.Combine(_configPath, "refactoriq.db"); connectionString = $"Data Source={dbPath}"; } options.ConnectionString = connectionString; // Configure database options for solution-specific databases options.Database.UseSolutionSpecificDatabase = bool.Parse(_configuration["RefactorIQ:Database:UseSolutionSpecificDatabase"] ?? "true"); // Use project ID pattern for better uniqueness and multi-tenant safety options.Database.DatabaseNamePattern = _configuration["RefactorIQ:Database:DatabaseNamePattern"] ?? "{ProjectId}.refactoriq.sqlite"; options.Database.DefaultPath = _configuration["RefactorIQ:Database:DefaultPath"] ?? "RefactorIQ.sqlite"; // Configure OpenAI settings var openAIApiKey = _configuration["RefactorIQ:OpenAI:ApiKey"] ?? Environment.GetEnvironmentVariable("OPENAI_API_KEY"); if (!string.IsNullOrEmpty(openAIApiKey)) { options.OpenAI.ApiKey = openAIApiKey; options.OpenAI.Model = _configuration["RefactorIQ:OpenAI:Model"] ?? "text-embedding-3-small"; options.OpenAI.MaxRetries = int.Parse(_configuration["RefactorIQ:OpenAI:MaxRetries"] ?? "3"); } // Configure embedding settings options.Embedding.BatchSize = int.Parse(_configuration["RefactorIQ:Embedding:BatchSize"] ?? "10"); options.Embedding.EnableProgressSaving = bool.Parse(_configuration["RefactorIQ:Embedding:EnableProgressSaving"] ?? "true"); options.Embedding.ProgressSaveInterval = int.Parse(_configuration["RefactorIQ:Embedding:ProgressSaveInterval"] ?? "10"); }); // Add performance optimizations (caching, parallel processing) services.AddPerformanceOptimization(_configuration); var serviceProvider = services.BuildServiceProvider(); // Ensure database schema is created by running migrations (same fix as in main app) using (var scope = serviceProvider.CreateScope()) { var dbContext = scope.ServiceProvider.GetRequiredService(); try { // Run migrations to create the database schema (including Solutions table) dbContext.Database.Migrate(); } catch (Exception migrationEx) { // If migrations fail, try EnsureCreated as fallback var logger = serviceProvider.GetRequiredService>(); logger.LogWarning(migrationEx, "Failed to run migrations, falling back to EnsureCreated"); dbContext.Database.EnsureCreated(); } } return serviceProvider; } /// /// Index a multi-language project (TypeScript, JavaScript, Python, PHP, Java, Go) /// public virtual async Task IndexProjectAsync(string projectPath, ProjectType type = ProjectType.Unknown) { var result = new RefactorIQResult { StartTime = DateTime.UtcNow, Operation = "IndexProject" }; try { Console.WriteLine($"🔍 Starting RefactorIQ multi-language project indexing for: {projectPath}"); // Use IndexSolutionAsync which can handle both .sln files and project directories // The MultiLanguageIndexerService will detect if it's a project directory and use IndexProjectAsync internally var indexResult = await _client.IndexSolutionAsync(projectPath, type, CancellationToken.None); result.Success = indexResult.IsSuccess; result.Error = indexResult.ErrorMessage; if (indexResult.IsSuccess && indexResult.Data != null) { var indexedSolution = indexResult.Data; var types = indexedSolution.TypeIndex.Types; result.SymbolCount = types.Sum(t => t.Members.Count); result.TypeCount = types.Count; result.ProjectCount = types.Select(t => t.ProjectName).Distinct().Count(); Console.WriteLine($"✅ RefactorIQ project indexing completed: {result.SymbolCount} symbols in {result.TypeCount} types"); Console.WriteLine($" Languages detected: {string.Join(", ", types.Select(t => t.Language).Distinct())}"); // Generate AI embeddings if configured if (!string.IsNullOrEmpty(_configuration["RefactorIQ:OpenAI:ApiKey"])) { Console.WriteLine("🧠 Generating AI embeddings for multi-language project..."); var embeddingProgress = new Progress(p => { Console.WriteLine($"🤖 Embedding progress: {p.ProcessedItems}/{p.TotalItems} items"); }); var embeddingResult = await _client.GenerateEmbeddingsAsync(projectPath, embeddingProgress); if (embeddingResult.IsSuccess) { Console.WriteLine("✅ AI embeddings generated for multi-language project"); } else { Console.WriteLine($"⚠️ Embedding generation failed: {embeddingResult.ErrorMessage}"); } } } else { Console.WriteLine($"❌ RefactorIQ project indexing failed: {result.Error}"); } } catch (Exception ex) { result.Success = false; result.Error = ex.Message; Console.WriteLine($"❌ RefactorIQ project indexing failed: {ex.Message}"); } finally { result.EndTime = DateTime.UtcNow; result.Duration = result.EndTime - result.StartTime; } return result; } public virtual async Task IndexSolutionAsync(string solutionPath, ProjectType type = ProjectType.Unknown) { var result = new RefactorIQResult { StartTime = DateTime.UtcNow, Operation = "Index" }; try { Console.WriteLine("🔍 Starting RefactorIQ indexing with enhanced services..."); // Use the correct IRefactorIQClient API var indexResult = await _client.IndexSolutionAsync(solutionPath, type, CancellationToken.None); result.Success = indexResult.IsSuccess; result.Error = indexResult.ErrorMessage; if (indexResult.IsSuccess && indexResult.Data != null) { var indexedSolution = indexResult.Data; var types = indexedSolution.TypeIndex.Types; result.SymbolCount = types.Sum(t => t.Members.Count); result.TypeCount = types.Count; result.ProjectCount = types.Select(t => t.ProjectName).Distinct().Count(); Console.WriteLine($"✅ RefactorIQ indexing completed: {result.SymbolCount} symbols in {result.TypeCount} types across {result.ProjectCount} projects"); // Generate AI embeddings if configured if (!string.IsNullOrEmpty(_configuration["RefactorIQ:OpenAI:ApiKey"])) { Console.WriteLine("🧠 Generating AI embeddings..."); var embeddingProgress = new Progress(p => { Console.WriteLine($"🤖 Embedding progress: {p.ProcessedItems}/{p.TotalItems} items"); }); var embeddingResult = await _client.GenerateEmbeddingsAsync(solutionPath, embeddingProgress); if (embeddingResult.IsSuccess) { Console.WriteLine("✅ AI embeddings generated"); } else { Console.WriteLine($"⚠️ Embedding generation failed: {embeddingResult.ErrorMessage}"); } } } else { Console.WriteLine($"❌ RefactorIQ indexing failed: {result.Error}"); } } catch (Exception ex) { result.Success = false; result.Error = ex.Message; Console.WriteLine($"❌ RefactorIQ indexing failed: {ex.Message}"); } finally { result.EndTime = DateTime.UtcNow; result.Duration = result.EndTime - result.StartTime; } return result; } public async Task RefreshSolutionAsync(string solutionPath, ProjectType type = ProjectType.Unknown) { var result = new RefactorIQResult { StartTime = DateTime.UtcNow, Operation = "Refresh" }; try { Console.WriteLine("🔄 Refreshing RefactorIQ database with incremental updates..."); var refreshResult = await _client.RefreshSolutionAsync(solutionPath, type, CancellationToken.None); result.Success = refreshResult.IsSuccess; result.Error = refreshResult.ErrorMessage; if (refreshResult.IsSuccess && refreshResult.Data != null) { var indexedSolution = refreshResult.Data; var types = indexedSolution.TypeIndex.Types; result.SymbolCount = types.Sum(t => t.Members.Count); result.TypeCount = types.Count; result.ProjectCount = types.Select(t => t.ProjectName).Distinct().Count(); Console.WriteLine($"✅ RefactorIQ refresh completed: {result.SymbolCount} symbols in {result.TypeCount} types across {result.ProjectCount} projects"); // Refresh embeddings if AI features are enabled if (!string.IsNullOrEmpty(_configuration["RefactorIQ:OpenAI:ApiKey"])) { Console.WriteLine("🧠 Refreshing AI embeddings for modified files..."); var embeddingProgress = new Progress(p => { Console.WriteLine($"🤖 Embedding refresh: {p.ProcessedItems}/{p.TotalItems} items"); }); // Get project names from the indexed solution var projects = types.Select(t => t.ProjectName).Distinct().ToList(); foreach (var project in projects) { var embeddingResult = await _client.GenerateIncrementalEmbeddingsAsync(project, embeddingProgress); if (!embeddingResult.IsSuccess) { Console.WriteLine($"⚠️ Embedding refresh failed for {project}: {embeddingResult.ErrorMessage}"); } } Console.WriteLine("✅ AI embeddings refreshed"); } } else { Console.WriteLine($"❌ RefactorIQ refresh failed: {result.Error}"); } } catch (Exception ex) { result.Success = false; result.Error = ex.Message; Console.WriteLine($"❌ RefactorIQ refresh failed: {ex.Message}"); } finally { result.EndTime = DateTime.UtcNow; result.Duration = result.EndTime - result.StartTime; } return result; } /// /// Search for similar code patterns using AI embeddings /// public async Task> SearchSimilarCodeAsync(string query, string? projectName = null, int maxResults = 10) { try { Console.WriteLine($"🔍 Searching for similar code: '{query}'"); var searchResult = await _client.SearchSimilarAsync(query, projectName, maxResults); if (searchResult.IsSuccess && searchResult.Data != null) { Console.WriteLine($"✅ Found {searchResult.Data.Count} similar code patterns"); return searchResult.Data; } else { Console.WriteLine($"❌ Search failed: {searchResult.ErrorMessage}"); return new List(); } } catch (Exception ex) { Console.WriteLine($"❌ Semantic search failed: {ex.Message}"); return new List(); } } /// /// Get detailed embedding statistics /// public async Task> GetEmbeddingStatsAsync() { try { var statsResult = await _client.GetEmbeddingStatsAsync(); if (statsResult.IsSuccess && statsResult.Data != null) { return statsResult.Data; } else { Console.WriteLine($"❌ Failed to get embedding stats: {statsResult.ErrorMessage}"); return new Dictionary(); } } catch (Exception ex) { Console.WriteLine($"❌ Failed to get embedding stats: {ex.Message}"); return new Dictionary(); } } /// /// Get list of indexed projects /// public async Task> GetIndexedProjectsAsync() { try { var projectsResult = await _client.GetProjectNamesAsync(); if (projectsResult.IsSuccess && projectsResult.Data != null) { return projectsResult.Data; } else { Console.WriteLine($"❌ Failed to get project names: {projectsResult.ErrorMessage}"); return new List(); } } catch (Exception ex) { Console.WriteLine($"❌ Failed to get project names: {ex.Message}"); return new List(); } } /// /// Get indexed types from a solution for method extraction /// public async Task> GetIndexedTypesAsync(string solutionPath, ProjectType type = ProjectType.Unknown) { try { Console.WriteLine($"🔍 Getting indexed types from RefactorIQ for: {solutionPath}"); var indexResult = await _client.IndexSolutionAsync(solutionPath, type, CancellationToken.None); if (indexResult.IsSuccess && indexResult.Data != null) { var indexedSolution = indexResult.Data; var types = indexedSolution.TypeIndex.Types.ToList(); Console.WriteLine($"✅ Retrieved {types.Count} indexed types from RefactorIQ"); return types; } else { Console.WriteLine($"❌ Failed to get indexed types: {indexResult.ErrorMessage}"); return new List(); } } catch (Exception ex) { Console.WriteLine($"❌ Exception getting indexed types: {ex.Message}"); return new List(); } } /// /// Clean up resources /// public void Dispose() { if (_serviceProvider is IDisposable disposable) { disposable.Dispose(); } } } public class RefactorIQResult { public DateTime StartTime { get; set; } public DateTime EndTime { get; set; } public TimeSpan Duration { get; set; } public string Operation { get; set; } public bool Success { get; set; } public string Error { get; set; } public int SymbolCount { get; set; } public int TypeCount { get; set; } public int ProjectCount { get; set; } } }