diff --git a/docs/library-symbols-feature.md b/docs/library-symbols-feature.md new file mode 100644 index 0000000..bb8f291 --- /dev/null +++ b/docs/library-symbols-feature.md @@ -0,0 +1,470 @@ +# Library Symbols — Feature Design Document + +## 1. Visão Geral + +Sistema de importação e resolução de símbolos de bibliotecas externas (ex: Borland OWL, MFC, Qt, .NET BCL). O parser consome dumps de símbolos (formato TurboDump inicialmente), armazena-os como referência autoritativa, e cruza automaticamente com os resultados de análise estática para resolver imports opacos (ordinais) em nomes semânticos. + +A informação resolvida é injetada no pipeline de embeddings (pgvector) e na construção de contexto do chat RAG, melhorando dramaticamente a qualidade das respostas do LLM. + +**Fase atual (1):** Parser TurboDump + integração completa com AI/RAG. Sem sessões de chat temático (Fase 2 futura). + +## 2. Arquitetura + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ LIBRARY UPLOAD FLOW │ +│ │ +│ POST /libraries/upload │ +│ │ {parserType, rawContent, libraryName, aliases[]} │ +│ ▼ │ +│ LibraryService.parseAndStore() │ +│ ├─ LibrarySymbolParser (interface) │ +│ │ └─ TurboDumpParser ← implementação atual │ +│ │ └─ DefFileParser ← futuro │ +│ │ └─ MapFileParser ← futuro │ +│ ├─ Cria Library entity │ +│ ├─ Cria LibraryAlias entities (mapeamento DLL ↔ library) │ +│ ├─ Chama parser.parse() → List │ +│ └─ Salva LibrarySymbol entities em lote │ +└──────────────────────────────────────────────────────────────────┘ + +┌──────────────────────────────────────────────────────────────────┐ +│ SYMBOL RESOLUTION FLOW │ +│ │ +│ StaticAnalysisCompletedEvent (disparado após IDA/Ghidra) │ +│ ▼ │ +│ SymbolResolutionService.resolveImports(analysisId) │ +│ ├─ Carrega todos os StaticImport do analysis │ +│ ├─ Para cada import com ordinal != 0: │ +│ │ ├─ Busca LibraryAlias pelo campo dll do import │ +│ │ ├─ Se encontrado: busca LibrarySymbol por (library, ordinal)│ +│ │ └─ Se encontrado: atualiza StaticImport.resolvedName │ +│ │ e StaticImport.librarySymbolId │ +│ └─ Loga estatísticas: quantos imports resolvidos │ +└──────────────────────────────────────────────────────────────────┘ + +┌──────────────────────────────────────────────────────────────────┐ +│ AI / RAG ENRICHMENT │ +│ │ +│ EmbeddingChunkBuilder │ +│ ├─ buildImportChunks(): inclui resolvedName + className │ +│ ├─ buildMetadataChunk(): inclui Library Diagnostics section │ +│ └─ buildFunctionChunkContent(): callees resolvidos via OWL │ +│ │ +│ ChatContextBuilder │ +│ ├─ buildSystemPrompt(): inclui [LIBRARY CONTEXT] seção │ +│ └─ detectFunctionReferences(): busca por classes/métodos OWL │ +└──────────────────────────────────────────────────────────────────┘ +``` + +## 3. Modelo de Dados + +### 3.1 Tabelas Novas + +#### `library` +| Coluna | Tipo | Descrição | +|--------|------|-----------| +| `id` | UUID PK | Identificador | +| `name` | VARCHAR(255) NOT NULL | Nome da biblioteca (ex: "Borland OWL 2.52") | +| `description` | TEXT | Descrição opcional (ex: "Object Windows Library for Borland C++") | +| `created_at` | TIMESTAMPTZ NOT NULL | Data de criação | + +#### `library_alias` +| Coluna | Tipo | Descrição | +|--------|------|-----------| +| `id` | UUID PK | Identificador | +| `library_id` | UUID FK → library(id) ON DELETE CASCADE | Biblioteca pai | +| `alias` | VARCHAR(255) NOT NULL | Nome do DLL/arquivo no import table (ex: "OWLWI", "owl252.dll") | +| `UNIQUE(library_id, alias)` | | Um alias pertence a uma única biblioteca | + +#### `library_symbol` +| Coluna | Tipo | Descrição | +|--------|------|-----------| +| `id` | UUID PK | Identificador | +| `library_id` | UUID FK → library(id) ON DELETE CASCADE | Biblioteca pai | +| `module_name` | VARCHAR(255) NOT NULL | Módulo original (ex: "Owl252" do TurboDump) | +| `class_name` | VARCHAR(500) | Classe C++ (nulo para funções livres) | +| `symbol_name` | VARCHAR(500) NOT NULL | Nome completo do símbolo (ex: "TButtonGadget::Paint(TDC*)") | +| `ordinal` | INTEGER NOT NULL | Número ordinal de importação | +| `symbol_type` | VARCHAR(50) | Tipo: CONSTRUCTOR, DESTRUCTOR, METHOD, FUNCTION, STATIC_MEMBER | +| `parameters` | VARCHAR(1000) | Assinatura dos parâmetros (extraído do nome) | +| `category` | VARCHAR(100) | Categoria (ex: "UI/Gadget", "Core/Application", "Graphics/Bitmap") | +| `created_at` | TIMESTAMPTZ NOT NULL | Data de criação | + +Índices: `idx_library_symbol_library_ordinal ON library_symbol (library_id, ordinal)` + `idx_library_symbol_class ON library_symbol (class_name) WHERE class_name IS NOT NULL` + +### 3.2 Tabela Alterada + +#### `static_import` (adições) +| Coluna | Tipo | Descrição | +|--------|------|-----------| +| `library_symbol_id` | UUID FK → library_symbol(id) ON SET NULL | Símbolo de biblioteca que corresponde a este import | +| `resolved_name` | VARCHAR(500) | Nome resolvido (ex: "TButtonGadget::Paint(TDC*)") | + +Índice: `idx_static_import_library_symbol ON static_import (library_symbol_id) WHERE library_symbol_id IS NOT NULL` + +## 4. Fluxo de Dados Detalhado + +### 4.1 Upload da Biblioteca + +``` +POST /libraries/upload +Content-Type: application/json + +{ + "name": "Borland OWL 2.52", + "description": "Object Windows Library for Borland C++ 5.0", + "parserType": "TURBO_DUMP", + "aliases": ["OWLWI", "owl252.dll", "Owl252"], + "rawContent": "Turbo Dump Version 5.0.1..." +} +``` + +1. `LibraryService.parseAndStore()` recebe o request +2. Busca um `LibrarySymbolParser` pelo `parserType` (pode ser um Map ou via qualifier annotation) +3. Cria `Library` entity e salva +4. Cria `LibraryAlias` entities para cada alias +5. Chama `parser.parse(rawContent, libraryId)` → `List` +6. Salva `LibrarySymbol` entities em batch (500 por vez) +7. Retorna `LibraryResponse` com contagem de símbolos + +### 4.2 Resolução de Import Symbols + +Após a análise IDA/Ghidra completar, o `StaticAnalysisCompletedEvent` é publicado. O listener `SymbolResolutionListener` (ou `SymbolResolutionService`) escuta o evento: + +1. Carrega `StaticImport` para o `analysisId` +2. Para cada import com `ordinal != 0`: + a. Busca `LibraryAlias` onde `alias ILIKE dll` (case-insensitive) + b. Se encontrado, busca `LibrarySymbol` por `(library_id, ordinal)` + c. Se encontrado, atualiza `StaticImport.librarySymbolId` e `resolvedName` +3. Loga: `Resolved 152/200 imports (76%) for analysis ` +4. Opcional: publica um evento `SymbolsResolvedEvent` para notificar downstream + +Alternativamente, se o listener não for viável (timing), o `StaticAnalysisHandler` chama `symbolResolutionService.resolveImports(analysisId)` no final do `process()`. + +### 4.3 Enriquecimento de Embedding Chunks + +#### Import Chunks +```java +// Antes +"[IMPORT] OWLWI.ord_75 @ 0x00405A00" + +// Depois (com library symbol resolvido) +"[IMPORT] OWLWI.TButtonGadget::Paint(TDC*) @ 0x00405A00 " + +" Library: Borland OWL 2.52 | Class: TButtonGadget | Method: Paint" +``` + +#### Metadata Chunk — Nova seção [LIBRARY DIAGNOSTICS] +``` +[LIBRARY DIAGNOSTICS] +Framework: Borland OWL 2.52 (Object Windows Library) +Detected classes (15): TApplication, TFrameWindow, TButtonGadget, TBitmap, + TControlBar, TChooseColorDialog, TChooseFontDialog, TClipboard, TBrush, + TDC, TCursor, TBitSet, TCharSet, TCommonDialog, TControlGadget +Category distribution: UI/Gadget (6), Core/Application (3), Graphics (4), + UI/Window (2) +Architecture: Event-driven GUI, message dispatch via TEventHandler::Find() +``` + +#### Function Chunks +No `buildFunctionChunkContent()`, ao listar callees, se o callee for um import com `resolvedName`: +```java +// Antes +"Callees: sub_401000, ordinal_75, ordinal_67" + +// Depois +"Callees: sub_401000, TButtonGadget::Paint [OWL/ord75], TButtonGadget::TButtonGadget [OWL/ord67]" +``` + +### 4.4 Enriquecimento do System Prompt + +```java +// Antes +String.format(""" + [BINARY OVERVIEW] + Filename: %s + Format: %s + Architecture: %s + Compiler: %s + """, ...); + +// Depois +String.format(""" + [BINARY OVERVIEW] + Filename: %s + Format: %s + Architecture: %s + Compiler: Borland C++ + + [LIBRARY CONTEXT] + This binary links against Borland OWL 2.52 (Object Windows Library), a + C++ framework for Windows GUI applications. Key classes detected: + %s + + OWL uses a message-driven architecture similar to MFC. Windows messages + are routed through TEventHandler response tables. Common patterns: + - TFrameWindow is the main application window + - TButtonGadget handles button controls with Paint() for WM_PAINT + - Dialogs extend TCommonDialog via TChooseColorDialog/TChooseFontDialog + - Graphics use TDC (Device Context wrapper) with TBrush/TPen/TBitmap + + When analyzing functions that call OWL imports, consider the OWL class + hierarchy and event dispatch model to infer the function's role. + """, ..., libraryDiagnostics); +``` + +## 5. API REST + +### 5.1 Endpoints + +| Método | Path | Descrição | +|--------|------|-----------| +| `POST` | `/libraries/upload` | Upload de um dump de símbolos de biblioteca | +| `GET` | `/libraries` | Listar todas as bibliotecas carregadas | +| `GET` | `/libraries/{id}` | Detalhes de uma biblioteca | +| `DELETE` | `/libraries/{id}` | Remover uma biblioteca e seus símbolos | +| `GET` | `/libraries/{id}/symbols` | Listar símbolos (com paginação, filtro por nome/ordinal) | +| `POST` | `/analysis/{analysisId}/resolve-symbols` | Forçar resolução de símbolos (debug) | + +### 5.2 DTOs + +```java +// Request +public record LibraryUploadRequest( + @NotBlank String name, + String description, + @NotBlank String parserType, + @NotEmpty List aliases, + @NotBlank String rawContent +) {} + +// Response +public record LibraryResponse( + UUID id, String name, String description, int symbolCount, + int aliasCount, Instant createdAt +) { + public static LibraryResponse from(Library lib) { ... } +} + +public record LibrarySymbolResponse( + UUID id, String moduleName, String className, String symbolName, + int ordinal, String symbolType, String parameters, String category +) { + public static LibrarySymbolResponse from(LibrarySymbol sym) { ... } +} +``` + +### 5.3 ImportResponse (estendido) + +```java +public record ImportResponse( + UUID id, String dll, String name, String address, int ordinal, + // Novos campos opcionais: + String resolvedName, // nulo se não resolvido + String libraryName, // nulo se não resolvido + String className, // nulo se não for método de classe + UUID librarySymbolId // nulo se não resolvido +) { + public static ImportResponse from(StaticImport imp) { + var sym = imp.getLibrarySymbol(); + return new ImportResponse( + imp.getId(), imp.getDll(), imp.getName(), imp.getAddress(), + imp.getOrdinal(), + imp.getResolvedName(), + Objects.nonNull(sym) ? sym.getLibrary().getName() : null, + Objects.nonNull(sym) ? sym.getClassName() : null, + imp.getLibrarySymbolId() + ); + } +} +``` + +## 6. Parser Architecture + +### 6.1 Interface + +```java +package ai.decompile.analysis.service.parser; + +import java.util.List; + +public interface LibrarySymbolParser { + /** Identificador único do formato (ex: "TURBO_DUMP", "DEF_FILE") */ + String getFormatName(); + + /** Valida se o conteúdo é compatível com este parser */ + boolean canParse(String rawContent); + + /** Parseia o conteúdo e retorna símbolos para association com libraryId */ + List parse(String rawContent); +} + +/** Registro intermediário entre parser e serviço */ +public record ParsedSymbol( + String moduleName, + String className, + String symbolName, + int ordinal, + String symbolType, + String parameters, + String category +) {} +``` + +### 6.2 TurboDumpParser + +Implementação concreta para o formato do utilitário `TDUMP.EXE` da Borland: + +**Formato de entrada:** +``` +000020 THEADR TAppDictionary::TAppDictionary() +00003E COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: TAppDictionary::TAppDictionary() + Module Name: Owl252 + ordinal: 2 +000069 MODEND +``` + +**Lógica de parsing:** +1. Divide o texto por blocos delimitados por `MODEND` +2. Para cada bloco: + - Extrai `symbolName` da linha `THEADR` + - Extrai `moduleName` do campo `Module Name:` + - Extrai `ordinal` do campo `ordinal:` + - Se não encontrar `ordinal:` ou o bloco não for `IMPDEF`, pula + - Classifica `symbolType`: + - Contém `::~` → `DESTRUCTOR` + - Contém `::` e termina em `()` sem `~` → `CONSTRUCTOR` + - Contém `::` → `METHOD` + - Sem `::` → `FUNCTION` + - Extrai `className` do trecho antes de `::` + - Extrai `parameters` dos parênteses após o nome do método + - Classifica `category` por prefixo de classe: + ``` + TApp, TModule, TStatus, TXCompatibility → Core/Application + TFrame, TWindow, TDialog, TClipboard → UI/Window + TButton, TGadget, TControl, TSlider → UI/Control + TBitmap, TPalette, TDib, TIcon, TDC → Graphics + TBrush, TPen, TFont → Graphics/Drawing + TColor → Graphics/Color + TCursor → UI/Cursor + TBitSet, TCharSet → Core/DataTypes + default → Core/Other + ``` + +### 6.3 Registro de Parsers + +```java +@Component +@RequiredArgsConstructor +public class LibrarySymbolParserRegistry { + // Spring injeta todos os beans LibrarySymbolParser + private final Map parsers; + + // Construtor monta o mapa formatName → parser + public LibrarySymbolParserRegistry(List parserList) { + this.parsers = parserList.stream() + .collect(Collectors.toMap( + LibrarySymbolParser::getFormatName, + Function.identity())); + } + + public LibrarySymbolParser get(String formatName) { + var parser = parsers.get(formatName); + if (Objects.isNull(parser)) { + throw new IllegalArgumentException( + "No parser found for format '" + formatName + "'. " + + "Available: " + parsers.keySet()); + } + return parser; + } + + public List getAvailableFormats() { + return List.copyOf(parsers.keySet()); + } +} +``` + +## 7. Estrutura de Código + +``` +src/main/java/ai/decompile/analysis/ +├── model/ +│ ├── entity/ +│ │ ├── Library.java (novo) +│ │ ├── LibraryAlias.java (novo) +│ │ ├── LibrarySymbol.java (novo) +│ │ └── StaticImport.java (modificado: +resolvedName, +librarySymbolId, +librarySymbol) +│ ├── repository/ +│ │ ├── LibraryRepository.java (novo) +│ │ ├── LibraryAliasRepository.java (novo) +│ │ └── LibrarySymbolRepository.java (novo) +│ └── dto/ +│ ├── LibraryResponse.java (novo) +│ ├── LibrarySymbolResponse.java (novo) +│ ├── LibraryUploadRequest.java (novo) +│ └── ImportResponse.java (modificado) +├── service/ +│ ├── parser/ +│ │ ├── LibrarySymbolParser.java (novo - interface) +│ │ ├── ParsedSymbol.java (novo - record) +│ │ ├── LibrarySymbolParserRegistry.java (novo) +│ │ └── TurboDumpParser.java (novo) +│ ├── LibraryService.java (novo) +│ ├── SymbolResolutionService.java (novo) +│ └── AnalysisService.java (modificado: chama resolution após saveResult) +├── event/ +│ └── SymbolsResolvedEvent.java (novo, opcional) +└── controller/ + └── LibraryController.java (novo) + +src/main/resources/db/migration/ +├── V021__create_library_tables.sql (novo) +└── V022__alter_static_import_library.sql (novo) + +src/main/java/ai/decompile/ai/service/ +├── EmbeddingChunkBuilder.java (modificado: enriquecido com library info) +└── ChatContextBuilder.java (modificado: system prompt + context enriquecido) +``` + +## 8. Casos de Teste + +### 8.1 TurboDumpParser + +| Teste | Entrada | Esperado | +|-------|---------|----------| +| Construtor de classe | `TAppDictionary::TAppDictionary()` | type=CONSTRUCTOR, class=TAppDictionary | +| Destrutor | `TBitmap::~TBitmap()` | type=DESTRUCTOR, class=TBitmap | +| Método com parâmetros | `TDC::SelectObject(const TPen*)` | type=METHOD, class=TDC, params=const TPen* | +| Método sem parâmetros | `TBitSet::IsEmpty()` | type=METHOD, params=null | +| Função livre | `NColors(unsigned short)` | type=FUNCTION, className=null | +| Bloco sem ordinal | Qualquer não-IMPDEF | Ignorado (não retorna símbolo) | +| Parser chamado 2x | Conteúdo com 1854 IMPDEF | 1854 símbolos retornados | + +### 8.2 SymbolResolutionService + +| Teste | Setup | Esperado | +|-------|-------|----------| +| Match por ordinal + alias | Import(dll="OWLWI", ordinal=75), LibrarySymbol(ordinal=75, library=L1), Alias("OWLWI"→L1) | resolvedName = "TButtonGadget::Paint(TDC*)" | +| Ordinal zero (nomeado) | Import(ordinal=0) | Não resolvido (pula) | +| Alias não encontrado | Import(dll="UNKNOWN", ordinal=5) | Não resolvido | +| Ordinal não encontrado | Import(dll="OWLWI", ordinal=9999) | Não resolvido | +| Sem bibliotecas carregadas | Nenhuma Library no banco | Nenhum import resolvido | +| Case insensitive alias | Import(dll="owlwi"), Alias("OWLWI") | Resolvido | + +### 8.3 EmbeddingChunkBuilder (enriquecido) + +| Teste | Setup | Esperado | +|-------|-------|----------| +| Import com resolvedName | StaticImport com resolvedName="TDC::Paint(TDC*)" | Chunk content contém o resolvedName | +| Metadata com library diagnostics | Library + LibrarySymbol carregados | Seção [LIBRARY DIAGNOSTICS] presente | +| Function callees resolvidos | Function com xrefs para imports resolvidos | Callee list mostra class::method [OWL] | + +## 9. Rollout e Rollback + +- **Nova feature, sem quebra de compatibilidade**: migration adiciona colunas nullable ao `static_import` +- **Sem bibliotecas carregadas**: comportamento idêntico ao atual (resolvedName = null, nenhum chunk afetado) +- **Rollback**: reverter migrations V021 e V022, remover código novo (as APIs novas simplesmente não existiriam mais) diff --git a/src/main/java/ai/decompile/ai/service/ChatContextBuilder.java b/src/main/java/ai/decompile/ai/service/ChatContextBuilder.java index 6b3bc9a..f43bc4b 100644 --- a/src/main/java/ai/decompile/ai/service/ChatContextBuilder.java +++ b/src/main/java/ai/decompile/ai/service/ChatContextBuilder.java @@ -7,6 +7,7 @@ import ai.decompile.analysis.model.entity.StaticFunction; import ai.decompile.analysis.model.entity.StaticXref; import ai.decompile.analysis.model.repository.StaticAnalysisRepository; import ai.decompile.analysis.model.repository.StaticFunctionRepository; +import ai.decompile.analysis.model.repository.StaticImportRepository; import ai.decompile.analysis.model.repository.StaticXrefRepository; import ai.decompile.workspace.model.entity.Binary; import java.util.ArrayList; @@ -42,6 +43,7 @@ public class ChatContextBuilder { private final StaticAnalysisRepository analysisRepository; private final StaticFunctionRepository functionRepository; private final StaticXrefRepository xrefRepository; + private final StaticImportRepository importRepository; @Value("${app.ai.rag.top-k:10}") private int topK; @@ -63,7 +65,9 @@ public class ChatContextBuilder { enrichWithCallGraph(inlineSections, merged, query, binaryId); var analysis = getLatestAnalysis(binaryId); - return formatContext(inlineSections, merged, binary.getFilename(), analysis); + var libraryDiagnostics = buildLibraryDiagnostics(analysis); + return formatContext( + inlineSections, merged, binary.getFilename(), analysis, libraryDiagnostics); } public String buildSystemPrompt(String context, Binary binary) { @@ -92,6 +96,8 @@ public class ChatContextBuilder { - Be concise but thorough in your technical analysis - If the context includes a [REQUESTED FUNCTION ANALYSIS] section with full assembly, analyze that function in depth rather than saying you don't have access to it + - If a [LIBRARY CONTEXT] section is present, use the framework/library knowledge + to infer architectural patterns (e.g., class hierarchy, event dispatch, GUI structure) """, AiUtils.orUnknown(binary.getFilename()), AiUtils.orUnknown(binary.getFormat()), @@ -178,7 +184,8 @@ public class ChatContextBuilder { List inlineSections, LinkedHashSet merged, String filename, - StaticAnalysis analysis) { + StaticAnalysis analysis, + String libraryDiagnostics) { if (merged.isEmpty() && inlineSections.isEmpty()) { return "[No relevant code sections found for this query in binary " + filename + ".]"; } @@ -189,6 +196,10 @@ public class ChatContextBuilder { sb.append("[ENTRY POINT] ").append(analysis.getEntryPoint()).append("\n\n"); } + if (Objects.nonNull(libraryDiagnostics) && !libraryDiagnostics.isEmpty()) { + sb.append(libraryDiagnostics).append("\n"); + } + if (!inlineSections.isEmpty()) { sb.append("[REQUESTED FUNCTION ANALYSIS]\n\n"); for (var section : inlineSections) { @@ -208,6 +219,73 @@ public class ChatContextBuilder { return sb.toString(); } + private String buildLibraryDiagnostics(StaticAnalysis analysis) { + if (Objects.isNull(analysis)) { + return ""; + } + + var imports = importRepository.findByAnalysisIdOrderByDllAscNameAsc(analysis.getId()); + if (imports.isEmpty()) { + return ""; + } + + var resolvedImports = + imports.stream().filter(i -> Objects.nonNull(i.getResolvedName())).toList(); + + if (resolvedImports.isEmpty()) { + return ""; + } + + var libraryNames = new LinkedHashSet(); + var classNames = new LinkedHashSet(); + var categoryCounts = new java.util.HashMap(); + + for (var imp : resolvedImports) { + var sym = imp.getLibrarySymbol(); + if (Objects.nonNull(sym)) { + libraryNames.add(sym.getLibrary().getName()); + + if (Objects.nonNull(sym.getClassName())) { + classNames.add(sym.getClassName()); + var category = sym.getCategory(); + if (Objects.nonNull(category)) { + categoryCounts.merge(category, 1, Integer::sum); + } + } + } + } + + var sb = new StringBuilder(); + sb.append("[LIBRARY CONTEXT]\n"); + sb.append("This binary links against: ").append(String.join(", ", libraryNames)).append("\n"); + + if (!classNames.isEmpty()) { + var topClasses = classNames.stream().sorted().limit(15).toList(); + sb.append("Detected classes (").append(classNames.size()).append("): "); + sb.append(String.join(", ", topClasses)); + sb.append("\n"); + } + + if (!categoryCounts.isEmpty()) { + sb.append("Category breakdown: "); + var cats = + categoryCounts.entrySet().stream() + .sorted((a, b) -> b.getValue().compareTo(a.getValue())) + .map(e -> e.getKey() + "=" + e.getValue()) + .limit(8) + .toList(); + sb.append(String.join(", ", cats)).append("\n"); + } + + sb.append("Resolved imports: ") + .append(resolvedImports.size()) + .append("/") + .append(imports.size()) + .append("\n"); + + return sb.toString(); + } + private List semanticSearch(UUID binaryId, String query) { var queryEmbedding = embeddingService.embed(query); var vectorStr = AiUtils.toVectorString(queryEmbedding); diff --git a/src/main/java/ai/decompile/ai/service/EmbeddingChunkBuilder.java b/src/main/java/ai/decompile/ai/service/EmbeddingChunkBuilder.java index fe02715..f00e374 100644 --- a/src/main/java/ai/decompile/ai/service/EmbeddingChunkBuilder.java +++ b/src/main/java/ai/decompile/ai/service/EmbeddingChunkBuilder.java @@ -116,16 +116,24 @@ public class EmbeddingChunkBuilder { } var imports = importRepository.findByAnalysisIdOrderByDllAscNameAsc(analysis.getId()); + var libraryDiagnostics = new LibraryDiagnostics(); if (!imports.isEmpty()) { sb.append("Imports:\n"); imports.stream() .limit(METADATA_TOP_IMPORTS) .forEach( - i -> - sb.append( - String.format(" %s.%s @ %s\n", i.getDll(), i.getName(), i.getAddress()))); + i -> { + sb.append(String.format(" %s.%s @ %s", i.getDll(), i.getName(), i.getAddress())); + if (Objects.nonNull(i.getResolvedName())) { + sb.append(" → ").append(i.getResolvedName()); + collectLibraryInfo(libraryDiagnostics, i); + } + sb.append("\n"); + }); } + appendLibraryDiagnostics(sb, libraryDiagnostics, imports); + var globalVars = dataLabelRepository.findByAnalysisIdOrderByNameAsc(analysis.getId()); if (!globalVars.isEmpty()) { sb.append("Global variables:\n"); @@ -214,8 +222,8 @@ public class EmbeddingChunkBuilder { public List buildImportChunks(List imports, Binary binary) { var chunks = new ArrayList(); for (var imp : imports) { - var content = - String.format("[IMPORT] %s.%s @ %s", imp.getDll(), imp.getName(), imp.getAddress()); + var content = buildImportContent(imp); + var metadata = buildImportMetadata(imp); chunks.add( EmbeddingChunk.builder() .id(UUID.randomUUID()) @@ -224,11 +232,7 @@ public class EmbeddingChunkBuilder { .sourceType("STATIC_IMPORT") .sourceId(imp.getId()) .content(content) - .metadata( - AiUtils.toJson( - objectMapper, - Map.of( - "dll", imp.getDll(), "name", imp.getName(), "address", imp.getAddress()))) + .metadata(AiUtils.toJson(objectMapper, metadata)) .tokenCount(AiUtils.estimateTokens(content)) .createdAt(java.time.Instant.now()) .build()); @@ -237,6 +241,51 @@ public class EmbeddingChunkBuilder { return chunks; } + private String buildImportContent(StaticImport imp) { + var sb = new StringBuilder(); + sb.append(String.format("[IMPORT] %s.%s @ %s", imp.getDll(), imp.getName(), imp.getAddress())); + + if (Objects.nonNull(imp.getResolvedName())) { + sb.append("\n Resolved: ").append(imp.getResolvedName()); + + var sym = imp.getLibrarySymbol(); + if (Objects.nonNull(sym)) { + sb.append(" [").append(sym.getLibrary().getName()).append("]"); + + if (Objects.nonNull(sym.getClassName())) { + sb.append(" Class: ").append(sym.getClassName()); + } + + if (Objects.nonNull(sym.getCategory())) { + sb.append(" Category: ").append(sym.getCategory()); + } + } + } + + return sb.toString(); + } + + private Map buildImportMetadata(StaticImport imp) { + var metadata = new HashMap(); + metadata.put("dll", imp.getDll()); + metadata.put("name", imp.getName()); + metadata.put("address", imp.getAddress()); + metadata.put("ordinal", imp.getOrdinal()); + + if (Objects.nonNull(imp.getResolvedName())) { + metadata.put("resolved_name", imp.getResolvedName()); + + var sym = imp.getLibrarySymbol(); + if (Objects.nonNull(sym)) { + metadata.put("library_name", sym.getLibrary().getName()); + metadata.put("class_name", Objects.nonNull(sym.getClassName()) ? sym.getClassName() : ""); + metadata.put("category", Objects.nonNull(sym.getCategory()) ? sym.getCategory() : ""); + } + } + + return metadata; + } + public List buildGlobalVarChunks( List globalVars, Binary binary) { var chunks = new ArrayList(); @@ -465,6 +514,71 @@ public class EmbeddingChunkBuilder { return m; } + private void collectLibraryInfo(LibraryDiagnostics diagnostics, StaticImport imp) { + var sym = imp.getLibrarySymbol(); + if (Objects.isNull(sym)) { + return; + } + + var lib = sym.getLibrary(); + if (Objects.nonNull(lib)) { + diagnostics.libraryNames.add(lib.getName()); + } + + if (Objects.nonNull(sym.getClassName())) { + diagnostics.classNames.add(sym.getClassName()); + var category = sym.getCategory(); + if (Objects.nonNull(category)) { + diagnostics.categoryCounts.merge(category, 1, Integer::sum); + } + } + } + + private void appendLibraryDiagnostics( + StringBuilder sb, LibraryDiagnostics diagnostics, List imports) { + if (diagnostics.libraryNames.isEmpty()) { + return; + } + + var resolvedCount = imports.stream().filter(i -> Objects.nonNull(i.getResolvedName())).count(); + + sb.append("\n[LIBRARY DIAGNOSTICS]\n"); + sb.append("Resolved imports: ") + .append(resolvedCount) + .append("/") + .append(imports.size()) + .append("\n"); + + sb.append("Detected libraries: ") + .append(String.join(", ", diagnostics.libraryNames)) + .append("\n"); + + if (!diagnostics.classNames.isEmpty()) { + var classList = diagnostics.classNames.stream().sorted().toList(); + sb.append("Detected classes (") + .append(classList.size()) + .append("): ") + .append(String.join(", ", classList)) + .append("\n"); + } + + if (!diagnostics.categoryCounts.isEmpty()) { + sb.append("Category distribution: "); + var cats = + diagnostics.categoryCounts.entrySet().stream() + .sorted((a, b) -> b.getValue().compareTo(a.getValue())) + .map(e -> e.getKey() + " (" + e.getValue() + ")") + .toList(); + sb.append(String.join(", ", cats)).append("\n"); + } + } + + private static class LibraryDiagnostics { + final java.util.Set libraryNames = new java.util.LinkedHashSet<>(); + final java.util.Set classNames = new java.util.LinkedHashSet<>(); + final java.util.Map categoryCounts = new java.util.HashMap<>(); + } + record FunctionGraphContext( Map> nameToCallers, Map> nameToCallees, diff --git a/src/main/java/ai/decompile/analysis/controller/LibraryController.java b/src/main/java/ai/decompile/analysis/controller/LibraryController.java new file mode 100644 index 0000000..61f928a --- /dev/null +++ b/src/main/java/ai/decompile/analysis/controller/LibraryController.java @@ -0,0 +1,125 @@ +package ai.decompile.analysis.controller; + +import ai.decompile.analysis.model.dto.LibraryResponse; +import ai.decompile.analysis.model.dto.LibrarySymbolResponse; +import ai.decompile.analysis.service.LibraryService; +import ai.decompile.analysis.service.SymbolResolutionService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import lombok.RequiredArgsConstructor; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +@RestController +@RequiredArgsConstructor +public class LibraryController { + private final LibraryService libraryService; + private final SymbolResolutionService symbolResolutionService; + + @Operation( + summary = "Upload a library symbol dump", + description = "Parse and store symbols from a library dump file (e.g., TurboDump output).", + tags = {"Library Symbols"}, + operationId = "uploadLibrary") + @PostMapping(value = "/libraries/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity uploadLibrary( + @RequestPart MultipartFile file, + @Parameter(description = "Human-readable library name, e.g. 'Borland OWL 2.52'") @RequestParam + String name, + @Parameter(description = "Optional description of the library") + @RequestParam(required = false) + String description, + @Parameter(description = "Parser format: TURBO_DUMP") @RequestParam String parserType, + @Parameter( + description = + "Comma-separated DLL aliases, e.g. 'OWLWI,owl252.dll'. Optional — matching falls back to module name if omitted.") + @RequestParam(required = false) + String aliases) { + var lib = libraryService.parseAndStore(name, description, parserType, aliases, file); + return ResponseEntity.status(HttpStatus.CREATED).body(lib); + } + + @Operation( + summary = "List available parsers", + description = "Retrieve the list of supported library symbol parser formats.", + tags = {"Library Symbols"}, + operationId = "listParsers") + @GetMapping("/libraries/parsers") + public ResponseEntity> listParsers() { + return ResponseEntity.ok(libraryService.getAvailableParsers()); + } + + @Operation( + summary = "List all loaded libraries", + description = "Retrieve all library symbol databases that have been uploaded.", + tags = {"Library Symbols"}, + operationId = "listLibraries") + @GetMapping("/libraries") + public ResponseEntity> listLibraries() { + return ResponseEntity.ok(libraryService.listLibraries()); + } + + @Operation( + summary = "Get library details", + description = "Retrieve details of a specific library symbol database.", + tags = {"Library Symbols"}, + operationId = "getLibrary") + @GetMapping("/libraries/{libraryId}") + public ResponseEntity getLibrary(@PathVariable UUID libraryId) { + return ResponseEntity.ok(libraryService.getLibrary(libraryId)); + } + + @Operation( + summary = "Delete a library", + description = "Delete a library and all its associated symbols and aliases.", + tags = {"Library Symbols"}, + operationId = "deleteLibrary") + @DeleteMapping("/libraries/{libraryId}") + public ResponseEntity deleteLibrary(@PathVariable UUID libraryId) { + libraryService.deleteLibrary(libraryId); + return ResponseEntity.noContent().build(); + } + + @Operation( + summary = "List symbols in a library", + description = "Retrieve paginated list of symbols within a library symbol database.", + tags = {"Library Symbols"}, + operationId = "listLibrarySymbols") + @GetMapping("/libraries/{libraryId}/symbols") + public ResponseEntity> listSymbols( + @PathVariable UUID libraryId, Pageable pageable) { + return ResponseEntity.ok(libraryService.listSymbols(libraryId, pageable)); + } + + @Operation( + summary = "Resolve symbols for an analysis", + description = + "Cross-reference imports from a specific analysis against all loaded library symbol databases.", + tags = {"Library Symbols"}, + operationId = "resolveSymbols") + @PostMapping("/analysis/{analysisId}/resolve-symbols") + public ResponseEntity> resolveSymbols(@PathVariable UUID analysisId) { + var resolvedCount = symbolResolutionService.resolveImports(analysisId); + return ResponseEntity.ok(Map.of("analysisId", analysisId, "resolvedCount", resolvedCount)); + } + + @Operation( + summary = "Resolve all unresolved imports", + description = + "Cross-reference all unresolved imports across all analyses against all loaded library symbol databases.", + tags = {"Library Symbols"}, + operationId = "resolveAllSymbols") + @PostMapping("/libraries/resolve-all") + public ResponseEntity> resolveAllSymbols() { + var resolvedCount = symbolResolutionService.resolveAllUnresolved(); + return ResponseEntity.ok(Map.of("resolvedCount", resolvedCount)); + } +} diff --git a/src/main/java/ai/decompile/analysis/model/dto/FunctionDetailResponse.java b/src/main/java/ai/decompile/analysis/model/dto/FunctionDetailResponse.java index 087d7c4..9fc93c8 100644 --- a/src/main/java/ai/decompile/analysis/model/dto/FunctionDetailResponse.java +++ b/src/main/java/ai/decompile/analysis/model/dto/FunctionDetailResponse.java @@ -11,6 +11,7 @@ public record FunctionDetailResponse( UUID id, UUID analysisId, String name, + String resolvedName, String address, int size, String flags, @@ -27,10 +28,20 @@ public record FunctionDetailResponse( public static FunctionDetailResponse from( StaticFunction f, int callerCount, int calleeCount, List labels) { + return from(f, callerCount, calleeCount, labels, null); + } + + public static FunctionDetailResponse from( + StaticFunction f, + int callerCount, + int calleeCount, + List labels, + String resolvedName) { return new FunctionDetailResponse( f.getId(), f.getAnalysis().getId(), f.getName(), + resolvedName, f.getAddress(), f.getFunctionSize() != null ? f.getFunctionSize() : 0, f.getFlags(), diff --git a/src/main/java/ai/decompile/analysis/model/dto/FunctionSummaryResponse.java b/src/main/java/ai/decompile/analysis/model/dto/FunctionSummaryResponse.java index 6171e08..82d724f 100644 --- a/src/main/java/ai/decompile/analysis/model/dto/FunctionSummaryResponse.java +++ b/src/main/java/ai/decompile/analysis/model/dto/FunctionSummaryResponse.java @@ -1,15 +1,30 @@ package ai.decompile.analysis.model.dto; import ai.decompile.analysis.model.entity.StaticFunction; +import java.util.Map; +import java.util.Objects; import java.util.UUID; public record FunctionSummaryResponse( - UUID id, UUID analysisId, String name, String address, int size, String flags) { + UUID id, + UUID analysisId, + String name, + String resolvedName, + String address, + int size, + String flags) { + public static FunctionSummaryResponse from(StaticFunction f) { + return from(f, null); + } + + public static FunctionSummaryResponse from(StaticFunction f, Map resolution) { + var resolved = Objects.nonNull(resolution) ? resolution.get(f.getName()) : null; return new FunctionSummaryResponse( f.getId(), f.getAnalysis().getId(), f.getName(), + resolved, f.getAddress(), f.getFunctionSize() != null ? f.getFunctionSize() : 0, f.getFlags()); diff --git a/src/main/java/ai/decompile/analysis/model/dto/ImportResponse.java b/src/main/java/ai/decompile/analysis/model/dto/ImportResponse.java index cf98fd7..8dab4e9 100644 --- a/src/main/java/ai/decompile/analysis/model/dto/ImportResponse.java +++ b/src/main/java/ai/decompile/analysis/model/dto/ImportResponse.java @@ -1,11 +1,32 @@ package ai.decompile.analysis.model.dto; import ai.decompile.analysis.model.entity.StaticImport; +import java.util.Objects; import java.util.UUID; +import lombok.NonNull; -public record ImportResponse(UUID id, String dll, String name, String address, int ordinal) { - public static ImportResponse from(StaticImport imp) { +public record ImportResponse( + UUID id, + String dll, + String name, + String address, + int ordinal, + String resolvedName, + String libraryName, + String className, + UUID librarySymbolId) { + + public static ImportResponse from(@NonNull StaticImport imp) { + var sym = imp.getLibrarySymbol(); return new ImportResponse( - imp.getId(), imp.getDll(), imp.getName(), imp.getAddress(), imp.getOrdinal()); + imp.getId(), + imp.getDll(), + imp.getName(), + imp.getAddress(), + imp.getOrdinal(), + imp.getResolvedName(), + Objects.nonNull(sym) ? sym.getLibrary().getName() : null, + Objects.nonNull(sym) ? sym.getClassName() : null, + imp.getLibrarySymbolId()); } } diff --git a/src/main/java/ai/decompile/analysis/model/dto/LibraryResponse.java b/src/main/java/ai/decompile/analysis/model/dto/LibraryResponse.java new file mode 100644 index 0000000..be244a8 --- /dev/null +++ b/src/main/java/ai/decompile/analysis/model/dto/LibraryResponse.java @@ -0,0 +1,21 @@ +package ai.decompile.analysis.model.dto; + +import ai.decompile.analysis.model.entity.Library; +import java.time.Instant; +import java.util.Objects; +import java.util.UUID; +import lombok.NonNull; + +public record LibraryResponse( + UUID id, String name, String description, int symbolCount, int aliasCount, Instant createdAt) { + + public static LibraryResponse from(@NonNull Library lib) { + return new LibraryResponse( + lib.getId(), + lib.getName(), + lib.getDescription(), + Objects.nonNull(lib.getSymbols()) ? lib.getSymbols().size() : 0, + Objects.nonNull(lib.getAliases()) ? lib.getAliases().size() : 0, + lib.getCreatedAt()); + } +} diff --git a/src/main/java/ai/decompile/analysis/model/dto/LibrarySymbolResponse.java b/src/main/java/ai/decompile/analysis/model/dto/LibrarySymbolResponse.java new file mode 100644 index 0000000..39907bb --- /dev/null +++ b/src/main/java/ai/decompile/analysis/model/dto/LibrarySymbolResponse.java @@ -0,0 +1,28 @@ +package ai.decompile.analysis.model.dto; + +import ai.decompile.analysis.model.entity.LibrarySymbol; +import java.util.UUID; +import lombok.NonNull; + +public record LibrarySymbolResponse( + UUID id, + String moduleName, + String className, + String symbolName, + int ordinal, + String symbolType, + String parameters, + String category) { + + public static LibrarySymbolResponse from(@NonNull LibrarySymbol sym) { + return new LibrarySymbolResponse( + sym.getId(), + sym.getModuleName(), + sym.getClassName(), + sym.getSymbolName(), + sym.getOrdinal(), + sym.getSymbolType(), + sym.getParameters(), + sym.getCategory()); + } +} diff --git a/src/main/java/ai/decompile/analysis/model/dto/XrefResponse.java b/src/main/java/ai/decompile/analysis/model/dto/XrefResponse.java index aa4375e..559e602 100644 --- a/src/main/java/ai/decompile/analysis/model/dto/XrefResponse.java +++ b/src/main/java/ai/decompile/analysis/model/dto/XrefResponse.java @@ -1,24 +1,43 @@ package ai.decompile.analysis.model.dto; import ai.decompile.analysis.model.entity.StaticXref; +import java.util.Map; +import java.util.Objects; import java.util.UUID; public record XrefResponse( UUID id, UUID callerId, String callerName, + String resolvedCallerName, UUID calleeId, String calleeName, + String resolvedCalleeName, String type, String address) { + public static XrefResponse from(StaticXref xref) { + return from(xref, Map.of()); + } + + public static XrefResponse from(StaticXref xref, Map nameResolution) { return new XrefResponse( xref.getId(), xref.getCaller().getId(), xref.getCaller().getName(), + resolve(xref.getCaller().getName(), nameResolution), xref.getCallee().getId(), xref.getCallee().getName(), + resolve(xref.getCallee().getName(), nameResolution), xref.getType(), xref.getAddress()); } + + private static String resolve(String name, Map resolution) { + if (Objects.isNull(name) || resolution.isEmpty()) { + return null; + } + + return resolution.get(name); + } } diff --git a/src/main/java/ai/decompile/analysis/model/entity/Library.java b/src/main/java/ai/decompile/analysis/model/entity/Library.java new file mode 100644 index 0000000..ea12698 --- /dev/null +++ b/src/main/java/ai/decompile/analysis/model/entity/Library.java @@ -0,0 +1,40 @@ +package ai.decompile.analysis.model.entity; + +import jakarta.persistence.*; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import lombok.*; +import org.hibernate.annotations.CreationTimestamp; + +@Entity +@Table(name = "library") +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class Library { + @Id + @GeneratedValue(strategy = GenerationType.UUID) + private UUID id; + + @Column(nullable = false) + private String name; + + @Column(columnDefinition = "TEXT") + private String description; + + @OneToMany(mappedBy = "library", cascade = CascadeType.ALL, orphanRemoval = true) + @Builder.Default + private List aliases = new ArrayList<>(); + + @OneToMany(mappedBy = "library", cascade = CascadeType.ALL, orphanRemoval = true) + @Builder.Default + private List symbols = new ArrayList<>(); + + @CreationTimestamp + @Column(name = "created_at", nullable = false, updatable = false) + private Instant createdAt; +} diff --git a/src/main/java/ai/decompile/analysis/model/entity/LibraryAlias.java b/src/main/java/ai/decompile/analysis/model/entity/LibraryAlias.java new file mode 100644 index 0000000..64dc297 --- /dev/null +++ b/src/main/java/ai/decompile/analysis/model/entity/LibraryAlias.java @@ -0,0 +1,27 @@ +package ai.decompile.analysis.model.entity; + +import jakarta.persistence.*; +import java.util.UUID; +import lombok.*; + +@Entity +@Table( + name = "library_alias", + uniqueConstraints = {@UniqueConstraint(columnNames = {"library_id", "alias"})}) +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class LibraryAlias { + @Id + @GeneratedValue(strategy = GenerationType.UUID) + private UUID id; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "library_id", nullable = false) + private Library library; + + @Column(nullable = false) + private String alias; +} diff --git a/src/main/java/ai/decompile/analysis/model/entity/LibrarySymbol.java b/src/main/java/ai/decompile/analysis/model/entity/LibrarySymbol.java new file mode 100644 index 0000000..cc3b15e --- /dev/null +++ b/src/main/java/ai/decompile/analysis/model/entity/LibrarySymbol.java @@ -0,0 +1,49 @@ +package ai.decompile.analysis.model.entity; + +import jakarta.persistence.*; +import java.time.Instant; +import java.util.UUID; +import lombok.*; +import org.hibernate.annotations.CreationTimestamp; + +@Entity +@Table(name = "library_symbol") +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class LibrarySymbol { + @Id + @GeneratedValue(strategy = GenerationType.UUID) + private UUID id; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "library_id", nullable = false) + private Library library; + + @Column(name = "module_name", nullable = false) + private String moduleName; + + @Column(name = "class_name", length = 500) + private String className; + + @Column(name = "symbol_name", nullable = false, length = 500) + private String symbolName; + + @Column(nullable = false) + private int ordinal; + + @Column(name = "symbol_type", length = 50) + private String symbolType; + + @Column(length = 1000) + private String parameters; + + @Column(length = 100) + private String category; + + @CreationTimestamp + @Column(name = "created_at", nullable = false, updatable = false) + private Instant createdAt; +} diff --git a/src/main/java/ai/decompile/analysis/model/entity/StaticImport.java b/src/main/java/ai/decompile/analysis/model/entity/StaticImport.java index da246d5..f7cc915 100644 --- a/src/main/java/ai/decompile/analysis/model/entity/StaticImport.java +++ b/src/main/java/ai/decompile/analysis/model/entity/StaticImport.java @@ -32,4 +32,14 @@ public class StaticImport { @Column(nullable = false) @Builder.Default private int ordinal = 0; + + @ManyToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "library_symbol_id") + private LibrarySymbol librarySymbol; + + @Column(name = "library_symbol_id", insertable = false, updatable = false) + private UUID librarySymbolId; + + @Column(name = "resolved_name", length = 500) + private String resolvedName; } diff --git a/src/main/java/ai/decompile/analysis/model/repository/LibraryAliasRepository.java b/src/main/java/ai/decompile/analysis/model/repository/LibraryAliasRepository.java new file mode 100644 index 0000000..0e8e780 --- /dev/null +++ b/src/main/java/ai/decompile/analysis/model/repository/LibraryAliasRepository.java @@ -0,0 +1,7 @@ +package ai.decompile.analysis.model.repository; + +import ai.decompile.analysis.model.entity.LibraryAlias; +import java.util.UUID; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface LibraryAliasRepository extends JpaRepository {} diff --git a/src/main/java/ai/decompile/analysis/model/repository/LibraryRepository.java b/src/main/java/ai/decompile/analysis/model/repository/LibraryRepository.java new file mode 100644 index 0000000..d841ec5 --- /dev/null +++ b/src/main/java/ai/decompile/analysis/model/repository/LibraryRepository.java @@ -0,0 +1,7 @@ +package ai.decompile.analysis.model.repository; + +import ai.decompile.analysis.model.entity.Library; +import java.util.UUID; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface LibraryRepository extends JpaRepository {} diff --git a/src/main/java/ai/decompile/analysis/model/repository/LibrarySymbolRepository.java b/src/main/java/ai/decompile/analysis/model/repository/LibrarySymbolRepository.java new file mode 100644 index 0000000..4782d2e --- /dev/null +++ b/src/main/java/ai/decompile/analysis/model/repository/LibrarySymbolRepository.java @@ -0,0 +1,14 @@ +package ai.decompile.analysis.model.repository; + +import ai.decompile.analysis.model.entity.LibrarySymbol; +import java.util.Optional; +import java.util.UUID; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface LibrarySymbolRepository extends JpaRepository { + Page findByLibraryIdOrderByOrdinalAsc(UUID libraryId, Pageable pageable); + + Optional findByLibraryIdAndOrdinal(UUID libraryId, int ordinal); +} diff --git a/src/main/java/ai/decompile/analysis/service/AnalysisService.java b/src/main/java/ai/decompile/analysis/service/AnalysisService.java index 5d11191..ef5b607 100644 --- a/src/main/java/ai/decompile/analysis/service/AnalysisService.java +++ b/src/main/java/ai/decompile/analysis/service/AnalysisService.java @@ -38,6 +38,7 @@ import ai.decompile.workspace.model.entity.Binary; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.function.Function; import lombok.RequiredArgsConstructor; @@ -92,18 +93,21 @@ public class AnalysisService { public Page listFunctions(UUID analysisId, Pageable pageable) { assertAnalysisExists(analysisId); + var resolution = buildNameResolutionMap(analysisId); return functionRepository .findByAnalysisId(analysisId, pageable) - .map(FunctionSummaryResponse::from); + .map(f -> FunctionSummaryResponse.from(f, resolution)); } public FunctionDetailResponse getFunction(UUID functionId) { var func = findFunction(functionId); + var resolution = buildNameResolutionMap(func.getAnalysis().getId()); + var resolvedName = resolution.get(func.getName()); int calleeCount = xrefRepository.countByCallerId(functionId); int callerCount = xrefRepository.countByCalleeId(functionId); var labels = labelRepository.findByFunctionId(functionId).stream().map(LabelResponse::from).toList(); - return FunctionDetailResponse.from(func, callerCount, calleeCount, labels); + return FunctionDetailResponse.from(func, callerCount, calleeCount, labels, resolvedName); } public List getCallers(UUID functionId) { @@ -327,8 +331,32 @@ public class AnalysisService { } private List listXrefs(UUID functionId, Function> finder) { - findFunction(functionId); - return finder.apply(functionId).stream().map(XrefResponse::from).toList(); + var func = findFunction(functionId); + var resolution = buildNameResolutionMap(func.getAnalysis().getId()); + return finder.apply(functionId).stream().map(xr -> XrefResponse.from(xr, resolution)).toList(); + } + + private Map buildNameResolutionMap(UUID analysisId) { + var imports = importRepository.findByAnalysisIdOrderByDllAscNameAsc(analysisId); + if (imports.isEmpty()) { + return Map.of(); + } + + var map = new HashMap(); + for (var imp : imports) { + if (Objects.isNull(imp.getResolvedName())) { + continue; + } + + map.putIfAbsent(imp.getName(), imp.getResolvedName()); + if (imp.getOrdinal() > 0) { + map.putIfAbsent("ord_" + imp.getOrdinal(), imp.getResolvedName()); + map.putIfAbsent("j_ord_" + imp.getOrdinal(), imp.getResolvedName()); + map.putIfAbsent(imp.getDll() + "_" + imp.getOrdinal(), imp.getResolvedName()); + } + } + + return map; } private void assertAnalysisExists(UUID analysisId) { diff --git a/src/main/java/ai/decompile/analysis/service/LibraryService.java b/src/main/java/ai/decompile/analysis/service/LibraryService.java new file mode 100644 index 0000000..4829e86 --- /dev/null +++ b/src/main/java/ai/decompile/analysis/service/LibraryService.java @@ -0,0 +1,149 @@ +package ai.decompile.analysis.service; + +import ai.decompile.analysis.model.dto.LibraryResponse; +import ai.decompile.analysis.model.dto.LibrarySymbolResponse; +import ai.decompile.analysis.model.entity.Library; +import ai.decompile.analysis.model.entity.LibraryAlias; +import ai.decompile.analysis.model.entity.LibrarySymbol; +import ai.decompile.analysis.model.repository.LibraryAliasRepository; +import ai.decompile.analysis.model.repository.LibraryRepository; +import ai.decompile.analysis.model.repository.LibrarySymbolRepository; +import ai.decompile.analysis.service.parser.LibrarySymbolParserRegistry; +import ai.decompile.common.exception.NotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class LibraryService { + private static final int SYMBOL_BATCH_SIZE = 500; + + private final LibraryRepository libraryRepository; + private final LibraryAliasRepository aliasRepository; + private final LibrarySymbolRepository symbolRepository; + private final LibrarySymbolParserRegistry parserRegistry; + + @Transactional + public LibraryResponse parseAndStore( + String name, String description, String parserType, String aliases, MultipartFile file) { + var parser = parserRegistry.get(parserType); + + var library = + libraryRepository.save(Library.builder().name(name).description(description).build()); + + var aliasList = parseAliasList(aliases); + for (var alias : aliasList) { + aliasRepository.save(LibraryAlias.builder().library(library).alias(alias).build()); + } + + var rawContent = readFileContent(file); + var symbols = parser.parse(rawContent); + var batch = new ArrayList(SYMBOL_BATCH_SIZE); + + for (var parsed : symbols) { + batch.add( + LibrarySymbol.builder() + .library(library) + .moduleName(parsed.moduleName()) + .className(parsed.className()) + .symbolName(parsed.symbolName()) + .ordinal(parsed.ordinal()) + .symbolType(parsed.symbolType()) + .parameters(parsed.parameters()) + .category(parsed.category()) + .build()); + + if (batch.size() >= SYMBOL_BATCH_SIZE) { + symbolRepository.saveAll(batch); + batch.clear(); + } + } + + if (!batch.isEmpty()) { + symbolRepository.saveAll(batch); + } + + log.info( + "Library '{}' created with {} symbols and {} aliases", + name, + symbols.size(), + aliasList.size()); + + return LibraryResponse.from(library); + } + + @Transactional(readOnly = true) + public List listLibraries() { + return libraryRepository.findAll().stream().map(LibraryResponse::from).toList(); + } + + @Transactional(readOnly = true) + public LibraryResponse getLibrary(UUID id) { + return LibraryResponse.from(findLibrary(id)); + } + + @Transactional + public void deleteLibrary(UUID id) { + var library = findLibrary(id); + libraryRepository.delete(library); + log.info("Library '{}' deleted", library.getName()); + } + + @Transactional(readOnly = true) + public Page listSymbols(UUID libraryId, Pageable pageable) { + return symbolRepository + .findByLibraryIdOrderByOrdinalAsc(libraryId, pageable) + .map(LibrarySymbolResponse::from); + } + + @Transactional(readOnly = true) + public LibrarySymbolResponse getSymbol(UUID id) { + return LibrarySymbolResponse.from(findSymbol(id)); + } + + @Transactional(readOnly = true) + public List getAvailableParsers() { + return parserRegistry.getAvailableFormats(); + } + + private String readFileContent(MultipartFile file) { + try { + return new String(file.getBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException("Failed to read uploaded library dump file", e); + } + } + + private List parseAliasList(String aliases) { + if (Objects.isNull(aliases) || aliases.isBlank()) { + return List.of(); + } + + return Stream.of(aliases.split(",")).map(String::trim).filter(a -> !a.isEmpty()).toList(); + } + + Library findLibrary(UUID id) { + return libraryRepository + .findById(id) + .orElseThrow(() -> new NotFoundException("Library with id '" + id + "' not found")); + } + + LibrarySymbol findSymbol(UUID id) { + return symbolRepository + .findById(id) + .orElseThrow(() -> new NotFoundException("Library symbol with id '" + id + "' not found")); + } +} diff --git a/src/main/java/ai/decompile/analysis/service/SymbolResolutionService.java b/src/main/java/ai/decompile/analysis/service/SymbolResolutionService.java new file mode 100644 index 0000000..593102b --- /dev/null +++ b/src/main/java/ai/decompile/analysis/service/SymbolResolutionService.java @@ -0,0 +1,142 @@ +package ai.decompile.analysis.service; + +import ai.decompile.analysis.model.repository.LibraryAliasRepository; +import ai.decompile.analysis.model.repository.LibraryRepository; +import ai.decompile.analysis.model.repository.LibrarySymbolRepository; +import ai.decompile.analysis.model.repository.StaticImportRepository; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Log4j2 +@Service +@RequiredArgsConstructor +public class SymbolResolutionService { + private final LibraryAliasRepository aliasRepository; + private final LibrarySymbolRepository symbolRepository; + private final LibraryRepository libraryRepository; + private final StaticImportRepository importRepository; + + @Transactional + public int resolveImports(UUID analysisId) { + var imports = importRepository.findByAnalysisIdOrderByDllAscNameAsc(analysisId); + if (imports.isEmpty()) { + return 0; + } + + var lookup = buildLibraryLookup(); + if (lookup.isEmpty()) { + log.debug("No libraries with symbols found, skipping resolution for analysis {}", analysisId); + return 0; + } + + var resolvedCount = 0; + for (var imp : imports) { + if (imp.getOrdinal() == 0) { + continue; + } + + if (Objects.nonNull(imp.getResolvedName())) { + continue; + } + + var libraryId = lookup.get(imp.getDll().toLowerCase()); + if (Objects.isNull(libraryId)) { + libraryId = lookup.get(imp.getDll().toLowerCase().replace(".dll", "")); + } + + if (Objects.isNull(libraryId)) { + continue; + } + + var symbol = symbolRepository.findByLibraryIdAndOrdinal(libraryId, imp.getOrdinal()); + if (symbol.isEmpty()) { + continue; + } + + imp.setLibrarySymbol(symbol.get()); + imp.setResolvedName(symbol.get().getSymbolName()); + importRepository.save(imp); + resolvedCount++; + } + + var totalWithOrdinal = imports.stream().filter(i -> i.getOrdinal() != 0).count(); + log.info( + "Symbol resolution: resolved {}/{} ordinal imports ({}%) for analysis {}", + resolvedCount, + totalWithOrdinal, + totalWithOrdinal > 0 ? (resolvedCount * 100L / totalWithOrdinal) : 0, + analysisId); + + return resolvedCount; + } + + @Transactional + public int resolveAllUnresolved() { + var allUnresolved = + importRepository.findAll().stream() + .filter( + i -> + i.getOrdinal() != 0 + && Objects.isNull(i.getResolvedName()) + && Objects.isNull(i.getLibrarySymbolId())) + .toList(); + + if (allUnresolved.isEmpty()) { + return 0; + } + + var lookup = buildLibraryLookup(); + if (lookup.isEmpty()) { + return 0; + } + + var resolvedCount = 0; + for (var imp : allUnresolved) { + var libraryId = lookup.get(imp.getDll().toLowerCase()); + if (Objects.isNull(libraryId)) { + libraryId = lookup.get(imp.getDll().toLowerCase().replace(".dll", "")); + } + + if (Objects.isNull(libraryId)) { + continue; + } + + var symbol = symbolRepository.findByLibraryIdAndOrdinal(libraryId, imp.getOrdinal()); + if (symbol.isEmpty()) { + continue; + } + + imp.setLibrarySymbol(symbol.get()); + imp.setResolvedName(symbol.get().getSymbolName()); + importRepository.save(imp); + resolvedCount++; + } + + log.info("Global symbol resolution: resolved {} previously unresolved imports", resolvedCount); + return resolvedCount; + } + + private Map buildLibraryLookup() { + var lookup = new HashMap(); + + for (var alias : aliasRepository.findAll()) { + lookup.putIfAbsent(alias.getAlias().toLowerCase(), alias.getLibrary().getId()); + } + + for (var lib : libraryRepository.findAll()) { + for (var sym : lib.getSymbols()) { + if (Objects.nonNull(sym.getModuleName()) && !sym.getModuleName().isBlank()) { + lookup.putIfAbsent(sym.getModuleName().toLowerCase(), lib.getId()); + } + } + } + + return lookup; + } +} diff --git a/src/main/java/ai/decompile/analysis/service/parser/LibrarySymbolParser.java b/src/main/java/ai/decompile/analysis/service/parser/LibrarySymbolParser.java new file mode 100644 index 0000000..9ba4389 --- /dev/null +++ b/src/main/java/ai/decompile/analysis/service/parser/LibrarySymbolParser.java @@ -0,0 +1,11 @@ +package ai.decompile.analysis.service.parser; + +import java.util.List; + +public interface LibrarySymbolParser { + String getFormatName(); + + boolean canParse(String rawContent); + + List parse(String rawContent); +} diff --git a/src/main/java/ai/decompile/analysis/service/parser/LibrarySymbolParserRegistry.java b/src/main/java/ai/decompile/analysis/service/parser/LibrarySymbolParserRegistry.java new file mode 100644 index 0000000..5eedba1 --- /dev/null +++ b/src/main/java/ai/decompile/analysis/service/parser/LibrarySymbolParserRegistry.java @@ -0,0 +1,33 @@ +package ai.decompile.analysis.service.parser; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.springframework.stereotype.Component; + +@Component +public class LibrarySymbolParserRegistry { + private final Map parsers; + + public LibrarySymbolParserRegistry(List parserList) { + this.parsers = + parserList.stream() + .collect(Collectors.toMap(LibrarySymbolParser::getFormatName, Function.identity())); + } + + public LibrarySymbolParser get(String formatName) { + var parser = parsers.get(formatName); + if (Objects.isNull(parser)) { + throw new IllegalArgumentException( + "No parser found for format '" + formatName + "'. Available: " + parsers.keySet()); + } + + return parser; + } + + public List getAvailableFormats() { + return List.copyOf(parsers.keySet()); + } +} diff --git a/src/main/java/ai/decompile/analysis/service/parser/ParsedSymbol.java b/src/main/java/ai/decompile/analysis/service/parser/ParsedSymbol.java new file mode 100644 index 0000000..5bb339e --- /dev/null +++ b/src/main/java/ai/decompile/analysis/service/parser/ParsedSymbol.java @@ -0,0 +1,10 @@ +package ai.decompile.analysis.service.parser; + +public record ParsedSymbol( + String moduleName, + String className, + String symbolName, + int ordinal, + String symbolType, + String parameters, + String category) {} diff --git a/src/main/java/ai/decompile/analysis/service/parser/TurboDumpParser.java b/src/main/java/ai/decompile/analysis/service/parser/TurboDumpParser.java new file mode 100644 index 0000000..ac9ae1c --- /dev/null +++ b/src/main/java/ai/decompile/analysis/service/parser/TurboDumpParser.java @@ -0,0 +1,267 @@ +package ai.decompile.analysis.service.parser; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Component; + +@Log4j2 +@Component +public class TurboDumpParser implements LibrarySymbolParser { + private static final String FORMAT_NAME = "TURBO_DUMP"; + + private static final Map CLASS_CATEGORY_MAP = new HashMap<>(); + + static { + CLASS_CATEGORY_MAP.put("TAppDictionary", "Core/Application"); + CLASS_CATEGORY_MAP.put("TApplication", "Core/Application"); + CLASS_CATEGORY_MAP.put("TModule", "Core/Application"); + CLASS_CATEGORY_MAP.put("TStatus", "Core/Application"); + CLASS_CATEGORY_MAP.put("TXCompatibility", "Core/Application"); + CLASS_CATEGORY_MAP.put("TFrameWindow", "UI/Window"); + CLASS_CATEGORY_MAP.put("TWindow", "UI/Window"); + CLASS_CATEGORY_MAP.put("TDialog", "UI/Window"); + CLASS_CATEGORY_MAP.put("TCommonDialog", "UI/Dialog"); + CLASS_CATEGORY_MAP.put("TChooseColorDialog", "UI/Dialog"); + CLASS_CATEGORY_MAP.put("TChooseFontDialog", "UI/Dialog"); + CLASS_CATEGORY_MAP.put("TButtonGadget", "UI/Control"); + CLASS_CATEGORY_MAP.put("TButtonGadgetEnabler", "UI/Control"); + CLASS_CATEGORY_MAP.put("TGadget", "UI/Control"); + CLASS_CATEGORY_MAP.put("TControlGadget", "UI/Control"); + CLASS_CATEGORY_MAP.put("TControlBar", "UI/Control"); + CLASS_CATEGORY_MAP.put("TBorderGadget", "UI/Control"); + CLASS_CATEGORY_MAP.put("TBitmapGadget", "UI/Control"); + CLASS_CATEGORY_MAP.put("TBitmap", "Graphics/Bitmap"); + CLASS_CATEGORY_MAP.put("TDib", "Graphics/Bitmap"); + CLASS_CATEGORY_MAP.put("TDibDC", "Graphics/Bitmap"); + CLASS_CATEGORY_MAP.put("TPalette", "Graphics/Palette"); + CLASS_CATEGORY_MAP.put("TIcon", "Graphics/Icon"); + CLASS_CATEGORY_MAP.put("TBrush", "Graphics/Drawing"); + CLASS_CATEGORY_MAP.put("THatch8x8Brush", "Graphics/Drawing"); + CLASS_CATEGORY_MAP.put("TPen", "Graphics/Drawing"); + CLASS_CATEGORY_MAP.put("TFont", "Graphics/Font"); + CLASS_CATEGORY_MAP.put("TDC", "Graphics/DC"); + CLASS_CATEGORY_MAP.put("TCreatedDC", "Graphics/DC"); + CLASS_CATEGORY_MAP.put("TIC", "Graphics/DC"); + CLASS_CATEGORY_MAP.put("TColor", "Graphics/Color"); + CLASS_CATEGORY_MAP.put("TCursor", "UI/Cursor"); + CLASS_CATEGORY_MAP.put("TClipboard", "UI/Clipboard"); + CLASS_CATEGORY_MAP.put("TClipboardViewer", "UI/Clipboard"); + CLASS_CATEGORY_MAP.put("TBitSet", "Core/DataTypes"); + CLASS_CATEGORY_MAP.put("TCharSet", "Core/DataTypes"); + CLASS_CATEGORY_MAP.put("TCelArray", "UI/Control"); + CLASS_CATEGORY_MAP.put("TEventHandler", "Core/EventSystem"); + CLASS_CATEGORY_MAP.put("TResponseTableEntry", "Core/EventSystem"); + } + + @Override + public String getFormatName() { + return FORMAT_NAME; + } + + @Override + public boolean canParse(String rawContent) { + if (Objects.isNull(rawContent) || rawContent.isBlank()) { + return false; + } + + return rawContent.contains("Turbo Dump") && rawContent.contains("MSLIBR"); + } + + @Override + public List parse(String rawContent) { + if (Objects.isNull(rawContent) || rawContent.isBlank()) { + return List.of(); + } + + var text = rawContent.replace("\r\n", "\n"); + var symbols = new ArrayList(); + var lines = text.split("\n"); + + StringBuilder currentBlock = new StringBuilder(); + var inBlock = false; + + for (var line : lines) { + if (line.contains("THEADR ")) { + inBlock = true; + currentBlock = new StringBuilder(); + currentBlock.append(line).append("\n"); + } else if (inBlock) { + if (line.contains("MODEND")) { + currentBlock.append(line).append("\n"); + var symbol = parseBlock(currentBlock.toString()); + if (Objects.nonNull(symbol)) { + symbols.add(symbol); + } + + inBlock = false; + currentBlock = new StringBuilder(); + } else { + currentBlock.append(line).append("\n"); + } + } + } + + log.info("TurboDumpParser extracted {} symbols", symbols.size()); + return symbols; + } + + private ParsedSymbol parseBlock(String block) { + var lines = block.split("\n"); + + String symbolName = null; + String moduleName = null; + String internalName = null; + int ordinal = -1; + + for (var line : lines) { + var trimmed = line.trim(); + + if (trimmed.contains("THEADR ")) { + symbolName = extractAfter(extractAfterHexPrefix(trimmed), "THEADR "); + } else if (trimmed.startsWith("Module Name:")) { + moduleName = extractAfter(trimmed, "Module Name:"); + } else if (trimmed.startsWith("Internal Name:")) { + internalName = extractAfter(trimmed, "Internal Name:"); + } else if (trimmed.startsWith("ordinal:")) { + try { + ordinal = Integer.parseInt(extractAfter(trimmed, "ordinal:")); + } catch (NumberFormatException e) { + log.debug("Failed to parse ordinal from line: '{}'", trimmed); + } + } + } + + if (ordinal < 0 || Objects.isNull(symbolName) || symbolName.isBlank()) { + return null; + } + + var effectiveName = + Objects.nonNull(internalName) && !internalName.isBlank() ? internalName : symbolName; + var effectiveModule = Objects.nonNull(moduleName) ? moduleName : "unknown"; + + var type = classifySymbolType(effectiveName); + var className = extractClassName(effectiveName); + var params = extractParameters(effectiveName); + var category = classifyCategory(className, effectiveName); + + return new ParsedSymbol( + effectiveModule, className, effectiveName, ordinal, type, params, category); + } + + private String extractAfterHexPrefix(String text) { + if (text.length() >= 7 && text.substring(0, 6).matches("[0-9A-Fa-f]{6}")) { + return text.substring(6).trim(); + } + + return text; + } + + private String extractAfter(String text, String prefix) { + var idx = text.indexOf(prefix); + if (idx < 0) { + return ""; + } + + return text.substring(idx + prefix.length()).trim(); + } + + private String classifySymbolType(String name) { + if (name.contains("::~") + || (name.contains("::") && name.substring(name.lastIndexOf("::") + 2).startsWith("~"))) { + return "DESTRUCTOR"; + } + + if (name.contains("::")) { + var colonIdx = name.lastIndexOf("::"); + var methodPart = name.substring(colonIdx + 2); + var className = name.substring(0, colonIdx); + var methodName = + methodPart.contains("(") ? methodPart.substring(0, methodPart.indexOf('(')) : methodPart; + + if (methodName.equals(className)) { + return "CONSTRUCTOR"; + } + + return "METHOD"; + } + + return "FUNCTION"; + } + + private String extractClassName(String name) { + var colonIdx = name.indexOf("::"); + if (colonIdx < 0) { + return null; + } + + return name.substring(0, colonIdx).trim(); + } + + private String extractParameters(String name) { + var parenStart = name.indexOf('('); + if (parenStart < 0) { + return null; + } + + var parenEnd = name.lastIndexOf(')'); + if (parenEnd <= parenStart) { + return null; + } + + var params = name.substring(parenStart + 1, parenEnd).trim(); + if (params.isEmpty()) { + return null; + } + + return params; + } + + private String classifyCategory(String className, String symbolName) { + if (Objects.nonNull(className)) { + var category = CLASS_CATEGORY_MAP.get(className); + if (Objects.nonNull(category)) { + return category; + } + + if (className.startsWith("TFrame")) { + return "UI/Window"; + } + + if (className.startsWith("TButton") || className.startsWith("TControl")) { + return "UI/Control"; + } + + if (className.startsWith("TBrush") || className.startsWith("TPen")) { + return "Graphics/Drawing"; + } + + if (className.startsWith("TBitmap") || className.startsWith("TPalette")) { + return "Graphics/Bitmap"; + } + + if (className.startsWith("TDialog")) { + return "UI/Dialog"; + } + + if (className.startsWith("TDC") || className.startsWith("TIC")) { + return "Graphics/DC"; + } + } + + if (Objects.nonNull(symbolName)) { + if (symbolName.startsWith("_Owl") || symbolName.startsWith("GETTASK")) { + return "Core/Application"; + } + + if (symbolName.startsWith("NColors") || symbolName.startsWith("NBits")) { + return "Graphics/Utility"; + } + } + + return "Core/Other"; + } +} diff --git a/src/main/java/ai/decompile/job/service/handler/StaticAnalysisHandler.java b/src/main/java/ai/decompile/job/service/handler/StaticAnalysisHandler.java index 9b7a9d4..6a4b6bf 100644 --- a/src/main/java/ai/decompile/job/service/handler/StaticAnalysisHandler.java +++ b/src/main/java/ai/decompile/job/service/handler/StaticAnalysisHandler.java @@ -2,6 +2,7 @@ package ai.decompile.job.service.handler; import ai.decompile.analysis.event.StaticAnalysisCompletedEvent; import ai.decompile.analysis.service.AnalysisService; +import ai.decompile.analysis.service.SymbolResolutionService; import ai.decompile.engine.service.EngineService; import ai.decompile.job.model.entity.Job; import ai.decompile.job.model.enums.JobStatus; @@ -23,6 +24,7 @@ import org.springframework.stereotype.Component; public class StaticAnalysisHandler implements JobHandler { private final Map engineServices; private final AnalysisService analysisService; + private final SymbolResolutionService symbolResolutionService; private final BinaryService binaryService; private final ApplicationEventPublisher eventPublisher; private final ObjectMapper objectMapper; @@ -30,6 +32,7 @@ public class StaticAnalysisHandler implements JobHandler { public StaticAnalysisHandler( List engineServiceList, AnalysisService analysisService, + SymbolResolutionService symbolResolutionService, BinaryService binaryService, ApplicationEventPublisher eventPublisher, ObjectMapper objectMapper) { @@ -37,6 +40,7 @@ public class StaticAnalysisHandler implements JobHandler { engineServiceList.stream() .collect(Collectors.toMap(e -> e.supportedEngine().name(), Function.identity())); this.analysisService = analysisService; + this.symbolResolutionService = symbolResolutionService; this.binaryService = binaryService; this.eventPublisher = eventPublisher; this.objectMapper = objectMapper; @@ -67,6 +71,8 @@ public class StaticAnalysisHandler implements JobHandler { var binary = binaryService.getBinary(job.getBinaryId()); var analysisResponse = analysisService.saveResult(binary, job.getId(), engineName, result); + symbolResolutionService.resolveImports(analysisResponse.id()); + binaryService.markStaticAnalysisDone(job.getBinaryId()); job.setResult(toJson(Map.of("analysisId", analysisResponse.id().toString()))); diff --git a/src/main/resources/db/migration/V021__create_library_tables.sql b/src/main/resources/db/migration/V021__create_library_tables.sql new file mode 100644 index 0000000..2c29e46 --- /dev/null +++ b/src/main/resources/db/migration/V021__create_library_tables.sql @@ -0,0 +1,34 @@ +CREATE TABLE library +( + id UUID PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE library_alias +( + id UUID PRIMARY KEY, + library_id UUID NOT NULL REFERENCES library (id) ON DELETE CASCADE, + alias VARCHAR(255) NOT NULL, + UNIQUE (library_id, alias) +); + +CREATE INDEX idx_library_alias_lookup ON library_alias (alias); + +CREATE TABLE library_symbol +( + id UUID PRIMARY KEY, + library_id UUID NOT NULL REFERENCES library (id) ON DELETE CASCADE, + module_name VARCHAR(255) NOT NULL, + class_name VARCHAR(500), + symbol_name VARCHAR(500) NOT NULL, + ordinal INTEGER NOT NULL, + symbol_type VARCHAR(50), + parameters VARCHAR(1000), + category VARCHAR(100), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_library_symbol_library_ordinal ON library_symbol (library_id, ordinal); +CREATE INDEX idx_library_symbol_class ON library_symbol (class_name) WHERE class_name IS NOT NULL; diff --git a/src/main/resources/db/migration/V022__alter_static_import_library.sql b/src/main/resources/db/migration/V022__alter_static_import_library.sql new file mode 100644 index 0000000..db8f611 --- /dev/null +++ b/src/main/resources/db/migration/V022__alter_static_import_library.sql @@ -0,0 +1,7 @@ +ALTER TABLE static_import + ADD COLUMN library_symbol_id UUID REFERENCES library_symbol (id) ON DELETE SET NULL; + +ALTER TABLE static_import + ADD COLUMN resolved_name VARCHAR(500); + +CREATE INDEX idx_static_import_library_symbol ON static_import (library_symbol_id) WHERE library_symbol_id IS NOT NULL; diff --git a/src/test/java/ai/decompile/analysis/service/parser/TurboDumpParserTest.java b/src/test/java/ai/decompile/analysis/service/parser/TurboDumpParserTest.java new file mode 100644 index 0000000..cff14a5 --- /dev/null +++ b/src/test/java/ai/decompile/analysis/service/parser/TurboDumpParserTest.java @@ -0,0 +1,295 @@ +package ai.decompile.analysis.service.parser; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +class TurboDumpParserTest { + + private final TurboDumpParser parser = new TurboDumpParser(); + + @Test + void shouldReturnFormatName() { + assertEquals("TURBO_DUMP", parser.getFormatName()); + } + + @Test + void shouldRecognizeTurboDumpContent() { + assertTrue(parser.canParse("Turbo Dump Version 5.0.13.1\nMSLIBR Index begins at file offset")); + } + + @Test + void shouldRejectNonTurboDumpContent() { + assertFalse(parser.canParse("some random text")); + assertFalse(parser.canParse(null)); + assertFalse(parser.canParse("")); + } + + @Test + void shouldParseConstructor() { + var content = + """ + Turbo Dump Version 5.0.13.1 Copyright (c) 1988, 1997 Borland International + Display of File OWLWI.LIB + + 000000 MSLIBR Index begins at file offset 7ee00. Index is 521 blocks. + Library Page size is 32 bytes. + + 000020 THEADR TAppDictionary::TAppDictionary() + 00003E COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: TAppDictionary::TAppDictionary() + Module Name: Owl252 + ordinal: 2 + 000069 MODEND + """; + + var symbols = parser.parse(content); + + assertEquals(1, symbols.size()); + var sym = symbols.get(0); + assertEquals("Owl252", sym.moduleName()); + assertEquals("TAppDictionary", sym.className()); + assertEquals("TAppDictionary::TAppDictionary()", sym.symbolName()); + assertEquals(2, sym.ordinal()); + assertEquals("CONSTRUCTOR", sym.symbolType()); + assertEquals("Core/Application", sym.category()); + } + + @Test + void shouldParseDestructor() { + var content = + """ + Turbo Dump Version 5.0 + Display of File OWLWI.LIB + 000000 MSLIBR Index begins at file offset 7ee00. + 000080 THEADR TAppDictionary::~TAppDictionary() + 00009E COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: TAppDictionary::~TAppDictionary() + Module Name: Owl252 + ordinal: 3 + 0000C9 MODEND + """; + + var symbols = parser.parse(content); + + assertEquals(1, symbols.size()); + assertEquals("DESTRUCTOR", symbols.get(0).symbolType()); + assertEquals("TAppDictionary", symbols.get(0).className()); + assertEquals(3, symbols.get(0).ordinal()); + } + + @Test + void shouldParseMethodWithParameters() { + var content = + """ + Turbo Dump Version 5.0 + Display of File OWLWI.LIB + 000000 MSLIBR Index begins at file offset 7ee00. + 0000E0 THEADR TDC::SelectObject(const TPen*) + 0000F0 COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: TDC::SelectObject(const TPen*) + Module Name: Owl252 + ordinal: 178 + 000120 MODEND + """; + + var symbols = parser.parse(content); + + assertEquals(1, symbols.size()); + assertEquals("METHOD", symbols.get(0).symbolType()); + assertEquals("TDC", symbols.get(0).className()); + assertEquals("const TPen*", symbols.get(0).parameters()); + assertEquals(178, symbols.get(0).ordinal()); + assertEquals("Graphics/DC", symbols.get(0).category()); + } + + @Test + void shouldParseMethodWithoutParameters() { + var content = + """ + Turbo Dump Version 5.0 + Display of File OWLWI.LIB + 000000 MSLIBR Index begins at file offset 7ee00. + 0001E0 THEADR TBitSet::IsEmpty() + 0001F0 COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: TBitSet::IsEmpty() + Module Name: Owl252 + ordinal: 47 + 000220 MODEND + """; + + var symbols = parser.parse(content); + + assertEquals(1, symbols.size()); + assertEquals("METHOD", symbols.get(0).symbolType()); + assertNull(symbols.get(0).parameters()); + assertEquals("TBitSet", symbols.get(0).className()); + assertEquals("Core/DataTypes", symbols.get(0).category()); + } + + @Test + void shouldParseFreeFunction() { + var content = + """ + Turbo Dump Version 5.0 + Display of File OWLWI.LIB + 000000 MSLIBR Index begins at file offset 7ee00. + 000360 THEADR NColors(unsigned short) + 000370 COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: NColors(unsigned short) + Module Name: Owl252 + ordinal: 123 + 000390 MODEND + """; + + var symbols = parser.parse(content); + + assertEquals(1, symbols.size()); + assertEquals("FUNCTION", symbols.get(0).symbolType()); + assertNull(symbols.get(0).className()); + assertEquals("unsigned short", symbols.get(0).parameters()); + assertEquals("Graphics/Utility", symbols.get(0).category()); + } + + @Test + void shouldParseMultipleEntries() { + var content = + """ + Turbo Dump Version 5.0 + Display of File OWLWI.LIB + 000000 MSLIBR Index begins at file offset 7ee00. + + 000020 THEADR TAppDictionary::TAppDictionary() + 00003E COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: TAppDictionary::TAppDictionary() + Module Name: Owl252 + ordinal: 2 + 000069 MODEND + + =============== + + 000080 THEADR TBitmap::Width() const + 00009E COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: TBitmap::Width() const + Module Name: Owl252 + ordinal: 23 + 0000C9 MODEND + + =============== + + 0000E0 THEADR TButtonGadget::Paint(TDC*) + 0000FE COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: TButtonGadget::Paint(TDC*) + Module Name: Owl252 + ordinal: 75 + 00012D MODEND + """; + + var symbols = parser.parse(content); + + assertEquals(3, symbols.size()); + assertEquals("CONSTRUCTOR", symbols.get(0).symbolType()); + assertEquals("METHOD", symbols.get(1).symbolType()); + assertEquals("METHOD", symbols.get(2).symbolType()); + assertEquals(75, symbols.get(2).ordinal()); + assertEquals("TButtonGadget", symbols.get(2).className()); + assertEquals("TDC*", symbols.get(2).parameters()); + } + + @Test + void shouldIgnoreNonImportedEntries() { + var content = + """ + Turbo Dump Version 5.0 + Display of File OWLWI.LIB + 000000 MSLIBR Index begins at file offset 7ee00. + + 000020 THEADR SomeDataVariable + 000030 COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Not a dynamic link import + 000050 MODEND + + =============== + + 000060 THEADR TBrush::TBrush(TColor) + 00007E COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: TBrush::TBrush(TColor) + Module Name: Owl252 + ordinal: 53 + 0000B0 MODEND + """; + + var symbols = parser.parse(content); + + assertEquals(1, symbols.size()); + assertEquals(53, symbols.get(0).ordinal()); + } + + @Test + void shouldReturnEmptyForBlankInput() { + assertTrue(parser.parse("").isEmpty()); + assertTrue(parser.parse(null).isEmpty()); + } + + @Test + void shouldParseClassPrefixCategoryTFrameWindow() { + var content = + """ + Turbo Dump Version 5.0 + Display of File OWLWI.LIB + 000000 MSLIBR Index begins at file offset 7ee00. + 000020 THEADR TFrameWindow::TFrameWindow() + 00003E COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: TFrameWindow::TFrameWindow() + Module Name: Owl252 + ordinal: 200 + 000069 MODEND + """; + + var symbols = parser.parse(content); + + assertEquals("UI/Window", symbols.get(0).category()); + } + + @Test + void shouldParseClassPrefixCategoryTDialog() { + var content = + """ + Turbo Dump Version 5.0 + Display of File OWLWI.LIB + 000000 MSLIBR Index begins at file offset 7ee00. + 000020 THEADR TChooseColorDialog::DoExecute() + 00003E COMENT Purge: Yes, List: Yes, Class: 160 (0A0h) + Dynamic link import (IMPDEF) + Imported by: ordinal + Internal Name: TChooseColorDialog::DoExecute() + Module Name: Owl252 + ordinal: 100 + 000069 MODEND + """; + + var symbols = parser.parse(content); + + assertEquals("UI/Dialog", symbols.get(0).category()); + } +} diff --git a/src/test/java/ai/decompile/workspace/service/BinaryServiceTest.java b/src/test/java/ai/decompile/workspace/service/BinaryServiceTest.java index 784c176..09a473b 100644 --- a/src/test/java/ai/decompile/workspace/service/BinaryServiceTest.java +++ b/src/test/java/ai/decompile/workspace/service/BinaryServiceTest.java @@ -10,7 +10,6 @@ import ai.decompile.workspace.model.dto.BinaryResponse; import ai.decompile.workspace.model.entity.Binary; import ai.decompile.workspace.model.entity.Project; import ai.decompile.workspace.model.entity.Workspace; -import ai.decompile.workspace.model.repository.BinaryDependencyRepository; import ai.decompile.workspace.model.repository.BinaryRepository; import ai.decompile.workspace.service.storage.FileStorage; import java.io.IOException; @@ -35,8 +34,6 @@ class BinaryServiceTest { @Mock private BinaryRepository binaryRepository; - @Mock private BinaryDependencyRepository dependencyRepository; - @Mock private FileStorage fileStorage; @Mock private ApplicationEventPublisher eventPublisher;