feat(analysis): add string extraction and API support
This commit is contained in:
parent
e01948b9e4
commit
036f392d61
@ -11,6 +11,7 @@ import idaapi
|
|||||||
import idautils
|
import idautils
|
||||||
|
|
||||||
OUTPUT = "output.json"
|
OUTPUT = "output.json"
|
||||||
|
DEBUG_LOG = "extract_debug.log"
|
||||||
|
|
||||||
# Cross-reference type constants (IDA SDK values)
|
# Cross-reference type constants (IDA SDK values)
|
||||||
FL_CF = 16 # Call Far
|
FL_CF = 16 # Call Far
|
||||||
@ -18,6 +19,34 @@ FL_CN = 17 # Call Near
|
|||||||
FL_JF = 18 # Jump Far
|
FL_JF = 18 # Jump Far
|
||||||
FL_JN = 19 # Jump Near
|
FL_JN = 19 # Jump Near
|
||||||
|
|
||||||
|
# String type constants (may not be in idc namespace in all IDA versions)
|
||||||
|
try:
|
||||||
|
_ASCSTR_C = idc.ASCSTR_C
|
||||||
|
_ASCSTR_UNICODE = idc.ASCSTR_UNICODE
|
||||||
|
_ASCSTR_ULEN2 = idc.ASCSTR_ULEN2
|
||||||
|
_ASCSTR_ULEN4 = idc.ASCSTR_ULEN4
|
||||||
|
_ASCSTR_PASCAL = idc.ASCSTR_PASCAL
|
||||||
|
_ASCSTR_LEN2 = idc.ASCSTR_LEN2
|
||||||
|
_ASCSTR_LEN4 = idc.ASCSTR_LEN4
|
||||||
|
except AttributeError:
|
||||||
|
_ASCSTR_C = 0
|
||||||
|
_ASCSTR_UNICODE = 4
|
||||||
|
_ASCSTR_ULEN2 = 5
|
||||||
|
_ASCSTR_ULEN4 = 6
|
||||||
|
_ASCSTR_PASCAL = 1
|
||||||
|
_ASCSTR_LEN2 = 2
|
||||||
|
_ASCSTR_LEN4 = 3
|
||||||
|
|
||||||
|
_STRING_TYPE_NAMES = {
|
||||||
|
_ASCSTR_C: "ASCSTR_C",
|
||||||
|
_ASCSTR_UNICODE: "ASCSTR_UNICODE",
|
||||||
|
_ASCSTR_ULEN2: "ASCSTR_ULEN2",
|
||||||
|
_ASCSTR_ULEN4: "ASCSTR_ULEN4",
|
||||||
|
_ASCSTR_PASCAL: "ASCSTR_PASCAL",
|
||||||
|
_ASCSTR_LEN2: "ASCSTR_LEN2",
|
||||||
|
_ASCSTR_LEN4: "ASCSTR_LEN4",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def msg(text):
|
def msg(text):
|
||||||
idc.Message(text)
|
idc.Message(text)
|
||||||
@ -25,6 +54,15 @@ def msg(text):
|
|||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def dbg(text):
|
||||||
|
try:
|
||||||
|
with open(DEBUG_LOG, "a") as df:
|
||||||
|
df.write(text)
|
||||||
|
df.flush()
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _json_string(s):
|
def _json_string(s):
|
||||||
"""Escape a string for JSON — returns the quoted representation.
|
"""Escape a string for JSON — returns the quoted representation.
|
||||||
Handles raw bytes in disassembly that aren't valid UTF-8."""
|
Handles raw bytes in disassembly that aren't valid UTF-8."""
|
||||||
@ -32,7 +70,7 @@ def _json_string(s):
|
|||||||
return json.dumps(s)
|
return json.dumps(s)
|
||||||
try:
|
try:
|
||||||
return json.dumps(s)
|
return json.dumps(s)
|
||||||
except UnicodeDecodeError:
|
except (UnicodeDecodeError, UnicodeEncodeError):
|
||||||
return json.dumps(s.decode('latin-1'))
|
return json.dumps(s.decode('latin-1'))
|
||||||
|
|
||||||
|
|
||||||
@ -150,6 +188,43 @@ def extract(f):
|
|||||||
f.write("\n ],\n")
|
f.write("\n ],\n")
|
||||||
f.flush()
|
f.flush()
|
||||||
|
|
||||||
|
# ---- strings ----
|
||||||
|
f.write(" \"strings\": [\n")
|
||||||
|
f.flush()
|
||||||
|
try:
|
||||||
|
strings = list(idautils.Strings())
|
||||||
|
except Exception as e:
|
||||||
|
dbg("idautils.Strings() failed: %s\n" % str(e))
|
||||||
|
strings = []
|
||||||
|
string_count = 0
|
||||||
|
for i, s in enumerate(strings):
|
||||||
|
try:
|
||||||
|
str_value = str(s)
|
||||||
|
if len(str_value) < 2:
|
||||||
|
continue
|
||||||
|
str_type_id = idc.GetStringType(s.ea)
|
||||||
|
if str_type_id is None:
|
||||||
|
str_type_id = -1
|
||||||
|
encoding = _STRING_TYPE_NAMES.get(str_type_id, "UNKNOWN")
|
||||||
|
ref_parts = []
|
||||||
|
for xref in idautils.XrefsTo(s.ea):
|
||||||
|
func_name = idc.GetFunctionName(xref.frm)
|
||||||
|
if func_name:
|
||||||
|
ref_parts.append(func_name)
|
||||||
|
referenced_by = " ".join(ref_parts)
|
||||||
|
if string_count > 0:
|
||||||
|
f.write(",\n")
|
||||||
|
f.write(
|
||||||
|
" {\"address\": \"%s\", \"value\": %s, \"encoding\": \"%s\", \"referencedBy\": %s}"
|
||||||
|
% (_json_hex(s.ea), _json_string(str_value), encoding, _json_string(referenced_by))
|
||||||
|
)
|
||||||
|
string_count += 1
|
||||||
|
except Exception as e:
|
||||||
|
dbg("String at %s failed: %s\n" % (_json_hex(s.ea), str(e)))
|
||||||
|
f.write("\n ],\n")
|
||||||
|
f.flush()
|
||||||
|
msg("[extract.py] Extraidas %d strings.\n" % string_count)
|
||||||
|
|
||||||
# ---- functions ----
|
# ---- functions ----
|
||||||
f.write(" \"functions\": [\n")
|
f.write(" \"functions\": [\n")
|
||||||
f.flush()
|
f.flush()
|
||||||
@ -212,6 +287,10 @@ def main():
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
msg("[extract.py] ERRO na extracao: %s\n" % str(e))
|
msg("[extract.py] ERRO na extracao: %s\n" % str(e))
|
||||||
msg(traceback.format_exc() + "\n")
|
msg(traceback.format_exc() + "\n")
|
||||||
|
try:
|
||||||
|
dbg("FATAL in extract(): %s\n%s\n" % (str(e), traceback.format_exc()))
|
||||||
|
except:
|
||||||
|
pass
|
||||||
finally:
|
finally:
|
||||||
f.close()
|
f.close()
|
||||||
|
|
||||||
|
|||||||
@ -3,9 +3,11 @@ package ai.decompile.ai.service;
|
|||||||
import ai.decompile.ai.model.entity.EmbeddingChunk;
|
import ai.decompile.ai.model.entity.EmbeddingChunk;
|
||||||
import ai.decompile.analysis.model.entity.StaticAnalysis;
|
import ai.decompile.analysis.model.entity.StaticAnalysis;
|
||||||
import ai.decompile.analysis.model.entity.StaticFunction;
|
import ai.decompile.analysis.model.entity.StaticFunction;
|
||||||
|
import ai.decompile.analysis.model.entity.StaticString;
|
||||||
import ai.decompile.analysis.model.entity.StaticXref;
|
import ai.decompile.analysis.model.entity.StaticXref;
|
||||||
import ai.decompile.analysis.model.repository.StaticLabelRepository;
|
import ai.decompile.analysis.model.repository.StaticLabelRepository;
|
||||||
import ai.decompile.analysis.model.repository.StaticSegmentRepository;
|
import ai.decompile.analysis.model.repository.StaticSegmentRepository;
|
||||||
|
import ai.decompile.analysis.model.repository.StaticStringRepository;
|
||||||
import ai.decompile.analysis.model.repository.StaticXrefRepository;
|
import ai.decompile.analysis.model.repository.StaticXrefRepository;
|
||||||
import ai.decompile.workspace.model.entity.Binary;
|
import ai.decompile.workspace.model.entity.Binary;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
@ -28,11 +30,14 @@ public class EmbeddingChunkBuilder {
|
|||||||
private final StaticXrefRepository xrefRepository;
|
private final StaticXrefRepository xrefRepository;
|
||||||
private final StaticLabelRepository labelRepository;
|
private final StaticLabelRepository labelRepository;
|
||||||
private final StaticSegmentRepository segmentRepository;
|
private final StaticSegmentRepository segmentRepository;
|
||||||
|
private final StaticStringRepository stringRepository;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
@Value("${app.ai.embedding.chunk-max-chars:3000}")
|
@Value("${app.ai.embedding.chunk-max-chars:3000}")
|
||||||
private int chunkMaxChars;
|
private int chunkMaxChars;
|
||||||
|
|
||||||
|
private static final int METADATA_TOP_STRINGS = 20;
|
||||||
|
|
||||||
public List<EmbeddingChunk> buildFunctionChunks(List<StaticFunction> functions, Binary binary) {
|
public List<EmbeddingChunk> buildFunctionChunks(List<StaticFunction> functions, Binary binary) {
|
||||||
var graphContext = loadGraphContext(functions);
|
var graphContext = loadGraphContext(functions);
|
||||||
|
|
||||||
@ -81,6 +86,19 @@ public class EmbeddingChunkBuilder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var strings = stringRepository.findByAnalysisIdOrderByAddressAsc(analysis.getId());
|
||||||
|
if (!strings.isEmpty()) {
|
||||||
|
sb.append("Top strings:\n");
|
||||||
|
strings.stream()
|
||||||
|
.limit(METADATA_TOP_STRINGS)
|
||||||
|
.forEach(
|
||||||
|
s ->
|
||||||
|
sb.append(
|
||||||
|
String.format(
|
||||||
|
" %s (%s): \"%s\"\n",
|
||||||
|
s.getAddress(), s.getEncoding(), AiUtils.truncate(s.getValue(), 200))));
|
||||||
|
}
|
||||||
|
|
||||||
return EmbeddingChunk.builder()
|
return EmbeddingChunk.builder()
|
||||||
.id(UUID.randomUUID())
|
.id(UUID.randomUUID())
|
||||||
.binaryId(binary.getId())
|
.binaryId(binary.getId())
|
||||||
@ -94,6 +112,44 @@ public class EmbeddingChunkBuilder {
|
|||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public List<EmbeddingChunk> buildStringChunks(List<StaticString> strings, Binary binary) {
|
||||||
|
var chunks = new ArrayList<EmbeddingChunk>();
|
||||||
|
for (var s : strings) {
|
||||||
|
var sb = new StringBuilder();
|
||||||
|
sb.append(String.format("[STRING] at %s (%s)\n", s.getAddress(), s.getEncoding()));
|
||||||
|
if (Objects.nonNull(s.getReferencedBy()) && !s.getReferencedBy().isBlank()) {
|
||||||
|
sb.append("Referenced by: ").append(s.getReferencedBy()).append("\n");
|
||||||
|
}
|
||||||
|
sb.append("\"").append(s.getValue()).append("\"");
|
||||||
|
var content = sb.toString();
|
||||||
|
chunks.add(
|
||||||
|
EmbeddingChunk.builder()
|
||||||
|
.id(UUID.randomUUID())
|
||||||
|
.binaryId(binary.getId())
|
||||||
|
.chunkType("STRING")
|
||||||
|
.sourceType("STATIC_STRING")
|
||||||
|
.sourceId(s.getId())
|
||||||
|
.content(content)
|
||||||
|
.metadata(
|
||||||
|
AiUtils.toJson(
|
||||||
|
objectMapper,
|
||||||
|
Map.of(
|
||||||
|
"address",
|
||||||
|
s.getAddress(),
|
||||||
|
"encoding",
|
||||||
|
s.getEncoding(),
|
||||||
|
"length",
|
||||||
|
s.getValue().length(),
|
||||||
|
"referenced_by",
|
||||||
|
Objects.nonNull(s.getReferencedBy()) ? s.getReferencedBy() : "")))
|
||||||
|
.tokenCount(AiUtils.estimateTokens(content))
|
||||||
|
.createdAt(java.time.Instant.now())
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
|
||||||
|
return chunks;
|
||||||
|
}
|
||||||
|
|
||||||
private FunctionGraphContext loadGraphContext(List<StaticFunction> functions) {
|
private FunctionGraphContext loadGraphContext(List<StaticFunction> functions) {
|
||||||
var nameToCallers =
|
var nameToCallers =
|
||||||
loadXrefNames(functions, xrefRepository::findByCalleeId, xr -> xr.getCaller().getName());
|
loadXrefNames(functions, xrefRepository::findByCalleeId, xr -> xr.getCaller().getName());
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import ai.decompile.ai.model.entity.EmbeddingChunk;
|
|||||||
import ai.decompile.ai.model.repository.EmbeddingChunkRepository;
|
import ai.decompile.ai.model.repository.EmbeddingChunkRepository;
|
||||||
import ai.decompile.analysis.model.entity.StaticAnalysis;
|
import ai.decompile.analysis.model.entity.StaticAnalysis;
|
||||||
import ai.decompile.analysis.model.repository.StaticAnalysisRepository;
|
import ai.decompile.analysis.model.repository.StaticAnalysisRepository;
|
||||||
|
import ai.decompile.analysis.model.repository.StaticStringRepository;
|
||||||
import ai.decompile.workspace.service.BinaryService;
|
import ai.decompile.workspace.service.BinaryService;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@ -25,6 +26,7 @@ public class EmbeddingService {
|
|||||||
private final EmbeddingChunkBuilder chunkBuilder;
|
private final EmbeddingChunkBuilder chunkBuilder;
|
||||||
private final EmbeddingChunkRepository embeddingChunkRepository;
|
private final EmbeddingChunkRepository embeddingChunkRepository;
|
||||||
private final StaticAnalysisRepository analysisRepository;
|
private final StaticAnalysisRepository analysisRepository;
|
||||||
|
private final StaticStringRepository stringRepository;
|
||||||
private final BinaryService binaryService;
|
private final BinaryService binaryService;
|
||||||
|
|
||||||
@Value("${app.ai.embedding.batch-size:20}")
|
@Value("${app.ai.embedding.batch-size:20}")
|
||||||
@ -53,6 +55,12 @@ public class EmbeddingService {
|
|||||||
var functionChunks = chunkBuilder.buildFunctionChunks(functions, binary);
|
var functionChunks = chunkBuilder.buildFunctionChunks(functions, binary);
|
||||||
var allChunks = new ArrayList<>(functionChunks);
|
var allChunks = new ArrayList<>(functionChunks);
|
||||||
allChunks.add(chunkBuilder.buildMetadataChunk(binary, analysis, functions.size()));
|
allChunks.add(chunkBuilder.buildMetadataChunk(binary, analysis, functions.size()));
|
||||||
|
|
||||||
|
var strings = stringRepository.findByAnalysisIdOrderByAddressAsc(analysis.getId());
|
||||||
|
if (!strings.isEmpty()) {
|
||||||
|
allChunks.addAll(chunkBuilder.buildStringChunks(strings, binary));
|
||||||
|
}
|
||||||
|
|
||||||
embedAndPersistChunks(allChunks);
|
embedAndPersistChunks(allChunks);
|
||||||
|
|
||||||
log.info("Embedding complete for binary {}: {} chunks", binaryId, allChunks.size());
|
log.info("Embedding complete for binary {}: {} chunks", binaryId, allChunks.size());
|
||||||
|
|||||||
@ -3,6 +3,7 @@ package ai.decompile.analysis.controller;
|
|||||||
import ai.decompile.analysis.model.dto.AnalysisResponse;
|
import ai.decompile.analysis.model.dto.AnalysisResponse;
|
||||||
import ai.decompile.analysis.model.dto.FunctionSummaryResponse;
|
import ai.decompile.analysis.model.dto.FunctionSummaryResponse;
|
||||||
import ai.decompile.analysis.model.dto.SegmentResponse;
|
import ai.decompile.analysis.model.dto.SegmentResponse;
|
||||||
|
import ai.decompile.analysis.model.dto.StringResponse;
|
||||||
import ai.decompile.analysis.service.AnalysisService;
|
import ai.decompile.analysis.service.AnalysisService;
|
||||||
import io.swagger.v3.oas.annotations.Operation;
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@ -58,4 +59,14 @@ public class AnalysisController {
|
|||||||
public List<SegmentResponse> getSegments(@PathVariable UUID analysisId) {
|
public List<SegmentResponse> getSegments(@PathVariable UUID analysisId) {
|
||||||
return analysisService.getSegments(analysisId);
|
return analysisService.getSegments(analysisId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "List strings in an analysis",
|
||||||
|
description = "Retrieve all strings extracted during a static analysis.",
|
||||||
|
tags = {"Analysis"},
|
||||||
|
operationId = "listStrings")
|
||||||
|
@GetMapping("/analysis/{analysisId}/strings")
|
||||||
|
public List<StringResponse> getStrings(@PathVariable UUID analysisId) {
|
||||||
|
return analysisService.getStrings(analysisId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,12 @@
|
|||||||
|
package ai.decompile.analysis.model.dto;
|
||||||
|
|
||||||
|
import ai.decompile.analysis.model.entity.StaticString;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
public record StringResponse(
|
||||||
|
UUID id, String address, String value, String encoding, String referencedBy) {
|
||||||
|
public static StringResponse from(StaticString s) {
|
||||||
|
return new StringResponse(
|
||||||
|
s.getId(), s.getAddress(), s.getValue(), s.getEncoding(), s.getReferencedBy());
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,35 @@
|
|||||||
|
package ai.decompile.analysis.model.entity;
|
||||||
|
|
||||||
|
import jakarta.persistence.*;
|
||||||
|
import java.util.UUID;
|
||||||
|
import lombok.*;
|
||||||
|
|
||||||
|
@Entity
|
||||||
|
@Table(name = "static_string")
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
@NoArgsConstructor
|
||||||
|
@AllArgsConstructor
|
||||||
|
@Builder
|
||||||
|
public class StaticString {
|
||||||
|
@Id
|
||||||
|
@GeneratedValue(strategy = GenerationType.UUID)
|
||||||
|
private UUID id;
|
||||||
|
|
||||||
|
@ManyToOne(fetch = FetchType.LAZY)
|
||||||
|
@JoinColumn(name = "analysis_id", nullable = false)
|
||||||
|
private StaticAnalysis analysis;
|
||||||
|
|
||||||
|
@Column(nullable = false, length = 20)
|
||||||
|
private String address;
|
||||||
|
|
||||||
|
@Column(nullable = false, columnDefinition = "TEXT")
|
||||||
|
private String value;
|
||||||
|
|
||||||
|
@Column(nullable = false, length = 20)
|
||||||
|
private String encoding;
|
||||||
|
|
||||||
|
@Column(name = "referenced_by", nullable = false, columnDefinition = "TEXT DEFAULT ''")
|
||||||
|
@Builder.Default
|
||||||
|
private String referencedBy = "";
|
||||||
|
}
|
||||||
@ -0,0 +1,10 @@
|
|||||||
|
package ai.decompile.analysis.model.repository;
|
||||||
|
|
||||||
|
import ai.decompile.analysis.model.entity.StaticString;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.UUID;
|
||||||
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
|
|
||||||
|
public interface StaticStringRepository extends JpaRepository<StaticString, UUID> {
|
||||||
|
List<StaticString> findByAnalysisIdOrderByAddressAsc(UUID analysisId);
|
||||||
|
}
|
||||||
@ -5,16 +5,19 @@ import ai.decompile.analysis.model.dto.FunctionDetailResponse;
|
|||||||
import ai.decompile.analysis.model.dto.FunctionSummaryResponse;
|
import ai.decompile.analysis.model.dto.FunctionSummaryResponse;
|
||||||
import ai.decompile.analysis.model.dto.LabelResponse;
|
import ai.decompile.analysis.model.dto.LabelResponse;
|
||||||
import ai.decompile.analysis.model.dto.SegmentResponse;
|
import ai.decompile.analysis.model.dto.SegmentResponse;
|
||||||
|
import ai.decompile.analysis.model.dto.StringResponse;
|
||||||
import ai.decompile.analysis.model.dto.XrefResponse;
|
import ai.decompile.analysis.model.dto.XrefResponse;
|
||||||
import ai.decompile.analysis.model.entity.StaticAnalysis;
|
import ai.decompile.analysis.model.entity.StaticAnalysis;
|
||||||
import ai.decompile.analysis.model.entity.StaticFunction;
|
import ai.decompile.analysis.model.entity.StaticFunction;
|
||||||
import ai.decompile.analysis.model.entity.StaticLabel;
|
import ai.decompile.analysis.model.entity.StaticLabel;
|
||||||
import ai.decompile.analysis.model.entity.StaticSegment;
|
import ai.decompile.analysis.model.entity.StaticSegment;
|
||||||
|
import ai.decompile.analysis.model.entity.StaticString;
|
||||||
import ai.decompile.analysis.model.entity.StaticXref;
|
import ai.decompile.analysis.model.entity.StaticXref;
|
||||||
import ai.decompile.analysis.model.repository.StaticAnalysisRepository;
|
import ai.decompile.analysis.model.repository.StaticAnalysisRepository;
|
||||||
import ai.decompile.analysis.model.repository.StaticFunctionRepository;
|
import ai.decompile.analysis.model.repository.StaticFunctionRepository;
|
||||||
import ai.decompile.analysis.model.repository.StaticLabelRepository;
|
import ai.decompile.analysis.model.repository.StaticLabelRepository;
|
||||||
import ai.decompile.analysis.model.repository.StaticSegmentRepository;
|
import ai.decompile.analysis.model.repository.StaticSegmentRepository;
|
||||||
|
import ai.decompile.analysis.model.repository.StaticStringRepository;
|
||||||
import ai.decompile.analysis.model.repository.StaticXrefRepository;
|
import ai.decompile.analysis.model.repository.StaticXrefRepository;
|
||||||
import ai.decompile.common.exception.NotFoundException;
|
import ai.decompile.common.exception.NotFoundException;
|
||||||
import ai.decompile.engine.model.AnalysisResult;
|
import ai.decompile.engine.model.AnalysisResult;
|
||||||
@ -41,6 +44,7 @@ public class AnalysisService {
|
|||||||
private final StaticXrefRepository xrefRepository;
|
private final StaticXrefRepository xrefRepository;
|
||||||
private final StaticLabelRepository labelRepository;
|
private final StaticLabelRepository labelRepository;
|
||||||
private final StaticSegmentRepository segmentRepository;
|
private final StaticSegmentRepository segmentRepository;
|
||||||
|
private final StaticStringRepository stringRepository;
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public AnalysisResponse saveResult(
|
public AnalysisResponse saveResult(
|
||||||
@ -49,6 +53,7 @@ public class AnalysisService {
|
|||||||
|
|
||||||
var analysis = createAnalysis(binary, jobId, engine, result);
|
var analysis = createAnalysis(binary, jobId, engine, result);
|
||||||
saveSegments(analysis, result.segments());
|
saveSegments(analysis, result.segments());
|
||||||
|
saveStrings(analysis, result.strings());
|
||||||
var nameToId = saveFunctions(analysis, result);
|
var nameToId = saveFunctions(analysis, result);
|
||||||
saveXrefs(result, nameToId);
|
saveXrefs(result, nameToId);
|
||||||
|
|
||||||
@ -96,6 +101,13 @@ public class AnalysisService {
|
|||||||
.toList();
|
.toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public List<StringResponse> getStrings(UUID analysisId) {
|
||||||
|
assertAnalysisExists(analysisId);
|
||||||
|
return stringRepository.findByAnalysisIdOrderByAddressAsc(analysisId).stream()
|
||||||
|
.map(StringResponse::from)
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
|
||||||
private void replaceExistingAnalysis(Binary binary, String engine) {
|
private void replaceExistingAnalysis(Binary binary, String engine) {
|
||||||
analysisRepository
|
analysisRepository
|
||||||
.findByBinaryIdAndEngine(binary.getId(), engine)
|
.findByBinaryIdAndEngine(binary.getId(), engine)
|
||||||
@ -123,6 +135,20 @@ public class AnalysisService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void saveStrings(
|
||||||
|
StaticAnalysis analysis, List<ai.decompile.engine.model.StringEntry> entries) {
|
||||||
|
for (var entry : entries) {
|
||||||
|
stringRepository.save(
|
||||||
|
StaticString.builder()
|
||||||
|
.analysis(analysis)
|
||||||
|
.address(entry.address())
|
||||||
|
.value(entry.value())
|
||||||
|
.encoding(entry.encoding())
|
||||||
|
.referencedBy(entry.referencedBy())
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private StaticAnalysis createAnalysis(
|
private StaticAnalysis createAnalysis(
|
||||||
Binary binary, UUID jobId, String engine, AnalysisResult result) {
|
Binary binary, UUID jobId, String engine, AnalysisResult result) {
|
||||||
return analysisRepository.save(
|
return analysisRepository.save(
|
||||||
|
|||||||
@ -3,4 +3,13 @@ package ai.decompile.engine.model;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
public record AnalysisResult(
|
public record AnalysisResult(
|
||||||
String entryPoint, List<SegmentEntry> segments, List<FunctionResult> functions) {}
|
String entryPoint,
|
||||||
|
List<SegmentEntry> segments,
|
||||||
|
List<FunctionResult> functions,
|
||||||
|
List<StringEntry> strings) {
|
||||||
|
|
||||||
|
public AnalysisResult(
|
||||||
|
String entryPoint, List<SegmentEntry> segments, List<FunctionResult> functions) {
|
||||||
|
this(entryPoint, segments, functions, List.of());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
3
src/main/java/ai/decompile/engine/model/StringEntry.java
Normal file
3
src/main/java/ai/decompile/engine/model/StringEntry.java
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
package ai.decompile.engine.model;
|
||||||
|
|
||||||
|
public record StringEntry(String address, String value, String encoding, String referencedBy) {}
|
||||||
@ -6,6 +6,7 @@ import ai.decompile.engine.model.AnalysisResult;
|
|||||||
import ai.decompile.engine.model.FunctionResult;
|
import ai.decompile.engine.model.FunctionResult;
|
||||||
import ai.decompile.engine.model.LabelEntry;
|
import ai.decompile.engine.model.LabelEntry;
|
||||||
import ai.decompile.engine.model.SegmentEntry;
|
import ai.decompile.engine.model.SegmentEntry;
|
||||||
|
import ai.decompile.engine.model.StringEntry;
|
||||||
import ai.decompile.engine.model.XrefEntry;
|
import ai.decompile.engine.model.XrefEntry;
|
||||||
import com.github.dockerjava.api.model.Bind;
|
import com.github.dockerjava.api.model.Bind;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
@ -36,6 +37,11 @@ public abstract class AbstractIdaEngineService implements EngineService {
|
|||||||
private static final String KEY_SEGMENT_NAME = "\"name\": \"";
|
private static final String KEY_SEGMENT_NAME = "\"name\": \"";
|
||||||
private static final String KEY_SEGMENT_START = "\", \"start\": \"";
|
private static final String KEY_SEGMENT_START = "\", \"start\": \"";
|
||||||
private static final String KEY_SEGMENT_END = "\", \"end\": \"";
|
private static final String KEY_SEGMENT_END = "\", \"end\": \"";
|
||||||
|
private static final String KEY_STRINGS_START = "\"strings\": [";
|
||||||
|
private static final String KEY_STRING_ADDRESS = "\"address\": \"";
|
||||||
|
private static final String KEY_STRING_VALUE = "\", \"value\": \"";
|
||||||
|
private static final String KEY_STRING_ENCODING = "\", \"encoding\": \"";
|
||||||
|
private static final String KEY_STRING_REFERENCED_BY = ", \"referencedBy\": \"";
|
||||||
|
|
||||||
protected final ContainerService containers;
|
protected final ContainerService containers;
|
||||||
protected final EngineProperties engineProperties;
|
protected final EngineProperties engineProperties;
|
||||||
@ -144,6 +150,7 @@ public abstract class AbstractIdaEngineService implements EngineService {
|
|||||||
|
|
||||||
var entryPoint = parseEntryPoint(text);
|
var entryPoint = parseEntryPoint(text);
|
||||||
var segments = parseSegments(text);
|
var segments = parseSegments(text);
|
||||||
|
var strings = parseStrings(text);
|
||||||
|
|
||||||
var functions = new ArrayList<FunctionResult>();
|
var functions = new ArrayList<FunctionResult>();
|
||||||
|
|
||||||
@ -161,7 +168,7 @@ public abstract class AbstractIdaEngineService implements EngineService {
|
|||||||
raw.length() > 500 ? raw.substring(0, 500) : raw);
|
raw.length() > 500 ? raw.substring(0, 500) : raw);
|
||||||
}
|
}
|
||||||
|
|
||||||
return new AnalysisResult(entryPoint, segments, functions);
|
return new AnalysisResult(entryPoint, segments, functions, strings);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String parseEntryPoint(String text) {
|
private static String parseEntryPoint(String text) {
|
||||||
@ -333,6 +340,80 @@ public abstract class AbstractIdaEngineService implements EngineService {
|
|||||||
return new SegmentEntry(name, startAddress, endAddress);
|
return new SegmentEntry(name, startAddress, endAddress);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static List<StringEntry> parseStrings(String text) {
|
||||||
|
var strings = new ArrayList<StringEntry>();
|
||||||
|
var remainder = after(text, KEY_STRINGS_START);
|
||||||
|
if (Objects.isNull(remainder)) {
|
||||||
|
return strings;
|
||||||
|
}
|
||||||
|
|
||||||
|
var stringsBlock = extractField(remainder, "\n ],\n");
|
||||||
|
if (Objects.isNull(stringsBlock)) {
|
||||||
|
return strings;
|
||||||
|
}
|
||||||
|
|
||||||
|
var parts = stringsBlock.split("},\\s*\\{");
|
||||||
|
for (var part : parts) {
|
||||||
|
var string = parseStringEntry(part);
|
||||||
|
if (Objects.nonNull(string)) {
|
||||||
|
strings.add(string);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static StringEntry parseStringEntry(String part) {
|
||||||
|
var clean = part.replace("{", "").replace("}", "").trim();
|
||||||
|
if (clean.isBlank()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var addressRemainder = after(clean, KEY_STRING_ADDRESS);
|
||||||
|
if (Objects.isNull(addressRemainder)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var address = extractField(addressRemainder, KEY_STRING_VALUE);
|
||||||
|
if (Objects.isNull(address)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var valueRemainder = after(addressRemainder, KEY_STRING_VALUE);
|
||||||
|
if (Objects.isNull(valueRemainder)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var value = extractField(valueRemainder, KEY_STRING_ENCODING);
|
||||||
|
if (Objects.isNull(value)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var encodingRemainder = after(valueRemainder, KEY_STRING_ENCODING);
|
||||||
|
if (Objects.isNull(encodingRemainder)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var encoding = extractField(encodingRemainder, "\"");
|
||||||
|
if (Objects.isNull(encoding)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var afterEncoding = after(encodingRemainder, "\"");
|
||||||
|
var referencedBy = "";
|
||||||
|
if (Objects.nonNull(afterEncoding)) {
|
||||||
|
var refRemainder = after(afterEncoding, KEY_STRING_REFERENCED_BY);
|
||||||
|
if (Objects.nonNull(refRemainder)) {
|
||||||
|
var ref = extractField(refRemainder, "\"");
|
||||||
|
if (Objects.nonNull(ref)) {
|
||||||
|
referencedBy = ref;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return new StringEntry(address, value, encoding, referencedBy);
|
||||||
|
}
|
||||||
|
|
||||||
private static List<XrefEntry> parseXrefs(String raw) {
|
private static List<XrefEntry> parseXrefs(String raw) {
|
||||||
if (Objects.isNull(raw) || raw.isBlank()) {
|
if (Objects.isNull(raw) || raw.isBlank()) {
|
||||||
return List.of();
|
return List.of();
|
||||||
|
|||||||
@ -0,0 +1,12 @@
|
|||||||
|
CREATE TABLE IF NOT EXISTS static_string
|
||||||
|
(
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
analysis_id UUID NOT NULL REFERENCES static_analysis (id) ON DELETE CASCADE,
|
||||||
|
address VARCHAR(20) NOT NULL,
|
||||||
|
value TEXT NOT NULL,
|
||||||
|
encoding VARCHAR(20) NOT NULL,
|
||||||
|
referenced_by TEXT NOT NULL DEFAULT ''
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_static_string_analysis_id ON static_string (analysis_id);
|
||||||
|
|
||||||
@ -0,0 +1,157 @@
|
|||||||
|
package ai.decompile.engine.service;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
|
||||||
|
import ai.decompile.engine.model.AnalysisResult;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
class AbstractIdaEngineServiceTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void shouldParseStringsWhenPresent() {
|
||||||
|
var json =
|
||||||
|
"""
|
||||||
|
{
|
||||||
|
"entry_point": "0x00401000:start",
|
||||||
|
"segments": [],
|
||||||
|
"strings": [
|
||||||
|
{"address": "0x00403000", "value": "Hello World", "encoding": "ASCSTR_C", "referencedBy": "sub_1000 sub_2500"},
|
||||||
|
{"address": "0x00403010", "value": "path/to/file", "encoding": "ASCSTR_C", "referencedBy": ""},
|
||||||
|
{"address": "0x00403020", "value": "test", "encoding": "UNKNOWN", "referencedBy": "sub_3000"}
|
||||||
|
],
|
||||||
|
"functions": []
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
|
||||||
|
var result = invokeParseOutput(json);
|
||||||
|
|
||||||
|
assertNotNull(result);
|
||||||
|
assertEquals(3, result.strings().size());
|
||||||
|
assertEquals("0x00403000", result.strings().get(0).address());
|
||||||
|
assertEquals("Hello World", result.strings().get(0).value());
|
||||||
|
assertEquals("ASCSTR_C", result.strings().get(0).encoding());
|
||||||
|
assertEquals("sub_1000 sub_2500", result.strings().get(0).referencedBy());
|
||||||
|
assertEquals("0x00403010", result.strings().get(1).address());
|
||||||
|
assertEquals("path/to/file", result.strings().get(1).value());
|
||||||
|
assertEquals("", result.strings().get(1).referencedBy());
|
||||||
|
assertEquals("0x00403020", result.strings().get(2).address());
|
||||||
|
assertEquals("test", result.strings().get(2).value());
|
||||||
|
assertEquals("UNKNOWN", result.strings().get(2).encoding());
|
||||||
|
assertEquals("sub_3000", result.strings().get(2).referencedBy());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void shouldReturnEmptyStringsWhenBlockAbsent() {
|
||||||
|
var json =
|
||||||
|
"""
|
||||||
|
{
|
||||||
|
"entry_point": "0x00401000:start",
|
||||||
|
"segments": [],
|
||||||
|
"functions": []
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
|
||||||
|
var result = invokeParseOutput(json);
|
||||||
|
|
||||||
|
assertNotNull(result);
|
||||||
|
assertTrue(result.strings().isEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void shouldReturnEmptyStringsWhenBlockEmpty() {
|
||||||
|
var json =
|
||||||
|
"""
|
||||||
|
{
|
||||||
|
"entry_point": "0x00401000:start",
|
||||||
|
"segments": [],
|
||||||
|
"strings": [],
|
||||||
|
"functions": []
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
|
||||||
|
var result = invokeParseOutput(json);
|
||||||
|
|
||||||
|
assertNotNull(result);
|
||||||
|
assertTrue(result.strings().isEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void shouldHandleSegmentsAndStringsTogether() {
|
||||||
|
var json =
|
||||||
|
"""
|
||||||
|
{
|
||||||
|
"entry_point": "0x00401000:start",
|
||||||
|
"segments": [
|
||||||
|
{"name": ".text", "start": "0x00401000", "end": "0x00408000"}
|
||||||
|
],
|
||||||
|
"strings": [
|
||||||
|
{"address": "0x0040A000", "value": "error", "encoding": "ASCSTR_C", "referencedBy": ""}
|
||||||
|
],
|
||||||
|
"functions": []
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
|
||||||
|
var result = invokeParseOutput(json);
|
||||||
|
|
||||||
|
assertNotNull(result);
|
||||||
|
assertEquals(1, result.segments().size());
|
||||||
|
assertEquals(1, result.strings().size());
|
||||||
|
assertEquals("error", result.strings().get(0).value());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void shouldDefaultReferencedByToEmptyWhenFieldMissing() {
|
||||||
|
var json =
|
||||||
|
"""
|
||||||
|
{
|
||||||
|
"entry_point": "0x00401000:start",
|
||||||
|
"segments": [],
|
||||||
|
"strings": [
|
||||||
|
{"address": "0x00403000", "value": "legacy", "encoding": "ASCSTR_C"}
|
||||||
|
],
|
||||||
|
"functions": []
|
||||||
|
}
|
||||||
|
""";
|
||||||
|
|
||||||
|
var result = invokeParseOutput(json);
|
||||||
|
|
||||||
|
assertNotNull(result);
|
||||||
|
assertEquals(1, result.strings().size());
|
||||||
|
assertEquals("legacy", result.strings().get(0).value());
|
||||||
|
assertEquals("", result.strings().get(0).referencedBy());
|
||||||
|
}
|
||||||
|
|
||||||
|
private AnalysisResult invokeParseOutput(String json) {
|
||||||
|
try {
|
||||||
|
var method = AbstractIdaEngineService.class.getDeclaredMethod("parseOutput", String.class);
|
||||||
|
method.setAccessible(true);
|
||||||
|
|
||||||
|
var dummyService =
|
||||||
|
new AbstractIdaEngineService(null, null) {
|
||||||
|
@Override
|
||||||
|
protected String getDockerfilePath() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String getConfigKey() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String getTempDirPrefix() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ai.decompile.engine.model.EngineType supportedEngine() {
|
||||||
|
return ai.decompile.engine.model.EngineType.IDA66;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (AnalysisResult) method.invoke(dummyService, json);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user