feat: rename maps for labels/structs/globals, reindex triggers, comment validation
This commit is contained in:
parent
75275f992d
commit
0c54602fcf
@ -39,7 +39,7 @@ public class EmbeddingChunk {
|
|||||||
private String content;
|
private String content;
|
||||||
|
|
||||||
@JdbcTypeCode(SqlTypes.VECTOR)
|
@JdbcTypeCode(SqlTypes.VECTOR)
|
||||||
@Column(name = "embedding", columnDefinition = "vector(4096)")
|
@Column(name = "embedding", columnDefinition = "vector(768)")
|
||||||
private float[] embedding;
|
private float[] embedding;
|
||||||
|
|
||||||
@JdbcTypeCode(SqlTypes.JSON)
|
@JdbcTypeCode(SqlTypes.JSON)
|
||||||
|
|||||||
@ -51,6 +51,8 @@ public interface EmbeddingChunkRepository extends JpaRepository<EmbeddingChunk,
|
|||||||
|
|
||||||
void deleteByBinaryId(UUID binaryId);
|
void deleteByBinaryId(UUID binaryId);
|
||||||
|
|
||||||
|
void deleteByBinaryIdAndChunkType(UUID binaryId, String chunkType);
|
||||||
|
|
||||||
@Modifying
|
@Modifying
|
||||||
@Transactional
|
@Transactional
|
||||||
@Query("DELETE FROM EmbeddingChunk e WHERE e.sourceId = :sourceId AND e.sourceType = :sourceType")
|
@Query("DELETE FROM EmbeddingChunk e WHERE e.sourceId = :sourceId AND e.sourceType = :sourceType")
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
package ai.decompile.ai.service;
|
package ai.decompile.ai.service;
|
||||||
|
|
||||||
import ai.decompile.ai.model.entity.EmbeddingChunk;
|
import ai.decompile.ai.model.entity.EmbeddingChunk;
|
||||||
import ai.decompile.analysis.model.entity.MutationSuggestion;
|
|
||||||
import ai.decompile.analysis.model.entity.StaticAnalysis;
|
import ai.decompile.analysis.model.entity.StaticAnalysis;
|
||||||
import ai.decompile.analysis.model.entity.StaticDataLabel;
|
import ai.decompile.analysis.model.entity.StaticDataLabel;
|
||||||
import ai.decompile.analysis.model.entity.StaticEnum;
|
import ai.decompile.analysis.model.entity.StaticEnum;
|
||||||
@ -87,10 +86,31 @@ public class EmbeddingChunkBuilder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var analysisId = functions.getFirst().getAnalysis().getId();
|
var analysisId = functions.getFirst().getAnalysis().getId();
|
||||||
return mutationRepository.findApprovedFunctionRenamesByAnalysisId(analysisId).stream()
|
var functionRenames = mutationRepository.findApprovedFunctionRenamesByAnalysisId(analysisId);
|
||||||
.collect(
|
var labelRenames = mutationRepository.findApprovedLabelRenamesByAnalysisId(analysisId);
|
||||||
java.util.stream.Collectors.toMap(
|
var globalVarRenames =
|
||||||
MutationSuggestion::getOldValue, MutationSuggestion::getNewValue, (_, b) -> b));
|
mutationRepository.findApprovedGlobalVarRenamesByAnalysisId(analysisId);
|
||||||
|
var structRenames = mutationRepository.findApprovedStructRenamesByAnalysisId(analysisId);
|
||||||
|
|
||||||
|
var map = new HashMap<String, String>();
|
||||||
|
|
||||||
|
for (var m : labelRenames) {
|
||||||
|
map.put(m.getOldValue(), m.getNewValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var m : globalVarRenames) {
|
||||||
|
map.put(m.getOldValue(), m.getNewValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var m : structRenames) {
|
||||||
|
map.put(m.getOldValue(), m.getNewValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var m : functionRenames) {
|
||||||
|
map.put(m.getOldValue(), m.getNewValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
return map;
|
||||||
}
|
}
|
||||||
|
|
||||||
public EmbeddingChunk buildMetadataChunk(
|
public EmbeddingChunk buildMetadataChunk(
|
||||||
|
|||||||
@ -133,6 +133,32 @@ public class EmbeddingService {
|
|||||||
binaryId);
|
binaryId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void reindexAllFunctions(UUID binaryId) {
|
||||||
|
var analysis = getLatestAnalysis(binaryId);
|
||||||
|
if (Objects.isNull(analysis)) {
|
||||||
|
log.warn("reindexAllFunctions: no analysis found for binary {}, skipping", binaryId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var functions = analysis.getFunctions();
|
||||||
|
if (functions.isEmpty()) {
|
||||||
|
log.debug("reindexAllFunctions: no functions for binary {}, nothing to do", binaryId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var binary = binaryService.getBinary(binaryId);
|
||||||
|
|
||||||
|
embeddingChunkRepository.deleteByBinaryIdAndChunkType(binaryId, "FUNCTION");
|
||||||
|
embeddingChunkRepository.flush();
|
||||||
|
|
||||||
|
var chunks = chunkBuilder.buildFunctionChunks(functions, binary);
|
||||||
|
embedAndPersistChunks(chunks);
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
"Re-indexed all {} function chunks for binary {}", functions.size(), binaryId);
|
||||||
|
}
|
||||||
|
|
||||||
private void embedAndPersistChunks(List<EmbeddingChunk> chunks) {
|
private void embedAndPersistChunks(List<EmbeddingChunk> chunks) {
|
||||||
for (var i = 0; i < chunks.size(); i += batchSize) {
|
for (var i = 0; i < chunks.size(); i += batchSize) {
|
||||||
var batch = chunks.subList(i, Math.min(i + batchSize, chunks.size()));
|
var batch = chunks.subList(i, Math.min(i + batchSize, chunks.size()));
|
||||||
|
|||||||
@ -39,4 +39,38 @@ public interface MutationSuggestionRepository extends JpaRepository<MutationSugg
|
|||||||
""")
|
""")
|
||||||
List<MutationSuggestion> findApprovedFunctionRenamesByAnalysisId(
|
List<MutationSuggestion> findApprovedFunctionRenamesByAnalysisId(
|
||||||
@Param("analysisId") UUID analysisId);
|
@Param("analysisId") UUID analysisId);
|
||||||
|
|
||||||
|
@Query(
|
||||||
|
"""
|
||||||
|
SELECT m FROM MutationSuggestion m
|
||||||
|
JOIN StaticLabel l ON l.id = m.targetId
|
||||||
|
JOIN StaticFunction f ON f.id = l.function.id
|
||||||
|
JOIN StaticAnalysis a ON a.id = f.analysis.id
|
||||||
|
WHERE m.targetType = 'LABEL' AND m.fieldName = 'name'
|
||||||
|
AND a.id = :analysisId AND m.status = 'APPROVED'
|
||||||
|
""")
|
||||||
|
List<MutationSuggestion> findApprovedLabelRenamesByAnalysisId(
|
||||||
|
@Param("analysisId") UUID analysisId);
|
||||||
|
|
||||||
|
@Query(
|
||||||
|
"""
|
||||||
|
SELECT m FROM MutationSuggestion m
|
||||||
|
JOIN StaticDataLabel l ON l.id = m.targetId
|
||||||
|
JOIN StaticAnalysis a ON a.id = l.analysis.id
|
||||||
|
WHERE m.targetType = 'GLOBAL_VAR' AND m.fieldName = 'name'
|
||||||
|
AND a.id = :analysisId AND m.status = 'APPROVED'
|
||||||
|
""")
|
||||||
|
List<MutationSuggestion> findApprovedGlobalVarRenamesByAnalysisId(
|
||||||
|
@Param("analysisId") UUID analysisId);
|
||||||
|
|
||||||
|
@Query(
|
||||||
|
"""
|
||||||
|
SELECT m FROM MutationSuggestion m
|
||||||
|
JOIN StaticStruct s ON s.id = m.targetId
|
||||||
|
JOIN StaticAnalysis a ON a.id = s.analysis.id
|
||||||
|
WHERE m.targetType = 'STRUCT' AND m.fieldName = 'name'
|
||||||
|
AND a.id = :analysisId AND m.status = 'APPROVED'
|
||||||
|
""")
|
||||||
|
List<MutationSuggestion> findApprovedStructRenamesByAnalysisId(
|
||||||
|
@Param("analysisId") UUID analysisId);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -16,6 +16,7 @@ import java.util.Map;
|
|||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import org.springframework.context.ApplicationEventPublisher;
|
import org.springframework.context.ApplicationEventPublisher;
|
||||||
@ -175,8 +176,7 @@ public class MutationService {
|
|||||||
var fieldName = "comments";
|
var fieldName = "comments";
|
||||||
|
|
||||||
var oldValue = handler.getCurrentValue(functionId, fieldName);
|
var oldValue = handler.getCurrentValue(functionId, fieldName);
|
||||||
handler.applyComment(functionId, comment, address);
|
var newValue = handler.computeUpdatedComments(oldValue, comment, address);
|
||||||
var newValue = handler.getCurrentValue(functionId, fieldName);
|
|
||||||
|
|
||||||
var mutation =
|
var mutation =
|
||||||
buildMutation(targetType, functionId, fieldName, oldValue, newValue, source, "PENDING");
|
buildMutation(targetType, functionId, fieldName, oldValue, newValue, source, "PENDING");
|
||||||
@ -285,10 +285,31 @@ public class MutationService {
|
|||||||
|
|
||||||
@Transactional(readOnly = true)
|
@Transactional(readOnly = true)
|
||||||
public Map<String, String> buildRenameMap(UUID analysisId) {
|
public Map<String, String> buildRenameMap(UUID analysisId) {
|
||||||
return mutationRepository.findApprovedFunctionRenamesByAnalysisId(analysisId).stream()
|
var functionRenames = mutationRepository.findApprovedFunctionRenamesByAnalysisId(analysisId);
|
||||||
.collect(
|
var labelRenames = mutationRepository.findApprovedLabelRenamesByAnalysisId(analysisId);
|
||||||
Collectors.toMap(
|
var globalVarRenames =
|
||||||
MutationSuggestion::getOldValue, MutationSuggestion::getNewValue, (_, b) -> b));
|
mutationRepository.findApprovedGlobalVarRenamesByAnalysisId(analysisId);
|
||||||
|
var structRenames = mutationRepository.findApprovedStructRenamesByAnalysisId(analysisId);
|
||||||
|
|
||||||
|
var map = new java.util.HashMap<String, String>();
|
||||||
|
|
||||||
|
for (var m : labelRenames) {
|
||||||
|
map.put(m.getOldValue(), m.getNewValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var m : globalVarRenames) {
|
||||||
|
map.put(m.getOldValue(), m.getNewValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var m : structRenames) {
|
||||||
|
map.put(m.getOldValue(), m.getNewValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var m : functionRenames) {
|
||||||
|
map.put(m.getOldValue(), m.getNewValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
return map;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional(readOnly = true)
|
@Transactional(readOnly = true)
|
||||||
|
|||||||
@ -52,7 +52,13 @@ public class FunctionTargetHandler implements MutationTargetHandler {
|
|||||||
switch (fieldName) {
|
switch (fieldName) {
|
||||||
case "name" -> func.setName(newValue);
|
case "name" -> func.setName(newValue);
|
||||||
case "signature" -> func.setSignature(newValue);
|
case "signature" -> func.setSignature(newValue);
|
||||||
case "comments" -> func.setComments(appendComment(func.getComments(), newValue, ""));
|
case "comments" -> {
|
||||||
|
if ("SET".equals(operation)) {
|
||||||
|
func.setComments(newValue);
|
||||||
|
} else {
|
||||||
|
func.setComments(appendComment(func.getComments(), newValue, ""));
|
||||||
|
}
|
||||||
|
}
|
||||||
case "flags" -> func.setFlags(applyFlagOperation(func.getFlags(), newValue, operation));
|
case "flags" -> func.setFlags(applyFlagOperation(func.getFlags(), newValue, operation));
|
||||||
default -> throw new IllegalArgumentException("Unknown field: " + fieldName);
|
default -> throw new IllegalArgumentException("Unknown field: " + fieldName);
|
||||||
}
|
}
|
||||||
@ -72,7 +78,13 @@ public class FunctionTargetHandler implements MutationTargetHandler {
|
|||||||
functionRepository.save(func);
|
functionRepository.save(func);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String computeUpdatedComments(String existingComments, String comment, String address) {
|
||||||
|
return appendComment(existingComments, comment, address);
|
||||||
|
}
|
||||||
|
|
||||||
private String appendComment(String existingComments, String newCommentText, String address) {
|
private String appendComment(String existingComments, String newCommentText, String address) {
|
||||||
|
validateNotJsonArray(newCommentText);
|
||||||
|
|
||||||
var list = new ArrayList<Map<String, Object>>();
|
var list = new ArrayList<Map<String, Object>>();
|
||||||
|
|
||||||
if (Objects.nonNull(existingComments)
|
if (Objects.nonNull(existingComments)
|
||||||
@ -104,6 +116,14 @@ public class FunctionTargetHandler implements MutationTargetHandler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void validateNotJsonArray(String commentText) {
|
||||||
|
var trimmed = commentText.trim();
|
||||||
|
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"Comment text looks like a JSON array. Pass only the comment text, not the full comments array.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private String applyFlagOperation(String currentFlags, String flag, String operation) {
|
private String applyFlagOperation(String currentFlags, String flag, String operation) {
|
||||||
var parts = new ArrayList<String>();
|
var parts = new ArrayList<String>();
|
||||||
if (Objects.nonNull(currentFlags) && !currentFlags.isBlank()) {
|
if (Objects.nonNull(currentFlags) && !currentFlags.isBlank()) {
|
||||||
|
|||||||
@ -3,11 +3,13 @@ package ai.decompile.job.service;
|
|||||||
import ai.decompile.ai.service.EmbeddingService;
|
import ai.decompile.ai.service.EmbeddingService;
|
||||||
import ai.decompile.analysis.event.MutationAppliedEvent;
|
import ai.decompile.analysis.event.MutationAppliedEvent;
|
||||||
import ai.decompile.analysis.event.StaticAnalysisCompletedEvent;
|
import ai.decompile.analysis.event.StaticAnalysisCompletedEvent;
|
||||||
|
import ai.decompile.analysis.model.repository.StaticLabelRepository;
|
||||||
import ai.decompile.job.messaging.JobMessagePublisher;
|
import ai.decompile.job.messaging.JobMessagePublisher;
|
||||||
import ai.decompile.job.model.entity.Job;
|
import ai.decompile.job.model.entity.Job;
|
||||||
import ai.decompile.job.model.enums.JobStatus;
|
import ai.decompile.job.model.enums.JobStatus;
|
||||||
import ai.decompile.job.model.enums.JobType;
|
import ai.decompile.job.model.enums.JobType;
|
||||||
import ai.decompile.job.model.repository.JobRepository;
|
import ai.decompile.job.model.repository.JobRepository;
|
||||||
|
import java.util.Objects;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||||
@ -24,6 +26,7 @@ public class EmbeddingEventSubscriber {
|
|||||||
private final JobRepository jobRepository;
|
private final JobRepository jobRepository;
|
||||||
private final JobMessagePublisher jobMessagePublisher;
|
private final JobMessagePublisher jobMessagePublisher;
|
||||||
private final EmbeddingService embeddingService;
|
private final EmbeddingService embeddingService;
|
||||||
|
private final StaticLabelRepository labelRepository;
|
||||||
|
|
||||||
@ApplicationModuleListener
|
@ApplicationModuleListener
|
||||||
public void on(StaticAnalysisCompletedEvent event) {
|
public void on(StaticAnalysisCompletedEvent event) {
|
||||||
@ -48,8 +51,25 @@ public class EmbeddingEventSubscriber {
|
|||||||
event.fieldName(),
|
event.fieldName(),
|
||||||
event.targetId());
|
event.targetId());
|
||||||
|
|
||||||
if ("FUNCTION".equals(event.targetType()) && "name".equals(event.fieldName())) {
|
if (!"name".equals(event.fieldName())) {
|
||||||
embeddingService.reindexFunction(event.targetId(), event.binaryId());
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (event.targetType()) {
|
||||||
|
case "FUNCTION":
|
||||||
|
embeddingService.reindexFunction(event.targetId(), event.binaryId());
|
||||||
|
break;
|
||||||
|
case "LABEL":
|
||||||
|
var label = labelRepository.findById(event.targetId()).orElse(null);
|
||||||
|
if (Objects.nonNull(label)) {
|
||||||
|
embeddingService.reindexFunction(label.getFunction().getId(), event.binaryId());
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case "GLOBAL_VAR", "STRUCT":
|
||||||
|
embeddingService.reindexAllFunctions(event.binaryId());
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE embedding_chunk
|
||||||
|
ALTER COLUMN embedding TYPE vector(768);
|
||||||
Loading…
x
Reference in New Issue
Block a user