Compare commits

..

2 Commits

5 changed files with 123 additions and 22 deletions

View File

@ -1,10 +1,16 @@
package com.magamochi.image.model.repository;
import com.magamochi.image.model.entity.Image;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
public interface ImageRepository extends JpaRepository<Image, UUID> {
Optional<Image> findByFileHash(String fileHash);
@Query("SELECT i.objectKey FROM Image i WHERE i.objectKey IN :objectKeys")
List<String> findExistingObjectKeys(@Param("objectKeys") List<String> objectKeys);
}

View File

@ -4,7 +4,9 @@ import com.magamochi.common.exception.NotFoundException;
import com.magamochi.image.model.entity.Image;
import com.magamochi.image.model.repository.ImageRepository;
import java.io.InputStream;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
@ -49,8 +51,8 @@ public class ImageService {
.orElseThrow(() -> new NotFoundException("Image not found with ID " + id));
}
public List<Image> findAll() {
return imageRepository.findAll();
public Set<String> findExistingObjectKeys(List<String> objectKeys) {
return new HashSet<>(imageRepository.findExistingObjectKeys(objectKeys));
}
public InputStream getStream(Image image) {

View File

@ -4,7 +4,6 @@ import static java.util.Objects.nonNull;
import java.io.InputStream;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import lombok.Getter;
@ -42,8 +41,7 @@ public class S3Service {
return filename;
}
public List<String> listAllObjectKeys() {
var keys = new ArrayList<String>();
public void processObjectKeyPages(java.util.function.Consumer<List<String>> pageConsumer) {
String continuationToken = null;
do {
@ -55,13 +53,12 @@ public class S3Service {
var response = s3Client.listObjectsV2(requestBuilder.build());
response.contents().forEach(s3Object -> keys.add(s3Object.key()));
var page = response.contents().stream().map(S3Object::key).toList();
pageConsumer.accept(page);
continuationToken = response.isTruncated() ? response.nextContinuationToken() : null;
} while (nonNull(continuationToken));
return keys;
}
public void deleteObjects(Set<String> objectKeys) {

View File

@ -1,9 +1,7 @@
package com.magamochi.image.task;
import com.magamochi.image.model.entity.Image;
import com.magamochi.image.service.ImageService;
import com.magamochi.image.service.S3Service;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Value;
@ -31,21 +29,21 @@ public class ImageCleanupTask {
}
public void cleanupImages() {
log.info("Getting unused S3 object keys to remove.");
log.info("Scanning S3 pages for orphaned object keys.");
var keysToRemove = new java.util.HashSet<String>();
var imageKeys = s3Service.listAllObjectKeys();
s3Service.processObjectKeyPages(
page -> {
var existing = imageService.findExistingObjectKeys(page);
page.stream().filter(key -> !existing.contains(key)).forEach(keysToRemove::add);
});
var existingImages =
imageService.findAll().parallelStream()
.map(Image::getObjectKey)
.collect(Collectors.toSet());
if (keysToRemove.isEmpty()) {
log.info("No orphaned objects found.");
return;
}
var keysToRemove =
imageKeys.parallelStream()
.filter(imageKey -> !existingImages.contains(imageKey))
.collect(Collectors.toSet());
log.info("Removing {} objects from S3 storage", keysToRemove.size());
log.info("Removing {} orphaned objects from S3 storage", keysToRemove.size());
s3Service.deleteObjects(keysToRemove);

View File

@ -0,0 +1,98 @@
-- ============================================================
-- V0012 — Add performance indexes
-- ============================================================
-- mangas
-- follow: findByFollowTrue() hot path for the scheduled follow-update task
CREATE INDEX IF NOT EXISTS idx_mangas_follow ON mangas (follow);
-- state: natural filter (PENDING, ACTIVE, etc.)
CREATE INDEX IF NOT EXISTS idx_mangas_state ON mangas (state);
-- adult: MangaSpecification always filters adult = false by default
CREATE INDEX IF NOT EXISTS idx_mangas_adult ON mangas (adult);
-- score: MangaSpecification filters score >= ?
CREATE INDEX IF NOT EXISTS idx_mangas_score ON mangas (score);
-- status: plain index for exact-match; functional index for LOWER(status) IN (...)
CREATE INDEX IF NOT EXISTS idx_mangas_status ON mangas (status);
CREATE INDEX IF NOT EXISTS idx_mangas_status_lower ON mangas (LOWER(status));
-- title: findByTitleIgnoreCase + LIKE search via LOWER(title)
CREATE INDEX IF NOT EXISTS idx_mangas_title_lower ON mangas (LOWER(title));
-- ============================================================
-- images
-- object_key: needed for the page-by-page IN-query in ImageCleanupTask
CREATE INDEX IF NOT EXISTS idx_images_object_key ON images (object_key);
-- ============================================================
-- manga_contents
-- manga_content_provider_id: FK join for every chapter lookup
CREATE INDEX IF NOT EXISTS idx_manga_contents_provider_id ON manga_contents (manga_content_provider_id);
-- (manga_content_provider_id, LOWER(url)): covers existsByMangaContentProvider_IdAndUrlIgnoreCase
CREATE INDEX IF NOT EXISTS idx_manga_contents_provider_url ON manga_contents (manga_content_provider_id, LOWER(url));
-- downloaded: ingestion pipelines filter on downloaded = false
CREATE INDEX IF NOT EXISTS idx_manga_contents_downloaded ON manga_contents (downloaded);
-- ============================================================
-- manga_content_images
-- manga_content_id: findAllByMangaContent + existsByMangaContent_IdAndPosition
CREATE INDEX IF NOT EXISTS idx_manga_content_images_content_id ON manga_content_images (manga_content_id);
-- (manga_content_id, position): covers the position-existence check and ordered reads
CREATE INDEX IF NOT EXISTS idx_manga_content_images_content_position ON manga_content_images (manga_content_id, position);
-- image_id: FK into the 500k-row images table — cascade deletes and joins
CREATE INDEX IF NOT EXISTS idx_manga_content_images_image_id ON manga_content_images (image_id);
-- ============================================================
-- manga_content_provider
-- manga_id: findByManga_IdAndContentProvider_Id + @OneToMany collection load
CREATE INDEX IF NOT EXISTS idx_manga_content_provider_manga_id ON manga_content_provider (manga_id);
-- LOWER(manga_title): existsByMangaTitleIgnoreCaseAndContentProvider_Id
CREATE INDEX IF NOT EXISTS idx_manga_content_provider_title_lower ON manga_content_provider (LOWER(manga_title));
-- ============================================================
-- manga_ingest_reviews
-- content_provider_id: FK lookups and cascade deletes
CREATE INDEX IF NOT EXISTS idx_manga_ingest_reviews_provider_id ON manga_ingest_reviews (content_provider_id);
-- ============================================================
-- user_manga_content_read
-- user_id alone: countByUser, countDistinctMangaByUser
-- (The UNIQUE constraint on (user_id, manga_content_id) exists but a single-column index
-- on user_id allows efficient count/scan queries without reading the full composite key.)
CREATE INDEX IF NOT EXISTS idx_user_manga_content_read_user_id ON user_manga_content_read (user_id);
-- (user_id, created_at DESC): eliminates the sort step in findTop10ByUserOrderByCreatedAtDesc
CREATE INDEX IF NOT EXISTS idx_user_manga_content_read_user_created ON user_manga_content_read (user_id, created_at DESC);
-- ============================================================
-- user_manga_follow
-- user_id: findByUser, existsByUserAndManga
CREATE INDEX IF NOT EXISTS idx_user_manga_follow_user_id ON user_manga_follow (user_id);
-- manga_id: findByManga, existsByManga
CREATE INDEX IF NOT EXISTS idx_user_manga_follow_manga_id ON user_manga_follow (manga_id);
-- Composite unique: existsByUserAndManga / findByUserAndManga point lookups
-- (no constraint existed on this table before)
CREATE UNIQUE INDEX IF NOT EXISTS idx_user_manga_follow_user_manga ON user_manga_follow (user_id, manga_id);
-- ============================================================
-- manga_import_job
-- status: MangaImportJobSpecification filters on status
CREATE INDEX IF NOT EXISTS idx_manga_import_job_status ON manga_import_job (status);