Compare commits
2 Commits
1a41ae2a8d
...
40991371b7
| Author | SHA1 | Date | |
|---|---|---|---|
| 40991371b7 | |||
| da3114b85e |
@ -1,10 +1,16 @@
|
|||||||
package com.magamochi.image.model.repository;
|
package com.magamochi.image.model.repository;
|
||||||
|
|
||||||
import com.magamochi.image.model.entity.Image;
|
import com.magamochi.image.model.entity.Image;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import org.springframework.data.jpa.repository.JpaRepository;
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
|
import org.springframework.data.jpa.repository.Query;
|
||||||
|
import org.springframework.data.repository.query.Param;
|
||||||
|
|
||||||
public interface ImageRepository extends JpaRepository<Image, UUID> {
|
public interface ImageRepository extends JpaRepository<Image, UUID> {
|
||||||
Optional<Image> findByFileHash(String fileHash);
|
Optional<Image> findByFileHash(String fileHash);
|
||||||
|
|
||||||
|
@Query("SELECT i.objectKey FROM Image i WHERE i.objectKey IN :objectKeys")
|
||||||
|
List<String> findExistingObjectKeys(@Param("objectKeys") List<String> objectKeys);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,7 +4,9 @@ import com.magamochi.common.exception.NotFoundException;
|
|||||||
import com.magamochi.image.model.entity.Image;
|
import com.magamochi.image.model.entity.Image;
|
||||||
import com.magamochi.image.model.repository.ImageRepository;
|
import com.magamochi.image.model.repository.ImageRepository;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
@ -49,8 +51,8 @@ public class ImageService {
|
|||||||
.orElseThrow(() -> new NotFoundException("Image not found with ID " + id));
|
.orElseThrow(() -> new NotFoundException("Image not found with ID " + id));
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Image> findAll() {
|
public Set<String> findExistingObjectKeys(List<String> objectKeys) {
|
||||||
return imageRepository.findAll();
|
return new HashSet<>(imageRepository.findExistingObjectKeys(objectKeys));
|
||||||
}
|
}
|
||||||
|
|
||||||
public InputStream getStream(Image image) {
|
public InputStream getStream(Image image) {
|
||||||
|
|||||||
@ -4,7 +4,6 @@ import static java.util.Objects.nonNull;
|
|||||||
|
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
@ -42,8 +41,7 @@ public class S3Service {
|
|||||||
return filename;
|
return filename;
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<String> listAllObjectKeys() {
|
public void processObjectKeyPages(java.util.function.Consumer<List<String>> pageConsumer) {
|
||||||
var keys = new ArrayList<String>();
|
|
||||||
String continuationToken = null;
|
String continuationToken = null;
|
||||||
|
|
||||||
do {
|
do {
|
||||||
@ -55,13 +53,12 @@ public class S3Service {
|
|||||||
|
|
||||||
var response = s3Client.listObjectsV2(requestBuilder.build());
|
var response = s3Client.listObjectsV2(requestBuilder.build());
|
||||||
|
|
||||||
response.contents().forEach(s3Object -> keys.add(s3Object.key()));
|
var page = response.contents().stream().map(S3Object::key).toList();
|
||||||
|
pageConsumer.accept(page);
|
||||||
|
|
||||||
continuationToken = response.isTruncated() ? response.nextContinuationToken() : null;
|
continuationToken = response.isTruncated() ? response.nextContinuationToken() : null;
|
||||||
|
|
||||||
} while (nonNull(continuationToken));
|
} while (nonNull(continuationToken));
|
||||||
|
|
||||||
return keys;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteObjects(Set<String> objectKeys) {
|
public void deleteObjects(Set<String> objectKeys) {
|
||||||
|
|||||||
@ -1,9 +1,7 @@
|
|||||||
package com.magamochi.image.task;
|
package com.magamochi.image.task;
|
||||||
|
|
||||||
import com.magamochi.image.model.entity.Image;
|
|
||||||
import com.magamochi.image.service.ImageService;
|
import com.magamochi.image.service.ImageService;
|
||||||
import com.magamochi.image.service.S3Service;
|
import com.magamochi.image.service.S3Service;
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@ -31,21 +29,21 @@ public class ImageCleanupTask {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public void cleanupImages() {
|
public void cleanupImages() {
|
||||||
log.info("Getting unused S3 object keys to remove.");
|
log.info("Scanning S3 pages for orphaned object keys.");
|
||||||
|
var keysToRemove = new java.util.HashSet<String>();
|
||||||
|
|
||||||
var imageKeys = s3Service.listAllObjectKeys();
|
s3Service.processObjectKeyPages(
|
||||||
|
page -> {
|
||||||
|
var existing = imageService.findExistingObjectKeys(page);
|
||||||
|
page.stream().filter(key -> !existing.contains(key)).forEach(keysToRemove::add);
|
||||||
|
});
|
||||||
|
|
||||||
var existingImages =
|
if (keysToRemove.isEmpty()) {
|
||||||
imageService.findAll().parallelStream()
|
log.info("No orphaned objects found.");
|
||||||
.map(Image::getObjectKey)
|
return;
|
||||||
.collect(Collectors.toSet());
|
}
|
||||||
|
|
||||||
var keysToRemove =
|
log.info("Removing {} orphaned objects from S3 storage", keysToRemove.size());
|
||||||
imageKeys.parallelStream()
|
|
||||||
.filter(imageKey -> !existingImages.contains(imageKey))
|
|
||||||
.collect(Collectors.toSet());
|
|
||||||
|
|
||||||
log.info("Removing {} objects from S3 storage", keysToRemove.size());
|
|
||||||
|
|
||||||
s3Service.deleteObjects(keysToRemove);
|
s3Service.deleteObjects(keysToRemove);
|
||||||
|
|
||||||
|
|||||||
98
src/main/resources/db/migration/V0012__ADD_INDEXES.sql
Normal file
98
src/main/resources/db/migration/V0012__ADD_INDEXES.sql
Normal file
@ -0,0 +1,98 @@
|
|||||||
|
-- ============================================================
|
||||||
|
-- V0012 — Add performance indexes
|
||||||
|
-- ============================================================
|
||||||
|
|
||||||
|
-- mangas
|
||||||
|
-- follow: findByFollowTrue() hot path for the scheduled follow-update task
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mangas_follow ON mangas (follow);
|
||||||
|
|
||||||
|
-- state: natural filter (PENDING, ACTIVE, etc.)
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mangas_state ON mangas (state);
|
||||||
|
|
||||||
|
-- adult: MangaSpecification always filters adult = false by default
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mangas_adult ON mangas (adult);
|
||||||
|
|
||||||
|
-- score: MangaSpecification filters score >= ?
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mangas_score ON mangas (score);
|
||||||
|
|
||||||
|
-- status: plain index for exact-match; functional index for LOWER(status) IN (...)
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mangas_status ON mangas (status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mangas_status_lower ON mangas (LOWER(status));
|
||||||
|
|
||||||
|
-- title: findByTitleIgnoreCase + LIKE search via LOWER(title)
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_mangas_title_lower ON mangas (LOWER(title));
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
|
||||||
|
-- images
|
||||||
|
-- object_key: needed for the page-by-page IN-query in ImageCleanupTask
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_images_object_key ON images (object_key);
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
|
||||||
|
-- manga_contents
|
||||||
|
-- manga_content_provider_id: FK join for every chapter lookup
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manga_contents_provider_id ON manga_contents (manga_content_provider_id);
|
||||||
|
|
||||||
|
-- (manga_content_provider_id, LOWER(url)): covers existsByMangaContentProvider_IdAndUrlIgnoreCase
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manga_contents_provider_url ON manga_contents (manga_content_provider_id, LOWER(url));
|
||||||
|
|
||||||
|
-- downloaded: ingestion pipelines filter on downloaded = false
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manga_contents_downloaded ON manga_contents (downloaded);
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
|
||||||
|
-- manga_content_images
|
||||||
|
-- manga_content_id: findAllByMangaContent + existsByMangaContent_IdAndPosition
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manga_content_images_content_id ON manga_content_images (manga_content_id);
|
||||||
|
|
||||||
|
-- (manga_content_id, position): covers the position-existence check and ordered reads
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manga_content_images_content_position ON manga_content_images (manga_content_id, position);
|
||||||
|
|
||||||
|
-- image_id: FK into the 500k-row images table — cascade deletes and joins
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manga_content_images_image_id ON manga_content_images (image_id);
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
|
||||||
|
-- manga_content_provider
|
||||||
|
-- manga_id: findByManga_IdAndContentProvider_Id + @OneToMany collection load
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manga_content_provider_manga_id ON manga_content_provider (manga_id);
|
||||||
|
|
||||||
|
-- LOWER(manga_title): existsByMangaTitleIgnoreCaseAndContentProvider_Id
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manga_content_provider_title_lower ON manga_content_provider (LOWER(manga_title));
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
|
||||||
|
-- manga_ingest_reviews
|
||||||
|
-- content_provider_id: FK lookups and cascade deletes
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manga_ingest_reviews_provider_id ON manga_ingest_reviews (content_provider_id);
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
|
||||||
|
-- user_manga_content_read
|
||||||
|
-- user_id alone: countByUser, countDistinctMangaByUser
|
||||||
|
-- (The UNIQUE constraint on (user_id, manga_content_id) exists but a single-column index
|
||||||
|
-- on user_id allows efficient count/scan queries without reading the full composite key.)
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_user_manga_content_read_user_id ON user_manga_content_read (user_id);
|
||||||
|
|
||||||
|
-- (user_id, created_at DESC): eliminates the sort step in findTop10ByUserOrderByCreatedAtDesc
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_user_manga_content_read_user_created ON user_manga_content_read (user_id, created_at DESC);
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
|
||||||
|
-- user_manga_follow
|
||||||
|
-- user_id: findByUser, existsByUserAndManga
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_user_manga_follow_user_id ON user_manga_follow (user_id);
|
||||||
|
|
||||||
|
-- manga_id: findByManga, existsByManga
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_user_manga_follow_manga_id ON user_manga_follow (manga_id);
|
||||||
|
|
||||||
|
-- Composite unique: existsByUserAndManga / findByUserAndManga point lookups
|
||||||
|
-- (no constraint existed on this table before)
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_user_manga_follow_user_manga ON user_manga_follow (user_id, manga_id);
|
||||||
|
|
||||||
|
-- ============================================================
|
||||||
|
|
||||||
|
-- manga_import_job
|
||||||
|
-- status: MangaImportJobSpecification filters on status
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_manga_import_job_status ON manga_import_job (status);
|
||||||
Loading…
x
Reference in New Issue
Block a user