Compare commits
No commits in common. "40991371b728a480eb88a7fd3c0d0db5bf27ba10" and "1a41ae2a8d090386ee57a94d270bef2e15b80f70" have entirely different histories.
40991371b7
...
1a41ae2a8d
@ -1,16 +1,10 @@
|
|||||||
package com.magamochi.image.model.repository;
|
package com.magamochi.image.model.repository;
|
||||||
|
|
||||||
import com.magamochi.image.model.entity.Image;
|
import com.magamochi.image.model.entity.Image;
|
||||||
import java.util.List;
|
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import org.springframework.data.jpa.repository.JpaRepository;
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
import org.springframework.data.jpa.repository.Query;
|
|
||||||
import org.springframework.data.repository.query.Param;
|
|
||||||
|
|
||||||
public interface ImageRepository extends JpaRepository<Image, UUID> {
|
public interface ImageRepository extends JpaRepository<Image, UUID> {
|
||||||
Optional<Image> findByFileHash(String fileHash);
|
Optional<Image> findByFileHash(String fileHash);
|
||||||
|
|
||||||
@Query("SELECT i.objectKey FROM Image i WHERE i.objectKey IN :objectKeys")
|
|
||||||
List<String> findExistingObjectKeys(@Param("objectKeys") List<String> objectKeys);
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,9 +4,7 @@ import com.magamochi.common.exception.NotFoundException;
|
|||||||
import com.magamochi.image.model.entity.Image;
|
import com.magamochi.image.model.entity.Image;
|
||||||
import com.magamochi.image.model.repository.ImageRepository;
|
import com.magamochi.image.model.repository.ImageRepository;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
@ -51,8 +49,8 @@ public class ImageService {
|
|||||||
.orElseThrow(() -> new NotFoundException("Image not found with ID " + id));
|
.orElseThrow(() -> new NotFoundException("Image not found with ID " + id));
|
||||||
}
|
}
|
||||||
|
|
||||||
public Set<String> findExistingObjectKeys(List<String> objectKeys) {
|
public List<Image> findAll() {
|
||||||
return new HashSet<>(imageRepository.findExistingObjectKeys(objectKeys));
|
return imageRepository.findAll();
|
||||||
}
|
}
|
||||||
|
|
||||||
public InputStream getStream(Image image) {
|
public InputStream getStream(Image image) {
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import static java.util.Objects.nonNull;
|
|||||||
|
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import lombok.Getter;
|
import lombok.Getter;
|
||||||
@ -41,7 +42,8 @@ public class S3Service {
|
|||||||
return filename;
|
return filename;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void processObjectKeyPages(java.util.function.Consumer<List<String>> pageConsumer) {
|
public List<String> listAllObjectKeys() {
|
||||||
|
var keys = new ArrayList<String>();
|
||||||
String continuationToken = null;
|
String continuationToken = null;
|
||||||
|
|
||||||
do {
|
do {
|
||||||
@ -53,12 +55,13 @@ public class S3Service {
|
|||||||
|
|
||||||
var response = s3Client.listObjectsV2(requestBuilder.build());
|
var response = s3Client.listObjectsV2(requestBuilder.build());
|
||||||
|
|
||||||
var page = response.contents().stream().map(S3Object::key).toList();
|
response.contents().forEach(s3Object -> keys.add(s3Object.key()));
|
||||||
pageConsumer.accept(page);
|
|
||||||
|
|
||||||
continuationToken = response.isTruncated() ? response.nextContinuationToken() : null;
|
continuationToken = response.isTruncated() ? response.nextContinuationToken() : null;
|
||||||
|
|
||||||
} while (nonNull(continuationToken));
|
} while (nonNull(continuationToken));
|
||||||
|
|
||||||
|
return keys;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteObjects(Set<String> objectKeys) {
|
public void deleteObjects(Set<String> objectKeys) {
|
||||||
|
|||||||
@ -1,7 +1,9 @@
|
|||||||
package com.magamochi.image.task;
|
package com.magamochi.image.task;
|
||||||
|
|
||||||
|
import com.magamochi.image.model.entity.Image;
|
||||||
import com.magamochi.image.service.ImageService;
|
import com.magamochi.image.service.ImageService;
|
||||||
import com.magamochi.image.service.S3Service;
|
import com.magamochi.image.service.S3Service;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
@ -29,21 +31,21 @@ public class ImageCleanupTask {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public void cleanupImages() {
|
public void cleanupImages() {
|
||||||
log.info("Scanning S3 pages for orphaned object keys.");
|
log.info("Getting unused S3 object keys to remove.");
|
||||||
var keysToRemove = new java.util.HashSet<String>();
|
|
||||||
|
|
||||||
s3Service.processObjectKeyPages(
|
var imageKeys = s3Service.listAllObjectKeys();
|
||||||
page -> {
|
|
||||||
var existing = imageService.findExistingObjectKeys(page);
|
|
||||||
page.stream().filter(key -> !existing.contains(key)).forEach(keysToRemove::add);
|
|
||||||
});
|
|
||||||
|
|
||||||
if (keysToRemove.isEmpty()) {
|
var existingImages =
|
||||||
log.info("No orphaned objects found.");
|
imageService.findAll().parallelStream()
|
||||||
return;
|
.map(Image::getObjectKey)
|
||||||
}
|
.collect(Collectors.toSet());
|
||||||
|
|
||||||
log.info("Removing {} orphaned objects from S3 storage", keysToRemove.size());
|
var keysToRemove =
|
||||||
|
imageKeys.parallelStream()
|
||||||
|
.filter(imageKey -> !existingImages.contains(imageKey))
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
|
||||||
|
log.info("Removing {} objects from S3 storage", keysToRemove.size());
|
||||||
|
|
||||||
s3Service.deleteObjects(keysToRemove);
|
s3Service.deleteObjects(keysToRemove);
|
||||||
|
|
||||||
|
|||||||
@ -1,98 +0,0 @@
|
|||||||
-- ============================================================
|
|
||||||
-- V0012 — Add performance indexes
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
-- mangas
|
|
||||||
-- follow: findByFollowTrue() hot path for the scheduled follow-update task
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_mangas_follow ON mangas (follow);
|
|
||||||
|
|
||||||
-- state: natural filter (PENDING, ACTIVE, etc.)
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_mangas_state ON mangas (state);
|
|
||||||
|
|
||||||
-- adult: MangaSpecification always filters adult = false by default
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_mangas_adult ON mangas (adult);
|
|
||||||
|
|
||||||
-- score: MangaSpecification filters score >= ?
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_mangas_score ON mangas (score);
|
|
||||||
|
|
||||||
-- status: plain index for exact-match; functional index for LOWER(status) IN (...)
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_mangas_status ON mangas (status);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_mangas_status_lower ON mangas (LOWER(status));
|
|
||||||
|
|
||||||
-- title: findByTitleIgnoreCase + LIKE search via LOWER(title)
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_mangas_title_lower ON mangas (LOWER(title));
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
-- images
|
|
||||||
-- object_key: needed for the page-by-page IN-query in ImageCleanupTask
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_images_object_key ON images (object_key);
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
-- manga_contents
|
|
||||||
-- manga_content_provider_id: FK join for every chapter lookup
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_manga_contents_provider_id ON manga_contents (manga_content_provider_id);
|
|
||||||
|
|
||||||
-- (manga_content_provider_id, LOWER(url)): covers existsByMangaContentProvider_IdAndUrlIgnoreCase
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_manga_contents_provider_url ON manga_contents (manga_content_provider_id, LOWER(url));
|
|
||||||
|
|
||||||
-- downloaded: ingestion pipelines filter on downloaded = false
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_manga_contents_downloaded ON manga_contents (downloaded);
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
-- manga_content_images
|
|
||||||
-- manga_content_id: findAllByMangaContent + existsByMangaContent_IdAndPosition
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_manga_content_images_content_id ON manga_content_images (manga_content_id);
|
|
||||||
|
|
||||||
-- (manga_content_id, position): covers the position-existence check and ordered reads
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_manga_content_images_content_position ON manga_content_images (manga_content_id, position);
|
|
||||||
|
|
||||||
-- image_id: FK into the 500k-row images table — cascade deletes and joins
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_manga_content_images_image_id ON manga_content_images (image_id);
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
-- manga_content_provider
|
|
||||||
-- manga_id: findByManga_IdAndContentProvider_Id + @OneToMany collection load
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_manga_content_provider_manga_id ON manga_content_provider (manga_id);
|
|
||||||
|
|
||||||
-- LOWER(manga_title): existsByMangaTitleIgnoreCaseAndContentProvider_Id
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_manga_content_provider_title_lower ON manga_content_provider (LOWER(manga_title));
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
-- manga_ingest_reviews
|
|
||||||
-- content_provider_id: FK lookups and cascade deletes
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_manga_ingest_reviews_provider_id ON manga_ingest_reviews (content_provider_id);
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
-- user_manga_content_read
|
|
||||||
-- user_id alone: countByUser, countDistinctMangaByUser
|
|
||||||
-- (The UNIQUE constraint on (user_id, manga_content_id) exists but a single-column index
|
|
||||||
-- on user_id allows efficient count/scan queries without reading the full composite key.)
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_user_manga_content_read_user_id ON user_manga_content_read (user_id);
|
|
||||||
|
|
||||||
-- (user_id, created_at DESC): eliminates the sort step in findTop10ByUserOrderByCreatedAtDesc
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_user_manga_content_read_user_created ON user_manga_content_read (user_id, created_at DESC);
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
-- user_manga_follow
|
|
||||||
-- user_id: findByUser, existsByUserAndManga
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_user_manga_follow_user_id ON user_manga_follow (user_id);
|
|
||||||
|
|
||||||
-- manga_id: findByManga, existsByManga
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_user_manga_follow_manga_id ON user_manga_follow (manga_id);
|
|
||||||
|
|
||||||
-- Composite unique: existsByUserAndManga / findByUserAndManga point lookups
|
|
||||||
-- (no constraint existed on this table before)
|
|
||||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_user_manga_follow_user_manga ON user_manga_follow (user_id, manga_id);
|
|
||||||
|
|
||||||
-- ============================================================
|
|
||||||
|
|
||||||
-- manga_import_job
|
|
||||||
-- status: MangaImportJobSpecification filters on status
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_manga_import_job_status ON manga_import_job (status);
|
|
||||||
Loading…
x
Reference in New Issue
Block a user