refactor-architecture #31
@ -2,9 +2,13 @@ package com.magamochi.catalog.model.repository;
|
|||||||
|
|
||||||
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
||||||
import jakarta.validation.constraints.NotBlank;
|
import jakarta.validation.constraints.NotBlank;
|
||||||
|
import java.util.Optional;
|
||||||
import org.springframework.data.jpa.repository.JpaRepository;
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
|
|
||||||
public interface MangaContentProviderRepository extends JpaRepository<MangaContentProvider, Long> {
|
public interface MangaContentProviderRepository extends JpaRepository<MangaContentProvider, Long> {
|
||||||
boolean existsByMangaTitleIgnoreCaseAndContentProvider_Id(
|
boolean existsByMangaTitleIgnoreCaseAndContentProvider_Id(
|
||||||
@NotBlank String mangaTitle, long contentProviderId);
|
@NotBlank String mangaTitle, long contentProviderId);
|
||||||
|
|
||||||
|
Optional<MangaContentProvider> findByManga_IdAndContentProvider_Id(
|
||||||
|
long mangaId, long contentProviderId);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,8 +1,10 @@
|
|||||||
package com.magamochi.catalog.service;
|
package com.magamochi.catalog.service;
|
||||||
|
|
||||||
|
import com.magamochi.catalog.model.entity.Manga;
|
||||||
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
||||||
import com.magamochi.catalog.model.repository.MangaContentProviderRepository;
|
import com.magamochi.catalog.model.repository.MangaContentProviderRepository;
|
||||||
import com.magamochi.common.exception.NotFoundException;
|
import com.magamochi.common.exception.NotFoundException;
|
||||||
|
import com.magamochi.ingestion.model.entity.ContentProvider;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
@ -19,4 +21,17 @@ public class MangaContentProviderService {
|
|||||||
new NotFoundException(
|
new NotFoundException(
|
||||||
"MangaContentProvider not found - ID: " + mangaContentProviderId));
|
"MangaContentProvider not found - ID: " + mangaContentProviderId));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public MangaContentProvider findOrCreate(Manga manga, ContentProvider contentProvider) {
|
||||||
|
return mangaContentProviderRepository
|
||||||
|
.findByManga_IdAndContentProvider_Id(manga.getId(), contentProvider.getId())
|
||||||
|
.orElseGet(
|
||||||
|
() ->
|
||||||
|
mangaContentProviderRepository.save(
|
||||||
|
MangaContentProvider.builder()
|
||||||
|
.manga(manga)
|
||||||
|
.mangaTitle(manga.getTitle())
|
||||||
|
.contentProvider(contentProvider)
|
||||||
|
.build()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -98,6 +98,10 @@ public class MangaResolutionService {
|
|||||||
return Optional.of(new ProviderResult(bestTitle, malId));
|
return Optional.of(new ProviderResult(bestTitle, malId));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Manga findOrCreateManga(Long aniListId, Long malId) {
|
||||||
|
return findOrCreateManga(null, aniListId, malId);
|
||||||
|
}
|
||||||
|
|
||||||
private Manga findOrCreateManga(String canonicalTitle, Long aniListId, Long malId) {
|
private Manga findOrCreateManga(String canonicalTitle, Long aniListId, Long malId) {
|
||||||
if (nonNull(aniListId)) {
|
if (nonNull(aniListId)) {
|
||||||
var existingByAniList = mangaRepository.findByAniListId(aniListId);
|
var existingByAniList = mangaRepository.findByAniListId(aniListId);
|
||||||
@ -113,20 +117,24 @@ public class MangaResolutionService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return mangaRepository
|
if (nonNull(canonicalTitle)) {
|
||||||
.findByTitleIgnoreCase(canonicalTitle)
|
var existingByTitle = mangaRepository.findByTitleIgnoreCase(canonicalTitle);
|
||||||
.orElseGet(
|
if (existingByTitle.isPresent()) {
|
||||||
() -> {
|
return existingByTitle.get();
|
||||||
var newManga =
|
}
|
||||||
Manga.builder().title(canonicalTitle).malId(malId).aniListId(aniListId).build();
|
}
|
||||||
|
|
||||||
var savedManga = mangaRepository.save(newManga);
|
return createAndNotifyManga(canonicalTitle, aniListId, malId);
|
||||||
|
}
|
||||||
|
|
||||||
mangaUpdateProducer.sendMangaUpdateCommand(
|
private Manga createAndNotifyManga(String title, Long aniListId, Long malId) {
|
||||||
new MangaUpdateCommand(savedManga.getId()));
|
var manga =
|
||||||
|
mangaRepository.save(
|
||||||
|
Manga.builder().title(title).aniListId(aniListId).malId(malId).build());
|
||||||
|
|
||||||
return savedManga;
|
mangaUpdateProducer.sendMangaUpdateCommand(new MangaUpdateCommand(manga.getId()));
|
||||||
});
|
|
||||||
|
return manga;
|
||||||
}
|
}
|
||||||
|
|
||||||
private record ProviderResult(String title, Long externalId) {}
|
private record ProviderResult(String title, Long externalId) {}
|
||||||
|
|||||||
@ -77,7 +77,7 @@ public class MyAnimeListService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private MangaStatus mapStatus(String malStatus) {
|
private MangaStatus mapStatus(String malStatus) {
|
||||||
return switch (malStatus) {
|
return switch (malStatus.toLowerCase()) {
|
||||||
case "finished" -> MangaStatus.COMPLETED;
|
case "finished" -> MangaStatus.COMPLETED;
|
||||||
case "publishing" -> MangaStatus.ONGOING;
|
case "publishing" -> MangaStatus.ONGOING;
|
||||||
case "on hiatus" -> MangaStatus.HIATUS;
|
case "on hiatus" -> MangaStatus.HIATUS;
|
||||||
|
|||||||
@ -1,8 +1,9 @@
|
|||||||
package com.magamochi.ingestion.providers;
|
package com.magamochi.common;
|
||||||
|
|
||||||
public class ContentProviders {
|
public class ContentProviders {
|
||||||
public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog";
|
public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog";
|
||||||
public static final String MANGA_LIVRE_TO = "Manga Livre.to";
|
public static final String MANGA_LIVRE_TO = "Manga Livre.to";
|
||||||
public static final String PINK_ROSA_SCAN = "Pink Rosa Scan";
|
public static final String PINK_ROSA_SCAN = "Pink Rosa Scan";
|
||||||
public static final String MANGA_DEX = "MangaDex";
|
public static final String MANGA_DEX = "MangaDex";
|
||||||
|
public static final String MANUAL_IMPORT = "Manual Import";
|
||||||
}
|
}
|
||||||
@ -37,6 +37,9 @@ public class RabbitConfig {
|
|||||||
@Value("${queues.image-fetch}")
|
@Value("${queues.image-fetch}")
|
||||||
private String imageFetchQueue;
|
private String imageFetchQueue;
|
||||||
|
|
||||||
|
@Value("${queues.file-import}")
|
||||||
|
private String fileImportQueue;
|
||||||
|
|
||||||
@Value("${topics.image-updates}")
|
@Value("${topics.image-updates}")
|
||||||
private String imageUpdatesTopic;
|
private String imageUpdatesTopic;
|
||||||
|
|
||||||
@ -68,6 +71,11 @@ public class RabbitConfig {
|
|||||||
return new Queue(mangaCoverUpdateQueue, false);
|
return new Queue(mangaCoverUpdateQueue, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Bean
|
||||||
|
public Queue fileImportQueue() {
|
||||||
|
return new Queue(fileImportQueue, false);
|
||||||
|
}
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
public Binding bindingMangaCoverUpdateQueue(
|
public Binding bindingMangaCoverUpdateQueue(
|
||||||
Queue mangaCoverUpdateQueue, TopicExchange imageUpdatesExchange) {
|
Queue mangaCoverUpdateQueue, TopicExchange imageUpdatesExchange) {
|
||||||
|
|||||||
@ -1,10 +1,12 @@
|
|||||||
package com.magamochi.content.controller;
|
package com.magamochi.content.controller;
|
||||||
|
|
||||||
import com.magamochi.common.model.dto.DefaultResponseDTO;
|
import com.magamochi.common.model.dto.DefaultResponseDTO;
|
||||||
|
import com.magamochi.content.model.dto.FileImportRequestDTO;
|
||||||
import com.magamochi.content.model.dto.MangaContentDTO;
|
import com.magamochi.content.model.dto.MangaContentDTO;
|
||||||
import com.magamochi.content.model.dto.MangaContentImagesDTO;
|
import com.magamochi.content.model.dto.MangaContentImagesDTO;
|
||||||
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
|
import com.magamochi.content.model.enumeration.ContentArchiveFileType;
|
||||||
import com.magamochi.content.service.ContentDownloadService;
|
import com.magamochi.content.service.ContentDownloadService;
|
||||||
|
import com.magamochi.content.service.ContentImportService;
|
||||||
import com.magamochi.content.service.ContentService;
|
import com.magamochi.content.service.ContentService;
|
||||||
import io.swagger.v3.oas.annotations.Operation;
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
import io.swagger.v3.oas.annotations.media.Content;
|
import io.swagger.v3.oas.annotations.media.Content;
|
||||||
@ -25,6 +27,7 @@ import org.springframework.web.bind.annotation.*;
|
|||||||
public class ContentController {
|
public class ContentController {
|
||||||
private final ContentService contentService;
|
private final ContentService contentService;
|
||||||
private final ContentDownloadService contentDownloadService;
|
private final ContentDownloadService contentDownloadService;
|
||||||
|
private final ContentImportService contentImportService;
|
||||||
|
|
||||||
@Operation(
|
@Operation(
|
||||||
summary = "Get the content for a specific manga/content provider combination",
|
summary = "Get the content for a specific manga/content provider combination",
|
||||||
@ -76,4 +79,18 @@ public class ContentController {
|
|||||||
.header("Content-Disposition", "attachment; filename=\"" + response.filename() + "\"")
|
.header("Content-Disposition", "attachment; filename=\"" + response.filename() + "\"")
|
||||||
.body(response.content());
|
.body(response.content());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Operation(
|
||||||
|
summary = "Import multiple files",
|
||||||
|
description = "Accepts multiple content files via multipart/form-data and processes them.",
|
||||||
|
tags = {"Content"},
|
||||||
|
operationId = "importContentFiles")
|
||||||
|
@PostMapping(value = "/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||||
|
public DefaultResponseDTO<Void> importContentFiles(
|
||||||
|
@ModelAttribute FileImportRequestDTO requestDTO) {
|
||||||
|
contentImportService.importFiles(
|
||||||
|
requestDTO.malId(), requestDTO.aniListId(), requestDTO.files());
|
||||||
|
|
||||||
|
return DefaultResponseDTO.ok().build();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,6 @@
|
|||||||
|
package com.magamochi.content.model.dto;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
|
||||||
|
public record FileImportRequestDTO(String malId, String aniListId, List<MultipartFile> files) {}
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
package com.magamochi.content.queue.command;
|
||||||
|
|
||||||
|
public record FileImportCommand(long mangaContentProviderId, String filename) {}
|
||||||
@ -0,0 +1,21 @@
|
|||||||
|
package com.magamochi.content.queue.consumer;
|
||||||
|
|
||||||
|
import com.magamochi.content.queue.command.FileImportCommand;
|
||||||
|
import com.magamochi.content.service.ContentImportService;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.log4j.Log4j2;
|
||||||
|
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
@Log4j2
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class FileImportConsumer {
|
||||||
|
private final ContentImportService contentImportService;
|
||||||
|
|
||||||
|
@RabbitListener(queues = "${queues.file-import}")
|
||||||
|
public void receiveFileImportCommand(FileImportCommand command) {
|
||||||
|
log.info("Received file import command: {}", command);
|
||||||
|
contentImportService.importFile(command.mangaContentProviderId(), command.filename());
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,23 @@
|
|||||||
|
package com.magamochi.content.queue.producer;
|
||||||
|
|
||||||
|
import com.magamochi.content.queue.command.FileImportCommand;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.log4j.Log4j2;
|
||||||
|
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
|
||||||
|
@Log4j2
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class FileImportProducer {
|
||||||
|
private final RabbitTemplate rabbitTemplate;
|
||||||
|
|
||||||
|
@Value("${queues.file-import}")
|
||||||
|
private String fileImportQueue;
|
||||||
|
|
||||||
|
public void sendFileImportCommand(FileImportCommand command) {
|
||||||
|
rabbitTemplate.convertAndSend(fileImportQueue, command);
|
||||||
|
log.info("Sent file import command: {}", command);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,151 @@
|
|||||||
|
package com.magamochi.content.service;
|
||||||
|
|
||||||
|
import static java.util.Objects.isNull;
|
||||||
|
import static org.apache.commons.lang3.StringUtils.isBlank;
|
||||||
|
|
||||||
|
import com.magamochi.catalog.service.MangaContentProviderService;
|
||||||
|
import com.magamochi.catalog.service.MangaResolutionService;
|
||||||
|
import com.magamochi.common.exception.UnprocessableException;
|
||||||
|
import com.magamochi.common.model.enumeration.ContentType;
|
||||||
|
import com.magamochi.content.model.entity.MangaContentImage;
|
||||||
|
import com.magamochi.content.model.repository.MangaContentImageRepository;
|
||||||
|
import com.magamochi.content.queue.command.FileImportCommand;
|
||||||
|
import com.magamochi.content.queue.producer.FileImportProducer;
|
||||||
|
import com.magamochi.image.service.ImageFetchService;
|
||||||
|
import com.magamochi.image.service.ImageService;
|
||||||
|
import com.magamochi.image.service.S3Service;
|
||||||
|
import com.magamochi.ingestion.service.ContentProviderService;
|
||||||
|
import jakarta.validation.constraints.NotNull;
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.zip.ZipEntry;
|
||||||
|
import java.util.zip.ZipInputStream;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.log4j.Log4j2;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
|
||||||
|
@Log4j2
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class ContentImportService {
|
||||||
|
private final ContentProviderService contentProviderService;
|
||||||
|
private final MangaResolutionService mangaResolutionService;
|
||||||
|
private final MangaContentProviderService mangaContentProviderService;
|
||||||
|
private final ContentIngestService contentIngestService;
|
||||||
|
private final ImageFetchService imageFetchService;
|
||||||
|
private final S3Service s3Service;
|
||||||
|
|
||||||
|
private final FileImportProducer fileImportProducer;
|
||||||
|
private final MangaContentImageRepository mangaContentImageRepository;
|
||||||
|
private final ImageService imageService;
|
||||||
|
|
||||||
|
public void importFiles(String malId, String aniListId, @NotNull List<MultipartFile> files) {
|
||||||
|
if (isBlank(malId) && isBlank(aniListId)) {
|
||||||
|
throw new UnprocessableException("Either MyAnimeList or AniList IDs are required.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (files.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var manga =
|
||||||
|
mangaResolutionService.findOrCreateManga(
|
||||||
|
isBlank(aniListId) ? null : Long.parseLong(aniListId),
|
||||||
|
isBlank(malId) ? null : Long.parseLong(malId));
|
||||||
|
var contentProvider = contentProviderService.findManualImportContentProvider();
|
||||||
|
|
||||||
|
var mangaContentProvider = mangaContentProviderService.findOrCreate(manga, contentProvider);
|
||||||
|
|
||||||
|
var sortedFiles = files.stream().sorted(Comparator.comparing(MultipartFile::getName)).toList();
|
||||||
|
|
||||||
|
sortedFiles.forEach(
|
||||||
|
file -> {
|
||||||
|
try {
|
||||||
|
var filename =
|
||||||
|
s3Service.uploadFile(
|
||||||
|
file.getBytes(),
|
||||||
|
file.getContentType(),
|
||||||
|
"temp/import/" + file.getOriginalFilename());
|
||||||
|
log.info("Temp file uploaded to S3: {}", filename);
|
||||||
|
|
||||||
|
fileImportProducer.sendFileImportCommand(
|
||||||
|
new FileImportCommand(mangaContentProvider.getId(), filename));
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new UnprocessableException("Failed to upload file to S3");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public void importFile(Long mangaContentProviderId, String filename) {
|
||||||
|
var mangaContent =
|
||||||
|
contentIngestService.ingest(
|
||||||
|
mangaContentProviderId,
|
||||||
|
removeImportPrefix(removeFileExtension(filename)),
|
||||||
|
null,
|
||||||
|
"en-US");
|
||||||
|
|
||||||
|
try {
|
||||||
|
var is = s3Service.getFileStream(filename);
|
||||||
|
var zis = new ZipInputStream(is);
|
||||||
|
|
||||||
|
ZipEntry entry;
|
||||||
|
var position = 0;
|
||||||
|
|
||||||
|
while ((entry = zis.getNextEntry()) != null) {
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var os = new ByteArrayOutputStream();
|
||||||
|
zis.transferTo(os);
|
||||||
|
var bytes = os.toByteArray();
|
||||||
|
|
||||||
|
var imageId = imageFetchService.uploadImage(bytes, null, ContentType.CONTENT_IMAGE);
|
||||||
|
var image = imageService.find(imageId);
|
||||||
|
|
||||||
|
mangaContentImageRepository.save(
|
||||||
|
MangaContentImage.builder()
|
||||||
|
.image(image)
|
||||||
|
.mangaContent(mangaContent)
|
||||||
|
.position(position)
|
||||||
|
.build());
|
||||||
|
|
||||||
|
zis.closeEntry();
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
mangaContent.setDownloaded(true);
|
||||||
|
s3Service.deleteObjects(Set.of(filename));
|
||||||
|
}
|
||||||
|
|
||||||
|
private String removeFileExtension(String filename) {
|
||||||
|
if (isBlank(filename)) {
|
||||||
|
return filename;
|
||||||
|
}
|
||||||
|
|
||||||
|
int lastDotIndex = filename.lastIndexOf('.');
|
||||||
|
|
||||||
|
// No dot, or dot is the first character (like .gitignore)
|
||||||
|
if (lastDotIndex <= 0) {
|
||||||
|
return filename;
|
||||||
|
}
|
||||||
|
|
||||||
|
return filename.substring(0, lastDotIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String removeImportPrefix(String path) {
|
||||||
|
if (isNull(path)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return path.replace("temp/import/", "");
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,6 +1,7 @@
|
|||||||
package com.magamochi.content.service;
|
package com.magamochi.content.service;
|
||||||
|
|
||||||
import static java.util.Objects.isNull;
|
import static java.util.Objects.isNull;
|
||||||
|
import static java.util.Objects.nonNull;
|
||||||
|
|
||||||
import com.magamochi.catalog.service.LanguageService;
|
import com.magamochi.catalog.service.LanguageService;
|
||||||
import com.magamochi.catalog.service.MangaContentProviderService;
|
import com.magamochi.catalog.service.MangaContentProviderService;
|
||||||
@ -34,22 +35,24 @@ public class ContentIngestService {
|
|||||||
private final ImageFetchProducer imageFetchProducer;
|
private final ImageFetchProducer imageFetchProducer;
|
||||||
private final ImageService imageService;
|
private final ImageService imageService;
|
||||||
|
|
||||||
public void ingest(
|
public MangaContent ingest(
|
||||||
long mangaContentProviderId,
|
long mangaContentProviderId,
|
||||||
@NotBlank String title,
|
@NotBlank String title,
|
||||||
@NotBlank String url,
|
String url,
|
||||||
@NotBlank String languageCode) {
|
@NotBlank String languageCode) {
|
||||||
log.info("Ingesting Manga Content ({}) for provider {}", title, mangaContentProviderId);
|
log.info("Ingesting Manga Content ({}) for provider {}", title, mangaContentProviderId);
|
||||||
|
|
||||||
var mangaContentProvider = mangaContentProviderService.find(mangaContentProviderId);
|
var mangaContentProvider = mangaContentProviderService.find(mangaContentProviderId);
|
||||||
|
|
||||||
|
if (nonNull(url)) {
|
||||||
if (mangaContentRepository.existsByMangaContentProvider_IdAndUrlIgnoreCase(
|
if (mangaContentRepository.existsByMangaContentProvider_IdAndUrlIgnoreCase(
|
||||||
mangaContentProvider.getId(), url)) {
|
mangaContentProvider.getId(), url)) {
|
||||||
log.info(
|
log.info(
|
||||||
"Manga Content ({}) for provider {} already exists. Skipped.",
|
"Manga Content ({}) for provider {} already exists. Skipped.",
|
||||||
title,
|
title,
|
||||||
mangaContentProviderId);
|
mangaContentProviderId);
|
||||||
return;
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var language = languageService.find(languageCode);
|
var language = languageService.find(languageCode);
|
||||||
@ -68,6 +71,8 @@ public class ContentIngestService {
|
|||||||
title,
|
title,
|
||||||
mangaContentProviderId,
|
mangaContentProviderId,
|
||||||
mangaContent.getId());
|
mangaContent.getId());
|
||||||
|
|
||||||
|
return mangaContent;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
|
|||||||
@ -6,15 +6,8 @@ import com.magamochi.model.dto.ImportRequestDTO;
|
|||||||
// import com.magamochi.service.MangaImportService;
|
// import com.magamochi.service.MangaImportService;
|
||||||
import com.magamochi.service.ProviderManualMangaImportService;
|
import com.magamochi.service.ProviderManualMangaImportService;
|
||||||
import io.swagger.v3.oas.annotations.Operation;
|
import io.swagger.v3.oas.annotations.Operation;
|
||||||
import io.swagger.v3.oas.annotations.Parameter;
|
|
||||||
import io.swagger.v3.oas.annotations.media.Content;
|
|
||||||
import io.swagger.v3.oas.annotations.media.Schema;
|
|
||||||
import jakarta.validation.constraints.NotBlank;
|
|
||||||
import jakarta.validation.constraints.NotNull;
|
|
||||||
import java.util.List;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.springframework.web.bind.annotation.*;
|
import org.springframework.web.bind.annotation.*;
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequestMapping("/manga/import")
|
@RequestMapping("/manga/import")
|
||||||
@ -34,29 +27,4 @@ public class MangaImportController {
|
|||||||
return DefaultResponseDTO.ok(
|
return DefaultResponseDTO.ok(
|
||||||
providerManualMangaImportService.importFromProvider(providerId, requestDTO));
|
providerManualMangaImportService.importFromProvider(providerId, requestDTO));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Operation(
|
|
||||||
summary = "Upload multiple files",
|
|
||||||
description = "Accepts multiple files via multipart/form-data and processes them.",
|
|
||||||
tags = {"Manga Import"},
|
|
||||||
operationId = "importMultipleFiles")
|
|
||||||
@PostMapping(
|
|
||||||
value = "/upload",
|
|
||||||
consumes = {"multipart/form-data"})
|
|
||||||
public DefaultResponseDTO<Void> uploadMultipleFiles(
|
|
||||||
@RequestPart("malId") @NotBlank String malId,
|
|
||||||
@Parameter(
|
|
||||||
description = "List of files to upload",
|
|
||||||
required = true,
|
|
||||||
content =
|
|
||||||
@Content(
|
|
||||||
mediaType = "multipart/form-data",
|
|
||||||
schema = @Schema(type = "array", format = "binary")))
|
|
||||||
@RequestPart("files")
|
|
||||||
@NotNull
|
|
||||||
List<MultipartFile> files) {
|
|
||||||
// mangaImportService.importMangaFiles(malId, files);
|
|
||||||
|
|
||||||
return DefaultResponseDTO.ok().build();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -37,19 +37,26 @@ public class ImageFetchService {
|
|||||||
|
|
||||||
var imageBytes = response.body();
|
var imageBytes = response.body();
|
||||||
|
|
||||||
var fileContentType = resolveContentType(response, imageBytes);
|
return uploadImage(imageBytes, response, contentType);
|
||||||
|
|
||||||
var fileHash = computeHash(imageBytes);
|
|
||||||
|
|
||||||
return imageManagerService.upload(
|
|
||||||
imageBytes, fileContentType, contentType.name().toLowerCase(), fileHash);
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("Failed to fetch image from URL: {}", imageUrl, e);
|
log.error("Failed to fetch image from URL: {}", imageUrl, e);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public UUID uploadImage(
|
||||||
|
byte[] imageBytes, HttpResponse<byte[]> httpResponse, ContentType contentType)
|
||||||
|
throws NoSuchAlgorithmException {
|
||||||
|
var fileContentType = resolveContentType(httpResponse, imageBytes);
|
||||||
|
|
||||||
|
var fileHash = computeHash(imageBytes);
|
||||||
|
|
||||||
|
return imageManagerService.upload(
|
||||||
|
imageBytes, fileContentType, contentType.name().toLowerCase(), fileHash);
|
||||||
|
}
|
||||||
|
|
||||||
private String resolveContentType(HttpResponse<byte[]> response, byte[] fileBytes) {
|
private String resolveContentType(HttpResponse<byte[]> response, byte[] fileBytes) {
|
||||||
|
if (nonNull(response)) {
|
||||||
var headerType =
|
var headerType =
|
||||||
response
|
response
|
||||||
.headers()
|
.headers()
|
||||||
@ -60,6 +67,7 @@ public class ImageFetchService {
|
|||||||
if (nonNull(headerType) && headerType.startsWith("image/")) {
|
if (nonNull(headerType) && headerType.startsWith("image/")) {
|
||||||
return headerType;
|
return headerType;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return tika.detect(fileBytes);
|
return tika.detect(fileBytes);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,6 +1,9 @@
|
|||||||
package com.magamochi.ingestion.model.repository;
|
package com.magamochi.ingestion.model.repository;
|
||||||
|
|
||||||
import com.magamochi.ingestion.model.entity.ContentProvider;
|
import com.magamochi.ingestion.model.entity.ContentProvider;
|
||||||
|
import java.util.Optional;
|
||||||
import org.springframework.data.jpa.repository.JpaRepository;
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
|
|
||||||
public interface ContentProviderRepository extends JpaRepository<ContentProvider, Long> {}
|
public interface ContentProviderRepository extends JpaRepository<ContentProvider, Long> {
|
||||||
|
Optional<ContentProvider> findByNameIgnoreCase(String name);
|
||||||
|
}
|
||||||
|
|||||||
@ -5,11 +5,11 @@ import static java.util.Objects.isNull;
|
|||||||
import com.google.common.util.concurrent.RateLimiter;
|
import com.google.common.util.concurrent.RateLimiter;
|
||||||
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
||||||
import com.magamochi.client.MangaDexClient;
|
import com.magamochi.client.MangaDexClient;
|
||||||
|
import com.magamochi.common.ContentProviders;
|
||||||
import com.magamochi.common.exception.UnprocessableException;
|
import com.magamochi.common.exception.UnprocessableException;
|
||||||
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
|
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
|
||||||
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
|
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
|
||||||
import com.magamochi.ingestion.providers.ContentProvider;
|
import com.magamochi.ingestion.providers.ContentProvider;
|
||||||
import com.magamochi.ingestion.providers.ContentProviders;
|
|
||||||
import com.magamochi.ingestion.providers.ManualImportContentProvider;
|
import com.magamochi.ingestion.providers.ManualImportContentProvider;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.stream.IntStream;
|
import java.util.stream.IntStream;
|
||||||
|
|||||||
@ -1,11 +1,11 @@
|
|||||||
package com.magamochi.ingestion.providers.impl;
|
package com.magamochi.ingestion.providers.impl;
|
||||||
|
|
||||||
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
||||||
|
import com.magamochi.common.ContentProviders;
|
||||||
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
|
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
|
||||||
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
|
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
|
||||||
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
|
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
|
||||||
import com.magamochi.ingestion.providers.ContentProvider;
|
import com.magamochi.ingestion.providers.ContentProvider;
|
||||||
import com.magamochi.ingestion.providers.ContentProviders;
|
|
||||||
import com.magamochi.ingestion.providers.PagedContentProvider;
|
import com.magamochi.ingestion.providers.PagedContentProvider;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|||||||
@ -3,11 +3,11 @@ package com.magamochi.ingestion.providers.impl;
|
|||||||
import static java.util.Objects.nonNull;
|
import static java.util.Objects.nonNull;
|
||||||
|
|
||||||
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
||||||
|
import com.magamochi.common.ContentProviders;
|
||||||
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
|
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
|
||||||
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
|
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
|
||||||
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
|
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
|
||||||
import com.magamochi.ingestion.providers.ContentProvider;
|
import com.magamochi.ingestion.providers.ContentProvider;
|
||||||
import com.magamochi.ingestion.providers.ContentProviders;
|
|
||||||
import com.magamochi.ingestion.providers.PagedContentProvider;
|
import com.magamochi.ingestion.providers.PagedContentProvider;
|
||||||
import com.magamochi.ingestion.service.FlareService;
|
import com.magamochi.ingestion.service.FlareService;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|||||||
@ -4,11 +4,11 @@ import static java.util.Objects.isNull;
|
|||||||
import static org.apache.commons.lang3.StringUtils.isBlank;
|
import static org.apache.commons.lang3.StringUtils.isBlank;
|
||||||
|
|
||||||
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
import com.magamochi.catalog.model.entity.MangaContentProvider;
|
||||||
|
import com.magamochi.common.ContentProviders;
|
||||||
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
|
import com.magamochi.ingestion.model.dto.ContentImageInfoDTO;
|
||||||
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
|
import com.magamochi.ingestion.model.dto.ContentInfoDTO;
|
||||||
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
|
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
|
||||||
import com.magamochi.ingestion.providers.ContentProvider;
|
import com.magamochi.ingestion.providers.ContentProvider;
|
||||||
import com.magamochi.ingestion.providers.ContentProviders;
|
|
||||||
import com.magamochi.ingestion.providers.PagedContentProvider;
|
import com.magamochi.ingestion.providers.PagedContentProvider;
|
||||||
import com.magamochi.ingestion.service.FlareService;
|
import com.magamochi.ingestion.service.FlareService;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
|||||||
@ -2,6 +2,7 @@ package com.magamochi.ingestion.service;
|
|||||||
|
|
||||||
import static java.util.Objects.nonNull;
|
import static java.util.Objects.nonNull;
|
||||||
|
|
||||||
|
import com.magamochi.common.ContentProviders;
|
||||||
import com.magamochi.common.exception.NotFoundException;
|
import com.magamochi.common.exception.NotFoundException;
|
||||||
import com.magamochi.ingestion.model.dto.ContentProviderListDTO;
|
import com.magamochi.ingestion.model.dto.ContentProviderListDTO;
|
||||||
import com.magamochi.ingestion.model.entity.ContentProvider;
|
import com.magamochi.ingestion.model.entity.ContentProvider;
|
||||||
@ -32,4 +33,10 @@ public class ContentProviderService {
|
|||||||
new NotFoundException(
|
new NotFoundException(
|
||||||
"Content Provider not found (ID: " + contentProviderId + ")."));
|
"Content Provider not found (ID: " + contentProviderId + ")."));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ContentProvider findManualImportContentProvider() {
|
||||||
|
return contentProviderRepository
|
||||||
|
.findByNameIgnoreCase(ContentProviders.MANUAL_IMPORT)
|
||||||
|
.orElseThrow(() -> new NotFoundException("Manual Import Content Provider not found"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,438 +0,0 @@
|
|||||||
// package com.magamochi.service;
|
|
||||||
//
|
|
||||||
// import static java.util.Objects.isNull;
|
|
||||||
// import static java.util.Objects.nonNull;
|
|
||||||
//
|
|
||||||
// import com.google.common.util.concurrent.RateLimiter;
|
|
||||||
// import com.magamochi.catalog.model.entity.Genre;
|
|
||||||
// import com.magamochi.catalog.model.repository.GenreRepository;
|
|
||||||
// import com.magamochi.catalog.client.AniListClient;
|
|
||||||
// import com.magamochi.catalog.client.JikanClient;
|
|
||||||
// import com.magamochi.common.exception.NotFoundException;
|
|
||||||
// import com.magamochi.ingestion.model.entity.ContentProvider;
|
|
||||||
// import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
|
|
||||||
// import com.magamochi.model.entity.*;
|
|
||||||
// import com.magamochi.model.repository.*;
|
|
||||||
// import com.magamochi.catalog.util.DoubleUtil;
|
|
||||||
// import java.io.*;
|
|
||||||
// import java.net.URI;
|
|
||||||
// import java.net.URISyntaxException;
|
|
||||||
// import java.net.URL;
|
|
||||||
// import java.time.OffsetDateTime;
|
|
||||||
// import java.time.ZoneOffset;
|
|
||||||
// import java.util.ArrayList;
|
|
||||||
// import java.util.Comparator;
|
|
||||||
// import java.util.List;
|
|
||||||
// import java.util.stream.IntStream;
|
|
||||||
// import java.util.zip.ZipEntry;
|
|
||||||
// import java.util.zip.ZipInputStream;
|
|
||||||
// import lombok.RequiredArgsConstructor;
|
|
||||||
// import lombok.extern.log4j.Log4j2;
|
|
||||||
// import org.apache.commons.lang3.StringUtils;
|
|
||||||
// import org.springframework.stereotype.Service;
|
|
||||||
// import org.springframework.web.multipart.MultipartFile;
|
|
||||||
//
|
|
||||||
// @Log4j2
|
|
||||||
//// @Service
|
|
||||||
// @RequiredArgsConstructor
|
|
||||||
// public class MangaImportService {
|
|
||||||
// private final ProviderService providerService;
|
|
||||||
// private final MangaCreationService mangaCreationService;
|
|
||||||
// private final ImageService imageService;
|
|
||||||
// private final LanguageService languageService;
|
|
||||||
//
|
|
||||||
// private final GenreRepository genreRepository;
|
|
||||||
// private final MangaGenreRepository mangaGenreRepository;
|
|
||||||
// private final MangaContentProviderRepository mangaContentProviderRepository;
|
|
||||||
// private final AuthorRepository authorRepository;
|
|
||||||
// private final MangaAuthorRepository mangaAuthorRepository;
|
|
||||||
// private final MangaChapterRepository mangaChapterRepository;
|
|
||||||
// private final MangaRepository mangaRepository;
|
|
||||||
//
|
|
||||||
// private final JikanClient jikanClient;
|
|
||||||
// private final AniListClient aniListClient;
|
|
||||||
// private final MangaChapterImageRepository mangaChapterImageRepository;
|
|
||||||
// private final MangaAlternativeTitlesRepository mangaAlternativeTitlesRepository;
|
|
||||||
//
|
|
||||||
// private final RateLimiter jikanRateLimiter;
|
|
||||||
//
|
|
||||||
// public void importMangaFiles(String malId, List<MultipartFile> files) {
|
|
||||||
// log.info("Importing manga files for MAL ID {}", malId);
|
|
||||||
// var provider = providerService.getOrCreateProvider("Manual Import", false);
|
|
||||||
//
|
|
||||||
// jikanRateLimiter.acquire();
|
|
||||||
// var mangaData = jikanClient.getMangaById(Long.parseLong(malId));
|
|
||||||
//
|
|
||||||
// var mangaProvider = getOrCreateMangaProvider(mangaData.data().title(), provider);
|
|
||||||
//
|
|
||||||
// var sortedFiles =
|
|
||||||
// files.stream().sorted(Comparator.comparing(MultipartFile::getName)).toList();
|
|
||||||
//
|
|
||||||
// IntStream.rangeClosed(1, sortedFiles.size())
|
|
||||||
// .forEach(
|
|
||||||
// fileIndex -> {
|
|
||||||
// var file = sortedFiles.get(fileIndex - 1);
|
|
||||||
// log.info(
|
|
||||||
// "Importing file {}/{}: {}, for Mangá {}",
|
|
||||||
// fileIndex,
|
|
||||||
// sortedFiles.size(),
|
|
||||||
// file.getOriginalFilename(),
|
|
||||||
// mangaProvider.getManga().getTitle());
|
|
||||||
//
|
|
||||||
// var chapter =
|
|
||||||
// persistMangaChapter(
|
|
||||||
// mangaProvider,
|
|
||||||
// new ContentProviderMangaChapterResponseDTO(
|
|
||||||
// removeFileExtension(file.getOriginalFilename()),
|
|
||||||
// "manual_" + file.getOriginalFilename(),
|
|
||||||
// file.getOriginalFilename(),
|
|
||||||
// "en-US"));
|
|
||||||
//
|
|
||||||
// List<MangaChapterImage> allChapterImages = new ArrayList<>();
|
|
||||||
// try (InputStream is = file.getInputStream();
|
|
||||||
// ZipInputStream zis = new ZipInputStream(is)) {
|
|
||||||
// ZipEntry entry;
|
|
||||||
// var position = 0;
|
|
||||||
//
|
|
||||||
// while ((entry = zis.getNextEntry()) != null) {
|
|
||||||
// if (entry.isDirectory()) {
|
|
||||||
// continue;
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// var os = new ByteArrayOutputStream();
|
|
||||||
// zis.transferTo(os);
|
|
||||||
// var bytes = os.toByteArray();
|
|
||||||
//
|
|
||||||
// var image =
|
|
||||||
// imageService.uploadImage(bytes, "image/jpeg", "chapter/" + chapter.getId());
|
|
||||||
//
|
|
||||||
// var chapterImage =
|
|
||||||
// MangaChapterImage.builder()
|
|
||||||
// .position(position++)
|
|
||||||
// .image(image)
|
|
||||||
// .mangaChapter(chapter)
|
|
||||||
// .build();
|
|
||||||
//
|
|
||||||
// allChapterImages.add(chapterImage);
|
|
||||||
// zis.closeEntry();
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// log.info("Chapter images added for chapter {}", chapter.getTitle());
|
|
||||||
// } catch (IOException e) {
|
|
||||||
// throw new RuntimeException(e);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// mangaChapterImageRepository.saveAll(allChapterImages);
|
|
||||||
// chapter.setDownloaded(true);
|
|
||||||
// mangaChapterRepository.save(chapter);
|
|
||||||
// });
|
|
||||||
//
|
|
||||||
// log.info("Import manga files for MAL ID {} completed.", malId);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// public void updateMangaData(Long mangaId) {
|
|
||||||
// var manga =
|
|
||||||
// mangaRepository
|
|
||||||
// .findById(mangaId)
|
|
||||||
// .orElseThrow(() -> new NotFoundException("Manga not found for ID: " + mangaId));
|
|
||||||
//
|
|
||||||
// updateMangaData(manga);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// public void updateMangaData(Manga manga) {
|
|
||||||
// log.info("Updating manga {}", manga.getTitle());
|
|
||||||
//
|
|
||||||
// if (nonNull(manga.getMalId())) {
|
|
||||||
// try {
|
|
||||||
// updateFromJikan(manga);
|
|
||||||
// return;
|
|
||||||
// } catch (Exception e) {
|
|
||||||
// log.warn(
|
|
||||||
// "Error updating manga data from Jikan for manga {}. Trying AniList... Error: {}",
|
|
||||||
// manga.getTitle(),
|
|
||||||
// e.getMessage());
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// if (nonNull(manga.getAniListId())) {
|
|
||||||
// try {
|
|
||||||
// updateFromAniList(manga);
|
|
||||||
// return;
|
|
||||||
// } catch (Exception e) {
|
|
||||||
// log.warn(
|
|
||||||
// "Error updating manga data from AniList for manga {}. Error: {}",
|
|
||||||
// manga.getTitle(),
|
|
||||||
// e.getMessage());
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// log.warn(
|
|
||||||
// "Could not update manga data for {}. No provider data available/found.",
|
|
||||||
// manga.getTitle());
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private void updateFromJikan(Manga manga) throws IOException, URISyntaxException {
|
|
||||||
// jikanRateLimiter.acquire();
|
|
||||||
// var mangaData = jikanClient.getMangaById(manga.getMalId());
|
|
||||||
//
|
|
||||||
// manga.setSynopsis(mangaData.data().synopsis());
|
|
||||||
// manga.setStatus(mangaData.data().status());
|
|
||||||
// manga.setScore(DoubleUtil.round((double) mangaData.data().score(), 2));
|
|
||||||
// manga.setPublishedFrom(mangaData.data().published().from());
|
|
||||||
// manga.setPublishedTo(mangaData.data().published().to());
|
|
||||||
// manga.setChapterCount(mangaData.data().chapters());
|
|
||||||
//
|
|
||||||
// var authors =
|
|
||||||
// mangaData.data().authors().stream()
|
|
||||||
// .map(
|
|
||||||
// authorData ->
|
|
||||||
// authorRepository
|
|
||||||
// .findByMalId(authorData.mal_id())
|
|
||||||
// .orElseGet(
|
|
||||||
// () ->
|
|
||||||
// authorRepository.save(
|
|
||||||
// Author.builder()
|
|
||||||
// .malId(authorData.mal_id())
|
|
||||||
// .name(authorData.name())
|
|
||||||
// .build())))
|
|
||||||
// .toList();
|
|
||||||
//
|
|
||||||
// updateMangaAuthors(manga, authors);
|
|
||||||
//
|
|
||||||
// var genres =
|
|
||||||
// mangaData.data().genres().stream()
|
|
||||||
// .map(
|
|
||||||
// genreData ->
|
|
||||||
// genreRepository
|
|
||||||
// .findByMalId(genreData.mal_id())
|
|
||||||
// .orElseGet(
|
|
||||||
// () ->
|
|
||||||
// genreRepository.save(
|
|
||||||
// Genre.builder()
|
|
||||||
// .malId(genreData.mal_id())
|
|
||||||
// .name(genreData.name())
|
|
||||||
// .build())))
|
|
||||||
// .toList();
|
|
||||||
//
|
|
||||||
// updateMangaGenres(manga, genres);
|
|
||||||
//
|
|
||||||
// if (isNull(manga.getCoverImage())) {
|
|
||||||
// downloadCoverImage(manga, mangaData.data().images().jpg().large_image_url());
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// var mangaEntity = mangaRepository.save(manga);
|
|
||||||
// var alternativeTitles =
|
|
||||||
// mangaData.data().title_synonyms().stream()
|
|
||||||
// .map(at -> MangaAlternativeTitle.builder().manga(mangaEntity).title(at).build())
|
|
||||||
// .toList();
|
|
||||||
// mangaAlternativeTitlesRepository.saveAll(alternativeTitles);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private void updateFromAniList(Manga manga) throws IOException, URISyntaxException {
|
|
||||||
// var query =
|
|
||||||
// """
|
|
||||||
// query ($id: Int) {
|
|
||||||
// Media (id: $id, type: MANGA) {
|
|
||||||
// startDate { year month day }
|
|
||||||
// endDate { year month day }
|
|
||||||
// description
|
|
||||||
// status
|
|
||||||
// averageScore
|
|
||||||
// chapters
|
|
||||||
// coverImage { large }
|
|
||||||
// genres
|
|
||||||
// staff {
|
|
||||||
// edges {
|
|
||||||
// role
|
|
||||||
// node {
|
|
||||||
// name {
|
|
||||||
// full
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// """;
|
|
||||||
// var request =
|
|
||||||
// new AniListClient.GraphQLRequest(
|
|
||||||
// query, new AniListClient.GraphQLRequest.Variables(manga.getAniListId()));
|
|
||||||
// var media = aniListClient.getManga(request).data().Media();
|
|
||||||
//
|
|
||||||
// manga.setSynopsis(media.description());
|
|
||||||
// manga.setStatus(mapAniListStatus(media.status()));
|
|
||||||
// manga.setScore(DoubleUtil.round((double) media.averageScore() / 10, 2)); // 0-100 -> 0-10
|
|
||||||
// manga.setPublishedFrom(convertFuzzyDate(media.startDate()));
|
|
||||||
// manga.setPublishedTo(convertFuzzyDate(media.endDate()));
|
|
||||||
// manga.setChapterCount(media.chapters());
|
|
||||||
//
|
|
||||||
// var authors =
|
|
||||||
// media.staff().edges().stream()
|
|
||||||
// .filter(edge -> isAuthorRole(edge.role()))
|
|
||||||
// .map(edge -> edge.node().name().full())
|
|
||||||
// .distinct()
|
|
||||||
// .map(
|
|
||||||
// name ->
|
|
||||||
// authorRepository
|
|
||||||
// .findByName(name)
|
|
||||||
// .orElseGet(
|
|
||||||
// () -> authorRepository.save(Author.builder().name(name).build())))
|
|
||||||
// .toList();
|
|
||||||
//
|
|
||||||
// updateMangaAuthors(manga, authors);
|
|
||||||
//
|
|
||||||
// var genres =
|
|
||||||
// media.genres().stream()
|
|
||||||
// .map(
|
|
||||||
// name ->
|
|
||||||
// genreRepository
|
|
||||||
// .findByName(name)
|
|
||||||
// .orElseGet(() ->
|
|
||||||
// genreRepository.save(Genre.builder().name(name).build())))
|
|
||||||
// .toList();
|
|
||||||
//
|
|
||||||
// updateMangaGenres(manga, genres);
|
|
||||||
//
|
|
||||||
// if (isNull(manga.getCoverImage())) {
|
|
||||||
// downloadCoverImage(manga, media.coverImage().large());
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// mangaRepository.save(manga);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private boolean isAuthorRole(String role) {
|
|
||||||
// return role.equalsIgnoreCase("Story & Art")
|
|
||||||
// || role.equalsIgnoreCase("Story")
|
|
||||||
// || role.equalsIgnoreCase("Art");
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private String mapAniListStatus(String status) {
|
|
||||||
// return switch (status) {
|
|
||||||
// case "RELEASING" -> "Publishing";
|
|
||||||
// case "FINISHED" -> "Finished";
|
|
||||||
// case "NOT_YET_RELEASED" -> "Not yet published";
|
|
||||||
// default -> "Unknown";
|
|
||||||
// };
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private OffsetDateTime convertFuzzyDate(AniListClient.MangaResponse.Manga.FuzzyDate date) {
|
|
||||||
// if (isNull(date) || isNull(date.year())) {
|
|
||||||
// return null;
|
|
||||||
// }
|
|
||||||
// return OffsetDateTime.of(
|
|
||||||
// date.year(),
|
|
||||||
// isNull(date.month()) ? 1 : date.month(),
|
|
||||||
// isNull(date.day()) ? 1 : date.day(),
|
|
||||||
// 0,
|
|
||||||
// 0,
|
|
||||||
// 0,
|
|
||||||
// 0,
|
|
||||||
// ZoneOffset.UTC);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private void updateMangaAuthors(Manga manga, List<Author> authors) {
|
|
||||||
// var mangaAuthors =
|
|
||||||
// authors.stream()
|
|
||||||
// .map(
|
|
||||||
// author ->
|
|
||||||
// mangaAuthorRepository
|
|
||||||
// .findByMangaAndAuthor(manga, author)
|
|
||||||
// .orElseGet(
|
|
||||||
// () ->
|
|
||||||
// mangaAuthorRepository.save(
|
|
||||||
// MangaAuthor.builder().manga(manga).author(author).build())))
|
|
||||||
// .toList();
|
|
||||||
// manga.setMangaAuthors(mangaAuthors);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private void updateMangaGenres(Manga manga, List<Genre> genres) {
|
|
||||||
// var mangaGenres =
|
|
||||||
// genres.stream()
|
|
||||||
// .map(
|
|
||||||
// genre ->
|
|
||||||
// mangaGenreRepository
|
|
||||||
// .findByMangaAndGenre(manga, genre)
|
|
||||||
// .orElseGet(
|
|
||||||
// () ->
|
|
||||||
// mangaGenreRepository.save(
|
|
||||||
// MangaGenre.builder().manga(manga).genre(genre).build())))
|
|
||||||
// .toList();
|
|
||||||
// manga.setMangaGenres(mangaGenres);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private void downloadCoverImage(Manga manga, String imageUrl)
|
|
||||||
// throws IOException, URISyntaxException {
|
|
||||||
// var inputStream =
|
|
||||||
// new BufferedInputStream(new URL(new URI(imageUrl).toASCIIString()).openStream());
|
|
||||||
//
|
|
||||||
// var bytes = inputStream.readAllBytes();
|
|
||||||
//
|
|
||||||
// inputStream.close();
|
|
||||||
// var image = imageService.uploadImage(bytes, "image/jpeg", "cover");
|
|
||||||
//
|
|
||||||
// manga.setCoverImage(image);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// public MangaChapter persistMangaChapter(
|
|
||||||
// MangaContentProvider mangaContentProvider, ContentProviderMangaChapterResponseDTO chapter) {
|
|
||||||
// var mangaChapter =
|
|
||||||
// mangaChapterRepository
|
|
||||||
// .findByMangaContentProviderAndUrlIgnoreCase(mangaContentProvider,
|
|
||||||
// chapter.url())
|
|
||||||
// .orElseGet(MangaChapter::new);
|
|
||||||
//
|
|
||||||
// mangaChapter.setMangaContentProvider(mangaContentProvider);
|
|
||||||
// mangaChapter.setTitle(chapter.title());
|
|
||||||
// mangaChapter.setUrl(chapter.url());
|
|
||||||
//
|
|
||||||
// var language = languageService.getOrThrow(chapter.languageCode());
|
|
||||||
// mangaChapter.setLanguage(language);
|
|
||||||
//
|
|
||||||
// if (nonNull(chapter.chapter())) {
|
|
||||||
// try {
|
|
||||||
// mangaChapter.setChapterNumber(Integer.parseInt(chapter.chapter()));
|
|
||||||
// } catch (NumberFormatException e) {
|
|
||||||
// log.warn(
|
|
||||||
// "Could not parse chapter number {} from manga {}",
|
|
||||||
// chapter.chapter(),
|
|
||||||
// mangaContentProvider.getManga().getTitle());
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// return mangaChapterRepository.save(mangaChapter);
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private MangaContentProvider getOrCreateMangaProvider(
|
|
||||||
// String title, ContentProvider contentProvider) {
|
|
||||||
// return mangaContentProviderRepository
|
|
||||||
// .findByMangaTitleIgnoreCaseAndContentProvider(title, contentProvider)
|
|
||||||
// .orElseGet(
|
|
||||||
// () -> {
|
|
||||||
// jikanRateLimiter.acquire();
|
|
||||||
// var manga = mangaCreationService.getOrCreateManga(title, "manual", contentProvider);
|
|
||||||
//
|
|
||||||
// return mangaContentProviderRepository.save(
|
|
||||||
// MangaContentProvider.builder()
|
|
||||||
// .manga(manga)
|
|
||||||
// .mangaTitle(manga.getTitle())
|
|
||||||
// .contentProvider(contentProvider)
|
|
||||||
// .url("manual")
|
|
||||||
// .build());
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// private String removeFileExtension(String filename) {
|
|
||||||
// if (StringUtils.isBlank(filename)) {
|
|
||||||
// return filename;
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// int lastDotIndex = filename.lastIndexOf('.');
|
|
||||||
//
|
|
||||||
// // No dot, or dot is the first character (like .gitignore)
|
|
||||||
// if (lastDotIndex <= 0) {
|
|
||||||
// return filename;
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// return filename.substring(0, lastDotIndex);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
@ -5,7 +5,6 @@ import com.magamochi.common.exception.NotFoundException;
|
|||||||
import com.magamochi.content.model.entity.MangaContent;
|
import com.magamochi.content.model.entity.MangaContent;
|
||||||
import com.magamochi.model.dto.*;
|
import com.magamochi.model.dto.*;
|
||||||
import com.magamochi.queue.MangaChapterDownloadProducer;
|
import com.magamochi.queue.MangaChapterDownloadProducer;
|
||||||
import com.magamochi.user.service.UserService;
|
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.log4j.Log4j2;
|
import lombok.extern.log4j.Log4j2;
|
||||||
@ -15,8 +14,6 @@ import org.springframework.stereotype.Service;
|
|||||||
@Service
|
@Service
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class OldMangaService {
|
public class OldMangaService {
|
||||||
private final UserService userService;
|
|
||||||
|
|
||||||
private final MangaContentProviderRepository mangaContentProviderRepository;
|
private final MangaContentProviderRepository mangaContentProviderRepository;
|
||||||
|
|
||||||
private final MangaChapterDownloadProducer mangaChapterDownloadProducer;
|
private final MangaChapterDownloadProducer mangaChapterDownloadProducer;
|
||||||
|
|||||||
@ -68,6 +68,7 @@ queues:
|
|||||||
provider-page-ingest: ${PROVIDER_PAGE_INGEST_QUEUE:mangamochi.provider.page.ingest}
|
provider-page-ingest: ${PROVIDER_PAGE_INGEST_QUEUE:mangamochi.provider.page.ingest}
|
||||||
image-fetch: ${IMAGE_FETCH_QUEUE:mangamochi.image.fetch}
|
image-fetch: ${IMAGE_FETCH_QUEUE:mangamochi.image.fetch}
|
||||||
manga-cover-update: ${MANGA_COVER_UDPATE_QUEUE:mangamochi.manga.cover.update}
|
manga-cover-update: ${MANGA_COVER_UDPATE_QUEUE:mangamochi.manga.cover.update}
|
||||||
|
file-import: ${FILE_IMPORT_QUEUE:mangamochi.file.import}
|
||||||
|
|
||||||
routing-key:
|
routing-key:
|
||||||
image-update: ${IMAGE_UPDATE_ROUTING_KEY:mangamochi.image.update}
|
image-update: ${IMAGE_UPDATE_ROUTING_KEY:mangamochi.image.update}
|
||||||
|
|||||||
@ -0,0 +1,6 @@
|
|||||||
|
ALTER TABLE content_providers
|
||||||
|
ALTER COLUMN url DROP NOT NULL;
|
||||||
|
|
||||||
|
INSERT INTO content_providers(name, url, active, supports_content_fetch, manual_import)
|
||||||
|
VALUES ('MangaDex', NULL, TRUE, TRUE, TRUE),
|
||||||
|
('Manual Import', NULL, TRUE, FALSE, FALSE);
|
||||||
2
src/main/resources/db/migration/V0005__MANGA_CONTENT.sql
Normal file
2
src/main/resources/db/migration/V0005__MANGA_CONTENT.sql
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE manga_contents
|
||||||
|
ALTER COLUMN url DROP NOT NULL;
|
||||||
3
src/main/resources/db/migration/V0006__MANGA_CONTENT.sql
Normal file
3
src/main/resources/db/migration/V0006__MANGA_CONTENT.sql
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE manga_content_provider
|
||||||
|
ALTER COLUMN url DROP NOT NULL,
|
||||||
|
ALTER COLUMN manga_title DROP NOT NULL;
|
||||||
Loading…
x
Reference in New Issue
Block a user