188 lines
8.4 KiB
Java
188 lines
8.4 KiB
Java
package com.magamochi.ingestion.service;
|
|
|
|
import com.magamochi.catalog.service.MangaContentProviderService;
|
|
import com.magamochi.common.queue.command.MangaContentImageIngestCommand;
|
|
import com.magamochi.common.queue.command.MangaContentIngestCommand;
|
|
import com.magamochi.common.queue.command.MangaIngestCommand;
|
|
import com.magamochi.content.service.ContentService;
|
|
import com.magamochi.ingestion.providers.ContentProviderFactory;
|
|
import com.magamochi.ingestion.providers.PagedContentProviderFactory;
|
|
import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand;
|
|
import com.magamochi.ingestion.queue.producer.MangaContentImageIngestProducer;
|
|
import com.magamochi.ingestion.queue.producer.MangaContentIngestProducer;
|
|
import com.magamochi.ingestion.queue.producer.MangaIngestProducer;
|
|
import com.magamochi.ingestion.queue.producer.ProviderPageIngestProducer;
|
|
import java.util.stream.IntStream;
|
|
import lombok.RequiredArgsConstructor;
|
|
import org.springframework.stereotype.Service;
|
|
|
|
@Service
|
|
@RequiredArgsConstructor
|
|
public class IngestionService {
|
|
private final ContentProviderService contentProviderService;
|
|
private final ContentService contentService;
|
|
private final MangaContentProviderService mangaContentProviderService;
|
|
|
|
private final ContentProviderFactory contentProviderFactory;
|
|
private final PagedContentProviderFactory pagedContentProviderFactory;
|
|
|
|
private final ProviderPageIngestProducer providerPageIngestProducer;
|
|
private final MangaIngestProducer mangaIngestProducer;
|
|
private final MangaContentIngestProducer mangaContentIngestProducer;
|
|
private final MangaContentImageIngestProducer mangaContentImageIngestProducer;
|
|
|
|
public void fetchContentProviderMangas(long contentProviderId) {
|
|
var contentProvider = contentProviderService.find(contentProviderId);
|
|
var pagedContentProvider =
|
|
pagedContentProviderFactory.getPagedContentProvider(contentProvider.getName());
|
|
var pages = pagedContentProvider.getTotalPages();
|
|
|
|
IntStream.rangeClosed(1, pages)
|
|
.forEach(
|
|
page ->
|
|
providerPageIngestProducer.sendProviderPageIngestCommand(
|
|
new ProviderPageIngestCommand(contentProvider.getId(), page)));
|
|
}
|
|
|
|
public void fetchProviderPageMangas(long providerId, int page) {
|
|
var contentProvider = contentProviderService.find(providerId);
|
|
var pagedContentProvider =
|
|
pagedContentProviderFactory.getPagedContentProvider(contentProvider.getName());
|
|
|
|
var mangas = pagedContentProvider.getMangasFromPage(page);
|
|
|
|
mangas.forEach(
|
|
manga ->
|
|
mangaIngestProducer.sendMangaIngestCommand(
|
|
new MangaIngestCommand(contentProvider.getId(), manga.title(), manga.url())));
|
|
}
|
|
|
|
public void fetchAllContentProviderMangas() {
|
|
var contentProviders = contentProviderService.getProviders(null);
|
|
contentProviders.providers().forEach(dto -> fetchContentProviderMangas(dto.id()));
|
|
}
|
|
|
|
public void fetchMangaContentProviderContentList(Long mangaContentProviderId) {
|
|
var mangaContentProvider = mangaContentProviderService.find(mangaContentProviderId);
|
|
|
|
var contentProvider =
|
|
contentProviderFactory.getContentProvider(
|
|
mangaContentProvider.getContentProvider().getName());
|
|
|
|
var availableChapters = contentProvider.getAvailableChapters(mangaContentProvider);
|
|
|
|
availableChapters.forEach(
|
|
content ->
|
|
mangaContentIngestProducer.sendMangaContentIngestCommand(
|
|
new MangaContentIngestCommand(
|
|
mangaContentProvider.getId(),
|
|
content.title(),
|
|
content.url(),
|
|
content.languageCode())));
|
|
}
|
|
|
|
public void fetchContent(Long mangaContentId) {
|
|
var mangaContent = contentService.find(mangaContentId);
|
|
var mangaContentProvider = mangaContent.getMangaContentProvider();
|
|
|
|
var contentProvider =
|
|
contentProviderFactory.getContentProvider(
|
|
mangaContentProvider.getContentProvider().getName());
|
|
|
|
var chapterImagesUrl = contentProvider.getContentImages(mangaContent.getUrl());
|
|
|
|
IntStream.range(0, chapterImagesUrl.size())
|
|
.forEach(
|
|
i -> {
|
|
var item = chapterImagesUrl.get(i);
|
|
|
|
var isLast = i == chapterImagesUrl.size() - 1;
|
|
|
|
mangaContentImageIngestProducer.sendMangaContentImageIngestCommand(
|
|
new MangaContentImageIngestCommand(
|
|
mangaContent.getId(), item.url(), item.position(), isLast));
|
|
});
|
|
}
|
|
|
|
// @Transactional
|
|
// public void fetchChapter(Long chapterId) {
|
|
//
|
|
// var retryConfig = retryRegistry.retry("ImageDownloadRetry").getRetryConfig();
|
|
//
|
|
// var chapterImages =
|
|
// chapterImagesUrls.entrySet().parallelStream()
|
|
// .map(
|
|
// entry -> {
|
|
// imageDownloadRateLimiter.acquire();
|
|
//
|
|
// try {
|
|
// var finalUrl = new
|
|
// URI(entry.getValue().trim()).toASCIIString().trim();
|
|
// var retry =
|
|
// Retry.of("image-download-" + chapterId + "-" +
|
|
// entry.getKey(), retryConfig);
|
|
//
|
|
// retry
|
|
// .getEventPublisher()
|
|
// .onRetry(
|
|
// event ->
|
|
// log.warn(
|
|
// "Retrying image download {}/{}
|
|
// for chapter {}. Attempt #{}. Error: {}",
|
|
// entry.getKey() + 1,
|
|
// chapterImagesUrls.size(),
|
|
// chapterId,
|
|
//
|
|
// event.getNumberOfRetryAttempts(),
|
|
//
|
|
// event.getLastThrowable().getMessage()));
|
|
//
|
|
// return retry.executeCheckedSupplier(
|
|
// () -> {
|
|
// var url = new URL(finalUrl);
|
|
// var connection = url.openConnection();
|
|
// connection.setConnectTimeout(5000);
|
|
// connection.setReadTimeout(5000);
|
|
//
|
|
// try (var inputStream =
|
|
// new
|
|
// BufferedInputStream(connection.getInputStream())) {
|
|
// var bytes = inputStream.readAllBytes();
|
|
//
|
|
// var image =
|
|
// oldImageService.uploadImage(
|
|
// bytes, "image/jpeg", "chapter/" +
|
|
// chapterId);
|
|
//
|
|
// log.info(
|
|
// "Downloaded image {}/{} for manga {} chapter
|
|
// {}: {}",
|
|
// entry.getKey() + 1,
|
|
// chapterImagesUrls.size(),
|
|
//
|
|
// chapter.getMangaContentProvider().getManga().getTitle(),
|
|
// chapterId,
|
|
// entry.getValue());
|
|
//
|
|
// return MangaContentImage.builder()
|
|
// .mangaContent(chapter)
|
|
// .position(entry.getKey())
|
|
// .image(image)
|
|
// .build();
|
|
// }
|
|
// });
|
|
// } catch (Throwable e) {
|
|
// throw new UnprocessableException(
|
|
// "Could not download image for chapter ID: " + chapterId,
|
|
// e);
|
|
// }
|
|
// })
|
|
// .toList();
|
|
//
|
|
// mangaChapterImageRepository.saveAll(chapterImages);
|
|
//
|
|
// chapter.setDownloaded(true);
|
|
// mangaContentRepository.save(chapter);
|
|
// }
|
|
}
|