mangamochi/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: content provider ingestion

Browse Source
This commit is contained in:
Rodrigo Verdiani 2026-03-17 20:37:25 -03:00
parent a6f01bba30
commit 5f66490dae
43 changed files with 295 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/com/magamochi/config/RabbitConfig.java → src/main/java/com/magamochi/common/config/RabbitConfig.java
View File

@ -1,4 +1,4 @@
package com.magamochi.config;
package com.magamochi.common.config;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.rabbit.connection.ConnectionFactory;
@ -10,6 +10,24 @@ import org.springframework.context.annotation.Configuration;
@Configuration
public class RabbitConfig {
@Value("${queues.manga-ingest}")
private String mangaIngestQueue;
@Value("${queues.provider-page-ingest}")
private String providerPageIngestQueue;
@Bean
public Queue mangaIngestQueue() {
return new Queue(mangaIngestQueue, false);
}
@Bean
public Queue providerPageIngestQueue() {
return new Queue(providerPageIngestQueue, false);
}
// TODO: remove unused queues
@Value("${rabbit-mq.queues.manga-data-update}")
private String mangaDataUpdateQueue;

6
src/main/java/com/magamochi/common/queue/command/MangaIngestCommand.java Normal file
View File

@ -0,0 +1,6 @@
package com.magamochi.common.queue.command;
import jakarta.validation.constraints.NotBlank;
public record MangaIngestCommand(
long providerId, @NotBlank String mangaTitle, @NotBlank String url) {}

28
src/main/java/com/magamochi/controller/ManagementController.java
View File

@ -2,12 +2,12 @@ package com.magamochi.controller;
import com.magamochi.client.NtfyClient;
import com.magamochi.common.dto.DefaultResponseDTO;
import com.magamochi.ingestion.task.IngestFromContentProvidersTask;
import com.magamochi.model.dto.UpdateMangaDataCommand;
import com.magamochi.model.repository.UserRepository;
import com.magamochi.queue.UpdateMangaDataProducer;
import com.magamochi.task.ImageCleanupTask;
import com.magamochi.task.MangaFollowUpdateTask;
import com.magamochi.task.UpdateMangaListTask;
import io.swagger.v3.oas.annotations.Operation;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
@ -16,7 +16,7 @@ import org.springframework.web.bind.annotation.*;
@RequestMapping("/management")
@RequiredArgsConstructor
public class ManagementController {
private final UpdateMangaListTask updateMangaListTask;
private final IngestFromContentProvidersTask ingestFromContentProvidersTask;
private final ImageCleanupTask imageCleanupTask;
private final MangaFollowUpdateTask mangaFollowUpdateTask;
private final UserRepository userRepository;
@ -35,30 +35,6 @@ public class ManagementController {
return DefaultResponseDTO.ok().build();
}
@Operation(
summary = "Queue update manga list",
description = "Queue the retrieval of the manga lists from the content providers",
tags = {"Management"},
operationId = "updateMangaList")
@PostMapping("update-manga-list")
public DefaultResponseDTO<Void> updateMangaList() {
updateMangaListTask.updateMangaList();
return DefaultResponseDTO.ok().build();
}
@Operation(
summary = "Queue update provider manga list",
description = "Queue the retrieval of the manga list for a specific provider",
tags = {"Management"},
operationId = "updateProviderMangaList")
@PostMapping("update-provider-manga-list")
public DefaultResponseDTO<Void> updateProviderMangaList(@RequestParam Long providerId) {
updateMangaListTask.updateProviderMangaList(providerId);
return DefaultResponseDTO.ok().build();
}
@Operation(
summary = "Cleanup unused S3 images",
description = "Triggers the cleanup of untracked S3 images",

2
src/main/java/com/magamochi/client/FlareClient.java → src/main/java/com/magamochi/ingestion/client/FlareClient.java
View File

@ -1,4 +1,4 @@
package com.magamochi.client;
package com.magamochi.ingestion.client;
import io.github.resilience4j.retry.annotation.Retry;
import java.util.List;

33
src/main/java/com/magamochi/ingestion/controller/IngestionController.java
View File

@ -3,18 +3,17 @@ package com.magamochi.ingestion.controller;
import com.magamochi.common.dto.DefaultResponseDTO;
import com.magamochi.ingestion.model.dto.ContentProviderListDTO;
import com.magamochi.ingestion.service.ContentProviderService;
import com.magamochi.ingestion.service.IngestionService;
import io.swagger.v3.oas.annotations.Operation;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
@RestController
@RequestMapping("/ingestion")
@RequiredArgsConstructor
public class IngestionController {
private final ContentProviderService contentProviderService;
private final IngestionService ingestionService;
@Operation(
summary = "Get a list of content providers",
@ -26,4 +25,30 @@ public class IngestionController {
@RequestParam(name = "manualImport", required = false) Boolean manualImport) {
return DefaultResponseDTO.ok(contentProviderService.getProviders(manualImport));
}
@Operation(
summary = "Fetch mangas from a content provider",
description =
"Triggers the ingestion process for a specific content provider, fetching manga data and queuing it for processing.",
tags = {"Ingestion"},
operationId = "fetchContentProviderMangas")
@PostMapping("/providers/{providerId}/fetch")
public DefaultResponseDTO<Void> fetchContentProviderMangas(@PathVariable Long providerId) {
ingestionService.fetchContentProviderMangas(providerId);
return DefaultResponseDTO.ok().build();
}
@Operation(
summary = "Fetch mangas from all content providers",
description =
"Triggers the ingestion process for all content providers, fetching manga data and queuing them for processing.",
tags = {"Ingestion"},
operationId = "fetchAllContentProviderMangas")
@PostMapping("/providers/fetch")
public DefaultResponseDTO<Void> fetchAllContentProviderMangas() {
ingestionService.fetchAllContentProviderMangas();
return DefaultResponseDTO.ok().build();
}
}

2
src/main/java/com/magamochi/ingestion/model/dto/ContentProviderListDTO.java
View File

@ -11,7 +11,7 @@ public record ContentProviderListDTO(@NotNull List<ContentProviderDTO> providers
contentProviders.stream().map(ContentProviderDTO::from).toList());
}
record ContentProviderDTO(long id, @NotBlank String name) {
public record ContentProviderDTO(long id, @NotBlank String name) {
public static ContentProviderDTO from(ContentProvider contentProvider) {
return new ContentProviderDTO(contentProvider.getId(), contentProvider.getName());
}

6
src/main/java/com/magamochi/ingestion/model/dto/MangaInfoDTO.java Normal file
View File

@ -0,0 +1,6 @@
package com.magamochi.ingestion.model.dto;
import com.magamochi.model.enumeration.MangaStatus;
import jakarta.validation.constraints.NotBlank;
public record MangaInfoDTO(@NotBlank String title, @NotBlank String url, MangaStatus status) {}

2
src/main/java/com/magamochi/service/providers/ContentProvider.java → src/main/java/com/magamochi/ingestion/providers/ContentProvider.java
View File

@ -1,4 +1,4 @@
package com.magamochi.service.providers;
package com.magamochi.ingestion.providers;
import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
import com.magamochi.model.entity.MangaContentProvider;

2
src/main/java/com/magamochi/service/providers/ContentProviderFactory.java → src/main/java/com/magamochi/ingestion/providers/ContentProviderFactory.java
View File

@ -1,4 +1,4 @@
package com.magamochi.service.providers;
package com.magamochi.ingestion.providers;
import java.util.Map;
import java.util.Objects;

7
src/main/java/com/magamochi/service/providers/ContentProviders.java → src/main/java/com/magamochi/ingestion/providers/ContentProviders.java
View File

@ -1,9 +1,8 @@
package com.magamochi.service.providers;
package com.magamochi.ingestion.providers;
public class ContentProviders {
public static final String MANGA_LIVRE_TO = "Manga Livre.to";
public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog";
public static final String MANGA_DEX = "MangaDex";
public static final String MANGA_LIVRE_TO = "Manga Livre.to";
public static final String PINK_ROSA_SCAN = "Pink Rosa Scan";
public static final String BATO = "Bato";
public static final String MANGA_DEX = "MangaDex";
}

2
src/main/java/com/magamochi/service/providers/ManualImportContentProvider.java → src/main/java/com/magamochi/ingestion/providers/ManualImportContentProvider.java
View File

@ -1,4 +1,4 @@
package com.magamochi.service.providers;
package com.magamochi.ingestion.providers;
public interface ManualImportContentProvider {
String getMangaTitle(String value);

2
src/main/java/com/magamochi/service/providers/ManualImportContentProviderFactory.java → src/main/java/com/magamochi/ingestion/providers/ManualImportContentProviderFactory.java
View File

@ -1,4 +1,4 @@
package com.magamochi.service.providers;
package com.magamochi.ingestion.providers;
import java.util.Map;
import java.util.Objects;

10
src/main/java/com/magamochi/ingestion/providers/PagedContentProvider.java Normal file
View File

@ -0,0 +1,10 @@
package com.magamochi.ingestion.providers;
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
import java.util.List;
public interface PagedContentProvider {
int getTotalPages();
List<MangaInfoDTO> getMangasFromPage(int page);
}

2
src/main/java/com/magamochi/service/providers/PagedContentProviderFactory.java → src/main/java/com/magamochi/ingestion/providers/PagedContentProviderFactory.java
View File

@ -1,4 +1,4 @@
package com.magamochi.service.providers;
package com.magamochi.ingestion.providers;
import java.util.Map;
import java.util.Objects;

8
src/main/java/com/magamochi/service/providers/impl/MangaDexProvider.java → src/main/java/com/magamochi/ingestion/providers/impl/MangaDexProvider.java
View File

@ -1,15 +1,15 @@
package com.magamochi.service.providers.impl;
package com.magamochi.ingestion.providers.impl;
import static java.util.Objects.isNull;
import com.google.common.util.concurrent.RateLimiter;
import com.magamochi.client.MangaDexClient;
import com.magamochi.common.exception.UnprocessableException;
import com.magamochi.ingestion.providers.ContentProvider;
import com.magamochi.ingestion.providers.ContentProviders;
import com.magamochi.ingestion.providers.ManualImportContentProvider;
import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
import com.magamochi.model.entity.MangaContentProvider;
import com.magamochi.service.providers.ContentProvider;
import com.magamochi.service.providers.ContentProviders;
import com.magamochi.service.providers.ManualImportContentProvider;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

18
src/main/java/com/magamochi/service/providers/impl/MangaLivreBlogProvider.java → src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreBlogProvider.java
View File

@ -1,12 +1,12 @@
package com.magamochi.service.providers.impl;
package com.magamochi.ingestion.providers.impl;
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
import com.magamochi.ingestion.providers.ContentProvider;
import com.magamochi.ingestion.providers.ContentProviders;
import com.magamochi.ingestion.providers.PagedContentProvider;
import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO;
import com.magamochi.model.entity.MangaContentProvider;
import com.magamochi.model.enumeration.MangaStatus;
import com.magamochi.service.providers.ContentProvider;
import com.magamochi.service.providers.ContentProviders;
import com.magamochi.service.providers.PagedContentProvider;
import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;
@ -101,7 +101,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv
}
@Override
public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
public List<MangaInfoDTO> getMangasFromPage(int page) {
log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE_BLOG, page);
try {
@ -134,7 +134,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv
default -> MangaStatus.UNKNOWN;
};
return new ContentProviderMangaInfoResponseDTO(title, url, status);
return new MangaInfoDTO(title, url, status);
} catch (Exception e) {
return null;
}
@ -148,7 +148,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv
}
@Override
public Integer getTotalPages() {
public int getTotalPages() {
log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE_BLOG);
try {
@ -166,7 +166,7 @@ public class MangaLivreBlogProvider implements ContentProvider, PagedContentProv
return pageNumbers.stream().max(Integer::compareTo).orElse(null);
} catch (IOException | NoSuchElementException e) {
log.error("Error fetching total pages from MangaLivre", e);
return null;
return 0;
}
}
}

19
src/main/java/com/magamochi/service/providers/impl/MangaLivreProvider.java → src/main/java/com/magamochi/ingestion/providers/impl/MangaLivreProvider.java
View File

@ -1,15 +1,15 @@
package com.magamochi.service.providers.impl;
package com.magamochi.ingestion.providers.impl;
import static java.util.Objects.nonNull;
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
import com.magamochi.ingestion.providers.ContentProvider;
import com.magamochi.ingestion.providers.ContentProviders;
import com.magamochi.ingestion.providers.PagedContentProvider;
import com.magamochi.ingestion.service.FlareService;
import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO;
import com.magamochi.model.entity.MangaContentProvider;
import com.magamochi.model.enumeration.MangaStatus;
import com.magamochi.service.providers.ContentProvider;
import com.magamochi.service.providers.ContentProviders;
import com.magamochi.service.providers.PagedContentProvider;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@ -90,7 +90,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
}
@Override
public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
public List<MangaInfoDTO> getMangasFromPage(int page) {
log.info("Getting mangas from {}, page {}", ContentProviders.MANGA_LIVRE_TO, page);
try {
@ -116,8 +116,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
}
}
return new ContentProviderMangaInfoResponseDTO(
title.trim(), url.trim(), MangaStatus.UNKNOWN);
return new MangaInfoDTO(title.trim(), url.trim(), MangaStatus.UNKNOWN);
})
.toList();
} catch (NoSuchElementException e) {
@ -127,7 +126,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
}
@Override
public Integer getTotalPages() {
public int getTotalPages() {
log.info("Getting total pages for {}", ContentProviders.MANGA_LIVRE_TO);
try {
@ -140,7 +139,7 @@ public class MangaLivreProvider implements ContentProvider, PagedContentProvider
return (int) Math.ceil((double) totalMangas / MANGAS_PER_PAGE);
} catch (NoSuchElementException e) {
log.error("Error parsing total pages from MangaLivre", e);
return null;
return 0;
}
}
}

16
src/main/java/com/magamochi/service/providers/impl/PinkRosaScanProvider.java → src/main/java/com/magamochi/ingestion/providers/impl/PinkRosaScanProvider.java
View File

@ -1,16 +1,16 @@
package com.magamochi.service.providers.impl;
package com.magamochi.ingestion.providers.impl;
import static java.util.Objects.isNull;
import static org.apache.commons.lang3.StringUtils.isBlank;
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
import com.magamochi.ingestion.providers.ContentProvider;
import com.magamochi.ingestion.providers.ContentProviders;
import com.magamochi.ingestion.providers.PagedContentProvider;
import com.magamochi.ingestion.service.FlareService;
import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO;
import com.magamochi.model.entity.MangaContentProvider;
import com.magamochi.model.enumeration.MangaStatus;
import com.magamochi.service.providers.ContentProvider;
import com.magamochi.service.providers.ContentProviders;
import com.magamochi.service.providers.PagedContentProvider;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@ -110,12 +110,12 @@ public class PinkRosaScanProvider implements ContentProvider, PagedContentProvid
}
@Override
public Integer getTotalPages() {
public int getTotalPages() {
return 1;
}
@Override
public List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page) {
public List<MangaInfoDTO> getMangasFromPage(int page) {
log.info("Getting mangas from {}", ContentProviders.PINK_ROSA_SCAN);
try {
@ -142,7 +142,7 @@ public class PinkRosaScanProvider implements ContentProvider, PagedContentProvid
var textElement = linkElement.getElementsByTag("h3");
var title = textElement.text().trim();
return new ContentProviderMangaInfoResponseDTO(title, url, MangaStatus.UNKNOWN);
return new MangaInfoDTO(title, url, MangaStatus.UNKNOWN);
})
.toList();
} catch (NoSuchElementException e) {

3
src/main/java/com/magamochi/ingestion/queue/command/ProviderPageIngestCommand.java Normal file
View File

@ -0,0 +1,3 @@
package com.magamochi.ingestion.queue.command;
public record ProviderPageIngestCommand(long providerId, int page) {}

21
src/main/java/com/magamochi/ingestion/queue/consumer/ProviderPageIngestConsumer.java Normal file
View File

@ -0,0 +1,21 @@
package com.magamochi.ingestion.queue.consumer;
import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand;
import com.magamochi.ingestion.service.IngestionService;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class ProviderPageIngestConsumer {
private final IngestionService ingestionService;
@RabbitListener(queues = "${queues.provider-page-ingest}")
public void receiveProviderPageIngestCommand(ProviderPageIngestCommand command) {
log.info("Received provider page ingest command: {}", command);
ingestionService.fetchProviderPageMangas(command.providerId(), command.page());
}
}

23
src/main/java/com/magamochi/ingestion/queue/producer/MangaIngestProducer.java Normal file
View File

@ -0,0 +1,23 @@
package com.magamochi.ingestion.queue.producer;
import com.magamochi.common.queue.command.MangaIngestCommand;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class MangaIngestProducer {
private final RabbitTemplate rabbitTemplate;
@Value("${queues.manga-ingest}")
private String mangaIngestQueue;
public void sendMangaIngestCommand(MangaIngestCommand command) {
rabbitTemplate.convertAndSend(mangaIngestQueue, command);
log.info("Sent manga ingest command: {}", command);
}
}

23
src/main/java/com/magamochi/ingestion/queue/producer/ProviderPageIngestProducer.java Normal file
View File

@ -0,0 +1,23 @@
package com.magamochi.ingestion.queue.producer;
import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Log4j2
@Service
@RequiredArgsConstructor
public class ProviderPageIngestProducer {
private final RabbitTemplate rabbitTemplate;
@Value("${queues.provider-page-ingest}")
private String providerPageIngestQueue;
public void sendProviderPageIngestCommand(ProviderPageIngestCommand command) {
rabbitTemplate.convertAndSend(providerPageIngestQueue, command);
log.info("Sent provider page ingest command: {}", command);
}
}

10
src/main/java/com/magamochi/ingestion/service/ContentProviderService.java
View File

@ -2,6 +2,7 @@ package com.magamochi.ingestion.service;
import static java.util.Objects.nonNull;
import com.magamochi.common.exception.NotFoundException;
import com.magamochi.ingestion.model.dto.ContentProviderListDTO;
import com.magamochi.ingestion.model.entity.ContentProvider;
import com.magamochi.ingestion.model.repository.ContentProviderRepository;
@ -22,4 +23,13 @@ public class ContentProviderService {
return ContentProviderListDTO.from(providers);
}
public ContentProvider find(Long contentProviderId) {
return contentProviderRepository
.findById(contentProviderId)
.orElseThrow(
() ->
new NotFoundException(
"Content Provider not found (ID: " + contentProviderId + ")."));
}
}

2
src/main/java/com/magamochi/ingestion/service/FlareService.java
View File

@ -1,6 +1,6 @@
package com.magamochi.ingestion.service;
import com.magamochi.client.FlareClient;
import com.magamochi.ingestion.client.FlareClient;
import lombok.RequiredArgsConstructor;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

3
src/main/java/com/magamochi/ingestion/service/FlareSessionManager.java
View File

@ -3,9 +3,8 @@ package com.magamochi.ingestion.service;
import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import com.magamochi.client.FlareClient;
import com.magamochi.ingestion.client.FlareClient;
import com.magamochi.ingestion.model.entity.FlareSession;
import com.magamochi.registry.FlareSessionRegistry;
import java.time.Duration;
import java.time.Instant;
import lombok.RequiredArgsConstructor;

2
src/main/java/com/magamochi/registry/FlareSessionRegistry.java → src/main/java/com/magamochi/ingestion/service/FlareSessionRegistry.java
View File

@ -1,4 +1,4 @@
package com.magamochi.registry;
package com.magamochi.ingestion.service;
import com.magamochi.ingestion.model.entity.FlareSession;
import java.util.concurrent.ConcurrentHashMap;

51
src/main/java/com/magamochi/ingestion/service/IngestionService.java Normal file
View File

@ -0,0 +1,51 @@
package com.magamochi.ingestion.service;
import com.magamochi.common.queue.command.MangaIngestCommand;
import com.magamochi.ingestion.providers.PagedContentProviderFactory;
import com.magamochi.ingestion.queue.command.ProviderPageIngestCommand;
import com.magamochi.ingestion.queue.producer.MangaIngestProducer;
import com.magamochi.ingestion.queue.producer.ProviderPageIngestProducer;
import java.util.stream.IntStream;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class IngestionService {
private final ContentProviderService contentProviderService;
private final PagedContentProviderFactory pagedContentProviderFactory;
private final ProviderPageIngestProducer providerPageIngestProducer;
private final MangaIngestProducer mangaIngestProducer;
public void fetchContentProviderMangas(long contentProviderId) {
var contentProvider = contentProviderService.find(contentProviderId);
var pagedContentProvider =
pagedContentProviderFactory.getPagedContentProvider(contentProvider.getName());
var pages = pagedContentProvider.getTotalPages();
IntStream.rangeClosed(1, pages)
.forEach(
page ->
providerPageIngestProducer.sendProviderPageIngestCommand(
new ProviderPageIngestCommand(contentProvider.getId(), page)));
}
public void fetchProviderPageMangas(long providerId, int page) {
var contentProvider = contentProviderService.find(providerId);
var pagedContentProvider =
pagedContentProviderFactory.getPagedContentProvider(contentProvider.getName());
var mangas = pagedContentProvider.getMangasFromPage(page);
mangas.forEach(
manga ->
mangaIngestProducer.sendMangaIngestCommand(
new MangaIngestCommand(contentProvider.getId(), manga.title(), manga.url())));
}
public void fetchAllContentProviderMangas() {
var contentProviders = contentProviderService.getProviders(null);
contentProviders.providers().forEach(dto -> fetchContentProviderMangas(dto.id()));
}
}

4
src/main/java/com/magamochi/ingestion/task/FlareSessionCleanupTask.java
View File

@ -1,7 +1,7 @@
package com.magamochi.ingestion.task;
import com.magamochi.client.FlareClient;
import com.magamochi.registry.FlareSessionRegistry;
import com.magamochi.ingestion.client.FlareClient;
import com.magamochi.ingestion.service.FlareSessionRegistry;
import java.time.Duration;
import java.time.Instant;
import lombok.RequiredArgsConstructor;

2
src/main/java/com/magamochi/ingestion/task/FlareStartupCleanupTask.java
View File

@ -1,6 +1,6 @@
package com.magamochi.ingestion.task;
import com.magamochi.client.FlareClient;
import com.magamochi.ingestion.client.FlareClient;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.boot.context.event.ApplicationReadyEvent;

31
src/main/java/com/magamochi/ingestion/task/IngestFromContentProvidersTask.java Normal file
View File

@ -0,0 +1,31 @@
package com.magamochi.ingestion.task;
import com.magamochi.ingestion.service.IngestionService;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Log4j2
@Component
@RequiredArgsConstructor
public class IngestFromContentProvidersTask {
@Value("${content-providers.update-enabled}")
private Boolean updateEnabled;
private final IngestionService ingestionService;
@Scheduled(cron = "${content-providers.cron-expression}")
public void updateMangaListScheduled() {
if (!updateEnabled) {
return;
}
log.info("Starting scheduling ingest from Content Providers.");
ingestionService.fetchAllContentProviderMangas();
log.info("Finished scheduling ingest from Content Providers.");
}
}

7
src/main/java/com/magamochi/model/dto/ContentProviderMangaInfoResponseDTO.java
View File

@ -1,7 +0,0 @@
package com.magamochi.model.dto;
import com.magamochi.model.enumeration.MangaStatus;
import jakarta.validation.constraints.NotBlank;
public record ContentProviderMangaInfoResponseDTO(
@NotBlank String title, @NotBlank String url, MangaStatus status) {}

3
src/main/java/com/magamochi/model/dto/MangaMessageDTO.java
View File

@ -1,6 +1,7 @@
package com.magamochi.model.dto;
import com.magamochi.ingestion.model.dto.MangaInfoDTO;
import java.util.List;
public record MangaMessageDTO(
String contentProviderName, List<ContentProviderMangaInfoResponseDTO> mangaInfoResponseDTOs) {}
String contentProviderName, List<MangaInfoDTO> mangaInfoResponseDTOs) {}

2
src/main/java/com/magamochi/service/MangaChapterService.java
View File

@ -2,6 +2,7 @@ package com.magamochi.service;
import com.google.common.util.concurrent.RateLimiter;
import com.magamochi.common.exception.UnprocessableException;
import com.magamochi.ingestion.providers.ContentProviderFactory;
import com.magamochi.model.dto.MangaChapterArchiveDTO;
import com.magamochi.model.dto.MangaChapterImagesDTO;
import com.magamochi.model.entity.MangaChapter;
@ -9,7 +10,6 @@ import com.magamochi.model.entity.MangaChapterImage;
import com.magamochi.model.enumeration.ArchiveFileType;
import com.magamochi.model.repository.MangaChapterImageRepository;
import com.magamochi.model.repository.MangaChapterRepository;
import com.magamochi.service.providers.ContentProviderFactory;
import io.github.resilience4j.retry.Retry;
import io.github.resilience4j.retry.RetryRegistry;
import java.io.BufferedInputStream;

2
src/main/java/com/magamochi/service/MangaListService.java
View File

@ -2,9 +2,9 @@ package com.magamochi.service;
import static java.util.Objects.isNull;
import com.magamochi.ingestion.providers.PagedContentProviderFactory;
import com.magamochi.model.entity.MangaContentProvider;
import com.magamochi.model.repository.MangaContentProviderRepository;
import com.magamochi.service.providers.PagedContentProviderFactory;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;

2
src/main/java/com/magamochi/service/MangaService.java
View File

@ -4,6 +4,7 @@ import static java.util.Objects.nonNull;
import com.magamochi.client.NtfyClient;
import com.magamochi.common.exception.NotFoundException;
import com.magamochi.ingestion.providers.ContentProviderFactory;
import com.magamochi.model.dto.*;
import com.magamochi.model.entity.Manga;
import com.magamochi.model.entity.MangaChapter;
@ -12,7 +13,6 @@ import com.magamochi.model.entity.UserMangaFollow;
import com.magamochi.model.repository.*;
import com.magamochi.model.specification.MangaSpecification;
import com.magamochi.queue.MangaChapterDownloadProducer;
import com.magamochi.service.providers.ContentProviderFactory;
import java.util.Comparator;
import java.util.List;
import java.util.Set;

2
src/main/java/com/magamochi/service/ProviderManualMangaImportService.java
View File

@ -6,11 +6,11 @@ import static java.util.Objects.nonNull;
import com.magamochi.common.exception.NotFoundException;
import com.magamochi.ingestion.model.entity.ContentProvider;
import com.magamochi.ingestion.model.repository.ContentProviderRepository;
import com.magamochi.ingestion.providers.ManualImportContentProviderFactory;
import com.magamochi.model.dto.ImportMangaResponseDTO;
import com.magamochi.model.dto.ImportRequestDTO;
import com.magamochi.model.entity.MangaContentProvider;
import com.magamochi.model.repository.MangaContentProviderRepository;
import com.magamochi.service.providers.ManualImportContentProviderFactory;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;

10
src/main/java/com/magamochi/service/providers/PagedContentProvider.java
View File

@ -1,10 +0,0 @@
package com.magamochi.service.providers;
import com.magamochi.model.dto.ContentProviderMangaInfoResponseDTO;
import java.util.List;
public interface PagedContentProvider {
Integer getTotalPages();
List<ContentProviderMangaInfoResponseDTO> getMangasFromPage(Integer page);
}

99
src/main/java/com/magamochi/service/providers/impl/BatoProvider.java
View File

@ -1,99 +0,0 @@
package com.magamochi.service.providers.impl;
import static java.util.Objects.isNull;
import com.magamochi.common.exception.UnprocessableException;
import com.magamochi.ingestion.service.FlareService;
import com.magamochi.model.dto.ContentProviderMangaChapterResponseDTO;
import com.magamochi.model.entity.MangaContentProvider;
import com.magamochi.service.providers.ContentProvider;
import com.magamochi.service.providers.ContentProviders;
import com.magamochi.service.providers.ManualImportContentProvider;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.stereotype.Service;
@Log4j2
@Service(ContentProviders.BATO)
@RequiredArgsConstructor
public class BatoProvider implements ContentProvider, ManualImportContentProvider {
private static final String URL = "https://battwo.com";
private final FlareService flareService;
@Override
public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(
MangaContentProvider provider) {
try {
var document =
flareService.getContentAsJsoupDocument(provider.getUrl(), ContentProviders.BATO);
// Direct selector for chapter links
var chapterLinks = document.select("div.scrollable-panel a[href*=/title/]");
// TODO: fix chapter and language code
return chapterLinks.stream()
.map(
chapterLink ->
new ContentProviderMangaChapterResponseDTO(
chapterLink.text(), chapterLink.attr("href"), null, null))
.toList();
} catch (Exception e) {
log.warn(e.getMessage());
return null;
}
}
@Override
public Map<Integer, String> getChapterImagesUrls(String chapterUrl) {
try {
var document =
flareService.getContentAsJsoupDocument(
URL + chapterUrl + "?load=2", ContentProviders.BATO);
// Select all chapter page images
var imgElements = document.select("img.z-10.w-full.h-full");
List<String> imageUrls = new ArrayList<>();
for (var img : imgElements) {
String src = img.attr("src");
// Normalize if needed
if (!src.startsWith("http")) {
src = "https://battwo.com" + src;
}
imageUrls.add(src);
}
return IntStream.range(0, imageUrls.size())
.boxed()
.collect(
Collectors.toMap(
i -> i, imageUrls::get, (existing, replacement) -> existing, LinkedHashMap::new));
} catch (Exception e) {
log.warn(e.getMessage());
return null;
}
}
@Override
public String getMangaTitle(String value) {
var document = flareService.getContentAsJsoupDocument(value, ContentProviders.BATO);
var titleElement = document.selectFirst("h3 a[href*=/title/]");
if (isNull(titleElement)) {
titleElement = document.selectFirst("h3.item-title > a");
if (isNull(titleElement)) {
throw new UnprocessableException("Manga title not found for url: " + value);
}
}
return titleElement.text();
}
}

30
src/main/java/com/magamochi/task/UpdateMangaDataTask.java
View File

@ -1,30 +0,0 @@
package com.magamochi.task;
import static java.util.Objects.isNull;
import com.magamochi.model.dto.UpdateMangaDataCommand;
import com.magamochi.model.repository.MangaRepository;
import com.magamochi.queue.UpdateMangaDataProducer;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Log4j2
@Component
@RequiredArgsConstructor
public class UpdateMangaDataTask {
private final MangaRepository mangaRepository;
private final UpdateMangaDataProducer updateMangaDataProducer;
@Scheduled(cron = "@daily")
public void updateMangaData() {
var mangas =
mangaRepository.findAll().stream().filter(manga -> isNull(manga.getScore())).toList();
mangas.forEach(
manga ->
updateMangaDataProducer.sendUpdateMangaDataCommand(
new UpdateMangaDataCommand(manga.getId())));
}
}

67
src/main/java/com/magamochi/task/UpdateMangaListTask.java
View File

@ -1,67 +0,0 @@
package com.magamochi.task;
import com.magamochi.common.exception.NotFoundException;
import com.magamochi.ingestion.model.repository.ContentProviderRepository;
import com.magamochi.model.dto.MangaListUpdateCommand;
import com.magamochi.queue.UpdateMangaListProducer;
import com.magamochi.service.providers.PagedContentProvider;
import com.magamochi.service.providers.PagedContentProviderFactory;
import java.util.stream.IntStream;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Log4j2
@Component
@RequiredArgsConstructor
public class UpdateMangaListTask {
@Value("${content-providers.update-enabled}")
private Boolean updateEnabled;
private final PagedContentProviderFactory contentProviderFactory;
private final UpdateMangaListProducer updateMangaListProducer;
private final ContentProviderRepository contentProviderRepository;
@Scheduled(cron = "${content-providers.cron-expression}")
public void updateMangaListScheduled() {
if (!updateEnabled) {
return;
}
updateMangaList();
}
public void updateMangaList() {
log.info("Queuing manga list updates...");
var contentProviders = contentProviderFactory.getContentProviders();
contentProviders.forEach(this::updateProviderMangaList);
}
public void updateProviderMangaList(Long providerId) {
var provider =
contentProviderRepository
.findById(providerId)
.orElseThrow(() -> new NotFoundException("Provider not found"));
var contentProvider = contentProviderFactory.getPagedContentProvider(provider.getName());
updateProviderMangaList(provider.getName(), contentProvider);
}
private void updateProviderMangaList(
String contentProviderName, PagedContentProvider contentProvider) {
log.info("Getting total pages for provider {}", contentProviderName);
var pages = contentProvider.getTotalPages();
IntStream.rangeClosed(1, pages)
.forEach(
page ->
updateMangaListProducer.sendUpdateMangaListCommand(
new MangaListUpdateCommand(contentProviderName, page)));
log.info("Manga list update queued for content provider {}.", contentProviderName);
}
}

4
src/main/resources/application.yml
View File

@ -88,6 +88,10 @@ resilience4j:
- java.io.IOException
- java.net.SocketTimeoutException
queues:
manga-ingest: ${MANGA_INGEST_QUEUE:mangaIngest}
provider-page-ingest: ${PROVIDER_PAGE_INGEST_QUEUE:providerPageIngest}
rabbit-mq:
queues:
manga-data-update: ${MANGA_DATA_UPDATE_QUEUE:mangaDataUpdateQueue}

1
src/main/resources/db/migration/V0001__CREATE_TABLES.sql
View File

@ -42,6 +42,7 @@ CREATE TABLE content_providers
(
id BIGSERIAL NOT NULL PRIMARY KEY,
name VARCHAR NOT NULL,
url VARCHAR NOT NULL,
active BOOLEAN NOT NULL DEFAULT TRUE,
supports_chapter_fetch BOOLEAN NOT NULL DEFAULT TRUE,
manual_import BOOLEAN NOT NULL DEFAULT FALSE,

4
src/main/resources/db/migration/V0002__CONTENT_PROVIDERS.sql Normal file
View File

@ -0,0 +1,4 @@
INSERT INTO content_providers(name, url, active, supports_chapter_fetch, manual_import)
VALUES ('Manga Livre Blog', 'https://mangalivre.blog', TRUE, TRUE, FALSE),
('Manga Livre.to', 'https://mangalivre.to', TRUE, TRUE, FALSE),
('Pink Rosa Scan', 'https://scanpinkrosa.blogspot.com', TRUE, TRUE, FALSE);