Merge pull request 'feat: implement MangaDex import functionality' (#2) from feat/mangadex-import into main

Reviewed-on: #2

.env

@@ -7,3 +7,8 @@ MINIO_USER=admin
 MINIO_PASS=!E9v4i0v3
 
 WEBSCRAPPER_ENDPOINT=http://localhost:8090/url
+
+MANGADEX_USER=rocverde
+MANGADEX_PASS=!A3u8e4s0
+MANGADEX_CLIENT_ID=personal-client-3c21667a-6de3-4273-94c4-e6014690f128-68830913
+MANGADEX_CLIENT_SECRET=fXwbnGLhXqqpGrznQeX3uYQDxj6hyWbS

src/main/java/com/magamochi/mangamochi/client/JikanClient.java

@@ -25,6 +25,7 @@ public interface JikanClient {
     public record MangaData(
         Long mal_id,
         ImageData images,
+        String title,
         List<String> title_synonyms,
         String status,
         boolean publishing,

src/main/java/com/magamochi/mangamochi/client/MangaDexAuthenticationClient.java

@@ -0,0 +1,33 @@
+package com.magamochi.mangamochi.client;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.http.MediaType;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+import org.springframework.web.bind.annotation.*;
+
+@FeignClient(name = "mangaDexAuthentication", url = "https://auth.mangadex.org")
+public interface MangaDexAuthenticationClient {
+
+  //    @Headers("Content-Type: application/x-www-form-urlencoded")
+  @PostMapping(
+      value = "/realms/mangadex/protocol/openid-connect/token",
+      consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
+  AuthTokenResponse authenticate(@RequestBody MultiValueMap<String, String> form);
+
+  public record AuthTokenResponse(
+      @JsonProperty("access_token") String accessToken,
+      @JsonProperty("refresh_token") String refreshToken) {}
+
+  public static MultiValueMap<String, String> build(
+      String username, String password, String clientId, String clientSecret) {
+    MultiValueMap<String, String> form = new LinkedMultiValueMap<>();
+    form.add("grant_type", "password");
+    form.add("username", username);
+    form.add("password", password);
+    form.add("client_id", clientId);
+    form.add("client_secret", clientSecret);
+    return form;
+  }
+}

src/main/java/com/magamochi/mangamochi/client/MangaDexClient.java

@@ -0,0 +1,45 @@
+package com.magamochi.mangamochi.client;
+
+import com.magamochi.mangamochi.model.dto.MangaDexMangaDTO;
+import java.util.List;
+import java.util.UUID;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.web.bind.annotation.*;
+
+@FeignClient(name = "mangaDex", url = "https://api.mangadex.org")
+public interface MangaDexClient {
+  @GetMapping("/manga/{id}")
+  MangaDexMangaDTO getManga(
+      @PathVariable UUID id, @RequestHeader("Authorization") String authorization);
+
+  @GetMapping("/manga/{id}/feed")
+  MangaDexMangaFeedDTO getMangaFeed(
+      @PathVariable UUID id, @RequestHeader("Authorization") String authorization);
+
+  @GetMapping("/manga/{id}/feed")
+  MangaDexMangaFeedDTO getMangaFeed(
+      @PathVariable UUID id,
+      @RequestParam int limit,
+      @RequestParam int offset,
+      @RequestHeader("Authorization") String authorization);
+
+  @GetMapping("/at-home/server/{chapterId}")
+  MangaChapterDataDTO getMangaChapter(
+      @PathVariable UUID chapterId, @RequestHeader("Authorization") String authorization);
+
+  record MangaDexMangaFeedDTO(
+      List<MangaFeedData> data, Integer limit, Integer offset, Integer total) {
+    public record MangaFeedData(UUID id, String type, MangaFeedAttributes attributes) {
+      public record MangaFeedAttributes(
+          String volume,
+          String chapter,
+          String title,
+          String translatedLanguage,
+          Boolean isUnavailable) {}
+    }
+  }
+
+  record MangaChapterDataDTO(String baseUrl, ChapterData chapter) {
+    public record ChapterData(String hash, List<String> data) {}
+  }
+}

src/main/java/com/magamochi/mangamochi/client/RapidFuzzClient.java

@@ -5,7 +5,7 @@ import org.springframework.cloud.openfeign.FeignClient;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 
-@FeignClient(name = "rapidFuzz", url = "http://127.0.0.1:9000/match-title")
+@FeignClient(name = "rapidFuzz", url = "http://127.0.0.1:8000/match-title")
 public interface RapidFuzzClient {
   @PostMapping
   Response mangaSearch(@RequestBody Request dto);

src/main/java/com/magamochi/mangamochi/controller/MangaImportController.java

@@ -0,0 +1,57 @@
+package com.magamochi.mangamochi.controller;
+
+import com.magamochi.mangamochi.model.dto.*;
+import com.magamochi.mangamochi.service.MangaImportService;
+import com.magamochi.mangamochi.service.providers.impl.MangaDexProvider;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.NotNull;
+import java.util.List;
+import lombok.RequiredArgsConstructor;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
+
+@RestController
+@RequestMapping("/manga/import")
+@RequiredArgsConstructor
+public class MangaImportController {
+  private final MangaDexProvider mangaDexProvider;
+  private final MangaImportService mangaImportService;
+
+  @Operation(
+      summary = "Import manga from MangaDex",
+      description = "Imports manga data from MangaDex into the local database.",
+      tags = {"Manga Import"},
+      operationId = "importFromMangaDex")
+  @PostMapping("/manga-dex")
+  public ImportMangaDexResponseDTO importFromMangaDex(
+      @RequestBody ImportMangaDexRequestDTO requestDTO) {
+    return mangaDexProvider.importManga(requestDTO.id());
+  }
+
+  @Operation(
+      summary = "Upload multiple files",
+      description = "Accepts multiple files via multipart/form-data and processes them.",
+      tags = {"Manga Import"},
+      operationId = "importMultipleFiles")
+  @PostMapping(
+      value = "/upload",
+      consumes = {"multipart/form-data"})
+  public void uploadMultipleFiles(
+      @RequestPart("malId") @NotBlank String malId,
+      @Parameter(
+              description = "List of files to upload",
+              required = true,
+              content =
+                  @Content(
+                      mediaType = "multipart/form-data",
+                      schema = @Schema(type = "array", format = "binary")))
+          @RequestPart("files")
+          @NotNull
+          List<MultipartFile> files) {
+    mangaImportService.importMangaFiles(malId, files);
+  }
+}

src/main/java/com/magamochi/mangamochi/model/dto/ContentProviderMangaChapterResponseDTO.java

@@ -3,4 +3,4 @@ package com.magamochi.mangamochi.model.dto;
 import jakarta.validation.constraints.NotBlank;
 
 public record ContentProviderMangaChapterResponseDTO(
-    @NotBlank String chapterTitle, @NotBlank String chapterUrl) {}
+    @NotBlank String chapterTitle, @NotBlank String chapterUrl, String chapter, String language) {}

src/main/java/com/magamochi/mangamochi/model/dto/ImportMangaDexRequestDTO.java

@@ -0,0 +1,6 @@
+package com.magamochi.mangamochi.model.dto;
+
+import jakarta.validation.constraints.NotNull;
+import java.util.UUID;
+
+public record ImportMangaDexRequestDTO(@NotNull UUID id) {}

src/main/java/com/magamochi/mangamochi/model/dto/ImportMangaDexResponseDTO.java

@@ -0,0 +1,5 @@
+package com.magamochi.mangamochi.model.dto;
+
+import jakarta.validation.constraints.NotNull;
+
+public record ImportMangaDexResponseDTO(@NotNull Long id) {}

src/main/java/com/magamochi/mangamochi/model/dto/MangaDexMangaDTO.java

@@ -0,0 +1,11 @@
+package com.magamochi.mangamochi.model.dto;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+public record MangaDexMangaDTO(MangaData data) {
+  public record MangaData(UUID id, AttributesData attributes) {
+    public record AttributesData(Map<String, String> title, List<Map<String, String>> altTitles) {}
+  }
+}

src/main/java/com/magamochi/mangamochi/model/entity/MangaChapter.java

@@ -37,4 +37,8 @@ public class MangaChapter {
 
   @OneToMany(mappedBy = "mangaChapter")
   private List<MangaChapterImage> mangaChapterImages;
+
+  private String language;
+
+  private Integer chapterNumber;
 }

src/main/java/com/magamochi/mangamochi/service/ImageService.java

@@ -4,8 +4,10 @@ import com.magamochi.mangamochi.model.entity.Image;
 import com.magamochi.mangamochi.model.repository.ImageRepository;
 import java.io.InputStream;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
 import org.springframework.stereotype.Service;
 
+@Log4j2
 @Service
 @RequiredArgsConstructor
 public class ImageService {
@@ -13,6 +15,7 @@ public class ImageService {
   private final ImageRepository imageRepository;
 
   public Image uploadImage(byte[] data, String contentType, String path) {
+    log.info("Uploading image {} to S3", path);
     var fileKey = s3Service.uploadFile(data, contentType, path);
 
     return imageRepository.save(Image.builder().fileKey(fileKey).build());

src/main/java/com/magamochi/mangamochi/service/MangaCreationService.java

@@ -0,0 +1,82 @@
+package com.magamochi.mangamochi.service;
+
+import com.magamochi.mangamochi.client.JikanClient;
+import com.magamochi.mangamochi.client.RapidFuzzClient;
+import com.magamochi.mangamochi.model.entity.Manga;
+import com.magamochi.mangamochi.model.entity.MangaImportReview;
+import com.magamochi.mangamochi.model.entity.Provider;
+import com.magamochi.mangamochi.model.repository.MangaImportReviewRepository;
+import com.magamochi.mangamochi.model.repository.MangaRepository;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Service;
+
+@Log4j2
+@Service
+@RequiredArgsConstructor
+public class MangaCreationService {
+  private final MangaRepository mangaRepository;
+  private final MangaImportReviewRepository mangaImportReviewRepository;
+
+  private final JikanClient jikanClient;
+  private final RapidFuzzClient rapidFuzzClient;
+
+  public Manga getOrCreateManga(String title, String url, Provider provider) {
+    var existingManga = mangaRepository.findByTitleIgnoreCase(title);
+    if (existingManga.isPresent()) {
+      return existingManga.get();
+    }
+
+    var jikanResults = jikanClient.mangaSearch(title).data();
+    if (jikanResults.isEmpty()) {
+      createMangaImportReview(title, url, provider);
+      log.warn("No manga found with title {}", title);
+      return null;
+    }
+
+    var request =
+        new RapidFuzzClient.Request(
+            title,
+            jikanResults.stream()
+                .flatMap(
+                    results ->
+                        results.titles().stream()
+                            .map(JikanClient.SearchResponse.MangaData.TitleData::title))
+                .toList());
+
+    var fuzzResults = rapidFuzzClient.mangaSearch(request);
+    if (!fuzzResults.match_found()) {
+      createMangaImportReview(title, url, provider);
+      log.warn("No match found for manga with title {}", title);
+      return null;
+    }
+
+    var resultOptional =
+        jikanResults.stream()
+            .filter(
+                results ->
+                    results.titles().stream()
+                        .map(JikanClient.SearchResponse.MangaData.TitleData::title)
+                        .toList()
+                        .contains(fuzzResults.best_match()))
+            .findFirst();
+    if (resultOptional.isEmpty()) {
+      createMangaImportReview(title, url, provider);
+      log.warn("No match found for manga with title {}", title);
+      return null;
+    }
+
+    var result = resultOptional.get();
+
+    existingManga = mangaRepository.findByTitleIgnoreCase(result.title());
+    return existingManga.orElseGet(
+        () ->
+            mangaRepository.save(
+                Manga.builder().title(result.title()).malId(result.mal_id()).build()));
+  }
+
+  private void createMangaImportReview(String title, String url, Provider provider) {
+    mangaImportReviewRepository.save(
+        MangaImportReview.builder().title(title).url(url).provider(provider).build());
+  }
+}

src/main/java/com/magamochi/mangamochi/service/MangaImportService.java

@@ -0,0 +1,276 @@
+package com.magamochi.mangamochi.service;
+
+import static java.util.Objects.isNull;
+import static java.util.Objects.nonNull;
+
+import com.google.common.util.concurrent.RateLimiter;
+import com.magamochi.mangamochi.client.JikanClient;
+import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
+import com.magamochi.mangamochi.model.entity.*;
+import com.magamochi.mangamochi.model.repository.*;
+import com.magamochi.mangamochi.util.DoubleUtil;
+import java.io.*;
+import java.net.URI;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.IntStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
+
+@Log4j2
+@Service
+@RequiredArgsConstructor
+public class MangaImportService {
+  private final ProviderService providerService;
+  private final MangaCreationService mangaCreationService;
+  private final ImageService imageService;
+
+  private final GenreRepository genreRepository;
+  private final MangaGenreRepository mangaGenreRepository;
+  private final MangaProviderRepository mangaProviderRepository;
+  private final AuthorRepository authorRepository;
+  private final MangaAuthorRepository mangaAuthorRepository;
+  private final MangaChapterRepository mangaChapterRepository;
+  private final MangaRepository mangaRepository;
+
+  private final JikanClient jikanClient;
+  private final MangaChapterImageRepository mangaChapterImageRepository;
+
+  RateLimiter rateLimiter = RateLimiter.create(1);
+
+  public void importMangaFiles(String malId, List<MultipartFile> files) {
+    var provider = providerService.getOrCreateProvider("Manual Import");
+
+    rateLimiter.acquire();
+    var mangaData = jikanClient.getMangaById(Long.parseLong(malId));
+
+    var mangaProvider = getOrCreateMangaProvider(mangaData.data().title(), provider);
+
+    var sortedFiles = files.stream().sorted(Comparator.comparing(MultipartFile::getName)).toList();
+
+    IntStream.rangeClosed(1, sortedFiles.size())
+        .forEach(
+            fileIndex -> {
+              var file = sortedFiles.get(fileIndex - 1);
+              log.info(
+                  "Importing file {}/{}: {}, for Mangá {}",
+                  fileIndex,
+                  sortedFiles.size(),
+                  file.getOriginalFilename(),
+                  mangaProvider.getManga().getTitle());
+
+              var chapter =
+                  persistMangaChapter(
+                      mangaProvider,
+                      new ContentProviderMangaChapterResponseDTO(
+                          removeFileExtension(file.getOriginalFilename()),
+                          "manual_" + file.getOriginalFilename(),
+                          file.getOriginalFilename(),
+                          "pt-br"));
+
+              List<MangaChapterImage> allChapterImages = new ArrayList<>();
+              try (InputStream is = file.getInputStream();
+                  ZipInputStream zis = new ZipInputStream(is)) {
+                ZipEntry entry;
+                var position = 0;
+
+                while ((entry = zis.getNextEntry()) != null) {
+                  if (entry.isDirectory()) {
+                    continue;
+                  }
+
+                  var os = new ByteArrayOutputStream();
+                  zis.transferTo(os);
+                  var bytes = os.toByteArray();
+
+                  var image =
+                      imageService.uploadImage(bytes, "image/jpeg", "chapter/" + chapter.getId());
+
+                  var chapterImage =
+                      MangaChapterImage.builder()
+                          .position(position++)
+                          .image(image)
+                          .mangaChapter(chapter)
+                          .build();
+
+                  allChapterImages.add(chapterImage);
+                  zis.closeEntry();
+                }
+
+                log.info("Chapter images added for chapter {}", chapter.getTitle());
+              } catch (IOException e) {
+                throw new RuntimeException(e);
+              }
+
+              mangaChapterImageRepository.saveAll(allChapterImages);
+              chapter.setDownloaded(true);
+              mangaChapterRepository.save(chapter);
+            });
+
+    log.warn("test");
+  }
+
+  public void updateMangaData(Manga manga) {
+    log.info("Updating manga {}", manga.getTitle());
+
+    try {
+      rateLimiter.acquire();
+      var mangaData = jikanClient.getMangaById(manga.getMalId());
+
+      manga.setAlternativeTitles(mangaData.data().title_synonyms());
+      manga.setSynopsis(mangaData.data().synopsis());
+      manga.setStatus(mangaData.data().status());
+      manga.setScore(DoubleUtil.round((double) mangaData.data().score(), 2));
+      manga.setPublishedFrom(mangaData.data().published().from());
+      manga.setPublishedTo(mangaData.data().published().to());
+      manga.setChapterCount(mangaData.data().chapters());
+
+      var authors =
+          mangaData.data().authors().stream()
+              .map(
+                  authorData -> {
+                    return authorRepository
+                        .findByMalId(authorData.mal_id())
+                        .orElseGet(
+                            () ->
+                                authorRepository.save(
+                                    Author.builder()
+                                        .malId(authorData.mal_id())
+                                        .name(authorData.name())
+                                        .build()));
+                  })
+              .toList();
+
+      var mangaAuthors =
+          authors.stream()
+              .map(
+                  author -> {
+                    return mangaAuthorRepository
+                        .findByMangaAndAuthor(manga, author)
+                        .orElseGet(
+                            () ->
+                                mangaAuthorRepository.save(
+                                    MangaAuthor.builder().manga(manga).author(author).build()));
+                  })
+              .toList();
+
+      manga.setMangaAuthors(mangaAuthors);
+
+      var genres =
+          mangaData.data().genres().stream()
+              .map(
+                  genreData -> {
+                    return genreRepository
+                        .findByMalId(genreData.mal_id())
+                        .orElseGet(
+                            () ->
+                                genreRepository.save(
+                                    Genre.builder()
+                                        .malId(genreData.mal_id())
+                                        .name(genreData.name())
+                                        .build()));
+                  })
+              .toList();
+
+      var mangaGenres =
+          genres.stream()
+              .map(
+                  genre -> {
+                    return mangaGenreRepository
+                        .findByMangaAndGenre(manga, genre)
+                        .orElseGet(
+                            () ->
+                                mangaGenreRepository.save(
+                                    MangaGenre.builder().manga(manga).genre(genre).build()));
+                  })
+              .toList();
+
+      manga.setMangaGenres(mangaGenres);
+
+      if (isNull(manga.getCoverImage())) {
+        var inputStream =
+            new BufferedInputStream(
+                new URL(new URI(mangaData.data().images().jpg().large_image_url()).toASCIIString())
+                    .openStream());
+
+        var bytes = inputStream.readAllBytes();
+
+        inputStream.close();
+        var image = imageService.uploadImage(bytes, "image/jpeg", "cover");
+
+        manga.setCoverImage(image);
+      }
+
+      mangaRepository.save(manga);
+
+    } catch (Exception e) {
+      log.warn("Error updating manga data for manga {}. {}", manga.getTitle(), e);
+    }
+  }
+
+  public MangaChapter persistMangaChapter(
+      MangaProvider mangaProvider, ContentProviderMangaChapterResponseDTO chapter) {
+    var mangaChapter =
+        mangaChapterRepository
+            .findByMangaProviderAndUrlIgnoreCase(mangaProvider, chapter.chapterUrl())
+            .orElseGet(MangaChapter::new);
+
+    mangaChapter.setMangaProvider(mangaProvider);
+    mangaChapter.setTitle(chapter.chapterTitle());
+    mangaChapter.setUrl(chapter.chapterUrl());
+    mangaChapter.setLanguage(chapter.language());
+
+    if (nonNull(chapter.chapter())) {
+      try {
+        mangaChapter.setChapterNumber(Integer.parseInt(chapter.chapter()));
+      } catch (NumberFormatException e) {
+        log.warn(
+            "Could not parse chapter number {} from manga {}",
+            chapter.chapter(),
+            mangaProvider.getManga().getTitle());
+      }
+    }
+
+    return mangaChapterRepository.save(mangaChapter);
+  }
+
+  private MangaProvider getOrCreateMangaProvider(String title, Provider provider) {
+    return mangaProviderRepository
+        .findByMangaTitleIgnoreCaseAndProvider(title, provider)
+        .orElseGet(
+            () -> {
+              rateLimiter.acquire();
+              var manga = mangaCreationService.getOrCreateManga(title, "manual", provider);
+
+              return mangaProviderRepository.save(
+                  MangaProvider.builder()
+                      .manga(manga)
+                      .mangaTitle(manga.getTitle())
+                      .provider(provider)
+                      .url("manual")
+                      .build());
+            });
+  }
+
+  private String removeFileExtension(String filename) {
+    if (StringUtils.isBlank(filename)) {
+      return filename;
+    }
+
+    int lastDotIndex = filename.lastIndexOf('.');
+
+    // No dot, or dot is the first character (like .gitignore)
+    if (lastDotIndex <= 0) {
+      return filename;
+    }
+
+    return filename.substring(0, lastDotIndex);
+  }
+}

src/main/java/com/magamochi/mangamochi/service/MangaListService.java

@@ -3,18 +3,9 @@ package com.magamochi.mangamochi.service;
 import static java.util.Objects.isNull;
 
 import com.google.common.util.concurrent.RateLimiter;
-import com.magamochi.mangamochi.client.JikanClient;
-import com.magamochi.mangamochi.client.RapidFuzzClient;
 import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
-import com.magamochi.mangamochi.model.entity.Manga;
-import com.magamochi.mangamochi.model.entity.MangaImportReview;
 import com.magamochi.mangamochi.model.entity.MangaProvider;
-import com.magamochi.mangamochi.model.entity.Provider;
-import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
-import com.magamochi.mangamochi.model.repository.MangaImportReviewRepository;
 import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
-import com.magamochi.mangamochi.model.repository.MangaRepository;
-import com.magamochi.mangamochi.model.repository.ProviderRepository;
 import java.util.List;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
@@ -24,28 +15,16 @@ import org.springframework.stereotype.Service;
 @Service
 @RequiredArgsConstructor
 public class MangaListService {
-  private final ProviderRepository providerRepository;
+  private final ProviderService providerService;
-  private final MangaRepository mangaRepository;
+  private final MangaCreationService mangaCreationService;
-  private final MangaProviderRepository mangaProviderRepository;
-  private final MangaImportReviewRepository mangaImportReviewRepository;
 
-  private final JikanClient jikanClient;
+  private final MangaProviderRepository mangaProviderRepository;
-  private final RapidFuzzClient rapidFuzzClient;
 
   public void updateMangaList(
       String contentProviderName, List<ContentProviderMangaInfoResponseDTO> mangaInfoResponseDTOs) {
     var rateLimiter = RateLimiter.create(1);
 
-    var provider =
+    var provider = providerService.getOrCreateProvider(contentProviderName);
-        providerRepository
-            .findByNameIgnoreCase(contentProviderName)
-            .orElseGet(
-                () ->
-                    providerRepository.save(
-                        Provider.builder()
-                            .name(contentProviderName)
-                            .status(ProviderStatus.ACTIVE)
-                            .build()));
 
     mangaInfoResponseDTOs.forEach(
         mangaResponse -> {
@@ -58,14 +37,15 @@ public class MangaListService {
           }
 
           rateLimiter.acquire();
-          var manga = getOrCreateManga(mangaResponse.title(), mangaResponse.url(), provider);
+          var manga =
+              mangaCreationService.getOrCreateManga(
+                  mangaResponse.title(), mangaResponse.url(), provider);
 
           if (isNull(manga)) {
             return;
           }
 
           try {
-
             mangaProviderRepository.save(
                 MangaProvider.builder()
                     .manga(manga)
@@ -78,63 +58,4 @@ public class MangaListService {
           }
         });
   }
-
-  private Manga getOrCreateManga(String title, String url, Provider provider) {
-    var existingManga = mangaRepository.findByTitleIgnoreCase(title);
-    if (existingManga.isPresent()) {
-      return existingManga.get();
-    }
-
-    var jikanResults = jikanClient.mangaSearch(title).data();
-    if (jikanResults.isEmpty()) {
-      createMangaImportReview(title, url, provider);
-      log.warn("No manga found with title {}", title);
-      return null;
-    }
-
-    var request =
-        new RapidFuzzClient.Request(
-            title,
-            jikanResults.stream()
-                .flatMap(
-                    results ->
-                        results.titles().stream()
-                            .map(JikanClient.SearchResponse.MangaData.TitleData::title))
-                .toList());
-
-    var fuzzResults = rapidFuzzClient.mangaSearch(request);
-    if (!fuzzResults.match_found()) {
-      createMangaImportReview(title, url, provider);
-      log.warn("No match found for manga with title {}", title);
-      return null;
-    }
-
-    var resultOptional =
-        jikanResults.stream()
-            .filter(
-                results ->
-                    results.titles().stream()
-                        .map(JikanClient.SearchResponse.MangaData.TitleData::title)
-                        .toList()
-                        .contains(fuzzResults.best_match()))
-            .findFirst();
-    if (resultOptional.isEmpty()) {
-      createMangaImportReview(title, url, provider);
-      log.warn("No match found for manga with title {}", title);
-      return null;
-    }
-
-    var result = resultOptional.get();
-
-    existingManga = mangaRepository.findByTitleIgnoreCase(result.title());
-    return existingManga.orElseGet(
-        () ->
-            mangaRepository.save(
-                Manga.builder().title(result.title()).malId(result.mal_id()).build()));
-  }
-
-  private void createMangaImportReview(String title, String url, Provider provider) {
-    mangaImportReviewRepository.save(
-        MangaImportReview.builder().title(title).url(url).provider(provider).build());
-  }
 }

src/main/java/com/magamochi/mangamochi/service/MangaService.java

@@ -7,7 +7,6 @@ import com.magamochi.mangamochi.model.dto.*;
 import com.magamochi.mangamochi.model.entity.Manga;
 import com.magamochi.mangamochi.model.entity.MangaChapter;
 import com.magamochi.mangamochi.model.entity.MangaChapterImage;
-import com.magamochi.mangamochi.model.entity.MangaProvider;
 import com.magamochi.mangamochi.model.enumeration.ArchiveFileType;
 import com.magamochi.mangamochi.model.repository.*;
 import com.magamochi.mangamochi.model.specification.MangaSpecification;
@@ -25,14 +24,17 @@ import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
 import org.apache.tomcat.util.http.fileupload.IOUtils;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.stereotype.Service;
 
+@Log4j2
 @Service
 @RequiredArgsConstructor
 public class MangaService {
+  private final MangaImportService mangaImportService;
   private final UserService userService;
   private final MangaChapterRepository mangaChapterRepository;
   private final MangaRepository mangaRepository;
@@ -72,10 +74,13 @@ public class MangaService {
             .findById(mangaProviderId)
             .orElseThrow(() -> new RuntimeException("manga provider not found"));
 
-    return mangaProvider.getMangaChapters().stream()
+    var chapters =
-        .sorted(Comparator.comparing(MangaChapter::getId))
+        mangaProvider.getMangaChapters().stream()
-        .map(MangaChapterDTO::from)
+            .sorted(Comparator.comparing(MangaChapter::getId))
-        .toList();
+            .map(MangaChapterDTO::from)
+            .toList();
+
+    return chapters;
   }
 
   public void fetchChapter(Long chapterId) {
@@ -109,6 +114,14 @@ public class MangaService {
                       var image =
                           imageService.uploadImage(bytes, "image/jpeg", "chapter/" + chapterId);
 
+                      log.info(
+                          "Downloaded image {}/{} for manga {} chapter {}: {}",
+                          entry.getKey() + 1,
+                          chapterImagesUrls.size(),
+                          chapter.getMangaProvider().getManga().getTitle(),
+                          chapterId,
+                          entry.getValue());
+
                       return MangaChapterImage.builder()
                           .mangaChapter(chapter)
                           .position(entry.getKey())
@@ -173,20 +186,6 @@ public class MangaService {
     return byteArrayOutputStream;
   }
 
-  private void persistMangaChapters(
-      MangaProvider mangaProvider, ContentProviderMangaChapterResponseDTO chapter) {
-    var mangaChapter =
-        mangaChapterRepository
-            .findByMangaProviderAndUrlIgnoreCase(mangaProvider, chapter.chapterUrl())
-            .orElseGet(MangaChapter::new);
-
-    mangaChapter.setMangaProvider(mangaProvider);
-    mangaChapter.setTitle(chapter.chapterTitle());
-    mangaChapter.setUrl(chapter.chapterUrl());
-
-    mangaChapterRepository.save(mangaChapter);
-  }
-
   public void downloadAllChapters(Long mangaProviderId) {
     var mangaProvider =
         mangaProviderRepository
@@ -230,7 +229,8 @@ public class MangaService {
         contentProviderFactory.getContentProvider(mangaProvider.getProvider().getName());
     var availableChapters = contentProvider.getAvailableChapters(mangaProvider);
 
-    availableChapters.forEach(chapter -> persistMangaChapters(mangaProvider, chapter));
+    availableChapters.forEach(
+        chapter -> mangaImportService.persistMangaChapter(mangaProvider, chapter));
   }
 
   public MangaChapterImagesDTO getMangaChapterImages(Long chapterId) {

src/main/java/com/magamochi/mangamochi/service/ProviderService.java

@@ -0,0 +1,22 @@
+package com.magamochi.mangamochi.service;
+
+import com.magamochi.mangamochi.model.entity.Provider;
+import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
+import com.magamochi.mangamochi.model.repository.ProviderRepository;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+
+@Service
+@RequiredArgsConstructor
+public class ProviderService {
+  private final ProviderRepository providerRepository;
+
+  public Provider getOrCreateProvider(String providerName) {
+    return providerRepository
+        .findByNameIgnoreCase(providerName)
+        .orElseGet(
+            () ->
+                providerRepository.save(
+                    Provider.builder().name(providerName).status(ProviderStatus.ACTIVE).build()));
+  }
+}

src/main/java/com/magamochi/mangamochi/service/providers/ContentProviders.java

@@ -3,4 +3,5 @@ package com.magamochi.mangamochi.service.providers;
 public class ContentProviders {
   public static final String MANGA_LIVRE = "Manga Livre";
   public static final String MANGA_LIVRE_BLOG = "Manga Livre Blog";
+  public static final String MANGA_DEX = "MangaDex";
 }

src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaDexProvider.java

@@ -0,0 +1,193 @@
+package com.magamochi.mangamochi.service.providers.impl;
+
+import static java.util.Objects.isNull;
+
+import com.google.common.util.concurrent.RateLimiter;
+import com.magamochi.mangamochi.client.MangaDexAuthenticationClient;
+import com.magamochi.mangamochi.client.MangaDexClient;
+import com.magamochi.mangamochi.model.dto.ContentProviderMangaChapterResponseDTO;
+import com.magamochi.mangamochi.model.dto.ContentProviderMangaInfoResponseDTO;
+import com.magamochi.mangamochi.model.dto.ImportMangaDexResponseDTO;
+import com.magamochi.mangamochi.model.entity.MangaProvider;
+import com.magamochi.mangamochi.model.entity.Provider;
+import com.magamochi.mangamochi.model.enumeration.ProviderStatus;
+import com.magamochi.mangamochi.model.repository.MangaProviderRepository;
+import com.magamochi.mangamochi.model.repository.ProviderRepository;
+import com.magamochi.mangamochi.service.MangaCreationService;
+import com.magamochi.mangamochi.service.providers.ContentProvider;
+import com.magamochi.mangamochi.service.providers.ContentProviders;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+@Log4j2
+@Service(ContentProviders.MANGA_DEX)
+@RequiredArgsConstructor
+public class MangaDexProvider implements ContentProvider {
+  @Value("${manga-dex.username}")
+  private String mangaDexUsername;
+
+  @Value("${manga-dex.password}")
+  private String mangaDexPassword;
+
+  @Value("${manga-dex.client-id}")
+  private String mangaDexClientId;
+
+  @Value("${manga-dex.client-secret}")
+  private String mangaDexClientSecret;
+
+  private final MangaDexClient mangaDexClient;
+  private final MangaDexAuthenticationClient mangaDexAuthenticationClient;
+  private final MangaCreationService mangaCreationService;
+  private final ProviderRepository providerRepository;
+  private final MangaProviderRepository mangaProviderRepository;
+
+  RateLimiter rateLimiter = RateLimiter.create(1);
+
+  private String authorizationToken;
+
+  @Override
+  public List<ContentProviderMangaInfoResponseDTO> getAvailableMangas() {
+    // MangaDex API does not provide an endpoint to list all mangas directly.
+    // As there is lots and lots of mangas, this is not feasible to implement here.
+    // The frontend has a function to import mangas by their IDs instead.
+    return List.of();
+  }
+
+  @Override
+  public List<ContentProviderMangaChapterResponseDTO> getAvailableChapters(MangaProvider provider) {
+    rateLimiter.acquire();
+    var response =
+        mangaDexClient.getMangaFeed(UUID.fromString(provider.getUrl()), getAuthorizationToken());
+
+    var mangas = new ArrayList<>(response.data());
+    var totalPages = (int) Math.ceil((double) response.total() / 100); // Default page size is 100
+
+    IntStream.range(1, totalPages)
+        .forEach(
+            i -> {
+              rateLimiter.acquire();
+
+              var pagedResponse =
+                  mangaDexClient.getMangaFeed(
+                      UUID.fromString(provider.getUrl()), 100, i * 100, getAuthorizationToken());
+
+              mangas.addAll(pagedResponse.data());
+            });
+
+    // TODO this is filtering only pt-br chapters for now, we may want to make this configurable
+    // later
+    return mangas.stream()
+        .filter(
+            c ->
+                c.type().equals("chapter")
+                    && c.attributes().isUnavailable().equals(Boolean.FALSE)
+                    && c.attributes().translatedLanguage().equals("pt-br"))
+        .sorted(
+            (o1, o2) -> {
+              try {
+                Float chapter1 = Float.parseFloat(o1.attributes().chapter());
+                Float chapter2 = Float.parseFloat(o2.attributes().chapter());
+                return chapter2.compareTo(chapter1);
+              } catch (NumberFormatException e) {
+                return o2.attributes().chapter().compareTo(o1.attributes().chapter());
+              }
+            })
+        .map(
+            c ->
+                new ContentProviderMangaChapterResponseDTO(
+                    c.attributes().chapter() + " - " + c.attributes().title(),
+                    c.id().toString(),
+                    c.attributes().chapter(),
+                    c.attributes().translatedLanguage()))
+        .toList();
+  }
+
+  @Override
+  public Map<Integer, String> getChapterImagesUrls(String chapterUrl) {
+    rateLimiter.acquire();
+    var chapter =
+        mangaDexClient.getMangaChapter(UUID.fromString(chapterUrl), getAuthorizationToken());
+
+    var chapterImageHashes =
+        chapter.chapter().data().stream()
+            .map(s -> chapter.baseUrl() + "/data/" + chapter.chapter().hash() + "/" + s)
+            .toList();
+
+    var map =
+        IntStream.range(0, chapterImageHashes.size())
+            .boxed()
+            .collect(
+                Collectors.toMap(
+                    i -> i,
+                    chapterImageHashes::get,
+                    (existing, replacement) -> existing,
+                    LinkedHashMap::new));
+
+    return map;
+  }
+
+  public ImportMangaDexResponseDTO importManga(UUID id) {
+    var token = getAuthorizationToken();
+
+    rateLimiter.acquire();
+    var resultData = mangaDexClient.getManga(id, token).data();
+
+    if (resultData.attributes().title().isEmpty()) {
+      throw new NoSuchElementException("Manga title not found for ID: " + id);
+    }
+
+    var mangaTitle =
+        resultData
+            .attributes()
+            .title()
+            .getOrDefault(
+                "en",
+                resultData.attributes().title().values().stream()
+                    .findFirst()
+                    .orElseThrow(() -> new NoSuchElementException("No title available")));
+
+    var provider =
+        providerRepository
+            .findByNameIgnoreCase("MangaDex")
+            .orElseGet(
+                () ->
+                    providerRepository.save(
+                        Provider.builder().name("MangaDex").status(ProviderStatus.ACTIVE).build()));
+
+    var manga = mangaCreationService.getOrCreateManga(mangaTitle, id.toString(), provider);
+
+    if (isNull(manga)) {
+      throw new NoSuchElementException("Manga not found for ID: " + id);
+    }
+
+    mangaProviderRepository.save(
+        MangaProvider.builder()
+            .manga(manga)
+            .mangaTitle(mangaTitle)
+            .provider(provider)
+            .url(id.toString())
+            .build());
+
+    return new ImportMangaDexResponseDTO(manga.getId());
+  }
+
+  private String getAuthorizationToken() {
+    if (isNull(authorizationToken)) {
+      rateLimiter.acquire();
+
+      var authResponse =
+          mangaDexAuthenticationClient.authenticate(
+              MangaDexAuthenticationClient.build(
+                  mangaDexUsername, mangaDexPassword, mangaDexClientId, mangaDexClientSecret));
+
+      authorizationToken = "Bearer " + authResponse.accessToken();
+    }
+
+    return authorizationToken;
+  }
+}

src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreBlogProvider.java

@@ -62,7 +62,7 @@ public class MangaLivreBlogProvider implements ContentProvider {
                     linkElement.getElementsByClass("chapter-number").getFirst();
 
                 return new ContentProviderMangaChapterResponseDTO(
-                    chapterNumberElement.text(), linkElement.attr("href"));
+                    chapterNumberElement.text(), linkElement.attr("href"), null, null);
               })
           .toList();
     } catch (IOException | NoSuchElementException e) {

src/main/java/com/magamochi/mangamochi/service/providers/impl/MangaLivreProvider.java

@@ -51,7 +51,7 @@ public class MangaLivreProvider implements ContentProvider {
                 var linkElement = chapterItemElement.getElementsByTag("a").getFirst();
 
                 return new ContentProviderMangaChapterResponseDTO(
-                    linkElement.text(), linkElement.attr("href"));
+                    linkElement.text(), linkElement.attr("href"), null, null);
               })
           .toList();
     } catch (NoSuchElementException | IOException e) {

src/main/java/com/magamochi/mangamochi/task/UpdateMangaDataTask.java

@@ -3,144 +3,27 @@ package com.magamochi.mangamochi.task;
 import static java.util.Objects.isNull;
 
 import com.google.common.util.concurrent.RateLimiter;
-import com.magamochi.mangamochi.client.JikanClient;
-import com.magamochi.mangamochi.model.entity.Author;
-import com.magamochi.mangamochi.model.entity.Genre;
-import com.magamochi.mangamochi.model.entity.MangaAuthor;
-import com.magamochi.mangamochi.model.entity.MangaGenre;
 import com.magamochi.mangamochi.model.repository.*;
-import com.magamochi.mangamochi.service.ImageService;
+import com.magamochi.mangamochi.service.MangaImportService;
-import com.magamochi.mangamochi.util.DoubleUtil;
-import java.io.BufferedInputStream;
-import java.net.URI;
-import java.net.URL;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 @Log4j2
 @Component
 @RequiredArgsConstructor
 public class UpdateMangaDataTask {
-  private final AuthorRepository authorRepository;
-  private final MangaAuthorRepository mangaAuthorRepository;
   private final MangaRepository mangaRepository;
+  private final MangaImportService mangaImportService;
 
-  private final JikanClient jikanClient;
+  @Scheduled(fixedDelayString = "1d")
-
-  private final ImageService imageService;
-  private final GenreRepository genreRepository;
-  private final MangaGenreRepository mangaGenreRepository;
-
-  //  @Scheduled(fixedDelayString = "1d")
   public void updateMangaData() {
     var rateLimiter = RateLimiter.create(1);
 
     var mangas =
         mangaRepository.findAll().stream().filter(manga -> isNull(manga.getScore())).toList();
 
-    mangas.forEach(
+    mangas.forEach(mangaImportService::updateMangaData);
-        manga -> {
-          log.info("Updating manga {}", manga.getTitle());
-
-          try {
-            rateLimiter.acquire();
-            var mangaData = jikanClient.getMangaById(manga.getMalId());
-
-            manga.setAlternativeTitles(mangaData.data().title_synonyms());
-            manga.setSynopsis(mangaData.data().synopsis());
-            manga.setStatus(mangaData.data().status());
-            manga.setScore(DoubleUtil.round((double) mangaData.data().score(), 2));
-            manga.setPublishedFrom(mangaData.data().published().from());
-            manga.setPublishedTo(mangaData.data().published().to());
-            manga.setChapterCount(mangaData.data().chapters());
-
-            var authors =
-                mangaData.data().authors().stream()
-                    .map(
-                        authorData -> {
-                          return authorRepository
-                              .findByMalId(authorData.mal_id())
-                              .orElseGet(
-                                  () ->
-                                      authorRepository.save(
-                                          Author.builder()
-                                              .malId(authorData.mal_id())
-                                              .name(authorData.name())
-                                              .build()));
-                        })
-                    .toList();
-
-            var mangaAuthors =
-                authors.stream()
-                    .map(
-                        author -> {
-                          return mangaAuthorRepository
-                              .findByMangaAndAuthor(manga, author)
-                              .orElseGet(
-                                  () ->
-                                      mangaAuthorRepository.save(
-                                          MangaAuthor.builder()
-                                              .manga(manga)
-                                              .author(author)
-                                              .build()));
-                        })
-                    .toList();
-
-            manga.setMangaAuthors(mangaAuthors);
-
-            var genres =
-                mangaData.data().genres().stream()
-                    .map(
-                        genreData -> {
-                          return genreRepository
-                              .findByMalId(genreData.mal_id())
-                              .orElseGet(
-                                  () ->
-                                      genreRepository.save(
-                                          Genre.builder()
-                                              .malId(genreData.mal_id())
-                                              .name(genreData.name())
-                                              .build()));
-                        })
-                    .toList();
-
-            var mangaGenres =
-                genres.stream()
-                    .map(
-                        genre -> {
-                          return mangaGenreRepository
-                              .findByMangaAndGenre(manga, genre)
-                              .orElseGet(
-                                  () ->
-                                      mangaGenreRepository.save(
-                                          MangaGenre.builder().manga(manga).genre(genre).build()));
-                        })
-                    .toList();
-
-            manga.setMangaGenres(mangaGenres);
-
-            if (isNull(manga.getCoverImage())) {
-              var inputStream =
-                  new BufferedInputStream(
-                      new URL(
-                              new URI(mangaData.data().images().jpg().large_image_url())
-                                  .toASCIIString())
-                          .openStream());
-
-              var bytes = inputStream.readAllBytes();
-
-              inputStream.close();
-              var image = imageService.uploadImage(bytes, "image/jpeg", "cover");
-
-              manga.setCoverImage(image);
-            }
-
-            mangaRepository.save(manga);
-
-          } catch (Exception e) {
-            log.warn("Error updating manga data for manga {}. {}", manga.getTitle(), e);
-          }
-        });
   }
 }

src/main/resources/application.yml

@@ -14,6 +14,10 @@ spring:
     schemas:
       - mangamochi
     default-schema: mangamochi
+  servlet:
+    multipart:
+      max-file-size: 250MB
+      max-request-size: 2GB
 
 springdoc:
   api-docs:
@@ -32,3 +36,9 @@ minio:
 jwt:
   secret: mySecretKeymySecretKeymySecretKeymySecretKeymySecretKeymySecretKey
   expiration: 86400000 # 24 hours in milliseconds
+
+manga-dex:
+  username: ${MANGADEX_USER}
+  password: ${MANGADEX_PASS}
+  client-id: ${MANGADEX_CLIENT_ID}
+  client-secret: ${MANGADEX_CLIENT_SECRET}

src/main/resources/db/migration/V0013__MANGA_CHAPTER_LANGUAGE.sql

@@ -0,0 +1,3 @@
+ALTER TABLE manga_chapters
+    ADD COLUMN language VARCHAR(10) DEFAULT 'pt-br',
+    ADD COLUMN chapter_number INTEGER;