From 616eaaa23a5552594223dbaf86542e2bd620745b Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Mon, 13 Feb 2023 17:54:49 +0400 Subject: [PATCH] #81 -- Add async tasks --- .../branch/model/IndexingStatus.java | 2 +- .../branch/repository/BranchRepository.java | 3 ++ .../branch/service/BranchService.java | 7 ++++ .../controler/GitIndexingController.java | 23 +++++++----- .../service/GitRepositoryService.java | 27 +++++++++----- .../gitrepository/service/IndexService.java | 37 +++++++++++-------- 6 files changed, 64 insertions(+), 35 deletions(-) diff --git a/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java b/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java index 3683817..67f2cec 100644 --- a/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java +++ b/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java @@ -1,5 +1,5 @@ package ru.ulstu.extractor.branch.model; public enum IndexingStatus { - EMPTY, INDEXING, FINISHED + EMPTY, INDEXING, ERROR, FINISHED } diff --git a/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java b/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java index 5a8b1f6..8ac8523 100644 --- a/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java +++ b/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java @@ -9,10 +9,13 @@ import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.gitrepository.model.GitRepository; import java.util.List; +import java.util.Optional; public interface BranchRepository extends JpaRepository { Branch findByGitRepositoryAndName(GitRepository gitRepository, String name); + Optional findByGitRepository_UrlAndName(String repositoryUrl, String name); + @Query("select count(c) from Commit c LEFT JOIN c.branch b LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2") int getCommitsCount(Integer repositoryId, String name); diff --git a/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java b/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java index 0a11945..cb8537e 100644 --- a/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java +++ b/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java @@ -62,6 +62,13 @@ public class BranchService { return branchRepository.findByGitRepositoryAndName(gitRepository, branchName); } + public Branch findByRepositoryAndNameOrCreate(GitRepository gitRepository, String branchName) { + Branch branch = branchRepository.findByGitRepositoryAndName(gitRepository, branchName); + return branch == null + ? branchRepository.save(new Branch(gitRepository, branchName)) + : branch; + } + public Optional findByBranchId(Integer branchId) { return branchRepository.findById(branchId); } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java b/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java index 3343754..bcd7917 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java @@ -17,8 +17,10 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.servlet.mvc.support.RedirectAttributes; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.branch.service.BranchService; import ru.ulstu.extractor.gitrepository.model.FilterForm; +import ru.ulstu.extractor.gitrepository.model.GitRepository; import ru.ulstu.extractor.gitrepository.model.RepoForm; import ru.ulstu.extractor.gitrepository.service.GitRepositoryService; import ru.ulstu.extractor.gitrepository.service.IndexService; @@ -27,8 +29,8 @@ import springfox.documentation.annotations.ApiIgnore; import java.io.IOException; import java.util.List; -import static ru.ulstu.extractor.core.Route.FILTER_COMMITS; import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY; +import static ru.ulstu.extractor.core.Route.LIST_REPOSITORY_BRANCHES; import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH; @Controller @@ -70,16 +72,18 @@ public class GitIndexingController { if (repoForm.getBranch() == null) { return INDEXING_NEW_REPOSITORY; } else { + GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(repoForm.getRepo()); + Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, repoForm.getBranch()); + branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); try { - indexService.index(repoForm.getRepo(), repoForm.getBranch()); + indexService.index(gitRepository, branch); } catch (IOException | GitAPIException ex) { ex.printStackTrace(); model.addAttribute("error", ex.getMessage()); return INDEXING_NEW_REPOSITORY; } - redirectAttributes.addAttribute("repositoryUrl", repoForm.getRepo()); - redirectAttributes.addAttribute("branchName", repoForm.getBranch()); - return "redirect:/" + FILTER_COMMITS; + redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId()); + return "redirect:/" + LIST_REPOSITORY_BRANCHES; } } @@ -87,6 +91,9 @@ public class GitIndexingController { public String reindexBranch(Model model, RedirectAttributes redirectAttributes, @RequestParam Integer branchId) { + + Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id")); + branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); try { indexService.index(branchId); } catch (IOException | GitAPIException ex) { @@ -94,9 +101,7 @@ public class GitIndexingController { model.addAttribute("error", ex.getMessage()); return INDEXING_NEW_REPOSITORY; } - Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id")); - redirectAttributes.addAttribute("repositoryUrl", branch.getGitRepository().getUrl()); - redirectAttributes.addAttribute("branchName", branch.getName()); - return "redirect:/" + FILTER_COMMITS; + redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId()); + return "redirect:/" + LIST_REPOSITORY_BRANCHES; } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java index 2d451e1..f6ead24 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java @@ -187,7 +187,6 @@ public class GitRepositoryService { return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl)); } - public void remove(String repositoryUrl) throws IOException { FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl)); } @@ -283,7 +282,7 @@ public class GitRepositoryService { List changes = new ArrayList<>(); String[] strings = output.split("\n"); Map> filesContent = getFilesContent(strings); - for(Map.Entry> fileSterings: filesContent.entrySet()) { + for (Map.Entry> fileSterings : filesContent.entrySet()) { FileChange fileChange = new FileChange(); fileChange.setFile(fileSterings.getKey()); Future futureEntity = executorService.submit(() -> structuralUnitService.containsEntity(getContent(repository, commit, fileSterings.getKey()))); @@ -327,7 +326,7 @@ public class GitRepositoryService { private Map> getFilesContent(String[] commitStrings) { int i = 0; - Map> result = new HashMap<>(); + Map> result = new HashMap<>(); while (i < commitStrings.length) { Optional maybeFileName = getFileName(commitStrings[i]); if (maybeFileName.isEmpty()) { @@ -368,15 +367,15 @@ public class GitRepositoryService { return index; } - private FileChange getChange(FileChange fileChange, List fileContent){ + private FileChange getChange(FileChange fileChange, List fileContent) { int addedLine = 0; int removedLine = 0; StringBuilder builder = new StringBuilder(); boolean isRemoved = false; boolean isAdded = false; - for (String line : fileContent){ + for (String line : fileContent) { LineChange lineChange = new LineChange(); - if (line.startsWith("-")){ + if (line.startsWith("-")) { isRemoved = true; if (isAdded) { isAdded = false; @@ -385,7 +384,7 @@ public class GitRepositoryService { } builder.append(line).append("\n"); removedLine++; - } else if(line.startsWith("+")){ + } else if (line.startsWith("+")) { isAdded = true; if (isRemoved) { isRemoved = false; @@ -398,7 +397,7 @@ public class GitRepositoryService { if (isRemoved) { lineChange = setRemoved(lineChange, builder); builder.setLength(0); - } else if (isAdded){ + } else if (isAdded) { lineChange = setAdded(lineChange, builder); builder.setLength(0); } @@ -409,17 +408,18 @@ public class GitRepositoryService { return fileChange; } - private LineChange setRemoved(LineChange lineChange, StringBuilder builder){ + private LineChange setRemoved(LineChange lineChange, StringBuilder builder) { lineChange.setLineFrom(builder.toString()); lineChange.setRemoved(true); return lineChange; } - private LineChange setAdded(LineChange lineChange, StringBuilder builder){ + private LineChange setAdded(LineChange lineChange, StringBuilder builder) { lineChange.setLineTo(builder.toString()); lineChange.setAdded(true); return lineChange; } + public Page findAll(Pageable pageable) { return gitRepositoryRepository.findAll(pageable); } @@ -427,4 +427,11 @@ public class GitRepositoryService { public GitRepository findById(Integer id) { return gitRepositoryRepository.getOne(id); } + + public GitRepository findByUrlOrCreate(String gitRepositoryUrl) { + GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl); + return gitRepository == null + ? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl)) + : gitRepository; + } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java index a442286..7c3b4bb 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java @@ -23,6 +23,8 @@ import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; @Service public class IndexService { @@ -32,6 +34,7 @@ public class IndexService { private final GitRepositoryRepository gitRepositoryRepository; private final BranchService branchService; private final List timeSeriesCreators; + private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10); public IndexService(GitRepositoryService gitRepositoryService, GitRepositoryRepository gitRepositoryRepository, @@ -43,38 +46,43 @@ public class IndexService { this.timeSeriesCreators = timeSeriesCreators; } + @Transactional + @Async public void index(@NotNull Integer branchId) throws GitAPIException, IOException { Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id")); - index(branch.getGitRepository().getUrl(), branch.getName()); + index(branch.getGitRepository(), branch); } - public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException { - GitRepository gitRepository = gitRepositoryRepository.findByUrl(repositoryUrl); - if (gitRepository == null) { - gitRepository = gitRepositoryRepository.save(new GitRepository(repositoryUrl)); - } - Branch branch = branchService.findByRepositoryAndName(gitRepository, branchName); - if (branch == null) { - branch = new Branch(gitRepository, branchName); - } + @Transactional + @Async + public void index(String gitRepositoryUrl, String branchName) throws GitAPIException, IOException { + GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(gitRepositoryUrl); + Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, branchName); + branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); + index(gitRepository, branch); + } + + @Transactional + @Async + public void index(GitRepository gitRepository, Branch branch) throws GitAPIException, IOException { branchService.save(branch, Collections.emptyList()); branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); int commitsFrom = 0; int commitsTo = COMMITS_PAGE_SIZE; - List commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true); + List commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, true); while (!commits.isEmpty()) { LOG.debug("{} commits loaded.", commits.size()); branchService.addCommits(branch, commits); - LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName); + LOG.debug("{} commits successfully saved. {} {}", commits.size(), gitRepository.getUrl(), branch.getName()); commitsFrom += COMMITS_PAGE_SIZE; commitsTo += COMMITS_PAGE_SIZE; - commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false); + commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, false); } Integer repositoryId = gitRepository.getId(); final Branch branchForSave = branch; timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave)); branchService.updateStatus(branch, IndexingStatus.FINISHED); - LOG.debug("Complete indexing {} branch", branchName); + LOG.debug("Complete indexing {} branch", branch.getName()); } @Transactional @@ -89,7 +97,6 @@ public class IndexService { try { index(failedBranch.getId()); LOG.info("Завершено индексирование ветки {}", failedBranch.getName()); - branchService.updateStatus(failedBranch, IndexingStatus.FINISHED); } catch (Exception ex) { LOG.warn(ex.getMessage()); }