From f9fd839a14f13fbc20e97593277ac624f4f6ceaf Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Tue, 7 Feb 2023 22:30:47 +0400 Subject: [PATCH 1/5] #81 -- Add branch indexing status --- .../ru/ulstu/extractor/branch/model/Branch.java | 15 +++++++++++++++ .../extractor/branch/model/IndexingStatus.java | 5 +++++ .../db/changelog-20221012_170000-schema.xml | 8 ++++++++ 3 files changed, 28 insertions(+) create mode 100644 src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java diff --git a/src/main/java/ru/ulstu/extractor/branch/model/Branch.java b/src/main/java/ru/ulstu/extractor/branch/model/Branch.java index 046b8c4..97c03df 100644 --- a/src/main/java/ru/ulstu/extractor/branch/model/Branch.java +++ b/src/main/java/ru/ulstu/extractor/branch/model/Branch.java @@ -13,6 +13,8 @@ import ru.ulstu.extractor.gitrepository.model.GitRepository; import javax.persistence.CascadeType; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.FetchType; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; @@ -20,6 +22,8 @@ import javax.persistence.OneToMany; import java.util.ArrayList; import java.util.List; +import static ru.ulstu.extractor.branch.model.IndexingStatus.EMPTY; + @Entity public class Branch extends BaseEntity { private String name; @@ -32,6 +36,9 @@ public class Branch extends BaseEntity { @Fetch(FetchMode.SUBSELECT) private List commits = new ArrayList<>(); + @Enumerated(EnumType.STRING) + private IndexingStatus indexingStatus = EMPTY; + public Branch() { } @@ -67,4 +74,12 @@ public class Branch extends BaseEntity { public void setCommits(List commits) { this.commits = commits; } + + public IndexingStatus getIndexingStatus() { + return indexingStatus; + } + + public void setIndexingStatus(IndexingStatus indexingStatus) { + this.indexingStatus = indexingStatus; + } } diff --git a/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java b/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java new file mode 100644 index 0000000..3683817 --- /dev/null +++ b/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java @@ -0,0 +1,5 @@ +package ru.ulstu.extractor.branch.model; + +public enum IndexingStatus { + EMPTY, INDEXING, FINISHED +} diff --git a/src/main/resources/db/changelog-20221012_170000-schema.xml b/src/main/resources/db/changelog-20221012_170000-schema.xml index 3e4dc33..6ee4e83 100644 --- a/src/main/resources/db/changelog-20221012_170000-schema.xml +++ b/src/main/resources/db/changelog-20221012_170000-schema.xml @@ -99,4 +99,12 @@ OR branch_id is null; + + + + + + + + \ No newline at end of file -- 2.34.1 From 98ec8217484f4bf9edc2d5cbf75555c7cbf2df9c Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Tue, 7 Feb 2023 22:44:04 +0400 Subject: [PATCH 2/5] #81 -- Show indexing status --- src/main/resources/templates/listBranches.html | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/resources/templates/listBranches.html b/src/main/resources/templates/listBranches.html index b8e32f8..2ecdaae 100644 --- a/src/main/resources/templates/listBranches.html +++ b/src/main/resources/templates/listBranches.html @@ -12,20 +12,24 @@ Ветки - + + th:text="${branch.name}"> + Индексируется... + -- 2.34.1 From 11b42d7e09d7c9c318155aa6495ec704628ec761 Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Tue, 7 Feb 2023 23:29:13 +0400 Subject: [PATCH 3/5] #81 -- Start indexing on failed tasks --- .../extractor/GitExtractorApplication.java | 17 +++++++++++ .../branch/repository/BranchRepository.java | 3 ++ .../branch/service/BranchService.java | 16 +++++++++++ .../gitrepository/service/IndexService.java | 28 +++++++++++++++++++ .../resources/templates/listBranches.html | 2 +- 5 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/ulstu/extractor/GitExtractorApplication.java b/src/main/java/ru/ulstu/extractor/GitExtractorApplication.java index f9a272d..1cf7928 100644 --- a/src/main/java/ru/ulstu/extractor/GitExtractorApplication.java +++ b/src/main/java/ru/ulstu/extractor/GitExtractorApplication.java @@ -2,12 +2,29 @@ package ru.ulstu.extractor; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; +import ru.ulstu.extractor.gitrepository.service.IndexService; @SpringBootApplication @EnableScheduling +@EnableAsync public class GitExtractorApplication { + private final IndexService indexService; + + public GitExtractorApplication(IndexService indexService) { + this.indexService = indexService; + } + public static void main(String[] args) { SpringApplication.run(GitExtractorApplication.class, args); } + + @EventListener(ApplicationReadyEvent.class) + public void doSomethingAfterStartup() { + indexService.indexFailedBranchesOnStart(); + } + } diff --git a/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java b/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java index af14f53..5a8b1f6 100644 --- a/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java +++ b/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java @@ -5,6 +5,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.gitrepository.model.GitRepository; import java.util.List; @@ -18,4 +19,6 @@ public interface BranchRepository extends JpaRepository { List findByGitRepositoryId(Integer repositoryId); Page findByGitRepository(GitRepository gitRepository, Pageable pageable); + + List findAllByIndexingStatus(IndexingStatus indexingStatus); } diff --git a/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java b/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java index ce37a85..0a11945 100644 --- a/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java +++ b/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java @@ -10,6 +10,7 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.branch.repository.BranchRepository; import ru.ulstu.extractor.commit.model.Commit; import ru.ulstu.extractor.commit.service.CommitService; @@ -32,6 +33,11 @@ public class BranchService { this.commitService = commitService; } + @Transactional + public Branch save(Branch branch) { + return branchRepository.save(branch); + } + @Transactional public Branch save(Branch branch, List commits) { LOG.debug("Start save {} branch with {} commits ", branch.getName(), commits.size()); @@ -63,4 +69,14 @@ public class BranchService { public List findAll() { return branchRepository.findAll(); } + + public List findAllByIndexingStatus(IndexingStatus indexingStatus) { + return branchRepository.findAllByIndexingStatus(indexingStatus); + } + + @Transactional + public Branch updateStatus(Branch branch, IndexingStatus indexingStatus) { + branch.setIndexingStatus(indexingStatus); + return branchRepository.save(branch); + } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java index b510338..a442286 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java @@ -9,8 +9,11 @@ import com.sun.istack.NotNull; import org.eclipse.jgit.api.errors.GitAPIException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.branch.service.BranchService; import ru.ulstu.extractor.commit.model.Commit; import ru.ulstu.extractor.gitrepository.model.GitRepository; @@ -55,6 +58,7 @@ public class IndexService { branch = new Branch(gitRepository, branchName); } branchService.save(branch, Collections.emptyList()); + branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); int commitsFrom = 0; int commitsTo = COMMITS_PAGE_SIZE; List commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true); @@ -69,6 +73,30 @@ public class IndexService { Integer repositoryId = gitRepository.getId(); final Branch branchForSave = branch; timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave)); + branchService.updateStatus(branch, IndexingStatus.FINISHED); LOG.debug("Complete indexing {} branch", branchName); } + + @Transactional + @Async + public void indexFailedBranchesOnStart() { + LOG.info("Старт проверки незавершенных задач для индексирований..."); + + List failedBranches = branchService.findAllByIndexingStatus(IndexingStatus.INDEXING); + if (failedBranches.size() > 0) { + LOG.info("Найдено {} незавершенных задач для индексирования", failedBranches.size()); + failedBranches.forEach(failedBranch -> { + try { + index(failedBranch.getId()); + LOG.info("Завершено индексирование ветки {}", failedBranch.getName()); + branchService.updateStatus(failedBranch, IndexingStatus.FINISHED); + } catch (Exception ex) { + LOG.warn(ex.getMessage()); + } + }); + LOG.info("Завершено индексирование незавершенных задач"); + } else { + LOG.info("Не найдено незавершенных веток для индексирования"); + } + } } diff --git a/src/main/resources/templates/listBranches.html b/src/main/resources/templates/listBranches.html index 2ecdaae..a9cb074 100644 --- a/src/main/resources/templates/listBranches.html +++ b/src/main/resources/templates/listBranches.html @@ -30,7 +30,7 @@ Индексируется... - -- 2.34.1 From 425ff706834923b7a4ab8e430cdef85136b12cdf Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Mon, 13 Feb 2023 12:03:13 +0400 Subject: [PATCH 4/5] #81 -- Add new time series type --- .../extractor/ts/creator/AuthorsCommitTS.java | 55 +++++++++++++------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/src/main/java/ru/ulstu/extractor/ts/creator/AuthorsCommitTS.java b/src/main/java/ru/ulstu/extractor/ts/creator/AuthorsCommitTS.java index 2c6927d..86ec31d 100644 --- a/src/main/java/ru/ulstu/extractor/ts/creator/AuthorsCommitTS.java +++ b/src/main/java/ru/ulstu/extractor/ts/creator/AuthorsCommitTS.java @@ -1,30 +1,39 @@ package ru.ulstu.extractor.ts.creator; import org.springframework.stereotype.Component; +import ru.ulstu.extractor.author.model.Author; +import ru.ulstu.extractor.author.service.AuthorService; import ru.ulstu.extractor.branch.service.BranchService; +import ru.ulstu.extractor.commit.model.Commit; +import ru.ulstu.extractor.commit.service.CommitService; import ru.ulstu.extractor.gitrepository.model.GitRepository; import ru.ulstu.extractor.gitrepository.service.GitRepositoryService; import ru.ulstu.extractor.ts.model.TimeSeries; import ru.ulstu.extractor.ts.model.TimeSeriesType; +import ru.ulstu.extractor.ts.model.TimeSeriesValue; import ru.ulstu.extractor.ts.service.TimeSeriesService; -import ru.ulstu.extractor.ts.util.Dummy; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; @Component public class AuthorsCommitTS extends AbstractTimeSeriesCreator { private final TimeSeriesService timeSeriesService; + private final CommitService commitService; + private final AuthorService authorService; + private final GitRepositoryService gitRepositoryService; private final BranchService branchService; - private final GitRepositoryService gitRepositoryService; - public AuthorsCommitTS(TimeSeriesService timeSeriesService, - BranchService branchService, - GitRepositoryService gitRepositoryService) { + CommitService commitService, + AuthorService authorService, + GitRepositoryService gitRepositoryService, + BranchService branchService) { this.timeSeriesService = timeSeriesService; - this.branchService = branchService; + this.commitService = commitService; + this.authorService = authorService; this.gitRepositoryService = gitRepositoryService; + this.branchService = branchService; } @Override @@ -34,17 +43,29 @@ public class AuthorsCommitTS extends AbstractTimeSeriesCreator { @Override public List getTimeSeries(Integer repositoryId, String branchName) { + List timeSeriesResult = new ArrayList<>(); + List authors = authorService.findAll(); GitRepository gitRepository = gitRepositoryService.findById(repositoryId); - //TODO: добавить постраничное чтение - return Collections.singletonList( - new TimeSeries( - String.format("%s %s %s", - gitRepository.getName(), - branchName, - getTimeSeriesType().getDescription()), - branchService.findByRepositoryAndName(gitRepository, branchName), - getTimeSeriesType(), - Dummy.getDefaultTimeSeries())); + List commits = new ArrayList<>(commitService.findByRepositoryIdAndName(repositoryId, branchName)); + for (Author author : authors) { + TimeSeries timeSeries = new TimeSeries( + String.format("%s %s %s %s", + gitRepository.getName(), + branchName, + author.getName(), + getTimeSeriesType().getDescription()), + branchService.findByRepositoryAndName(gitRepository, branchName), + getTimeSeriesType()); + for (Commit commit : commits) { + if (commit.getAuthor().equals(author)) { + timeSeries.getValues().add(new TimeSeriesValue(commit.getDate(), 1.0)); + } + } + if (!timeSeries.getValues().isEmpty()) { + timeSeriesResult.add(timeSeries); + } + } + return timeSeriesResult; } @Override -- 2.34.1 From 616eaaa23a5552594223dbaf86542e2bd620745b Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Mon, 13 Feb 2023 17:54:49 +0400 Subject: [PATCH 5/5] #81 -- Add async tasks --- .../branch/model/IndexingStatus.java | 2 +- .../branch/repository/BranchRepository.java | 3 ++ .../branch/service/BranchService.java | 7 ++++ .../controler/GitIndexingController.java | 23 +++++++----- .../service/GitRepositoryService.java | 27 +++++++++----- .../gitrepository/service/IndexService.java | 37 +++++++++++-------- 6 files changed, 64 insertions(+), 35 deletions(-) diff --git a/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java b/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java index 3683817..67f2cec 100644 --- a/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java +++ b/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java @@ -1,5 +1,5 @@ package ru.ulstu.extractor.branch.model; public enum IndexingStatus { - EMPTY, INDEXING, FINISHED + EMPTY, INDEXING, ERROR, FINISHED } diff --git a/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java b/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java index 5a8b1f6..8ac8523 100644 --- a/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java +++ b/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java @@ -9,10 +9,13 @@ import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.gitrepository.model.GitRepository; import java.util.List; +import java.util.Optional; public interface BranchRepository extends JpaRepository { Branch findByGitRepositoryAndName(GitRepository gitRepository, String name); + Optional findByGitRepository_UrlAndName(String repositoryUrl, String name); + @Query("select count(c) from Commit c LEFT JOIN c.branch b LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2") int getCommitsCount(Integer repositoryId, String name); diff --git a/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java b/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java index 0a11945..cb8537e 100644 --- a/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java +++ b/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java @@ -62,6 +62,13 @@ public class BranchService { return branchRepository.findByGitRepositoryAndName(gitRepository, branchName); } + public Branch findByRepositoryAndNameOrCreate(GitRepository gitRepository, String branchName) { + Branch branch = branchRepository.findByGitRepositoryAndName(gitRepository, branchName); + return branch == null + ? branchRepository.save(new Branch(gitRepository, branchName)) + : branch; + } + public Optional findByBranchId(Integer branchId) { return branchRepository.findById(branchId); } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java b/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java index 3343754..bcd7917 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java @@ -17,8 +17,10 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.servlet.mvc.support.RedirectAttributes; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.branch.service.BranchService; import ru.ulstu.extractor.gitrepository.model.FilterForm; +import ru.ulstu.extractor.gitrepository.model.GitRepository; import ru.ulstu.extractor.gitrepository.model.RepoForm; import ru.ulstu.extractor.gitrepository.service.GitRepositoryService; import ru.ulstu.extractor.gitrepository.service.IndexService; @@ -27,8 +29,8 @@ import springfox.documentation.annotations.ApiIgnore; import java.io.IOException; import java.util.List; -import static ru.ulstu.extractor.core.Route.FILTER_COMMITS; import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY; +import static ru.ulstu.extractor.core.Route.LIST_REPOSITORY_BRANCHES; import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH; @Controller @@ -70,16 +72,18 @@ public class GitIndexingController { if (repoForm.getBranch() == null) { return INDEXING_NEW_REPOSITORY; } else { + GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(repoForm.getRepo()); + Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, repoForm.getBranch()); + branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); try { - indexService.index(repoForm.getRepo(), repoForm.getBranch()); + indexService.index(gitRepository, branch); } catch (IOException | GitAPIException ex) { ex.printStackTrace(); model.addAttribute("error", ex.getMessage()); return INDEXING_NEW_REPOSITORY; } - redirectAttributes.addAttribute("repositoryUrl", repoForm.getRepo()); - redirectAttributes.addAttribute("branchName", repoForm.getBranch()); - return "redirect:/" + FILTER_COMMITS; + redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId()); + return "redirect:/" + LIST_REPOSITORY_BRANCHES; } } @@ -87,6 +91,9 @@ public class GitIndexingController { public String reindexBranch(Model model, RedirectAttributes redirectAttributes, @RequestParam Integer branchId) { + + Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id")); + branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); try { indexService.index(branchId); } catch (IOException | GitAPIException ex) { @@ -94,9 +101,7 @@ public class GitIndexingController { model.addAttribute("error", ex.getMessage()); return INDEXING_NEW_REPOSITORY; } - Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id")); - redirectAttributes.addAttribute("repositoryUrl", branch.getGitRepository().getUrl()); - redirectAttributes.addAttribute("branchName", branch.getName()); - return "redirect:/" + FILTER_COMMITS; + redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId()); + return "redirect:/" + LIST_REPOSITORY_BRANCHES; } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java index 2d451e1..f6ead24 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java @@ -187,7 +187,6 @@ public class GitRepositoryService { return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl)); } - public void remove(String repositoryUrl) throws IOException { FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl)); } @@ -283,7 +282,7 @@ public class GitRepositoryService { List changes = new ArrayList<>(); String[] strings = output.split("\n"); Map> filesContent = getFilesContent(strings); - for(Map.Entry> fileSterings: filesContent.entrySet()) { + for (Map.Entry> fileSterings : filesContent.entrySet()) { FileChange fileChange = new FileChange(); fileChange.setFile(fileSterings.getKey()); Future futureEntity = executorService.submit(() -> structuralUnitService.containsEntity(getContent(repository, commit, fileSterings.getKey()))); @@ -327,7 +326,7 @@ public class GitRepositoryService { private Map> getFilesContent(String[] commitStrings) { int i = 0; - Map> result = new HashMap<>(); + Map> result = new HashMap<>(); while (i < commitStrings.length) { Optional maybeFileName = getFileName(commitStrings[i]); if (maybeFileName.isEmpty()) { @@ -368,15 +367,15 @@ public class GitRepositoryService { return index; } - private FileChange getChange(FileChange fileChange, List fileContent){ + private FileChange getChange(FileChange fileChange, List fileContent) { int addedLine = 0; int removedLine = 0; StringBuilder builder = new StringBuilder(); boolean isRemoved = false; boolean isAdded = false; - for (String line : fileContent){ + for (String line : fileContent) { LineChange lineChange = new LineChange(); - if (line.startsWith("-")){ + if (line.startsWith("-")) { isRemoved = true; if (isAdded) { isAdded = false; @@ -385,7 +384,7 @@ public class GitRepositoryService { } builder.append(line).append("\n"); removedLine++; - } else if(line.startsWith("+")){ + } else if (line.startsWith("+")) { isAdded = true; if (isRemoved) { isRemoved = false; @@ -398,7 +397,7 @@ public class GitRepositoryService { if (isRemoved) { lineChange = setRemoved(lineChange, builder); builder.setLength(0); - } else if (isAdded){ + } else if (isAdded) { lineChange = setAdded(lineChange, builder); builder.setLength(0); } @@ -409,17 +408,18 @@ public class GitRepositoryService { return fileChange; } - private LineChange setRemoved(LineChange lineChange, StringBuilder builder){ + private LineChange setRemoved(LineChange lineChange, StringBuilder builder) { lineChange.setLineFrom(builder.toString()); lineChange.setRemoved(true); return lineChange; } - private LineChange setAdded(LineChange lineChange, StringBuilder builder){ + private LineChange setAdded(LineChange lineChange, StringBuilder builder) { lineChange.setLineTo(builder.toString()); lineChange.setAdded(true); return lineChange; } + public Page findAll(Pageable pageable) { return gitRepositoryRepository.findAll(pageable); } @@ -427,4 +427,11 @@ public class GitRepositoryService { public GitRepository findById(Integer id) { return gitRepositoryRepository.getOne(id); } + + public GitRepository findByUrlOrCreate(String gitRepositoryUrl) { + GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl); + return gitRepository == null + ? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl)) + : gitRepository; + } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java index a442286..7c3b4bb 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java @@ -23,6 +23,8 @@ import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; @Service public class IndexService { @@ -32,6 +34,7 @@ public class IndexService { private final GitRepositoryRepository gitRepositoryRepository; private final BranchService branchService; private final List timeSeriesCreators; + private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10); public IndexService(GitRepositoryService gitRepositoryService, GitRepositoryRepository gitRepositoryRepository, @@ -43,38 +46,43 @@ public class IndexService { this.timeSeriesCreators = timeSeriesCreators; } + @Transactional + @Async public void index(@NotNull Integer branchId) throws GitAPIException, IOException { Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id")); - index(branch.getGitRepository().getUrl(), branch.getName()); + index(branch.getGitRepository(), branch); } - public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException { - GitRepository gitRepository = gitRepositoryRepository.findByUrl(repositoryUrl); - if (gitRepository == null) { - gitRepository = gitRepositoryRepository.save(new GitRepository(repositoryUrl)); - } - Branch branch = branchService.findByRepositoryAndName(gitRepository, branchName); - if (branch == null) { - branch = new Branch(gitRepository, branchName); - } + @Transactional + @Async + public void index(String gitRepositoryUrl, String branchName) throws GitAPIException, IOException { + GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(gitRepositoryUrl); + Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, branchName); + branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); + index(gitRepository, branch); + } + + @Transactional + @Async + public void index(GitRepository gitRepository, Branch branch) throws GitAPIException, IOException { branchService.save(branch, Collections.emptyList()); branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); int commitsFrom = 0; int commitsTo = COMMITS_PAGE_SIZE; - List commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true); + List commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, true); while (!commits.isEmpty()) { LOG.debug("{} commits loaded.", commits.size()); branchService.addCommits(branch, commits); - LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName); + LOG.debug("{} commits successfully saved. {} {}", commits.size(), gitRepository.getUrl(), branch.getName()); commitsFrom += COMMITS_PAGE_SIZE; commitsTo += COMMITS_PAGE_SIZE; - commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false); + commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, false); } Integer repositoryId = gitRepository.getId(); final Branch branchForSave = branch; timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave)); branchService.updateStatus(branch, IndexingStatus.FINISHED); - LOG.debug("Complete indexing {} branch", branchName); + LOG.debug("Complete indexing {} branch", branch.getName()); } @Transactional @@ -89,7 +97,6 @@ public class IndexService { try { index(failedBranch.getId()); LOG.info("Завершено индексирование ветки {}", failedBranch.getName()); - branchService.updateStatus(failedBranch, IndexingStatus.FINISHED); } catch (Exception ex) { LOG.warn(ex.getMessage()); } -- 2.34.1