diff --git a/src/main/java/ru/ulstu/extractor/GitExtractorApplication.java b/src/main/java/ru/ulstu/extractor/GitExtractorApplication.java index f9a272d..1cf7928 100644 --- a/src/main/java/ru/ulstu/extractor/GitExtractorApplication.java +++ b/src/main/java/ru/ulstu/extractor/GitExtractorApplication.java @@ -2,12 +2,29 @@ package ru.ulstu.extractor; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; +import ru.ulstu.extractor.gitrepository.service.IndexService; @SpringBootApplication @EnableScheduling +@EnableAsync public class GitExtractorApplication { + private final IndexService indexService; + + public GitExtractorApplication(IndexService indexService) { + this.indexService = indexService; + } + public static void main(String[] args) { SpringApplication.run(GitExtractorApplication.class, args); } + + @EventListener(ApplicationReadyEvent.class) + public void doSomethingAfterStartup() { + indexService.indexFailedBranchesOnStart(); + } + } diff --git a/src/main/java/ru/ulstu/extractor/branch/model/Branch.java b/src/main/java/ru/ulstu/extractor/branch/model/Branch.java index 046b8c4..97c03df 100644 --- a/src/main/java/ru/ulstu/extractor/branch/model/Branch.java +++ b/src/main/java/ru/ulstu/extractor/branch/model/Branch.java @@ -13,6 +13,8 @@ import ru.ulstu.extractor.gitrepository.model.GitRepository; import javax.persistence.CascadeType; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.FetchType; import javax.persistence.JoinColumn; import javax.persistence.ManyToOne; @@ -20,6 +22,8 @@ import javax.persistence.OneToMany; import java.util.ArrayList; import java.util.List; +import static ru.ulstu.extractor.branch.model.IndexingStatus.EMPTY; + @Entity public class Branch extends BaseEntity { private String name; @@ -32,6 +36,9 @@ public class Branch extends BaseEntity { @Fetch(FetchMode.SUBSELECT) private List commits = new ArrayList<>(); + @Enumerated(EnumType.STRING) + private IndexingStatus indexingStatus = EMPTY; + public Branch() { } @@ -67,4 +74,12 @@ public class Branch extends BaseEntity { public void setCommits(List commits) { this.commits = commits; } + + public IndexingStatus getIndexingStatus() { + return indexingStatus; + } + + public void setIndexingStatus(IndexingStatus indexingStatus) { + this.indexingStatus = indexingStatus; + } } diff --git a/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java b/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java new file mode 100644 index 0000000..67f2cec --- /dev/null +++ b/src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java @@ -0,0 +1,5 @@ +package ru.ulstu.extractor.branch.model; + +public enum IndexingStatus { + EMPTY, INDEXING, ERROR, FINISHED +} diff --git a/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java b/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java index af14f53..8ac8523 100644 --- a/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java +++ b/src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java @@ -5,17 +5,23 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.gitrepository.model.GitRepository; import java.util.List; +import java.util.Optional; public interface BranchRepository extends JpaRepository { Branch findByGitRepositoryAndName(GitRepository gitRepository, String name); + Optional findByGitRepository_UrlAndName(String repositoryUrl, String name); + @Query("select count(c) from Commit c LEFT JOIN c.branch b LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2") int getCommitsCount(Integer repositoryId, String name); List findByGitRepositoryId(Integer repositoryId); Page findByGitRepository(GitRepository gitRepository, Pageable pageable); + + List findAllByIndexingStatus(IndexingStatus indexingStatus); } diff --git a/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java b/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java index ce37a85..cb8537e 100644 --- a/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java +++ b/src/main/java/ru/ulstu/extractor/branch/service/BranchService.java @@ -10,6 +10,7 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.branch.repository.BranchRepository; import ru.ulstu.extractor.commit.model.Commit; import ru.ulstu.extractor.commit.service.CommitService; @@ -32,6 +33,11 @@ public class BranchService { this.commitService = commitService; } + @Transactional + public Branch save(Branch branch) { + return branchRepository.save(branch); + } + @Transactional public Branch save(Branch branch, List commits) { LOG.debug("Start save {} branch with {} commits ", branch.getName(), commits.size()); @@ -56,6 +62,13 @@ public class BranchService { return branchRepository.findByGitRepositoryAndName(gitRepository, branchName); } + public Branch findByRepositoryAndNameOrCreate(GitRepository gitRepository, String branchName) { + Branch branch = branchRepository.findByGitRepositoryAndName(gitRepository, branchName); + return branch == null + ? branchRepository.save(new Branch(gitRepository, branchName)) + : branch; + } + public Optional findByBranchId(Integer branchId) { return branchRepository.findById(branchId); } @@ -63,4 +76,14 @@ public class BranchService { public List findAll() { return branchRepository.findAll(); } + + public List findAllByIndexingStatus(IndexingStatus indexingStatus) { + return branchRepository.findAllByIndexingStatus(indexingStatus); + } + + @Transactional + public Branch updateStatus(Branch branch, IndexingStatus indexingStatus) { + branch.setIndexingStatus(indexingStatus); + return branchRepository.save(branch); + } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java b/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java index 3343754..bcd7917 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java @@ -17,8 +17,10 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.servlet.mvc.support.RedirectAttributes; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.branch.service.BranchService; import ru.ulstu.extractor.gitrepository.model.FilterForm; +import ru.ulstu.extractor.gitrepository.model.GitRepository; import ru.ulstu.extractor.gitrepository.model.RepoForm; import ru.ulstu.extractor.gitrepository.service.GitRepositoryService; import ru.ulstu.extractor.gitrepository.service.IndexService; @@ -27,8 +29,8 @@ import springfox.documentation.annotations.ApiIgnore; import java.io.IOException; import java.util.List; -import static ru.ulstu.extractor.core.Route.FILTER_COMMITS; import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY; +import static ru.ulstu.extractor.core.Route.LIST_REPOSITORY_BRANCHES; import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH; @Controller @@ -70,16 +72,18 @@ public class GitIndexingController { if (repoForm.getBranch() == null) { return INDEXING_NEW_REPOSITORY; } else { + GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(repoForm.getRepo()); + Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, repoForm.getBranch()); + branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); try { - indexService.index(repoForm.getRepo(), repoForm.getBranch()); + indexService.index(gitRepository, branch); } catch (IOException | GitAPIException ex) { ex.printStackTrace(); model.addAttribute("error", ex.getMessage()); return INDEXING_NEW_REPOSITORY; } - redirectAttributes.addAttribute("repositoryUrl", repoForm.getRepo()); - redirectAttributes.addAttribute("branchName", repoForm.getBranch()); - return "redirect:/" + FILTER_COMMITS; + redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId()); + return "redirect:/" + LIST_REPOSITORY_BRANCHES; } } @@ -87,6 +91,9 @@ public class GitIndexingController { public String reindexBranch(Model model, RedirectAttributes redirectAttributes, @RequestParam Integer branchId) { + + Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id")); + branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); try { indexService.index(branchId); } catch (IOException | GitAPIException ex) { @@ -94,9 +101,7 @@ public class GitIndexingController { model.addAttribute("error", ex.getMessage()); return INDEXING_NEW_REPOSITORY; } - Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id")); - redirectAttributes.addAttribute("repositoryUrl", branch.getGitRepository().getUrl()); - redirectAttributes.addAttribute("branchName", branch.getName()); - return "redirect:/" + FILTER_COMMITS; + redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId()); + return "redirect:/" + LIST_REPOSITORY_BRANCHES; } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java index 2d451e1..f6ead24 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java @@ -187,7 +187,6 @@ public class GitRepositoryService { return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl)); } - public void remove(String repositoryUrl) throws IOException { FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl)); } @@ -283,7 +282,7 @@ public class GitRepositoryService { List changes = new ArrayList<>(); String[] strings = output.split("\n"); Map> filesContent = getFilesContent(strings); - for(Map.Entry> fileSterings: filesContent.entrySet()) { + for (Map.Entry> fileSterings : filesContent.entrySet()) { FileChange fileChange = new FileChange(); fileChange.setFile(fileSterings.getKey()); Future futureEntity = executorService.submit(() -> structuralUnitService.containsEntity(getContent(repository, commit, fileSterings.getKey()))); @@ -327,7 +326,7 @@ public class GitRepositoryService { private Map> getFilesContent(String[] commitStrings) { int i = 0; - Map> result = new HashMap<>(); + Map> result = new HashMap<>(); while (i < commitStrings.length) { Optional maybeFileName = getFileName(commitStrings[i]); if (maybeFileName.isEmpty()) { @@ -368,15 +367,15 @@ public class GitRepositoryService { return index; } - private FileChange getChange(FileChange fileChange, List fileContent){ + private FileChange getChange(FileChange fileChange, List fileContent) { int addedLine = 0; int removedLine = 0; StringBuilder builder = new StringBuilder(); boolean isRemoved = false; boolean isAdded = false; - for (String line : fileContent){ + for (String line : fileContent) { LineChange lineChange = new LineChange(); - if (line.startsWith("-")){ + if (line.startsWith("-")) { isRemoved = true; if (isAdded) { isAdded = false; @@ -385,7 +384,7 @@ public class GitRepositoryService { } builder.append(line).append("\n"); removedLine++; - } else if(line.startsWith("+")){ + } else if (line.startsWith("+")) { isAdded = true; if (isRemoved) { isRemoved = false; @@ -398,7 +397,7 @@ public class GitRepositoryService { if (isRemoved) { lineChange = setRemoved(lineChange, builder); builder.setLength(0); - } else if (isAdded){ + } else if (isAdded) { lineChange = setAdded(lineChange, builder); builder.setLength(0); } @@ -409,17 +408,18 @@ public class GitRepositoryService { return fileChange; } - private LineChange setRemoved(LineChange lineChange, StringBuilder builder){ + private LineChange setRemoved(LineChange lineChange, StringBuilder builder) { lineChange.setLineFrom(builder.toString()); lineChange.setRemoved(true); return lineChange; } - private LineChange setAdded(LineChange lineChange, StringBuilder builder){ + private LineChange setAdded(LineChange lineChange, StringBuilder builder) { lineChange.setLineTo(builder.toString()); lineChange.setAdded(true); return lineChange; } + public Page findAll(Pageable pageable) { return gitRepositoryRepository.findAll(pageable); } @@ -427,4 +427,11 @@ public class GitRepositoryService { public GitRepository findById(Integer id) { return gitRepositoryRepository.getOne(id); } + + public GitRepository findByUrlOrCreate(String gitRepositoryUrl) { + GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl); + return gitRepository == null + ? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl)) + : gitRepository; + } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java index b510338..7c3b4bb 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java @@ -9,8 +9,11 @@ import com.sun.istack.NotNull; import org.eclipse.jgit.api.errors.GitAPIException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.branch.service.BranchService; import ru.ulstu.extractor.commit.model.Commit; import ru.ulstu.extractor.gitrepository.model.GitRepository; @@ -20,6 +23,8 @@ import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; @Service public class IndexService { @@ -29,6 +34,7 @@ public class IndexService { private final GitRepositoryRepository gitRepositoryRepository; private final BranchService branchService; private final List timeSeriesCreators; + private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10); public IndexService(GitRepositoryService gitRepositoryService, GitRepositoryRepository gitRepositoryRepository, @@ -40,35 +46,64 @@ public class IndexService { this.timeSeriesCreators = timeSeriesCreators; } + @Transactional + @Async public void index(@NotNull Integer branchId) throws GitAPIException, IOException { Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id")); - index(branch.getGitRepository().getUrl(), branch.getName()); + index(branch.getGitRepository(), branch); } - public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException { - GitRepository gitRepository = gitRepositoryRepository.findByUrl(repositoryUrl); - if (gitRepository == null) { - gitRepository = gitRepositoryRepository.save(new GitRepository(repositoryUrl)); - } - Branch branch = branchService.findByRepositoryAndName(gitRepository, branchName); - if (branch == null) { - branch = new Branch(gitRepository, branchName); - } + @Transactional + @Async + public void index(String gitRepositoryUrl, String branchName) throws GitAPIException, IOException { + GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(gitRepositoryUrl); + Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, branchName); + branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); + index(gitRepository, branch); + } + + @Transactional + @Async + public void index(GitRepository gitRepository, Branch branch) throws GitAPIException, IOException { branchService.save(branch, Collections.emptyList()); + branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); int commitsFrom = 0; int commitsTo = COMMITS_PAGE_SIZE; - List commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true); + List commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, true); while (!commits.isEmpty()) { LOG.debug("{} commits loaded.", commits.size()); branchService.addCommits(branch, commits); - LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName); + LOG.debug("{} commits successfully saved. {} {}", commits.size(), gitRepository.getUrl(), branch.getName()); commitsFrom += COMMITS_PAGE_SIZE; commitsTo += COMMITS_PAGE_SIZE; - commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false); + commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, false); } Integer repositoryId = gitRepository.getId(); final Branch branchForSave = branch; timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave)); - LOG.debug("Complete indexing {} branch", branchName); + branchService.updateStatus(branch, IndexingStatus.FINISHED); + LOG.debug("Complete indexing {} branch", branch.getName()); + } + + @Transactional + @Async + public void indexFailedBranchesOnStart() { + LOG.info("Старт проверки незавершенных задач для индексирований..."); + + List failedBranches = branchService.findAllByIndexingStatus(IndexingStatus.INDEXING); + if (failedBranches.size() > 0) { + LOG.info("Найдено {} незавершенных задач для индексирования", failedBranches.size()); + failedBranches.forEach(failedBranch -> { + try { + index(failedBranch.getId()); + LOG.info("Завершено индексирование ветки {}", failedBranch.getName()); + } catch (Exception ex) { + LOG.warn(ex.getMessage()); + } + }); + LOG.info("Завершено индексирование незавершенных задач"); + } else { + LOG.info("Не найдено незавершенных веток для индексирования"); + } } } diff --git a/src/main/java/ru/ulstu/extractor/ts/creator/AuthorsCommitTS.java b/src/main/java/ru/ulstu/extractor/ts/creator/AuthorsCommitTS.java index 2c6927d..86ec31d 100644 --- a/src/main/java/ru/ulstu/extractor/ts/creator/AuthorsCommitTS.java +++ b/src/main/java/ru/ulstu/extractor/ts/creator/AuthorsCommitTS.java @@ -1,30 +1,39 @@ package ru.ulstu.extractor.ts.creator; import org.springframework.stereotype.Component; +import ru.ulstu.extractor.author.model.Author; +import ru.ulstu.extractor.author.service.AuthorService; import ru.ulstu.extractor.branch.service.BranchService; +import ru.ulstu.extractor.commit.model.Commit; +import ru.ulstu.extractor.commit.service.CommitService; import ru.ulstu.extractor.gitrepository.model.GitRepository; import ru.ulstu.extractor.gitrepository.service.GitRepositoryService; import ru.ulstu.extractor.ts.model.TimeSeries; import ru.ulstu.extractor.ts.model.TimeSeriesType; +import ru.ulstu.extractor.ts.model.TimeSeriesValue; import ru.ulstu.extractor.ts.service.TimeSeriesService; -import ru.ulstu.extractor.ts.util.Dummy; -import java.util.Collections; +import java.util.ArrayList; import java.util.List; @Component public class AuthorsCommitTS extends AbstractTimeSeriesCreator { private final TimeSeriesService timeSeriesService; - private final BranchService branchService; - + private final CommitService commitService; + private final AuthorService authorService; private final GitRepositoryService gitRepositoryService; + private final BranchService branchService; public AuthorsCommitTS(TimeSeriesService timeSeriesService, - BranchService branchService, - GitRepositoryService gitRepositoryService) { + CommitService commitService, + AuthorService authorService, + GitRepositoryService gitRepositoryService, + BranchService branchService) { this.timeSeriesService = timeSeriesService; - this.branchService = branchService; + this.commitService = commitService; + this.authorService = authorService; this.gitRepositoryService = gitRepositoryService; + this.branchService = branchService; } @Override @@ -34,17 +43,29 @@ public class AuthorsCommitTS extends AbstractTimeSeriesCreator { @Override public List getTimeSeries(Integer repositoryId, String branchName) { + List timeSeriesResult = new ArrayList<>(); + List authors = authorService.findAll(); GitRepository gitRepository = gitRepositoryService.findById(repositoryId); - //TODO: добавить постраничное чтение - return Collections.singletonList( - new TimeSeries( - String.format("%s %s %s", - gitRepository.getName(), - branchName, - getTimeSeriesType().getDescription()), - branchService.findByRepositoryAndName(gitRepository, branchName), - getTimeSeriesType(), - Dummy.getDefaultTimeSeries())); + List commits = new ArrayList<>(commitService.findByRepositoryIdAndName(repositoryId, branchName)); + for (Author author : authors) { + TimeSeries timeSeries = new TimeSeries( + String.format("%s %s %s %s", + gitRepository.getName(), + branchName, + author.getName(), + getTimeSeriesType().getDescription()), + branchService.findByRepositoryAndName(gitRepository, branchName), + getTimeSeriesType()); + for (Commit commit : commits) { + if (commit.getAuthor().equals(author)) { + timeSeries.getValues().add(new TimeSeriesValue(commit.getDate(), 1.0)); + } + } + if (!timeSeries.getValues().isEmpty()) { + timeSeriesResult.add(timeSeries); + } + } + return timeSeriesResult; } @Override diff --git a/src/main/resources/db/changelog-20221012_170000-schema.xml b/src/main/resources/db/changelog-20221012_170000-schema.xml index 3e4dc33..6ee4e83 100644 --- a/src/main/resources/db/changelog-20221012_170000-schema.xml +++ b/src/main/resources/db/changelog-20221012_170000-schema.xml @@ -99,4 +99,12 @@ OR branch_id is null; + + + + + + + + \ No newline at end of file diff --git a/src/main/resources/templates/listBranches.html b/src/main/resources/templates/listBranches.html index b8e32f8..a9cb074 100644 --- a/src/main/resources/templates/listBranches.html +++ b/src/main/resources/templates/listBranches.html @@ -12,21 +12,25 @@ Ветки - + + th:text="${branch.name}"> - Индексируется... + +