From 4ae01bb3f170d5665ff8a8ab59748bd6574538c9 Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Wed, 28 Apr 2021 09:17:40 +0400 Subject: [PATCH] add commit pagination on indexing --- .../extractor/controller/RepoController.java | 4 ++- .../extractor/service/BranchService.java | 8 ++++++ .../extractor/service/CommitService.java | 22 --------------- .../service/GitRepositoryService.java | 27 ++++++++++--------- .../ulstu/extractor/service/IndexService.java | 18 ++++++++++--- 5 files changed, 39 insertions(+), 40 deletions(-) diff --git a/src/main/java/ru/ulstu/extractor/controller/RepoController.java b/src/main/java/ru/ulstu/extractor/controller/RepoController.java index 341508f..9be6c93 100644 --- a/src/main/java/ru/ulstu/extractor/controller/RepoController.java +++ b/src/main/java/ru/ulstu/extractor/controller/RepoController.java @@ -40,9 +40,11 @@ public class RepoController { @GetMapping("commits") public List getCommits(@RequestParam("repositoryUrl") String repositoryUrl, @RequestParam("branchName") String branchName, + int commitsFrom, + int commitsTo, HttpServletRequest request) throws GitAPIException, IOException { LOG.debug("Get commits {} {}. User ip {}", repositoryUrl, branchName, HttpUtils.getUserIp(request)); - return gitRepositoryService.getCommits(repositoryUrl, branchName); + return gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true); } @GetMapping("index") diff --git a/src/main/java/ru/ulstu/extractor/service/BranchService.java b/src/main/java/ru/ulstu/extractor/service/BranchService.java index ea0fb60..0b5649b 100644 --- a/src/main/java/ru/ulstu/extractor/service/BranchService.java +++ b/src/main/java/ru/ulstu/extractor/service/BranchService.java @@ -40,6 +40,14 @@ public class BranchService { return branch; } + public Branch addCommits(Branch branch, List commits) { + LOG.debug("Start add commits to {} branch with {} commits ", branch.getName(), commits.size()); + branch.getCommits().addAll(commitService.save(commits)); + LOG.debug("Save branch {} ", branch.getName()); + branch = branchRepository.save(branch); + return branch; + } + public Branch findByRepositoryAndName(Repository repository, String branchName) { return branchRepository.findByRepositoryAndName(repository, branchName); } diff --git a/src/main/java/ru/ulstu/extractor/service/CommitService.java b/src/main/java/ru/ulstu/extractor/service/CommitService.java index d19a713..c61b2ce 100644 --- a/src/main/java/ru/ulstu/extractor/service/CommitService.java +++ b/src/main/java/ru/ulstu/extractor/service/CommitService.java @@ -5,17 +5,10 @@ package ru.ulstu.extractor.service; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageImpl; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.repository.CommitRepository; -import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -33,21 +26,6 @@ public class CommitService { this.authorService = authorService; } - public Page findPaginated(Pageable pageable, String repositoryUrl, String branchName) throws GitAPIException, IOException { - int pageSize = pageable.getPageSize(); - int currentPage = pageable.getPageNumber(); - int startItem = currentPage * pageSize; - List commits = gitRepositoryService.getCommits(repositoryUrl, branchName); - - if (commits.size() < startItem) { - commits = Collections.emptyList(); - } else { - int toIndex = Math.min(startItem + pageSize, commits.size()); - commits = commits.subList(startItem, toIndex); - } - return new PageImpl<>(commits, PageRequest.of(currentPage, pageSize), commits.size()); - } - public void delete(List commitsToRemove) { commitRepository.deleteAll(commitsToRemove); } diff --git a/src/main/java/ru/ulstu/extractor/service/GitRepositoryService.java b/src/main/java/ru/ulstu/extractor/service/GitRepositoryService.java index d42369f..530fa47 100644 --- a/src/main/java/ru/ulstu/extractor/service/GitRepositoryService.java +++ b/src/main/java/ru/ulstu/extractor/service/GitRepositoryService.java @@ -98,9 +98,11 @@ public class GitRepositoryService { return branches; } - public List getCommits(String repositoryUrl, String branchName) throws GitAPIException, IOException { + public List getCommits(String repositoryUrl, String branchName, int commitsFrom, int commitsTo, boolean needUpdate) throws GitAPIException, IOException { LOG.debug("Get commits of {}. Branch {}", repositoryUrl, branchName); - cloneOrUpdateRepo(repositoryUrl, branchName); + if (needUpdate) { + cloneOrUpdateRepo(repositoryUrl, branchName); + } Repository localRepo = new FileRepository(getProjectGitDirectory(repositoryUrl)); Git git = new Git(localRepo); @@ -110,9 +112,13 @@ public class GitRepositoryService { List list = new ArrayList<>(); RevCommit prevCommit = null; LOG.debug("Start analyse {} commits", commits.size()); - int counter = commits.size(); - for (RevCommit revCommit : commits) { - LOG.debug(" {} of {} commits", counter--, commits.size()); + commitsFrom = commitsFrom > commits.size() ? commits.size() - 1 : commitsFrom; + commitsFrom = Math.max(commitsFrom, 0); + commitsTo = commitsTo > commits.size() ? commits.size() - 1 : commitsTo; + commitsTo = Math.max(commitsTo, 0); + for (int i = commitsFrom; i < commitsTo; i++) { + RevCommit revCommit = commits.get(i); + LOG.debug(" {} to {} commits", i, commitsTo); Commit commit = new Commit( revCommit.getFullMessage(), new Author(revCommit.getAuthorIdent().getName()), @@ -229,7 +235,7 @@ public class GitRepositoryService { if (laterCommit == null || earlierCommit == null) { return null; } - String output = null; + String output; try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { DiffFormatter diffFormatter = new DiffFormatter(out); diffFormatter.setRepository(localRepo); @@ -252,13 +258,8 @@ public class GitRepositoryService { if (maybeFileName.isPresent()) { fileChange = new FileChange(); fileChange.setFile(maybeFileName.get()); - Future futureEntity = executorService.submit(() -> { - return structuralUnitService.containsEntity(getContent(repository, commit, maybeFileName.get())); - }); - - Future futureBL = executorService.submit(() -> { - return structuralUnitService.containsBusinessLogic(getContent(repository, commit, maybeFileName.get())); - }); + Future futureEntity = executorService.submit(() -> structuralUnitService.containsEntity(getContent(repository, commit, maybeFileName.get()))); + Future futureBL = executorService.submit(() -> structuralUnitService.containsBusinessLogic(getContent(repository, commit, maybeFileName.get()))); try { fileChange.setContainsBusinessLogic(futureBL.get()); fileChange.setContainsEntity(futureEntity.get()); diff --git a/src/main/java/ru/ulstu/extractor/service/IndexService.java b/src/main/java/ru/ulstu/extractor/service/IndexService.java index 00c1565..bd22b8b 100644 --- a/src/main/java/ru/ulstu/extractor/service/IndexService.java +++ b/src/main/java/ru/ulstu/extractor/service/IndexService.java @@ -17,11 +17,13 @@ import ru.ulstu.extractor.model.Repository; import ru.ulstu.extractor.repository.RepositoryRepository; import java.io.IOException; +import java.util.Collections; import java.util.List; @Service public class IndexService { private final static Logger LOG = LoggerFactory.getLogger(IndexService.class); + private final static int COMMITS_PAGE_SIZE = 10; private final GitRepositoryService gitRepositoryService; private final RepositoryRepository repositoryRepository; private final BranchService branchService; @@ -44,9 +46,17 @@ public class IndexService { if (branch == null) { branch = new Branch(repository, branchName); } - List commits = gitRepositoryService.getCommits(repositoryUrl, branchName); - LOG.debug("{} commits loaded.", commits.size()); - branchService.save(branch, commits); - LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName); + branchService.save(branch, Collections.emptyList()); + int commitsFrom = 0; + int commitsTo = COMMITS_PAGE_SIZE; + List commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true); + while (!commits.isEmpty()) { + LOG.debug("{} commits loaded.", commits.size()); + branch = branchService.addCommits(branch, commits); + LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName); + commitsFrom += COMMITS_PAGE_SIZE; + commitsTo += COMMITS_PAGE_SIZE; + commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false); + } } }