async indexing #88

Merged
romanov73 merged 5 commits from 81-async-indexing into master 2023-02-13 17:56:23 +04:00
6 changed files with 64 additions and 35 deletions
Showing only changes of commit 616eaaa23a - Show all commits

View File

@ -1,5 +1,5 @@
package ru.ulstu.extractor.branch.model; package ru.ulstu.extractor.branch.model;
public enum IndexingStatus { public enum IndexingStatus {
EMPTY, INDEXING, FINISHED EMPTY, INDEXING, ERROR, FINISHED
} }

View File

@ -9,10 +9,13 @@ import ru.ulstu.extractor.branch.model.IndexingStatus;
import ru.ulstu.extractor.gitrepository.model.GitRepository; import ru.ulstu.extractor.gitrepository.model.GitRepository;
import java.util.List; import java.util.List;
import java.util.Optional;
public interface BranchRepository extends JpaRepository<Branch, Integer> { public interface BranchRepository extends JpaRepository<Branch, Integer> {
Branch findByGitRepositoryAndName(GitRepository gitRepository, String name); Branch findByGitRepositoryAndName(GitRepository gitRepository, String name);
Optional<Branch> findByGitRepository_UrlAndName(String repositoryUrl, String name);
@Query("select count(c) from Commit c LEFT JOIN c.branch b LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2") @Query("select count(c) from Commit c LEFT JOIN c.branch b LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2")
int getCommitsCount(Integer repositoryId, String name); int getCommitsCount(Integer repositoryId, String name);

View File

@ -62,6 +62,13 @@ public class BranchService {
return branchRepository.findByGitRepositoryAndName(gitRepository, branchName); return branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
} }
public Branch findByRepositoryAndNameOrCreate(GitRepository gitRepository, String branchName) {
Branch branch = branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
return branch == null
? branchRepository.save(new Branch(gitRepository, branchName))
: branch;
}
public Optional<Branch> findByBranchId(Integer branchId) { public Optional<Branch> findByBranchId(Integer branchId) {
return branchRepository.findById(branchId); return branchRepository.findById(branchId);
} }

View File

@ -17,8 +17,10 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.servlet.mvc.support.RedirectAttributes; import org.springframework.web.servlet.mvc.support.RedirectAttributes;
import ru.ulstu.extractor.branch.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.branch.model.IndexingStatus;
import ru.ulstu.extractor.branch.service.BranchService; import ru.ulstu.extractor.branch.service.BranchService;
import ru.ulstu.extractor.gitrepository.model.FilterForm; import ru.ulstu.extractor.gitrepository.model.FilterForm;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
import ru.ulstu.extractor.gitrepository.model.RepoForm; import ru.ulstu.extractor.gitrepository.model.RepoForm;
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService; import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
import ru.ulstu.extractor.gitrepository.service.IndexService; import ru.ulstu.extractor.gitrepository.service.IndexService;
@ -27,8 +29,8 @@ import springfox.documentation.annotations.ApiIgnore;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import static ru.ulstu.extractor.core.Route.FILTER_COMMITS;
import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY; import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY;
import static ru.ulstu.extractor.core.Route.LIST_REPOSITORY_BRANCHES;
import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH; import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH;
@Controller @Controller
@ -70,16 +72,18 @@ public class GitIndexingController {
if (repoForm.getBranch() == null) { if (repoForm.getBranch() == null) {
return INDEXING_NEW_REPOSITORY; return INDEXING_NEW_REPOSITORY;
} else { } else {
GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(repoForm.getRepo());
Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, repoForm.getBranch());
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
try { try {
indexService.index(repoForm.getRepo(), repoForm.getBranch()); indexService.index(gitRepository, branch);
} catch (IOException | GitAPIException ex) { } catch (IOException | GitAPIException ex) {
ex.printStackTrace(); ex.printStackTrace();
model.addAttribute("error", ex.getMessage()); model.addAttribute("error", ex.getMessage());
return INDEXING_NEW_REPOSITORY; return INDEXING_NEW_REPOSITORY;
} }
redirectAttributes.addAttribute("repositoryUrl", repoForm.getRepo()); redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
redirectAttributes.addAttribute("branchName", repoForm.getBranch()); return "redirect:/" + LIST_REPOSITORY_BRANCHES;
return "redirect:/" + FILTER_COMMITS;
} }
} }
@ -87,6 +91,9 @@ public class GitIndexingController {
public String reindexBranch(Model model, public String reindexBranch(Model model,
RedirectAttributes redirectAttributes, RedirectAttributes redirectAttributes,
@RequestParam Integer branchId) { @RequestParam Integer branchId) {
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id"));
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
try { try {
indexService.index(branchId); indexService.index(branchId);
} catch (IOException | GitAPIException ex) { } catch (IOException | GitAPIException ex) {
@ -94,9 +101,7 @@ public class GitIndexingController {
model.addAttribute("error", ex.getMessage()); model.addAttribute("error", ex.getMessage());
return INDEXING_NEW_REPOSITORY; return INDEXING_NEW_REPOSITORY;
} }
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id")); redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
redirectAttributes.addAttribute("repositoryUrl", branch.getGitRepository().getUrl()); return "redirect:/" + LIST_REPOSITORY_BRANCHES;
redirectAttributes.addAttribute("branchName", branch.getName());
return "redirect:/" + FILTER_COMMITS;
} }
} }

View File

@ -187,7 +187,6 @@ public class GitRepositoryService {
return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl)); return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl));
} }
public void remove(String repositoryUrl) throws IOException { public void remove(String repositoryUrl) throws IOException {
FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl)); FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl));
} }
@ -420,6 +419,7 @@ public class GitRepositoryService {
lineChange.setAdded(true); lineChange.setAdded(true);
return lineChange; return lineChange;
} }
public Page<GitRepository> findAll(Pageable pageable) { public Page<GitRepository> findAll(Pageable pageable) {
return gitRepositoryRepository.findAll(pageable); return gitRepositoryRepository.findAll(pageable);
} }
@ -427,4 +427,11 @@ public class GitRepositoryService {
public GitRepository findById(Integer id) { public GitRepository findById(Integer id) {
return gitRepositoryRepository.getOne(id); return gitRepositoryRepository.getOne(id);
} }
public GitRepository findByUrlOrCreate(String gitRepositoryUrl) {
GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl);
return gitRepository == null
? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl))
: gitRepository;
}
} }

View File

@ -23,6 +23,8 @@ import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Service @Service
public class IndexService { public class IndexService {
@ -32,6 +34,7 @@ public class IndexService {
private final GitRepositoryRepository gitRepositoryRepository; private final GitRepositoryRepository gitRepositoryRepository;
private final BranchService branchService; private final BranchService branchService;
private final List<AbstractTimeSeriesCreator> timeSeriesCreators; private final List<AbstractTimeSeriesCreator> timeSeriesCreators;
private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10);
public IndexService(GitRepositoryService gitRepositoryService, public IndexService(GitRepositoryService gitRepositoryService,
GitRepositoryRepository gitRepositoryRepository, GitRepositoryRepository gitRepositoryRepository,
@ -43,38 +46,43 @@ public class IndexService {
this.timeSeriesCreators = timeSeriesCreators; this.timeSeriesCreators = timeSeriesCreators;
} }
@Transactional
@Async
public void index(@NotNull Integer branchId) throws GitAPIException, IOException { public void index(@NotNull Integer branchId) throws GitAPIException, IOException {
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id")); Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id"));
index(branch.getGitRepository().getUrl(), branch.getName()); index(branch.getGitRepository(), branch);
} }
public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException { @Transactional
GitRepository gitRepository = gitRepositoryRepository.findByUrl(repositoryUrl); @Async
if (gitRepository == null) { public void index(String gitRepositoryUrl, String branchName) throws GitAPIException, IOException {
gitRepository = gitRepositoryRepository.save(new GitRepository(repositoryUrl)); GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(gitRepositoryUrl);
} Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, branchName);
Branch branch = branchService.findByRepositoryAndName(gitRepository, branchName); branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
if (branch == null) { index(gitRepository, branch);
branch = new Branch(gitRepository, branchName);
} }
@Transactional
@Async
public void index(GitRepository gitRepository, Branch branch) throws GitAPIException, IOException {
branchService.save(branch, Collections.emptyList()); branchService.save(branch, Collections.emptyList());
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
int commitsFrom = 0; int commitsFrom = 0;
int commitsTo = COMMITS_PAGE_SIZE; int commitsTo = COMMITS_PAGE_SIZE;
List<Commit> commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true); List<Commit> commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, true);
while (!commits.isEmpty()) { while (!commits.isEmpty()) {
LOG.debug("{} commits loaded.", commits.size()); LOG.debug("{} commits loaded.", commits.size());
branchService.addCommits(branch, commits); branchService.addCommits(branch, commits);
LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName); LOG.debug("{} commits successfully saved. {} {}", commits.size(), gitRepository.getUrl(), branch.getName());
commitsFrom += COMMITS_PAGE_SIZE; commitsFrom += COMMITS_PAGE_SIZE;
commitsTo += COMMITS_PAGE_SIZE; commitsTo += COMMITS_PAGE_SIZE;
commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false); commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, false);
} }
Integer repositoryId = gitRepository.getId(); Integer repositoryId = gitRepository.getId();
final Branch branchForSave = branch; final Branch branchForSave = branch;
timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave)); timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave));
branchService.updateStatus(branch, IndexingStatus.FINISHED); branchService.updateStatus(branch, IndexingStatus.FINISHED);
LOG.debug("Complete indexing {} branch", branchName); LOG.debug("Complete indexing {} branch", branch.getName());
} }
@Transactional @Transactional
@ -89,7 +97,6 @@ public class IndexService {
try { try {
index(failedBranch.getId()); index(failedBranch.getId());
LOG.info("Завершено индексирование ветки {}", failedBranch.getName()); LOG.info("Завершено индексирование ветки {}", failedBranch.getName());
branchService.updateStatus(failedBranch, IndexingStatus.FINISHED);
} catch (Exception ex) { } catch (Exception ex) {
LOG.warn(ex.getMessage()); LOG.warn(ex.getMessage());
} }