#81 -- Add async tasks

src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java

@@ -1,5 +1,5 @@
 package ru.ulstu.extractor.branch.model;
 
 public enum IndexingStatus {
-    EMPTY, INDEXING, FINISHED
+    EMPTY, INDEXING, ERROR, FINISHED
 }

src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java

@@ -9,10 +9,13 @@ import ru.ulstu.extractor.branch.model.IndexingStatus;
 import ru.ulstu.extractor.gitrepository.model.GitRepository;
 
 import java.util.List;
+import java.util.Optional;
 
 public interface BranchRepository extends JpaRepository<Branch, Integer> {
     Branch findByGitRepositoryAndName(GitRepository gitRepository, String name);
 
+    Optional<Branch> findByGitRepository_UrlAndName(String repositoryUrl, String name);
+
     @Query("select count(c) from Commit c LEFT JOIN c.branch b  LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2")
     int getCommitsCount(Integer repositoryId, String name);
 

src/main/java/ru/ulstu/extractor/branch/service/BranchService.java

@@ -62,6 +62,13 @@ public class BranchService {
         return branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
     }
 
+    public Branch findByRepositoryAndNameOrCreate(GitRepository gitRepository, String branchName) {
+        Branch branch = branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
+        return branch == null
+                ? branchRepository.save(new Branch(gitRepository, branchName))
+                : branch;
+    }
+
     public Optional<Branch> findByBranchId(Integer branchId) {
         return branchRepository.findById(branchId);
     }

src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java

@@ -17,8 +17,10 @@ import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.servlet.mvc.support.RedirectAttributes;
 import ru.ulstu.extractor.branch.model.Branch;
+import ru.ulstu.extractor.branch.model.IndexingStatus;
 import ru.ulstu.extractor.branch.service.BranchService;
 import ru.ulstu.extractor.gitrepository.model.FilterForm;
+import ru.ulstu.extractor.gitrepository.model.GitRepository;
 import ru.ulstu.extractor.gitrepository.model.RepoForm;
 import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
 import ru.ulstu.extractor.gitrepository.service.IndexService;
@@ -27,8 +29,8 @@ import springfox.documentation.annotations.ApiIgnore;
 import java.io.IOException;
 import java.util.List;
 
-import static ru.ulstu.extractor.core.Route.FILTER_COMMITS;
 import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY;
+import static ru.ulstu.extractor.core.Route.LIST_REPOSITORY_BRANCHES;
 import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH;
 
 @Controller
@@ -70,16 +72,18 @@ public class GitIndexingController {
         if (repoForm.getBranch() == null) {
             return INDEXING_NEW_REPOSITORY;
         } else {
+            GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(repoForm.getRepo());
+            Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, repoForm.getBranch());
+            branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
             try {
-                indexService.index(repoForm.getRepo(), repoForm.getBranch());
+                indexService.index(gitRepository, branch);
             } catch (IOException | GitAPIException ex) {
                 ex.printStackTrace();
                 model.addAttribute("error", ex.getMessage());
                 return INDEXING_NEW_REPOSITORY;
             }
-            redirectAttributes.addAttribute("repositoryUrl", repoForm.getRepo());
+            redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
-            redirectAttributes.addAttribute("branchName", repoForm.getBranch());
+            return "redirect:/" + LIST_REPOSITORY_BRANCHES;
-            return "redirect:/" + FILTER_COMMITS;
         }
     }
 
@@ -87,6 +91,9 @@ public class GitIndexingController {
     public String reindexBranch(Model model,
                                 RedirectAttributes redirectAttributes,
                                 @RequestParam Integer branchId) {
+
+        Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id"));
+        branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
         try {
             indexService.index(branchId);
         } catch (IOException | GitAPIException ex) {
@@ -94,9 +101,7 @@ public class GitIndexingController {
             model.addAttribute("error", ex.getMessage());
             return INDEXING_NEW_REPOSITORY;
         }
-        Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id"));
+        redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
-        redirectAttributes.addAttribute("repositoryUrl", branch.getGitRepository().getUrl());
+        return "redirect:/" + LIST_REPOSITORY_BRANCHES;
-        redirectAttributes.addAttribute("branchName", branch.getName());
-        return "redirect:/" + FILTER_COMMITS;
     }
 }

src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java

@@ -187,7 +187,6 @@ public class GitRepositoryService {
         return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl));
     }
 
-
     public void remove(String repositoryUrl) throws IOException {
         FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl));
     }
@@ -283,7 +282,7 @@ public class GitRepositoryService {
         List<FileChange> changes = new ArrayList<>();
         String[] strings = output.split("\n");
         Map<String, List<String>> filesContent = getFilesContent(strings);
-        for(Map.Entry<String, List<String>> fileSterings: filesContent.entrySet()) {
+        for (Map.Entry<String, List<String>> fileSterings : filesContent.entrySet()) {
             FileChange fileChange = new FileChange();
             fileChange.setFile(fileSterings.getKey());
             Future<Boolean> futureEntity = executorService.submit(() -> structuralUnitService.containsEntity(getContent(repository, commit, fileSterings.getKey())));
@@ -368,15 +367,15 @@ public class GitRepositoryService {
         return index;
     }
 
-    private FileChange getChange(FileChange fileChange, List<String> fileContent){
+    private FileChange getChange(FileChange fileChange, List<String> fileContent) {
         int addedLine = 0;
         int removedLine = 0;
         StringBuilder builder = new StringBuilder();
         boolean isRemoved = false;
         boolean isAdded = false;
-        for (String line : fileContent){
+        for (String line : fileContent) {
             LineChange lineChange = new LineChange();
-            if (line.startsWith("-")){
+            if (line.startsWith("-")) {
                 isRemoved = true;
                 if (isAdded) {
                     isAdded = false;
@@ -385,7 +384,7 @@ public class GitRepositoryService {
                 }
                 builder.append(line).append("\n");
                 removedLine++;
-            } else if(line.startsWith("+")){
+            } else if (line.startsWith("+")) {
                 isAdded = true;
                 if (isRemoved) {
                     isRemoved = false;
@@ -398,7 +397,7 @@ public class GitRepositoryService {
                 if (isRemoved) {
                     lineChange = setRemoved(lineChange, builder);
                     builder.setLength(0);
-                } else if (isAdded){
+                } else if (isAdded) {
                     lineChange = setAdded(lineChange, builder);
                     builder.setLength(0);
                 }
@@ -409,17 +408,18 @@ public class GitRepositoryService {
         return fileChange;
     }
 
-    private LineChange setRemoved(LineChange lineChange, StringBuilder builder){
+    private LineChange setRemoved(LineChange lineChange, StringBuilder builder) {
         lineChange.setLineFrom(builder.toString());
         lineChange.setRemoved(true);
         return lineChange;
     }
 
-    private LineChange setAdded(LineChange lineChange, StringBuilder builder){
+    private LineChange setAdded(LineChange lineChange, StringBuilder builder) {
         lineChange.setLineTo(builder.toString());
         lineChange.setAdded(true);
         return lineChange;
     }
+
     public Page<GitRepository> findAll(Pageable pageable) {
         return gitRepositoryRepository.findAll(pageable);
     }
@@ -427,4 +427,11 @@ public class GitRepositoryService {
     public GitRepository findById(Integer id) {
         return gitRepositoryRepository.getOne(id);
     }
+
+    public GitRepository findByUrlOrCreate(String gitRepositoryUrl) {
+        GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl);
+        return gitRepository == null
+                ? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl))
+                : gitRepository;
+    }
 }

src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java

@@ -23,6 +23,8 @@ import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 @Service
 public class IndexService {
@@ -32,6 +34,7 @@ public class IndexService {
     private final GitRepositoryRepository gitRepositoryRepository;
     private final BranchService branchService;
     private final List<AbstractTimeSeriesCreator> timeSeriesCreators;
+    private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10);
 
     public IndexService(GitRepositoryService gitRepositoryService,
                         GitRepositoryRepository gitRepositoryRepository,
@@ -43,38 +46,43 @@ public class IndexService {
         this.timeSeriesCreators = timeSeriesCreators;
     }
 
+    @Transactional
+    @Async
     public void index(@NotNull Integer branchId) throws GitAPIException, IOException {
         Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id"));
-        index(branch.getGitRepository().getUrl(), branch.getName());
+        index(branch.getGitRepository(), branch);
     }
 
-    public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException {
+    @Transactional
-        GitRepository gitRepository = gitRepositoryRepository.findByUrl(repositoryUrl);
+    @Async
-        if (gitRepository == null) {
+    public void index(String gitRepositoryUrl, String branchName) throws GitAPIException, IOException {
-            gitRepository = gitRepositoryRepository.save(new GitRepository(repositoryUrl));
+        GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(gitRepositoryUrl);
-        }
+        Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, branchName);
-        Branch branch = branchService.findByRepositoryAndName(gitRepository, branchName);
+        branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
-        if (branch == null) {
+        index(gitRepository, branch);
-            branch = new Branch(gitRepository, branchName);
     }
+
+    @Transactional
+    @Async
+    public void index(GitRepository gitRepository, Branch branch) throws GitAPIException, IOException {
         branchService.save(branch, Collections.emptyList());
         branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
         int commitsFrom = 0;
         int commitsTo = COMMITS_PAGE_SIZE;
-        List<Commit> commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true);
+        List<Commit> commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, true);
         while (!commits.isEmpty()) {
             LOG.debug("{} commits loaded.", commits.size());
             branchService.addCommits(branch, commits);
-            LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName);
+            LOG.debug("{} commits successfully saved. {} {}", commits.size(), gitRepository.getUrl(), branch.getName());
             commitsFrom += COMMITS_PAGE_SIZE;
             commitsTo += COMMITS_PAGE_SIZE;
-            commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false);
+            commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, false);
         }
         Integer repositoryId = gitRepository.getId();
         final Branch branchForSave = branch;
         timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave));
         branchService.updateStatus(branch, IndexingStatus.FINISHED);
-        LOG.debug("Complete indexing {} branch", branchName);
+        LOG.debug("Complete indexing {} branch", branch.getName());
     }
 
     @Transactional
@@ -89,7 +97,6 @@ public class IndexService {
                 try {
                     index(failedBranch.getId());
                     LOG.info("Завершено индексирование ветки {}", failedBranch.getName());
-                    branchService.updateStatus(failedBranch, IndexingStatus.FINISHED);
                 } catch (Exception ex) {
                     LOG.warn(ex.getMessage());
                 }