Merge pull request 'async indexing' (#88) from 81-async-indexing into master

Reviewed-on: #88

src/main/java/ru/ulstu/extractor/GitExtractorApplication.java

@@ -2,12 +2,29 @@ package ru.ulstu.extractor;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.scheduling.annotation.EnableScheduling;
+import ru.ulstu.extractor.gitrepository.service.IndexService;
 
 @SpringBootApplication
 @EnableScheduling
+@EnableAsync
 public class GitExtractorApplication {
+    private final IndexService indexService;
+
+    public GitExtractorApplication(IndexService indexService) {
+        this.indexService = indexService;
+    }
+
     public static void main(String[] args) {
         SpringApplication.run(GitExtractorApplication.class, args);
     }
+
+    @EventListener(ApplicationReadyEvent.class)
+    public void doSomethingAfterStartup() {
+        indexService.indexFailedBranchesOnStart();
+    }
+
 }

src/main/java/ru/ulstu/extractor/branch/model/Branch.java

@@ -13,6 +13,8 @@ import ru.ulstu.extractor.gitrepository.model.GitRepository;
 
 import javax.persistence.CascadeType;
 import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
 import javax.persistence.FetchType;
 import javax.persistence.JoinColumn;
 import javax.persistence.ManyToOne;
@@ -20,6 +22,8 @@ import javax.persistence.OneToMany;
 import java.util.ArrayList;
 import java.util.List;
 
+import static ru.ulstu.extractor.branch.model.IndexingStatus.EMPTY;
+
 @Entity
 public class Branch extends BaseEntity {
     private String name;
@@ -32,6 +36,9 @@ public class Branch extends BaseEntity {
     @Fetch(FetchMode.SUBSELECT)
     private List<Commit> commits = new ArrayList<>();
 
+    @Enumerated(EnumType.STRING)
+    private IndexingStatus indexingStatus = EMPTY;
+
     public Branch() {
     }
 
@@ -67,4 +74,12 @@ public class Branch extends BaseEntity {
     public void setCommits(List<Commit> commits) {
         this.commits = commits;
     }
+
+    public IndexingStatus getIndexingStatus() {
+        return indexingStatus;
+    }
+
+    public void setIndexingStatus(IndexingStatus indexingStatus) {
+        this.indexingStatus = indexingStatus;
+    }
 }

src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java

@@ -0,0 +1,5 @@
+package ru.ulstu.extractor.branch.model;
+
+public enum IndexingStatus {
+    EMPTY, INDEXING, ERROR, FINISHED
+}

src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java

@@ -5,17 +5,23 @@ import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Query;
 import ru.ulstu.extractor.branch.model.Branch;
+import ru.ulstu.extractor.branch.model.IndexingStatus;
 import ru.ulstu.extractor.gitrepository.model.GitRepository;
 
 import java.util.List;
+import java.util.Optional;
 
 public interface BranchRepository extends JpaRepository<Branch, Integer> {
     Branch findByGitRepositoryAndName(GitRepository gitRepository, String name);
 
+    Optional<Branch> findByGitRepository_UrlAndName(String repositoryUrl, String name);
+
     @Query("select count(c) from Commit c LEFT JOIN c.branch b  LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2")
     int getCommitsCount(Integer repositoryId, String name);
 
     List<Branch> findByGitRepositoryId(Integer repositoryId);
 
     Page<Branch> findByGitRepository(GitRepository gitRepository, Pageable pageable);
+
+    List<Branch> findAllByIndexingStatus(IndexingStatus indexingStatus);
 }

src/main/java/ru/ulstu/extractor/branch/service/BranchService.java

@@ -10,6 +10,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import ru.ulstu.extractor.branch.model.Branch;
+import ru.ulstu.extractor.branch.model.IndexingStatus;
 import ru.ulstu.extractor.branch.repository.BranchRepository;
 import ru.ulstu.extractor.commit.model.Commit;
 import ru.ulstu.extractor.commit.service.CommitService;
@@ -32,6 +33,11 @@ public class BranchService {
         this.commitService = commitService;
     }
 
+    @Transactional
+    public Branch save(Branch branch) {
+        return branchRepository.save(branch);
+    }
+
     @Transactional
     public Branch save(Branch branch, List<Commit> commits) {
         LOG.debug("Start save {} branch with {} commits ", branch.getName(), commits.size());
@@ -56,6 +62,13 @@ public class BranchService {
         return branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
     }
 
+    public Branch findByRepositoryAndNameOrCreate(GitRepository gitRepository, String branchName) {
+        Branch branch = branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
+        return branch == null
+                ? branchRepository.save(new Branch(gitRepository, branchName))
+                : branch;
+    }
+
     public Optional<Branch> findByBranchId(Integer branchId) {
         return branchRepository.findById(branchId);
     }
@@ -63,4 +76,14 @@ public class BranchService {
     public List<Branch> findAll() {
         return branchRepository.findAll();
     }
+
+    public List<Branch> findAllByIndexingStatus(IndexingStatus indexingStatus) {
+        return branchRepository.findAllByIndexingStatus(indexingStatus);
+    }
+
+    @Transactional
+    public Branch updateStatus(Branch branch, IndexingStatus indexingStatus) {
+        branch.setIndexingStatus(indexingStatus);
+        return branchRepository.save(branch);
+    }
 }

src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java

@@ -17,8 +17,10 @@ import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.servlet.mvc.support.RedirectAttributes;
 import ru.ulstu.extractor.branch.model.Branch;
+import ru.ulstu.extractor.branch.model.IndexingStatus;
 import ru.ulstu.extractor.branch.service.BranchService;
 import ru.ulstu.extractor.gitrepository.model.FilterForm;
+import ru.ulstu.extractor.gitrepository.model.GitRepository;
 import ru.ulstu.extractor.gitrepository.model.RepoForm;
 import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
 import ru.ulstu.extractor.gitrepository.service.IndexService;
@@ -27,8 +29,8 @@ import springfox.documentation.annotations.ApiIgnore;
 import java.io.IOException;
 import java.util.List;
 
-import static ru.ulstu.extractor.core.Route.FILTER_COMMITS;
 import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY;
+import static ru.ulstu.extractor.core.Route.LIST_REPOSITORY_BRANCHES;
 import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH;
 
 @Controller
@@ -70,16 +72,18 @@ public class GitIndexingController {
         if (repoForm.getBranch() == null) {
             return INDEXING_NEW_REPOSITORY;
         } else {
+            GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(repoForm.getRepo());
+            Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, repoForm.getBranch());
+            branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
             try {
-                indexService.index(repoForm.getRepo(), repoForm.getBranch());
+                indexService.index(gitRepository, branch);
             } catch (IOException | GitAPIException ex) {
                 ex.printStackTrace();
                 model.addAttribute("error", ex.getMessage());
                 return INDEXING_NEW_REPOSITORY;
             }
-            redirectAttributes.addAttribute("repositoryUrl", repoForm.getRepo());
+            redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
-            redirectAttributes.addAttribute("branchName", repoForm.getBranch());
+            return "redirect:/" + LIST_REPOSITORY_BRANCHES;
-            return "redirect:/" + FILTER_COMMITS;
         }
     }
 
@@ -87,6 +91,9 @@ public class GitIndexingController {
     public String reindexBranch(Model model,
                                 RedirectAttributes redirectAttributes,
                                 @RequestParam Integer branchId) {
+
+        Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id"));
+        branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
         try {
             indexService.index(branchId);
         } catch (IOException | GitAPIException ex) {
@@ -94,9 +101,7 @@ public class GitIndexingController {
             model.addAttribute("error", ex.getMessage());
             return INDEXING_NEW_REPOSITORY;
         }
-        Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id"));
+        redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
-        redirectAttributes.addAttribute("repositoryUrl", branch.getGitRepository().getUrl());
+        return "redirect:/" + LIST_REPOSITORY_BRANCHES;
-        redirectAttributes.addAttribute("branchName", branch.getName());
-        return "redirect:/" + FILTER_COMMITS;
     }
 }

src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java

@@ -187,7 +187,6 @@ public class GitRepositoryService {
         return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl));
     }
 
-
     public void remove(String repositoryUrl) throws IOException {
         FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl));
     }
@@ -420,6 +419,7 @@ public class GitRepositoryService {
         lineChange.setAdded(true);
         return lineChange;
     }
+
     public Page<GitRepository> findAll(Pageable pageable) {
         return gitRepositoryRepository.findAll(pageable);
     }
@@ -427,4 +427,11 @@ public class GitRepositoryService {
     public GitRepository findById(Integer id) {
         return gitRepositoryRepository.getOne(id);
     }
+
+    public GitRepository findByUrlOrCreate(String gitRepositoryUrl) {
+        GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl);
+        return gitRepository == null
+                ? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl))
+                : gitRepository;
+    }
 }

src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java

@@ -9,8 +9,11 @@ import com.sun.istack.NotNull;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 import ru.ulstu.extractor.branch.model.Branch;
+import ru.ulstu.extractor.branch.model.IndexingStatus;
 import ru.ulstu.extractor.branch.service.BranchService;
 import ru.ulstu.extractor.commit.model.Commit;
 import ru.ulstu.extractor.gitrepository.model.GitRepository;
@@ -20,6 +23,8 @@ import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 @Service
 public class IndexService {
@@ -29,6 +34,7 @@ public class IndexService {
     private final GitRepositoryRepository gitRepositoryRepository;
     private final BranchService branchService;
     private final List<AbstractTimeSeriesCreator> timeSeriesCreators;
+    private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10);
 
     public IndexService(GitRepositoryService gitRepositoryService,
                         GitRepositoryRepository gitRepositoryRepository,
@@ -40,35 +46,64 @@ public class IndexService {
         this.timeSeriesCreators = timeSeriesCreators;
     }
 
+    @Transactional
+    @Async
     public void index(@NotNull Integer branchId) throws GitAPIException, IOException {
         Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id"));
-        index(branch.getGitRepository().getUrl(), branch.getName());
+        index(branch.getGitRepository(), branch);
     }
 
-    public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException {
+    @Transactional
-        GitRepository gitRepository = gitRepositoryRepository.findByUrl(repositoryUrl);
+    @Async
-        if (gitRepository == null) {
+    public void index(String gitRepositoryUrl, String branchName) throws GitAPIException, IOException {
-            gitRepository = gitRepositoryRepository.save(new GitRepository(repositoryUrl));
+        GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(gitRepositoryUrl);
-        }
+        Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, branchName);
-        Branch branch = branchService.findByRepositoryAndName(gitRepository, branchName);
+        branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
-        if (branch == null) {
+        index(gitRepository, branch);
-            branch = new Branch(gitRepository, branchName);
     }
+
+    @Transactional
+    @Async
+    public void index(GitRepository gitRepository, Branch branch) throws GitAPIException, IOException {
         branchService.save(branch, Collections.emptyList());
+        branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
         int commitsFrom = 0;
         int commitsTo = COMMITS_PAGE_SIZE;
-        List<Commit> commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true);
+        List<Commit> commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, true);
         while (!commits.isEmpty()) {
             LOG.debug("{} commits loaded.", commits.size());
             branchService.addCommits(branch, commits);
-            LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName);
+            LOG.debug("{} commits successfully saved. {} {}", commits.size(), gitRepository.getUrl(), branch.getName());
             commitsFrom += COMMITS_PAGE_SIZE;
             commitsTo += COMMITS_PAGE_SIZE;
-            commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false);
+            commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, false);
         }
         Integer repositoryId = gitRepository.getId();
         final Branch branchForSave = branch;
         timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave));
-        LOG.debug("Complete indexing {} branch", branchName);
+        branchService.updateStatus(branch, IndexingStatus.FINISHED);
+        LOG.debug("Complete indexing {} branch", branch.getName());
+    }
+
+    @Transactional
+    @Async
+    public void indexFailedBranchesOnStart() {
+        LOG.info("Старт проверки незавершенных задач для индексирований...");
+
+        List<Branch> failedBranches = branchService.findAllByIndexingStatus(IndexingStatus.INDEXING);
+        if (failedBranches.size() > 0) {
+            LOG.info("Найдено {} незавершенных задач для индексирования", failedBranches.size());
+            failedBranches.forEach(failedBranch -> {
+                try {
+                    index(failedBranch.getId());
+                    LOG.info("Завершено индексирование ветки {}", failedBranch.getName());
+                } catch (Exception ex) {
+                    LOG.warn(ex.getMessage());
+                }
+            });
+            LOG.info("Завершено индексирование незавершенных задач");
+        } else {
+            LOG.info("Не найдено незавершенных веток для индексирования");
+        }
     }
 }

src/main/java/ru/ulstu/extractor/ts/creator/AuthorsCommitTS.java

@@ -1,30 +1,39 @@
 package ru.ulstu.extractor.ts.creator;
 
 import org.springframework.stereotype.Component;
+import ru.ulstu.extractor.author.model.Author;
+import ru.ulstu.extractor.author.service.AuthorService;
 import ru.ulstu.extractor.branch.service.BranchService;
+import ru.ulstu.extractor.commit.model.Commit;
+import ru.ulstu.extractor.commit.service.CommitService;
 import ru.ulstu.extractor.gitrepository.model.GitRepository;
 import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
 import ru.ulstu.extractor.ts.model.TimeSeries;
 import ru.ulstu.extractor.ts.model.TimeSeriesType;
+import ru.ulstu.extractor.ts.model.TimeSeriesValue;
 import ru.ulstu.extractor.ts.service.TimeSeriesService;
-import ru.ulstu.extractor.ts.util.Dummy;
 
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.List;
 
 @Component
 public class AuthorsCommitTS extends AbstractTimeSeriesCreator {
     private final TimeSeriesService timeSeriesService;
+    private final CommitService commitService;
+    private final AuthorService authorService;
+    private final GitRepositoryService gitRepositoryService;
     private final BranchService branchService;
 
-    private final GitRepositoryService gitRepositoryService;
-
     public AuthorsCommitTS(TimeSeriesService timeSeriesService,
-                           BranchService branchService,
+                           CommitService commitService,
-                           GitRepositoryService gitRepositoryService) {
+                           AuthorService authorService,
+                           GitRepositoryService gitRepositoryService,
+                           BranchService branchService) {
         this.timeSeriesService = timeSeriesService;
-        this.branchService = branchService;
+        this.commitService = commitService;
+        this.authorService = authorService;
         this.gitRepositoryService = gitRepositoryService;
+        this.branchService = branchService;
     }
 
     @Override
@@ -34,17 +43,29 @@ public class AuthorsCommitTS extends AbstractTimeSeriesCreator {
 
     @Override
     public List<TimeSeries> getTimeSeries(Integer repositoryId, String branchName) {
+        List<TimeSeries> timeSeriesResult = new ArrayList<>();
+        List<Author> authors = authorService.findAll();
         GitRepository gitRepository = gitRepositoryService.findById(repositoryId);
-        //TODO: добавить постраничное чтение
+        List<Commit> commits = new ArrayList<>(commitService.findByRepositoryIdAndName(repositoryId, branchName));
-        return Collections.singletonList(
+        for (Author author : authors) {
-                new TimeSeries(
+            TimeSeries timeSeries = new TimeSeries(
-                        String.format("%s %s %s",
+                    String.format("%s %s %s %s",
                             gitRepository.getName(),
                             branchName,
+                            author.getName(),
                             getTimeSeriesType().getDescription()),
                     branchService.findByRepositoryAndName(gitRepository, branchName),
-                        getTimeSeriesType(),
+                    getTimeSeriesType());
-                        Dummy.getDefaultTimeSeries()));
+            for (Commit commit : commits) {
+                if (commit.getAuthor().equals(author)) {
+                    timeSeries.getValues().add(new TimeSeriesValue(commit.getDate(), 1.0));
+                }
+            }
+            if (!timeSeries.getValues().isEmpty()) {
+                timeSeriesResult.add(timeSeries);
+            }
+        }
+        return timeSeriesResult;
     }
 
     @Override

src/main/resources/db/changelog-20221012_170000-schema.xml

@@ -99,4 +99,12 @@
                OR branch_id is null;
         </sql>
     </changeSet>
+    <changeSet author="orion" id="20230207-220000-1">
+        <addColumn tableName="branch">
+            <column name="indexing_status" type="varchar(100)"/>
+        </addColumn>
+        <update tableName="branch">
+            <column name="indexing_status" value="FINISHED"/>
+        </update>
+    </changeSet>
 </databaseChangeLog>

src/main/resources/templates/listBranches.html

@@ -12,21 +12,25 @@
         <thead class="thead-dark">
         <tr>
             <th scope="col">Ветки</th>
-            <th width="10%"></th>
+            <th width="20%"></th>
         </tr>
         </thead>
         <tbody>
         <tr th:each="branch: ${branches}">
             <td>
                 <a th:href="@{${@route.FILTER_COMMITS} + '?branchName='+${branch.name}+'&repositoryUrl='+${repository.url}}"
-                   th:text="${branch.name}"/>
+                   th:text="${branch.name}"></a>
             </td>
             <td>
                 <a role="button" class="btn btn-primary" title="Повторить индексацию ветки"
+                   th:if="${branch.indexingStatus.name() != 'INDEXING'}"
                    th:href="@{'reindexBranch?branchId=' + ${branch.id}}">
                     <i class="fa fa-repeat" aria-hidden="true"></i>
                 </a>
-                <a role="button" class="btn btn-danger"
+                <a role="button" class="btn btn-dark disabled"
+                   th:if="${branch.indexingStatus.name() == 'INDEXING'}">Индексируется...
+                </a>
+                <a role="button" class="btn btn-danger" th:if="${branch.indexingStatus.name() != 'INDEXING'}"
                    th:href="@{'deleteBranch?id=' + ${branch.id} + '&repositoryId='+${repository.id}}"
                    onclick="return confirm('Удалить ветку?')">
                     <i class="fa fa-times" aria-hidden="true"></i>