async indexing #88

Merged
romanov73 merged 5 commits from 81-async-indexing into master 2023-02-13 17:56:23 +04:00
5 changed files with 65 additions and 1 deletions
Showing only changes of commit 11b42d7e09 - Show all commits

View File

@ -2,12 +2,29 @@ package ru.ulstu.extractor;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.EnableScheduling;
import ru.ulstu.extractor.gitrepository.service.IndexService;
@SpringBootApplication @SpringBootApplication
@EnableScheduling @EnableScheduling
@EnableAsync
public class GitExtractorApplication { public class GitExtractorApplication {
private final IndexService indexService;
public GitExtractorApplication(IndexService indexService) {
this.indexService = indexService;
}
public static void main(String[] args) { public static void main(String[] args) {
SpringApplication.run(GitExtractorApplication.class, args); SpringApplication.run(GitExtractorApplication.class, args);
} }
@EventListener(ApplicationReadyEvent.class)
public void doSomethingAfterStartup() {
indexService.indexFailedBranchesOnStart();
}
} }

View File

@ -5,6 +5,7 @@ import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query; import org.springframework.data.jpa.repository.Query;
import ru.ulstu.extractor.branch.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.branch.model.IndexingStatus;
import ru.ulstu.extractor.gitrepository.model.GitRepository; import ru.ulstu.extractor.gitrepository.model.GitRepository;
import java.util.List; import java.util.List;
@ -18,4 +19,6 @@ public interface BranchRepository extends JpaRepository<Branch, Integer> {
List<Branch> findByGitRepositoryId(Integer repositoryId); List<Branch> findByGitRepositoryId(Integer repositoryId);
Page<Branch> findByGitRepository(GitRepository gitRepository, Pageable pageable); Page<Branch> findByGitRepository(GitRepository gitRepository, Pageable pageable);
List<Branch> findAllByIndexingStatus(IndexingStatus indexingStatus);
} }

View File

@ -10,6 +10,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import ru.ulstu.extractor.branch.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.branch.model.IndexingStatus;
import ru.ulstu.extractor.branch.repository.BranchRepository; import ru.ulstu.extractor.branch.repository.BranchRepository;
import ru.ulstu.extractor.commit.model.Commit; import ru.ulstu.extractor.commit.model.Commit;
import ru.ulstu.extractor.commit.service.CommitService; import ru.ulstu.extractor.commit.service.CommitService;
@ -32,6 +33,11 @@ public class BranchService {
this.commitService = commitService; this.commitService = commitService;
} }
@Transactional
public Branch save(Branch branch) {
return branchRepository.save(branch);
}
@Transactional @Transactional
public Branch save(Branch branch, List<Commit> commits) { public Branch save(Branch branch, List<Commit> commits) {
LOG.debug("Start save {} branch with {} commits ", branch.getName(), commits.size()); LOG.debug("Start save {} branch with {} commits ", branch.getName(), commits.size());
@ -63,4 +69,14 @@ public class BranchService {
public List<Branch> findAll() { public List<Branch> findAll() {
return branchRepository.findAll(); return branchRepository.findAll();
} }
public List<Branch> findAllByIndexingStatus(IndexingStatus indexingStatus) {
return branchRepository.findAllByIndexingStatus(indexingStatus);
}
@Transactional
public Branch updateStatus(Branch branch, IndexingStatus indexingStatus) {
branch.setIndexingStatus(indexingStatus);
return branchRepository.save(branch);
}
} }

View File

@ -9,8 +9,11 @@ import com.sun.istack.NotNull;
import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.GitAPIException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.ulstu.extractor.branch.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.branch.model.IndexingStatus;
import ru.ulstu.extractor.branch.service.BranchService; import ru.ulstu.extractor.branch.service.BranchService;
import ru.ulstu.extractor.commit.model.Commit; import ru.ulstu.extractor.commit.model.Commit;
import ru.ulstu.extractor.gitrepository.model.GitRepository; import ru.ulstu.extractor.gitrepository.model.GitRepository;
@ -55,6 +58,7 @@ public class IndexService {
branch = new Branch(gitRepository, branchName); branch = new Branch(gitRepository, branchName);
} }
branchService.save(branch, Collections.emptyList()); branchService.save(branch, Collections.emptyList());
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
int commitsFrom = 0; int commitsFrom = 0;
int commitsTo = COMMITS_PAGE_SIZE; int commitsTo = COMMITS_PAGE_SIZE;
List<Commit> commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true); List<Commit> commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true);
@ -69,6 +73,30 @@ public class IndexService {
Integer repositoryId = gitRepository.getId(); Integer repositoryId = gitRepository.getId();
final Branch branchForSave = branch; final Branch branchForSave = branch;
timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave)); timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave));
branchService.updateStatus(branch, IndexingStatus.FINISHED);
LOG.debug("Complete indexing {} branch", branchName); LOG.debug("Complete indexing {} branch", branchName);
} }
@Transactional
@Async
public void indexFailedBranchesOnStart() {
LOG.info("Старт проверки незавершенных задач для индексирований...");
List<Branch> failedBranches = branchService.findAllByIndexingStatus(IndexingStatus.INDEXING);
if (failedBranches.size() > 0) {
LOG.info("Найдено {} незавершенных задач для индексирования", failedBranches.size());
failedBranches.forEach(failedBranch -> {
try {
index(failedBranch.getId());
LOG.info("Завершено индексирование ветки {}", failedBranch.getName());
branchService.updateStatus(failedBranch, IndexingStatus.FINISHED);
} catch (Exception ex) {
LOG.warn(ex.getMessage());
}
});
LOG.info("Завершено индексирование незавершенных задач");
} else {
LOG.info("Не найдено незавершенных веток для индексирования");
}
}
} }

View File

@ -30,7 +30,7 @@
<a role="button" class="btn btn-dark disabled" <a role="button" class="btn btn-dark disabled"
th:if="${branch.indexingStatus.name() == 'INDEXING'}">Индексируется... th:if="${branch.indexingStatus.name() == 'INDEXING'}">Индексируется...
</a> </a>
<a role="button" class="btn btn-danger" <a role="button" class="btn btn-danger" th:if="${branch.indexingStatus.name() != 'INDEXING'}"
th:href="@{'deleteBranch?id=' + ${branch.id} + '&repositoryId='+${repository.id}}" th:href="@{'deleteBranch?id=' + ${branch.id} + '&repositoryId='+${repository.id}}"
onclick="return confirm('Удалить ветку?')"> onclick="return confirm('Удалить ветку?')">
<i class="fa fa-times" aria-hidden="true"></i> <i class="fa fa-times" aria-hidden="true"></i>