Merge pull request 'async indexing' (#88) from 81-async-indexing into master
Reviewed-on: #88
This commit is contained in:
commit
b8c6aaf9da
@ -2,12 +2,29 @@ package ru.ulstu.extractor;
|
|||||||
|
|
||||||
import org.springframework.boot.SpringApplication;
|
import org.springframework.boot.SpringApplication;
|
||||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||||
|
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||||
|
import org.springframework.context.event.EventListener;
|
||||||
|
import org.springframework.scheduling.annotation.EnableAsync;
|
||||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||||
|
import ru.ulstu.extractor.gitrepository.service.IndexService;
|
||||||
|
|
||||||
@SpringBootApplication
|
@SpringBootApplication
|
||||||
@EnableScheduling
|
@EnableScheduling
|
||||||
|
@EnableAsync
|
||||||
public class GitExtractorApplication {
|
public class GitExtractorApplication {
|
||||||
|
private final IndexService indexService;
|
||||||
|
|
||||||
|
public GitExtractorApplication(IndexService indexService) {
|
||||||
|
this.indexService = indexService;
|
||||||
|
}
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
SpringApplication.run(GitExtractorApplication.class, args);
|
SpringApplication.run(GitExtractorApplication.class, args);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@EventListener(ApplicationReadyEvent.class)
|
||||||
|
public void doSomethingAfterStartup() {
|
||||||
|
indexService.indexFailedBranchesOnStart();
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -13,6 +13,8 @@ import ru.ulstu.extractor.gitrepository.model.GitRepository;
|
|||||||
|
|
||||||
import javax.persistence.CascadeType;
|
import javax.persistence.CascadeType;
|
||||||
import javax.persistence.Entity;
|
import javax.persistence.Entity;
|
||||||
|
import javax.persistence.EnumType;
|
||||||
|
import javax.persistence.Enumerated;
|
||||||
import javax.persistence.FetchType;
|
import javax.persistence.FetchType;
|
||||||
import javax.persistence.JoinColumn;
|
import javax.persistence.JoinColumn;
|
||||||
import javax.persistence.ManyToOne;
|
import javax.persistence.ManyToOne;
|
||||||
@ -20,6 +22,8 @@ import javax.persistence.OneToMany;
|
|||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
import static ru.ulstu.extractor.branch.model.IndexingStatus.EMPTY;
|
||||||
|
|
||||||
@Entity
|
@Entity
|
||||||
public class Branch extends BaseEntity {
|
public class Branch extends BaseEntity {
|
||||||
private String name;
|
private String name;
|
||||||
@ -32,6 +36,9 @@ public class Branch extends BaseEntity {
|
|||||||
@Fetch(FetchMode.SUBSELECT)
|
@Fetch(FetchMode.SUBSELECT)
|
||||||
private List<Commit> commits = new ArrayList<>();
|
private List<Commit> commits = new ArrayList<>();
|
||||||
|
|
||||||
|
@Enumerated(EnumType.STRING)
|
||||||
|
private IndexingStatus indexingStatus = EMPTY;
|
||||||
|
|
||||||
public Branch() {
|
public Branch() {
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -67,4 +74,12 @@ public class Branch extends BaseEntity {
|
|||||||
public void setCommits(List<Commit> commits) {
|
public void setCommits(List<Commit> commits) {
|
||||||
this.commits = commits;
|
this.commits = commits;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public IndexingStatus getIndexingStatus() {
|
||||||
|
return indexingStatus;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIndexingStatus(IndexingStatus indexingStatus) {
|
||||||
|
this.indexingStatus = indexingStatus;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,5 @@
|
|||||||
|
package ru.ulstu.extractor.branch.model;
|
||||||
|
|
||||||
|
public enum IndexingStatus {
|
||||||
|
EMPTY, INDEXING, ERROR, FINISHED
|
||||||
|
}
|
@ -5,17 +5,23 @@ import org.springframework.data.domain.Pageable;
|
|||||||
import org.springframework.data.jpa.repository.JpaRepository;
|
import org.springframework.data.jpa.repository.JpaRepository;
|
||||||
import org.springframework.data.jpa.repository.Query;
|
import org.springframework.data.jpa.repository.Query;
|
||||||
import ru.ulstu.extractor.branch.model.Branch;
|
import ru.ulstu.extractor.branch.model.Branch;
|
||||||
|
import ru.ulstu.extractor.branch.model.IndexingStatus;
|
||||||
import ru.ulstu.extractor.gitrepository.model.GitRepository;
|
import ru.ulstu.extractor.gitrepository.model.GitRepository;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
public interface BranchRepository extends JpaRepository<Branch, Integer> {
|
public interface BranchRepository extends JpaRepository<Branch, Integer> {
|
||||||
Branch findByGitRepositoryAndName(GitRepository gitRepository, String name);
|
Branch findByGitRepositoryAndName(GitRepository gitRepository, String name);
|
||||||
|
|
||||||
|
Optional<Branch> findByGitRepository_UrlAndName(String repositoryUrl, String name);
|
||||||
|
|
||||||
@Query("select count(c) from Commit c LEFT JOIN c.branch b LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2")
|
@Query("select count(c) from Commit c LEFT JOIN c.branch b LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2")
|
||||||
int getCommitsCount(Integer repositoryId, String name);
|
int getCommitsCount(Integer repositoryId, String name);
|
||||||
|
|
||||||
List<Branch> findByGitRepositoryId(Integer repositoryId);
|
List<Branch> findByGitRepositoryId(Integer repositoryId);
|
||||||
|
|
||||||
Page<Branch> findByGitRepository(GitRepository gitRepository, Pageable pageable);
|
Page<Branch> findByGitRepository(GitRepository gitRepository, Pageable pageable);
|
||||||
|
|
||||||
|
List<Branch> findAllByIndexingStatus(IndexingStatus indexingStatus);
|
||||||
}
|
}
|
||||||
|
@ -10,6 +10,7 @@ import org.slf4j.LoggerFactory;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import ru.ulstu.extractor.branch.model.Branch;
|
import ru.ulstu.extractor.branch.model.Branch;
|
||||||
|
import ru.ulstu.extractor.branch.model.IndexingStatus;
|
||||||
import ru.ulstu.extractor.branch.repository.BranchRepository;
|
import ru.ulstu.extractor.branch.repository.BranchRepository;
|
||||||
import ru.ulstu.extractor.commit.model.Commit;
|
import ru.ulstu.extractor.commit.model.Commit;
|
||||||
import ru.ulstu.extractor.commit.service.CommitService;
|
import ru.ulstu.extractor.commit.service.CommitService;
|
||||||
@ -32,6 +33,11 @@ public class BranchService {
|
|||||||
this.commitService = commitService;
|
this.commitService = commitService;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public Branch save(Branch branch) {
|
||||||
|
return branchRepository.save(branch);
|
||||||
|
}
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public Branch save(Branch branch, List<Commit> commits) {
|
public Branch save(Branch branch, List<Commit> commits) {
|
||||||
LOG.debug("Start save {} branch with {} commits ", branch.getName(), commits.size());
|
LOG.debug("Start save {} branch with {} commits ", branch.getName(), commits.size());
|
||||||
@ -56,6 +62,13 @@ public class BranchService {
|
|||||||
return branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
|
return branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Branch findByRepositoryAndNameOrCreate(GitRepository gitRepository, String branchName) {
|
||||||
|
Branch branch = branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
|
||||||
|
return branch == null
|
||||||
|
? branchRepository.save(new Branch(gitRepository, branchName))
|
||||||
|
: branch;
|
||||||
|
}
|
||||||
|
|
||||||
public Optional<Branch> findByBranchId(Integer branchId) {
|
public Optional<Branch> findByBranchId(Integer branchId) {
|
||||||
return branchRepository.findById(branchId);
|
return branchRepository.findById(branchId);
|
||||||
}
|
}
|
||||||
@ -63,4 +76,14 @@ public class BranchService {
|
|||||||
public List<Branch> findAll() {
|
public List<Branch> findAll() {
|
||||||
return branchRepository.findAll();
|
return branchRepository.findAll();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public List<Branch> findAllByIndexingStatus(IndexingStatus indexingStatus) {
|
||||||
|
return branchRepository.findAllByIndexingStatus(indexingStatus);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
public Branch updateStatus(Branch branch, IndexingStatus indexingStatus) {
|
||||||
|
branch.setIndexingStatus(indexingStatus);
|
||||||
|
return branchRepository.save(branch);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -17,8 +17,10 @@ import org.springframework.web.bind.annotation.RequestMethod;
|
|||||||
import org.springframework.web.bind.annotation.RequestParam;
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
import org.springframework.web.servlet.mvc.support.RedirectAttributes;
|
import org.springframework.web.servlet.mvc.support.RedirectAttributes;
|
||||||
import ru.ulstu.extractor.branch.model.Branch;
|
import ru.ulstu.extractor.branch.model.Branch;
|
||||||
|
import ru.ulstu.extractor.branch.model.IndexingStatus;
|
||||||
import ru.ulstu.extractor.branch.service.BranchService;
|
import ru.ulstu.extractor.branch.service.BranchService;
|
||||||
import ru.ulstu.extractor.gitrepository.model.FilterForm;
|
import ru.ulstu.extractor.gitrepository.model.FilterForm;
|
||||||
|
import ru.ulstu.extractor.gitrepository.model.GitRepository;
|
||||||
import ru.ulstu.extractor.gitrepository.model.RepoForm;
|
import ru.ulstu.extractor.gitrepository.model.RepoForm;
|
||||||
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
|
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
|
||||||
import ru.ulstu.extractor.gitrepository.service.IndexService;
|
import ru.ulstu.extractor.gitrepository.service.IndexService;
|
||||||
@ -27,8 +29,8 @@ import springfox.documentation.annotations.ApiIgnore;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import static ru.ulstu.extractor.core.Route.FILTER_COMMITS;
|
|
||||||
import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY;
|
import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY;
|
||||||
|
import static ru.ulstu.extractor.core.Route.LIST_REPOSITORY_BRANCHES;
|
||||||
import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH;
|
import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH;
|
||||||
|
|
||||||
@Controller
|
@Controller
|
||||||
@ -70,16 +72,18 @@ public class GitIndexingController {
|
|||||||
if (repoForm.getBranch() == null) {
|
if (repoForm.getBranch() == null) {
|
||||||
return INDEXING_NEW_REPOSITORY;
|
return INDEXING_NEW_REPOSITORY;
|
||||||
} else {
|
} else {
|
||||||
|
GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(repoForm.getRepo());
|
||||||
|
Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, repoForm.getBranch());
|
||||||
|
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
|
||||||
try {
|
try {
|
||||||
indexService.index(repoForm.getRepo(), repoForm.getBranch());
|
indexService.index(gitRepository, branch);
|
||||||
} catch (IOException | GitAPIException ex) {
|
} catch (IOException | GitAPIException ex) {
|
||||||
ex.printStackTrace();
|
ex.printStackTrace();
|
||||||
model.addAttribute("error", ex.getMessage());
|
model.addAttribute("error", ex.getMessage());
|
||||||
return INDEXING_NEW_REPOSITORY;
|
return INDEXING_NEW_REPOSITORY;
|
||||||
}
|
}
|
||||||
redirectAttributes.addAttribute("repositoryUrl", repoForm.getRepo());
|
redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
|
||||||
redirectAttributes.addAttribute("branchName", repoForm.getBranch());
|
return "redirect:/" + LIST_REPOSITORY_BRANCHES;
|
||||||
return "redirect:/" + FILTER_COMMITS;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -87,6 +91,9 @@ public class GitIndexingController {
|
|||||||
public String reindexBranch(Model model,
|
public String reindexBranch(Model model,
|
||||||
RedirectAttributes redirectAttributes,
|
RedirectAttributes redirectAttributes,
|
||||||
@RequestParam Integer branchId) {
|
@RequestParam Integer branchId) {
|
||||||
|
|
||||||
|
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id"));
|
||||||
|
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
|
||||||
try {
|
try {
|
||||||
indexService.index(branchId);
|
indexService.index(branchId);
|
||||||
} catch (IOException | GitAPIException ex) {
|
} catch (IOException | GitAPIException ex) {
|
||||||
@ -94,9 +101,7 @@ public class GitIndexingController {
|
|||||||
model.addAttribute("error", ex.getMessage());
|
model.addAttribute("error", ex.getMessage());
|
||||||
return INDEXING_NEW_REPOSITORY;
|
return INDEXING_NEW_REPOSITORY;
|
||||||
}
|
}
|
||||||
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id"));
|
redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
|
||||||
redirectAttributes.addAttribute("repositoryUrl", branch.getGitRepository().getUrl());
|
return "redirect:/" + LIST_REPOSITORY_BRANCHES;
|
||||||
redirectAttributes.addAttribute("branchName", branch.getName());
|
|
||||||
return "redirect:/" + FILTER_COMMITS;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -187,7 +187,6 @@ public class GitRepositoryService {
|
|||||||
return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl));
|
return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public void remove(String repositoryUrl) throws IOException {
|
public void remove(String repositoryUrl) throws IOException {
|
||||||
FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl));
|
FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl));
|
||||||
}
|
}
|
||||||
@ -283,7 +282,7 @@ public class GitRepositoryService {
|
|||||||
List<FileChange> changes = new ArrayList<>();
|
List<FileChange> changes = new ArrayList<>();
|
||||||
String[] strings = output.split("\n");
|
String[] strings = output.split("\n");
|
||||||
Map<String, List<String>> filesContent = getFilesContent(strings);
|
Map<String, List<String>> filesContent = getFilesContent(strings);
|
||||||
for(Map.Entry<String, List<String>> fileSterings: filesContent.entrySet()) {
|
for (Map.Entry<String, List<String>> fileSterings : filesContent.entrySet()) {
|
||||||
FileChange fileChange = new FileChange();
|
FileChange fileChange = new FileChange();
|
||||||
fileChange.setFile(fileSterings.getKey());
|
fileChange.setFile(fileSterings.getKey());
|
||||||
Future<Boolean> futureEntity = executorService.submit(() -> structuralUnitService.containsEntity(getContent(repository, commit, fileSterings.getKey())));
|
Future<Boolean> futureEntity = executorService.submit(() -> structuralUnitService.containsEntity(getContent(repository, commit, fileSterings.getKey())));
|
||||||
@ -327,7 +326,7 @@ public class GitRepositoryService {
|
|||||||
|
|
||||||
private Map<String, List<String>> getFilesContent(String[] commitStrings) {
|
private Map<String, List<String>> getFilesContent(String[] commitStrings) {
|
||||||
int i = 0;
|
int i = 0;
|
||||||
Map<String, List<String>> result = new HashMap<>();
|
Map<String, List<String>> result = new HashMap<>();
|
||||||
while (i < commitStrings.length) {
|
while (i < commitStrings.length) {
|
||||||
Optional<String> maybeFileName = getFileName(commitStrings[i]);
|
Optional<String> maybeFileName = getFileName(commitStrings[i]);
|
||||||
if (maybeFileName.isEmpty()) {
|
if (maybeFileName.isEmpty()) {
|
||||||
@ -368,15 +367,15 @@ public class GitRepositoryService {
|
|||||||
return index;
|
return index;
|
||||||
}
|
}
|
||||||
|
|
||||||
private FileChange getChange(FileChange fileChange, List<String> fileContent){
|
private FileChange getChange(FileChange fileChange, List<String> fileContent) {
|
||||||
int addedLine = 0;
|
int addedLine = 0;
|
||||||
int removedLine = 0;
|
int removedLine = 0;
|
||||||
StringBuilder builder = new StringBuilder();
|
StringBuilder builder = new StringBuilder();
|
||||||
boolean isRemoved = false;
|
boolean isRemoved = false;
|
||||||
boolean isAdded = false;
|
boolean isAdded = false;
|
||||||
for (String line : fileContent){
|
for (String line : fileContent) {
|
||||||
LineChange lineChange = new LineChange();
|
LineChange lineChange = new LineChange();
|
||||||
if (line.startsWith("-")){
|
if (line.startsWith("-")) {
|
||||||
isRemoved = true;
|
isRemoved = true;
|
||||||
if (isAdded) {
|
if (isAdded) {
|
||||||
isAdded = false;
|
isAdded = false;
|
||||||
@ -385,7 +384,7 @@ public class GitRepositoryService {
|
|||||||
}
|
}
|
||||||
builder.append(line).append("\n");
|
builder.append(line).append("\n");
|
||||||
removedLine++;
|
removedLine++;
|
||||||
} else if(line.startsWith("+")){
|
} else if (line.startsWith("+")) {
|
||||||
isAdded = true;
|
isAdded = true;
|
||||||
if (isRemoved) {
|
if (isRemoved) {
|
||||||
isRemoved = false;
|
isRemoved = false;
|
||||||
@ -398,7 +397,7 @@ public class GitRepositoryService {
|
|||||||
if (isRemoved) {
|
if (isRemoved) {
|
||||||
lineChange = setRemoved(lineChange, builder);
|
lineChange = setRemoved(lineChange, builder);
|
||||||
builder.setLength(0);
|
builder.setLength(0);
|
||||||
} else if (isAdded){
|
} else if (isAdded) {
|
||||||
lineChange = setAdded(lineChange, builder);
|
lineChange = setAdded(lineChange, builder);
|
||||||
builder.setLength(0);
|
builder.setLength(0);
|
||||||
}
|
}
|
||||||
@ -409,17 +408,18 @@ public class GitRepositoryService {
|
|||||||
return fileChange;
|
return fileChange;
|
||||||
}
|
}
|
||||||
|
|
||||||
private LineChange setRemoved(LineChange lineChange, StringBuilder builder){
|
private LineChange setRemoved(LineChange lineChange, StringBuilder builder) {
|
||||||
lineChange.setLineFrom(builder.toString());
|
lineChange.setLineFrom(builder.toString());
|
||||||
lineChange.setRemoved(true);
|
lineChange.setRemoved(true);
|
||||||
return lineChange;
|
return lineChange;
|
||||||
}
|
}
|
||||||
|
|
||||||
private LineChange setAdded(LineChange lineChange, StringBuilder builder){
|
private LineChange setAdded(LineChange lineChange, StringBuilder builder) {
|
||||||
lineChange.setLineTo(builder.toString());
|
lineChange.setLineTo(builder.toString());
|
||||||
lineChange.setAdded(true);
|
lineChange.setAdded(true);
|
||||||
return lineChange;
|
return lineChange;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Page<GitRepository> findAll(Pageable pageable) {
|
public Page<GitRepository> findAll(Pageable pageable) {
|
||||||
return gitRepositoryRepository.findAll(pageable);
|
return gitRepositoryRepository.findAll(pageable);
|
||||||
}
|
}
|
||||||
@ -427,4 +427,11 @@ public class GitRepositoryService {
|
|||||||
public GitRepository findById(Integer id) {
|
public GitRepository findById(Integer id) {
|
||||||
return gitRepositoryRepository.getOne(id);
|
return gitRepositoryRepository.getOne(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public GitRepository findByUrlOrCreate(String gitRepositoryUrl) {
|
||||||
|
GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl);
|
||||||
|
return gitRepository == null
|
||||||
|
? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl))
|
||||||
|
: gitRepository;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -9,8 +9,11 @@ import com.sun.istack.NotNull;
|
|||||||
import org.eclipse.jgit.api.errors.GitAPIException;
|
import org.eclipse.jgit.api.errors.GitAPIException;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.scheduling.annotation.Async;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import ru.ulstu.extractor.branch.model.Branch;
|
import ru.ulstu.extractor.branch.model.Branch;
|
||||||
|
import ru.ulstu.extractor.branch.model.IndexingStatus;
|
||||||
import ru.ulstu.extractor.branch.service.BranchService;
|
import ru.ulstu.extractor.branch.service.BranchService;
|
||||||
import ru.ulstu.extractor.commit.model.Commit;
|
import ru.ulstu.extractor.commit.model.Commit;
|
||||||
import ru.ulstu.extractor.gitrepository.model.GitRepository;
|
import ru.ulstu.extractor.gitrepository.model.GitRepository;
|
||||||
@ -20,6 +23,8 @@ import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class IndexService {
|
public class IndexService {
|
||||||
@ -29,6 +34,7 @@ public class IndexService {
|
|||||||
private final GitRepositoryRepository gitRepositoryRepository;
|
private final GitRepositoryRepository gitRepositoryRepository;
|
||||||
private final BranchService branchService;
|
private final BranchService branchService;
|
||||||
private final List<AbstractTimeSeriesCreator> timeSeriesCreators;
|
private final List<AbstractTimeSeriesCreator> timeSeriesCreators;
|
||||||
|
private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10);
|
||||||
|
|
||||||
public IndexService(GitRepositoryService gitRepositoryService,
|
public IndexService(GitRepositoryService gitRepositoryService,
|
||||||
GitRepositoryRepository gitRepositoryRepository,
|
GitRepositoryRepository gitRepositoryRepository,
|
||||||
@ -40,35 +46,64 @@ public class IndexService {
|
|||||||
this.timeSeriesCreators = timeSeriesCreators;
|
this.timeSeriesCreators = timeSeriesCreators;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
@Async
|
||||||
public void index(@NotNull Integer branchId) throws GitAPIException, IOException {
|
public void index(@NotNull Integer branchId) throws GitAPIException, IOException {
|
||||||
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id"));
|
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id"));
|
||||||
index(branch.getGitRepository().getUrl(), branch.getName());
|
index(branch.getGitRepository(), branch);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException {
|
@Transactional
|
||||||
GitRepository gitRepository = gitRepositoryRepository.findByUrl(repositoryUrl);
|
@Async
|
||||||
if (gitRepository == null) {
|
public void index(String gitRepositoryUrl, String branchName) throws GitAPIException, IOException {
|
||||||
gitRepository = gitRepositoryRepository.save(new GitRepository(repositoryUrl));
|
GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(gitRepositoryUrl);
|
||||||
}
|
Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, branchName);
|
||||||
Branch branch = branchService.findByRepositoryAndName(gitRepository, branchName);
|
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
|
||||||
if (branch == null) {
|
index(gitRepository, branch);
|
||||||
branch = new Branch(gitRepository, branchName);
|
}
|
||||||
}
|
|
||||||
|
@Transactional
|
||||||
|
@Async
|
||||||
|
public void index(GitRepository gitRepository, Branch branch) throws GitAPIException, IOException {
|
||||||
branchService.save(branch, Collections.emptyList());
|
branchService.save(branch, Collections.emptyList());
|
||||||
|
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
|
||||||
int commitsFrom = 0;
|
int commitsFrom = 0;
|
||||||
int commitsTo = COMMITS_PAGE_SIZE;
|
int commitsTo = COMMITS_PAGE_SIZE;
|
||||||
List<Commit> commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true);
|
List<Commit> commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, true);
|
||||||
while (!commits.isEmpty()) {
|
while (!commits.isEmpty()) {
|
||||||
LOG.debug("{} commits loaded.", commits.size());
|
LOG.debug("{} commits loaded.", commits.size());
|
||||||
branchService.addCommits(branch, commits);
|
branchService.addCommits(branch, commits);
|
||||||
LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName);
|
LOG.debug("{} commits successfully saved. {} {}", commits.size(), gitRepository.getUrl(), branch.getName());
|
||||||
commitsFrom += COMMITS_PAGE_SIZE;
|
commitsFrom += COMMITS_PAGE_SIZE;
|
||||||
commitsTo += COMMITS_PAGE_SIZE;
|
commitsTo += COMMITS_PAGE_SIZE;
|
||||||
commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false);
|
commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, false);
|
||||||
}
|
}
|
||||||
Integer repositoryId = gitRepository.getId();
|
Integer repositoryId = gitRepository.getId();
|
||||||
final Branch branchForSave = branch;
|
final Branch branchForSave = branch;
|
||||||
timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave));
|
timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave));
|
||||||
LOG.debug("Complete indexing {} branch", branchName);
|
branchService.updateStatus(branch, IndexingStatus.FINISHED);
|
||||||
|
LOG.debug("Complete indexing {} branch", branch.getName());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Transactional
|
||||||
|
@Async
|
||||||
|
public void indexFailedBranchesOnStart() {
|
||||||
|
LOG.info("Старт проверки незавершенных задач для индексирований...");
|
||||||
|
|
||||||
|
List<Branch> failedBranches = branchService.findAllByIndexingStatus(IndexingStatus.INDEXING);
|
||||||
|
if (failedBranches.size() > 0) {
|
||||||
|
LOG.info("Найдено {} незавершенных задач для индексирования", failedBranches.size());
|
||||||
|
failedBranches.forEach(failedBranch -> {
|
||||||
|
try {
|
||||||
|
index(failedBranch.getId());
|
||||||
|
LOG.info("Завершено индексирование ветки {}", failedBranch.getName());
|
||||||
|
} catch (Exception ex) {
|
||||||
|
LOG.warn(ex.getMessage());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
LOG.info("Завершено индексирование незавершенных задач");
|
||||||
|
} else {
|
||||||
|
LOG.info("Не найдено незавершенных веток для индексирования");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,30 +1,39 @@
|
|||||||
package ru.ulstu.extractor.ts.creator;
|
package ru.ulstu.extractor.ts.creator;
|
||||||
|
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
import ru.ulstu.extractor.author.model.Author;
|
||||||
|
import ru.ulstu.extractor.author.service.AuthorService;
|
||||||
import ru.ulstu.extractor.branch.service.BranchService;
|
import ru.ulstu.extractor.branch.service.BranchService;
|
||||||
|
import ru.ulstu.extractor.commit.model.Commit;
|
||||||
|
import ru.ulstu.extractor.commit.service.CommitService;
|
||||||
import ru.ulstu.extractor.gitrepository.model.GitRepository;
|
import ru.ulstu.extractor.gitrepository.model.GitRepository;
|
||||||
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
|
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
|
||||||
import ru.ulstu.extractor.ts.model.TimeSeries;
|
import ru.ulstu.extractor.ts.model.TimeSeries;
|
||||||
import ru.ulstu.extractor.ts.model.TimeSeriesType;
|
import ru.ulstu.extractor.ts.model.TimeSeriesType;
|
||||||
|
import ru.ulstu.extractor.ts.model.TimeSeriesValue;
|
||||||
import ru.ulstu.extractor.ts.service.TimeSeriesService;
|
import ru.ulstu.extractor.ts.service.TimeSeriesService;
|
||||||
import ru.ulstu.extractor.ts.util.Dummy;
|
|
||||||
|
|
||||||
import java.util.Collections;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
public class AuthorsCommitTS extends AbstractTimeSeriesCreator {
|
public class AuthorsCommitTS extends AbstractTimeSeriesCreator {
|
||||||
private final TimeSeriesService timeSeriesService;
|
private final TimeSeriesService timeSeriesService;
|
||||||
|
private final CommitService commitService;
|
||||||
|
private final AuthorService authorService;
|
||||||
|
private final GitRepositoryService gitRepositoryService;
|
||||||
private final BranchService branchService;
|
private final BranchService branchService;
|
||||||
|
|
||||||
private final GitRepositoryService gitRepositoryService;
|
|
||||||
|
|
||||||
public AuthorsCommitTS(TimeSeriesService timeSeriesService,
|
public AuthorsCommitTS(TimeSeriesService timeSeriesService,
|
||||||
BranchService branchService,
|
CommitService commitService,
|
||||||
GitRepositoryService gitRepositoryService) {
|
AuthorService authorService,
|
||||||
|
GitRepositoryService gitRepositoryService,
|
||||||
|
BranchService branchService) {
|
||||||
this.timeSeriesService = timeSeriesService;
|
this.timeSeriesService = timeSeriesService;
|
||||||
this.branchService = branchService;
|
this.commitService = commitService;
|
||||||
|
this.authorService = authorService;
|
||||||
this.gitRepositoryService = gitRepositoryService;
|
this.gitRepositoryService = gitRepositoryService;
|
||||||
|
this.branchService = branchService;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -34,17 +43,29 @@ public class AuthorsCommitTS extends AbstractTimeSeriesCreator {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<TimeSeries> getTimeSeries(Integer repositoryId, String branchName) {
|
public List<TimeSeries> getTimeSeries(Integer repositoryId, String branchName) {
|
||||||
|
List<TimeSeries> timeSeriesResult = new ArrayList<>();
|
||||||
|
List<Author> authors = authorService.findAll();
|
||||||
GitRepository gitRepository = gitRepositoryService.findById(repositoryId);
|
GitRepository gitRepository = gitRepositoryService.findById(repositoryId);
|
||||||
//TODO: добавить постраничное чтение
|
List<Commit> commits = new ArrayList<>(commitService.findByRepositoryIdAndName(repositoryId, branchName));
|
||||||
return Collections.singletonList(
|
for (Author author : authors) {
|
||||||
new TimeSeries(
|
TimeSeries timeSeries = new TimeSeries(
|
||||||
String.format("%s %s %s",
|
String.format("%s %s %s %s",
|
||||||
gitRepository.getName(),
|
gitRepository.getName(),
|
||||||
branchName,
|
branchName,
|
||||||
getTimeSeriesType().getDescription()),
|
author.getName(),
|
||||||
branchService.findByRepositoryAndName(gitRepository, branchName),
|
getTimeSeriesType().getDescription()),
|
||||||
getTimeSeriesType(),
|
branchService.findByRepositoryAndName(gitRepository, branchName),
|
||||||
Dummy.getDefaultTimeSeries()));
|
getTimeSeriesType());
|
||||||
|
for (Commit commit : commits) {
|
||||||
|
if (commit.getAuthor().equals(author)) {
|
||||||
|
timeSeries.getValues().add(new TimeSeriesValue(commit.getDate(), 1.0));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!timeSeries.getValues().isEmpty()) {
|
||||||
|
timeSeriesResult.add(timeSeries);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return timeSeriesResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -99,4 +99,12 @@
|
|||||||
OR branch_id is null;
|
OR branch_id is null;
|
||||||
</sql>
|
</sql>
|
||||||
</changeSet>
|
</changeSet>
|
||||||
|
<changeSet author="orion" id="20230207-220000-1">
|
||||||
|
<addColumn tableName="branch">
|
||||||
|
<column name="indexing_status" type="varchar(100)"/>
|
||||||
|
</addColumn>
|
||||||
|
<update tableName="branch">
|
||||||
|
<column name="indexing_status" value="FINISHED"/>
|
||||||
|
</update>
|
||||||
|
</changeSet>
|
||||||
</databaseChangeLog>
|
</databaseChangeLog>
|
@ -12,21 +12,25 @@
|
|||||||
<thead class="thead-dark">
|
<thead class="thead-dark">
|
||||||
<tr>
|
<tr>
|
||||||
<th scope="col">Ветки</th>
|
<th scope="col">Ветки</th>
|
||||||
<th width="10%"></th>
|
<th width="20%"></th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr th:each="branch: ${branches}">
|
<tr th:each="branch: ${branches}">
|
||||||
<td>
|
<td>
|
||||||
<a th:href="@{${@route.FILTER_COMMITS} + '?branchName='+${branch.name}+'&repositoryUrl='+${repository.url}}"
|
<a th:href="@{${@route.FILTER_COMMITS} + '?branchName='+${branch.name}+'&repositoryUrl='+${repository.url}}"
|
||||||
th:text="${branch.name}"/>
|
th:text="${branch.name}"></a>
|
||||||
</td>
|
</td>
|
||||||
<td>
|
<td>
|
||||||
<a role="button" class="btn btn-primary" title="Повторить индексацию ветки"
|
<a role="button" class="btn btn-primary" title="Повторить индексацию ветки"
|
||||||
|
th:if="${branch.indexingStatus.name() != 'INDEXING'}"
|
||||||
th:href="@{'reindexBranch?branchId=' + ${branch.id}}">
|
th:href="@{'reindexBranch?branchId=' + ${branch.id}}">
|
||||||
<i class="fa fa-repeat" aria-hidden="true"></i>
|
<i class="fa fa-repeat" aria-hidden="true"></i>
|
||||||
</a>
|
</a>
|
||||||
<a role="button" class="btn btn-danger"
|
<a role="button" class="btn btn-dark disabled"
|
||||||
|
th:if="${branch.indexingStatus.name() == 'INDEXING'}">Индексируется...
|
||||||
|
</a>
|
||||||
|
<a role="button" class="btn btn-danger" th:if="${branch.indexingStatus.name() != 'INDEXING'}"
|
||||||
th:href="@{'deleteBranch?id=' + ${branch.id} + '&repositoryId='+${repository.id}}"
|
th:href="@{'deleteBranch?id=' + ${branch.id} + '&repositoryId='+${repository.id}}"
|
||||||
onclick="return confirm('Удалить ветку?')">
|
onclick="return confirm('Удалить ветку?')">
|
||||||
<i class="fa fa-times" aria-hidden="true"></i>
|
<i class="fa fa-times" aria-hidden="true"></i>
|
||||||
|
Loading…
Reference in New Issue
Block a user