romanov73/git-extractor
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

async indexing #88

Merged
romanov73 merged 5 commits from 81-async-indexing into master 2023-02-13 17:56:23 +04:00
Conversation 1 Commits 5 Files Changed 11 +64 -35
6 changed files with 64 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 616eaaa23a - Show all commits

2
src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java
View File

@ -1,5 +1,5 @@
package ru.ulstu.extractor.branch.model;
public enum IndexingStatus {
EMPTY, INDEXING, FINISHED
EMPTY, INDEXING, ERROR, FINISHED
}

3
src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java
View File

@ -9,10 +9,13 @@ import ru.ulstu.extractor.branch.model.IndexingStatus;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
import java.util.List;
import java.util.Optional;
public interface BranchRepository extends JpaRepository<Branch, Integer> {
Branch findByGitRepositoryAndName(GitRepository gitRepository, String name);
Optional<Branch> findByGitRepository_UrlAndName(String repositoryUrl, String name);
@Query("select count(c) from Commit c LEFT JOIN c.branch b LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2")
int getCommitsCount(Integer repositoryId, String name);

7
src/main/java/ru/ulstu/extractor/branch/service/BranchService.java
View File

@ -62,6 +62,13 @@ public class BranchService {
return branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
}
public Branch findByRepositoryAndNameOrCreate(GitRepository gitRepository, String branchName) {
Branch branch = branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
return branch == null
? branchRepository.save(new Branch(gitRepository, branchName))
: branch;
}
public Optional<Branch> findByBranchId(Integer branchId) {
return branchRepository.findById(branchId);
}

23
src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java
View File

@ -17,8 +17,10 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.servlet.mvc.support.RedirectAttributes;
import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.branch.model.IndexingStatus;
import ru.ulstu.extractor.branch.service.BranchService;
import ru.ulstu.extractor.gitrepository.model.FilterForm;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
import ru.ulstu.extractor.gitrepository.model.RepoForm;
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
import ru.ulstu.extractor.gitrepository.service.IndexService;
@ -27,8 +29,8 @@ import springfox.documentation.annotations.ApiIgnore;
import java.io.IOException;
import java.util.List;
import static ru.ulstu.extractor.core.Route.FILTER_COMMITS;
import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY;
import static ru.ulstu.extractor.core.Route.LIST_REPOSITORY_BRANCHES;
import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH;
@Controller
@ -70,16 +72,18 @@ public class GitIndexingController {
if (repoForm.getBranch() == null) {
return INDEXING_NEW_REPOSITORY;
} else {
GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(repoForm.getRepo());
Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, repoForm.getBranch());
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
try {
indexService.index(repoForm.getRepo(), repoForm.getBranch());
indexService.index(gitRepository, branch);
} catch (IOException | GitAPIException ex) {
ex.printStackTrace();
model.addAttribute("error", ex.getMessage());
return INDEXING_NEW_REPOSITORY;
}
redirectAttributes.addAttribute("repositoryUrl", repoForm.getRepo());
redirectAttributes.addAttribute("branchName", repoForm.getBranch());
return "redirect:/" + FILTER_COMMITS;
redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
return "redirect:/" + LIST_REPOSITORY_BRANCHES;
}
}
@ -87,6 +91,9 @@ public class GitIndexingController {
public String reindexBranch(Model model,
RedirectAttributes redirectAttributes,
@RequestParam Integer branchId) {
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id"));
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
try {
indexService.index(branchId);
} catch (IOException | GitAPIException ex) {
@ -94,9 +101,7 @@ public class GitIndexingController {
model.addAttribute("error", ex.getMessage());
return INDEXING_NEW_REPOSITORY;
}
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id"));
redirectAttributes.addAttribute("repositoryUrl", branch.getGitRepository().getUrl());
redirectAttributes.addAttribute("branchName", branch.getName());
return "redirect:/" + FILTER_COMMITS;
redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
return "redirect:/" + LIST_REPOSITORY_BRANCHES;
}
}

27
src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java
View File

@ -187,7 +187,6 @@ public class GitRepositoryService {
return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl));
}
public void remove(String repositoryUrl) throws IOException {
FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl));
}
@ -283,7 +282,7 @@ public class GitRepositoryService {
List<FileChange> changes = new ArrayList<>();
String[] strings = output.split("\n");
Map<String, List<String>> filesContent = getFilesContent(strings);
for(Map.Entry<String, List<String>> fileSterings: filesContent.entrySet()) {
for (Map.Entry<String, List<String>> fileSterings : filesContent.entrySet()) {
FileChange fileChange = new FileChange();
fileChange.setFile(fileSterings.getKey());
Future<Boolean> futureEntity = executorService.submit(() -> structuralUnitService.containsEntity(getContent(repository, commit, fileSterings.getKey())));
@ -327,7 +326,7 @@ public class GitRepositoryService {
private Map<String, List<String>> getFilesContent(String[] commitStrings) {
int i = 0;
Map<String, List<String>> result = new HashMap<>();
Map<String, List<String>> result = new HashMap<>();
while (i < commitStrings.length) {
Optional<String> maybeFileName = getFileName(commitStrings[i]);
if (maybeFileName.isEmpty()) {
@ -368,15 +367,15 @@ public class GitRepositoryService {
return index;
}
private FileChange getChange(FileChange fileChange, List<String> fileContent){
private FileChange getChange(FileChange fileChange, List<String> fileContent) {
int addedLine = 0;
int removedLine = 0;
StringBuilder builder = new StringBuilder();
boolean isRemoved = false;
boolean isAdded = false;
for (String line : fileContent){
for (String line : fileContent) {
LineChange lineChange = new LineChange();
if (line.startsWith("-")){
if (line.startsWith("-")) {
isRemoved = true;
if (isAdded) {
isAdded = false;
@ -385,7 +384,7 @@ public class GitRepositoryService {
}
builder.append(line).append("\n");
removedLine++;
} else if(line.startsWith("+")){
} else if (line.startsWith("+")) {
isAdded = true;
if (isRemoved) {
isRemoved = false;
@ -398,7 +397,7 @@ public class GitRepositoryService {
if (isRemoved) {
lineChange = setRemoved(lineChange, builder);
builder.setLength(0);
} else if (isAdded){
} else if (isAdded) {
lineChange = setAdded(lineChange, builder);
builder.setLength(0);
}
@ -409,17 +408,18 @@ public class GitRepositoryService {
return fileChange;
}
private LineChange setRemoved(LineChange lineChange, StringBuilder builder){
private LineChange setRemoved(LineChange lineChange, StringBuilder builder) {
lineChange.setLineFrom(builder.toString());
lineChange.setRemoved(true);
return lineChange;
}
private LineChange setAdded(LineChange lineChange, StringBuilder builder){
private LineChange setAdded(LineChange lineChange, StringBuilder builder) {
lineChange.setLineTo(builder.toString());
lineChange.setAdded(true);
return lineChange;
}
public Page<GitRepository> findAll(Pageable pageable) {
return gitRepositoryRepository.findAll(pageable);
}
@ -427,4 +427,11 @@ public class GitRepositoryService {
public GitRepository findById(Integer id) {
return gitRepositoryRepository.getOne(id);
}
public GitRepository findByUrlOrCreate(String gitRepositoryUrl) {
GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl);
return gitRepository == null
? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl))
: gitRepository;
}
}

37
src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java
View File

@ -23,6 +23,8 @@ import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Service
public class IndexService {
@ -32,6 +34,7 @@ public class IndexService {
private final GitRepositoryRepository gitRepositoryRepository;
private final BranchService branchService;
private final List<AbstractTimeSeriesCreator> timeSeriesCreators;
private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10);
public IndexService(GitRepositoryService gitRepositoryService,
GitRepositoryRepository gitRepositoryRepository,
@ -43,38 +46,43 @@ public class IndexService {
this.timeSeriesCreators = timeSeriesCreators;
}
@Transactional
@Async
public void index(@NotNull Integer branchId) throws GitAPIException, IOException {
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id"));
index(branch.getGitRepository().getUrl(), branch.getName());
index(branch.getGitRepository(), branch);
}
public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException {
GitRepository gitRepository = gitRepositoryRepository.findByUrl(repositoryUrl);
if (gitRepository == null) {
gitRepository = gitRepositoryRepository.save(new GitRepository(repositoryUrl));
}
Branch branch = branchService.findByRepositoryAndName(gitRepository, branchName);
if (branch == null) {
branch = new Branch(gitRepository, branchName);
}
@Transactional
@Async
public void index(String gitRepositoryUrl, String branchName) throws GitAPIException, IOException {
GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(gitRepositoryUrl);
Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, branchName);
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
index(gitRepository, branch);
}
@Transactional
@Async
public void index(GitRepository gitRepository, Branch branch) throws GitAPIException, IOException {
branchService.save(branch, Collections.emptyList());
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
int commitsFrom = 0;
int commitsTo = COMMITS_PAGE_SIZE;
List<Commit> commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true);
List<Commit> commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, true);
while (!commits.isEmpty()) {
LOG.debug("{} commits loaded.", commits.size());
branchService.addCommits(branch, commits);
LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName);
LOG.debug("{} commits successfully saved. {} {}", commits.size(), gitRepository.getUrl(), branch.getName());
commitsFrom += COMMITS_PAGE_SIZE;
commitsTo += COMMITS_PAGE_SIZE;
commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false);
commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, false);
}
Integer repositoryId = gitRepository.getId();
final Branch branchForSave = branch;
timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave));
branchService.updateStatus(branch, IndexingStatus.FINISHED);
LOG.debug("Complete indexing {} branch", branchName);
LOG.debug("Complete indexing {} branch", branch.getName());
}
@Transactional
@ -89,7 +97,6 @@ public class IndexService {
try {
index(failedBranch.getId());
LOG.info("Завершено индексирование ветки {}", failedBranch.getName());
branchService.updateStatus(failedBranch, IndexingStatus.FINISHED);
} catch (Exception ex) {
LOG.warn(ex.getMessage());
}