romanov73/git-extractor
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

async indexing #88

Merged
romanov73 merged 5 commits from 81-async-indexing into master 2023-02-13 17:56:23 +04:00
Conversation 1 Commits 5 Files Changed 11 +64 -35
6 changed files with 64 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 616eaaa23a - Show all commits

2
src/main/java/ru/ulstu/extractor/branch/model/IndexingStatus.java
View File

@ -1,5 +1,5 @@
package ru.ulstu.extractor.branch.model; package ru.ulstu.extractor.branch.model;
public enum IndexingStatus { public enum IndexingStatus {
EMPTY, INDEXING, FINISHED EMPTY, INDEXING, ERROR, FINISHED
} }

3
src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java
View File

@ -9,10 +9,13 @@ import ru.ulstu.extractor.branch.model.IndexingStatus;
import ru.ulstu.extractor.gitrepository.model.GitRepository; import ru.ulstu.extractor.gitrepository.model.GitRepository;
import java.util.List; import java.util.List;
import java.util.Optional;
public interface BranchRepository extends JpaRepository<Branch, Integer> { public interface BranchRepository extends JpaRepository<Branch, Integer> {
Branch findByGitRepositoryAndName(GitRepository gitRepository, String name); Branch findByGitRepositoryAndName(GitRepository gitRepository, String name);
Optional<Branch> findByGitRepository_UrlAndName(String repositoryUrl, String name);
@Query("select count(c) from Commit c LEFT JOIN c.branch b LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2") @Query("select count(c) from Commit c LEFT JOIN c.branch b LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2")
int getCommitsCount(Integer repositoryId, String name); int getCommitsCount(Integer repositoryId, String name);

7
src/main/java/ru/ulstu/extractor/branch/service/BranchService.java
View File

@ -62,6 +62,13 @@ public class BranchService {
return branchRepository.findByGitRepositoryAndName(gitRepository, branchName); return branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
} }
public Branch findByRepositoryAndNameOrCreate(GitRepository gitRepository, String branchName) {
Branch branch = branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
return branch == null
? branchRepository.save(new Branch(gitRepository, branchName))
: branch;
}
public Optional<Branch> findByBranchId(Integer branchId) { public Optional<Branch> findByBranchId(Integer branchId) {
return branchRepository.findById(branchId); return branchRepository.findById(branchId);
} }

23
src/main/java/ru/ulstu/extractor/gitrepository/controler/GitIndexingController.java
View File

@ -17,8 +17,10 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.servlet.mvc.support.RedirectAttributes; import org.springframework.web.servlet.mvc.support.RedirectAttributes;
import ru.ulstu.extractor.branch.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.branch.model.IndexingStatus;
import ru.ulstu.extractor.branch.service.BranchService; import ru.ulstu.extractor.branch.service.BranchService;
import ru.ulstu.extractor.gitrepository.model.FilterForm; import ru.ulstu.extractor.gitrepository.model.FilterForm;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
import ru.ulstu.extractor.gitrepository.model.RepoForm; import ru.ulstu.extractor.gitrepository.model.RepoForm;
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService; import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
import ru.ulstu.extractor.gitrepository.service.IndexService; import ru.ulstu.extractor.gitrepository.service.IndexService;
@ -27,8 +29,8 @@ import springfox.documentation.annotations.ApiIgnore;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import static ru.ulstu.extractor.core.Route.FILTER_COMMITS;
import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY; import static ru.ulstu.extractor.core.Route.INDEXING_NEW_REPOSITORY;
import static ru.ulstu.extractor.core.Route.LIST_REPOSITORY_BRANCHES;
import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH; import static ru.ulstu.extractor.core.Route.REINDEX_BRANCH;
@Controller @Controller
@ -70,16 +72,18 @@ public class GitIndexingController {
if (repoForm.getBranch() == null) { if (repoForm.getBranch() == null) {
return INDEXING_NEW_REPOSITORY; return INDEXING_NEW_REPOSITORY;
} else { } else {
GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(repoForm.getRepo());
Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, repoForm.getBranch());
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
try { try {
indexService.index(repoForm.getRepo(), repoForm.getBranch()); indexService.index(gitRepository, branch);
} catch (IOException | GitAPIException ex) { } catch (IOException | GitAPIException ex) {
ex.printStackTrace(); ex.printStackTrace();
model.addAttribute("error", ex.getMessage()); model.addAttribute("error", ex.getMessage());
return INDEXING_NEW_REPOSITORY; return INDEXING_NEW_REPOSITORY;
} }
redirectAttributes.addAttribute("repositoryUrl", repoForm.getRepo()); redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
redirectAttributes.addAttribute("branchName", repoForm.getBranch()); return "redirect:/" + LIST_REPOSITORY_BRANCHES;
return "redirect:/" + FILTER_COMMITS;
} }
} }
@ -87,6 +91,9 @@ public class GitIndexingController {
public String reindexBranch(Model model, public String reindexBranch(Model model,
RedirectAttributes redirectAttributes, RedirectAttributes redirectAttributes,
@RequestParam Integer branchId) { @RequestParam Integer branchId) {
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id"));
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
try { try {
indexService.index(branchId); indexService.index(branchId);
} catch (IOException | GitAPIException ex) { } catch (IOException | GitAPIException ex) {
@ -94,9 +101,7 @@ public class GitIndexingController {
model.addAttribute("error", ex.getMessage()); model.addAttribute("error", ex.getMessage());
return INDEXING_NEW_REPOSITORY; return INDEXING_NEW_REPOSITORY;
} }
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка не найдена по id")); redirectAttributes.addAttribute("repositoryId", branch.getGitRepository().getId());
redirectAttributes.addAttribute("repositoryUrl", branch.getGitRepository().getUrl()); return "redirect:/" + LIST_REPOSITORY_BRANCHES;
redirectAttributes.addAttribute("branchName", branch.getName());
return "redirect:/" + FILTER_COMMITS;
} }
} }

27
src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java
View File

@ -187,7 +187,6 @@ public class GitRepositoryService {
return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl)); return structuralUnitService.getResource(getProjectDirectoryFile(repositoryUrl));
} }
public void remove(String repositoryUrl) throws IOException { public void remove(String repositoryUrl) throws IOException {
FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl)); FileUtils.deleteDirectory(getProjectDirectoryFile(repositoryUrl));
} }
@ -283,7 +282,7 @@ public class GitRepositoryService {
List<FileChange> changes = new ArrayList<>(); List<FileChange> changes = new ArrayList<>();
String[] strings = output.split("\n"); String[] strings = output.split("\n");
Map<String, List<String>> filesContent = getFilesContent(strings); Map<String, List<String>> filesContent = getFilesContent(strings);
for(Map.Entry<String, List<String>> fileSterings: filesContent.entrySet()) { for (Map.Entry<String, List<String>> fileSterings : filesContent.entrySet()) {
FileChange fileChange = new FileChange(); FileChange fileChange = new FileChange();
fileChange.setFile(fileSterings.getKey()); fileChange.setFile(fileSterings.getKey());
Future<Boolean> futureEntity = executorService.submit(() -> structuralUnitService.containsEntity(getContent(repository, commit, fileSterings.getKey()))); Future<Boolean> futureEntity = executorService.submit(() -> structuralUnitService.containsEntity(getContent(repository, commit, fileSterings.getKey())));
@ -327,7 +326,7 @@ public class GitRepositoryService {
private Map<String, List<String>> getFilesContent(String[] commitStrings) { private Map<String, List<String>> getFilesContent(String[] commitStrings) {
int i = 0; int i = 0;
Map<String, List<String>> result = new HashMap<>(); Map<String, List<String>> result = new HashMap<>();
while (i < commitStrings.length) { while (i < commitStrings.length) {
Optional<String> maybeFileName = getFileName(commitStrings[i]); Optional<String> maybeFileName = getFileName(commitStrings[i]);
if (maybeFileName.isEmpty()) { if (maybeFileName.isEmpty()) {
@ -368,15 +367,15 @@ public class GitRepositoryService {
return index; return index;
} }
private FileChange getChange(FileChange fileChange, List<String> fileContent){ private FileChange getChange(FileChange fileChange, List<String> fileContent) {
int addedLine = 0; int addedLine = 0;
int removedLine = 0; int removedLine = 0;
StringBuilder builder = new StringBuilder(); StringBuilder builder = new StringBuilder();
boolean isRemoved = false; boolean isRemoved = false;
boolean isAdded = false; boolean isAdded = false;
for (String line : fileContent){ for (String line : fileContent) {
LineChange lineChange = new LineChange(); LineChange lineChange = new LineChange();
if (line.startsWith("-")){ if (line.startsWith("-")) {
isRemoved = true; isRemoved = true;
if (isAdded) { if (isAdded) {
isAdded = false; isAdded = false;
@ -385,7 +384,7 @@ public class GitRepositoryService {
} }
builder.append(line).append("\n"); builder.append(line).append("\n");
removedLine++; removedLine++;
} else if(line.startsWith("+")){ } else if (line.startsWith("+")) {
isAdded = true; isAdded = true;
if (isRemoved) { if (isRemoved) {
isRemoved = false; isRemoved = false;
@ -398,7 +397,7 @@ public class GitRepositoryService {
if (isRemoved) { if (isRemoved) {
lineChange = setRemoved(lineChange, builder); lineChange = setRemoved(lineChange, builder);
builder.setLength(0); builder.setLength(0);
} else if (isAdded){ } else if (isAdded) {
lineChange = setAdded(lineChange, builder); lineChange = setAdded(lineChange, builder);
builder.setLength(0); builder.setLength(0);
} }
@ -409,17 +408,18 @@ public class GitRepositoryService {
return fileChange; return fileChange;
} }
private LineChange setRemoved(LineChange lineChange, StringBuilder builder){ private LineChange setRemoved(LineChange lineChange, StringBuilder builder) {
lineChange.setLineFrom(builder.toString()); lineChange.setLineFrom(builder.toString());
lineChange.setRemoved(true); lineChange.setRemoved(true);
return lineChange; return lineChange;
} }
private LineChange setAdded(LineChange lineChange, StringBuilder builder){ private LineChange setAdded(LineChange lineChange, StringBuilder builder) {
lineChange.setLineTo(builder.toString()); lineChange.setLineTo(builder.toString());
lineChange.setAdded(true); lineChange.setAdded(true);
return lineChange; return lineChange;
} }
public Page<GitRepository> findAll(Pageable pageable) { public Page<GitRepository> findAll(Pageable pageable) {
return gitRepositoryRepository.findAll(pageable); return gitRepositoryRepository.findAll(pageable);
} }
@ -427,4 +427,11 @@ public class GitRepositoryService {
public GitRepository findById(Integer id) { public GitRepository findById(Integer id) {
return gitRepositoryRepository.getOne(id); return gitRepositoryRepository.getOne(id);
} }
public GitRepository findByUrlOrCreate(String gitRepositoryUrl) {
GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl);
return gitRepository == null
? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl))
: gitRepository;
}
} }

37
src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java
View File

@ -23,6 +23,8 @@ import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Service @Service
public class IndexService { public class IndexService {
@ -32,6 +34,7 @@ public class IndexService {
private final GitRepositoryRepository gitRepositoryRepository; private final GitRepositoryRepository gitRepositoryRepository;
private final BranchService branchService; private final BranchService branchService;
private final List<AbstractTimeSeriesCreator> timeSeriesCreators; private final List<AbstractTimeSeriesCreator> timeSeriesCreators;
private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10);
public IndexService(GitRepositoryService gitRepositoryService, public IndexService(GitRepositoryService gitRepositoryService,
GitRepositoryRepository gitRepositoryRepository, GitRepositoryRepository gitRepositoryRepository,
@ -43,38 +46,43 @@ public class IndexService {
this.timeSeriesCreators = timeSeriesCreators; this.timeSeriesCreators = timeSeriesCreators;
} }
@Transactional
@Async
public void index(@NotNull Integer branchId) throws GitAPIException, IOException { public void index(@NotNull Integer branchId) throws GitAPIException, IOException {
Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id")); Branch branch = branchService.findByBranchId(branchId).orElseThrow(() -> new RuntimeException("Ветка репозитория не найдена по id"));
index(branch.getGitRepository().getUrl(), branch.getName()); index(branch.getGitRepository(), branch);
} }
public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException { @Transactional
GitRepository gitRepository = gitRepositoryRepository.findByUrl(repositoryUrl); @Async
if (gitRepository == null) { public void index(String gitRepositoryUrl, String branchName) throws GitAPIException, IOException {
gitRepository = gitRepositoryRepository.save(new GitRepository(repositoryUrl)); GitRepository gitRepository = gitRepositoryService.findByUrlOrCreate(gitRepositoryUrl);
} Branch branch = branchService.findByRepositoryAndNameOrCreate(gitRepository, branchName);
Branch branch = branchService.findByRepositoryAndName(gitRepository, branchName); branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
if (branch == null) { index(gitRepository, branch);
branch = new Branch(gitRepository, branchName); }
}
@Transactional
@Async
public void index(GitRepository gitRepository, Branch branch) throws GitAPIException, IOException {
branchService.save(branch, Collections.emptyList()); branchService.save(branch, Collections.emptyList());
branch = branchService.updateStatus(branch, IndexingStatus.INDEXING); branch = branchService.updateStatus(branch, IndexingStatus.INDEXING);
int commitsFrom = 0; int commitsFrom = 0;
int commitsTo = COMMITS_PAGE_SIZE; int commitsTo = COMMITS_PAGE_SIZE;
List<Commit> commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true); List<Commit> commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, true);
while (!commits.isEmpty()) { while (!commits.isEmpty()) {
LOG.debug("{} commits loaded.", commits.size()); LOG.debug("{} commits loaded.", commits.size());
branchService.addCommits(branch, commits); branchService.addCommits(branch, commits);
LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName); LOG.debug("{} commits successfully saved. {} {}", commits.size(), gitRepository.getUrl(), branch.getName());
commitsFrom += COMMITS_PAGE_SIZE; commitsFrom += COMMITS_PAGE_SIZE;
commitsTo += COMMITS_PAGE_SIZE; commitsTo += COMMITS_PAGE_SIZE;
commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false); commits = gitRepositoryService.getCommits(gitRepository.getUrl(), branch.getName(), commitsFrom, commitsTo, false);
} }
Integer repositoryId = gitRepository.getId(); Integer repositoryId = gitRepository.getId();
final Branch branchForSave = branch; final Branch branchForSave = branch;
timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave)); timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchForSave));
branchService.updateStatus(branch, IndexingStatus.FINISHED); branchService.updateStatus(branch, IndexingStatus.FINISHED);
LOG.debug("Complete indexing {} branch", branchName); LOG.debug("Complete indexing {} branch", branch.getName());
} }
@Transactional @Transactional
@ -89,7 +97,6 @@ public class IndexService {
try { try {
index(failedBranch.getId()); index(failedBranch.getId());
LOG.info("Завершено индексирование ветки {}", failedBranch.getName()); LOG.info("Завершено индексирование ветки {}", failedBranch.getName());
branchService.updateStatus(failedBranch, IndexingStatus.FINISHED);
} catch (Exception ex) { } catch (Exception ex) {
LOG.warn(ex.getMessage()); LOG.warn(ex.getMessage());
} }