romanov73/git-extractor
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

fix out of memory

Browse Source
This commit is contained in:
Anton Romanov 2021-04-28 11:44:27 +04:00
parent 743e730397
commit 5054224ff3
3 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/ru/ulstu/extractor/service/BranchService.java
View File

@ -42,9 +42,9 @@ public class BranchService {
public Branch addCommits(Branch branch, List<Commit> commits) { public Branch addCommits(Branch branch, List<Commit> commits) {
LOG.debug("Start add commits to {} branch with {} commits ", branch.getName(), commits.size()); LOG.debug("Start add commits to {} branch with {} commits ", branch.getName(), commits.size());
branch.getCommits().addAll(commitService.save(commits)); commitService.save(commits);
LOG.debug("Save branch {} ", branch.getName()); LOG.debug("Save branch {} ", branch.getName());
branch = branchRepository.save(branch); commitService.updateBranchId(commits, branch.getId());
return branch; return branch;
} }

18
src/main/java/ru/ulstu/extractor/service/CommitService.java
View File

@ -5,6 +5,7 @@
package ru.ulstu.extractor.service; package ru.ulstu.extractor.service;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.repository.CommitRepository; import ru.ulstu.extractor.repository.CommitRepository;
@ -14,16 +15,16 @@ import java.util.stream.Collectors;
@Service @Service
public class CommitService { public class CommitService {
private final GitRepositoryService gitRepositoryService;
private final CommitRepository commitRepository; private final CommitRepository commitRepository;
private final AuthorService authorService; private final AuthorService authorService;
private final JdbcTemplate jdbcTemplate;
public CommitService(GitRepositoryService gitRepositoryService, public CommitService(CommitRepository commitRepository,
CommitRepository commitRepository, AuthorService authorService,
AuthorService authorService) { JdbcTemplate jdbcTemplate) {
this.gitRepositoryService = gitRepositoryService;
this.commitRepository = commitRepository; this.commitRepository = commitRepository;
this.authorService = authorService; this.authorService = authorService;
this.jdbcTemplate = jdbcTemplate;
} }
public void delete(List<Commit> commitsToRemove) { public void delete(List<Commit> commitsToRemove) {
@ -41,6 +42,13 @@ public class CommitService {
public void deleteWithEmptyIds() { public void deleteWithEmptyIds() {
commitRepository.deleteByBranchIsNull(); commitRepository.deleteByBranchIsNull();
} }
public void updateBranchId(List<Commit> commits, Integer branchId) {
List<String> commitIds = commits.stream().map(c -> c.getId().toString()).collect(Collectors.toList());
String updateQuery = "update commit set branch_id = ? where id in (%s)";
updateQuery = String.format(updateQuery, String.join(",", commitIds));
jdbcTemplate.update(updateQuery, branchId);
}
} }

4
src/main/java/ru/ulstu/extractor/service/IndexService.java
View File

@ -10,7 +10,6 @@ import org.eclipse.jgit.api.errors.GitAPIException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.ulstu.extractor.model.Branch; import ru.ulstu.extractor.model.Branch;
import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.model.Repository; import ru.ulstu.extractor.model.Repository;
@ -36,7 +35,6 @@ public class IndexService {
this.branchService = branchService; this.branchService = branchService;
} }
@Transactional
public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException { public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException {
Repository repository = repositoryRepository.findByUrl(repositoryUrl); Repository repository = repositoryRepository.findByUrl(repositoryUrl);
if (repository == null) { if (repository == null) {
@ -52,7 +50,7 @@ public class IndexService {
List<Commit> commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true); List<Commit> commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, true);
while (!commits.isEmpty()) { while (!commits.isEmpty()) {
LOG.debug("{} commits loaded.", commits.size()); LOG.debug("{} commits loaded.", commits.size());
branch = branchService.addCommits(branch, commits); branchService.addCommits(branch, commits);
LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName); LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName);
commitsFrom += COMMITS_PAGE_SIZE; commitsFrom += COMMITS_PAGE_SIZE;
commitsTo += COMMITS_PAGE_SIZE; commitsTo += COMMITS_PAGE_SIZE;