diff --git a/src/main/java/ru/ulstu/extractor/controller/RepoController.java b/src/main/java/ru/ulstu/extractor/controller/RepoController.java index b7da394..341508f 100644 --- a/src/main/java/ru/ulstu/extractor/controller/RepoController.java +++ b/src/main/java/ru/ulstu/extractor/controller/RepoController.java @@ -6,6 +6,8 @@ package ru.ulstu.extractor.controller; import org.eclipse.jgit.api.errors.GitAPIException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; @@ -13,7 +15,9 @@ import org.springframework.web.bind.annotation.RestController; import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.service.GitRepositoryService; import ru.ulstu.extractor.service.IndexService; +import ru.ulstu.extractor.util.HttpUtils; +import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.util.List; @@ -22,6 +26,7 @@ import static ru.ulstu.extractor.controller.RepoController.URL; @RestController @RequestMapping(URL) public class RepoController { + private final static Logger LOG = LoggerFactory.getLogger(RepoController.class); public static final String URL = "/"; private final GitRepositoryService gitRepositoryService; private final IndexService indexService; @@ -34,13 +39,17 @@ public class RepoController { @GetMapping("commits") public List getCommits(@RequestParam("repositoryUrl") String repositoryUrl, - @RequestParam("branchName") String branchName) throws GitAPIException, IOException { + @RequestParam("branchName") String branchName, + HttpServletRequest request) throws GitAPIException, IOException { + LOG.debug("Get commits {} {}. User ip {}", repositoryUrl, branchName, HttpUtils.getUserIp(request)); return gitRepositoryService.getCommits(repositoryUrl, branchName); } @GetMapping("index") public Boolean indexRepository(@RequestParam("repositoryUrl") String repositoryUrl, - @RequestParam("branchName") String branchName) throws GitAPIException, IOException { + @RequestParam("branchName") String branchName, + HttpServletRequest request) throws GitAPIException, IOException { + LOG.debug("Index {} {}. User ip {}", repositoryUrl, branchName, HttpUtils.getUserIp(request)); indexService.index(repositoryUrl, branchName); return true; } diff --git a/src/main/java/ru/ulstu/extractor/heuristic/controller/StructuralUnitController.java b/src/main/java/ru/ulstu/extractor/heuristic/controller/StructuralUnitController.java index 639d420..0939e08 100644 --- a/src/main/java/ru/ulstu/extractor/heuristic/controller/StructuralUnitController.java +++ b/src/main/java/ru/ulstu/extractor/heuristic/controller/StructuralUnitController.java @@ -6,6 +6,8 @@ package ru.ulstu.extractor.heuristic.controller; import org.eclipse.jgit.api.errors.GitAPIException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -13,13 +15,16 @@ import ru.ulstu.extractor.heuristic.model.BusinessLogicUnit; import ru.ulstu.extractor.heuristic.model.EntityUnit; import ru.ulstu.extractor.heuristic.model.ResourceUnit; import ru.ulstu.extractor.service.GitRepositoryService; +import ru.ulstu.extractor.util.HttpUtils; +import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.util.List; @RestController @RequestMapping("StructuralUnitController") public class StructuralUnitController { + private final static Logger LOG = LoggerFactory.getLogger(StructuralUnitController.class); private final GitRepositoryService gitRepositoryService; public StructuralUnitController(GitRepositoryService gitRepositoryService) { @@ -27,17 +32,26 @@ public class StructuralUnitController { } @GetMapping("get-entities") - public List getEntities(String repositoryUrl, String branchName) throws IOException, GitAPIException { + public List getEntities(String repositoryUrl, + String branchName, + HttpServletRequest request) throws IOException, GitAPIException { + LOG.debug("Get entities {} {}. User ip {}", repositoryUrl, branchName, HttpUtils.getUserIp(request)); return gitRepositoryService.getEntities(repositoryUrl, branchName); } @GetMapping("get-business-logic") - public List getBusinessLogic(String repositoryUrl, String branchName) throws IOException, GitAPIException { + public List getBusinessLogic(String repositoryUrl, + String branchName, + HttpServletRequest request) throws IOException, GitAPIException { + LOG.debug("Get business logic {} {}. User ip {}", repositoryUrl, branchName, HttpUtils.getUserIp(request)); return gitRepositoryService.getBusinessLogic(repositoryUrl, branchName); } @GetMapping("get-resources") - public List getResource(String repositoryUrl, String branchName) throws IOException, GitAPIException { + public List getResource(String repositoryUrl, + String branchName, + HttpServletRequest request) throws IOException, GitAPIException { + LOG.debug("Get resources {} {}. User ip {}", repositoryUrl, branchName, HttpUtils.getUserIp(request)); return gitRepositoryService.getResource(repositoryUrl, branchName); } } diff --git a/src/main/java/ru/ulstu/extractor/model/Commit.java b/src/main/java/ru/ulstu/extractor/model/Commit.java index 6f13cb1..daf9087 100644 --- a/src/main/java/ru/ulstu/extractor/model/Commit.java +++ b/src/main/java/ru/ulstu/extractor/model/Commit.java @@ -29,7 +29,7 @@ public class Commit extends BaseEntity { @JoinColumn(name = "commit_id") @Fetch(FetchMode.SUBSELECT) private List fileChanges = new ArrayList<>(); - @ManyToOne(cascade = CascadeType.ALL, fetch = FetchType.LAZY) + @ManyToOne(fetch = FetchType.LAZY) private Branch branch; public Commit() { diff --git a/src/main/java/ru/ulstu/extractor/repository/CommitRepository.java b/src/main/java/ru/ulstu/extractor/repository/CommitRepository.java index 5059f5f..68a58cd 100644 --- a/src/main/java/ru/ulstu/extractor/repository/CommitRepository.java +++ b/src/main/java/ru/ulstu/extractor/repository/CommitRepository.java @@ -38,4 +38,5 @@ public interface CommitRepository extends JpaRepository { @Query("SELECT new ru.ulstu.extractor.model.CommitTimeStatistic(cast(c.date as date), COUNT(DISTINCT c.hash)) FROM Commit c JOIN c.fileChanges f WHERE f.containsEntity = true GROUP by cast(c.date as date) ORDER by cast(c.date as date)") List getCommitTimeEntityStatistic(); + void deleteByBranchIsNull(); } diff --git a/src/main/java/ru/ulstu/extractor/service/AuthorService.java b/src/main/java/ru/ulstu/extractor/service/AuthorService.java index d829740..c04cfd1 100644 --- a/src/main/java/ru/ulstu/extractor/service/AuthorService.java +++ b/src/main/java/ru/ulstu/extractor/service/AuthorService.java @@ -5,25 +5,27 @@ package ru.ulstu.extractor.service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import ru.ulstu.extractor.model.Author; import ru.ulstu.extractor.repository.AuthorRepository; -import javax.transaction.Transactional; import java.util.Optional; @Service public class AuthorService { + private final static Logger LOG = LoggerFactory.getLogger(AuthorService.class); private final AuthorRepository authorRepository; public AuthorService(AuthorRepository authorRepository) { this.authorRepository = authorRepository; } - @Transactional public Author findOrCreate(Author author) { Optional newAuthor = authorRepository.findByName(author.getName()).stream().findAny(); if (newAuthor.isEmpty()) { + LOG.debug("Author {} was saved.", author.getName()); return authorRepository.save(author); } return newAuthor.get(); diff --git a/src/main/java/ru/ulstu/extractor/service/BranchService.java b/src/main/java/ru/ulstu/extractor/service/BranchService.java index a38997f..ea0fb60 100644 --- a/src/main/java/ru/ulstu/extractor/service/BranchService.java +++ b/src/main/java/ru/ulstu/extractor/service/BranchService.java @@ -5,17 +5,21 @@ package ru.ulstu.extractor.service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import ru.ulstu.extractor.model.BaseEntity; import ru.ulstu.extractor.model.Branch; import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.model.Repository; import ru.ulstu.extractor.repository.BranchRepository; -import javax.transaction.Transactional; import java.util.List; +import java.util.stream.Collectors; @Service public class BranchService { + private final static Logger LOG = LoggerFactory.getLogger(BranchService.class); private final BranchRepository branchRepository; private final CommitService commitService; @@ -25,13 +29,15 @@ public class BranchService { this.commitService = commitService; } - @Transactional public Branch save(Branch branch, List commits) { - List commitsToRemove = branch.getCommits(); - branch.getCommits().clear(); - commitService.delete(commitsToRemove); + LOG.debug("Start save {} branch with {} commits ", branch.getName(), commits.size()); + List commitsToRemoveIds = branch.getCommits().stream().map(BaseEntity::getId).collect(Collectors.toList()); branch.setCommits(commitService.save(commits)); - return branchRepository.save(branch); + LOG.debug("Save branch {} ", branch.getName()); + branch = branchRepository.save(branch); + LOG.debug("Clear {} commits ", commitsToRemoveIds.size()); + commitService.deleteWithEmptyIds(); + return branch; } public Branch findByRepositoryAndName(Repository repository, String branchName) { diff --git a/src/main/java/ru/ulstu/extractor/service/CommitService.java b/src/main/java/ru/ulstu/extractor/service/CommitService.java index 7b76509..d19a713 100644 --- a/src/main/java/ru/ulstu/extractor/service/CommitService.java +++ b/src/main/java/ru/ulstu/extractor/service/CommitService.java @@ -14,7 +14,6 @@ import org.springframework.stereotype.Service; import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.repository.CommitRepository; -import javax.transaction.Transactional; import java.io.IOException; import java.util.Collections; import java.util.List; @@ -53,7 +52,6 @@ public class CommitService { commitRepository.deleteAll(commitsToRemove); } - @Transactional public List save(List commits) { return commits.stream() .map(commit -> { @@ -61,6 +59,10 @@ public class CommitService { return commitRepository.save(commit); }).collect(Collectors.toList()); } + + public void deleteWithEmptyIds() { + commitRepository.deleteByBranchIsNull(); + } } diff --git a/src/main/java/ru/ulstu/extractor/service/GitRepositoryService.java b/src/main/java/ru/ulstu/extractor/service/GitRepositoryService.java index 17238d1..641d5c3 100644 --- a/src/main/java/ru/ulstu/extractor/service/GitRepositoryService.java +++ b/src/main/java/ru/ulstu/extractor/service/GitRepositoryService.java @@ -18,6 +18,8 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.treewalk.TreeWalk; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import ru.ulstu.extractor.heuristic.model.BusinessLogicUnit; @@ -43,15 +45,20 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.stream.Collectors; import static org.apache.logging.log4j.util.Strings.isBlank; @Service public class GitRepositoryService { + private final static Logger LOG = LoggerFactory.getLogger(GitRepositoryService.class); private static final String BRANCH_PREFIX = "refs/remotes/origin/"; @Value("${extractor.custom-projects-dir}") private String customProjectsDir; + private final ExecutorService executorService = Executors.newFixedThreadPool(4); private final StructuralUnitService structuralUnitService; @@ -60,7 +67,9 @@ public class GitRepositoryService { } public List getRemoteBranches(String url) throws GitAPIException, IOException { + LOG.debug("Get remote branches of {}. Clone", url); cloneOrUpdateRepo(url); + LOG.debug("Get remote branches of {}. Get branches", url); Repository localRepo = new FileRepository(getProjectGitDirectory(url)); Git git = new Git(localRepo); List branches = git.branchList().setListMode(ListBranchCommand.ListMode.REMOTE) @@ -74,7 +83,9 @@ public class GitRepositoryService { } public List getLocalBranches(String url) throws GitAPIException, IOException { + LOG.debug("Get local branches of {}. Clone", url); cloneOrUpdateRepo(url); + LOG.debug("Get local branches of {}. Get branches", url); Repository localRepo = new FileRepository(getProjectGitDirectory(url)); Git git = new Git(localRepo); List branches = git.branchList() @@ -88,6 +99,7 @@ public class GitRepositoryService { } public List getCommits(String repositoryUrl, String branchName) throws GitAPIException, IOException { + LOG.debug("Get commits of {}. Branch {}", repositoryUrl, branchName); cloneOrUpdateRepo(repositoryUrl, branchName); Repository localRepo = new FileRepository(getProjectGitDirectory(repositoryUrl)); Git git = new Git(localRepo); @@ -97,7 +109,10 @@ public class GitRepositoryService { List list = new ArrayList<>(); RevCommit prevCommit = null; + LOG.debug("Start analyse {} commits", commits.size()); + int counter = commits.size(); for (RevCommit revCommit : commits) { + LOG.debug(" {} of {} commits", counter--, commits.size()); Commit commit = new Commit( revCommit.getFullMessage(), new Author(revCommit.getAuthorIdent().getName()), @@ -115,6 +130,7 @@ public class GitRepositoryService { } private void checkoutBranch(String repositoryUrl, Git git, Repository localRepo, String branchName) throws GitAPIException, IOException { + LOG.debug("Checkout branch {} {}", repositoryUrl, branchName); git.pull().call(); if (!localRepo.getBranch().equals(branchName)) { if (getLocalBranches(repositoryUrl).stream().anyMatch(localBranch -> localBranch.getName().contains(branchName))) { @@ -236,12 +252,19 @@ public class GitRepositoryService { if (maybeFileName.isPresent()) { fileChange = new FileChange(); fileChange.setFile(maybeFileName.get()); - fileChange.setContainsEntity( - structuralUnitService.containsEntity(getContent(repository, commit, maybeFileName.get())) - ); - fileChange.setContainsBusinessLogic( - structuralUnitService.containsBusinessLogic(getContent(repository, commit, maybeFileName.get())) - ); + Future futureEntity = executorService.submit(() -> { + return structuralUnitService.containsEntity(getContent(repository, commit, maybeFileName.get())); + }); + + Future futureBL = executorService.submit(() -> { + return structuralUnitService.containsBusinessLogic(getContent(repository, commit, maybeFileName.get())); + }); + try { + fileChange.setContainsBusinessLogic(futureBL.get()); + fileChange.setContainsEntity(futureEntity.get()); + } catch (Exception ex) { + LOG.warn(ex.getMessage()); + } /// вытащить другие изменения из коммита changes.add(fileChange); } diff --git a/src/main/java/ru/ulstu/extractor/service/IndexService.java b/src/main/java/ru/ulstu/extractor/service/IndexService.java index 74d1782..00c1565 100644 --- a/src/main/java/ru/ulstu/extractor/service/IndexService.java +++ b/src/main/java/ru/ulstu/extractor/service/IndexService.java @@ -7,12 +7,13 @@ package ru.ulstu.extractor.service; import com.sun.istack.NotNull; import org.eclipse.jgit.api.errors.GitAPIException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import ru.ulstu.extractor.model.Branch; import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.model.Repository; -import ru.ulstu.extractor.repository.CommitRepository; import ru.ulstu.extractor.repository.RepositoryRepository; import java.io.IOException; @@ -20,22 +21,17 @@ import java.util.List; @Service public class IndexService { + private final static Logger LOG = LoggerFactory.getLogger(IndexService.class); private final GitRepositoryService gitRepositoryService; private final RepositoryRepository repositoryRepository; private final BranchService branchService; - private final CommitRepository commitRepository; - private final AuthorService authorService; public IndexService(GitRepositoryService gitRepositoryService, RepositoryRepository repositoryRepository, - BranchService branchService, - CommitRepository commitRepository, - AuthorService authorService) { + BranchService branchService) { this.gitRepositoryService = gitRepositoryService; this.repositoryRepository = repositoryRepository; this.branchService = branchService; - this.commitRepository = commitRepository; - this.authorService = authorService; } @Transactional @@ -49,6 +45,8 @@ public class IndexService { branch = new Branch(repository, branchName); } List commits = gitRepositoryService.getCommits(repositoryUrl, branchName); + LOG.debug("{} commits loaded.", commits.size()); branchService.save(branch, commits); + LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName); } } diff --git a/src/main/java/ru/ulstu/extractor/util/HttpUtils.java b/src/main/java/ru/ulstu/extractor/util/HttpUtils.java new file mode 100644 index 0000000..5561566 --- /dev/null +++ b/src/main/java/ru/ulstu/extractor/util/HttpUtils.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2021 Anton Romanov - All Rights Reserved + * You may use, distribute and modify this code, please write to: romanov73@gmail.com. + */ + +package ru.ulstu.extractor.util; + +import javax.servlet.http.HttpServletRequest; + +public class HttpUtils { + public static String getUserIp(HttpServletRequest request) { + String xfHeader = request.getHeader("X-Forwarded-For"); + if (xfHeader == null) { + return request.getRemoteAddr(); + } + return xfHeader.split(",")[0]; + } +}