romanov73/git-extractor
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Merge branch '27-logs' into 'master'

Browse Source 
Resolve "Добавить логирование всех действий по индексации"

Closes #27

See merge request romanov73/git-extractor!25
This commit is contained in:
Anton Romanov 2021-04-27 14:02:53 +00:00
commit 27d7cc17b5
10 changed files with 103 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/ru/ulstu/extractor/controller/RepoController.java
View File

@ -6,6 +6,8 @@
package ru.ulstu.extractor.controller;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@ -13,7 +15,9 @@ import org.springframework.web.bind.annotation.RestController;
import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.service.GitRepositoryService;
import ru.ulstu.extractor.service.IndexService;
import ru.ulstu.extractor.util.HttpUtils;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.util.List;
@ -22,6 +26,7 @@ import static ru.ulstu.extractor.controller.RepoController.URL;
@RestController
@RequestMapping(URL)
public class RepoController {
private final static Logger LOG = LoggerFactory.getLogger(RepoController.class);
public static final String URL = "/";
private final GitRepositoryService gitRepositoryService;
private final IndexService indexService;
@ -34,13 +39,17 @@ public class RepoController {
@GetMapping("commits")
public List<Commit> getCommits(@RequestParam("repositoryUrl") String repositoryUrl,
@RequestParam("branchName") String branchName) throws GitAPIException, IOException {
@RequestParam("branchName") String branchName,
HttpServletRequest request) throws GitAPIException, IOException {
LOG.debug("Get commits {} {}. User ip {}", repositoryUrl, branchName, HttpUtils.getUserIp(request));
return gitRepositoryService.getCommits(repositoryUrl, branchName);
}
@GetMapping("index")
public Boolean indexRepository(@RequestParam("repositoryUrl") String repositoryUrl,
@RequestParam("branchName") String branchName) throws GitAPIException, IOException {
@RequestParam("branchName") String branchName,
HttpServletRequest request) throws GitAPIException, IOException {
LOG.debug("Index {} {}. User ip {}", repositoryUrl, branchName, HttpUtils.getUserIp(request));
indexService.index(repositoryUrl, branchName);
return true;
}

20
src/main/java/ru/ulstu/extractor/heuristic/controller/StructuralUnitController.java
View File

@ -6,6 +6,8 @@
package ru.ulstu.extractor.heuristic.controller;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@ -13,13 +15,16 @@ import ru.ulstu.extractor.heuristic.model.BusinessLogicUnit;
import ru.ulstu.extractor.heuristic.model.EntityUnit;
import ru.ulstu.extractor.heuristic.model.ResourceUnit;
import ru.ulstu.extractor.service.GitRepositoryService;
import ru.ulstu.extractor.util.HttpUtils;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.util.List;
@RestController
@RequestMapping("StructuralUnitController")
public class StructuralUnitController {
private final static Logger LOG = LoggerFactory.getLogger(StructuralUnitController.class);
private final GitRepositoryService gitRepositoryService;
public StructuralUnitController(GitRepositoryService gitRepositoryService) {
@ -27,17 +32,26 @@ public class StructuralUnitController {
}
@GetMapping("get-entities")
public List<EntityUnit> getEntities(String repositoryUrl, String branchName) throws IOException, GitAPIException {
public List<EntityUnit> getEntities(String repositoryUrl,
String branchName,
HttpServletRequest request) throws IOException, GitAPIException {
LOG.debug("Get entities {} {}. User ip {}", repositoryUrl, branchName, HttpUtils.getUserIp(request));
return gitRepositoryService.getEntities(repositoryUrl, branchName);
}
@GetMapping("get-business-logic")
public List<BusinessLogicUnit> getBusinessLogic(String repositoryUrl, String branchName) throws IOException, GitAPIException {
public List<BusinessLogicUnit> getBusinessLogic(String repositoryUrl,
String branchName,
HttpServletRequest request) throws IOException, GitAPIException {
LOG.debug("Get business logic {} {}. User ip {}", repositoryUrl, branchName, HttpUtils.getUserIp(request));
return gitRepositoryService.getBusinessLogic(repositoryUrl, branchName);
}
@GetMapping("get-resources")
public List<ResourceUnit> getResource(String repositoryUrl, String branchName) throws IOException, GitAPIException {
public List<ResourceUnit> getResource(String repositoryUrl,
String branchName,
HttpServletRequest request) throws IOException, GitAPIException {
LOG.debug("Get resources {} {}. User ip {}", repositoryUrl, branchName, HttpUtils.getUserIp(request));
return gitRepositoryService.getResource(repositoryUrl, branchName);
}
}

2
src/main/java/ru/ulstu/extractor/model/Commit.java
View File

@ -29,7 +29,7 @@ public class Commit extends BaseEntity {
@JoinColumn(name = "commit_id")
@Fetch(FetchMode.SUBSELECT)
private List<FileChange> fileChanges = new ArrayList<>();
@ManyToOne(cascade = CascadeType.ALL, fetch = FetchType.LAZY)
@ManyToOne(fetch = FetchType.LAZY)
private Branch branch;
public Commit() {

1
src/main/java/ru/ulstu/extractor/repository/CommitRepository.java
View File

@ -38,4 +38,5 @@ public interface CommitRepository extends JpaRepository<Commit, Integer> {
@Query("SELECT new ru.ulstu.extractor.model.CommitTimeStatistic(cast(c.date as date), COUNT(DISTINCT c.hash)) FROM Commit c JOIN c.fileChanges f WHERE f.containsEntity = true GROUP by cast(c.date as date) ORDER by cast(c.date as date)")
List<CommitTimeStatistic> getCommitTimeEntityStatistic();
void deleteByBranchIsNull();
}

6
src/main/java/ru/ulstu/extractor/service/AuthorService.java
View File

@ -5,25 +5,27 @@
package ru.ulstu.extractor.service;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import ru.ulstu.extractor.model.Author;
import ru.ulstu.extractor.repository.AuthorRepository;
import javax.transaction.Transactional;
import java.util.Optional;
@Service
public class AuthorService {
private final static Logger LOG = LoggerFactory.getLogger(AuthorService.class);
private final AuthorRepository authorRepository;
public AuthorService(AuthorRepository authorRepository) {
this.authorRepository = authorRepository;
}
@Transactional
public Author findOrCreate(Author author) {
Optional<Author> newAuthor = authorRepository.findByName(author.getName()).stream().findAny();
if (newAuthor.isEmpty()) {
LOG.debug("Author {} was saved.", author.getName());
return authorRepository.save(author);
}
return newAuthor.get();

18
src/main/java/ru/ulstu/extractor/service/BranchService.java
View File

@ -5,17 +5,21 @@
package ru.ulstu.extractor.service;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import ru.ulstu.extractor.model.BaseEntity;
import ru.ulstu.extractor.model.Branch;
import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.model.Repository;
import ru.ulstu.extractor.repository.BranchRepository;
import javax.transaction.Transactional;
import java.util.List;
import java.util.stream.Collectors;
@Service
public class BranchService {
private final static Logger LOG = LoggerFactory.getLogger(BranchService.class);
private final BranchRepository branchRepository;
private final CommitService commitService;
@ -25,13 +29,15 @@ public class BranchService {
this.commitService = commitService;
}
@Transactional
public Branch save(Branch branch, List<Commit> commits) {
List<Commit> commitsToRemove = branch.getCommits();
branch.getCommits().clear();
commitService.delete(commitsToRemove);
LOG.debug("Start save {} branch with {} commits ", branch.getName(), commits.size());
List<Integer> commitsToRemoveIds = branch.getCommits().stream().map(BaseEntity::getId).collect(Collectors.toList());
branch.setCommits(commitService.save(commits));
return branchRepository.save(branch);
LOG.debug("Save branch {} ", branch.getName());
branch = branchRepository.save(branch);
LOG.debug("Clear {} commits ", commitsToRemoveIds.size());
commitService.deleteWithEmptyIds();
return branch;
}
public Branch findByRepositoryAndName(Repository repository, String branchName) {

6
src/main/java/ru/ulstu/extractor/service/CommitService.java
View File

@ -14,7 +14,6 @@ import org.springframework.stereotype.Service;
import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.repository.CommitRepository;
import javax.transaction.Transactional;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
@ -53,7 +52,6 @@ public class CommitService {
commitRepository.deleteAll(commitsToRemove);
}
@Transactional
public List<Commit> save(List<Commit> commits) {
return commits.stream()
.map(commit -> {
@ -61,6 +59,10 @@ public class CommitService {
return commitRepository.save(commit);
}).collect(Collectors.toList());
}
public void deleteWithEmptyIds() {
commitRepository.deleteByBranchIsNull();
}
}

35
src/main/java/ru/ulstu/extractor/service/GitRepositoryService.java
View File

@ -18,6 +18,8 @@ import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import ru.ulstu.extractor.heuristic.model.BusinessLogicUnit;
@ -43,15 +45,20 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import static org.apache.logging.log4j.util.Strings.isBlank;
@Service
public class GitRepositoryService {
private final static Logger LOG = LoggerFactory.getLogger(GitRepositoryService.class);
private static final String BRANCH_PREFIX = "refs/remotes/origin/";
@Value("${extractor.custom-projects-dir}")
private String customProjectsDir;
private final ExecutorService executorService = Executors.newFixedThreadPool(4);
private final StructuralUnitService structuralUnitService;
@ -60,7 +67,9 @@ public class GitRepositoryService {
}
public List<Branch> getRemoteBranches(String url) throws GitAPIException, IOException {
LOG.debug("Get remote branches of {}. Clone", url);
cloneOrUpdateRepo(url);
LOG.debug("Get remote branches of {}. Get branches", url);
Repository localRepo = new FileRepository(getProjectGitDirectory(url));
Git git = new Git(localRepo);
List<Branch> branches = git.branchList().setListMode(ListBranchCommand.ListMode.REMOTE)
@ -74,7 +83,9 @@ public class GitRepositoryService {
}
public List<Branch> getLocalBranches(String url) throws GitAPIException, IOException {
LOG.debug("Get local branches of {}. Clone", url);
cloneOrUpdateRepo(url);
LOG.debug("Get local branches of {}. Get branches", url);
Repository localRepo = new FileRepository(getProjectGitDirectory(url));
Git git = new Git(localRepo);
List<Branch> branches = git.branchList()
@ -88,6 +99,7 @@ public class GitRepositoryService {
}
public List<Commit> getCommits(String repositoryUrl, String branchName) throws GitAPIException, IOException {
LOG.debug("Get commits of {}. Branch {}", repositoryUrl, branchName);
cloneOrUpdateRepo(repositoryUrl, branchName);
Repository localRepo = new FileRepository(getProjectGitDirectory(repositoryUrl));
Git git = new Git(localRepo);
@ -97,7 +109,10 @@ public class GitRepositoryService {
List<Commit> list = new ArrayList<>();
RevCommit prevCommit = null;
LOG.debug("Start analyse {} commits", commits.size());
int counter = commits.size();
for (RevCommit revCommit : commits) {
LOG.debug(" {} of {} commits", counter--, commits.size());
Commit commit = new Commit(
revCommit.getFullMessage(),
new Author(revCommit.getAuthorIdent().getName()),
@ -115,6 +130,7 @@ public class GitRepositoryService {
}
private void checkoutBranch(String repositoryUrl, Git git, Repository localRepo, String branchName) throws GitAPIException, IOException {
LOG.debug("Checkout branch {} {}", repositoryUrl, branchName);
git.pull().call();
if (!localRepo.getBranch().equals(branchName)) {
if (getLocalBranches(repositoryUrl).stream().anyMatch(localBranch -> localBranch.getName().contains(branchName))) {
@ -236,12 +252,19 @@ public class GitRepositoryService {
if (maybeFileName.isPresent()) {
fileChange = new FileChange();
fileChange.setFile(maybeFileName.get());
fileChange.setContainsEntity(
structuralUnitService.containsEntity(getContent(repository, commit, maybeFileName.get()))
);
fileChange.setContainsBusinessLogic(
structuralUnitService.containsBusinessLogic(getContent(repository, commit, maybeFileName.get()))
);
Future<Boolean> futureEntity = executorService.submit(() -> {
return structuralUnitService.containsEntity(getContent(repository, commit, maybeFileName.get()));
});
Future<Boolean> futureBL = executorService.submit(() -> {
return structuralUnitService.containsBusinessLogic(getContent(repository, commit, maybeFileName.get()));
});
try {
fileChange.setContainsBusinessLogic(futureBL.get());
fileChange.setContainsEntity(futureEntity.get());
} catch (Exception ex) {
LOG.warn(ex.getMessage());
}
/// вытащить другие изменения из коммита
changes.add(fileChange);
}

14
src/main/java/ru/ulstu/extractor/service/IndexService.java
View File

@ -7,12 +7,13 @@ package ru.ulstu.extractor.service;
import com.sun.istack.NotNull;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.ulstu.extractor.model.Branch;
import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.model.Repository;
import ru.ulstu.extractor.repository.CommitRepository;
import ru.ulstu.extractor.repository.RepositoryRepository;
import java.io.IOException;
@ -20,22 +21,17 @@ import java.util.List;
@Service
public class IndexService {
private final static Logger LOG = LoggerFactory.getLogger(IndexService.class);
private final GitRepositoryService gitRepositoryService;
private final RepositoryRepository repositoryRepository;
private final BranchService branchService;
private final CommitRepository commitRepository;
private final AuthorService authorService;
public IndexService(GitRepositoryService gitRepositoryService,
RepositoryRepository repositoryRepository,
BranchService branchService,
CommitRepository commitRepository,
AuthorService authorService) {
BranchService branchService) {
this.gitRepositoryService = gitRepositoryService;
this.repositoryRepository = repositoryRepository;
this.branchService = branchService;
this.commitRepository = commitRepository;
this.authorService = authorService;
}
@Transactional
@ -49,6 +45,8 @@ public class IndexService {
branch = new Branch(repository, branchName);
}
List<Commit> commits = gitRepositoryService.getCommits(repositoryUrl, branchName);
LOG.debug("{} commits loaded.", commits.size());
branchService.save(branch, commits);
LOG.debug("{} commits successfully saved. {} {}", commits.size(), repositoryUrl, branchName);
}
}

18
src/main/java/ru/ulstu/extractor/util/HttpUtils.java Normal file
View File

@ -0,0 +1,18 @@
/*
* Copyright (C) 2021 Anton Romanov - All Rights Reserved
* You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/
package ru.ulstu.extractor.util;
import javax.servlet.http.HttpServletRequest;
public class HttpUtils {
public static String getUserIp(HttpServletRequest request) {
String xfHeader = request.getHeader("X-Forwarded-For");
if (xfHeader == null) {
return request.getRemoteAddr();
}
return xfHeader.split(",")[0];
}
}