From de6c83c2393542ecb1d78c177a492a83c7c768ee Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Mon, 20 Feb 2023 14:18:37 +0400 Subject: [PATCH] #92 -- Move id repository to model --- .../gitrepository/model/GitRepository.java | 23 +++++++++-------- .../gitrepository/service/GitApi.java | 2 ++ .../gitrepository/service/GitAtheneApi.java | 5 ++++ .../service/GitRepositoryService.java | 25 ++++++++++++++++--- .../gitrepository/service/GithubApi.java | 5 ++++ .../gitrepository/service/GitlabApi.java | 21 +++++++++------- .../gitrepository/service/IndexService.java | 4 --- .../ts/creator/scheduled/BranchTS.java | 11 +++++++- .../ts/creator/scheduled/IssuesTS.java | 13 ++++++++-- .../scheduled/ScheduledTimeSeriesCreator.java | 16 ++---------- .../ts/creator/scheduled/StarTS.java | 13 ++++++++-- .../ru/ulstu/extractor/util/StringUtils.java | 10 ++++++++ .../db/changelog-20221012_170000-schema.xml | 20 +++++++++++++++ 13 files changed, 122 insertions(+), 46 deletions(-) diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/model/GitRepository.java b/src/main/java/ru/ulstu/extractor/gitrepository/model/GitRepository.java index 652faf3..f4707c7 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/model/GitRepository.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/model/GitRepository.java @@ -1,18 +1,21 @@ package ru.ulstu.extractor.gitrepository.model; import ru.ulstu.extractor.core.BaseEntity; +import ru.ulstu.extractor.util.StringUtils; import javax.persistence.Entity; @Entity public class GitRepository extends BaseEntity { private String url; + private String repositoryId; public GitRepository() { } - public GitRepository(String repositoryUrl) { - url = repositoryUrl; + public GitRepository(String repositoryUrl, String repositoryId) { + this.url = repositoryUrl; + this.repositoryId = repositoryId; } public String getUrl() { @@ -23,16 +26,14 @@ public class GitRepository extends BaseEntity { this.url = url; } - public String getName() { - int lastDelimiterIndex = url.lastIndexOf("/"); - return (lastDelimiterIndex > 0 && lastDelimiterIndex < url.length()) - ? removeDotGit(url.substring(lastDelimiterIndex + 1)) - : removeDotGit(url); + public String getRepositoryId() { + return repositoryId; } - private String removeDotGit(String prevName) { - return prevName.lastIndexOf(".git") > 0 - ? prevName.substring(0, prevName.lastIndexOf(".git")) - : prevName; + public String getName() { + int lastDelimiterIndex = url.lastIndexOf("/"); + return StringUtils.removeDotGit((lastDelimiterIndex > 0 && lastDelimiterIndex < url.length()) + ? url.substring(lastDelimiterIndex + 1) + : url); } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitApi.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitApi.java index 4be8736..33d3a14 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitApi.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitApi.java @@ -12,4 +12,6 @@ public interface GitApi { String getFormattedUrl(Branch branch, String template); Integer getAuthorsCompletedIssues(Branch branch); + + String getRepositoryId(String repositoryUrl); } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitAtheneApi.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitAtheneApi.java index d6644e1..b1fbf55 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitAtheneApi.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitAtheneApi.java @@ -50,4 +50,9 @@ public class GitAtheneApi implements GitApi { .get(getFormattedUrl(branch, COMPLETED_ISSUES_URL)) .length(); } + + @Override + public String getRepositoryId(String repositoryUrl) { + return null; + } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java index f6ead24..e400670 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitRepositoryService.java @@ -15,6 +15,7 @@ import org.eclipse.jgit.treewalk.TreeWalk; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationContext; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; @@ -29,6 +30,7 @@ import ru.ulstu.extractor.heuristic.model.BusinessLogicUnit; import ru.ulstu.extractor.heuristic.model.EntityUnit; import ru.ulstu.extractor.heuristic.model.ResourceUnit; import ru.ulstu.extractor.heuristic.service.StructuralUnitService; +import ru.ulstu.extractor.util.StringUtils; import java.io.ByteArrayOutputStream; import java.io.File; @@ -55,7 +57,11 @@ import static org.apache.logging.log4j.util.Strings.isBlank; @Service public class GitRepositoryService { - private final static Logger LOG = LoggerFactory.getLogger(GitRepositoryService.class); + private final Map> gitApiServiceHolder = Map.of( + "git.athene.tech", GitAtheneApi.class, + "github.com", GithubApi.class, + "gitlab.com", GitlabApi.class); + private static final Logger LOG = LoggerFactory.getLogger(GitRepositoryService.class); private static final String BRANCH_PREFIX = "refs/remotes/origin/"; @Value("${extractor.custom-projects-dir}") private String customProjectsDir; @@ -63,11 +69,14 @@ public class GitRepositoryService { private final ExecutorService executorServiceCommits = Executors.newFixedThreadPool(8); private final StructuralUnitService structuralUnitService; private final GitRepositoryRepository gitRepositoryRepository; + private final ApplicationContext applicationContext; public GitRepositoryService(StructuralUnitService structuralUnitService, - GitRepositoryRepository gitRepositoryRepository) { + GitRepositoryRepository gitRepositoryRepository, + ApplicationContext applicationContext) { this.structuralUnitService = structuralUnitService; this.gitRepositoryRepository = gitRepositoryRepository; + this.applicationContext = applicationContext; } public List getRemoteBranches(String url) throws GitAPIException, IOException { @@ -430,8 +439,18 @@ public class GitRepositoryService { public GitRepository findByUrlOrCreate(String gitRepositoryUrl) { GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl); + return gitRepository == null - ? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl)) + ? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl, + applicationContext.getBean(getGitApiServiceClass(gitRepositoryUrl)).getRepositoryId(gitRepositoryUrl))) : gitRepository; } + + public Class getGitApiServiceClass(Branch branch) { + return getGitApiServiceClass(branch.getGitRepository().getUrl()); + } + + public Class getGitApiServiceClass(String url) { + return gitApiServiceHolder.get(StringUtils.getServerDomain(url)); + } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/GithubApi.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/GithubApi.java index e744bb7..717594f 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/GithubApi.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/GithubApi.java @@ -50,4 +50,9 @@ public class GithubApi implements GitApi { .get(getFormattedUrl(branch, AUTHOR_COMPLETED_ISSUES_URL)) .length(); } + + @Override + public String getRepositoryId(String repositoryUrl) { + return null; + } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitlabApi.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitlabApi.java index 60feab5..9f774a3 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/GitlabApi.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/GitlabApi.java @@ -5,6 +5,9 @@ import org.springframework.stereotype.Service; import ru.ulstu.extractor.branch.model.Branch; import ru.ulstu.extractor.http.HttpService; +import static ru.ulstu.extractor.util.StringUtils.getUrlParts; +import static ru.ulstu.extractor.util.StringUtils.removeDotGit; + @Service public class GitlabApi implements GitApi { private final HttpService httpService; @@ -42,14 +45,14 @@ public class GitlabApi implements GitApi { @Override public String getFormattedUrl(Branch branch, String template) { - String[] urlParts = branch.getGitRepository().getUrl().split("/"); - return String.format(template, urlParts[0] + "/" + urlParts[1] + "/" + urlParts[2], getProjectId(branch)); + String[] urlParts = getUrlParts(branch.getGitRepository().getUrl()); + return String.format(template, urlParts[0] + "/" + urlParts[1] + "/" + urlParts[2], branch.getGitRepository().getRepositoryId()); } - public String getFormattedUrlForProjectId(Branch branch, String template) { - String[] urlParts = branch.getGitRepository().getUrl().split("/"); + public String getFormattedUrlForProjectId(String repositoryUrl, String template) { + String[] urlParts = getUrlParts(repositoryUrl); // получаем корректное название репозитория, gitlab всегда добавляет .git в конец - return String.format(template, urlParts[3], branch.getGitRepository().getName()); + return String.format(template, urlParts[3], removeDotGit(urlParts[4])); } @Override @@ -59,13 +62,13 @@ public class GitlabApi implements GitApi { .length(); } - private String getProjectId(Branch branch) { - JSONArray projects = httpService.get(getFormattedUrlForProjectId(branch, PROJECT_ID_URL)); + public String getRepositoryId(String repositoryUrl) { + JSONArray projects = httpService.get(getFormattedUrlForProjectId(repositoryUrl, PROJECT_ID_URL)); for (int i = 0; i < projects.length(); i++) { - if (projects.getJSONObject(i).get("name").equals(branch.getGitRepository().getName())) { + if (projects.getJSONObject(i).get("name").equals(removeDotGit(getUrlParts(repositoryUrl)[4]))) { return String.valueOf(projects.getJSONObject(i).getInt("id")); } } - throw new RuntimeException("Id проекта не найден: " + branch.getGitRepository().getName()); + throw new RuntimeException("Id проекта не найден: " + repositoryUrl); } } diff --git a/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java b/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java index 7921c21..6d43000 100644 --- a/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java +++ b/src/main/java/ru/ulstu/extractor/gitrepository/service/IndexService.java @@ -17,7 +17,6 @@ import ru.ulstu.extractor.branch.model.IndexingStatus; import ru.ulstu.extractor.branch.service.BranchService; import ru.ulstu.extractor.commit.model.Commit; import ru.ulstu.extractor.gitrepository.model.GitRepository; -import ru.ulstu.extractor.gitrepository.repository.GitRepositoryRepository; import ru.ulstu.extractor.ts.creator.db.DBTimeSeriesCreator; import java.io.IOException; @@ -31,17 +30,14 @@ public class IndexService { private final static Logger LOG = LoggerFactory.getLogger(IndexService.class); private final static int COMMITS_PAGE_SIZE = 10; private final GitRepositoryService gitRepositoryService; - private final GitRepositoryRepository gitRepositoryRepository; private final BranchService branchService; private final List timeSeriesCreators; private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10); public IndexService(GitRepositoryService gitRepositoryService, - GitRepositoryRepository gitRepositoryRepository, BranchService branchService, List timeSeriesCreators) { this.gitRepositoryService = gitRepositoryService; - this.gitRepositoryRepository = gitRepositoryRepository; this.branchService = branchService; this.timeSeriesCreators = timeSeriesCreators; } diff --git a/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/BranchTS.java b/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/BranchTS.java index 81289db..2ae2510 100644 --- a/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/BranchTS.java +++ b/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/BranchTS.java @@ -3,6 +3,7 @@ package ru.ulstu.extractor.ts.creator.scheduled; import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.gitrepository.service.GitRepositoryService; import ru.ulstu.extractor.ts.model.TimeSeriesType; import ru.ulstu.extractor.ts.model.TimeSeriesValue; import ru.ulstu.extractor.ts.service.TimeSeriesService; @@ -10,11 +11,14 @@ import ru.ulstu.extractor.ts.service.TimeSeriesService; @Component public class BranchTS extends ScheduledTimeSeriesCreator { private final TimeSeriesService timeSeriesService; + private final GitRepositoryService gitRepositoryService; private final ApplicationContext applicationContext; public BranchTS(TimeSeriesService timeSeriesService, + GitRepositoryService gitRepositoryService, ApplicationContext applicationContext) { this.timeSeriesService = timeSeriesService; + this.gitRepositoryService = gitRepositoryService; this.applicationContext = applicationContext; } @@ -28,9 +32,14 @@ public class BranchTS extends ScheduledTimeSeriesCreator { return timeSeriesService; } + @Override + public GitRepositoryService getGitRepositoryService() { + return gitRepositoryService; + } + @Override public TimeSeriesValue getNewTimeSeriesValue(Branch branch) { - return new TimeSeriesValue(applicationContext.getBean(getGitApiServiceClass(branch)) + return new TimeSeriesValue(applicationContext.getBean(getGitRepositoryService().getGitApiServiceClass(branch)) .getBranchesCount(branch)); } } diff --git a/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/IssuesTS.java b/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/IssuesTS.java index 8d26648..4f9e9f6 100644 --- a/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/IssuesTS.java +++ b/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/IssuesTS.java @@ -3,6 +3,7 @@ package ru.ulstu.extractor.ts.creator.scheduled; import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.gitrepository.service.GitRepositoryService; import ru.ulstu.extractor.ts.model.TimeSeriesType; import ru.ulstu.extractor.ts.model.TimeSeriesValue; import ru.ulstu.extractor.ts.service.TimeSeriesService; @@ -11,16 +12,19 @@ import ru.ulstu.extractor.ts.service.TimeSeriesService; public class IssuesTS extends ScheduledTimeSeriesCreator { private final TimeSeriesService timeSeriesService; private final ApplicationContext applicationContext; + private final GitRepositoryService gitRepositoryService; public IssuesTS(TimeSeriesService timeSeriesService, - ApplicationContext applicationContext) { + ApplicationContext applicationContext, + GitRepositoryService gitRepositoryService) { this.timeSeriesService = timeSeriesService; this.applicationContext = applicationContext; + this.gitRepositoryService = gitRepositoryService; } @Override public TimeSeriesValue getNewTimeSeriesValue(Branch branch) { - return new TimeSeriesValue(applicationContext.getBean(getGitApiServiceClass(branch)) + return new TimeSeriesValue(applicationContext.getBean(getGitRepositoryService().getGitApiServiceClass(branch)) .getOpenIssuesCount(branch)); } @@ -33,4 +37,9 @@ public class IssuesTS extends ScheduledTimeSeriesCreator { public TimeSeriesType getTimeSeriesType() { return TimeSeriesType.ISSUES; } + + @Override + public GitRepositoryService getGitRepositoryService() { + return gitRepositoryService; + } } diff --git a/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/ScheduledTimeSeriesCreator.java b/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/ScheduledTimeSeriesCreator.java index 0bb3786..032bb40 100644 --- a/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/ScheduledTimeSeriesCreator.java +++ b/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/ScheduledTimeSeriesCreator.java @@ -2,24 +2,14 @@ package ru.ulstu.extractor.ts.creator.scheduled; import org.springframework.transaction.annotation.Transactional; import ru.ulstu.extractor.branch.model.Branch; -import ru.ulstu.extractor.gitrepository.service.GitApi; -import ru.ulstu.extractor.gitrepository.service.GitAtheneApi; -import ru.ulstu.extractor.gitrepository.service.GithubApi; -import ru.ulstu.extractor.gitrepository.service.GitlabApi; +import ru.ulstu.extractor.gitrepository.service.GitRepositoryService; import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator; import ru.ulstu.extractor.ts.model.TimeSeries; import ru.ulstu.extractor.ts.model.TimeSeriesValue; -import ru.ulstu.extractor.util.StringUtils; import java.util.Collections; -import java.util.Map; public abstract class ScheduledTimeSeriesCreator extends AbstractTimeSeriesCreator { - private final Map> gitApiServiceHolder = Map.of( - "git.athene.tech", GitAtheneApi.class, - "github.com", GithubApi.class, - "gitlab.com", GitlabApi.class); - @Transactional public void addTimeSeriesValue(Branch branch) { TimeSeries timeSeries = getTimeSeriesService().findByBranchAndTypeOrCreate(branch, getTimeSeriesType()); @@ -29,7 +19,5 @@ public abstract class ScheduledTimeSeriesCreator extends AbstractTimeSeriesCreat public abstract TimeSeriesValue getNewTimeSeriesValue(Branch branch); - public Class getGitApiServiceClass(Branch branch) { - return gitApiServiceHolder.get(StringUtils.getServerDomain(branch.getGitRepository().getUrl())); - } + public abstract GitRepositoryService getGitRepositoryService(); } diff --git a/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/StarTS.java b/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/StarTS.java index 27becfa..46f5f28 100644 --- a/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/StarTS.java +++ b/src/main/java/ru/ulstu/extractor/ts/creator/scheduled/StarTS.java @@ -3,6 +3,7 @@ package ru.ulstu.extractor.ts.creator.scheduled; import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import ru.ulstu.extractor.branch.model.Branch; +import ru.ulstu.extractor.gitrepository.service.GitRepositoryService; import ru.ulstu.extractor.ts.model.TimeSeriesType; import ru.ulstu.extractor.ts.model.TimeSeriesValue; import ru.ulstu.extractor.ts.service.TimeSeriesService; @@ -11,11 +12,14 @@ import ru.ulstu.extractor.ts.service.TimeSeriesService; public class StarTS extends ScheduledTimeSeriesCreator { private final TimeSeriesService timeSeriesService; private final ApplicationContext applicationContext; + private final GitRepositoryService gitRepositoryService; public StarTS(TimeSeriesService timeSeriesService, - ApplicationContext applicationContext) { + ApplicationContext applicationContext, + GitRepositoryService gitRepositoryService) { this.timeSeriesService = timeSeriesService; this.applicationContext = applicationContext; + this.gitRepositoryService = gitRepositoryService; } @Override @@ -28,9 +32,14 @@ public class StarTS extends ScheduledTimeSeriesCreator { return TimeSeriesType.STARS; } + @Override + public GitRepositoryService getGitRepositoryService() { + return gitRepositoryService; + } + @Override public TimeSeriesValue getNewTimeSeriesValue(Branch branch) { - return new TimeSeriesValue(applicationContext.getBean(getGitApiServiceClass(branch)) + return new TimeSeriesValue(applicationContext.getBean(getGitRepositoryService().getGitApiServiceClass(branch)) .getStarsCount(branch)); } } diff --git a/src/main/java/ru/ulstu/extractor/util/StringUtils.java b/src/main/java/ru/ulstu/extractor/util/StringUtils.java index 9d137e5..0c57bcb 100644 --- a/src/main/java/ru/ulstu/extractor/util/StringUtils.java +++ b/src/main/java/ru/ulstu/extractor/util/StringUtils.java @@ -31,4 +31,14 @@ public class StringUtils { String[] urlParts = url.split("/"); return urlParts[2]; } + + public static String[] getUrlParts(String gitRepositoryUrl) { + return gitRepositoryUrl.split("/"); + } + + public static String removeDotGit(String repositoryName) { + return repositoryName.lastIndexOf(".git") > 0 + ? repositoryName.substring(0, repositoryName.lastIndexOf(".git")) + : repositoryName; + } } diff --git a/src/main/resources/db/changelog-20221012_170000-schema.xml b/src/main/resources/db/changelog-20221012_170000-schema.xml index 6ee4e83..6b8df1a 100644 --- a/src/main/resources/db/changelog-20221012_170000-schema.xml +++ b/src/main/resources/db/changelog-20221012_170000-schema.xml @@ -107,4 +107,24 @@ + + + + + + + + + + + + + \ No newline at end of file