#92 -- gitlab repository id #93

Merged
romanov73 merged 3 commits from 92-id-gitlab into master 2023-02-20 14:25:20 +04:00
13 changed files with 122 additions and 46 deletions
Showing only changes of commit de6c83c239 - Show all commits

View File

@ -1,18 +1,21 @@
package ru.ulstu.extractor.gitrepository.model; package ru.ulstu.extractor.gitrepository.model;
import ru.ulstu.extractor.core.BaseEntity; import ru.ulstu.extractor.core.BaseEntity;
import ru.ulstu.extractor.util.StringUtils;
import javax.persistence.Entity; import javax.persistence.Entity;
@Entity @Entity
public class GitRepository extends BaseEntity { public class GitRepository extends BaseEntity {
private String url; private String url;
private String repositoryId;
public GitRepository() { public GitRepository() {
} }
public GitRepository(String repositoryUrl) { public GitRepository(String repositoryUrl, String repositoryId) {
url = repositoryUrl; this.url = repositoryUrl;
this.repositoryId = repositoryId;
} }
public String getUrl() { public String getUrl() {
@ -23,16 +26,14 @@ public class GitRepository extends BaseEntity {
this.url = url; this.url = url;
} }
public String getName() { public String getRepositoryId() {
int lastDelimiterIndex = url.lastIndexOf("/"); return repositoryId;
return (lastDelimiterIndex > 0 && lastDelimiterIndex < url.length())
? removeDotGit(url.substring(lastDelimiterIndex + 1))
: removeDotGit(url);
} }
private String removeDotGit(String prevName) { public String getName() {
return prevName.lastIndexOf(".git") > 0 int lastDelimiterIndex = url.lastIndexOf("/");
? prevName.substring(0, prevName.lastIndexOf(".git")) return StringUtils.removeDotGit((lastDelimiterIndex > 0 && lastDelimiterIndex < url.length())
: prevName; ? url.substring(lastDelimiterIndex + 1)
: url);
} }
} }

View File

@ -12,4 +12,6 @@ public interface GitApi {
String getFormattedUrl(Branch branch, String template); String getFormattedUrl(Branch branch, String template);
Integer getAuthorsCompletedIssues(Branch branch); Integer getAuthorsCompletedIssues(Branch branch);
String getRepositoryId(String repositoryUrl);
} }

View File

@ -50,4 +50,9 @@ public class GitAtheneApi implements GitApi {
.get(getFormattedUrl(branch, COMPLETED_ISSUES_URL)) .get(getFormattedUrl(branch, COMPLETED_ISSUES_URL))
.length(); .length();
} }
@Override
public String getRepositoryId(String repositoryUrl) {
return null;
}
} }

View File

@ -15,6 +15,7 @@ import org.eclipse.jgit.treewalk.TreeWalk;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -29,6 +30,7 @@ import ru.ulstu.extractor.heuristic.model.BusinessLogicUnit;
import ru.ulstu.extractor.heuristic.model.EntityUnit; import ru.ulstu.extractor.heuristic.model.EntityUnit;
import ru.ulstu.extractor.heuristic.model.ResourceUnit; import ru.ulstu.extractor.heuristic.model.ResourceUnit;
import ru.ulstu.extractor.heuristic.service.StructuralUnitService; import ru.ulstu.extractor.heuristic.service.StructuralUnitService;
import ru.ulstu.extractor.util.StringUtils;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
@ -55,7 +57,11 @@ import static org.apache.logging.log4j.util.Strings.isBlank;
@Service @Service
public class GitRepositoryService { public class GitRepositoryService {
private final static Logger LOG = LoggerFactory.getLogger(GitRepositoryService.class); private final Map<String, Class<? extends GitApi>> gitApiServiceHolder = Map.of(
"git.athene.tech", GitAtheneApi.class,
"github.com", GithubApi.class,
"gitlab.com", GitlabApi.class);
private static final Logger LOG = LoggerFactory.getLogger(GitRepositoryService.class);
private static final String BRANCH_PREFIX = "refs/remotes/origin/"; private static final String BRANCH_PREFIX = "refs/remotes/origin/";
@Value("${extractor.custom-projects-dir}") @Value("${extractor.custom-projects-dir}")
private String customProjectsDir; private String customProjectsDir;
@ -63,11 +69,14 @@ public class GitRepositoryService {
private final ExecutorService executorServiceCommits = Executors.newFixedThreadPool(8); private final ExecutorService executorServiceCommits = Executors.newFixedThreadPool(8);
private final StructuralUnitService structuralUnitService; private final StructuralUnitService structuralUnitService;
private final GitRepositoryRepository gitRepositoryRepository; private final GitRepositoryRepository gitRepositoryRepository;
private final ApplicationContext applicationContext;
public GitRepositoryService(StructuralUnitService structuralUnitService, public GitRepositoryService(StructuralUnitService structuralUnitService,
GitRepositoryRepository gitRepositoryRepository) { GitRepositoryRepository gitRepositoryRepository,
ApplicationContext applicationContext) {
this.structuralUnitService = structuralUnitService; this.structuralUnitService = structuralUnitService;
this.gitRepositoryRepository = gitRepositoryRepository; this.gitRepositoryRepository = gitRepositoryRepository;
this.applicationContext = applicationContext;
} }
public List<Branch> getRemoteBranches(String url) throws GitAPIException, IOException { public List<Branch> getRemoteBranches(String url) throws GitAPIException, IOException {
@ -430,8 +439,18 @@ public class GitRepositoryService {
public GitRepository findByUrlOrCreate(String gitRepositoryUrl) { public GitRepository findByUrlOrCreate(String gitRepositoryUrl) {
GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl); GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl);
return gitRepository == null return gitRepository == null
? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl)) ? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl,
applicationContext.getBean(getGitApiServiceClass(gitRepositoryUrl)).getRepositoryId(gitRepositoryUrl)))
: gitRepository; : gitRepository;
} }
public Class<? extends GitApi> getGitApiServiceClass(Branch branch) {
return getGitApiServiceClass(branch.getGitRepository().getUrl());
}
public Class<? extends GitApi> getGitApiServiceClass(String url) {
return gitApiServiceHolder.get(StringUtils.getServerDomain(url));
}
} }

View File

@ -50,4 +50,9 @@ public class GithubApi implements GitApi {
.get(getFormattedUrl(branch, AUTHOR_COMPLETED_ISSUES_URL)) .get(getFormattedUrl(branch, AUTHOR_COMPLETED_ISSUES_URL))
.length(); .length();
} }
@Override
public String getRepositoryId(String repositoryUrl) {
return null;
}
} }

View File

@ -5,6 +5,9 @@ import org.springframework.stereotype.Service;
import ru.ulstu.extractor.branch.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.http.HttpService; import ru.ulstu.extractor.http.HttpService;
import static ru.ulstu.extractor.util.StringUtils.getUrlParts;
import static ru.ulstu.extractor.util.StringUtils.removeDotGit;
@Service @Service
public class GitlabApi implements GitApi { public class GitlabApi implements GitApi {
private final HttpService httpService; private final HttpService httpService;
@ -42,14 +45,14 @@ public class GitlabApi implements GitApi {
@Override @Override
public String getFormattedUrl(Branch branch, String template) { public String getFormattedUrl(Branch branch, String template) {
String[] urlParts = branch.getGitRepository().getUrl().split("/"); String[] urlParts = getUrlParts(branch.getGitRepository().getUrl());
return String.format(template, urlParts[0] + "/" + urlParts[1] + "/" + urlParts[2], getProjectId(branch)); return String.format(template, urlParts[0] + "/" + urlParts[1] + "/" + urlParts[2], branch.getGitRepository().getRepositoryId());
} }
public String getFormattedUrlForProjectId(Branch branch, String template) { public String getFormattedUrlForProjectId(String repositoryUrl, String template) {
String[] urlParts = branch.getGitRepository().getUrl().split("/"); String[] urlParts = getUrlParts(repositoryUrl);
// получаем корректное название репозитория, gitlab всегда добавляет .git в конец // получаем корректное название репозитория, gitlab всегда добавляет .git в конец
return String.format(template, urlParts[3], branch.getGitRepository().getName()); return String.format(template, urlParts[3], removeDotGit(urlParts[4]));
} }
@Override @Override
@ -59,13 +62,13 @@ public class GitlabApi implements GitApi {
.length(); .length();
} }
private String getProjectId(Branch branch) { public String getRepositoryId(String repositoryUrl) {
JSONArray projects = httpService.get(getFormattedUrlForProjectId(branch, PROJECT_ID_URL)); JSONArray projects = httpService.get(getFormattedUrlForProjectId(repositoryUrl, PROJECT_ID_URL));
for (int i = 0; i < projects.length(); i++) { for (int i = 0; i < projects.length(); i++) {
if (projects.getJSONObject(i).get("name").equals(branch.getGitRepository().getName())) { if (projects.getJSONObject(i).get("name").equals(removeDotGit(getUrlParts(repositoryUrl)[4]))) {
return String.valueOf(projects.getJSONObject(i).getInt("id")); return String.valueOf(projects.getJSONObject(i).getInt("id"));
} }
} }
throw new RuntimeException("Id проекта не найден: " + branch.getGitRepository().getName()); throw new RuntimeException("Id проекта не найден: " + repositoryUrl);
} }
} }

View File

@ -17,7 +17,6 @@ import ru.ulstu.extractor.branch.model.IndexingStatus;
import ru.ulstu.extractor.branch.service.BranchService; import ru.ulstu.extractor.branch.service.BranchService;
import ru.ulstu.extractor.commit.model.Commit; import ru.ulstu.extractor.commit.model.Commit;
import ru.ulstu.extractor.gitrepository.model.GitRepository; import ru.ulstu.extractor.gitrepository.model.GitRepository;
import ru.ulstu.extractor.gitrepository.repository.GitRepositoryRepository;
import ru.ulstu.extractor.ts.creator.db.DBTimeSeriesCreator; import ru.ulstu.extractor.ts.creator.db.DBTimeSeriesCreator;
import java.io.IOException; import java.io.IOException;
@ -31,17 +30,14 @@ public class IndexService {
private final static Logger LOG = LoggerFactory.getLogger(IndexService.class); private final static Logger LOG = LoggerFactory.getLogger(IndexService.class);
private final static int COMMITS_PAGE_SIZE = 10; private final static int COMMITS_PAGE_SIZE = 10;
private final GitRepositoryService gitRepositoryService; private final GitRepositoryService gitRepositoryService;
private final GitRepositoryRepository gitRepositoryRepository;
private final BranchService branchService; private final BranchService branchService;
private final List<DBTimeSeriesCreator> timeSeriesCreators; private final List<DBTimeSeriesCreator> timeSeriesCreators;
private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10); private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10);
public IndexService(GitRepositoryService gitRepositoryService, public IndexService(GitRepositoryService gitRepositoryService,
GitRepositoryRepository gitRepositoryRepository,
BranchService branchService, BranchService branchService,
List<DBTimeSeriesCreator> timeSeriesCreators) { List<DBTimeSeriesCreator> timeSeriesCreators) {
this.gitRepositoryService = gitRepositoryService; this.gitRepositoryService = gitRepositoryService;
this.gitRepositoryRepository = gitRepositoryRepository;
this.branchService = branchService; this.branchService = branchService;
this.timeSeriesCreators = timeSeriesCreators; this.timeSeriesCreators = timeSeriesCreators;
} }

View File

@ -3,6 +3,7 @@ package ru.ulstu.extractor.ts.creator.scheduled;
import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import ru.ulstu.extractor.branch.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
import ru.ulstu.extractor.ts.model.TimeSeriesType; import ru.ulstu.extractor.ts.model.TimeSeriesType;
import ru.ulstu.extractor.ts.model.TimeSeriesValue; import ru.ulstu.extractor.ts.model.TimeSeriesValue;
import ru.ulstu.extractor.ts.service.TimeSeriesService; import ru.ulstu.extractor.ts.service.TimeSeriesService;
@ -10,11 +11,14 @@ import ru.ulstu.extractor.ts.service.TimeSeriesService;
@Component @Component
public class BranchTS extends ScheduledTimeSeriesCreator { public class BranchTS extends ScheduledTimeSeriesCreator {
private final TimeSeriesService timeSeriesService; private final TimeSeriesService timeSeriesService;
private final GitRepositoryService gitRepositoryService;
private final ApplicationContext applicationContext; private final ApplicationContext applicationContext;
public BranchTS(TimeSeriesService timeSeriesService, public BranchTS(TimeSeriesService timeSeriesService,
GitRepositoryService gitRepositoryService,
ApplicationContext applicationContext) { ApplicationContext applicationContext) {
this.timeSeriesService = timeSeriesService; this.timeSeriesService = timeSeriesService;
this.gitRepositoryService = gitRepositoryService;
this.applicationContext = applicationContext; this.applicationContext = applicationContext;
} }
@ -28,9 +32,14 @@ public class BranchTS extends ScheduledTimeSeriesCreator {
return timeSeriesService; return timeSeriesService;
} }
@Override
public GitRepositoryService getGitRepositoryService() {
return gitRepositoryService;
}
@Override @Override
public TimeSeriesValue getNewTimeSeriesValue(Branch branch) { public TimeSeriesValue getNewTimeSeriesValue(Branch branch) {
return new TimeSeriesValue(applicationContext.getBean(getGitApiServiceClass(branch)) return new TimeSeriesValue(applicationContext.getBean(getGitRepositoryService().getGitApiServiceClass(branch))
.getBranchesCount(branch)); .getBranchesCount(branch));
} }
} }

View File

@ -3,6 +3,7 @@ package ru.ulstu.extractor.ts.creator.scheduled;
import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import ru.ulstu.extractor.branch.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
import ru.ulstu.extractor.ts.model.TimeSeriesType; import ru.ulstu.extractor.ts.model.TimeSeriesType;
import ru.ulstu.extractor.ts.model.TimeSeriesValue; import ru.ulstu.extractor.ts.model.TimeSeriesValue;
import ru.ulstu.extractor.ts.service.TimeSeriesService; import ru.ulstu.extractor.ts.service.TimeSeriesService;
@ -11,16 +12,19 @@ import ru.ulstu.extractor.ts.service.TimeSeriesService;
public class IssuesTS extends ScheduledTimeSeriesCreator { public class IssuesTS extends ScheduledTimeSeriesCreator {
private final TimeSeriesService timeSeriesService; private final TimeSeriesService timeSeriesService;
private final ApplicationContext applicationContext; private final ApplicationContext applicationContext;
private final GitRepositoryService gitRepositoryService;
public IssuesTS(TimeSeriesService timeSeriesService, public IssuesTS(TimeSeriesService timeSeriesService,
ApplicationContext applicationContext) { ApplicationContext applicationContext,
GitRepositoryService gitRepositoryService) {
this.timeSeriesService = timeSeriesService; this.timeSeriesService = timeSeriesService;
this.applicationContext = applicationContext; this.applicationContext = applicationContext;
this.gitRepositoryService = gitRepositoryService;
} }
@Override @Override
public TimeSeriesValue getNewTimeSeriesValue(Branch branch) { public TimeSeriesValue getNewTimeSeriesValue(Branch branch) {
return new TimeSeriesValue(applicationContext.getBean(getGitApiServiceClass(branch)) return new TimeSeriesValue(applicationContext.getBean(getGitRepositoryService().getGitApiServiceClass(branch))
.getOpenIssuesCount(branch)); .getOpenIssuesCount(branch));
} }
@ -33,4 +37,9 @@ public class IssuesTS extends ScheduledTimeSeriesCreator {
public TimeSeriesType getTimeSeriesType() { public TimeSeriesType getTimeSeriesType() {
return TimeSeriesType.ISSUES; return TimeSeriesType.ISSUES;
} }
@Override
public GitRepositoryService getGitRepositoryService() {
return gitRepositoryService;
}
} }

View File

@ -2,24 +2,14 @@ package ru.ulstu.extractor.ts.creator.scheduled;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import ru.ulstu.extractor.branch.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.gitrepository.service.GitApi; import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
import ru.ulstu.extractor.gitrepository.service.GitAtheneApi;
import ru.ulstu.extractor.gitrepository.service.GithubApi;
import ru.ulstu.extractor.gitrepository.service.GitlabApi;
import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator; import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator;
import ru.ulstu.extractor.ts.model.TimeSeries; import ru.ulstu.extractor.ts.model.TimeSeries;
import ru.ulstu.extractor.ts.model.TimeSeriesValue; import ru.ulstu.extractor.ts.model.TimeSeriesValue;
import ru.ulstu.extractor.util.StringUtils;
import java.util.Collections; import java.util.Collections;
import java.util.Map;
public abstract class ScheduledTimeSeriesCreator extends AbstractTimeSeriesCreator { public abstract class ScheduledTimeSeriesCreator extends AbstractTimeSeriesCreator {
private final Map<String, Class<? extends GitApi>> gitApiServiceHolder = Map.of(
"git.athene.tech", GitAtheneApi.class,
"github.com", GithubApi.class,
"gitlab.com", GitlabApi.class);
@Transactional @Transactional
public void addTimeSeriesValue(Branch branch) { public void addTimeSeriesValue(Branch branch) {
TimeSeries timeSeries = getTimeSeriesService().findByBranchAndTypeOrCreate(branch, getTimeSeriesType()); TimeSeries timeSeries = getTimeSeriesService().findByBranchAndTypeOrCreate(branch, getTimeSeriesType());
@ -29,7 +19,5 @@ public abstract class ScheduledTimeSeriesCreator extends AbstractTimeSeriesCreat
public abstract TimeSeriesValue getNewTimeSeriesValue(Branch branch); public abstract TimeSeriesValue getNewTimeSeriesValue(Branch branch);
public Class<? extends GitApi> getGitApiServiceClass(Branch branch) { public abstract GitRepositoryService getGitRepositoryService();
return gitApiServiceHolder.get(StringUtils.getServerDomain(branch.getGitRepository().getUrl()));
}
} }

View File

@ -3,6 +3,7 @@ package ru.ulstu.extractor.ts.creator.scheduled;
import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import ru.ulstu.extractor.branch.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
import ru.ulstu.extractor.ts.model.TimeSeriesType; import ru.ulstu.extractor.ts.model.TimeSeriesType;
import ru.ulstu.extractor.ts.model.TimeSeriesValue; import ru.ulstu.extractor.ts.model.TimeSeriesValue;
import ru.ulstu.extractor.ts.service.TimeSeriesService; import ru.ulstu.extractor.ts.service.TimeSeriesService;
@ -11,11 +12,14 @@ import ru.ulstu.extractor.ts.service.TimeSeriesService;
public class StarTS extends ScheduledTimeSeriesCreator { public class StarTS extends ScheduledTimeSeriesCreator {
private final TimeSeriesService timeSeriesService; private final TimeSeriesService timeSeriesService;
private final ApplicationContext applicationContext; private final ApplicationContext applicationContext;
private final GitRepositoryService gitRepositoryService;
public StarTS(TimeSeriesService timeSeriesService, public StarTS(TimeSeriesService timeSeriesService,
ApplicationContext applicationContext) { ApplicationContext applicationContext,
GitRepositoryService gitRepositoryService) {
this.timeSeriesService = timeSeriesService; this.timeSeriesService = timeSeriesService;
this.applicationContext = applicationContext; this.applicationContext = applicationContext;
this.gitRepositoryService = gitRepositoryService;
} }
@Override @Override
@ -28,9 +32,14 @@ public class StarTS extends ScheduledTimeSeriesCreator {
return TimeSeriesType.STARS; return TimeSeriesType.STARS;
} }
@Override
public GitRepositoryService getGitRepositoryService() {
return gitRepositoryService;
}
@Override @Override
public TimeSeriesValue getNewTimeSeriesValue(Branch branch) { public TimeSeriesValue getNewTimeSeriesValue(Branch branch) {
return new TimeSeriesValue(applicationContext.getBean(getGitApiServiceClass(branch)) return new TimeSeriesValue(applicationContext.getBean(getGitRepositoryService().getGitApiServiceClass(branch))
.getStarsCount(branch)); .getStarsCount(branch));
} }
} }

View File

@ -31,4 +31,14 @@ public class StringUtils {
String[] urlParts = url.split("/"); String[] urlParts = url.split("/");
return urlParts[2]; return urlParts[2];
} }
public static String[] getUrlParts(String gitRepositoryUrl) {
return gitRepositoryUrl.split("/");
}
public static String removeDotGit(String repositoryName) {
return repositoryName.lastIndexOf(".git") > 0
? repositoryName.substring(0, repositoryName.lastIndexOf(".git"))
: repositoryName;
}
} }

View File

@ -107,4 +107,24 @@
<column name="indexing_status" value="FINISHED"/> <column name="indexing_status" value="FINISHED"/>
</update> </update>
</changeSet> </changeSet>
<changeSet author="orion" id="20230220-120000-1">
<addColumn tableName="git_repository">
<column name="repository_id" type="varchar(100)"/>
</addColumn>
</changeSet>
<changeSet author="orion" id="20230220-120000-2">
<dropForeignKeyConstraint baseTableName="time_series"
constraintName="fk_time_series_branch"/>
<addForeignKeyConstraint baseTableName="time_series" baseColumnNames="branch_id"
constraintName="fk_time_series_branch" referencedTableName="branch"
referencedColumnNames="id" onDelete="CASCADE"/>
</changeSet>
<changeSet author="orion" id="20230220-120000-3">
<dropForeignKeyConstraint baseTableName="time_series_value"
constraintName="fk_time_series"/>
<addForeignKeyConstraint baseTableName="time_series_value" baseColumnNames="time_series_id"
constraintName="fk_time_series"
referencedTableName="time_series"
referencedColumnNames="id" onDelete="CASCADE"/>
</changeSet>
</databaseChangeLog> </databaseChangeLog>