Reviewed-on: #93
This commit is contained in:
commit
513c2cb7ba
@ -1,18 +1,21 @@
|
||||
package ru.ulstu.extractor.gitrepository.model;
|
||||
|
||||
import ru.ulstu.extractor.core.BaseEntity;
|
||||
import ru.ulstu.extractor.util.StringUtils;
|
||||
|
||||
import javax.persistence.Entity;
|
||||
|
||||
@Entity
|
||||
public class GitRepository extends BaseEntity {
|
||||
private String url;
|
||||
private String repositoryId;
|
||||
|
||||
public GitRepository() {
|
||||
}
|
||||
|
||||
public GitRepository(String repositoryUrl) {
|
||||
url = repositoryUrl;
|
||||
public GitRepository(String repositoryUrl, String repositoryId) {
|
||||
this.url = repositoryUrl;
|
||||
this.repositoryId = repositoryId;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
@ -23,14 +26,14 @@ public class GitRepository extends BaseEntity {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
int lastDelimiterIndex = url.lastIndexOf("/");
|
||||
return (lastDelimiterIndex > 0 && lastDelimiterIndex < url.length())
|
||||
? removeDotGit(url.substring(lastDelimiterIndex + 1))
|
||||
: removeDotGit(url);
|
||||
public String getRepositoryId() {
|
||||
return repositoryId;
|
||||
}
|
||||
|
||||
private String removeDotGit(String prevName) {
|
||||
return prevName.substring(0, prevName.lastIndexOf(".git"));
|
||||
public String getName() {
|
||||
int lastDelimiterIndex = url.lastIndexOf("/");
|
||||
return StringUtils.removeDotGit((lastDelimiterIndex > 0 && lastDelimiterIndex < url.length())
|
||||
? url.substring(lastDelimiterIndex + 1)
|
||||
: url);
|
||||
}
|
||||
}
|
||||
|
@ -12,4 +12,6 @@ public interface GitApi {
|
||||
String getFormattedUrl(Branch branch, String template);
|
||||
|
||||
Integer getAuthorsCompletedIssues(Branch branch);
|
||||
|
||||
String getRepositoryId(String repositoryUrl);
|
||||
}
|
||||
|
@ -50,4 +50,9 @@ public class GitAtheneApi implements GitApi {
|
||||
.get(getFormattedUrl(branch, COMPLETED_ISSUES_URL))
|
||||
.length();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getRepositoryId(String repositoryUrl) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
@ -15,6 +15,7 @@ import org.eclipse.jgit.treewalk.TreeWalk;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.stereotype.Service;
|
||||
@ -29,6 +30,7 @@ import ru.ulstu.extractor.heuristic.model.BusinessLogicUnit;
|
||||
import ru.ulstu.extractor.heuristic.model.EntityUnit;
|
||||
import ru.ulstu.extractor.heuristic.model.ResourceUnit;
|
||||
import ru.ulstu.extractor.heuristic.service.StructuralUnitService;
|
||||
import ru.ulstu.extractor.util.StringUtils;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
@ -55,7 +57,11 @@ import static org.apache.logging.log4j.util.Strings.isBlank;
|
||||
|
||||
@Service
|
||||
public class GitRepositoryService {
|
||||
private final static Logger LOG = LoggerFactory.getLogger(GitRepositoryService.class);
|
||||
private final Map<String, Class<? extends GitApi>> gitApiServiceHolder = Map.of(
|
||||
"git.athene.tech", GitAtheneApi.class,
|
||||
"github.com", GithubApi.class,
|
||||
"gitlab.com", GitlabApi.class);
|
||||
private static final Logger LOG = LoggerFactory.getLogger(GitRepositoryService.class);
|
||||
private static final String BRANCH_PREFIX = "refs/remotes/origin/";
|
||||
@Value("${extractor.custom-projects-dir}")
|
||||
private String customProjectsDir;
|
||||
@ -63,11 +69,14 @@ public class GitRepositoryService {
|
||||
private final ExecutorService executorServiceCommits = Executors.newFixedThreadPool(8);
|
||||
private final StructuralUnitService structuralUnitService;
|
||||
private final GitRepositoryRepository gitRepositoryRepository;
|
||||
private final ApplicationContext applicationContext;
|
||||
|
||||
public GitRepositoryService(StructuralUnitService structuralUnitService,
|
||||
GitRepositoryRepository gitRepositoryRepository) {
|
||||
GitRepositoryRepository gitRepositoryRepository,
|
||||
ApplicationContext applicationContext) {
|
||||
this.structuralUnitService = structuralUnitService;
|
||||
this.gitRepositoryRepository = gitRepositoryRepository;
|
||||
this.applicationContext = applicationContext;
|
||||
}
|
||||
|
||||
public List<Branch> getRemoteBranches(String url) throws GitAPIException, IOException {
|
||||
@ -430,8 +439,18 @@ public class GitRepositoryService {
|
||||
|
||||
public GitRepository findByUrlOrCreate(String gitRepositoryUrl) {
|
||||
GitRepository gitRepository = gitRepositoryRepository.findByUrl(gitRepositoryUrl);
|
||||
|
||||
return gitRepository == null
|
||||
? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl))
|
||||
? gitRepositoryRepository.save(new GitRepository(gitRepositoryUrl,
|
||||
applicationContext.getBean(getGitApiServiceClass(gitRepositoryUrl)).getRepositoryId(gitRepositoryUrl)))
|
||||
: gitRepository;
|
||||
}
|
||||
|
||||
public Class<? extends GitApi> getGitApiServiceClass(Branch branch) {
|
||||
return getGitApiServiceClass(branch.getGitRepository().getUrl());
|
||||
}
|
||||
|
||||
public Class<? extends GitApi> getGitApiServiceClass(String url) {
|
||||
return gitApiServiceHolder.get(StringUtils.getServerDomain(url));
|
||||
}
|
||||
}
|
||||
|
@ -50,4 +50,9 @@ public class GithubApi implements GitApi {
|
||||
.get(getFormattedUrl(branch, AUTHOR_COMPLETED_ISSUES_URL))
|
||||
.length();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getRepositoryId(String repositoryUrl) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
@ -5,6 +5,9 @@ import org.springframework.stereotype.Service;
|
||||
import ru.ulstu.extractor.branch.model.Branch;
|
||||
import ru.ulstu.extractor.http.HttpService;
|
||||
|
||||
import static ru.ulstu.extractor.util.StringUtils.getUrlParts;
|
||||
import static ru.ulstu.extractor.util.StringUtils.removeDotGit;
|
||||
|
||||
@Service
|
||||
public class GitlabApi implements GitApi {
|
||||
private final HttpService httpService;
|
||||
@ -42,14 +45,14 @@ public class GitlabApi implements GitApi {
|
||||
|
||||
@Override
|
||||
public String getFormattedUrl(Branch branch, String template) {
|
||||
String[] urlParts = branch.getGitRepository().getUrl().split("/");
|
||||
return String.format(template, urlParts[0] + "/" + urlParts[1] + "/" + urlParts[2], getProjectId(branch));
|
||||
String[] urlParts = getUrlParts(branch.getGitRepository().getUrl());
|
||||
return String.format(template, urlParts[0] + "/" + urlParts[1] + "/" + urlParts[2], branch.getGitRepository().getRepositoryId());
|
||||
}
|
||||
|
||||
public String getFormattedUrlForProjectId(Branch branch, String template) {
|
||||
String[] urlParts = branch.getGitRepository().getUrl().split("/");
|
||||
public String getFormattedUrlForProjectId(String repositoryUrl, String template) {
|
||||
String[] urlParts = getUrlParts(repositoryUrl);
|
||||
// получаем корректное название репозитория, gitlab всегда добавляет .git в конец
|
||||
return String.format(template, urlParts[3], branch.getGitRepository().getName());
|
||||
return String.format(template, urlParts[3], removeDotGit(urlParts[4]));
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -59,13 +62,13 @@ public class GitlabApi implements GitApi {
|
||||
.length();
|
||||
}
|
||||
|
||||
private String getProjectId(Branch branch) {
|
||||
JSONArray projects = httpService.get(getFormattedUrlForProjectId(branch, PROJECT_ID_URL));
|
||||
public String getRepositoryId(String repositoryUrl) {
|
||||
JSONArray projects = httpService.get(getFormattedUrlForProjectId(repositoryUrl, PROJECT_ID_URL));
|
||||
for (int i = 0; i < projects.length(); i++) {
|
||||
if (projects.getJSONObject(i).get("name").equals(branch.getGitRepository().getName())) {
|
||||
if (projects.getJSONObject(i).get("name").equals(removeDotGit(getUrlParts(repositoryUrl)[4]))) {
|
||||
return String.valueOf(projects.getJSONObject(i).getInt("id"));
|
||||
}
|
||||
}
|
||||
throw new RuntimeException("Id проекта не найден: " + branch.getGitRepository().getName());
|
||||
throw new RuntimeException("Id проекта не найден: " + repositoryUrl);
|
||||
}
|
||||
}
|
||||
|
@ -17,7 +17,6 @@ import ru.ulstu.extractor.branch.model.IndexingStatus;
|
||||
import ru.ulstu.extractor.branch.service.BranchService;
|
||||
import ru.ulstu.extractor.commit.model.Commit;
|
||||
import ru.ulstu.extractor.gitrepository.model.GitRepository;
|
||||
import ru.ulstu.extractor.gitrepository.repository.GitRepositoryRepository;
|
||||
import ru.ulstu.extractor.ts.creator.db.DBTimeSeriesCreator;
|
||||
|
||||
import java.io.IOException;
|
||||
@ -31,17 +30,14 @@ public class IndexService {
|
||||
private final static Logger LOG = LoggerFactory.getLogger(IndexService.class);
|
||||
private final static int COMMITS_PAGE_SIZE = 10;
|
||||
private final GitRepositoryService gitRepositoryService;
|
||||
private final GitRepositoryRepository gitRepositoryRepository;
|
||||
private final BranchService branchService;
|
||||
private final List<DBTimeSeriesCreator> timeSeriesCreators;
|
||||
private final static ExecutorService EXECUTOR = Executors.newFixedThreadPool(10);
|
||||
|
||||
public IndexService(GitRepositoryService gitRepositoryService,
|
||||
GitRepositoryRepository gitRepositoryRepository,
|
||||
BranchService branchService,
|
||||
List<DBTimeSeriesCreator> timeSeriesCreators) {
|
||||
this.gitRepositoryService = gitRepositoryService;
|
||||
this.gitRepositoryRepository = gitRepositoryRepository;
|
||||
this.branchService = branchService;
|
||||
this.timeSeriesCreators = timeSeriesCreators;
|
||||
}
|
||||
|
@ -3,6 +3,7 @@ package ru.ulstu.extractor.ts.creator.scheduled;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
import ru.ulstu.extractor.branch.model.Branch;
|
||||
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
|
||||
import ru.ulstu.extractor.ts.model.TimeSeriesType;
|
||||
import ru.ulstu.extractor.ts.model.TimeSeriesValue;
|
||||
import ru.ulstu.extractor.ts.service.TimeSeriesService;
|
||||
@ -10,11 +11,14 @@ import ru.ulstu.extractor.ts.service.TimeSeriesService;
|
||||
@Component
|
||||
public class BranchTS extends ScheduledTimeSeriesCreator {
|
||||
private final TimeSeriesService timeSeriesService;
|
||||
private final GitRepositoryService gitRepositoryService;
|
||||
private final ApplicationContext applicationContext;
|
||||
|
||||
public BranchTS(TimeSeriesService timeSeriesService,
|
||||
GitRepositoryService gitRepositoryService,
|
||||
ApplicationContext applicationContext) {
|
||||
this.timeSeriesService = timeSeriesService;
|
||||
this.gitRepositoryService = gitRepositoryService;
|
||||
this.applicationContext = applicationContext;
|
||||
}
|
||||
|
||||
@ -28,9 +32,14 @@ public class BranchTS extends ScheduledTimeSeriesCreator {
|
||||
return timeSeriesService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GitRepositoryService getGitRepositoryService() {
|
||||
return gitRepositoryService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TimeSeriesValue getNewTimeSeriesValue(Branch branch) {
|
||||
return new TimeSeriesValue(applicationContext.getBean(getGitApiServiceClass(branch))
|
||||
return new TimeSeriesValue(applicationContext.getBean(getGitRepositoryService().getGitApiServiceClass(branch))
|
||||
.getBranchesCount(branch));
|
||||
}
|
||||
}
|
||||
|
@ -3,6 +3,7 @@ package ru.ulstu.extractor.ts.creator.scheduled;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
import ru.ulstu.extractor.branch.model.Branch;
|
||||
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
|
||||
import ru.ulstu.extractor.ts.model.TimeSeriesType;
|
||||
import ru.ulstu.extractor.ts.model.TimeSeriesValue;
|
||||
import ru.ulstu.extractor.ts.service.TimeSeriesService;
|
||||
@ -11,16 +12,19 @@ import ru.ulstu.extractor.ts.service.TimeSeriesService;
|
||||
public class IssuesTS extends ScheduledTimeSeriesCreator {
|
||||
private final TimeSeriesService timeSeriesService;
|
||||
private final ApplicationContext applicationContext;
|
||||
private final GitRepositoryService gitRepositoryService;
|
||||
|
||||
public IssuesTS(TimeSeriesService timeSeriesService,
|
||||
ApplicationContext applicationContext) {
|
||||
ApplicationContext applicationContext,
|
||||
GitRepositoryService gitRepositoryService) {
|
||||
this.timeSeriesService = timeSeriesService;
|
||||
this.applicationContext = applicationContext;
|
||||
this.gitRepositoryService = gitRepositoryService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TimeSeriesValue getNewTimeSeriesValue(Branch branch) {
|
||||
return new TimeSeriesValue(applicationContext.getBean(getGitApiServiceClass(branch))
|
||||
return new TimeSeriesValue(applicationContext.getBean(getGitRepositoryService().getGitApiServiceClass(branch))
|
||||
.getOpenIssuesCount(branch));
|
||||
}
|
||||
|
||||
@ -33,4 +37,9 @@ public class IssuesTS extends ScheduledTimeSeriesCreator {
|
||||
public TimeSeriesType getTimeSeriesType() {
|
||||
return TimeSeriesType.ISSUES;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GitRepositoryService getGitRepositoryService() {
|
||||
return gitRepositoryService;
|
||||
}
|
||||
}
|
||||
|
@ -2,24 +2,14 @@ package ru.ulstu.extractor.ts.creator.scheduled;
|
||||
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import ru.ulstu.extractor.branch.model.Branch;
|
||||
import ru.ulstu.extractor.gitrepository.service.GitApi;
|
||||
import ru.ulstu.extractor.gitrepository.service.GitAtheneApi;
|
||||
import ru.ulstu.extractor.gitrepository.service.GithubApi;
|
||||
import ru.ulstu.extractor.gitrepository.service.GitlabApi;
|
||||
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
|
||||
import ru.ulstu.extractor.ts.creator.AbstractTimeSeriesCreator;
|
||||
import ru.ulstu.extractor.ts.model.TimeSeries;
|
||||
import ru.ulstu.extractor.ts.model.TimeSeriesValue;
|
||||
import ru.ulstu.extractor.util.StringUtils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
|
||||
public abstract class ScheduledTimeSeriesCreator extends AbstractTimeSeriesCreator {
|
||||
private final Map<String, Class<? extends GitApi>> gitApiServiceHolder = Map.of(
|
||||
"git.athene.tech", GitAtheneApi.class,
|
||||
"github.com", GithubApi.class,
|
||||
"gitlab.com", GitlabApi.class);
|
||||
|
||||
@Transactional
|
||||
public void addTimeSeriesValue(Branch branch) {
|
||||
TimeSeries timeSeries = getTimeSeriesService().findByBranchAndTypeOrCreate(branch, getTimeSeriesType());
|
||||
@ -29,7 +19,5 @@ public abstract class ScheduledTimeSeriesCreator extends AbstractTimeSeriesCreat
|
||||
|
||||
public abstract TimeSeriesValue getNewTimeSeriesValue(Branch branch);
|
||||
|
||||
public Class<? extends GitApi> getGitApiServiceClass(Branch branch) {
|
||||
return gitApiServiceHolder.get(StringUtils.getServerDomain(branch.getGitRepository().getUrl()));
|
||||
}
|
||||
public abstract GitRepositoryService getGitRepositoryService();
|
||||
}
|
||||
|
@ -3,6 +3,7 @@ package ru.ulstu.extractor.ts.creator.scheduled;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
import ru.ulstu.extractor.branch.model.Branch;
|
||||
import ru.ulstu.extractor.gitrepository.service.GitRepositoryService;
|
||||
import ru.ulstu.extractor.ts.model.TimeSeriesType;
|
||||
import ru.ulstu.extractor.ts.model.TimeSeriesValue;
|
||||
import ru.ulstu.extractor.ts.service.TimeSeriesService;
|
||||
@ -11,11 +12,14 @@ import ru.ulstu.extractor.ts.service.TimeSeriesService;
|
||||
public class StarTS extends ScheduledTimeSeriesCreator {
|
||||
private final TimeSeriesService timeSeriesService;
|
||||
private final ApplicationContext applicationContext;
|
||||
private final GitRepositoryService gitRepositoryService;
|
||||
|
||||
public StarTS(TimeSeriesService timeSeriesService,
|
||||
ApplicationContext applicationContext) {
|
||||
ApplicationContext applicationContext,
|
||||
GitRepositoryService gitRepositoryService) {
|
||||
this.timeSeriesService = timeSeriesService;
|
||||
this.applicationContext = applicationContext;
|
||||
this.gitRepositoryService = gitRepositoryService;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -28,9 +32,14 @@ public class StarTS extends ScheduledTimeSeriesCreator {
|
||||
return TimeSeriesType.STARS;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GitRepositoryService getGitRepositoryService() {
|
||||
return gitRepositoryService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public TimeSeriesValue getNewTimeSeriesValue(Branch branch) {
|
||||
return new TimeSeriesValue(applicationContext.getBean(getGitApiServiceClass(branch))
|
||||
return new TimeSeriesValue(applicationContext.getBean(getGitRepositoryService().getGitApiServiceClass(branch))
|
||||
.getStarsCount(branch));
|
||||
}
|
||||
}
|
||||
|
@ -22,7 +22,7 @@ public class ScheduledTimeSeriesService {
|
||||
this.branchService = branchService;
|
||||
}
|
||||
|
||||
@Scheduled(cron = "0 0 * * * *", zone = "Europe/Samara")
|
||||
@Scheduled(cron = "0 0 8 * * *", zone = "Europe/Samara")
|
||||
public void addTimeSeriesPoints() {
|
||||
log.debug("Старт добавления новых точек временного ряда");
|
||||
List<Branch> branches = branchService.findAll();
|
||||
|
@ -31,4 +31,14 @@ public class StringUtils {
|
||||
String[] urlParts = url.split("/");
|
||||
return urlParts[2];
|
||||
}
|
||||
|
||||
public static String[] getUrlParts(String gitRepositoryUrl) {
|
||||
return gitRepositoryUrl.split("/");
|
||||
}
|
||||
|
||||
public static String removeDotGit(String repositoryName) {
|
||||
return repositoryName.lastIndexOf(".git") > 0
|
||||
? repositoryName.substring(0, repositoryName.lastIndexOf(".git"))
|
||||
: repositoryName;
|
||||
}
|
||||
}
|
||||
|
@ -107,4 +107,24 @@
|
||||
<column name="indexing_status" value="FINISHED"/>
|
||||
</update>
|
||||
</changeSet>
|
||||
<changeSet author="orion" id="20230220-120000-1">
|
||||
<addColumn tableName="git_repository">
|
||||
<column name="repository_id" type="varchar(100)"/>
|
||||
</addColumn>
|
||||
</changeSet>
|
||||
<changeSet author="orion" id="20230220-120000-2">
|
||||
<dropForeignKeyConstraint baseTableName="time_series"
|
||||
constraintName="fk_time_series_branch"/>
|
||||
<addForeignKeyConstraint baseTableName="time_series" baseColumnNames="branch_id"
|
||||
constraintName="fk_time_series_branch" referencedTableName="branch"
|
||||
referencedColumnNames="id" onDelete="CASCADE"/>
|
||||
</changeSet>
|
||||
<changeSet author="orion" id="20230220-120000-3">
|
||||
<dropForeignKeyConstraint baseTableName="time_series_value"
|
||||
constraintName="fk_time_series"/>
|
||||
<addForeignKeyConstraint baseTableName="time_series_value" baseColumnNames="time_series_id"
|
||||
constraintName="fk_time_series"
|
||||
referencedTableName="time_series"
|
||||
referencedColumnNames="id" onDelete="CASCADE"/>
|
||||
</changeSet>
|
||||
</databaseChangeLog>
|
Loading…
Reference in New Issue
Block a user