romanov73/git-extractor
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

#58 -- save time series on indexing

Browse Source
This commit is contained in:
Anton Romanov 2022-06-21 00:51:52 +04:00
parent 4f8d25e56e
commit 156cdbc672
11 changed files with 75 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/ru/ulstu/extractor/controller/BranchController.java → src/main/java/ru/ulstu/extractor/branch/BranchController.java
View File

@ -3,13 +3,12 @@
* You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/
package ru.ulstu.extractor.controller;
package ru.ulstu.extractor.branch;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import ru.ulstu.extractor.branch.BranchRepository;
import ru.ulstu.extractor.gitrepository.GitRepositoryRepository;
import springfox.documentation.annotations.ApiIgnore;

10
src/main/java/ru/ulstu/extractor/branch/BranchService.java
View File

@ -7,8 +7,6 @@ package ru.ulstu.extractor.branch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.ulstu.extractor.model.BaseEntity;
@ -56,14 +54,6 @@ public class BranchService {
return branchRepository.findByRepositoryAndName(repository, branchName);
}
public int getCommitsCount(Integer repositoryId, String branchName) {
return branchRepository.getCommitsCount(repositoryId, branchName);
}
public Page<Branch> findByRepository(Repository repository, Pageable pageable) {
return branchRepository.findByRepository(repository, pageable);
}
public List<Branch> findAll() {
return branchRepository.findAll();
}

69
src/main/java/ru/ulstu/extractor/loader/GitScheduler.java
View File

@ -1,69 +0,0 @@
package ru.ulstu.extractor.loader;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.domain.Page;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import ru.ulstu.extractor.branch.BranchService;
import ru.ulstu.extractor.gitrepository.GitRepositoryService;
import ru.ulstu.extractor.model.Branch;
import ru.ulstu.extractor.model.OffsetablePageRequest;
import ru.ulstu.extractor.model.Repository;
import ru.ulstu.extractor.service.IndexService;
import ru.ulstu.extractor.ts.AbstractTSExtractor;
import java.io.IOException;
import java.util.List;
@Service
public class GitScheduler {
private final static int DEFAULT_PAGE_SIZE = 100;
private final Logger log = LoggerFactory.getLogger(GitScheduler.class);
private final GitRepositoryService gitRepositoryService;
private final BranchService branchService;
private final IndexService indexService;
private final List<AbstractTSExtractor> tsExtractors;
public GitScheduler(GitRepositoryService gitRepositoryService,
BranchService branchService,
IndexService indexService,
List<AbstractTSExtractor> tsExtractors) {
this.gitRepositoryService = gitRepositoryService;
this.branchService = branchService;
this.indexService = indexService;
this.tsExtractors = tsExtractors;
}
@Scheduled(cron = "* */5 * * * *")
public void getCommitsCount() {
log.debug("Load commits count started");
int repositoryPageNumber = 0;
Page<Repository> repositoryPage;
do {
int branchPageNumber = 0;
repositoryPage = gitRepositoryService.findAll(
new OffsetablePageRequest(repositoryPageNumber, DEFAULT_PAGE_SIZE));
Page<Branch> branchPage;
for (Repository repository : repositoryPage.getContent()) {
do {
branchPage = branchService.findByRepository(repository,
new OffsetablePageRequest(branchPageNumber, DEFAULT_PAGE_SIZE));
for (Branch branch : branchPage.getContent()) {
try {
indexService.index(repository.getUrl(), branch.getName());
tsExtractors.forEach(tsExtractor -> tsExtractor.addPoint(repository.getId(), branch.getName()));
} catch (GitAPIException | IOException ex) {
log.warn(ex.getMessage());
ex.printStackTrace();
}
}
branchPageNumber++;
} while (!branchPage.isEmpty());
}
repositoryPageNumber++;
} while (!repositoryPage.isEmpty());
log.debug("Load commits count finished");
}
}

3
src/main/java/ru/ulstu/extractor/repository/CommitRepository.java
View File

@ -39,4 +39,7 @@ public interface CommitRepository extends JpaRepository<Commit, Integer> {
List<CommitTimeStatistic> getCommitTimeEntityStatistic(@Param("branchId") Integer branchId, @Param("author") String author);
void deleteByBranchIsNull();
@Query("SELECT b.commits FROM Branch b WHERE b.name = ?2 and b.repository.id = ?1")
List<Commit> findByRepositoryIdAndBranchName(Integer repositoryId, String name);
}

5
src/main/java/ru/ulstu/extractor/service/CommitService.java
View File

@ -49,6 +49,11 @@ public class CommitService {
updateQuery = String.format(updateQuery, String.join(",", commitIds));
jdbcTemplate.update(updateQuery, branchId);
}
public List<Commit> findByRepositoryIdAndName(Integer repositoryId, String branchName) {
return commitRepository.findByRepositoryIdAndBranchName(repositoryId, branchName);
}
}

10
src/main/java/ru/ulstu/extractor/service/IndexService.java
View File

@ -10,13 +10,13 @@ import org.eclipse.jgit.api.errors.GitAPIException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.ulstu.extractor.branch.BranchService;
import ru.ulstu.extractor.gitrepository.GitRepositoryRepository;
import ru.ulstu.extractor.gitrepository.GitRepositoryService;
import ru.ulstu.extractor.model.Branch;
import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.model.Repository;
import ru.ulstu.extractor.ts.AbstractTimeSeriesCreator;
import java.io.IOException;
import java.util.Collections;
@ -29,16 +29,18 @@ public class IndexService {
private final GitRepositoryService gitRepositoryService;
private final GitRepositoryRepository gitRepositoryRepository;
private final BranchService branchService;
private final List<AbstractTimeSeriesCreator> timeSeriesCreators;
public IndexService(GitRepositoryService gitRepositoryService,
GitRepositoryRepository gitRepositoryRepository,
BranchService branchService) {
BranchService branchService,
List<AbstractTimeSeriesCreator> timeSeriesCreators) {
this.gitRepositoryService = gitRepositoryService;
this.gitRepositoryRepository = gitRepositoryRepository;
this.branchService = branchService;
this.timeSeriesCreators = timeSeriesCreators;
}
@Transactional
public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException {
Repository repository = gitRepositoryRepository.findByUrl(repositoryUrl);
if (repository == null) {
@ -60,6 +62,8 @@ public class IndexService {
commitsTo += COMMITS_PAGE_SIZE;
commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false);
}
Integer repositoryId = repository.getId();
timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchName));
LOG.debug("Complete indexing {} branch", branchName);
}
}

21
src/main/java/ru/ulstu/extractor/service/TimeSeriesService.java
View File

@ -13,6 +13,7 @@ import ru.ulstu.extractor.model.TimeSeries;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.repository.TimeSeriesRepository;
import ru.ulstu.extractor.repository.TimeSeriesValueRepository;
import ru.ulstu.extractor.ts.TimeSeriesDateMapper;
import java.util.Date;
import java.util.List;
@ -24,21 +25,23 @@ public class TimeSeriesService {
private final static Logger LOG = LoggerFactory.getLogger(TimeSeriesService.class);
private final TimeSeriesRepository timeSeriesRepository;
private final TimeSeriesValueRepository timeSeriesValueRepository;
private final TimeSeriesDateMapper.TimeSeriesInterval timeSeriesInterval = TimeSeriesDateMapper.TimeSeriesInterval.HOUR;
public TimeSeriesService(TimeSeriesRepository timeSeriesRepository, TimeSeriesValueRepository timeSeriesValueRepository) {
this.timeSeriesRepository = timeSeriesRepository;
this.timeSeriesValueRepository = timeSeriesValueRepository;
}
public TimeSeries save(TimeSeries timeSeries, List<TimeSeriesValue> timeSeriesValues) {
LOG.debug("Start save {} time series with {} time series values ", timeSeries.getName(), timeSeriesValues.size());
public TimeSeries save(String timeSeriesName, List<TimeSeriesValue> timeSeriesValues) {
LOG.debug("Start save {} time series with {} time series values ", timeSeriesName, timeSeriesValues.size());
final TimeSeries timeSeries = findOrCreate(timeSeriesName);
List<Integer> timeSeriesValuesToRemoveIds = timeSeries.getValues().stream().map(BaseEntity::getId).collect(Collectors.toList());
timeSeriesValues.forEach(v -> v.setTimeSeries(timeSeries));
timeSeries.setValues(timeSeriesValues);
LOG.debug("Save time series {} ", timeSeries.getName());
timeSeries = timeSeriesRepository.save(timeSeries);
TimeSeries savedTimeSeries = timeSeriesRepository.save(timeSeries);
LOG.debug("Clear {} ime series values ", timeSeriesValuesToRemoveIds.size());
return timeSeries;
return savedTimeSeries;
}
public TimeSeries findOrCreate(String timeSeriesName) {
@ -64,12 +67,12 @@ public class TimeSeriesService {
TimeSeries timeSeries = findOrCreate(timeSeriesName);
timeSeriesValueRepository.save(new TimeSeriesValue(timeSeries, date, value));
}
//
// public TimeSeriesValue findByTimeSeriesAndName(TimeSeries timeSeries, String name) {
// return timeSeriesValueRepository.findByTimeSeriesAndName(timeSeries, name);
// }
public List<TimeSeriesValue> findAll() {
return timeSeriesValueRepository.findAll();
}
public TimeSeriesDateMapper.TimeSeriesInterval getTimeSeriesInterval() {
return timeSeriesInterval;
}
}

21
src/main/java/ru/ulstu/extractor/ts/AbstractTSExtractor.java
View File

@ -1,21 +0,0 @@
package ru.ulstu.extractor.ts;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.Date;
public abstract class AbstractTSExtractor {
public abstract String getTSName();
public abstract int getNewTSValue(Integer repositoryId, String branchName);
public abstract TimeSeriesService getTimeSeriesService();
public void addPoint(Integer repositoryId, String branchName) {
getTimeSeriesService().addTimeSeriesValue(
String.format("%s %s %s", getTSName(), repositoryId, branchName),
new Date(),
getNewTSValue(repositoryId, branchName));
}
}

22
src/main/java/ru/ulstu/extractor/ts/AbstractTimeSeriesCreator.java Normal file
View File

@ -0,0 +1,22 @@
package ru.ulstu.extractor.ts;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.List;
public abstract class AbstractTimeSeriesCreator {
public abstract String getTimeSeriesName();
public abstract List<TimeSeriesValue> getTimeSeriesValues(Integer repositoryId, String branchName);
public abstract TimeSeriesService getTimeSeriesService();
public void addTimeSeries(Integer repositoryId, String branchName) {
getTimeSeriesService().save(
String.format("%s %s %s", getTimeSeriesName(), repositoryId, branchName),
TimeSeriesDateMapper.mapTimeSeriesToInterval(getTimeSeriesService().getTimeSeriesInterval(),
getTimeSeriesValues(repositoryId, branchName)));
}
}

23
src/main/java/ru/ulstu/extractor/ts/CommitsTS.java
View File

@ -1,28 +1,35 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.branch.BranchService;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.CommitService;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.List;
import java.util.stream.Collectors;
@Component
public class CommitsTS extends AbstractTSExtractor {
public class CommitsTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
private final BranchService branchService;
private final CommitService commitService;
public CommitsTS(TimeSeriesService timeSeriesService,
BranchService branchService) {
CommitService commitService) {
this.timeSeriesService = timeSeriesService;
this.branchService = branchService;
this.commitService = commitService;
}
@Override
public String getTSName() {
public String getTimeSeriesName() {
return "Количество коммитов во времени";
}
@Override
public int getNewTSValue(Integer repositoryId, String branchName) {
return branchService.getCommitsCount(repositoryId, branchName);
public List<TimeSeriesValue> getTimeSeriesValues(Integer repositoryId, String branchName) {
return commitService.findByRepositoryIdAndName(repositoryId, branchName)
.stream()
.map(c -> new TimeSeriesValue(c.getDate(), 1))
.collect(Collectors.toList());
}
@Override

15
src/main/java/ru/ulstu/extractor/ts/TimeSeriesDateMapper.java
View File

@ -17,8 +17,8 @@ import java.util.stream.Collectors;
*/
public class TimeSeriesDateMapper {
public TimeSeries mapTimeSeriesToInterval(TimeSeriesInterval timeSeriesInterval, TimeSeries timeSeries) {
List<TimeSeriesValue> trimmedTimeSeriesValues = timeSeries.getValues()
public static List<TimeSeriesValue> mapTimeSeriesToInterval(TimeSeriesInterval timeSeriesInterval, List<TimeSeriesValue> timeSeriesValues) {
List<TimeSeriesValue> trimmedTimeSeriesValues = timeSeriesValues
.stream()
.map(timeSeriesValue -> new TimeSeriesValue(trimTo(timeSeriesInterval, timeSeriesValue.getDate()),
timeSeriesValue.getValue()))
@ -28,13 +28,18 @@ public class TimeSeriesDateMapper {
.collect(Collectors.groupingBy(TimeSeriesValue::getDate,
Collectors.summingInt(TimeSeriesValue::getValue)));
return new TimeSeries(timeSeries.getName(), groupedTimeSeriesValues.entrySet()
return groupedTimeSeriesValues.entrySet()
.stream()
.map(e -> new TimeSeriesValue(e.getKey(), e.getValue()))
.collect(Collectors.toList()));
.collect(Collectors.toList());
}
private Date trimTo(TimeSeriesInterval timeSeriesInterval, Date date) {
public static TimeSeries mapTimeSeriesToInterval(TimeSeriesInterval timeSeriesInterval, TimeSeries timeSeries) {
timeSeries.setValues(mapTimeSeriesToInterval(timeSeriesInterval, timeSeries.getValues()));
return timeSeries;
}
private static Date trimTo(TimeSeriesInterval timeSeriesInterval, Date date) {
return DateUtils.truncate(date, timeSeriesInterval.calendarField);
}