From 156cdbc67230bbbfc16f2e9c5a78224d750056d2 Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Tue, 21 Jun 2022 00:51:52 +0400 Subject: [PATCH] #58 -- save time series on indexing --- .../BranchController.java | 3 +- .../ulstu/extractor/branch/BranchService.java | 10 --- .../ulstu/extractor/loader/GitScheduler.java | 69 ------------------- .../repository/CommitRepository.java | 3 + .../extractor/service/CommitService.java | 5 ++ .../ulstu/extractor/service/IndexService.java | 10 ++- .../extractor/service/TimeSeriesService.java | 21 +++--- .../extractor/ts/AbstractTSExtractor.java | 21 ------ .../ts/AbstractTimeSeriesCreator.java | 22 ++++++ .../java/ru/ulstu/extractor/ts/CommitsTS.java | 23 ++++--- .../extractor/ts/TimeSeriesDateMapper.java | 15 ++-- 11 files changed, 75 insertions(+), 127 deletions(-) rename src/main/java/ru/ulstu/extractor/{controller => branch}/BranchController.java (95%) delete mode 100644 src/main/java/ru/ulstu/extractor/loader/GitScheduler.java delete mode 100644 src/main/java/ru/ulstu/extractor/ts/AbstractTSExtractor.java create mode 100644 src/main/java/ru/ulstu/extractor/ts/AbstractTimeSeriesCreator.java diff --git a/src/main/java/ru/ulstu/extractor/controller/BranchController.java b/src/main/java/ru/ulstu/extractor/branch/BranchController.java similarity index 95% rename from src/main/java/ru/ulstu/extractor/controller/BranchController.java rename to src/main/java/ru/ulstu/extractor/branch/BranchController.java index 64ccecc..47f823e 100644 --- a/src/main/java/ru/ulstu/extractor/controller/BranchController.java +++ b/src/main/java/ru/ulstu/extractor/branch/BranchController.java @@ -3,13 +3,12 @@ * You may use, distribute and modify this code, please write to: romanov73@gmail.com. */ -package ru.ulstu.extractor.controller; +package ru.ulstu.extractor.branch; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestParam; -import ru.ulstu.extractor.branch.BranchRepository; import ru.ulstu.extractor.gitrepository.GitRepositoryRepository; import springfox.documentation.annotations.ApiIgnore; diff --git a/src/main/java/ru/ulstu/extractor/branch/BranchService.java b/src/main/java/ru/ulstu/extractor/branch/BranchService.java index 8cae8e6..e047bcb 100644 --- a/src/main/java/ru/ulstu/extractor/branch/BranchService.java +++ b/src/main/java/ru/ulstu/extractor/branch/BranchService.java @@ -7,8 +7,6 @@ package ru.ulstu.extractor.branch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import ru.ulstu.extractor.model.BaseEntity; @@ -56,14 +54,6 @@ public class BranchService { return branchRepository.findByRepositoryAndName(repository, branchName); } - public int getCommitsCount(Integer repositoryId, String branchName) { - return branchRepository.getCommitsCount(repositoryId, branchName); - } - - public Page findByRepository(Repository repository, Pageable pageable) { - return branchRepository.findByRepository(repository, pageable); - } - public List findAll() { return branchRepository.findAll(); } diff --git a/src/main/java/ru/ulstu/extractor/loader/GitScheduler.java b/src/main/java/ru/ulstu/extractor/loader/GitScheduler.java deleted file mode 100644 index 4a16fdc..0000000 --- a/src/main/java/ru/ulstu/extractor/loader/GitScheduler.java +++ /dev/null @@ -1,69 +0,0 @@ -package ru.ulstu.extractor.loader; - -import org.eclipse.jgit.api.errors.GitAPIException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.domain.Page; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import ru.ulstu.extractor.branch.BranchService; -import ru.ulstu.extractor.gitrepository.GitRepositoryService; -import ru.ulstu.extractor.model.Branch; -import ru.ulstu.extractor.model.OffsetablePageRequest; -import ru.ulstu.extractor.model.Repository; -import ru.ulstu.extractor.service.IndexService; -import ru.ulstu.extractor.ts.AbstractTSExtractor; - -import java.io.IOException; -import java.util.List; - -@Service -public class GitScheduler { - private final static int DEFAULT_PAGE_SIZE = 100; - private final Logger log = LoggerFactory.getLogger(GitScheduler.class); - private final GitRepositoryService gitRepositoryService; - private final BranchService branchService; - private final IndexService indexService; - private final List tsExtractors; - - public GitScheduler(GitRepositoryService gitRepositoryService, - BranchService branchService, - IndexService indexService, - List tsExtractors) { - this.gitRepositoryService = gitRepositoryService; - this.branchService = branchService; - this.indexService = indexService; - this.tsExtractors = tsExtractors; - } - - @Scheduled(cron = "* */5 * * * *") - public void getCommitsCount() { - log.debug("Load commits count started"); - int repositoryPageNumber = 0; - Page repositoryPage; - do { - int branchPageNumber = 0; - repositoryPage = gitRepositoryService.findAll( - new OffsetablePageRequest(repositoryPageNumber, DEFAULT_PAGE_SIZE)); - Page branchPage; - for (Repository repository : repositoryPage.getContent()) { - do { - branchPage = branchService.findByRepository(repository, - new OffsetablePageRequest(branchPageNumber, DEFAULT_PAGE_SIZE)); - for (Branch branch : branchPage.getContent()) { - try { - indexService.index(repository.getUrl(), branch.getName()); - tsExtractors.forEach(tsExtractor -> tsExtractor.addPoint(repository.getId(), branch.getName())); - } catch (GitAPIException | IOException ex) { - log.warn(ex.getMessage()); - ex.printStackTrace(); - } - } - branchPageNumber++; - } while (!branchPage.isEmpty()); - } - repositoryPageNumber++; - } while (!repositoryPage.isEmpty()); - log.debug("Load commits count finished"); - } -} diff --git a/src/main/java/ru/ulstu/extractor/repository/CommitRepository.java b/src/main/java/ru/ulstu/extractor/repository/CommitRepository.java index a1845b4..8bf5e98 100644 --- a/src/main/java/ru/ulstu/extractor/repository/CommitRepository.java +++ b/src/main/java/ru/ulstu/extractor/repository/CommitRepository.java @@ -39,4 +39,7 @@ public interface CommitRepository extends JpaRepository { List getCommitTimeEntityStatistic(@Param("branchId") Integer branchId, @Param("author") String author); void deleteByBranchIsNull(); + + @Query("SELECT b.commits FROM Branch b WHERE b.name = ?2 and b.repository.id = ?1") + List findByRepositoryIdAndBranchName(Integer repositoryId, String name); } diff --git a/src/main/java/ru/ulstu/extractor/service/CommitService.java b/src/main/java/ru/ulstu/extractor/service/CommitService.java index ed75ee1..8da7bda 100644 --- a/src/main/java/ru/ulstu/extractor/service/CommitService.java +++ b/src/main/java/ru/ulstu/extractor/service/CommitService.java @@ -49,6 +49,11 @@ public class CommitService { updateQuery = String.format(updateQuery, String.join(",", commitIds)); jdbcTemplate.update(updateQuery, branchId); } + + + public List findByRepositoryIdAndName(Integer repositoryId, String branchName) { + return commitRepository.findByRepositoryIdAndBranchName(repositoryId, branchName); + } } diff --git a/src/main/java/ru/ulstu/extractor/service/IndexService.java b/src/main/java/ru/ulstu/extractor/service/IndexService.java index 86f9454..875f357 100644 --- a/src/main/java/ru/ulstu/extractor/service/IndexService.java +++ b/src/main/java/ru/ulstu/extractor/service/IndexService.java @@ -10,13 +10,13 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import ru.ulstu.extractor.branch.BranchService; import ru.ulstu.extractor.gitrepository.GitRepositoryRepository; import ru.ulstu.extractor.gitrepository.GitRepositoryService; import ru.ulstu.extractor.model.Branch; import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.model.Repository; +import ru.ulstu.extractor.ts.AbstractTimeSeriesCreator; import java.io.IOException; import java.util.Collections; @@ -29,16 +29,18 @@ public class IndexService { private final GitRepositoryService gitRepositoryService; private final GitRepositoryRepository gitRepositoryRepository; private final BranchService branchService; + private final List timeSeriesCreators; public IndexService(GitRepositoryService gitRepositoryService, GitRepositoryRepository gitRepositoryRepository, - BranchService branchService) { + BranchService branchService, + List timeSeriesCreators) { this.gitRepositoryService = gitRepositoryService; this.gitRepositoryRepository = gitRepositoryRepository; this.branchService = branchService; + this.timeSeriesCreators = timeSeriesCreators; } - @Transactional public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException { Repository repository = gitRepositoryRepository.findByUrl(repositoryUrl); if (repository == null) { @@ -60,6 +62,8 @@ public class IndexService { commitsTo += COMMITS_PAGE_SIZE; commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false); } + Integer repositoryId = repository.getId(); + timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchName)); LOG.debug("Complete indexing {} branch", branchName); } } diff --git a/src/main/java/ru/ulstu/extractor/service/TimeSeriesService.java b/src/main/java/ru/ulstu/extractor/service/TimeSeriesService.java index 5653c3e..30a8327 100644 --- a/src/main/java/ru/ulstu/extractor/service/TimeSeriesService.java +++ b/src/main/java/ru/ulstu/extractor/service/TimeSeriesService.java @@ -13,6 +13,7 @@ import ru.ulstu.extractor.model.TimeSeries; import ru.ulstu.extractor.model.TimeSeriesValue; import ru.ulstu.extractor.repository.TimeSeriesRepository; import ru.ulstu.extractor.repository.TimeSeriesValueRepository; +import ru.ulstu.extractor.ts.TimeSeriesDateMapper; import java.util.Date; import java.util.List; @@ -24,21 +25,23 @@ public class TimeSeriesService { private final static Logger LOG = LoggerFactory.getLogger(TimeSeriesService.class); private final TimeSeriesRepository timeSeriesRepository; private final TimeSeriesValueRepository timeSeriesValueRepository; - + private final TimeSeriesDateMapper.TimeSeriesInterval timeSeriesInterval = TimeSeriesDateMapper.TimeSeriesInterval.HOUR; public TimeSeriesService(TimeSeriesRepository timeSeriesRepository, TimeSeriesValueRepository timeSeriesValueRepository) { this.timeSeriesRepository = timeSeriesRepository; this.timeSeriesValueRepository = timeSeriesValueRepository; } - public TimeSeries save(TimeSeries timeSeries, List timeSeriesValues) { - LOG.debug("Start save {} time series with {} time series values ", timeSeries.getName(), timeSeriesValues.size()); + public TimeSeries save(String timeSeriesName, List timeSeriesValues) { + LOG.debug("Start save {} time series with {} time series values ", timeSeriesName, timeSeriesValues.size()); + final TimeSeries timeSeries = findOrCreate(timeSeriesName); List timeSeriesValuesToRemoveIds = timeSeries.getValues().stream().map(BaseEntity::getId).collect(Collectors.toList()); + timeSeriesValues.forEach(v -> v.setTimeSeries(timeSeries)); timeSeries.setValues(timeSeriesValues); LOG.debug("Save time series {} ", timeSeries.getName()); - timeSeries = timeSeriesRepository.save(timeSeries); + TimeSeries savedTimeSeries = timeSeriesRepository.save(timeSeries); LOG.debug("Clear {} ime series values ", timeSeriesValuesToRemoveIds.size()); - return timeSeries; + return savedTimeSeries; } public TimeSeries findOrCreate(String timeSeriesName) { @@ -64,12 +67,12 @@ public class TimeSeriesService { TimeSeries timeSeries = findOrCreate(timeSeriesName); timeSeriesValueRepository.save(new TimeSeriesValue(timeSeries, date, value)); } -// -// public TimeSeriesValue findByTimeSeriesAndName(TimeSeries timeSeries, String name) { -// return timeSeriesValueRepository.findByTimeSeriesAndName(timeSeries, name); -// } public List findAll() { return timeSeriesValueRepository.findAll(); } + + public TimeSeriesDateMapper.TimeSeriesInterval getTimeSeriesInterval() { + return timeSeriesInterval; + } } diff --git a/src/main/java/ru/ulstu/extractor/ts/AbstractTSExtractor.java b/src/main/java/ru/ulstu/extractor/ts/AbstractTSExtractor.java deleted file mode 100644 index 14271ee..0000000 --- a/src/main/java/ru/ulstu/extractor/ts/AbstractTSExtractor.java +++ /dev/null @@ -1,21 +0,0 @@ -package ru.ulstu.extractor.ts; - -import ru.ulstu.extractor.service.TimeSeriesService; - -import java.util.Date; - -public abstract class AbstractTSExtractor { - - public abstract String getTSName(); - - public abstract int getNewTSValue(Integer repositoryId, String branchName); - - public abstract TimeSeriesService getTimeSeriesService(); - - public void addPoint(Integer repositoryId, String branchName) { - getTimeSeriesService().addTimeSeriesValue( - String.format("%s %s %s", getTSName(), repositoryId, branchName), - new Date(), - getNewTSValue(repositoryId, branchName)); - } -} diff --git a/src/main/java/ru/ulstu/extractor/ts/AbstractTimeSeriesCreator.java b/src/main/java/ru/ulstu/extractor/ts/AbstractTimeSeriesCreator.java new file mode 100644 index 0000000..2cc9f4f --- /dev/null +++ b/src/main/java/ru/ulstu/extractor/ts/AbstractTimeSeriesCreator.java @@ -0,0 +1,22 @@ +package ru.ulstu.extractor.ts; + +import ru.ulstu.extractor.model.TimeSeriesValue; +import ru.ulstu.extractor.service.TimeSeriesService; + +import java.util.List; + +public abstract class AbstractTimeSeriesCreator { + + public abstract String getTimeSeriesName(); + + public abstract List getTimeSeriesValues(Integer repositoryId, String branchName); + + public abstract TimeSeriesService getTimeSeriesService(); + + public void addTimeSeries(Integer repositoryId, String branchName) { + getTimeSeriesService().save( + String.format("%s %s %s", getTimeSeriesName(), repositoryId, branchName), + TimeSeriesDateMapper.mapTimeSeriesToInterval(getTimeSeriesService().getTimeSeriesInterval(), + getTimeSeriesValues(repositoryId, branchName))); + } +} diff --git a/src/main/java/ru/ulstu/extractor/ts/CommitsTS.java b/src/main/java/ru/ulstu/extractor/ts/CommitsTS.java index a50f988..6a2b6f2 100644 --- a/src/main/java/ru/ulstu/extractor/ts/CommitsTS.java +++ b/src/main/java/ru/ulstu/extractor/ts/CommitsTS.java @@ -1,28 +1,35 @@ package ru.ulstu.extractor.ts; import org.springframework.stereotype.Component; -import ru.ulstu.extractor.branch.BranchService; +import ru.ulstu.extractor.model.TimeSeriesValue; +import ru.ulstu.extractor.service.CommitService; import ru.ulstu.extractor.service.TimeSeriesService; +import java.util.List; +import java.util.stream.Collectors; + @Component -public class CommitsTS extends AbstractTSExtractor { +public class CommitsTS extends AbstractTimeSeriesCreator { private final TimeSeriesService timeSeriesService; - private final BranchService branchService; + private final CommitService commitService; public CommitsTS(TimeSeriesService timeSeriesService, - BranchService branchService) { + CommitService commitService) { this.timeSeriesService = timeSeriesService; - this.branchService = branchService; + this.commitService = commitService; } @Override - public String getTSName() { + public String getTimeSeriesName() { return "Количество коммитов во времени"; } @Override - public int getNewTSValue(Integer repositoryId, String branchName) { - return branchService.getCommitsCount(repositoryId, branchName); + public List getTimeSeriesValues(Integer repositoryId, String branchName) { + return commitService.findByRepositoryIdAndName(repositoryId, branchName) + .stream() + .map(c -> new TimeSeriesValue(c.getDate(), 1)) + .collect(Collectors.toList()); } @Override diff --git a/src/main/java/ru/ulstu/extractor/ts/TimeSeriesDateMapper.java b/src/main/java/ru/ulstu/extractor/ts/TimeSeriesDateMapper.java index d192b53..19298a0 100644 --- a/src/main/java/ru/ulstu/extractor/ts/TimeSeriesDateMapper.java +++ b/src/main/java/ru/ulstu/extractor/ts/TimeSeriesDateMapper.java @@ -17,8 +17,8 @@ import java.util.stream.Collectors; */ public class TimeSeriesDateMapper { - public TimeSeries mapTimeSeriesToInterval(TimeSeriesInterval timeSeriesInterval, TimeSeries timeSeries) { - List trimmedTimeSeriesValues = timeSeries.getValues() + public static List mapTimeSeriesToInterval(TimeSeriesInterval timeSeriesInterval, List timeSeriesValues) { + List trimmedTimeSeriesValues = timeSeriesValues .stream() .map(timeSeriesValue -> new TimeSeriesValue(trimTo(timeSeriesInterval, timeSeriesValue.getDate()), timeSeriesValue.getValue())) @@ -28,13 +28,18 @@ public class TimeSeriesDateMapper { .collect(Collectors.groupingBy(TimeSeriesValue::getDate, Collectors.summingInt(TimeSeriesValue::getValue))); - return new TimeSeries(timeSeries.getName(), groupedTimeSeriesValues.entrySet() + return groupedTimeSeriesValues.entrySet() .stream() .map(e -> new TimeSeriesValue(e.getKey(), e.getValue())) - .collect(Collectors.toList())); + .collect(Collectors.toList()); } - private Date trimTo(TimeSeriesInterval timeSeriesInterval, Date date) { + public static TimeSeries mapTimeSeriesToInterval(TimeSeriesInterval timeSeriesInterval, TimeSeries timeSeries) { + timeSeries.setValues(mapTimeSeriesToInterval(timeSeriesInterval, timeSeries.getValues())); + return timeSeries; + } + + private static Date trimTo(TimeSeriesInterval timeSeriesInterval, Date date) { return DateUtils.truncate(date, timeSeriesInterval.calendarField); }