#58 -- save time series on indexing

src/main/java/ru/ulstu/extractor/branch/BranchController.java

@@ -3,13 +3,12 @@
  * You may use, distribute and modify this code, please write to: romanov73@gmail.com.
  */
 
-package ru.ulstu.extractor.controller;
+package ru.ulstu.extractor.branch;
 
 import org.springframework.stereotype.Controller;
 import org.springframework.ui.Model;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestParam;
-import ru.ulstu.extractor.branch.BranchRepository;
 import ru.ulstu.extractor.gitrepository.GitRepositoryRepository;
 import springfox.documentation.annotations.ApiIgnore;
 

src/main/java/ru/ulstu/extractor/branch/BranchService.java

@@ -7,8 +7,6 @@ package ru.ulstu.extractor.branch;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.data.domain.Page;
-import org.springframework.data.domain.Pageable;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import ru.ulstu.extractor.model.BaseEntity;
@@ -56,14 +54,6 @@ public class BranchService {
         return branchRepository.findByRepositoryAndName(repository, branchName);
     }
 
-    public int getCommitsCount(Integer repositoryId, String branchName) {
-        return branchRepository.getCommitsCount(repositoryId, branchName);
-    }
-
-    public Page<Branch> findByRepository(Repository repository, Pageable pageable) {
-        return branchRepository.findByRepository(repository, pageable);
-    }
-
     public List<Branch> findAll() {
         return branchRepository.findAll();
     }

src/main/java/ru/ulstu/extractor/loader/GitScheduler.java

@@ -1,69 +0,0 @@
-package ru.ulstu.extractor.loader;
-
-import org.eclipse.jgit.api.errors.GitAPIException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.data.domain.Page;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Service;
-import ru.ulstu.extractor.branch.BranchService;
-import ru.ulstu.extractor.gitrepository.GitRepositoryService;
-import ru.ulstu.extractor.model.Branch;
-import ru.ulstu.extractor.model.OffsetablePageRequest;
-import ru.ulstu.extractor.model.Repository;
-import ru.ulstu.extractor.service.IndexService;
-import ru.ulstu.extractor.ts.AbstractTSExtractor;
-
-import java.io.IOException;
-import java.util.List;
-
-@Service
-public class GitScheduler {
-    private final static int DEFAULT_PAGE_SIZE = 100;
-    private final Logger log = LoggerFactory.getLogger(GitScheduler.class);
-    private final GitRepositoryService gitRepositoryService;
-    private final BranchService branchService;
-    private final IndexService indexService;
-    private final List<AbstractTSExtractor> tsExtractors;
-
-    public GitScheduler(GitRepositoryService gitRepositoryService,
-                        BranchService branchService,
-                        IndexService indexService,
-                        List<AbstractTSExtractor> tsExtractors) {
-        this.gitRepositoryService = gitRepositoryService;
-        this.branchService = branchService;
-        this.indexService = indexService;
-        this.tsExtractors = tsExtractors;
-    }
-
-    @Scheduled(cron = "* */5 * * * *")
-    public void getCommitsCount() {
-        log.debug("Load commits count started");
-        int repositoryPageNumber = 0;
-        Page<Repository> repositoryPage;
-        do {
-            int branchPageNumber = 0;
-            repositoryPage = gitRepositoryService.findAll(
-                    new OffsetablePageRequest(repositoryPageNumber, DEFAULT_PAGE_SIZE));
-            Page<Branch> branchPage;
-            for (Repository repository : repositoryPage.getContent()) {
-                do {
-                    branchPage = branchService.findByRepository(repository,
-                            new OffsetablePageRequest(branchPageNumber, DEFAULT_PAGE_SIZE));
-                    for (Branch branch : branchPage.getContent()) {
-                        try {
-                            indexService.index(repository.getUrl(), branch.getName());
-                            tsExtractors.forEach(tsExtractor -> tsExtractor.addPoint(repository.getId(), branch.getName()));
-                        } catch (GitAPIException | IOException ex) {
-                            log.warn(ex.getMessage());
-                            ex.printStackTrace();
-                        }
-                    }
-                    branchPageNumber++;
-                } while (!branchPage.isEmpty());
-            }
-            repositoryPageNumber++;
-        } while (!repositoryPage.isEmpty());
-        log.debug("Load commits count finished");
-    }
-}

src/main/java/ru/ulstu/extractor/repository/CommitRepository.java

@@ -39,4 +39,7 @@ public interface CommitRepository extends JpaRepository<Commit, Integer> {
     List<CommitTimeStatistic> getCommitTimeEntityStatistic(@Param("branchId") Integer branchId, @Param("author") String author);
 
     void deleteByBranchIsNull();
+
+    @Query("SELECT b.commits FROM Branch b WHERE b.name = ?2 and b.repository.id = ?1")
+    List<Commit> findByRepositoryIdAndBranchName(Integer repositoryId, String name);
 }

src/main/java/ru/ulstu/extractor/service/CommitService.java

@@ -49,6 +49,11 @@ public class CommitService {
         updateQuery = String.format(updateQuery, String.join(",", commitIds));
         jdbcTemplate.update(updateQuery, branchId);
     }
+
+
+    public List<Commit> findByRepositoryIdAndName(Integer repositoryId, String branchName) {
+        return commitRepository.findByRepositoryIdAndBranchName(repositoryId, branchName);
+    }
 }
 
 

src/main/java/ru/ulstu/extractor/service/IndexService.java

@@ -10,13 +10,13 @@ import org.eclipse.jgit.api.errors.GitAPIException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
 import ru.ulstu.extractor.branch.BranchService;
 import ru.ulstu.extractor.gitrepository.GitRepositoryRepository;
 import ru.ulstu.extractor.gitrepository.GitRepositoryService;
 import ru.ulstu.extractor.model.Branch;
 import ru.ulstu.extractor.model.Commit;
 import ru.ulstu.extractor.model.Repository;
+import ru.ulstu.extractor.ts.AbstractTimeSeriesCreator;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -29,16 +29,18 @@ public class IndexService {
     private final GitRepositoryService gitRepositoryService;
     private final GitRepositoryRepository gitRepositoryRepository;
     private final BranchService branchService;
+    private final List<AbstractTimeSeriesCreator> timeSeriesCreators;
 
     public IndexService(GitRepositoryService gitRepositoryService,
                         GitRepositoryRepository gitRepositoryRepository,
-                        BranchService branchService) {
+                        BranchService branchService,
+                        List<AbstractTimeSeriesCreator> timeSeriesCreators) {
         this.gitRepositoryService = gitRepositoryService;
         this.gitRepositoryRepository = gitRepositoryRepository;
         this.branchService = branchService;
+        this.timeSeriesCreators = timeSeriesCreators;
     }
 
-    @Transactional
     public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException {
         Repository repository = gitRepositoryRepository.findByUrl(repositoryUrl);
         if (repository == null) {
@@ -60,6 +62,8 @@ public class IndexService {
             commitsTo += COMMITS_PAGE_SIZE;
             commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false);
         }
+        Integer repositoryId = repository.getId();
+        timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchName));
         LOG.debug("Complete indexing {} branch", branchName);
     }
 }

src/main/java/ru/ulstu/extractor/service/TimeSeriesService.java

@@ -13,6 +13,7 @@ import ru.ulstu.extractor.model.TimeSeries;
 import ru.ulstu.extractor.model.TimeSeriesValue;
 import ru.ulstu.extractor.repository.TimeSeriesRepository;
 import ru.ulstu.extractor.repository.TimeSeriesValueRepository;
+import ru.ulstu.extractor.ts.TimeSeriesDateMapper;
 
 import java.util.Date;
 import java.util.List;
@@ -24,21 +25,23 @@ public class TimeSeriesService {
     private final static Logger LOG = LoggerFactory.getLogger(TimeSeriesService.class);
     private final TimeSeriesRepository timeSeriesRepository;
     private final TimeSeriesValueRepository timeSeriesValueRepository;
-
+    private final TimeSeriesDateMapper.TimeSeriesInterval timeSeriesInterval = TimeSeriesDateMapper.TimeSeriesInterval.HOUR;
 
     public TimeSeriesService(TimeSeriesRepository timeSeriesRepository, TimeSeriesValueRepository timeSeriesValueRepository) {
         this.timeSeriesRepository = timeSeriesRepository;
         this.timeSeriesValueRepository = timeSeriesValueRepository;
     }
 
-    public TimeSeries save(TimeSeries timeSeries, List<TimeSeriesValue> timeSeriesValues) {
+    public TimeSeries save(String timeSeriesName, List<TimeSeriesValue> timeSeriesValues) {
-        LOG.debug("Start save {} time series with {} time series values ", timeSeries.getName(), timeSeriesValues.size());
+        LOG.debug("Start save {} time series with {} time series values ", timeSeriesName, timeSeriesValues.size());
+        final TimeSeries timeSeries = findOrCreate(timeSeriesName);
         List<Integer> timeSeriesValuesToRemoveIds = timeSeries.getValues().stream().map(BaseEntity::getId).collect(Collectors.toList());
+        timeSeriesValues.forEach(v -> v.setTimeSeries(timeSeries));
         timeSeries.setValues(timeSeriesValues);
         LOG.debug("Save time series {} ", timeSeries.getName());
-        timeSeries = timeSeriesRepository.save(timeSeries);
+        TimeSeries savedTimeSeries = timeSeriesRepository.save(timeSeries);
         LOG.debug("Clear {} ime series values ", timeSeriesValuesToRemoveIds.size());
-        return timeSeries;
+        return savedTimeSeries;
     }
 
     public TimeSeries findOrCreate(String timeSeriesName) {
@@ -64,12 +67,12 @@ public class TimeSeriesService {
         TimeSeries timeSeries = findOrCreate(timeSeriesName);
         timeSeriesValueRepository.save(new TimeSeriesValue(timeSeries, date, value));
     }
-//
-//    public TimeSeriesValue findByTimeSeriesAndName(TimeSeries timeSeries, String name) {
-//        return timeSeriesValueRepository.findByTimeSeriesAndName(timeSeries, name);
-//    }
 
     public List<TimeSeriesValue> findAll() {
         return timeSeriesValueRepository.findAll();
     }
+
+    public TimeSeriesDateMapper.TimeSeriesInterval getTimeSeriesInterval() {
+        return timeSeriesInterval;
+    }
 }

src/main/java/ru/ulstu/extractor/ts/AbstractTSExtractor.java

@@ -1,21 +0,0 @@
-package ru.ulstu.extractor.ts;
-
-import ru.ulstu.extractor.service.TimeSeriesService;
-
-import java.util.Date;
-
-public abstract class AbstractTSExtractor {
-
-    public abstract String getTSName();
-
-    public abstract int getNewTSValue(Integer repositoryId, String branchName);
-
-    public abstract TimeSeriesService getTimeSeriesService();
-
-    public void addPoint(Integer repositoryId, String branchName) {
-        getTimeSeriesService().addTimeSeriesValue(
-                String.format("%s %s %s", getTSName(), repositoryId, branchName),
-                new Date(),
-                getNewTSValue(repositoryId, branchName));
-    }
-}

src/main/java/ru/ulstu/extractor/ts/AbstractTimeSeriesCreator.java

@@ -0,0 +1,22 @@
+package ru.ulstu.extractor.ts;
+
+import ru.ulstu.extractor.model.TimeSeriesValue;
+import ru.ulstu.extractor.service.TimeSeriesService;
+
+import java.util.List;
+
+public abstract class AbstractTimeSeriesCreator {
+
+    public abstract String getTimeSeriesName();
+
+    public abstract List<TimeSeriesValue> getTimeSeriesValues(Integer repositoryId, String branchName);
+
+    public abstract TimeSeriesService getTimeSeriesService();
+
+    public void addTimeSeries(Integer repositoryId, String branchName) {
+        getTimeSeriesService().save(
+                String.format("%s %s %s", getTimeSeriesName(), repositoryId, branchName),
+                TimeSeriesDateMapper.mapTimeSeriesToInterval(getTimeSeriesService().getTimeSeriesInterval(),
+                        getTimeSeriesValues(repositoryId, branchName)));
+    }
+}

src/main/java/ru/ulstu/extractor/ts/CommitsTS.java

@@ -1,28 +1,35 @@
 package ru.ulstu.extractor.ts;
 
 import org.springframework.stereotype.Component;
-import ru.ulstu.extractor.branch.BranchService;
+import ru.ulstu.extractor.model.TimeSeriesValue;
+import ru.ulstu.extractor.service.CommitService;
 import ru.ulstu.extractor.service.TimeSeriesService;
 
+import java.util.List;
+import java.util.stream.Collectors;
+
 @Component
-public class CommitsTS extends AbstractTSExtractor {
+public class CommitsTS extends AbstractTimeSeriesCreator {
     private final TimeSeriesService timeSeriesService;
-    private final BranchService branchService;
+    private final CommitService commitService;
 
     public CommitsTS(TimeSeriesService timeSeriesService,
-                     BranchService branchService) {
+                     CommitService commitService) {
         this.timeSeriesService = timeSeriesService;
-        this.branchService = branchService;
+        this.commitService = commitService;
     }
 
     @Override
-    public String getTSName() {
+    public String getTimeSeriesName() {
         return "Количество коммитов во времени";
     }
 
     @Override
-    public int getNewTSValue(Integer repositoryId, String branchName) {
+    public List<TimeSeriesValue> getTimeSeriesValues(Integer repositoryId, String branchName) {
-        return branchService.getCommitsCount(repositoryId, branchName);
+        return commitService.findByRepositoryIdAndName(repositoryId, branchName)
+                .stream()
+                .map(c -> new TimeSeriesValue(c.getDate(), 1))
+                .collect(Collectors.toList());
     }
 
     @Override

src/main/java/ru/ulstu/extractor/ts/TimeSeriesDateMapper.java

@@ -17,8 +17,8 @@ import java.util.stream.Collectors;
  */
 public class TimeSeriesDateMapper {
 
-    public TimeSeries mapTimeSeriesToInterval(TimeSeriesInterval timeSeriesInterval, TimeSeries timeSeries) {
+    public static List<TimeSeriesValue> mapTimeSeriesToInterval(TimeSeriesInterval timeSeriesInterval, List<TimeSeriesValue> timeSeriesValues) {
-        List<TimeSeriesValue> trimmedTimeSeriesValues = timeSeries.getValues()
+        List<TimeSeriesValue> trimmedTimeSeriesValues = timeSeriesValues
                 .stream()
                 .map(timeSeriesValue -> new TimeSeriesValue(trimTo(timeSeriesInterval, timeSeriesValue.getDate()),
                         timeSeriesValue.getValue()))
@@ -28,13 +28,18 @@ public class TimeSeriesDateMapper {
                 .collect(Collectors.groupingBy(TimeSeriesValue::getDate,
                         Collectors.summingInt(TimeSeriesValue::getValue)));
 
-        return new TimeSeries(timeSeries.getName(), groupedTimeSeriesValues.entrySet()
+        return groupedTimeSeriesValues.entrySet()
                 .stream()
                 .map(e -> new TimeSeriesValue(e.getKey(), e.getValue()))
-                .collect(Collectors.toList()));
+                .collect(Collectors.toList());
     }
 
-    private Date trimTo(TimeSeriesInterval timeSeriesInterval, Date date) {
+    public static TimeSeries mapTimeSeriesToInterval(TimeSeriesInterval timeSeriesInterval, TimeSeries timeSeries) {
+        timeSeries.setValues(mapTimeSeriesToInterval(timeSeriesInterval, timeSeries.getValues()));
+        return timeSeries;
+    }
+
+    private static Date trimTo(TimeSeriesInterval timeSeriesInterval, Date date) {
         return DateUtils.truncate(date, timeSeriesInterval.calendarField);
     }