romanov73/git-extractor
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Временные ряды #59

Merged
romanov73 merged 10 commits from influx into master 2022-10-06 16:09:37 +04:00
Conversation 0 Commits 10 Files Changed 57 +1313 -156
57 changed files with 1313 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
build.gradle
View File

@ -1,8 +1,3 @@
/*
* Copyright (C) 2021 Anton Romanov - All Rights Reserved
* You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/
buildscript { buildscript {
ext { ext {
versionSpringBoot = '2.3.9.RELEASE' versionSpringBoot = '2.3.9.RELEASE'
@ -53,6 +48,9 @@ dependencies {
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-jetty' implementation group: 'org.springframework.boot', name: 'spring-boot-starter-jetty'
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-thymeleaf' implementation group: 'org.springframework.boot', name: 'spring-boot-starter-thymeleaf'
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa' implementation group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa'
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-webflux'
implementation group: 'org.json', name: 'json', version: '20220320'
implementation group: 'nz.net.ultraq.thymeleaf', name: 'thymeleaf-layout-dialect' implementation group: 'nz.net.ultraq.thymeleaf', name: 'thymeleaf-layout-dialect'
implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-afterburner' implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-afterburner'
implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-hibernate5' implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-hibernate5'
@ -61,6 +59,7 @@ dependencies {
implementation group: 'commons-io', name: 'commons-io', version: '2.6' implementation group: 'commons-io', name: 'commons-io', version: '2.6'
implementation group: 'net.sourceforge.htmlunit', name: 'htmlunit', version: '2.35.0' implementation group: 'net.sourceforge.htmlunit', name: 'htmlunit', version: '2.35.0'
implementation group: 'com.github.javaparser', name: 'javaparser-core', version: '3.20.2' implementation group: 'com.github.javaparser', name: 'javaparser-core', version: '3.20.2'
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
implementation group: 'io.springfox', name: 'springfox-boot-starter', version: '3.0.0' implementation group: 'io.springfox', name: 'springfox-boot-starter', version: '3.0.0'

2
src/main/java/ru/ulstu/extractor/GitExtractorApplication.java
View File

@ -2,8 +2,10 @@ package ru.ulstu.extractor;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication @SpringBootApplication
@EnableScheduling
public class GitExtractorApplication { public class GitExtractorApplication {
public static void main(String[] args) { public static void main(String[] args) {
SpringApplication.run(GitExtractorApplication.class, args); SpringApplication.run(GitExtractorApplication.class, args);

20
src/main/java/ru/ulstu/extractor/controller/BranchController.java → src/main/java/ru/ulstu/extractor/branch/controller/BranchController.java
View File

@ -3,14 +3,14 @@
* You may use, distribute and modify this code, please write to: romanov73@gmail.com. * You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/ */
package ru.ulstu.extractor.controller; package ru.ulstu.extractor.branch.controller;
import org.springframework.stereotype.Controller; import org.springframework.stereotype.Controller;
import org.springframework.ui.Model; import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import ru.ulstu.extractor.repository.BranchRepository; import ru.ulstu.extractor.branch.repository.BranchRepository;
import ru.ulstu.extractor.repository.RepositoryRepository; import ru.ulstu.extractor.gitrepository.GitRepositoryRepository;
import springfox.documentation.annotations.ApiIgnore; import springfox.documentation.annotations.ApiIgnore;
import static ru.ulstu.extractor.controller.Route.DELETE_BRANCH; import static ru.ulstu.extractor.controller.Route.DELETE_BRANCH;
@ -19,11 +19,11 @@ import static ru.ulstu.extractor.controller.Route.LIST_REPOSITORY_BRANCHES;
@Controller @Controller
@ApiIgnore @ApiIgnore
public class BranchController { public class BranchController {
private final RepositoryRepository repositoryRepository; private final GitRepositoryRepository gitRepositoryRepository;
private final BranchRepository branchRepository; private final BranchRepository branchRepository;
public BranchController(RepositoryRepository repositoryRepository, BranchRepository branchRepository) { public BranchController(GitRepositoryRepository gitRepositoryRepository, BranchRepository branchRepository) {
this.repositoryRepository = repositoryRepository; this.gitRepositoryRepository = gitRepositoryRepository;
this.branchRepository = branchRepository; this.branchRepository = branchRepository;
} }
@ -31,8 +31,8 @@ public class BranchController {
public String indexBranch( public String indexBranch(
Model model, Model model,
@RequestParam int repositoryId) { @RequestParam int repositoryId) {
model.addAttribute("branches", branchRepository.findByRepositoryId(repositoryId)); model.addAttribute("branches", branchRepository.findByGitRepositoryId(repositoryId));
model.addAttribute("repository", repositoryRepository.findById(repositoryId).get()); model.addAttribute("repository", gitRepositoryRepository.findById(repositoryId).get());
return LIST_REPOSITORY_BRANCHES; return LIST_REPOSITORY_BRANCHES;
} }
@ -41,8 +41,8 @@ public class BranchController {
@RequestParam int repositoryId, @RequestParam int repositoryId,
@RequestParam Integer id) { @RequestParam Integer id) {
branchRepository.deleteById(id); branchRepository.deleteById(id);
model.addAttribute("branches", branchRepository.findByRepositoryId(repositoryId)); model.addAttribute("branches", branchRepository.findByGitRepositoryId(repositoryId));
model.addAttribute("repository", repositoryRepository.findById(repositoryId).get()); model.addAttribute("repository", gitRepositoryRepository.findById(repositoryId).get());
return LIST_REPOSITORY_BRANCHES; return LIST_REPOSITORY_BRANCHES;
} }
} }

19
src/main/java/ru/ulstu/extractor/model/Branch.java → src/main/java/ru/ulstu/extractor/branch/model/Branch.java
View File

@ -3,10 +3,13 @@
* You may use, distribute and modify this code, please write to: romanov73@gmail.com. * You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/ */
package ru.ulstu.extractor.model; package ru.ulstu.extractor.branch.model;
import org.hibernate.annotations.Fetch; import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode; import org.hibernate.annotations.FetchMode;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
import ru.ulstu.extractor.model.BaseEntity;
import ru.ulstu.extractor.model.Commit;
import javax.persistence.CascadeType; import javax.persistence.CascadeType;
import javax.persistence.Entity; import javax.persistence.Entity;
@ -22,7 +25,7 @@ public class Branch extends BaseEntity {
private String name; private String name;
@ManyToOne @ManyToOne
private Repository repository; private GitRepository gitRepository;
@OneToMany(cascade = CascadeType.ALL, fetch = FetchType.LAZY) @OneToMany(cascade = CascadeType.ALL, fetch = FetchType.LAZY)
@JoinColumn(name = "branch_id") @JoinColumn(name = "branch_id")
@ -36,8 +39,8 @@ public class Branch extends BaseEntity {
this.name = name; this.name = name;
} }
public Branch(Repository repository, String branchName) { public Branch(GitRepository gitRepository, String branchName) {
this.repository = repository; this.gitRepository = gitRepository;
this.name = branchName; this.name = branchName;
} }
@ -49,12 +52,12 @@ public class Branch extends BaseEntity {
this.name = name; this.name = name;
} }
public Repository getRepository() { public GitRepository getGitRepository() {
return repository; return gitRepository;
} }
public void setRepository(Repository repository) { public void setGitRepository(GitRepository gitRepository) {
this.repository = repository; this.gitRepository = gitRepository;
} }
public List<Commit> getCommits() { public List<Commit> getCommits() {

21
src/main/java/ru/ulstu/extractor/branch/repository/BranchRepository.java Normal file
View File

@ -0,0 +1,21 @@
package ru.ulstu.extractor.branch.repository;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
import java.util.List;
public interface BranchRepository extends JpaRepository<Branch, Integer> {
Branch findByGitRepositoryAndName(GitRepository gitRepository, String name);
@Query("select count(c) from Commit c LEFT JOIN c.branch b LEFT JOIN GitRepository r where r.id = ?1 AND b.name = ?2")
int getCommitsCount(Integer repositoryId, String name);
List<Branch> findByGitRepositoryId(Integer repositoryId);
Page<Branch> findByGitRepository(GitRepository gitRepository, Pageable pageable);
}

15
src/main/java/ru/ulstu/extractor/service/BranchService.java → src/main/java/ru/ulstu/extractor/branch/service/BranchService.java
View File

@ -3,16 +3,18 @@
* You may use, distribute and modify this code, please write to: romanov73@gmail.com. * You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/ */
package ru.ulstu.extractor.service; package ru.ulstu.extractor.branch.service;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.branch.repository.BranchRepository;
import ru.ulstu.extractor.commit.service.CommitService;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
import ru.ulstu.extractor.model.BaseEntity; import ru.ulstu.extractor.model.BaseEntity;
import ru.ulstu.extractor.model.Branch;
import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.model.Repository;
import ru.ulstu.extractor.repository.BranchRepository;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -29,6 +31,7 @@ public class BranchService {
this.commitService = commitService; this.commitService = commitService;
} }
@Transactional
public Branch save(Branch branch, List<Commit> commits) { public Branch save(Branch branch, List<Commit> commits) {
LOG.debug("Start save {} branch with {} commits ", branch.getName(), commits.size()); LOG.debug("Start save {} branch with {} commits ", branch.getName(), commits.size());
List<Integer> commitsToRemoveIds = branch.getCommits().stream().map(BaseEntity::getId).collect(Collectors.toList()); List<Integer> commitsToRemoveIds = branch.getCommits().stream().map(BaseEntity::getId).collect(Collectors.toList());
@ -48,8 +51,8 @@ public class BranchService {
return branch; return branch;
} }
public Branch findByRepositoryAndName(Repository repository, String branchName) { public Branch findByRepositoryAndName(GitRepository gitRepository, String branchName) {
return branchRepository.findByRepositoryAndName(repository, branchName); return branchRepository.findByGitRepositoryAndName(gitRepository, branchName);
} }
public List<Branch> findAll() { public List<Branch> findAll() {

1
src/main/java/ru/ulstu/extractor/model/Commit.java → src/main/java/ru/ulstu/extractor/commit/model/Commit.java
View File

@ -7,6 +7,7 @@ package ru.ulstu.extractor.model;
import org.hibernate.annotations.Fetch; import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode; import org.hibernate.annotations.FetchMode;
import ru.ulstu.extractor.branch.model.Branch;
import javax.persistence.CascadeType; import javax.persistence.CascadeType;
import javax.persistence.Entity; import javax.persistence.Entity;

2
src/main/java/ru/ulstu/extractor/model/CommitAuthorStatistic.java → src/main/java/ru/ulstu/extractor/commit/model/CommitAuthorStatistic.java
View File

@ -1,4 +1,4 @@
package ru.ulstu.extractor.model; package ru.ulstu.extractor.commit.model;
public class CommitAuthorStatistic { public class CommitAuthorStatistic {
private String author; private String author;

2
src/main/java/ru/ulstu/extractor/model/CommitEntityStatistic.java → src/main/java/ru/ulstu/extractor/commit/model/CommitEntityStatistic.java
View File

@ -1,4 +1,4 @@
package ru.ulstu.extractor.model; package ru.ulstu.extractor.commit.model;
public class CommitEntityStatistic { public class CommitEntityStatistic {
private Boolean entity; private Boolean entity;

2
src/main/java/ru/ulstu/extractor/model/CommitTimeStatistic.java → src/main/java/ru/ulstu/extractor/commit/model/CommitTimeStatistic.java
View File

@ -1,4 +1,4 @@
package ru.ulstu.extractor.model; package ru.ulstu.extractor.commit.model;
public class CommitTimeStatistic { public class CommitTimeStatistic {
private final static String DATE_TEMPLATE = "%s.%s"; private final static String DATE_TEMPLATE = "%s.%s";

2
src/main/java/ru/ulstu/extractor/model/CommitUrlStatistic.java → src/main/java/ru/ulstu/extractor/commit/model/CommitUrlStatistic.java
View File

@ -1,4 +1,4 @@
package ru.ulstu.extractor.model; package ru.ulstu.extractor.commit.model;
public class CommitUrlStatistic { public class CommitUrlStatistic {
private String url; private String url;

45
src/main/java/ru/ulstu/extractor/commit/repository/CommitRepository.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright (C) 2021 Anton Romanov - All Rights Reserved
* You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/
package ru.ulstu.extractor.commit.repository;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import ru.ulstu.extractor.commit.model.CommitAuthorStatistic;
import ru.ulstu.extractor.commit.model.CommitEntityStatistic;
import ru.ulstu.extractor.commit.model.CommitTimeStatistic;
import ru.ulstu.extractor.commit.model.CommitUrlStatistic;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
import ru.ulstu.extractor.model.Commit;
import java.util.List;
public interface CommitRepository extends JpaRepository<Commit, Integer> {
@Query("SELECT DISTINCT c FROM Commit c LEFT JOIN c.branch b LEFT JOIN c.fileChanges f LEFT JOIN c.author a LEFT JOIN b.gitRepository r WHERE r = :repository AND b.name = :branchName AND (:author IS NULL OR :author = '' OR a.name = :author) AND (:filter IS NULL OR :filter = '' OR lower(c.message) LIKE lower(concat('%', :filter,'%'))) AND (:entity IS NULL OR f.containsEntity = :entity)")
Page<Commit> findByRepositoryAndBranch(Pageable pageable, @Param("repository") GitRepository gitRepository, @Param("branchName") String branchName, @Param("author") String author, @Param("filter") String filter, @Param("entity") Boolean entity);
@Query("SELECT new ru.ulstu.extractor.commit.model.CommitAuthorStatistic(c.author.name, COUNT(DISTINCT c.hash)) FROM Commit c LEFT JOIN c.branch LEFT JOIN c.author a WHERE (:branchId IS NULL OR c.branch.id = :branchId) AND (:author IS NULL OR :author = '' OR a.name = :author) GROUP by c.author.name")
List<CommitAuthorStatistic> getCommitAuthorStatistic(@Param("branchId") Integer branchId, @Param("author") String author);
@Query("SELECT new ru.ulstu.extractor.commit.model.CommitUrlStatistic(c.branch.gitRepository.url, COUNT(DISTINCT c.hash)) FROM Commit c GROUP by c.branch.gitRepository.url")
List<CommitUrlStatistic> getCommitUrlStatistic();
@Query("SELECT new ru.ulstu.extractor.commit.model.CommitTimeStatistic(extract(month FROM c.date) as month, extract(year FROM c.date) as year, COUNT(DISTINCT c.hash)) FROM Commit c LEFT JOIN c.branch LEFT JOIN c.author a WHERE (:branchId IS NULL OR c.branch.id = :branchId) AND (:author IS NULL OR :author = '' OR a.name = :author) GROUP by extract(month from c.date), extract(year from c.date) ORDER by extract(year from c.date), extract(month from c.date)")
List<CommitTimeStatistic> getCommitTimeStatistic(@Param("branchId") Integer branchId, @Param("author") String author);
@Query("SELECT new ru.ulstu.extractor.commit.model.CommitEntityStatistic(f.containsEntity, COUNT(DISTINCT c.hash)) FROM Commit c LEFT JOIN c.branch LEFT JOIN c.author a LEFT JOIN c.fileChanges f WHERE (:branchId IS NULL OR c.branch.id = :branchId) AND (:author IS NULL OR :author = '' OR a.name = :author) GROUP by f.containsEntity")
List<CommitEntityStatistic> getCommitEntityStatistic(@Param("branchId") Integer branchId, @Param("author") String author);
@Query("SELECT new ru.ulstu.extractor.commit.model.CommitTimeStatistic(EXTRACT(MONTH FROM c.date), EXTRACT(YEAR FROM c.date), COUNT(DISTINCT c.hash)) FROM Commit c LEFT JOIN c.branch LEFT JOIN c.author a LEFT JOIN c.fileChanges f WHERE f.containsEntity = true AND (:branchId IS NULL OR c.branch.id = :branchId) AND (:author IS NULL OR :author = '' OR a.name = :author) GROUP by extract(month from c.date), extract(year from c.date) ORDER by extract(year from c.date), extract(month from c.date)")
List<CommitTimeStatistic> getCommitTimeEntityStatistic(@Param("branchId") Integer branchId, @Param("author") String author);
void deleteByBranchIsNull();
@Query("SELECT b.commits FROM Branch b WHERE b.name = ?2 and b.gitRepository.id = ?1")
List<Commit> findByRepositoryIdAndBranchName(Integer repositoryId, String name);
}

10
src/main/java/ru/ulstu/extractor/service/CommitService.java → src/main/java/ru/ulstu/extractor/commit/service/CommitService.java
View File

@ -3,12 +3,13 @@
* You may use, distribute and modify this code, please write to: romanov73@gmail.com. * You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/ */
package ru.ulstu.extractor.service; package ru.ulstu.extractor.commit.service;
import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import ru.ulstu.extractor.commit.repository.CommitRepository;
import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.repository.CommitRepository; import ru.ulstu.extractor.service.AuthorService;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -49,6 +50,11 @@ public class CommitService {
updateQuery = String.format(updateQuery, String.join(",", commitIds)); updateQuery = String.format(updateQuery, String.join(",", commitIds));
jdbcTemplate.update(updateQuery, branchId); jdbcTemplate.update(updateQuery, branchId);
} }
public List<Commit> findByRepositoryIdAndName(Integer repositoryId, String branchName) {
return commitRepository.findByRepositoryIdAndBranchName(repositoryId, branchName);
}
} }

13
src/main/java/ru/ulstu/extractor/config/WebClientConfiguration.java Normal file
View File

@ -0,0 +1,13 @@
package ru.ulstu.extractor.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.reactive.function.client.WebClient;
@Configuration
public class WebClientConfiguration {
@Bean
public WebClient webClient(WebClient.Builder webClientBuilder) {
return webClientBuilder.build();
}
}

4
src/main/java/ru/ulstu/extractor/controller/GitIndexingController.java
View File

@ -15,10 +15,10 @@ import org.springframework.web.bind.annotation.ModelAttribute;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.servlet.mvc.support.RedirectAttributes; import org.springframework.web.servlet.mvc.support.RedirectAttributes;
import ru.ulstu.extractor.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.gitrepository.GitRepositoryService;
import ru.ulstu.extractor.model.mvc.FilterForm; import ru.ulstu.extractor.model.mvc.FilterForm;
import ru.ulstu.extractor.model.mvc.RepoForm; import ru.ulstu.extractor.model.mvc.RepoForm;
import ru.ulstu.extractor.service.GitRepositoryService;
import ru.ulstu.extractor.service.IndexService; import ru.ulstu.extractor.service.IndexService;
import springfox.documentation.annotations.ApiIgnore; import springfox.documentation.annotations.ApiIgnore;

2
src/main/java/ru/ulstu/extractor/controller/RepoController.java
View File

@ -12,8 +12,8 @@ import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import ru.ulstu.extractor.gitrepository.GitRepositoryService;
import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.service.GitRepositoryService;
import ru.ulstu.extractor.service.IndexService; import ru.ulstu.extractor.service.IndexService;
import ru.ulstu.extractor.util.HttpUtils; import ru.ulstu.extractor.util.HttpUtils;

14
src/main/java/ru/ulstu/extractor/controller/RepositoryController.java
View File

@ -9,7 +9,7 @@ import org.springframework.stereotype.Controller;
import org.springframework.ui.Model; import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import ru.ulstu.extractor.repository.RepositoryRepository; import ru.ulstu.extractor.gitrepository.GitRepositoryRepository;
import springfox.documentation.annotations.ApiIgnore; import springfox.documentation.annotations.ApiIgnore;
import static ru.ulstu.extractor.controller.Route.DELETE_INDEXED_REPOSITORY; import static ru.ulstu.extractor.controller.Route.DELETE_INDEXED_REPOSITORY;
@ -18,23 +18,23 @@ import static ru.ulstu.extractor.controller.Route.LIST_INDEXED_REPOSITORIES;
@Controller @Controller
@ApiIgnore @ApiIgnore
public class RepositoryController { public class RepositoryController {
private final RepositoryRepository repositoryRepository; private final GitRepositoryRepository gitRepositoryRepository;
public RepositoryController(RepositoryRepository repositoryRepository) { public RepositoryController(GitRepositoryRepository gitRepositoryRepository) {
this.repositoryRepository = repositoryRepository; this.gitRepositoryRepository = gitRepositoryRepository;
} }
@GetMapping(LIST_INDEXED_REPOSITORIES) @GetMapping(LIST_INDEXED_REPOSITORIES)
public String indexNewRepo(Model model) { public String indexNewRepo(Model model) {
model.addAttribute("repositories", repositoryRepository.findAll()); model.addAttribute("repositories", gitRepositoryRepository.findAll());
return LIST_INDEXED_REPOSITORIES; return LIST_INDEXED_REPOSITORIES;
} }
@GetMapping(DELETE_INDEXED_REPOSITORY) @GetMapping(DELETE_INDEXED_REPOSITORY)
public String deleteRepo(Model model, public String deleteRepo(Model model,
@RequestParam Integer id) { @RequestParam Integer id) {
repositoryRepository.deleteById(id); gitRepositoryRepository.deleteById(id);
model.addAttribute("repositories", repositoryRepository.findAll()); model.addAttribute("repositories", gitRepositoryRepository.findAll());
return "redirect:/" + LIST_INDEXED_REPOSITORIES; return "redirect:/" + LIST_INDEXED_REPOSITORIES;
} }
} }

4
src/main/java/ru/ulstu/extractor/controller/StatisticController.java
View File

@ -9,9 +9,9 @@ import org.springframework.stereotype.Controller;
import org.springframework.ui.Model; import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import ru.ulstu.extractor.branch.service.BranchService;
import ru.ulstu.extractor.commit.repository.CommitRepository;
import ru.ulstu.extractor.model.mvc.FilterForm; import ru.ulstu.extractor.model.mvc.FilterForm;
import ru.ulstu.extractor.repository.CommitRepository;
import ru.ulstu.extractor.service.BranchService;
import ru.ulstu.extractor.service.FilteringService; import ru.ulstu.extractor.service.FilteringService;
import springfox.documentation.annotations.ApiIgnore; import springfox.documentation.annotations.ApiIgnore;

8
src/main/java/ru/ulstu/extractor/gitrepository/GitRepositoryRepository.java Normal file
View File

@ -0,0 +1,8 @@
package ru.ulstu.extractor.gitrepository;
import org.springframework.data.jpa.repository.JpaRepository;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
public interface GitRepositoryRepository extends JpaRepository<GitRepository, Integer> {
GitRepository findByUrl(String url);
}

60
src/main/java/ru/ulstu/extractor/service/GitRepositoryService.java → src/main/java/ru/ulstu/extractor/gitrepository/GitRepositoryService.java
View File

@ -1,9 +1,4 @@
/* package ru.ulstu.extractor.gitrepository;
* Copyright (C) 2021 Anton Romanov - All Rights Reserved
* You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/
package ru.ulstu.extractor.service;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.api.CreateBranchCommand; import org.eclipse.jgit.api.CreateBranchCommand;
@ -15,19 +10,21 @@ import org.eclipse.jgit.internal.storage.file.FileRepository;
import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.TreeWalk;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
import ru.ulstu.extractor.heuristic.model.BusinessLogicUnit; import ru.ulstu.extractor.heuristic.model.BusinessLogicUnit;
import ru.ulstu.extractor.heuristic.model.EntityUnit; import ru.ulstu.extractor.heuristic.model.EntityUnit;
import ru.ulstu.extractor.heuristic.model.ResourceUnit; import ru.ulstu.extractor.heuristic.model.ResourceUnit;
import ru.ulstu.extractor.heuristic.service.StructuralUnitService; import ru.ulstu.extractor.heuristic.service.StructuralUnitService;
import ru.ulstu.extractor.model.Author; import ru.ulstu.extractor.model.Author;
import ru.ulstu.extractor.model.Branch;
import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.model.FileChange; import ru.ulstu.extractor.model.FileChange;
import ru.ulstu.extractor.model.LineChange; import ru.ulstu.extractor.model.LineChange;
@ -61,18 +58,20 @@ public class GitRepositoryService {
private String customProjectsDir; private String customProjectsDir;
private final ExecutorService executorService = Executors.newFixedThreadPool(4); private final ExecutorService executorService = Executors.newFixedThreadPool(4);
private final ExecutorService executorServiceCommits = Executors.newFixedThreadPool(4); private final ExecutorService executorServiceCommits = Executors.newFixedThreadPool(4);
private final StructuralUnitService structuralUnitService; private final StructuralUnitService structuralUnitService;
private final GitRepositoryRepository gitRepositoryRepository;
public GitRepositoryService(StructuralUnitService structuralUnitService) { public GitRepositoryService(StructuralUnitService structuralUnitService,
GitRepositoryRepository gitRepositoryRepository) {
this.structuralUnitService = structuralUnitService; this.structuralUnitService = structuralUnitService;
this.gitRepositoryRepository = gitRepositoryRepository;
} }
public List<Branch> getRemoteBranches(String url) throws GitAPIException, IOException { public List<Branch> getRemoteBranches(String url) throws GitAPIException, IOException {
LOG.debug("Get remote branches of {}. Clone", url); LOG.debug("Get remote branches of {}. Clone", url);
cloneOrUpdateRepo(url); cloneOrUpdateRepo(url);
LOG.debug("Get remote branches of {}. Get branches", url); LOG.debug("Get remote branches of {}. Get branches", url);
Repository localRepo = new FileRepository(getProjectGitDirectory(url)); org.eclipse.jgit.lib.Repository localRepo = new FileRepository(getProjectGitDirectory(url));
Git git = new Git(localRepo); Git git = new Git(localRepo);
List<Branch> branches = git.branchList().setListMode(ListBranchCommand.ListMode.REMOTE) List<Branch> branches = git.branchList().setListMode(ListBranchCommand.ListMode.REMOTE)
.call() .call()
@ -88,7 +87,7 @@ public class GitRepositoryService {
LOG.debug("Get local branches of {}. Clone", url); LOG.debug("Get local branches of {}. Clone", url);
cloneOrUpdateRepo(url); cloneOrUpdateRepo(url);
LOG.debug("Get local branches of {}. Get branches", url); LOG.debug("Get local branches of {}. Get branches", url);
Repository localRepo = new FileRepository(getProjectGitDirectory(url)); org.eclipse.jgit.lib.Repository localRepo = new FileRepository(getProjectGitDirectory(url));
Git git = new Git(localRepo); Git git = new Git(localRepo);
List<Branch> branches = git.branchList() List<Branch> branches = git.branchList()
.call() .call()
@ -105,7 +104,7 @@ public class GitRepositoryService {
if (needUpdate) { if (needUpdate) {
cloneOrUpdateRepo(repositoryUrl, branchName); cloneOrUpdateRepo(repositoryUrl, branchName);
} }
Repository localRepo = new FileRepository(getProjectGitDirectory(repositoryUrl)); org.eclipse.jgit.lib.Repository localRepo = new FileRepository(getProjectGitDirectory(repositoryUrl));
Git git = new Git(localRepo); Git git = new Git(localRepo);
List<RevCommit> commits = new ArrayList<>(); List<RevCommit> commits = new ArrayList<>();
@ -144,7 +143,7 @@ public class GitRepositoryService {
return list; return list;
} }
private void checkoutBranch(String repositoryUrl, Git git, Repository localRepo, String branchName) throws GitAPIException, IOException { private void checkoutBranch(String repositoryUrl, Git git, org.eclipse.jgit.lib.Repository localRepo, String branchName) throws GitAPIException, IOException {
LOG.debug("Checkout branch {} {}", repositoryUrl, branchName); LOG.debug("Checkout branch {} {}", repositoryUrl, branchName);
git.pull().call(); git.pull().call();
if (!localRepo.getBranch().equals(branchName)) { if (!localRepo.getBranch().equals(branchName)) {
@ -192,7 +191,8 @@ public class GitRepositoryService {
private void cloneOrUpdateRepo(String repositoryUrl, String branchName) throws GitAPIException, IOException { private void cloneOrUpdateRepo(String repositoryUrl, String branchName) throws GitAPIException, IOException {
Git git; Git git;
Repository localRepo; org.eclipse.jgit.lib.Repository localRepo;
validateOrClearRepositoryDirectory(getProjectDirectoryFile(repositoryUrl));
if (projectDirExists(getProjectDirectoryFile(repositoryUrl))) { if (projectDirExists(getProjectDirectoryFile(repositoryUrl))) {
localRepo = new FileRepository(getProjectGitDirectory(repositoryUrl)); localRepo = new FileRepository(getProjectGitDirectory(repositoryUrl));
git = new Git(localRepo); git = new Git(localRepo);
@ -211,6 +211,26 @@ public class GitRepositoryService {
git.close(); git.close();
} }
private void validateOrClearRepositoryDirectory(File projectDirectory) {
if (projectDirectory.exists()) {
try {
Git.open(projectDirectory).status().call();
} catch (Exception e) {
deleteDirectory(projectDirectory);
}
}
}
private boolean deleteDirectory(File directoryToBeDeleted) {
File[] allContents = directoryToBeDeleted.listFiles();
if (allContents != null) {
for (File file : allContents) {
deleteDirectory(file);
}
}
return directoryToBeDeleted.delete();
}
private void cloneOrUpdateRepo(String url) throws GitAPIException, IOException { private void cloneOrUpdateRepo(String url) throws GitAPIException, IOException {
cloneOrUpdateRepo(url, null); cloneOrUpdateRepo(url, null);
} }
@ -240,7 +260,7 @@ public class GitRepositoryService {
return file.exists(); return file.exists();
} }
private List<FileChange> findDiffBetweenTwoRevisions(RevCommit laterCommit, RevCommit earlierCommit, Repository localRepo) { private List<FileChange> findDiffBetweenTwoRevisions(RevCommit laterCommit, RevCommit earlierCommit, org.eclipse.jgit.lib.Repository localRepo) {
if (laterCommit == null || earlierCommit == null) { if (laterCommit == null || earlierCommit == null) {
return null; return null;
} }
@ -256,7 +276,7 @@ public class GitRepositoryService {
return parseOutputDiff(output, localRepo, laterCommit); return parseOutputDiff(output, localRepo, laterCommit);
} }
private List<FileChange> parseOutputDiff(String output, Repository repository, RevCommit commit) { private List<FileChange> parseOutputDiff(String output, org.eclipse.jgit.lib.Repository repository, RevCommit commit) {
List<FileChange> changes = new ArrayList<>(); List<FileChange> changes = new ArrayList<>();
String[] strings = output.split("\n"); String[] strings = output.split("\n");
FileChange fileChange = new FileChange(); FileChange fileChange = new FileChange();
@ -322,7 +342,7 @@ public class GitRepositoryService {
return changes; return changes;
} }
private String getContent(Repository repository, RevCommit commit, String path) { private String getContent(org.eclipse.jgit.lib.Repository repository, RevCommit commit, String path) {
try (TreeWalk treeWalk = TreeWalk.forPath(repository, path, commit.getTree())) { try (TreeWalk treeWalk = TreeWalk.forPath(repository, path, commit.getTree())) {
if (treeWalk != null) { if (treeWalk != null) {
ObjectId blobId = treeWalk.getObjectId(0); ObjectId blobId = treeWalk.getObjectId(0);
@ -346,4 +366,8 @@ public class GitRepositoryService {
} }
return Optional.empty(); return Optional.empty();
} }
public Page<GitRepository> findAll(Pageable pageable) {
return gitRepositoryRepository.findAll(pageable);
}
} }

10
src/main/java/ru/ulstu/extractor/model/Repository.java → src/main/java/ru/ulstu/extractor/gitrepository/model/GitRepository.java
View File

@ -1,15 +1,17 @@
package ru.ulstu.extractor.model; package ru.ulstu.extractor.gitrepository.model;
import ru.ulstu.extractor.model.BaseEntity;
import javax.persistence.Entity; import javax.persistence.Entity;
@Entity @Entity
public class Repository extends BaseEntity { public class GitRepository extends BaseEntity {
private String url; private String url;
public Repository() { public GitRepository() {
} }
public Repository(String repositoryUrl) { public GitRepository(String repositoryUrl) {
url = repositoryUrl; url = repositoryUrl;
} }

2
src/main/java/ru/ulstu/extractor/heuristic/controller/StructuralUnitController.java
View File

@ -11,10 +11,10 @@ import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import ru.ulstu.extractor.gitrepository.GitRepositoryService;
import ru.ulstu.extractor.heuristic.model.BusinessLogicUnit; import ru.ulstu.extractor.heuristic.model.BusinessLogicUnit;
import ru.ulstu.extractor.heuristic.model.EntityUnit; import ru.ulstu.extractor.heuristic.model.EntityUnit;
import ru.ulstu.extractor.heuristic.model.ResourceUnit; import ru.ulstu.extractor.heuristic.model.ResourceUnit;
import ru.ulstu.extractor.service.GitRepositoryService;
import ru.ulstu.extractor.util.HttpUtils; import ru.ulstu.extractor.util.HttpUtils;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;

36
src/main/java/ru/ulstu/extractor/http/HttpService.java Normal file
View File

@ -0,0 +1,36 @@
package ru.ulstu.extractor.http;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.BodyInserters;
import org.springframework.web.reactive.function.client.WebClient;
import java.util.Optional;
@Service
public class HttpService {
private final Logger log = LoggerFactory.getLogger(HttpService.class);
private final WebClient client;
public HttpService(WebClient client) {
this.client = client;
}
public JSONObject post(String url, JSONObject postData) {
log.debug("Service call: {}", url);
JSONObject response = new JSONObject(Optional.ofNullable(client
.post()
.uri(url)
.contentType(MediaType.APPLICATION_JSON)
.body(BodyInserters.fromValue(postData.toString()))
.accept(MediaType.APPLICATION_JSON)
.retrieve()
.bodyToMono(String.class)
.block()).orElse("{response:\"empty\"}"));
log.debug("Service response: {}", response);
return response;
}
}

35
src/main/java/ru/ulstu/extractor/http/JsonTimeSeries.java Normal file
View File

@ -0,0 +1,35 @@
package ru.ulstu.extractor.http;
import ru.ulstu.extractor.model.TimeSeries;
import java.util.List;
import java.util.stream.Collectors;
public class JsonTimeSeries {
private String name;
private List<JsonTimeSeriesValue> values;
public JsonTimeSeries(TimeSeries timeSeries) {
this.name = timeSeries.getName();
this.values = timeSeries.getValues()
.stream()
.map(JsonTimeSeriesValue::new)
.collect(Collectors.toList());
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public List<JsonTimeSeriesValue> getValues() {
return values;
}
public void setValues(List<JsonTimeSeriesValue> values) {
this.values = values;
}
}

35
src/main/java/ru/ulstu/extractor/http/JsonTimeSeriesValue.java Normal file
View File

@ -0,0 +1,35 @@
package ru.ulstu.extractor.http;
import ru.ulstu.extractor.model.TimeSeriesValue;
import java.time.LocalDateTime;
import java.time.ZoneId;
public class JsonTimeSeriesValue {
private LocalDateTime date;
private Double value;
public JsonTimeSeriesValue(TimeSeriesValue timeSeriesValue) {
this.value = timeSeriesValue.getValue();
this.date = timeSeriesValue.getDate()
.toInstant()
.atZone(ZoneId.systemDefault())
.toLocalDateTime();
}
public LocalDateTime getDate() {
return date;
}
public void setDate(LocalDateTime date) {
this.date = date;
}
public Double getValue() {
return value;
}
public void setValue(Double value) {
this.value = value;
}
}

50
src/main/java/ru/ulstu/extractor/model/TimeSeries.java Normal file
View File

@ -0,0 +1,50 @@
package ru.ulstu.extractor.model;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.CascadeType;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.OneToMany;
import java.util.ArrayList;
import java.util.List;
@Entity
public class TimeSeries extends BaseEntity {
private String name;
@OneToMany(cascade = CascadeType.ALL, fetch = FetchType.LAZY)
@JoinColumn(name = "time_series_id")
@Fetch(FetchMode.SUBSELECT)
private List<TimeSeriesValue> values = new ArrayList<>();
public TimeSeries() {
}
public TimeSeries(String name) {
this.name = name;
}
public TimeSeries(String name, List<TimeSeriesValue> values) {
this.name = name;
this.values = values;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public List<TimeSeriesValue> getValues() {
return values;
}
public void setValues(List<TimeSeriesValue> values) {
this.values = values;
}
}

39
src/main/java/ru/ulstu/extractor/model/TimeSeriesValue.java Normal file
View File

@ -0,0 +1,39 @@
package ru.ulstu.extractor.model;
import javax.persistence.Entity;
import java.util.Date;
@Entity
public class TimeSeriesValue extends BaseEntity {
private Date date;
private Double value;
public TimeSeriesValue() {
}
public TimeSeriesValue(Date date, Double value) {
this.date = date;
this.value = value;
}
public TimeSeriesValue(TimeSeries timeSeries, Date date, Double value) {
this.date = date;
this.value = value;
}
public Date getDate() {
return date;
}
public Double getValue() {
return value;
}
public void setDate(Date date) {
this.date = date;
}
public void setValue(Double value) {
this.value = value;
}
}

6
src/main/java/ru/ulstu/extractor/repository/AuthorRepository.java
View File

@ -8,14 +8,14 @@ package ru.ulstu.extractor.repository;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query; import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param; import org.springframework.data.repository.query.Param;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
import ru.ulstu.extractor.model.Author; import ru.ulstu.extractor.model.Author;
import ru.ulstu.extractor.model.Repository;
import java.util.List; import java.util.List;
public interface AuthorRepository extends JpaRepository<Author, Integer> { public interface AuthorRepository extends JpaRepository<Author, Integer> {
@Query("SELECT DISTINCT a.name FROM Commit c, Repository r, Branch b, Author a WHERE c.author = a AND c.branch = b AND r = b.repository AND (:repository IS NULL OR r = :repository) AND (:branchName IS NULL OR :branchName = '' OR b.name = :branchName) AND a.name IS NOT NULL AND a.name <> '' ORDER BY a.name") @Query("SELECT DISTINCT a.name FROM Commit c, GitRepository r, Branch b, Author a WHERE c.author = a AND c.branch = b AND r = b.gitRepository AND (:repository IS NULL OR r = :repository) AND (:branchName IS NULL OR :branchName = '' OR b.name = :branchName) AND a.name IS NOT NULL AND a.name <> '' ORDER BY a.name")
List<String> findByRepositoryAndBranch(@Param("repository") Repository repository, @Param("branchName") String branchName); List<String> findByRepositoryAndBranch(@Param("repository") GitRepository gitRepository, @Param("branchName") String branchName);
List<Author> findByName(String name); List<Author> findByName(String name);

13
src/main/java/ru/ulstu/extractor/repository/BranchRepository.java
View File

@ -1,13 +0,0 @@
package ru.ulstu.extractor.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import ru.ulstu.extractor.model.Branch;
import ru.ulstu.extractor.model.Repository;
import java.util.List;
public interface BranchRepository extends JpaRepository<Branch, Integer> {
Branch findByRepositoryAndName(Repository repository, String name);
List<Branch> findByRepositoryId(Integer repositoryId);
}

42
src/main/java/ru/ulstu/extractor/repository/CommitRepository.java
View File

@ -1,42 +0,0 @@
/*
* Copyright (C) 2021 Anton Romanov - All Rights Reserved
* You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/
package ru.ulstu.extractor.repository;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.model.CommitAuthorStatistic;
import ru.ulstu.extractor.model.CommitEntityStatistic;
import ru.ulstu.extractor.model.CommitTimeStatistic;
import ru.ulstu.extractor.model.CommitUrlStatistic;
import ru.ulstu.extractor.model.Repository;
import java.util.List;
public interface CommitRepository extends JpaRepository<Commit, Integer> {
@Query("SELECT DISTINCT c FROM Commit c LEFT JOIN c.branch b LEFT JOIN c.fileChanges f LEFT JOIN c.author a LEFT JOIN b.repository r WHERE r = :repository AND b.name = :branchName AND (:author IS NULL OR :author = '' OR a.name = :author) AND (:filter IS NULL OR :filter = '' OR lower(c.message) LIKE lower(concat('%', :filter,'%'))) AND (:entity IS NULL OR f.containsEntity = :entity)")
Page<Commit> findByRepositoryAndBranch(Pageable pageable, @Param("repository") Repository repository, @Param("branchName") String branchName, @Param("author") String author, @Param("filter") String filter, @Param("entity") Boolean entity);
@Query("SELECT new ru.ulstu.extractor.model.CommitAuthorStatistic(c.author.name, COUNT(DISTINCT c.hash)) FROM Commit c LEFT JOIN c.branch LEFT JOIN c.author a WHERE (:branchId IS NULL OR c.branch.id = :branchId) AND (:author IS NULL OR :author = '' OR a.name = :author) GROUP by c.author.name")
List<CommitAuthorStatistic> getCommitAuthorStatistic(@Param("branchId") Integer branchId, @Param("author") String author);
@Query("SELECT new ru.ulstu.extractor.model.CommitUrlStatistic(c.branch.repository.url, COUNT(DISTINCT c.hash)) FROM Commit c GROUP by c.branch.repository.url")
List<CommitUrlStatistic> getCommitUrlStatistic();
@Query("SELECT new ru.ulstu.extractor.model.CommitTimeStatistic(extract(month FROM c.date) as month, extract(year FROM c.date) as year, COUNT(DISTINCT c.hash)) FROM Commit c LEFT JOIN c.branch LEFT JOIN c.author a WHERE (:branchId IS NULL OR c.branch.id = :branchId) AND (:author IS NULL OR :author = '' OR a.name = :author) GROUP by extract(month from c.date), extract(year from c.date) ORDER by extract(year from c.date), extract(month from c.date)")
List<CommitTimeStatistic> getCommitTimeStatistic(@Param("branchId") Integer branchId, @Param("author") String author);
@Query("SELECT new ru.ulstu.extractor.model.CommitEntityStatistic(f.containsEntity, COUNT(DISTINCT c.hash)) FROM Commit c LEFT JOIN c.branch LEFT JOIN c.author a LEFT JOIN c.fileChanges f WHERE (:branchId IS NULL OR c.branch.id = :branchId) AND (:author IS NULL OR :author = '' OR a.name = :author) GROUP by f.containsEntity")
List<CommitEntityStatistic> getCommitEntityStatistic(@Param("branchId") Integer branchId, @Param("author") String author);
@Query("SELECT new ru.ulstu.extractor.model.CommitTimeStatistic(EXTRACT(MONTH FROM c.date), EXTRACT(YEAR FROM c.date), COUNT(DISTINCT c.hash)) FROM Commit c LEFT JOIN c.branch LEFT JOIN c.author a LEFT JOIN c.fileChanges f WHERE f.containsEntity = true AND (:branchId IS NULL OR c.branch.id = :branchId) AND (:author IS NULL OR :author = '' OR a.name = :author) GROUP by extract(month from c.date), extract(year from c.date) ORDER by extract(year from c.date), extract(month from c.date)")
List<CommitTimeStatistic> getCommitTimeEntityStatistic(@Param("branchId") Integer branchId, @Param("author") String author);
void deleteByBranchIsNull();
}

8
src/main/java/ru/ulstu/extractor/repository/RepositoryRepository.java
View File

@ -1,8 +0,0 @@
package ru.ulstu.extractor.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import ru.ulstu.extractor.model.Repository;
public interface RepositoryRepository extends JpaRepository<Repository, Integer> {
Repository findByUrl(String url);
}

10
src/main/java/ru/ulstu/extractor/repository/TimeSeriesRepository.java Normal file
View File

@ -0,0 +1,10 @@
package ru.ulstu.extractor.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import ru.ulstu.extractor.model.TimeSeries;
import java.util.Optional;
public interface TimeSeriesRepository extends JpaRepository<TimeSeries, Integer> {
Optional<TimeSeries> findByName(String name);
}

14
src/main/java/ru/ulstu/extractor/repository/TimeSeriesValueRepository.java Normal file
View File

@ -0,0 +1,14 @@
package ru.ulstu.extractor.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import ru.ulstu.extractor.model.TimeSeriesValue;
import java.util.List;
public interface TimeSeriesValueRepository extends JpaRepository<TimeSeriesValue, Integer> {
//TimeSeriesValue findByTimeSeriesAndName(TimeSeries timeSeries, String name);
//List<TimeSeriesValue> findTimeSeriesValueById(Integer repositoryId);
void deleteAllByIdIn(List<Integer> ids);
}

14
src/main/java/ru/ulstu/extractor/service/FilteringService.java
View File

@ -9,10 +9,10 @@ import com.sun.istack.NotNull;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import ru.ulstu.extractor.commit.repository.CommitRepository;
import ru.ulstu.extractor.gitrepository.GitRepositoryRepository;
import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.repository.AuthorRepository; import ru.ulstu.extractor.repository.AuthorRepository;
import ru.ulstu.extractor.repository.CommitRepository;
import ru.ulstu.extractor.repository.RepositoryRepository;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -21,20 +21,20 @@ import java.util.Map;
public class FilteringService { public class FilteringService {
private final AuthorRepository authorRepository; private final AuthorRepository authorRepository;
private final CommitRepository commitRepository; private final CommitRepository commitRepository;
private final RepositoryRepository repositoryRepository; private final GitRepositoryRepository gitRepositoryRepository;
public FilteringService(AuthorRepository authorRepository, public FilteringService(AuthorRepository authorRepository,
CommitRepository commitRepository, CommitRepository commitRepository,
RepositoryRepository repositoryRepository) { GitRepositoryRepository gitRepositoryRepository) {
this.authorRepository = authorRepository; this.authorRepository = authorRepository;
this.commitRepository = commitRepository; this.commitRepository = commitRepository;
this.repositoryRepository = repositoryRepository; this.gitRepositoryRepository = gitRepositoryRepository;
} }
public List<String> getRepositoryAuthors(@NotNull String repositoryUrl, public List<String> getRepositoryAuthors(@NotNull String repositoryUrl,
@NotNull String branchName) { @NotNull String branchName) {
return authorRepository.findByRepositoryAndBranch( return authorRepository.findByRepositoryAndBranch(
repositoryRepository.findByUrl(repositoryUrl), gitRepositoryRepository.findByUrl(repositoryUrl),
branchName branchName
); );
} }
@ -51,7 +51,7 @@ public class FilteringService {
Pageable pageable) { Pageable pageable) {
return commitRepository.findByRepositoryAndBranch( return commitRepository.findByRepositoryAndBranch(
pageable, pageable,
repositoryRepository.findByUrl(repositoryUrl), gitRepositoryRepository.findByUrl(repositoryUrl),
branchName, branchName,
author, author,
filter, filter,

32
src/main/java/ru/ulstu/extractor/service/IndexService.java
View File

@ -10,10 +10,13 @@ import org.eclipse.jgit.api.errors.GitAPIException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import ru.ulstu.extractor.model.Branch; import ru.ulstu.extractor.branch.model.Branch;
import ru.ulstu.extractor.branch.service.BranchService;
import ru.ulstu.extractor.gitrepository.GitRepositoryRepository;
import ru.ulstu.extractor.gitrepository.GitRepositoryService;
import ru.ulstu.extractor.gitrepository.model.GitRepository;
import ru.ulstu.extractor.model.Commit; import ru.ulstu.extractor.model.Commit;
import ru.ulstu.extractor.model.Repository; import ru.ulstu.extractor.ts.AbstractTimeSeriesCreator;
import ru.ulstu.extractor.repository.RepositoryRepository;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
@ -24,25 +27,28 @@ public class IndexService {
private final static Logger LOG = LoggerFactory.getLogger(IndexService.class); private final static Logger LOG = LoggerFactory.getLogger(IndexService.class);
private final static int COMMITS_PAGE_SIZE = 10; private final static int COMMITS_PAGE_SIZE = 10;
private final GitRepositoryService gitRepositoryService; private final GitRepositoryService gitRepositoryService;
private final RepositoryRepository repositoryRepository; private final GitRepositoryRepository gitRepositoryRepository;
private final BranchService branchService; private final BranchService branchService;
private final List<AbstractTimeSeriesCreator> timeSeriesCreators;
public IndexService(GitRepositoryService gitRepositoryService, public IndexService(GitRepositoryService gitRepositoryService,
RepositoryRepository repositoryRepository, GitRepositoryRepository gitRepositoryRepository,
BranchService branchService) { BranchService branchService,
List<AbstractTimeSeriesCreator> timeSeriesCreators) {
this.gitRepositoryService = gitRepositoryService; this.gitRepositoryService = gitRepositoryService;
this.repositoryRepository = repositoryRepository; this.gitRepositoryRepository = gitRepositoryRepository;
this.branchService = branchService; this.branchService = branchService;
this.timeSeriesCreators = timeSeriesCreators;
} }
public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException { public void index(@NotNull String repositoryUrl, @NotNull String branchName) throws GitAPIException, IOException {
Repository repository = repositoryRepository.findByUrl(repositoryUrl); GitRepository gitRepository = gitRepositoryRepository.findByUrl(repositoryUrl);
if (repository == null) { if (gitRepository == null) {
repository = repositoryRepository.save(new Repository(repositoryUrl)); gitRepository = gitRepositoryRepository.save(new GitRepository(repositoryUrl));
} }
Branch branch = branchService.findByRepositoryAndName(repository, branchName); Branch branch = branchService.findByRepositoryAndName(gitRepository, branchName);
if (branch == null) { if (branch == null) {
branch = new Branch(repository, branchName); branch = new Branch(gitRepository, branchName);
} }
branchService.save(branch, Collections.emptyList()); branchService.save(branch, Collections.emptyList());
int commitsFrom = 0; int commitsFrom = 0;
@ -56,6 +62,8 @@ public class IndexService {
commitsTo += COMMITS_PAGE_SIZE; commitsTo += COMMITS_PAGE_SIZE;
commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false); commits = gitRepositoryService.getCommits(repositoryUrl, branchName, commitsFrom, commitsTo, false);
} }
Integer repositoryId = gitRepository.getId();
timeSeriesCreators.forEach(tsCreator -> tsCreator.addTimeSeries(repositoryId, branchName));
LOG.debug("Complete indexing {} branch", branchName); LOG.debug("Complete indexing {} branch", branchName);
} }
} }

116
src/main/java/ru/ulstu/extractor/service/TimeSeriesService.java Normal file
View File

@ -0,0 +1,116 @@
/*
* Copyright (C) 2021 Anton Romanov - All Rights Reserved
* You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/
package ru.ulstu.extractor.service;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import ru.ulstu.extractor.http.HttpService;
import ru.ulstu.extractor.http.JsonTimeSeries;
import ru.ulstu.extractor.model.TimeSeries;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.repository.TimeSeriesRepository;
import ru.ulstu.extractor.repository.TimeSeriesValueRepository;
import ru.ulstu.extractor.ts.TimeSeriesDateMapper;
import javax.transaction.Transactional;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@Service
public class TimeSeriesService {
private final static Logger LOG = LoggerFactory.getLogger(TimeSeriesService.class);
private final TimeSeriesRepository timeSeriesRepository;
private final TimeSeriesValueRepository timeSeriesValueRepository;
private final TimeSeriesDateMapper.TimeSeriesInterval timeSeriesInterval = TimeSeriesDateMapper.TimeSeriesInterval.HOUR;
private final HttpService httpService;
private final static String TIME_SERIES_SERVICE_URL = "http://time-series.athene.tech/api/1.0/add-time-series?setKey=git-extractor";
public TimeSeriesService(TimeSeriesRepository timeSeriesRepository,
TimeSeriesValueRepository timeSeriesValueRepository,
HttpService httpService) {
this.timeSeriesRepository = timeSeriesRepository;
this.timeSeriesValueRepository = timeSeriesValueRepository;
this.httpService = httpService;
}
/**
* Сохранить список временных рядов
*
* @param timeSeriesValues
* @return
*/
@Transactional
public List<TimeSeries> save(Map<String, List<TimeSeriesValue>> timeSeriesValues) {
List<TimeSeries> results = new ArrayList<>();
for (Map.Entry<String, List<TimeSeriesValue>> entry : timeSeriesValues.entrySet()) {
results.add(save(entry.getKey(), entry.getValue()));
}
return results;
}
@Transactional
public TimeSeries save(String timeSeriesName, List<TimeSeriesValue> timeSeriesValues) {
LOG.debug("Start save {} time series with {} time series values ", timeSeriesName, timeSeriesValues.size());
final TimeSeries timeSeries = findOrCreate(timeSeriesName);
List<TimeSeriesValue> timeSeriesValuesToRemove = timeSeries.getValues();
timeSeries.setValues(timeSeriesValues);
LOG.debug("Save time series {} ", timeSeries.getName());
TimeSeries savedTimeSeries = timeSeriesRepository.save(timeSeries);
LOG.debug("Clear {} time series values ", timeSeriesValuesToRemove.size());
timeSeriesValueRepository.deleteAll(timeSeriesValuesToRemove);
sendToTimeSeriesService(savedTimeSeries);
return savedTimeSeries;
}
public TimeSeries findOrCreate(String timeSeriesName) {
Optional<TimeSeries> maybeTimeSeries = timeSeriesRepository.findByName(timeSeriesName);
if (maybeTimeSeries.isPresent()) {
LOG.debug("TimeSeries {} exists.", maybeTimeSeries.get().getName());
return maybeTimeSeries.get();
}
return timeSeriesRepository.save(new TimeSeries(timeSeriesName));
}
public List<TimeSeriesValue> save(List<TimeSeriesValue> timeSeriesValues) {
return timeSeriesValues.stream()
.map(timeSeriesValue -> {
timeSeriesValue.setValue((timeSeriesValue.getValue()));
timeSeriesValue.setDate((timeSeriesValue.getDate()));
return timeSeriesValueRepository.save(timeSeriesValue);
}).collect(Collectors.toList());
}
public void addTimeSeriesValue(String timeSeriesName, Date date, Double value) {
LOG.debug("Start add time series values to {} time series values ", timeSeriesName);
TimeSeries timeSeries = findOrCreate(timeSeriesName);
timeSeriesValueRepository.save(new TimeSeriesValue(timeSeries, date, value));
}
public List<TimeSeriesValue> findAll() {
return timeSeriesValueRepository.findAll();
}
public TimeSeriesDateMapper.TimeSeriesInterval getTimeSeriesInterval() {
return timeSeriesInterval;
}
private void sendToTimeSeriesService(TimeSeries timeSeries) {
new Thread(() -> {
try {
httpService.post(TIME_SERIES_SERVICE_URL, new JSONObject(new JsonTimeSeries(timeSeries)));
LOG.debug("Успешно отправлен на сервис");
} catch (Exception ex) {
LOG.debug(ex.getMessage());
}
}).start();
}
}

40
src/main/java/ru/ulstu/extractor/ts/AbstractTimeSeriesCreator.java Normal file
View File

@ -0,0 +1,40 @@
package ru.ulstu.extractor.ts;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.List;
import java.util.Map;
import static ru.ulstu.extractor.ts.TimeSeriesDateMapper.mapTimeSeriesToInterval;
public abstract class AbstractTimeSeriesCreator {
public abstract String getTimeSeriesName();
/**
* Извлечь список точек временных рядов
*
* @param repositoryId
* @param branchName
* @return
*/
public abstract Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName);
public abstract TimeSeriesService getTimeSeriesService();
/**
* Сохранить извлеченные временные ряды
*
* @param repositoryId
* @param branchName
*/
public void addTimeSeries(Integer repositoryId, String branchName) {
// извлеченные точки временных рядов
Map<String, List<TimeSeriesValue>> timeSeriesValues = getTimeSeriesValues(repositoryId, branchName);
// сгруппированные по временным интервалам точки временных рядов
timeSeriesValues.replaceAll((k, v) -> mapTimeSeriesToInterval(getTimeSeriesService().getTimeSeriesInterval(), v));
getTimeSeriesService().save(timeSeriesValues);
}
}

33
src/main/java/ru/ulstu/extractor/ts/AuthorTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class AuthorTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public AuthorTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество Авторов";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/AuthorsCommentTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class AuthorsCommentTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public AuthorsCommentTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество коммитов авторов";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/AuthorsCompletedIssueTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class AuthorsCompletedIssueTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public AuthorsCompletedIssueTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество выполненных issues авторов";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/AuthorsIssueTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class AuthorsIssueTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public AuthorsIssueTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество issues авторов";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/BranchTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class BranchTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public BranchTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество веток";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/ClassTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class ClassTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public ClassTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество классов";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

45
src/main/java/ru/ulstu/extractor/ts/CommitsTS.java Normal file
View File

@ -0,0 +1,45 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.commit.service.CommitService;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Component
public class CommitsTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
private final CommitService commitService;
public CommitsTS(TimeSeriesService timeSeriesService,
CommitService commitService) {
this.timeSeriesService = timeSeriesService;
this.commitService = commitService;
}
@Override
public String getTimeSeriesName() {
return "Количество коммитов во времени";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
//TODO: добавить постраничное чтение
Map<String, List<TimeSeriesValue>> result = new HashMap<>();
result.put(String.format("%s %s %s", getTimeSeriesName(), repositoryId, branchName),
commitService.findByRepositoryIdAndName(repositoryId, branchName)
.stream()
.map(c -> new TimeSeriesValue(c.getDate(), 1.0))
.collect(Collectors.toList()));
return result;
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/DependenceTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class DependenceTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public DependenceTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество зависимостей";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/EntityTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class EntityTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public EntityTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество сущностей";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/FileTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class FileTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public FileTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество файлов";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/InterfaceTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class InterfaceTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public InterfaceTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество интерфейсов";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/IssuesTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class IssuesTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public IssuesTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество issues созданных во времени";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/ProcessTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class ProcessTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public ProcessTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество процессов";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

33
src/main/java/ru/ulstu/extractor/ts/StarTS.java Normal file
View File

@ -0,0 +1,33 @@
package ru.ulstu.extractor.ts;
import org.springframework.stereotype.Component;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.service.TimeSeriesService;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class StarTS extends AbstractTimeSeriesCreator {
private final TimeSeriesService timeSeriesService;
public StarTS(TimeSeriesService timeSeriesService) {
this.timeSeriesService = timeSeriesService;
}
@Override
public String getTimeSeriesName() {
return "Количество звезд";
}
@Override
public Map<String, List<TimeSeriesValue>> getTimeSeriesValues(Integer repositoryId, String branchName) {
return new HashMap<>();
}
@Override
public TimeSeriesService getTimeSeriesService() {
return timeSeriesService;
}
}

64
src/main/java/ru/ulstu/extractor/ts/TimeSeriesDateMapper.java Normal file
View File

@ -0,0 +1,64 @@
package ru.ulstu.extractor.ts;
import org.apache.commons.lang3.time.DateUtils;
import ru.ulstu.extractor.model.TimeSeries;
import ru.ulstu.extractor.model.TimeSeriesValue;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Класс для регулировки дискретности временного ряда.
* Можно создать временной ряд с точками с секундными отметками, затем суммировать значения,
* применив одно из значений TimeSeriesInterval
*/
public class TimeSeriesDateMapper {
public static List<TimeSeriesValue> mapTimeSeriesToInterval(TimeSeriesInterval timeSeriesInterval, List<TimeSeriesValue> timeSeriesValues) {
List<TimeSeriesValue> trimmedTimeSeriesValues = timeSeriesValues
.stream()
.map(timeSeriesValue -> new TimeSeriesValue(trimTo(timeSeriesInterval, timeSeriesValue.getDate()),
timeSeriesValue.getValue()))
.collect(Collectors.toList());
Map<Date, Double> groupedTimeSeriesValues = trimmedTimeSeriesValues
.stream()
.collect(Collectors.groupingBy(TimeSeriesValue::getDate,
Collectors.summingDouble(TimeSeriesValue::getValue)));
return groupedTimeSeriesValues.entrySet()
.stream()
.map(e -> new TimeSeriesValue(e.getKey(), e.getValue()))
.collect(Collectors.toList());
}
public static TimeSeries mapTimeSeriesToInterval(TimeSeriesInterval timeSeriesInterval, TimeSeries timeSeries) {
timeSeries.setValues(mapTimeSeriesToInterval(timeSeriesInterval, timeSeries.getValues()));
return timeSeries;
}
private static Date trimTo(TimeSeriesInterval timeSeriesInterval, Date date) {
return DateUtils.truncate(date, timeSeriesInterval.calendarField);
}
/**
* Интервальность временного ряда при преобразовании
*/
public enum TimeSeriesInterval {
SECOND(Calendar.SECOND),
MINUTE(Calendar.MINUTE),
HOUR(Calendar.HOUR_OF_DAY),
DAY(Calendar.DAY_OF_MONTH),
WEEK(Calendar.WEEK_OF_MONTH),
MONTH(Calendar.MONTH),
YEAR(Calendar.YEAR);
private final int calendarField;
TimeSeriesInterval(int calendarField) {
this.calendarField = calendarField;
}
}
}

44
src/main/resources/db/changelog-20220422_120000-schema.xml Normal file
View File

@ -0,0 +1,44 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd">
<changeSet author="orion" id="20220422-120000-1">
<createTable tableName="time_series">
<column name="id" type="integer">
<constraints nullable="false"/>
</column>
<column name="name" type="varchar(255)">
<constraints nullable="false"/>
</column>
</createTable>
<addPrimaryKey columnNames="id" constraintName="pk_time_series" tableName="time_series"/>
<createTable tableName="time_series_value">
<column name="id" type="integer">
<constraints nullable="false"/>
</column>
<column name="time_series_id" type="integer">
<constraints nullable="false"/>
</column>
<column name="date" type="timestamp">
<constraints nullable="false"/>
</column>
<column name="value" type="integer">
<constraints nullable="false"/>
</column>
</createTable>
<addPrimaryKey columnNames="id" constraintName="pk_time_series_value" tableName="time_series_value"/>
<addForeignKeyConstraint baseTableName="time_series_value" baseColumnNames="time_series_id"
constraintName="fk_time_series"
referencedTableName="time_series"
referencedColumnNames="id"/>
</changeSet>
<changeSet author="orion" id="20220422-120000-2">
<addColumn tableName="time_series">
<column name="version" type="integer"/>
</addColumn>
<addColumn tableName="time_series_value">
<column name="version" type="integer"/>
</addColumn>
</changeSet>
</databaseChangeLog>

23
src/main/resources/db/changelog-20220621_120000-schema.xml Normal file
View File

@ -0,0 +1,23 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd">
<changeSet author="orion" id="20220621-120000-1">
<renameColumn tableName="branch" oldColumnName="repository_id" newColumnName="git_repository_id"/>
</changeSet>
<changeSet author="orion" id="20220621-120000-2">
<renameTable oldTableName="repository" newTableName="git_repository"/>
</changeSet>
<changeSet author="orion" id="20221006-120000-1">
<dropNotNullConstraint tableName="time_series_value" columnName="time_series_id"/>
</changeSet>
<changeSet author="orion" id="20221006-120000-2">
<delete tableName="time_series_value"/>
<dropColumn tableName="time_series_value" columnName="value"/>
<addColumn tableName="time_series_value">
<column name="value" type="double">
<constraints nullable="false"/>
</column>
</addColumn>
</changeSet>
</databaseChangeLog>

2
src/main/resources/db/changelog-master.xml
View File

@ -12,4 +12,6 @@
<include file="db/changelog-20210326_170000-schema.xml"/> <include file="db/changelog-20210326_170000-schema.xml"/>
<include file="db/changelog-20210329_120000-schema.xml"/> <include file="db/changelog-20210329_120000-schema.xml"/>
<include file="db/changelog-20210412_100000-schema.xml"/> <include file="db/changelog-20210412_100000-schema.xml"/>
<include file="db/changelog-20220422_120000-schema.xml"/>
<include file="db/changelog-20220621_120000-schema.xml"/>
</databaseChangeLog> </databaseChangeLog>

2
src/main/resources/templates/statistic.html
View File

@ -193,7 +193,7 @@
<option value="">Все ветки</option> <option value="">Все ветки</option>
<option th:each="branch : ${branches}" <option th:each="branch : ${branches}"
th:value="${branch.id}" th:value="${branch.id}"
th:utext="${branch.repository.url} + ' - '+ ${branch.name}"> th:utext="${branch.gitRepository.url} + ' - '+ ${branch.name}">
</option> </option>
</select> </select>
<script th:inline="javascript"> <script th:inline="javascript">

103
src/test/java/ru/ulstu/TimeSeriesMapperTest.java Normal file
View File

@ -0,0 +1,103 @@
package ru.ulstu;
import org.junit.Assert;
import org.junit.Test;
import ru.ulstu.extractor.model.TimeSeries;
import ru.ulstu.extractor.model.TimeSeriesValue;
import ru.ulstu.extractor.ts.TimeSeriesDateMapper;
import java.util.Arrays;
import java.util.Calendar;
import java.util.GregorianCalendar;
public class TimeSeriesMapperTest {
@Test
public void testMappingByDay() {
Calendar c1 = GregorianCalendar.getInstance();
c1.set(2020, 5, 1, 1, 1, 1);
Calendar c2 = GregorianCalendar.getInstance();
c2.set(2020, 5, 2, 2, 1, 1);
TimeSeries timeSeries = new TimeSeries("Тестовый",
Arrays.asList(new TimeSeriesValue(c1.getTime(), 10),
new TimeSeriesValue(c2.getTime(), 10)));
TimeSeriesDateMapper mapper = new TimeSeriesDateMapper();
timeSeries = mapper.mapTimeSeriesToInterval(TimeSeriesDateMapper.TimeSeriesInterval.MONTH, timeSeries);
Assert.assertEquals(1, timeSeries.getValues().size());
Assert.assertEquals(Integer.valueOf(20), timeSeries.getValues().get(0).getValue());
}
@Test
public void testMappingByDayDifferent() {
Calendar c1 = GregorianCalendar.getInstance();
c1.set(2020, 5, 1, 1, 1, 1);
Calendar c2 = GregorianCalendar.getInstance();
c2.set(2020, 5, 2, 1, 1, 1);
TimeSeries timeSeries = new TimeSeries("Тестовый",
Arrays.asList(new TimeSeriesValue(c1.getTime(), 10),
new TimeSeriesValue(c2.getTime(), 10)));
TimeSeriesDateMapper mapper = new TimeSeriesDateMapper();
timeSeries = mapper.mapTimeSeriesToInterval(TimeSeriesDateMapper.TimeSeriesInterval.MONTH, timeSeries);
Assert.assertEquals(1, timeSeries.getValues().size());
Assert.assertEquals(Integer.valueOf(20), timeSeries.getValues().get(0).getValue());
}
@Test
public void testMappingByMonth() {
Calendar c1 = GregorianCalendar.getInstance();
c1.set(2020, 5, 1, 1, 1, 1);
Calendar c2 = GregorianCalendar.getInstance();
c2.set(2020, 5, 2, 1, 1, 1);
TimeSeries timeSeries = new TimeSeries("Тестовый",
Arrays.asList(new TimeSeriesValue(c1.getTime(), 10),
new TimeSeriesValue(c2.getTime(), 10)));
TimeSeriesDateMapper mapper = new TimeSeriesDateMapper();
timeSeries = mapper.mapTimeSeriesToInterval(TimeSeriesDateMapper.TimeSeriesInterval.MONTH, timeSeries);
Assert.assertEquals(1, timeSeries.getValues().size());
Assert.assertEquals(Integer.valueOf(20), timeSeries.getValues().get(0).getValue());
}
@Test
public void testMappingByMonthDifferent() {
Calendar c1 = GregorianCalendar.getInstance();
c1.set(2020, 5, 1, 1, 1, 1);
Calendar c2 = GregorianCalendar.getInstance();
c2.set(2020, 6, 2, 1, 1, 1);
TimeSeries timeSeries = new TimeSeries("Тестовый",
Arrays.asList(new TimeSeriesValue(c1.getTime(), 10),
new TimeSeriesValue(c2.getTime(), 10)));
TimeSeriesDateMapper mapper = new TimeSeriesDateMapper();
timeSeries = mapper.mapTimeSeriesToInterval(TimeSeriesDateMapper.TimeSeriesInterval.MONTH, timeSeries);
Assert.assertEquals(2, timeSeries.getValues().size());
Assert.assertEquals(Integer.valueOf(10), timeSeries.getValues().get(0).getValue());
}
@Test
public void testMappingByYear() {
Calendar c1 = GregorianCalendar.getInstance();
c1.set(2020, 5, 1, 1, 1, 1);
Calendar c2 = GregorianCalendar.getInstance();
c2.set(2020, 5, 2, 1, 1, 1);
TimeSeries timeSeries = new TimeSeries("Тестовый",
Arrays.asList(new TimeSeriesValue(c1.getTime(), 10),
new TimeSeriesValue(c2.getTime(), 10)));
TimeSeriesDateMapper mapper = new TimeSeriesDateMapper();
timeSeries = mapper.mapTimeSeriesToInterval(TimeSeriesDateMapper.TimeSeriesInterval.YEAR, timeSeries);
Assert.assertEquals(1, timeSeries.getValues().size());
Assert.assertEquals(Integer.valueOf(20), timeSeries.getValues().get(0).getValue());
}
@Test
public void testMappingByYearDifferent() {
Calendar c1 = GregorianCalendar.getInstance();
c1.set(2020, 5, 1, 1, 1, 1);
Calendar c2 = GregorianCalendar.getInstance();
c2.set(2021, 5, 2, 1, 1, 1);
TimeSeries timeSeries = new TimeSeries("Тестовый",
Arrays.asList(new TimeSeriesValue(c1.getTime(), 10),
new TimeSeriesValue(c2.getTime(), 10)));
TimeSeriesDateMapper mapper = new TimeSeriesDateMapper();
timeSeries = mapper.mapTimeSeriesToInterval(TimeSeriesDateMapper.TimeSeriesInterval.YEAR, timeSeries);
Assert.assertEquals(2, timeSeries.getValues().size());
Assert.assertEquals(Integer.valueOf(10), timeSeries.getValues().get(0).getValue());
}
}