Merge branch '120-kias' into 'dev'

Resolve "Загрузка грантов в сайта kias"

Closes #120

See merge request romanov73/ng-tracker!94
This commit is contained in:
Anton Romanov 2019-05-25 17:40:59 +00:00
commit 2b911017b1
20 changed files with 678 additions and 386 deletions

View File

@ -126,6 +126,6 @@ dependencies {
compile group: 'io.springfox', name: 'springfox-swagger-ui', version: '2.6.0'
testCompile group: 'org.springframework.boot', name: 'spring-boot-starter-test'
testCompile group: 'org.seleniumhq.selenium', name: 'selenium-java', version: '3.3.1'
compile group: 'org.seleniumhq.selenium', name: 'selenium-java', version: '3.3.1'
}

View File

@ -21,6 +21,8 @@ public class ApplicationProperties {
private boolean checkRun;
private String debugEmail;
public boolean isUseHttps() {
return useHttps;
}
@ -60,4 +62,12 @@ public class ApplicationProperties {
public void setCheckRun(boolean checkRun) {
this.checkRun = checkRun;
}
public String getDebugEmail() {
return debugEmail;
}
public void setDebugEmail(String debugEmail) {
this.debugEmail = debugEmail;
}
}

View File

@ -2,8 +2,15 @@ package ru.ulstu.deadline.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import ru.ulstu.deadline.model.Deadline;
import java.util.Date;
public interface DeadlineRepository extends JpaRepository<Deadline, Integer> {
@Query(
value = "SELECT date FROM Deadline d WHERE (d.grant_id = ?1) AND (d.date = ?2)",
nativeQuery = true)
Date findByGrantIdAndDate(Integer grantId, Date date);
}

View File

@ -5,6 +5,7 @@ import org.springframework.transaction.annotation.Transactional;
import ru.ulstu.deadline.model.Deadline;
import ru.ulstu.deadline.repository.DeadlineRepository;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
@ -46,4 +47,8 @@ public class DeadlineService {
public void remove(Integer deadlineId) {
deadlineRepository.delete(deadlineId);
}
public Date findByGrantIdAndDate(Integer id, Date date) {
return deadlineRepository.findByGrantIdAndDate(id, date);
}
}

View File

@ -20,9 +20,7 @@ import springfox.documentation.annotations.ApiIgnore;
import javax.validation.Valid;
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
import static org.springframework.util.StringUtils.isEmpty;
import static ru.ulstu.core.controller.Navigation.GRANTS_PAGE;
import static ru.ulstu.core.controller.Navigation.GRANT_PAGE;
import static ru.ulstu.core.controller.Navigation.REDIRECT_TO;
@ -62,17 +60,9 @@ public class GrantController {
@PostMapping(value = "/grant", params = "save")
public String save(@Valid GrantDto grantDto, Errors errors)
throws IOException {
filterEmptyDeadlines(grantDto);
if (grantDto.getDeadlines().isEmpty()) {
errors.rejectValue("deadlines", "errorCode", "Не может быть пусто");
}
if (grantDto.getLeaderId().equals(-1)) {
errors.rejectValue("leaderId", "errorCode", "Укажите руководителя");
}
if (errors.hasErrors()) {
if (!grantService.save(grantDto, errors)) {
return GRANT_PAGE;
}
grantService.save(grantDto);
return String.format(REDIRECT_TO, GRANTS_PAGE);
}
@ -89,7 +79,7 @@ public class GrantController {
@PostMapping(value = "/grant", params = "addDeadline")
public String addDeadline(@Valid GrantDto grantDto, Errors errors) {
filterEmptyDeadlines(grantDto);
grantService.filterEmptyDeadlines(grantDto);
if (errors.hasErrors()) {
return GRANT_PAGE;
}
@ -133,10 +123,4 @@ public class GrantController {
public List<PaperDto> getAllPapers() {
return grantService.getAllUncompletedPapers();
}
private void filterEmptyDeadlines(GrantDto grantDto) {
grantDto.setDeadlines(grantDto.getDeadlines().stream()
.filter(dto -> dto.getDate() != null || !isEmpty(dto.getDescription()))
.collect(Collectors.toList()));
}
}

View File

@ -6,18 +6,20 @@ import org.apache.commons.lang3.StringUtils;
import org.hibernate.validator.constraints.NotEmpty;
import ru.ulstu.deadline.model.Deadline;
import ru.ulstu.file.model.FileDataDto;
import ru.ulstu.name.NameContainer;
import ru.ulstu.paper.model.PaperDto;
import ru.ulstu.project.model.ProjectDto;
import ru.ulstu.user.model.UserDto;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static ru.ulstu.core.util.StreamApiUtils.convert;
public class GrantDto {
public class GrantDto extends NameContainer {
private final static int MAX_AUTHORS_LENGTH = 60;
private Integer id;
@ -95,6 +97,12 @@ public class GrantDto {
this.papers = convert(grant.getPapers(), PaperDto::new);
}
public GrantDto(String grantTitle, Date deadLineDate) {
this.title = grantTitle;
deadlines.add(new Deadline(deadLineDate, "Окончание приёма заявок"));
status = Grant.GrantStatus.LOADED_FROM_KIAS;
}
public Integer getId() {
return id;
}

View File

@ -0,0 +1,50 @@
package ru.ulstu.grant.page;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
public class KiasPage {
private final static String KIAS_GRANT_DATE_FORMAT = "dd.MM.yyyy HH:mm";
private WebDriver driver;
public KiasPage(WebDriver webDriver) {
this.driver = webDriver;
}
public boolean goToNextPage() {
try {
if (driver.findElements(By.id("js-ctrlNext")).size() > 0) {
driver.findElement(By.id("js-ctrlNext")).click();
return true;
}
} finally {
return false;
}
}
public List<WebElement> getPageOfGrants() {
WebElement listContest = driver.findElement(By.tagName("tBody"));
List<WebElement> grants = listContest.findElements(By.cssSelector("tr.tr"));
return grants;
}
public String getGrantTitle(WebElement grant) {
return grant.findElement(By.cssSelector("td.tertiary")).findElement(By.tagName("a")).getText();
}
public Date parseDeadLineDate(WebElement grantElement) throws ParseException {
String deadlineDate = getFirstDeadline(grantElement); //10.06.2019 23:59
SimpleDateFormat formatter = new SimpleDateFormat(KIAS_GRANT_DATE_FORMAT);
return formatter.parse(deadlineDate);
}
private String getFirstDeadline(WebElement grantElement) {
return grantElement.findElement(By.xpath("./td[5]")).getText();
}
}

View File

@ -1,11 +1,20 @@
package ru.ulstu.grant.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import ru.ulstu.grant.model.Grant;
import ru.ulstu.name.BaseRepository;
import java.util.List;
public interface GrantRepository extends JpaRepository<Grant, Integer> {
public interface GrantRepository extends JpaRepository<Grant, Integer>, BaseRepository {
List<Grant> findByStatus(Grant.GrantStatus status);
Grant findByTitle(String title);
@Override
@Query("SELECT title FROM Grant g WHERE (g.title = :name) AND (:id IS NULL OR g.id != :id) ")
String findByNameAndNotId(@Param("name") String name, @Param("id") Integer id);
}

View File

@ -5,6 +5,9 @@ import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.text.ParseException;
@Service
public class GrantScheduler {
private final static boolean IS_DEADLINE_NOTIFICATION_BEFORE_WEEK = true;
@ -27,4 +30,15 @@ public class GrantScheduler {
grantNotificationService.sendDeadlineNotifications(grantService.findAll(), IS_DEADLINE_NOTIFICATION_BEFORE_WEEK);
log.debug("GrantScheduler.checkDeadlineBeforeWeek finished");
}
@Scheduled(cron = "0 0 8 1 * ?", zone = "Europe/Samara")
public void loadGrantsFromKias() {
log.debug("GrantScheduler.loadGrantsFromKias started");
try {
grantService.createFromKias();
} catch (ParseException | IOException e) {
e.printStackTrace();
}
log.debug("GrantScheduler.loadGrantsFromKias finished");
}
}

View File

@ -1,8 +1,11 @@
package ru.ulstu.grant.service;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.Errors;
import ru.ulstu.deadline.model.Deadline;
import ru.ulstu.deadline.service.DeadlineService;
import ru.ulstu.file.model.FileDataDto;
@ -10,6 +13,7 @@ import ru.ulstu.file.service.FileService;
import ru.ulstu.grant.model.Grant;
import ru.ulstu.grant.model.GrantDto;
import ru.ulstu.grant.repository.GrantRepository;
import ru.ulstu.name.BaseService;
import ru.ulstu.paper.model.Paper;
import ru.ulstu.paper.model.PaperDto;
import ru.ulstu.paper.service.PaperService;
@ -21,12 +25,14 @@ import ru.ulstu.user.model.User;
import ru.ulstu.user.service.UserService;
import java.io.IOException;
import java.text.ParseException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static java.util.stream.Collectors.toList;
import static org.springframework.util.ObjectUtils.isEmpty;
@ -34,8 +40,9 @@ import static ru.ulstu.core.util.StreamApiUtils.convert;
import static ru.ulstu.grant.model.Grant.GrantStatus.APPLICATION;
@Service
public class GrantService {
public class GrantService extends BaseService {
private final static int MAX_DISPLAY_SIZE = 50;
private final Logger log = LoggerFactory.getLogger(GrantService.class);
private final GrantRepository grantRepository;
private final ProjectService projectService;
@ -45,6 +52,7 @@ public class GrantService {
private final PaperService paperService;
private final EventService eventService;
private final GrantNotificationService grantNotificationService;
private final KiasService kiasService;
public GrantService(GrantRepository grantRepository,
FileService fileService,
@ -53,8 +61,11 @@ public class GrantService {
UserService userService,
PaperService paperService,
EventService eventService,
GrantNotificationService grantNotificationService) {
GrantNotificationService grantNotificationService,
KiasService kiasService) {
this.grantRepository = grantRepository;
this.kiasService = kiasService;
this.baseRepository = grantRepository;
this.fileService = fileService;
this.deadlineService = deadlineService;
this.projectService = projectService;
@ -176,12 +187,58 @@ public class GrantService {
return grant;
}
public void save(GrantDto grantDto) throws IOException {
public boolean save(GrantDto grantDto, Errors errors) throws IOException {
grantDto.setName(grantDto.getTitle());
filterEmptyDeadlines(grantDto);
checkEmptyDeadlines(grantDto, errors);
checkEmptyLeader(grantDto, errors);
checkUniqueName(grantDto, errors, grantDto.getId(), "title", "Грант с таким именем уже существует");
if (errors.hasErrors()) {
return false;
}
if (isEmpty(grantDto.getId())) {
create(grantDto);
} else {
update(grantDto);
}
return true;
}
public boolean saveFromKias(GrantDto grantDto) throws IOException {
grantDto.setName(grantDto.getTitle());
String title = checkUniqueName(grantDto, grantDto.getId()); //проверка уникальности имени
if (title != null) {
Grant grantFromDB = grantRepository.findByTitle(title); //грант с таким же названием из бд
if (checkSameDeadline(grantDto, grantFromDB.getId())) { //если дедайны тоже совпадают
return false;
} else { //иначе грант уже был в системе, но в другом году, поэтому надо создать
create(grantDto);
return true;
}
} else { //иначе такого гранта ещё нет, поэтому надо создать
create(grantDto);
return true;
}
}
private void checkEmptyLeader(GrantDto grantDto, Errors errors) {
if (grantDto.getLeaderId().equals(-1)) {
errors.rejectValue("leaderId", "errorCode", "Укажите руководителя");
}
}
private void checkEmptyDeadlines(GrantDto grantDto, Errors errors) {
if (grantDto.getDeadlines().isEmpty()) {
errors.rejectValue("deadlines", "errorCode", "Не может быть пусто");
}
}
private boolean checkSameDeadline(GrantDto grantDto, Integer id) {
Date date = grantDto.getDeadlines().get(0).getDate(); //дата с сайта киас
if (deadlineService.findByGrantIdAndDate(id, date).compareTo(date) == 0) { //если есть такая строка с датой
return true;
}
return false;
}
public List<User> getGrantAuthors(GrantDto grantDto) {
@ -261,4 +318,21 @@ public class GrantService {
.filter(author -> Collections.frequency(authors, author) > 3)
.collect(toList());
}
public void filterEmptyDeadlines(GrantDto grantDto) {
grantDto.setDeadlines(grantDto.getDeadlines().stream()
.filter(dto -> dto.getDate() != null || !org.springframework.util.StringUtils.isEmpty(dto.getDescription()))
.collect(Collectors.toList()));
}
@Transactional
public void createFromKias() throws IOException, ParseException {
for (GrantDto grantDto : kiasService.getNewGrantsDto()) {
if (saveFromKias(grantDto)) {
log.debug("GrantScheduler.loadGrantsFromKias new grant was loaded");
} else {
log.debug("GrantScheduler.loadGrantsFromKias grant wasn't loaded, cause it's already exists");
}
}
}
}

View File

@ -0,0 +1,92 @@
package ru.ulstu.grant.service;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.springframework.stereotype.Service;
import ru.ulstu.grant.model.GrantDto;
import ru.ulstu.grant.page.KiasPage;
import ru.ulstu.user.service.UserService;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.List;
@Service
public class KiasService {
private final static String BASE_URL = "https://www.rfbr.ru/rffi/ru/contest_search?CONTEST_STATUS_ID=%s&CONTEST_TYPE=%s&CONTEST_YEAR=%s";
private final static String CONTEST_STATUS_ID = "1";
private final static String CONTEST_TYPE = "-1";
private final static String DRIVER_LOCATION = "drivers/%s";
private final static String WINDOWS_DRIVER = "chromedriver.exe";
private final static String LINUX_DRIVER = "chromedriver";
private final static String DRIVER_TYPE = "webdriver.chrome.driver";
private final UserService userService;
public KiasService(UserService userService) {
this.userService = userService;
}
public List<GrantDto> getNewGrantsDto() throws ParseException {
WebDriver webDriver = getDriver();
Integer leaderId = userService.findOneByLoginIgnoreCase("admin").getId();
List<GrantDto> grants = new ArrayList<>();
for (Integer year : generateGrantYears()) {
webDriver.get(String.format(BASE_URL, CONTEST_STATUS_ID, CONTEST_TYPE, year));
grants.addAll(getKiasGrants(webDriver));
}
grants.forEach(grantDto -> grantDto.setLeaderId(leaderId));
webDriver.quit();
return grants;
}
public List<GrantDto> getKiasGrants(WebDriver webDriver) throws ParseException {
List<GrantDto> newGrants = new ArrayList<>();
KiasPage kiasPage = new KiasPage(webDriver);
do {
newGrants.addAll(getGrantsFromPage(kiasPage));
} while (kiasPage.goToNextPage()); //проверка существования следующей страницы с грантами
return newGrants;
}
private List<GrantDto> getGrantsFromPage(KiasPage kiasPage) throws ParseException {
List<GrantDto> grants = new ArrayList<>();
for (WebElement grantElement : kiasPage.getPageOfGrants()) {
GrantDto grantDto = new GrantDto(
kiasPage.getGrantTitle(grantElement),
kiasPage.parseDeadLineDate(grantElement));
grants.add(grantDto);
}
return grants;
}
private List<Integer> generateGrantYears() {
return Arrays.asList(Calendar.getInstance().get(Calendar.YEAR),
Calendar.getInstance().get(Calendar.YEAR) + 1);
}
private WebDriver getDriver() {
System.setProperty(DRIVER_TYPE, getDriverExecutablePath());
final ChromeOptions chromeOptions = new ChromeOptions();
chromeOptions.addArguments("--headless");
return new ChromeDriver(chromeOptions);
}
private String getDriverExecutablePath() {
return KiasService.class.getClassLoader().getResource(
String.format(DRIVER_LOCATION, getDriverExecutable(isWindows()))).getFile();
}
private String getDriverExecutable(boolean isWindows) {
return isWindows ? WINDOWS_DRIVER : LINUX_DRIVER;
}
private boolean isWindows() {
return System.getProperty("os.name").toLowerCase().contains("windows");
}
}

View File

@ -13,4 +13,11 @@ public abstract class BaseService {
errors.rejectValue(checkField, "errorCode", errorMessage);
}
}
public String checkUniqueName(NameContainer nameContainer, Integer id) {
if (nameContainer.getName().equals(baseRepository.findByNameAndNotId(nameContainer.getName(), id))) {
return baseRepository.findByNameAndNotId(nameContainer.getName(), id);
}
return null;
}
}

View File

@ -1,5 +1,6 @@
package ru.ulstu.user.service;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.mail.MailProperties;
@ -21,11 +22,9 @@ import java.util.Map;
@Service
public class MailService {
private final Logger log = LoggerFactory.getLogger(MailService.class);
private static final String USER = "user";
private static final String BASE_URL = "baseUrl";
private final Logger log = LoggerFactory.getLogger(MailService.class);
private final JavaMailSender javaMailSender;
private final SpringTemplateEngine templateEngine;
private final MailProperties mailProperties;
@ -48,10 +47,18 @@ public class MailService {
message.setFrom(mailProperties.getUsername());
message.setSubject(subject);
message.setText(content, true);
modifyForDebug(message, subject);
javaMailSender.send(mimeMessage);
log.debug("Sent email to User '{}'", to);
}
private void modifyForDebug(MimeMessageHelper message, String originalSubject) throws MessagingException {
if (!StringUtils.isEmpty(applicationProperties.getDebugEmail())) {
message.setTo(applicationProperties.getDebugEmail());
message.setSubject("To " + applicationProperties.getDebugEmail() + "; " + originalSubject);
}
}
@Async
public void sendEmailFromTemplate(User user, String templateName, String subject) {
Context context = new Context();

View File

@ -348,4 +348,8 @@ public class UserService implements UserDetailsService {
throw new UserSendingMailException(email);
}
}
public User findOneByLoginIgnoreCase(String login) {
return userRepository.findOneByLoginIgnoreCase(login);
}
}

View File

@ -35,5 +35,6 @@ liquibase.change-log=classpath:db/changelog-master.xml
ng-tracker.base-url=http://127.0.0.1:8080
ng-tracker.undead-user-login=admin
ng-tracker.dev-mode=true
ng-tracker.debug_email=romanov73@gmail.com
ng-tracker.use-https=false
ng-tracker.check-run=false

View File

@ -0,0 +1,8 @@
<?xml version="1.1" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd">
<changeSet author="tanya" id="20190520_000000-1">
<dropColumn columnName="applicationFileName" tableName="grants"/>
</changeSet>
</databaseChangeLog>

View File

@ -42,5 +42,6 @@
<include file="db/changelog-20190507_000000-schema.xml"/>
<include file="db/changelog-20190507_000001-schema.xml"/>
<include file="db/changelog-20190511_000000-schema.xml"/>
<include file="db/changelog-20190520_000000-schema.xml"/>
<include file="db/changelog-20190523_000000-schema.xml"/>
</databaseChangeLog>

View File

@ -26,7 +26,7 @@ import java.util.Map;
@RunWith(SpringRunner.class)
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@SpringBootTest(classes = NgTrackerApplication.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT)
public class IndexConferenceTest extends TestTemplate {
public class ConferenceTest extends TestTemplate {
private final Map<PageObject, List<String>> navigationHolder = ImmutableMap.of(
new ConferencesPage(), Arrays.asList("КОНФЕРЕНЦИИ", "/conferences/conferences"),
new ConferencePage(), Arrays.asList("РЕДАКТИРОВАНИЕ КОНФЕРЕНЦИИ", "/conferences/conference?id=0"),

View File

@ -24,14 +24,14 @@ import java.util.Map;
@RunWith(SpringRunner.class)
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@SpringBootTest(classes = NgTrackerApplication.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT)
public class IndexTaskTest extends TestTemplate {
public class StudentTaskTest extends TestTemplate {
private final Map<PageObject, List<String>> navigationHolder = ImmutableMap.of(
new TasksPage(), Arrays.asList("Список задач", "/students/tasks"),
new TasksDashboardPage(), Arrays.asList("Панель управления", "/students/dashboard"),
new TaskPage(), Arrays.asList("Создать задачу", "/students/task?id=0")
);
private final String TAG = "ATag";
private final String tag = "ATag";
@Autowired
private ApplicationProperties applicationProperties;
@ -168,13 +168,13 @@ public class IndexTaskTest extends TestTemplate {
String taskName = "Task " + (new Date()).getTime();
taskPage.setName(taskName);
taskPage.setTag(TAG);
taskPage.setTag(tag);
Thread.sleep(1000);
taskPage.addDeadlineDate("01.01.2020", 0);
taskPage.addDeadlineDescription("Description", 0);
taskPage.save();
Assert.assertTrue(tasksPage.findTaskByTag(taskName, TAG));
Assert.assertTrue(tasksPage.findTaskByTag(taskName, tag));
}
@Test
@ -185,7 +185,7 @@ public class IndexTaskTest extends TestTemplate {
TasksPage tasksPage = (TasksPage) getContext().initPage(page.getKey());
Assert.assertTrue(tasksPage.findTag(TAG));
Assert.assertTrue(tasksPage.findTag(tag));
}
@Test
@ -194,9 +194,9 @@ public class IndexTaskTest extends TestTemplate {
getContext().goTo(applicationProperties.getBaseUrl() + page.getValue().get(1));
TasksPage tasksPage = (TasksPage) getContext().initPage(page.getKey());
tasksPage.selectTag(TAG);
tasksPage.selectTag(tag);
Assert.assertTrue(tasksPage.findTasksByTag(TAG));
Assert.assertTrue(tasksPage.findTasksByTag(tag));
}
@Test

View File

@ -0,0 +1,11 @@
package grant;
import core.PageObject;
import org.openqa.selenium.By;
public class KiasPage extends PageObject {
@Override
public String getSubTitle() {
return driver.findElement(By.tagName("h1")).getText();
}
}