diff --git a/build.gradle b/build.gradle index e57e620..6f67f77 100644 --- a/build.gradle +++ b/build.gradle @@ -124,9 +124,9 @@ dependencies { compile group: 'io.springfox', name: 'springfox-swagger2', version: '2.6.0' compile group: 'io.springfox', name: 'springfox-swagger-ui', version: '2.6.0' + compile group: 'net.sourceforge.htmlunit', name: 'htmlunit', version: '2.35.0' testCompile group: 'org.springframework.boot', name: 'spring-boot-starter-test' - compile group: 'org.seleniumhq.selenium', name: 'selenium-java', version: '3.3.1' testCompile group: 'org.seleniumhq.selenium', name: 'selenium-support', version: '3.3.1' testCompile group: 'com.google.guava', name: 'guava', version: '21.0' diff --git a/src/main/java/ru/ulstu/grant/page/KiasPage.java b/src/main/java/ru/ulstu/grant/page/KiasPage.java index 631f5db..0a83b43 100644 --- a/src/main/java/ru/ulstu/grant/page/KiasPage.java +++ b/src/main/java/ru/ulstu/grant/page/KiasPage.java @@ -1,8 +1,13 @@ package ru.ulstu.grant.page; -import org.openqa.selenium.By; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; +import com.gargoylesoftware.htmlunit.html.DomElement; +import com.gargoylesoftware.htmlunit.html.DomNode; +import com.gargoylesoftware.htmlunit.html.DomNodeList; +import com.gargoylesoftware.htmlunit.html.HtmlAnchor; +import com.gargoylesoftware.htmlunit.html.HtmlElement; +import com.gargoylesoftware.htmlunit.html.HtmlPage; +import com.gargoylesoftware.htmlunit.html.HtmlTableDataCell; +import com.gargoylesoftware.htmlunit.html.HtmlTableRow; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -11,16 +16,17 @@ import java.util.List; public class KiasPage { private final static String KIAS_GRANT_DATE_FORMAT = "dd.MM.yyyy HH:mm"; - private WebDriver driver; + private final HtmlPage page; - public KiasPage(WebDriver webDriver) { - this.driver = webDriver; + public KiasPage(HtmlPage page) { + this.page = page; } public boolean goToNextPage() { try { - if (driver.findElements(By.id("js-ctrlNext")).size() > 0) { - driver.findElement(By.id("js-ctrlNext")).click(); + HtmlElement nextPageLink = page.getHtmlElementById("js-ctrlNext"); + if (nextPageLink.isDisplayed()) { + nextPageLink.click(); return true; } } finally { @@ -28,23 +34,25 @@ public class KiasPage { } } - public List getPageOfGrants() { - WebElement listContest = driver.findElement(By.tagName("tBody")); - List grants = listContest.findElements(By.cssSelector("tr.tr")); - return grants; + public List getPageOfGrants() { + return page.getByXPath("/html/body/div[2]/div/div[2]/main/div[1]/table/tbody/tr"); } - public String getGrantTitle(WebElement grant) { - return grant.findElement(By.cssSelector("td.tertiary")).findElement(By.tagName("a")).getText(); + public String getGrantTitle(DomNode grant) { + return ((DomNode)grant.getFirstByXPath("td[@class='tertiary']/a")).getTextContent(); } - public Date parseDeadLineDate(WebElement grantElement) throws ParseException { + public Date parseDeadLineDate(DomNode grantElement) throws ParseException { String deadlineDate = getFirstDeadline(grantElement); //10.06.2019 23:59 SimpleDateFormat formatter = new SimpleDateFormat(KIAS_GRANT_DATE_FORMAT); return formatter.parse(deadlineDate); } - private String getFirstDeadline(WebElement grantElement) { - return grantElement.findElement(By.xpath("./td[5]")).getText(); + private String getFirstDeadline(DomNode grantElement) { + return ((DomNode)grantElement.getFirstByXPath("td[5]")).getTextContent(); + } + + public boolean isTrGrantLine(DomNode grantElement) { + return !((HtmlTableRow)grantElement).getAttribute("class").contains("pagerSavedHeightSpacer"); } } diff --git a/src/main/java/ru/ulstu/grant/service/KiasService.java b/src/main/java/ru/ulstu/grant/service/KiasService.java index c3a55dc..56474fa 100644 --- a/src/main/java/ru/ulstu/grant/service/KiasService.java +++ b/src/main/java/ru/ulstu/grant/service/KiasService.java @@ -1,35 +1,29 @@ package ru.ulstu.grant.service; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.chrome.ChromeDriver; -import org.openqa.selenium.chrome.ChromeOptions; +import com.gargoylesoftware.htmlunit.WebClient; +import com.gargoylesoftware.htmlunit.html.DomElement; +import com.gargoylesoftware.htmlunit.html.DomNode; +import com.gargoylesoftware.htmlunit.html.HtmlElement; +import com.gargoylesoftware.htmlunit.html.HtmlPage; import org.springframework.stereotype.Service; import ru.ulstu.configuration.ApplicationProperties; import ru.ulstu.grant.model.GrantDto; import ru.ulstu.grant.page.KiasPage; import ru.ulstu.user.service.UserService; -import java.nio.file.Paths; +import java.io.IOException; import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.List; -import static org.apache.commons.lang3.StringUtils.isEmpty; - @Service public class KiasService { private final static String BASE_URL = "https://www.rfbr.ru/rffi/ru/contest_search?CONTEST_STATUS_ID=%s&CONTEST_TYPE=%s&CONTEST_YEAR=%s"; private final static String CONTEST_STATUS_ID = "1"; private final static String CONTEST_TYPE = "-1"; - private final static String DRIVER_LOCATION = "drivers/%s"; - private final static String WINDOWS_DRIVER = "chromedriver.exe"; - private final static String LINUX_DRIVER = "chromedriver"; - private final static String DRIVER_TYPE = "webdriver.chrome.driver"; - private final UserService userService; private final ApplicationProperties applicationProperties; @@ -39,22 +33,22 @@ public class KiasService { this.applicationProperties = applicationProperties; } - public List getNewGrantsDto() throws ParseException { - WebDriver webDriver = getDriver(); + public List getNewGrantsDto() throws ParseException, IOException { Integer leaderId = userService.findOneByLoginIgnoreCase("admin").getId(); List grants = new ArrayList<>(); - for (Integer year : generateGrantYears()) { - webDriver.get(String.format(BASE_URL, CONTEST_STATUS_ID, CONTEST_TYPE, year)); - grants.addAll(getKiasGrants(webDriver)); + try (final WebClient webClient = new WebClient()) { + for (Integer year : generateGrantYears()) { + final HtmlPage page = webClient.getPage(String.format(BASE_URL, CONTEST_STATUS_ID, CONTEST_TYPE, year)); + grants.addAll(getKiasGrants(page)); + } } grants.forEach(grantDto -> grantDto.setLeaderId(leaderId)); - webDriver.quit(); return grants; } - public List getKiasGrants(WebDriver webDriver) throws ParseException { + public List getKiasGrants(HtmlPage page) throws ParseException { List newGrants = new ArrayList<>(); - KiasPage kiasPage = new KiasPage(webDriver); + KiasPage kiasPage = new KiasPage(page); do { newGrants.addAll(getGrantsFromPage(kiasPage)); } while (kiasPage.goToNextPage()); //проверка существования следующей страницы с грантами @@ -63,11 +57,13 @@ public class KiasService { private List getGrantsFromPage(KiasPage kiasPage) throws ParseException { List grants = new ArrayList<>(); - for (WebElement grantElement : kiasPage.getPageOfGrants()) { - GrantDto grantDto = new GrantDto( - kiasPage.getGrantTitle(grantElement), - kiasPage.parseDeadLineDate(grantElement)); - grants.add(grantDto); + for (DomNode grantElement : kiasPage.getPageOfGrants()) { + if (kiasPage.isTrGrantLine(grantElement)) { + GrantDto grantDto = new GrantDto( + kiasPage.getGrantTitle(grantElement), + kiasPage.parseDeadLineDate(grantElement)); + grants.add(grantDto); + } } return grants; } @@ -76,28 +72,4 @@ public class KiasService { return Arrays.asList(Calendar.getInstance().get(Calendar.YEAR), Calendar.getInstance().get(Calendar.YEAR) + 1); } - - private WebDriver getDriver() { - System.setProperty(DRIVER_TYPE, getDriverExecutablePath()); - System.out.println("drive search path: " + getDriverExecutablePath()); - final ChromeOptions chromeOptions = new ChromeOptions(); - chromeOptions.setBinary("/usr/bin/google-chrome"); - chromeOptions.addArguments("--headless"); - return new ChromeDriver(chromeOptions); - } - - private String getDriverExecutablePath() { - return isEmpty(applicationProperties.getDriverPath()) - ? KiasService.class.getClassLoader() - .getResource(String.format(DRIVER_LOCATION, getDriverExecutable(isWindows()))).getFile() - : Paths.get(applicationProperties.getDriverPath(), getDriverExecutable(isWindows())).toString(); - } - - private String getDriverExecutable(boolean isWindows) { - return isWindows ? WINDOWS_DRIVER : LINUX_DRIVER; - } - - private boolean isWindows() { - return System.getProperty("os.name").toLowerCase().contains("windows"); - } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f4d9c36..7b30ec0 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -13,6 +13,8 @@ server.ssl.key-store-password=secret server.ssl.key-password=password # Log settings (TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF) logging.level.ru.ulstu=DEBUG +#HtmlUnit +logging.level.com.gargoylesoftware.htmlunit=ERROR # Mail Settings spring.mail.host=smtp.yandex.ru spring.mail.port=465 @@ -31,6 +33,7 @@ spring.jpa.hibernate.ddl-auto=validate liquibase.drop-first=false liquibase.enabled=true liquibase.change-log=classpath:db/changelog-master.xml + # Application Settings ng-tracker.base-url=http://127.0.0.1:8080 ng-tracker.undead-user-login=admin diff --git a/src/main/resources/db/changelog-20190605_000000-schema.xml b/src/main/resources/db/changelog-20190605_000000-schema.xml index c035b35..2783660 100644 --- a/src/main/resources/db/changelog-20190605_000000-schema.xml +++ b/src/main/resources/db/changelog-20190605_000000-schema.xml @@ -8,4 +8,22 @@ + + + + + + + + +