use HtmlUnit instead of selenium webdriver for grants loading

build.gradle

@@ -124,9 +124,9 @@ dependencies {
 
     compile group: 'io.springfox', name: 'springfox-swagger2', version: '2.6.0'
     compile group: 'io.springfox', name: 'springfox-swagger-ui', version: '2.6.0'
+    compile group: 'net.sourceforge.htmlunit', name: 'htmlunit', version: '2.35.0'
 
     testCompile group: 'org.springframework.boot', name: 'spring-boot-starter-test'
-    compile group: 'org.seleniumhq.selenium', name: 'selenium-java', version: '3.3.1'
     testCompile group: 'org.seleniumhq.selenium', name: 'selenium-support', version: '3.3.1'
     testCompile group: 'com.google.guava', name: 'guava', version: '21.0'
 

src/main/java/ru/ulstu/grant/page/KiasPage.java

@@ -1,8 +1,13 @@
 package ru.ulstu.grant.page;
 
-import org.openqa.selenium.By;
+import com.gargoylesoftware.htmlunit.html.DomElement;
-import org.openqa.selenium.WebDriver;
+import com.gargoylesoftware.htmlunit.html.DomNode;
-import org.openqa.selenium.WebElement;
+import com.gargoylesoftware.htmlunit.html.DomNodeList;
+import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
+import com.gargoylesoftware.htmlunit.html.HtmlElement;
+import com.gargoylesoftware.htmlunit.html.HtmlPage;
+import com.gargoylesoftware.htmlunit.html.HtmlTableDataCell;
+import com.gargoylesoftware.htmlunit.html.HtmlTableRow;
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
@@ -11,16 +16,17 @@ import java.util.List;
 
 public class KiasPage {
     private final static String KIAS_GRANT_DATE_FORMAT = "dd.MM.yyyy HH:mm";
-    private WebDriver driver;
+    private final HtmlPage page;
 
-    public KiasPage(WebDriver webDriver) {
+    public KiasPage(HtmlPage page) {
-        this.driver = webDriver;
+        this.page = page;
     }
 
     public boolean goToNextPage() {
         try {
-            if (driver.findElements(By.id("js-ctrlNext")).size() > 0) {
+            HtmlElement nextPageLink = page.getHtmlElementById("js-ctrlNext");
-                driver.findElement(By.id("js-ctrlNext")).click();
+            if (nextPageLink.isDisplayed()) {
+                nextPageLink.click();
                 return true;
             }
         } finally {
@@ -28,23 +34,25 @@ public class KiasPage {
         }
     }
 
-    public List<WebElement> getPageOfGrants() {
+    public List<DomNode> getPageOfGrants() {
-        WebElement listContest = driver.findElement(By.tagName("tBody"));
+        return page.getByXPath("/html/body/div[2]/div/div[2]/main/div[1]/table/tbody/tr");
-        List<WebElement> grants = listContest.findElements(By.cssSelector("tr.tr"));
-        return grants;
     }
 
-    public String getGrantTitle(WebElement grant) {
+    public String getGrantTitle(DomNode grant) {
-        return grant.findElement(By.cssSelector("td.tertiary")).findElement(By.tagName("a")).getText();
+        return ((DomNode)grant.getFirstByXPath("td[@class='tertiary']/a")).getTextContent();
     }
 
-    public Date parseDeadLineDate(WebElement grantElement) throws ParseException {
+    public Date parseDeadLineDate(DomNode grantElement) throws ParseException {
         String deadlineDate = getFirstDeadline(grantElement); //10.06.2019 23:59
         SimpleDateFormat formatter = new SimpleDateFormat(KIAS_GRANT_DATE_FORMAT);
         return formatter.parse(deadlineDate);
     }
 
-    private String getFirstDeadline(WebElement grantElement) {
+    private String getFirstDeadline(DomNode grantElement) {
-        return grantElement.findElement(By.xpath("./td[5]")).getText();
+        return ((DomNode)grantElement.getFirstByXPath("td[5]")).getTextContent();
+    }
+
+    public boolean isTrGrantLine(DomNode grantElement) {
+        return !((HtmlTableRow)grantElement).getAttribute("class").contains("pagerSavedHeightSpacer");
     }
 }

src/main/java/ru/ulstu/grant/service/KiasService.java

@@ -1,35 +1,29 @@
 package ru.ulstu.grant.service;
 
-import org.openqa.selenium.WebDriver;
+import com.gargoylesoftware.htmlunit.WebClient;
-import org.openqa.selenium.WebElement;
+import com.gargoylesoftware.htmlunit.html.DomElement;
-import org.openqa.selenium.chrome.ChromeDriver;
+import com.gargoylesoftware.htmlunit.html.DomNode;
-import org.openqa.selenium.chrome.ChromeOptions;
+import com.gargoylesoftware.htmlunit.html.HtmlElement;
+import com.gargoylesoftware.htmlunit.html.HtmlPage;
 import org.springframework.stereotype.Service;
 import ru.ulstu.configuration.ApplicationProperties;
 import ru.ulstu.grant.model.GrantDto;
 import ru.ulstu.grant.page.KiasPage;
 import ru.ulstu.user.service.UserService;
 
-import java.nio.file.Paths;
+import java.io.IOException;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
 import java.util.List;
 
-import static org.apache.commons.lang3.StringUtils.isEmpty;
-
 @Service
 public class KiasService {
     private final static String BASE_URL = "https://www.rfbr.ru/rffi/ru/contest_search?CONTEST_STATUS_ID=%s&CONTEST_TYPE=%s&CONTEST_YEAR=%s";
     private final static String CONTEST_STATUS_ID = "1";
     private final static String CONTEST_TYPE = "-1";
 
-    private final static String DRIVER_LOCATION = "drivers/%s";
-    private final static String WINDOWS_DRIVER = "chromedriver.exe";
-    private final static String LINUX_DRIVER = "chromedriver";
-    private final static String DRIVER_TYPE = "webdriver.chrome.driver";
-
     private final UserService userService;
     private final ApplicationProperties applicationProperties;
 
@@ -39,22 +33,22 @@ public class KiasService {
         this.applicationProperties = applicationProperties;
     }
 
-    public List<GrantDto> getNewGrantsDto() throws ParseException {
+    public List<GrantDto> getNewGrantsDto() throws ParseException, IOException {
-        WebDriver webDriver = getDriver();
         Integer leaderId = userService.findOneByLoginIgnoreCase("admin").getId();
         List<GrantDto> grants = new ArrayList<>();
-        for (Integer year : generateGrantYears()) {
+        try (final WebClient webClient = new WebClient()) {
-            webDriver.get(String.format(BASE_URL, CONTEST_STATUS_ID, CONTEST_TYPE, year));
+            for (Integer year : generateGrantYears()) {
-            grants.addAll(getKiasGrants(webDriver));
+                final HtmlPage page = webClient.getPage(String.format(BASE_URL, CONTEST_STATUS_ID, CONTEST_TYPE, year));
+                grants.addAll(getKiasGrants(page));
+            }
         }
         grants.forEach(grantDto -> grantDto.setLeaderId(leaderId));
-        webDriver.quit();
         return grants;
     }
 
-    public List<GrantDto> getKiasGrants(WebDriver webDriver) throws ParseException {
+    public List<GrantDto> getKiasGrants(HtmlPage page) throws ParseException {
         List<GrantDto> newGrants = new ArrayList<>();
-        KiasPage kiasPage = new KiasPage(webDriver);
+        KiasPage kiasPage = new KiasPage(page);
         do {
             newGrants.addAll(getGrantsFromPage(kiasPage));
         } while (kiasPage.goToNextPage()); //проверка существования следующей страницы с грантами
@@ -63,11 +57,13 @@ public class KiasService {
 
     private List<GrantDto> getGrantsFromPage(KiasPage kiasPage) throws ParseException {
         List<GrantDto> grants = new ArrayList<>();
-        for (WebElement grantElement : kiasPage.getPageOfGrants()) {
+        for (DomNode grantElement : kiasPage.getPageOfGrants()) {
-            GrantDto grantDto = new GrantDto(
+            if (kiasPage.isTrGrantLine(grantElement)) {
-                    kiasPage.getGrantTitle(grantElement),
+                GrantDto grantDto = new GrantDto(
-                    kiasPage.parseDeadLineDate(grantElement));
+                        kiasPage.getGrantTitle(grantElement),
-            grants.add(grantDto);
+                        kiasPage.parseDeadLineDate(grantElement));
+                grants.add(grantDto);
+            }
         }
         return grants;
     }
@@ -76,28 +72,4 @@ public class KiasService {
         return Arrays.asList(Calendar.getInstance().get(Calendar.YEAR),
                 Calendar.getInstance().get(Calendar.YEAR) + 1);
     }
-
-    private WebDriver getDriver() {
-        System.setProperty(DRIVER_TYPE, getDriverExecutablePath());
-        System.out.println("drive search path: " + getDriverExecutablePath());
-        final ChromeOptions chromeOptions = new ChromeOptions();
-        chromeOptions.setBinary("/usr/bin/google-chrome");
-        chromeOptions.addArguments("--headless");
-        return new ChromeDriver(chromeOptions);
-    }
-
-    private String getDriverExecutablePath() {
-        return isEmpty(applicationProperties.getDriverPath())
-                ? KiasService.class.getClassLoader()
-                .getResource(String.format(DRIVER_LOCATION, getDriverExecutable(isWindows()))).getFile()
-                : Paths.get(applicationProperties.getDriverPath(), getDriverExecutable(isWindows())).toString();
-    }
-
-    private String getDriverExecutable(boolean isWindows) {
-        return isWindows ? WINDOWS_DRIVER : LINUX_DRIVER;
-    }
-
-    private boolean isWindows() {
-        return System.getProperty("os.name").toLowerCase().contains("windows");
-    }
 }

src/main/resources/application.properties

@@ -13,6 +13,8 @@ server.ssl.key-store-password=secret
 server.ssl.key-password=password
 # Log settings (TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF)
 logging.level.ru.ulstu=DEBUG
+#HtmlUnit
+logging.level.com.gargoylesoftware.htmlunit=ERROR
 # Mail Settings
 spring.mail.host=smtp.yandex.ru
 spring.mail.port=465
@@ -31,6 +33,7 @@ spring.jpa.hibernate.ddl-auto=validate
 liquibase.drop-first=false
 liquibase.enabled=true
 liquibase.change-log=classpath:db/changelog-master.xml
+
 # Application Settings
 ng-tracker.base-url=http://127.0.0.1:8080
 ng-tracker.undead-user-login=admin

src/main/resources/db/changelog-20190605_000000-schema.xml

@@ -8,4 +8,22 @@
             <column name="activity_id" type="integer"/>
         </addColumn>
     </changeSet>
+    <changeSet author="orion" id="20190525_000000-2">
+        <modifyDataType tableName="grants"
+                        columnName="title"
+                        newDataType="text"
+                        schemaName="public"/>
+    </changeSet>
+    <changeSet author="orion" id="20190525_000000-3">
+        <modifyDataType tableName="event"
+                        columnName="title"
+                        newDataType="text"
+                        schemaName="public"/>
+    </changeSet>
+    <changeSet author="orion" id="20190525_000000-4">
+        <modifyDataType tableName="event"
+                        columnName="description"
+                        newDataType="text"
+                        schemaName="public"/>
+    </changeSet>
 </databaseChangeLog>