romanov73/git-extractor
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

#12 -- Improve performance

Browse Source
This commit is contained in:
Anton Romanov 2021-04-10 10:23:33 +04:00
parent 7e429fca2c
commit cbd77dfb24
3 changed files with 30 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/ru/ulstu/extractor/heuristic/api/StructuralUnitIdentifier.java
View File

@ -51,7 +51,10 @@ public abstract class StructuralUnitIdentifier {
.filter(file -> StringUtils.fileInSubdirectory(file.getPath(), projectPath, subDirectory))
.forEach(projectFile -> {
try {
String detectedLanguage = getDetectorService().getDetectedLanguage(new String(Files.readAllBytes(projectFile.toPath())));
String detectedLanguage = getDetectorService().getDetectedLanguage(
projectFile.getName(),
new String(Files.readAllBytes(projectFile.toPath())),
DetectorService.LangDetectScrupulousness.LOW);
projectFileLanguageFrequency.put(detectedLanguage, projectFileLanguageFrequency.getOrDefault(detectedLanguage, 0) + 1);
} catch (IOException e) {
e.printStackTrace();

18
src/main/java/ru/ulstu/extractor/heuristic/service/DetectorService.java
View File

@ -1,3 +1,8 @@
/*
* Copyright (C) 2021 Anton Romanov - All Rights Reserved
* You may use, distribute and modify this code, please write to: romanov73@gmail.com.
*/
package ru.ulstu.extractor.heuristic.service;
import com.gargoylesoftware.htmlunit.WebClient;
@ -9,11 +14,22 @@ import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import static ru.ulstu.extractor.heuristic.service.DetectorService.LangDetectScrupulousness.LOW;
@Service
public class DetectorService {
public enum LangDetectScrupulousness {LOW, HIGH}
private final static String BASE_URL = "http://localhost:8080/lang-detector.html";
public String getDetectedLanguage(String code) {
public String getDetectedLanguage(String fileName, String code, LangDetectScrupulousness scrupulousness) {
if (scrupulousness == LOW) {
return DirectoryService.getFileExtension(fileName).orElse("");
}
return getDetectedLanguage(code);
}
private String getDetectedLanguage(String code) {
String selectedLang = null;
try (WebClient webClient = new WebClient()) {
webClient.setJavaScriptTimeout(60 * 1000);

9
src/main/java/ru/ulstu/extractor/heuristic/service/DirectoryService.java
View File

@ -16,6 +16,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
/**
@ -23,6 +24,7 @@ import java.util.stream.Collectors;
*/
@Service
public class DirectoryService {
private static final String FILE_EXTENSION_DELIMITER = ".";
/**
* Получить список файлов, рекурсивно обойдя все дерево каталогов проекта.
@ -51,4 +53,11 @@ public class DirectoryService {
}
return Collections.emptyList();
}
public static Optional<String> getFileExtension(String fileName) {
if (fileName == null || fileName.isEmpty() || fileName.lastIndexOf(FILE_EXTENSION_DELIMITER) < 0) {
return Optional.empty();
}
return Optional.of(fileName.substring(fileName.lastIndexOf(FILE_EXTENSION_DELIMITER) + 1));
}
}