#12 -- Improve performance
This commit is contained in:
parent
7e429fca2c
commit
cbd77dfb24
@ -51,7 +51,10 @@ public abstract class StructuralUnitIdentifier {
|
|||||||
.filter(file -> StringUtils.fileInSubdirectory(file.getPath(), projectPath, subDirectory))
|
.filter(file -> StringUtils.fileInSubdirectory(file.getPath(), projectPath, subDirectory))
|
||||||
.forEach(projectFile -> {
|
.forEach(projectFile -> {
|
||||||
try {
|
try {
|
||||||
String detectedLanguage = getDetectorService().getDetectedLanguage(new String(Files.readAllBytes(projectFile.toPath())));
|
String detectedLanguage = getDetectorService().getDetectedLanguage(
|
||||||
|
projectFile.getName(),
|
||||||
|
new String(Files.readAllBytes(projectFile.toPath())),
|
||||||
|
DetectorService.LangDetectScrupulousness.LOW);
|
||||||
projectFileLanguageFrequency.put(detectedLanguage, projectFileLanguageFrequency.getOrDefault(detectedLanguage, 0) + 1);
|
projectFileLanguageFrequency.put(detectedLanguage, projectFileLanguageFrequency.getOrDefault(detectedLanguage, 0) + 1);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
|
@ -1,3 +1,8 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Anton Romanov - All Rights Reserved
|
||||||
|
* You may use, distribute and modify this code, please write to: romanov73@gmail.com.
|
||||||
|
*/
|
||||||
|
|
||||||
package ru.ulstu.extractor.heuristic.service;
|
package ru.ulstu.extractor.heuristic.service;
|
||||||
|
|
||||||
import com.gargoylesoftware.htmlunit.WebClient;
|
import com.gargoylesoftware.htmlunit.WebClient;
|
||||||
@ -9,11 +14,22 @@ import java.util.Arrays;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
|
|
||||||
|
import static ru.ulstu.extractor.heuristic.service.DetectorService.LangDetectScrupulousness.LOW;
|
||||||
|
|
||||||
@Service
|
@Service
|
||||||
public class DetectorService {
|
public class DetectorService {
|
||||||
|
public enum LangDetectScrupulousness {LOW, HIGH}
|
||||||
|
|
||||||
private final static String BASE_URL = "http://localhost:8080/lang-detector.html";
|
private final static String BASE_URL = "http://localhost:8080/lang-detector.html";
|
||||||
|
|
||||||
public String getDetectedLanguage(String code) {
|
public String getDetectedLanguage(String fileName, String code, LangDetectScrupulousness scrupulousness) {
|
||||||
|
if (scrupulousness == LOW) {
|
||||||
|
return DirectoryService.getFileExtension(fileName).orElse("");
|
||||||
|
}
|
||||||
|
return getDetectedLanguage(code);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getDetectedLanguage(String code) {
|
||||||
String selectedLang = null;
|
String selectedLang = null;
|
||||||
try (WebClient webClient = new WebClient()) {
|
try (WebClient webClient = new WebClient()) {
|
||||||
webClient.setJavaScriptTimeout(60 * 1000);
|
webClient.setJavaScriptTimeout(60 * 1000);
|
||||||
|
@ -16,6 +16,7 @@ import java.util.Arrays;
|
|||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
|
import java.util.Optional;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -23,6 +24,7 @@ import java.util.stream.Collectors;
|
|||||||
*/
|
*/
|
||||||
@Service
|
@Service
|
||||||
public class DirectoryService {
|
public class DirectoryService {
|
||||||
|
private static final String FILE_EXTENSION_DELIMITER = ".";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Получить список файлов, рекурсивно обойдя все дерево каталогов проекта.
|
* Получить список файлов, рекурсивно обойдя все дерево каталогов проекта.
|
||||||
@ -51,4 +53,11 @@ public class DirectoryService {
|
|||||||
}
|
}
|
||||||
return Collections.emptyList();
|
return Collections.emptyList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static Optional<String> getFileExtension(String fileName) {
|
||||||
|
if (fileName == null || fileName.isEmpty() || fileName.lastIndexOf(FILE_EXTENSION_DELIMITER) < 0) {
|
||||||
|
return Optional.empty();
|
||||||
|
}
|
||||||
|
return Optional.of(fileName.substring(fileName.lastIndexOf(FILE_EXTENSION_DELIMITER) + 1));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user