From 371dfe566591f928f6299b0866f0d4cf5d4db3aa Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Thu, 21 Apr 2022 11:54:55 +0400 Subject: [PATCH 01/25] #3 -- add fuzzy time series for forecasting --- .../ulstu/method/ftransform/AComponent.java | 29 ++++++++---- .../ulstu/method/ftransform/FTransform.java | 47 ++++++++++++++----- 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/src/main/java/ru/ulstu/method/ftransform/AComponent.java b/src/main/java/ru/ulstu/method/ftransform/AComponent.java index 5ba4fe4..3292f37 100644 --- a/src/main/java/ru/ulstu/method/ftransform/AComponent.java +++ b/src/main/java/ru/ulstu/method/ftransform/AComponent.java @@ -1,21 +1,21 @@ package ru.ulstu.method.ftransform; /** - * Треугольная функция принадлежности + * Базисная функция */ public class AComponent { - private int start; // левая граница треугольника - private int end; // правая граница треугольника - private int top; // вершина треугольника + private double start; // левая граница треугольника + private double end; // правая граница треугольника + private double top; // вершина треугольника - public int getStart() { + public double getStart() { return start; } public AComponent() { } - public AComponent(int start, int top, int end) { + public AComponent(double start, double top, double end) { this.start = start; this.top = top; this.end = end; @@ -26,19 +26,23 @@ public class AComponent { } - public int getEnd() { + public double getEnd() { return end; } - public void setEnd(int end) { + public void setEnd(double end) { this.end = end; } - public int getTop() { + public double getTop() { return top; } - public void setTop(int top) { + public int getTopInt() { + return (int) Math.round(top); + } + + public void setTop(double top) { this.top = top; } @@ -54,4 +58,9 @@ public class AComponent { } return 0; } + + @Override + public String toString() { + return "start=" + start; + } } diff --git a/src/main/java/ru/ulstu/method/ftransform/FTransform.java b/src/main/java/ru/ulstu/method/ftransform/FTransform.java index e533c9c..eb266e1 100644 --- a/src/main/java/ru/ulstu/method/ftransform/FTransform.java +++ b/src/main/java/ru/ulstu/method/ftransform/FTransform.java @@ -3,14 +3,17 @@ package ru.ulstu.method.ftransform; import org.springframework.stereotype.Component; import ru.ulstu.datamodel.Model; import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.datamodel.ts.TimeSeriesValue; import ru.ulstu.method.Method; import ru.ulstu.method.MethodParamValue; import ru.ulstu.method.MethodParameter; +import java.util.ArrayList; import java.util.List; @Component public class FTransform extends Method { + private final static int NUMBER_OF_FUZZY_VALUES = 3; @Override protected FTransformModel getModelOfValidTimeSeries(TimeSeries ts, @@ -29,8 +32,8 @@ public class FTransform extends Method { sum1 += membership * ts.getNumericValue(j); sum2 += membership; } - piecewiseLinearTrend.addValue(ts.getValue(aComponent.getTop()), sum1 / sum2); - tsModel.addValue(ts.getValue(aComponent.getTop()), sum1 / sum2); + piecewiseLinearTrend.addValue(ts.getValue(aComponent.getTopInt()), sum1 / sum2); + tsModel.addValue(ts.getValue(aComponent.getTopInt()), sum1 / sum2); } return model; } @@ -41,27 +44,27 @@ public class FTransform extends Method { while (currentPoint < ts.getLength()) { int startPoint = (currentPoint == 0) ? 0 - : piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getTop(); - AComponent bf = new AComponent(startPoint, currentPoint, (int) (currentPoint + Math.round(numberOfCoveredPoints / 2.0))); - if (bf.getStart() < 0) { - bf.setStart(0); + : piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getTopInt(); + AComponent aComponent = new AComponent(startPoint, currentPoint, (int) (currentPoint + Math.round(numberOfCoveredPoints / 2.0))); + if (aComponent.getStart() < 0) { + aComponent.setStart(0); } - if (bf.getEnd() > ts.getLength() - 1) { - bf.setEnd(ts.getLength() - 1); + if (aComponent.getEnd() > ts.getLength() - 1) { + aComponent.setEnd(ts.getLength() - 1); } - if (bf.getTop() > ts.getLength() - 1) { - bf.setTop(ts.getLength() - 1); + if (aComponent.getTop() > ts.getLength() - 1) { + aComponent.setTop(ts.getLength() - 1); } - piecewiseLinearTrend.add(bf); + piecewiseLinearTrend.add(aComponent); currentPoint += deltaForTriangle; } if (piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getEnd() != piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getTop()) { - AComponent bf = new AComponent(piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getTop(), + AComponent aComponent = new AComponent(piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getTop(), piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getEnd(), piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getEnd()); - piecewiseLinearTrend.add(bf); + piecewiseLinearTrend.add(aComponent); } return piecewiseLinearTrend; } @@ -69,6 +72,24 @@ public class FTransform extends Method { @Override protected TimeSeries getForecastWithValidParams(Model model, TimeSeries forecast) { FTransformModel fTransformModel = (FTransformModel) model; + double minValue = fTransformModel.getPiecewiseLinearTrend().getValues().stream().map(TimeSeriesValue::getValue).min(Double::compareTo).orElse(0.0); + double maxValue = fTransformModel.getPiecewiseLinearTrend().getValues().stream().map(TimeSeriesValue::getValue).max(Double::compareTo).orElse(0.0); + List fuzzyValues = new ArrayList<>(); + double diff = (maxValue - minValue) / NUMBER_OF_FUZZY_VALUES; + for (int i = 0; i < NUMBER_OF_FUZZY_VALUES; i++) { + fuzzyValues.add(new AComponent(i * diff, i * diff + diff / 2, i * diff + diff)); + } + List fuzzyTimeSeries = new ArrayList<>(); + + for (int i = 0; i < model.getTimeSeriesModel().getLength(); i++) { + for (AComponent fuzzyValue : fuzzyValues) { + if (model.getTimeSeriesModel().getValue(i).getValue() >= fuzzyValue.getStart() + && model.getTimeSeriesModel().getValue(i).getValue() <= fuzzyValue.getEnd()) { + fuzzyTimeSeries.add(fuzzyValue); + } + } + } + for (int t = 1; t < forecast.getLength(); t++) { forecast.getValues().get(t).setValue(fTransformModel.getPiecewiseLinearTrend().getLastValue().getValue()); } -- 2.34.1 From 0a017f86678ac878efb9b6f6176c9655e266f9b0 Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Sat, 8 Oct 2022 11:56:36 +0400 Subject: [PATCH 02/25] #3 -- some fixes --- .../ru/ulstu/method/ftransform/FTransform.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/ulstu/method/ftransform/FTransform.java b/src/main/java/ru/ulstu/method/ftransform/FTransform.java index eb266e1..f84e613 100644 --- a/src/main/java/ru/ulstu/method/ftransform/FTransform.java +++ b/src/main/java/ru/ulstu/method/ftransform/FTransform.java @@ -9,7 +9,9 @@ import ru.ulstu.method.MethodParamValue; import ru.ulstu.method.MethodParameter; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; @Component public class FTransform extends Method { @@ -77,7 +79,7 @@ public class FTransform extends Method { List fuzzyValues = new ArrayList<>(); double diff = (maxValue - minValue) / NUMBER_OF_FUZZY_VALUES; for (int i = 0; i < NUMBER_OF_FUZZY_VALUES; i++) { - fuzzyValues.add(new AComponent(i * diff, i * diff + diff / 2, i * diff + diff)); + fuzzyValues.add(new AComponent(minValue + i * diff, minValue + i * diff + diff / 2, minValue + i * diff + diff)); } List fuzzyTimeSeries = new ArrayList<>(); @@ -90,6 +92,7 @@ public class FTransform extends Method { } } + for (int t = 1; t < forecast.getLength(); t++) { forecast.getValues().get(t).setValue(fTransformModel.getPiecewiseLinearTrend().getLastValue().getValue()); } @@ -101,6 +104,16 @@ public class FTransform extends Method { return FTransformModel.getAvailableParameters(); } + private AComponent findFuzzyValueByPrevious(List fuzzyTimeSeries, AComponent currentFuzzyValue) { + Map mostFrequentValues = new HashMap<>(); + for (int i = 0; i < fuzzyTimeSeries.size() - 1; i++) { + if (fuzzyTimeSeries.get(i).equals(currentFuzzyValue)) { + mostFrequentValues.put(fuzzyTimeSeries.get(i + 1), mostFrequentValues.getOrDefault(fuzzyTimeSeries.get(i + 1), 0) + 1); + } + } + mostFrequentValues.entrySet().stream() + } + @Override public String getName() { return "F - преобразование"; -- 2.34.1 From c06688d7440b4c3b016cb78e9be31c9afe5689f2 Mon Sep 17 00:00:00 2001 From: sam <318597anton> Date: Mon, 3 Apr 2023 22:45:22 +0400 Subject: [PATCH 03/25] add min and max --- .../java/ru/ulstu/datamodel/ts/TimeSeries.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java b/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java index 8260c49..dfa8373 100644 --- a/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java +++ b/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java @@ -88,6 +88,20 @@ public class TimeSeries { throw new RuntimeException("Индекс выходит за границы временного ряда"); } + public TimeSeriesValue getMax() { + if ((values.size() > 0)) { + return values.stream().max((o1, o2) -> o1.getValue().compareTo(o2.getValue())).get(); + } + throw new RuntimeException("Временной ряд пустой"); + } + + public TimeSeriesValue getMin() { + if ((values.size() > 0)) { + return values.stream().min((o1, o2) -> o1.getValue().compareTo(o2.getValue())).get(); + } + throw new RuntimeException("Временной ряд пустой"); + } + @Override public String toString() { return "TimeSeries{" + -- 2.34.1 From 682370f3da3f8391f63fcfa11f8d81aff1667042 Mon Sep 17 00:00:00 2001 From: sam <318597anton> Date: Mon, 3 Apr 2023 22:47:12 +0400 Subject: [PATCH 04/25] edit min and max --- src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java b/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java index dfa8373..3bc3146 100644 --- a/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java +++ b/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; public class TimeSeries { @@ -90,14 +91,14 @@ public class TimeSeries { public TimeSeriesValue getMax() { if ((values.size() > 0)) { - return values.stream().max((o1, o2) -> o1.getValue().compareTo(o2.getValue())).get(); + return values.stream().max(Comparator.comparing(TimeSeriesValue::getValue)).get(); } throw new RuntimeException("Временной ряд пустой"); } public TimeSeriesValue getMin() { if ((values.size() > 0)) { - return values.stream().min((o1, o2) -> o1.getValue().compareTo(o2.getValue())).get(); + return values.stream().min(Comparator.comparing(TimeSeriesValue::getValue)).get(); } throw new RuntimeException("Временной ряд пустой"); } -- 2.34.1 From ffe303c7affbea2bbb36bbbb0be921b3f08afb29 Mon Sep 17 00:00:00 2001 From: sam <318597anton> Date: Mon, 3 Apr 2023 22:49:23 +0400 Subject: [PATCH 05/25] comment some code --- .../ru/ulstu/method/ftransform/FTransform.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/ru/ulstu/method/ftransform/FTransform.java b/src/main/java/ru/ulstu/method/ftransform/FTransform.java index f84e613..4c9edd7 100644 --- a/src/main/java/ru/ulstu/method/ftransform/FTransform.java +++ b/src/main/java/ru/ulstu/method/ftransform/FTransform.java @@ -104,15 +104,15 @@ public class FTransform extends Method { return FTransformModel.getAvailableParameters(); } - private AComponent findFuzzyValueByPrevious(List fuzzyTimeSeries, AComponent currentFuzzyValue) { - Map mostFrequentValues = new HashMap<>(); - for (int i = 0; i < fuzzyTimeSeries.size() - 1; i++) { - if (fuzzyTimeSeries.get(i).equals(currentFuzzyValue)) { - mostFrequentValues.put(fuzzyTimeSeries.get(i + 1), mostFrequentValues.getOrDefault(fuzzyTimeSeries.get(i + 1), 0) + 1); - } - } - mostFrequentValues.entrySet().stream() - } +// private AComponent findFuzzyValueByPrevious(List fuzzyTimeSeries, AComponent currentFuzzyValue) { +// Map mostFrequentValues = new HashMap<>(); +// for (int i = 0; i < fuzzyTimeSeries.size() - 1; i++) { +// if (fuzzyTimeSeries.get(i).equals(currentFuzzyValue)) { +// mostFrequentValues.put(fuzzyTimeSeries.get(i + 1), mostFrequentValues.getOrDefault(fuzzyTimeSeries.get(i + 1), 0) + 1); +// } +// } +// mostFrequentValues.entrySet().stream() +// } @Override public String getName() { -- 2.34.1 From 4cb14518db029ee35c972da284dac9f70d9a0207 Mon Sep 17 00:00:00 2001 From: sam <318597anton> Date: Mon, 3 Apr 2023 22:50:42 +0400 Subject: [PATCH 06/25] add class Target --- src/main/java/ru/ulstu/target/Target.java | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/main/java/ru/ulstu/target/Target.java diff --git a/src/main/java/ru/ulstu/target/Target.java b/src/main/java/ru/ulstu/target/Target.java new file mode 100644 index 0000000..bd11e58 --- /dev/null +++ b/src/main/java/ru/ulstu/target/Target.java @@ -0,0 +1,22 @@ +package ru.ulstu.target; + +import ru.ulstu.datamodel.exception.ModelingException; +import ru.ulstu.datamodel.ts.TimeSeries; + +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ExecutionException; + +public abstract class Target { + private final String name; + + public Target(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public abstract TimeSeries calculate(TimeSeries model) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException; + +} -- 2.34.1 From 292e774d8ed94e0bd91f8698ab84a5ae69767ba6 Mon Sep 17 00:00:00 2001 From: sam <318597anton> Date: Mon, 3 Apr 2023 22:51:19 +0400 Subject: [PATCH 07/25] add Target implementation --- .../ru/ulstu/target/AnomalyDecompose.java | 17 +++++++++ .../target/AnomalyDifferenceSmoothed.java | 35 +++++++++++++++++++ .../ru/ulstu/target/AnomalyPredictive.java | 14 ++++++++ 3 files changed, 66 insertions(+) create mode 100644 src/main/java/ru/ulstu/target/AnomalyDecompose.java create mode 100644 src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java create mode 100644 src/main/java/ru/ulstu/target/AnomalyPredictive.java diff --git a/src/main/java/ru/ulstu/target/AnomalyDecompose.java b/src/main/java/ru/ulstu/target/AnomalyDecompose.java new file mode 100644 index 0000000..77dad83 --- /dev/null +++ b/src/main/java/ru/ulstu/target/AnomalyDecompose.java @@ -0,0 +1,17 @@ +package ru.ulstu.target; + +import org.springframework.stereotype.Service; +import ru.ulstu.datamodel.ts.TimeSeries; + +@Service +public class AnomalyDecompose extends Target { + public AnomalyDecompose() { + super("Разложение"); + } + + @Override + public TimeSeries calculate(TimeSeries model) { + + return null; + } +} \ No newline at end of file diff --git a/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java b/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java new file mode 100644 index 0000000..41755d7 --- /dev/null +++ b/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java @@ -0,0 +1,35 @@ +package ru.ulstu.target; + +import org.springframework.stereotype.Service; +import ru.ulstu.datamodel.ModelingResult; +import ru.ulstu.datamodel.exception.ModelingException; +import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.datamodel.ts.TimeSeriesValue; +import ru.ulstu.service.TimeSeriesService; + +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ExecutionException; + +@Service +public class AnomalyDifferenceSmoothed extends Target { + + private final TimeSeriesService timeSeriesService; + + public AnomalyDifferenceSmoothed(TimeSeriesService timeSeriesService) { + super("Модель сравнения со сглаженным ВР"); + this.timeSeriesService = timeSeriesService; + } + + @Override + public TimeSeries calculate(TimeSeries ts) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + TimeSeries anomalyPoints = new TimeSeries(); + TimeSeries residual = new TimeSeries(); + ModelingResult modelingResult = timeSeriesService.smoothTimeSeries(ts, "AddTrendAddSeason"); + var tsResult = modelingResult.getTimeSeries(); + + for (int i = 0; i < ts.getLength(); i++) { + residual.addValue(new TimeSeriesValue(ts.getValue(i).getDate(), Math.abs(ts.getNumericValue(i) - tsResult.getNumericValue(i)))); + } + return null; + } +} \ No newline at end of file diff --git a/src/main/java/ru/ulstu/target/AnomalyPredictive.java b/src/main/java/ru/ulstu/target/AnomalyPredictive.java new file mode 100644 index 0000000..49a1ac8 --- /dev/null +++ b/src/main/java/ru/ulstu/target/AnomalyPredictive.java @@ -0,0 +1,14 @@ +package ru.ulstu.target; + +import ru.ulstu.datamodel.ts.TimeSeries; + +public class AnomalyPredictive extends Target { + public AnomalyPredictive() { + super("Прогнозная модель, %"); + } + + @Override + public TimeSeries calculate(TimeSeries model) { + return null; + } +} \ No newline at end of file -- 2.34.1 From a8b6cf7d2902903ede2c3f97517703d9dc8217ab Mon Sep 17 00:00:00 2001 From: sam <318597anton> Date: Mon, 3 Apr 2023 22:51:39 +0400 Subject: [PATCH 08/25] add AnomalyController --- .../ulstu/controller/AnomalyController.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/main/java/ru/ulstu/controller/AnomalyController.java diff --git a/src/main/java/ru/ulstu/controller/AnomalyController.java b/src/main/java/ru/ulstu/controller/AnomalyController.java new file mode 100644 index 0000000..b2f26e7 --- /dev/null +++ b/src/main/java/ru/ulstu/controller/AnomalyController.java @@ -0,0 +1,40 @@ +package ru.ulstu.controller; + +import io.swagger.v3.oas.annotations.Operation; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import ru.ulstu.configuration.ApiConfiguration; +import ru.ulstu.datamodel.exception.ModelingException; +import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.estimate.CompressionMetricService; +import ru.ulstu.service.UtilService; +import ru.ulstu.target.AnomalyDifferenceSmoothed; +import ru.ulstu.target.Target; + +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ExecutionException; + +@RestController +@RequestMapping(ApiConfiguration.API_1_0) +public class AnomalyController { + + private final UtilService utilService; + private final Target target; + + public AnomalyController(UtilService utilService, CompressionMetricService compressionMetricService, AnomalyDifferenceSmoothed anomalyDifferenceSmoothed) { + this.utilService = utilService; + this.target = anomalyDifferenceSmoothed; + } + + @PostMapping("getAnomalyAtDifferenceSmothed") + @Operation(description = "Получить метрику сжатия") + public ResponseEntity getRandomTimeSeries(@RequestParam("length") int length) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + var randomTS = utilService.getRandomTimeSeries(length); + var tsResult = target.calculate(randomTS); + return new ResponseEntity<>( tsResult, HttpStatus.OK); + } +} -- 2.34.1 From 0da256106ed2ac928e47171612645e611757d9cf Mon Sep 17 00:00:00 2001 From: sam <318597anton> Date: Mon, 3 Apr 2023 22:51:59 +0400 Subject: [PATCH 09/25] add CompressionMetricClasses --- .../CompressionMetricController.java | 38 +++++++++++++++++++ .../estimate/CompressionMetricService.java | 38 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 src/main/java/ru/ulstu/controller/CompressionMetricController.java create mode 100644 src/main/java/ru/ulstu/estimate/CompressionMetricService.java diff --git a/src/main/java/ru/ulstu/controller/CompressionMetricController.java b/src/main/java/ru/ulstu/controller/CompressionMetricController.java new file mode 100644 index 0000000..f05eee8 --- /dev/null +++ b/src/main/java/ru/ulstu/controller/CompressionMetricController.java @@ -0,0 +1,38 @@ +package ru.ulstu.controller; + +import io.swagger.v3.oas.annotations.Operation; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import ru.ulstu.configuration.ApiConfiguration; +import ru.ulstu.datamodel.exception.ModelingException; +import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.estimate.CompressionMetricService; +import ru.ulstu.service.UtilService; + +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ExecutionException; + +@RestController +@RequestMapping(ApiConfiguration.API_1_0) +public class CompressionMetricController { + + private final UtilService utilService; + private final CompressionMetricService compressionMetricService; + + public CompressionMetricController(UtilService utilService, CompressionMetricService compressionMetricService) { + this.utilService = utilService; + this.compressionMetricService = compressionMetricService; + } + + @PostMapping("getMetricOfRandom") + @Operation(description = "Получить метрику сжатия") + public ResponseEntity getRandomTimeSeries(@RequestParam("length") int length) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + var randomTS = utilService.getRandomTimeSeries(length); + var ts1 = compressionMetricService.getCompressionTimeSeries(randomTS); + return new ResponseEntity<>( null/*Сюда вывод*/, HttpStatus.OK); + } +} diff --git a/src/main/java/ru/ulstu/estimate/CompressionMetricService.java b/src/main/java/ru/ulstu/estimate/CompressionMetricService.java new file mode 100644 index 0000000..0dfaa24 --- /dev/null +++ b/src/main/java/ru/ulstu/estimate/CompressionMetricService.java @@ -0,0 +1,38 @@ +package ru.ulstu.estimate; + + +import org.springframework.stereotype.Service; +import ru.ulstu.datamodel.ModelingResult; +import ru.ulstu.datamodel.exception.ModelingException; +import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.service.TimeSeriesService; + +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ExecutionException; + +@Service +public class CompressionMetricService { + + private final TimeSeriesService timeSeriesService; + + public CompressionMetricService(TimeSeriesService timeSeriesService) { + this.timeSeriesService = timeSeriesService; + } + + public void getCompressionMetric() { + + } + + public void getCompressionMetricForAnomaly() { + + } + + public TimeSeries getCompressionTimeSeries(TimeSeries ts) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + ModelingResult modelingResult = timeSeriesService.smoothTimeSeries(ts, "FTransform"); + //var ts1 = fTransform.getForecast(ts, ) + return modelingResult.getTimeSeries(); + } + + + +} -- 2.34.1 From e9adff9a8744f144dc22c8d0ab8ebbdd9e5be055 Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Thu, 21 Apr 2022 11:54:55 +0400 Subject: [PATCH 10/25] #3 -- add fuzzy time series for forecasting --- .../ulstu/method/ftransform/AComponent.java | 29 ++++++++---- .../ulstu/method/ftransform/FTransform.java | 47 ++++++++++++++----- 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/src/main/java/ru/ulstu/method/ftransform/AComponent.java b/src/main/java/ru/ulstu/method/ftransform/AComponent.java index 5ba4fe4..3292f37 100644 --- a/src/main/java/ru/ulstu/method/ftransform/AComponent.java +++ b/src/main/java/ru/ulstu/method/ftransform/AComponent.java @@ -1,21 +1,21 @@ package ru.ulstu.method.ftransform; /** - * Треугольная функция принадлежности + * Базисная функция */ public class AComponent { - private int start; // левая граница треугольника - private int end; // правая граница треугольника - private int top; // вершина треугольника + private double start; // левая граница треугольника + private double end; // правая граница треугольника + private double top; // вершина треугольника - public int getStart() { + public double getStart() { return start; } public AComponent() { } - public AComponent(int start, int top, int end) { + public AComponent(double start, double top, double end) { this.start = start; this.top = top; this.end = end; @@ -26,19 +26,23 @@ public class AComponent { } - public int getEnd() { + public double getEnd() { return end; } - public void setEnd(int end) { + public void setEnd(double end) { this.end = end; } - public int getTop() { + public double getTop() { return top; } - public void setTop(int top) { + public int getTopInt() { + return (int) Math.round(top); + } + + public void setTop(double top) { this.top = top; } @@ -54,4 +58,9 @@ public class AComponent { } return 0; } + + @Override + public String toString() { + return "start=" + start; + } } diff --git a/src/main/java/ru/ulstu/method/ftransform/FTransform.java b/src/main/java/ru/ulstu/method/ftransform/FTransform.java index e533c9c..eb266e1 100644 --- a/src/main/java/ru/ulstu/method/ftransform/FTransform.java +++ b/src/main/java/ru/ulstu/method/ftransform/FTransform.java @@ -3,14 +3,17 @@ package ru.ulstu.method.ftransform; import org.springframework.stereotype.Component; import ru.ulstu.datamodel.Model; import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.datamodel.ts.TimeSeriesValue; import ru.ulstu.method.Method; import ru.ulstu.method.MethodParamValue; import ru.ulstu.method.MethodParameter; +import java.util.ArrayList; import java.util.List; @Component public class FTransform extends Method { + private final static int NUMBER_OF_FUZZY_VALUES = 3; @Override protected FTransformModel getModelOfValidTimeSeries(TimeSeries ts, @@ -29,8 +32,8 @@ public class FTransform extends Method { sum1 += membership * ts.getNumericValue(j); sum2 += membership; } - piecewiseLinearTrend.addValue(ts.getValue(aComponent.getTop()), sum1 / sum2); - tsModel.addValue(ts.getValue(aComponent.getTop()), sum1 / sum2); + piecewiseLinearTrend.addValue(ts.getValue(aComponent.getTopInt()), sum1 / sum2); + tsModel.addValue(ts.getValue(aComponent.getTopInt()), sum1 / sum2); } return model; } @@ -41,27 +44,27 @@ public class FTransform extends Method { while (currentPoint < ts.getLength()) { int startPoint = (currentPoint == 0) ? 0 - : piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getTop(); - AComponent bf = new AComponent(startPoint, currentPoint, (int) (currentPoint + Math.round(numberOfCoveredPoints / 2.0))); - if (bf.getStart() < 0) { - bf.setStart(0); + : piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getTopInt(); + AComponent aComponent = new AComponent(startPoint, currentPoint, (int) (currentPoint + Math.round(numberOfCoveredPoints / 2.0))); + if (aComponent.getStart() < 0) { + aComponent.setStart(0); } - if (bf.getEnd() > ts.getLength() - 1) { - bf.setEnd(ts.getLength() - 1); + if (aComponent.getEnd() > ts.getLength() - 1) { + aComponent.setEnd(ts.getLength() - 1); } - if (bf.getTop() > ts.getLength() - 1) { - bf.setTop(ts.getLength() - 1); + if (aComponent.getTop() > ts.getLength() - 1) { + aComponent.setTop(ts.getLength() - 1); } - piecewiseLinearTrend.add(bf); + piecewiseLinearTrend.add(aComponent); currentPoint += deltaForTriangle; } if (piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getEnd() != piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getTop()) { - AComponent bf = new AComponent(piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getTop(), + AComponent aComponent = new AComponent(piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getTop(), piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getEnd(), piecewiseLinearTrend.get(piecewiseLinearTrend.size() - 1).getEnd()); - piecewiseLinearTrend.add(bf); + piecewiseLinearTrend.add(aComponent); } return piecewiseLinearTrend; } @@ -69,6 +72,24 @@ public class FTransform extends Method { @Override protected TimeSeries getForecastWithValidParams(Model model, TimeSeries forecast) { FTransformModel fTransformModel = (FTransformModel) model; + double minValue = fTransformModel.getPiecewiseLinearTrend().getValues().stream().map(TimeSeriesValue::getValue).min(Double::compareTo).orElse(0.0); + double maxValue = fTransformModel.getPiecewiseLinearTrend().getValues().stream().map(TimeSeriesValue::getValue).max(Double::compareTo).orElse(0.0); + List fuzzyValues = new ArrayList<>(); + double diff = (maxValue - minValue) / NUMBER_OF_FUZZY_VALUES; + for (int i = 0; i < NUMBER_OF_FUZZY_VALUES; i++) { + fuzzyValues.add(new AComponent(i * diff, i * diff + diff / 2, i * diff + diff)); + } + List fuzzyTimeSeries = new ArrayList<>(); + + for (int i = 0; i < model.getTimeSeriesModel().getLength(); i++) { + for (AComponent fuzzyValue : fuzzyValues) { + if (model.getTimeSeriesModel().getValue(i).getValue() >= fuzzyValue.getStart() + && model.getTimeSeriesModel().getValue(i).getValue() <= fuzzyValue.getEnd()) { + fuzzyTimeSeries.add(fuzzyValue); + } + } + } + for (int t = 1; t < forecast.getLength(); t++) { forecast.getValues().get(t).setValue(fTransformModel.getPiecewiseLinearTrend().getLastValue().getValue()); } -- 2.34.1 From 55275e17169fec8c5e2d79ff82c11fc3124162c4 Mon Sep 17 00:00:00 2001 From: Anton Romanov Date: Sat, 8 Oct 2022 11:56:36 +0400 Subject: [PATCH 11/25] #3 -- some fixes --- .../ru/ulstu/method/ftransform/FTransform.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/ulstu/method/ftransform/FTransform.java b/src/main/java/ru/ulstu/method/ftransform/FTransform.java index eb266e1..f84e613 100644 --- a/src/main/java/ru/ulstu/method/ftransform/FTransform.java +++ b/src/main/java/ru/ulstu/method/ftransform/FTransform.java @@ -9,7 +9,9 @@ import ru.ulstu.method.MethodParamValue; import ru.ulstu.method.MethodParameter; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; @Component public class FTransform extends Method { @@ -77,7 +79,7 @@ public class FTransform extends Method { List fuzzyValues = new ArrayList<>(); double diff = (maxValue - minValue) / NUMBER_OF_FUZZY_VALUES; for (int i = 0; i < NUMBER_OF_FUZZY_VALUES; i++) { - fuzzyValues.add(new AComponent(i * diff, i * diff + diff / 2, i * diff + diff)); + fuzzyValues.add(new AComponent(minValue + i * diff, minValue + i * diff + diff / 2, minValue + i * diff + diff)); } List fuzzyTimeSeries = new ArrayList<>(); @@ -90,6 +92,7 @@ public class FTransform extends Method { } } + for (int t = 1; t < forecast.getLength(); t++) { forecast.getValues().get(t).setValue(fTransformModel.getPiecewiseLinearTrend().getLastValue().getValue()); } @@ -101,6 +104,16 @@ public class FTransform extends Method { return FTransformModel.getAvailableParameters(); } + private AComponent findFuzzyValueByPrevious(List fuzzyTimeSeries, AComponent currentFuzzyValue) { + Map mostFrequentValues = new HashMap<>(); + for (int i = 0; i < fuzzyTimeSeries.size() - 1; i++) { + if (fuzzyTimeSeries.get(i).equals(currentFuzzyValue)) { + mostFrequentValues.put(fuzzyTimeSeries.get(i + 1), mostFrequentValues.getOrDefault(fuzzyTimeSeries.get(i + 1), 0) + 1); + } + } + mostFrequentValues.entrySet().stream() + } + @Override public String getName() { return "F - преобразование"; -- 2.34.1 From fae21d0f873c8186731f29af7dcd314ab0929196 Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Mon, 3 Apr 2023 22:45:22 +0400 Subject: [PATCH 12/25] add min and max --- .../java/ru/ulstu/datamodel/ts/TimeSeries.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java b/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java index 8260c49..dfa8373 100644 --- a/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java +++ b/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java @@ -88,6 +88,20 @@ public class TimeSeries { throw new RuntimeException("Индекс выходит за границы временного ряда"); } + public TimeSeriesValue getMax() { + if ((values.size() > 0)) { + return values.stream().max((o1, o2) -> o1.getValue().compareTo(o2.getValue())).get(); + } + throw new RuntimeException("Временной ряд пустой"); + } + + public TimeSeriesValue getMin() { + if ((values.size() > 0)) { + return values.stream().min((o1, o2) -> o1.getValue().compareTo(o2.getValue())).get(); + } + throw new RuntimeException("Временной ряд пустой"); + } + @Override public String toString() { return "TimeSeries{" + -- 2.34.1 From a81149bf01339ca5791133c754fa125dcbe416b5 Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Mon, 3 Apr 2023 22:47:12 +0400 Subject: [PATCH 13/25] edit min and max --- src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java b/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java index dfa8373..3bc3146 100644 --- a/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java +++ b/src/main/java/ru/ulstu/datamodel/ts/TimeSeries.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; public class TimeSeries { @@ -90,14 +91,14 @@ public class TimeSeries { public TimeSeriesValue getMax() { if ((values.size() > 0)) { - return values.stream().max((o1, o2) -> o1.getValue().compareTo(o2.getValue())).get(); + return values.stream().max(Comparator.comparing(TimeSeriesValue::getValue)).get(); } throw new RuntimeException("Временной ряд пустой"); } public TimeSeriesValue getMin() { if ((values.size() > 0)) { - return values.stream().min((o1, o2) -> o1.getValue().compareTo(o2.getValue())).get(); + return values.stream().min(Comparator.comparing(TimeSeriesValue::getValue)).get(); } throw new RuntimeException("Временной ряд пустой"); } -- 2.34.1 From 2b21708d9d1bf5101fabb2342795ad6a62f4adfa Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Mon, 3 Apr 2023 22:49:23 +0400 Subject: [PATCH 14/25] comment some code --- .../ru/ulstu/method/ftransform/FTransform.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/ru/ulstu/method/ftransform/FTransform.java b/src/main/java/ru/ulstu/method/ftransform/FTransform.java index f84e613..4c9edd7 100644 --- a/src/main/java/ru/ulstu/method/ftransform/FTransform.java +++ b/src/main/java/ru/ulstu/method/ftransform/FTransform.java @@ -104,15 +104,15 @@ public class FTransform extends Method { return FTransformModel.getAvailableParameters(); } - private AComponent findFuzzyValueByPrevious(List fuzzyTimeSeries, AComponent currentFuzzyValue) { - Map mostFrequentValues = new HashMap<>(); - for (int i = 0; i < fuzzyTimeSeries.size() - 1; i++) { - if (fuzzyTimeSeries.get(i).equals(currentFuzzyValue)) { - mostFrequentValues.put(fuzzyTimeSeries.get(i + 1), mostFrequentValues.getOrDefault(fuzzyTimeSeries.get(i + 1), 0) + 1); - } - } - mostFrequentValues.entrySet().stream() - } +// private AComponent findFuzzyValueByPrevious(List fuzzyTimeSeries, AComponent currentFuzzyValue) { +// Map mostFrequentValues = new HashMap<>(); +// for (int i = 0; i < fuzzyTimeSeries.size() - 1; i++) { +// if (fuzzyTimeSeries.get(i).equals(currentFuzzyValue)) { +// mostFrequentValues.put(fuzzyTimeSeries.get(i + 1), mostFrequentValues.getOrDefault(fuzzyTimeSeries.get(i + 1), 0) + 1); +// } +// } +// mostFrequentValues.entrySet().stream() +// } @Override public String getName() { -- 2.34.1 From 7a9044c090d9d1f71159f709ce258b170dbbd868 Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Mon, 3 Apr 2023 22:50:42 +0400 Subject: [PATCH 15/25] add class Target --- src/main/java/ru/ulstu/target/Target.java | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/main/java/ru/ulstu/target/Target.java diff --git a/src/main/java/ru/ulstu/target/Target.java b/src/main/java/ru/ulstu/target/Target.java new file mode 100644 index 0000000..bd11e58 --- /dev/null +++ b/src/main/java/ru/ulstu/target/Target.java @@ -0,0 +1,22 @@ +package ru.ulstu.target; + +import ru.ulstu.datamodel.exception.ModelingException; +import ru.ulstu.datamodel.ts.TimeSeries; + +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ExecutionException; + +public abstract class Target { + private final String name; + + public Target(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public abstract TimeSeries calculate(TimeSeries model) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException; + +} -- 2.34.1 From 63791ecfa018cd81d43afd7548b6c36b8e5ebd1f Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Mon, 3 Apr 2023 22:51:19 +0400 Subject: [PATCH 16/25] add Target implementation --- .../ru/ulstu/target/AnomalyDecompose.java | 17 +++++++++ .../target/AnomalyDifferenceSmoothed.java | 35 +++++++++++++++++++ .../ru/ulstu/target/AnomalyPredictive.java | 14 ++++++++ 3 files changed, 66 insertions(+) create mode 100644 src/main/java/ru/ulstu/target/AnomalyDecompose.java create mode 100644 src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java create mode 100644 src/main/java/ru/ulstu/target/AnomalyPredictive.java diff --git a/src/main/java/ru/ulstu/target/AnomalyDecompose.java b/src/main/java/ru/ulstu/target/AnomalyDecompose.java new file mode 100644 index 0000000..77dad83 --- /dev/null +++ b/src/main/java/ru/ulstu/target/AnomalyDecompose.java @@ -0,0 +1,17 @@ +package ru.ulstu.target; + +import org.springframework.stereotype.Service; +import ru.ulstu.datamodel.ts.TimeSeries; + +@Service +public class AnomalyDecompose extends Target { + public AnomalyDecompose() { + super("Разложение"); + } + + @Override + public TimeSeries calculate(TimeSeries model) { + + return null; + } +} \ No newline at end of file diff --git a/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java b/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java new file mode 100644 index 0000000..41755d7 --- /dev/null +++ b/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java @@ -0,0 +1,35 @@ +package ru.ulstu.target; + +import org.springframework.stereotype.Service; +import ru.ulstu.datamodel.ModelingResult; +import ru.ulstu.datamodel.exception.ModelingException; +import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.datamodel.ts.TimeSeriesValue; +import ru.ulstu.service.TimeSeriesService; + +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ExecutionException; + +@Service +public class AnomalyDifferenceSmoothed extends Target { + + private final TimeSeriesService timeSeriesService; + + public AnomalyDifferenceSmoothed(TimeSeriesService timeSeriesService) { + super("Модель сравнения со сглаженным ВР"); + this.timeSeriesService = timeSeriesService; + } + + @Override + public TimeSeries calculate(TimeSeries ts) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + TimeSeries anomalyPoints = new TimeSeries(); + TimeSeries residual = new TimeSeries(); + ModelingResult modelingResult = timeSeriesService.smoothTimeSeries(ts, "AddTrendAddSeason"); + var tsResult = modelingResult.getTimeSeries(); + + for (int i = 0; i < ts.getLength(); i++) { + residual.addValue(new TimeSeriesValue(ts.getValue(i).getDate(), Math.abs(ts.getNumericValue(i) - tsResult.getNumericValue(i)))); + } + return null; + } +} \ No newline at end of file diff --git a/src/main/java/ru/ulstu/target/AnomalyPredictive.java b/src/main/java/ru/ulstu/target/AnomalyPredictive.java new file mode 100644 index 0000000..49a1ac8 --- /dev/null +++ b/src/main/java/ru/ulstu/target/AnomalyPredictive.java @@ -0,0 +1,14 @@ +package ru.ulstu.target; + +import ru.ulstu.datamodel.ts.TimeSeries; + +public class AnomalyPredictive extends Target { + public AnomalyPredictive() { + super("Прогнозная модель, %"); + } + + @Override + public TimeSeries calculate(TimeSeries model) { + return null; + } +} \ No newline at end of file -- 2.34.1 From ab05f5a509ecb78a5c66e5de6ba9b057a5dd7c40 Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Mon, 3 Apr 2023 22:51:39 +0400 Subject: [PATCH 17/25] add AnomalyController --- .../ulstu/controller/AnomalyController.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/main/java/ru/ulstu/controller/AnomalyController.java diff --git a/src/main/java/ru/ulstu/controller/AnomalyController.java b/src/main/java/ru/ulstu/controller/AnomalyController.java new file mode 100644 index 0000000..b2f26e7 --- /dev/null +++ b/src/main/java/ru/ulstu/controller/AnomalyController.java @@ -0,0 +1,40 @@ +package ru.ulstu.controller; + +import io.swagger.v3.oas.annotations.Operation; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import ru.ulstu.configuration.ApiConfiguration; +import ru.ulstu.datamodel.exception.ModelingException; +import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.estimate.CompressionMetricService; +import ru.ulstu.service.UtilService; +import ru.ulstu.target.AnomalyDifferenceSmoothed; +import ru.ulstu.target.Target; + +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ExecutionException; + +@RestController +@RequestMapping(ApiConfiguration.API_1_0) +public class AnomalyController { + + private final UtilService utilService; + private final Target target; + + public AnomalyController(UtilService utilService, CompressionMetricService compressionMetricService, AnomalyDifferenceSmoothed anomalyDifferenceSmoothed) { + this.utilService = utilService; + this.target = anomalyDifferenceSmoothed; + } + + @PostMapping("getAnomalyAtDifferenceSmothed") + @Operation(description = "Получить метрику сжатия") + public ResponseEntity getRandomTimeSeries(@RequestParam("length") int length) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + var randomTS = utilService.getRandomTimeSeries(length); + var tsResult = target.calculate(randomTS); + return new ResponseEntity<>( tsResult, HttpStatus.OK); + } +} -- 2.34.1 From d236081a45997499b71d98cddc3f03516e359300 Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Mon, 3 Apr 2023 22:51:59 +0400 Subject: [PATCH 18/25] add CompressionMetricClasses --- .../CompressionMetricController.java | 38 +++++++++++++++++++ .../estimate/CompressionMetricService.java | 38 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 src/main/java/ru/ulstu/controller/CompressionMetricController.java create mode 100644 src/main/java/ru/ulstu/estimate/CompressionMetricService.java diff --git a/src/main/java/ru/ulstu/controller/CompressionMetricController.java b/src/main/java/ru/ulstu/controller/CompressionMetricController.java new file mode 100644 index 0000000..f05eee8 --- /dev/null +++ b/src/main/java/ru/ulstu/controller/CompressionMetricController.java @@ -0,0 +1,38 @@ +package ru.ulstu.controller; + +import io.swagger.v3.oas.annotations.Operation; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import ru.ulstu.configuration.ApiConfiguration; +import ru.ulstu.datamodel.exception.ModelingException; +import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.estimate.CompressionMetricService; +import ru.ulstu.service.UtilService; + +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ExecutionException; + +@RestController +@RequestMapping(ApiConfiguration.API_1_0) +public class CompressionMetricController { + + private final UtilService utilService; + private final CompressionMetricService compressionMetricService; + + public CompressionMetricController(UtilService utilService, CompressionMetricService compressionMetricService) { + this.utilService = utilService; + this.compressionMetricService = compressionMetricService; + } + + @PostMapping("getMetricOfRandom") + @Operation(description = "Получить метрику сжатия") + public ResponseEntity getRandomTimeSeries(@RequestParam("length") int length) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + var randomTS = utilService.getRandomTimeSeries(length); + var ts1 = compressionMetricService.getCompressionTimeSeries(randomTS); + return new ResponseEntity<>( null/*Сюда вывод*/, HttpStatus.OK); + } +} diff --git a/src/main/java/ru/ulstu/estimate/CompressionMetricService.java b/src/main/java/ru/ulstu/estimate/CompressionMetricService.java new file mode 100644 index 0000000..0dfaa24 --- /dev/null +++ b/src/main/java/ru/ulstu/estimate/CompressionMetricService.java @@ -0,0 +1,38 @@ +package ru.ulstu.estimate; + + +import org.springframework.stereotype.Service; +import ru.ulstu.datamodel.ModelingResult; +import ru.ulstu.datamodel.exception.ModelingException; +import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.service.TimeSeriesService; + +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ExecutionException; + +@Service +public class CompressionMetricService { + + private final TimeSeriesService timeSeriesService; + + public CompressionMetricService(TimeSeriesService timeSeriesService) { + this.timeSeriesService = timeSeriesService; + } + + public void getCompressionMetric() { + + } + + public void getCompressionMetricForAnomaly() { + + } + + public TimeSeries getCompressionTimeSeries(TimeSeries ts) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + ModelingResult modelingResult = timeSeriesService.smoothTimeSeries(ts, "FTransform"); + //var ts1 = fTransform.getForecast(ts, ) + return modelingResult.getTimeSeries(); + } + + + +} -- 2.34.1 From f4b8089f292bc9122a5f6c66a8481d9d436563d5 Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Tue, 4 Apr 2023 11:13:55 +0400 Subject: [PATCH 19/25] add calculate diff --- src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java b/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java index 41755d7..71a544d 100644 --- a/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java +++ b/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java @@ -27,6 +27,8 @@ public class AnomalyDifferenceSmoothed extends Target { ModelingResult modelingResult = timeSeriesService.smoothTimeSeries(ts, "AddTrendAddSeason"); var tsResult = modelingResult.getTimeSeries(); + double diff = anomalyPoints.getMax().getValue()-anomalyPoints.getMin().getValue(); + for (int i = 0; i < ts.getLength(); i++) { residual.addValue(new TimeSeriesValue(ts.getValue(i).getDate(), Math.abs(ts.getNumericValue(i) - tsResult.getNumericValue(i)))); } -- 2.34.1 From 57e588f4a0166f0f3b16d09fe1f09b30c9c0ed6e Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Tue, 4 Apr 2023 11:22:43 +0400 Subject: [PATCH 20/25] find anomalyPoints --- .../target/AnomalyDifferenceSmoothed.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java b/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java index 71a544d..27b6f56 100644 --- a/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java +++ b/src/main/java/ru/ulstu/target/AnomalyDifferenceSmoothed.java @@ -21,17 +21,26 @@ public class AnomalyDifferenceSmoothed extends Target { } @Override - public TimeSeries calculate(TimeSeries ts) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { - TimeSeries anomalyPoints = new TimeSeries(); + public TimeSeries calculate(TimeSeries timeSeries) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { TimeSeries residual = new TimeSeries(); - ModelingResult modelingResult = timeSeriesService.smoothTimeSeries(ts, "AddTrendAddSeason"); - var tsResult = modelingResult.getTimeSeries(); + TimeSeries anomalyPoints = new TimeSeries(); - double diff = anomalyPoints.getMax().getValue()-anomalyPoints.getMin().getValue(); + TimeSeries timeSeriesSmooth = timeSeriesService.smoothTimeSeries(timeSeries, "AddTrendAddSeason").getTimeSeries(); - for (int i = 0; i < ts.getLength(); i++) { - residual.addValue(new TimeSeriesValue(ts.getValue(i).getDate(), Math.abs(ts.getNumericValue(i) - tsResult.getNumericValue(i)))); + for (int i = 0; i < timeSeries.getLength(); i++) { + residual.addValue(new TimeSeriesValue(timeSeries.getValue(i).getDate(), + Math.abs(timeSeries.getNumericValue(i) - timeSeriesSmooth.getNumericValue(i)))); } - return null; + + double diff = timeSeries.getMax().getValue()-timeSeries.getMin().getValue(); + + for (int i = 0; i < residual.getLength(); i++) { + if (residual.getNumericValue(i)/diff > 0.05) { + anomalyPoints.addValue(new TimeSeriesValue(residual.getValue(i).getDate(), + timeSeries.getNumericValue(i))); + } + } + + return anomalyPoints; } } \ No newline at end of file -- 2.34.1 From cb52a2cce62719cbd36105540c83a9fa65216e29 Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Fri, 7 Apr 2023 00:09:49 +0400 Subject: [PATCH 21/25] add AnomalyCompressionScore --- .../ulstu/score/AnomalyCompressionScore.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/main/java/ru/ulstu/score/AnomalyCompressionScore.java diff --git a/src/main/java/ru/ulstu/score/AnomalyCompressionScore.java b/src/main/java/ru/ulstu/score/AnomalyCompressionScore.java new file mode 100644 index 0000000..883c989 --- /dev/null +++ b/src/main/java/ru/ulstu/score/AnomalyCompressionScore.java @@ -0,0 +1,32 @@ +package ru.ulstu.score; + +import org.springframework.stereotype.Component; +import ru.ulstu.datamodel.exception.ModelingException; +import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.datamodel.ts.TimeSeriesValue; +import ru.ulstu.target.AnomalyDifferenceSmoothed; + +import java.lang.reflect.InvocationTargetException; +import java.time.LocalDateTime; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static java.lang.Math.abs; + +@Component +public class AnomalyCompressionScore extends ScoreMethod { + + private final AnomalyDifferenceSmoothed anomalyDifferenceSmoothed; + public AnomalyCompressionScore(AnomalyDifferenceSmoothed anomalyDifferenceSmoothed) { + super("Smape, %"); + this.anomalyDifferenceSmoothed = anomalyDifferenceSmoothed; + } + + @Override + public Number evaluate(Map tsValues, TimeSeries model) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + var timeSeriesAnomalyResult = anomalyDifferenceSmoothed.calculate(model); + var timeSeriesAnomaly = anomalyDifferenceSmoothed.calculate(tsValues); + // туду: добавить сравнение аномальных точек + return model.getLength()/tsValues.size() + Math.abs(timeSeriesAnomaly.getLength() - timeSeriesAnomalyResult.getLength()); + } +} \ No newline at end of file -- 2.34.1 From 1c24fb805549de4d93d586f041a16d65e9df21fd Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Fri, 7 Apr 2023 00:10:37 +0400 Subject: [PATCH 22/25] edit some controllers --- .../ulstu/controller/AnomalyController.java | 26 +++++++++---------- .../CompressionMetricController.java | 26 ++++++++++++------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/main/java/ru/ulstu/controller/AnomalyController.java b/src/main/java/ru/ulstu/controller/AnomalyController.java index b2f26e7..4a93065 100644 --- a/src/main/java/ru/ulstu/controller/AnomalyController.java +++ b/src/main/java/ru/ulstu/controller/AnomalyController.java @@ -3,18 +3,18 @@ package ru.ulstu.controller; import io.swagger.v3.oas.annotations.Operation; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import ru.ulstu.configuration.ApiConfiguration; import ru.ulstu.datamodel.exception.ModelingException; import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.db.DbService; +import ru.ulstu.db.model.TimeSeriesSet; import ru.ulstu.estimate.CompressionMetricService; import ru.ulstu.service.UtilService; import ru.ulstu.target.AnomalyDifferenceSmoothed; import ru.ulstu.target.Target; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.concurrent.ExecutionException; @@ -22,19 +22,19 @@ import java.util.concurrent.ExecutionException; @RequestMapping(ApiConfiguration.API_1_0) public class AnomalyController { - private final UtilService utilService; + private final DbService dbService; private final Target target; - public AnomalyController(UtilService utilService, CompressionMetricService compressionMetricService, AnomalyDifferenceSmoothed anomalyDifferenceSmoothed) { - this.utilService = utilService; + public AnomalyController(CompressionMetricService compressionMetricService, DbService dbService, AnomalyDifferenceSmoothed anomalyDifferenceSmoothed) { + this.dbService = dbService; this.target = anomalyDifferenceSmoothed; } - @PostMapping("getAnomalyAtDifferenceSmothed") - @Operation(description = "Получить метрику сжатия") - public ResponseEntity getRandomTimeSeries(@RequestParam("length") int length) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { - var randomTS = utilService.getRandomTimeSeries(length); - var tsResult = target.calculate(randomTS); - return new ResponseEntity<>( tsResult, HttpStatus.OK); + @GetMapping ("getAnomalyAtDifferenceSmothed") + @Operation(description = "Получить аномальные значения") + public TimeSeries getAnomalyAtDifferenceSmothed(@RequestParam("setKey") String setKey, @RequestParam("timeSeriesKey") String timeSeriesKey) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, IOException { + var timeSeries = dbService.getTimeSeries(new TimeSeriesSet(setKey), timeSeriesKey); + var timeSeriesResult = target.calculate(timeSeries); + return timeSeriesResult; } } diff --git a/src/main/java/ru/ulstu/controller/CompressionMetricController.java b/src/main/java/ru/ulstu/controller/CompressionMetricController.java index f05eee8..f5265ee 100644 --- a/src/main/java/ru/ulstu/controller/CompressionMetricController.java +++ b/src/main/java/ru/ulstu/controller/CompressionMetricController.java @@ -3,16 +3,18 @@ package ru.ulstu.controller; import io.swagger.v3.oas.annotations.Operation; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import ru.ulstu.configuration.ApiConfiguration; import ru.ulstu.datamodel.exception.ModelingException; import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.db.DbService; +import ru.ulstu.db.model.TimeSeriesSet; import ru.ulstu.estimate.CompressionMetricService; import ru.ulstu.service.UtilService; +import ru.ulstu.target.AnomalyDifferenceSmoothed; +import ru.ulstu.target.Target; +import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.concurrent.ExecutionException; @@ -21,18 +23,22 @@ import java.util.concurrent.ExecutionException; public class CompressionMetricController { private final UtilService utilService; + private final DbService dbService; private final CompressionMetricService compressionMetricService; + private final Target target; - public CompressionMetricController(UtilService utilService, CompressionMetricService compressionMetricService) { + public CompressionMetricController(UtilService utilService, DbService dbService, CompressionMetricService compressionMetricService, AnomalyDifferenceSmoothed anomalyDifferenceSmoothed) { this.utilService = utilService; + this.dbService = dbService; this.compressionMetricService = compressionMetricService; + this.target = anomalyDifferenceSmoothed; } - @PostMapping("getMetricOfRandom") + @GetMapping("getMetricOfTimeSeries") @Operation(description = "Получить метрику сжатия") - public ResponseEntity getRandomTimeSeries(@RequestParam("length") int length) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { - var randomTS = utilService.getRandomTimeSeries(length); - var ts1 = compressionMetricService.getCompressionTimeSeries(randomTS); - return new ResponseEntity<>( null/*Сюда вывод*/, HttpStatus.OK); + public ResponseEntity getRandomTimeSeries(@RequestParam("setKey") String setKey, @RequestParam("timeSeriesKey") String timeSeriesKey) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, IOException { + var timeSeries = dbService.getTimeSeries(new TimeSeriesSet(setKey), timeSeriesKey); + var timeSeriesResult = compressionMetricService.getCompressionTimeSeries(timeSeries); + return new ResponseEntity<>(timeSeriesResult, HttpStatus.OK); } } -- 2.34.1 From 0cd1e535adeddc3fe0abaa48ee33cd15396d4ed4 Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Fri, 7 Apr 2023 00:12:38 +0400 Subject: [PATCH 23/25] add getCompressionTimeSeriesWithFTransform() --- .../estimate/CompressionMetricService.java | 48 ++++++++++++++++++- src/main/java/ru/ulstu/score/ScoreMethod.java | 6 ++- .../ulstu/service/MethodParamBruteForce.java | 4 +- src/main/java/ru/ulstu/target/Target.java | 10 ++++ 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/main/java/ru/ulstu/estimate/CompressionMetricService.java b/src/main/java/ru/ulstu/estimate/CompressionMetricService.java index 0dfaa24..3d26eaa 100644 --- a/src/main/java/ru/ulstu/estimate/CompressionMetricService.java +++ b/src/main/java/ru/ulstu/estimate/CompressionMetricService.java @@ -2,21 +2,38 @@ package ru.ulstu.estimate; import org.springframework.stereotype.Service; +import ru.ulstu.datamodel.Model; import ru.ulstu.datamodel.ModelingResult; import ru.ulstu.datamodel.exception.ModelingException; import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.datamodel.ts.TimeSeriesValue; +import ru.ulstu.method.MethodParamValue; +import ru.ulstu.method.ftransform.FTransform; +import ru.ulstu.score.ScoreMethod; +import ru.ulstu.service.MethodParamBruteForce; import ru.ulstu.service.TimeSeriesService; import java.lang.reflect.InvocationTargetException; -import java.util.concurrent.ExecutionException; +import java.time.LocalDateTime; +import java.util.*; +import java.util.concurrent.*; +import java.util.stream.Collectors; @Service public class CompressionMetricService { + private final int DEFAULT_THREAD_COUNT = 50; private final TimeSeriesService timeSeriesService; + private final FTransform fTransform; + private final MethodParamBruteForce methodParamBruteForce; + private final ScoreMethod scoreMethod; + private final ExecutorService executors = Executors.newFixedThreadPool(DEFAULT_THREAD_COUNT); - public CompressionMetricService(TimeSeriesService timeSeriesService) { + public CompressionMetricService(TimeSeriesService timeSeriesService, FTransform fTransform, MethodParamBruteForce methodParamBruteForce, ScoreMethod scoreMethod) { this.timeSeriesService = timeSeriesService; + this.fTransform = fTransform; + this.methodParamBruteForce = methodParamBruteForce; + this.scoreMethod = scoreMethod; } public void getCompressionMetric() { @@ -33,6 +50,33 @@ public class CompressionMetricService { return modelingResult.getTimeSeries(); } + protected ModelingResult getCompressionTimeSeriesWithFTransform(TimeSeries timeSeries) throws ExecutionException, InterruptedException { + List> results = new ArrayList<>(); + List results2 = new CopyOnWriteArrayList<>(); + Map tsValues = timeSeries.getValues().stream() + .collect(Collectors.toMap(TimeSeriesValue::getDate, TimeSeriesValue::getValue, + (oldValue, newValue) -> oldValue, LinkedHashMap::new)); + + + List> availableParametersValues = methodParamBruteForce.getAvailableParametersValues(timeSeries, fTransform.getAvailableParameters()); + for (List parametersValues : availableParametersValues) { + results.add(executors.submit(() -> { + Model model = fTransform.getModel(timeSeries, parametersValues); + return new ModelingResult(model.getTimeSeriesModel(), + null, + parametersValues, + scoreMethod.getScore(tsValues, model.getTimeSeriesModel()), fTransform); + })); + } + + + for (Future futureModelingResult : results) { + results2.add(futureModelingResult.get()); + } + return results2.stream() + .min(Comparator.comparing(modelingResult -> modelingResult.getScore().getDoubleValue())) + .orElse(null); + } } diff --git a/src/main/java/ru/ulstu/score/ScoreMethod.java b/src/main/java/ru/ulstu/score/ScoreMethod.java index 7459f29..7e8128f 100644 --- a/src/main/java/ru/ulstu/score/ScoreMethod.java +++ b/src/main/java/ru/ulstu/score/ScoreMethod.java @@ -5,9 +5,11 @@ import ru.ulstu.datamodel.exception.ModelingException; import ru.ulstu.datamodel.ts.TimeSeries; import ru.ulstu.datamodel.ts.TimeSeriesValue; +import java.lang.reflect.InvocationTargetException; import java.time.LocalDateTime; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutionException; public abstract class ScoreMethod { private final String name; @@ -16,11 +18,11 @@ public abstract class ScoreMethod { this.name = name; } - public Score getScore(Map tsValues, TimeSeries model) throws ModelingException { + public Score getScore(Map tsValues, TimeSeries model) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { return new Score(this, evaluate(tsValues, model)); } - public abstract Number evaluate(Map tsValues, TimeSeries model) throws ModelingException; + public abstract Number evaluate(Map tsValues, TimeSeries model) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException; public String getName() { return name; diff --git a/src/main/java/ru/ulstu/service/MethodParamBruteForce.java b/src/main/java/ru/ulstu/service/MethodParamBruteForce.java index c8e76a9..5c8573e 100644 --- a/src/main/java/ru/ulstu/service/MethodParamBruteForce.java +++ b/src/main/java/ru/ulstu/service/MethodParamBruteForce.java @@ -28,7 +28,7 @@ import java.util.concurrent.Future; import java.util.stream.Collectors; @Service -class MethodParamBruteForce { +public class MethodParamBruteForce { private final int DEFAULT_THREAD_COUNT = 50; private final List methods; private final ScoreMethod scoreMethod = new Smape(); @@ -175,7 +175,7 @@ class MethodParamBruteForce { return getSmoothedTimeSeries(timeSeries, methods); } - private List> getAvailableParametersValues(TimeSeries timeSeries, List availableParameters) { + public List> getAvailableParametersValues(TimeSeries timeSeries, List availableParameters) { List> result = new ArrayList<>(); Map parameterOffset = new TreeMap<>(); Map> parameterValues = new TreeMap<>(); diff --git a/src/main/java/ru/ulstu/target/Target.java b/src/main/java/ru/ulstu/target/Target.java index bd11e58..d834ae6 100644 --- a/src/main/java/ru/ulstu/target/Target.java +++ b/src/main/java/ru/ulstu/target/Target.java @@ -2,8 +2,11 @@ package ru.ulstu.target; import ru.ulstu.datamodel.exception.ModelingException; import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.datamodel.ts.TimeSeriesValue; import java.lang.reflect.InvocationTargetException; +import java.time.LocalDateTime; +import java.util.Map; import java.util.concurrent.ExecutionException; public abstract class Target { @@ -19,4 +22,11 @@ public abstract class Target { public abstract TimeSeries calculate(TimeSeries model) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException; + public TimeSeries calculate(Map values) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + TimeSeries model = new TimeSeries(); + for (var entry : values.entrySet()) { + model.addValue(new TimeSeriesValue(entry.getKey(), entry.getValue())); + } + return calculate(model); + } } -- 2.34.1 From d1b57b2f766a2ad9a07aa8a5aa0430a071d07320 Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Fri, 7 Apr 2023 22:25:54 +0400 Subject: [PATCH 24/25] add Prometheus HTTP API --- build.gradle | 2 + .../configuration/WebClientConfiguration.java | 14 ++++ src/main/java/ru/ulstu/http/HttpService.java | 66 +++++++++++++++++++ .../ru/ulstu/service/PrometheusService.java | 66 +++++++++++++++++++ src/main/resources/application.properties | 1 + 5 files changed, 149 insertions(+) create mode 100644 src/main/java/ru/ulstu/configuration/WebClientConfiguration.java create mode 100644 src/main/java/ru/ulstu/http/HttpService.java create mode 100644 src/main/java/ru/ulstu/service/PrometheusService.java diff --git a/build.gradle b/build.gradle index 96922ab..1cd9974 100644 --- a/build.gradle +++ b/build.gradle @@ -34,10 +34,12 @@ dependencies { implementation group: 'org.springframework.boot', name: 'spring-boot-starter-jetty' implementation group: 'org.springframework.boot', name: 'spring-boot-starter-thymeleaf' implementation group: 'org.springframework.boot', name: 'spring-boot-starter-test' + implementation group: 'org.springframework.boot', name: 'spring-boot-starter-webflux' implementation group: 'org.slf4j', name: 'slf4j-api', version: versionSLF4J implementation group: 'nz.net.ultraq.thymeleaf', name: 'thymeleaf-layout-dialect', version: '3.0.0' implementation group: 'org.javassist', name: 'javassist', version: '3.25.0-GA' implementation group: 'org.eclipse.jetty', name: 'jetty-servlet', version: versionJetty + implementation group: 'org.json', name: 'json', version: '20220320' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-annotations' diff --git a/src/main/java/ru/ulstu/configuration/WebClientConfiguration.java b/src/main/java/ru/ulstu/configuration/WebClientConfiguration.java new file mode 100644 index 0000000..e477de4 --- /dev/null +++ b/src/main/java/ru/ulstu/configuration/WebClientConfiguration.java @@ -0,0 +1,14 @@ +package ru.ulstu.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.*; +import org.springframework.web.reactive.function.client.WebClient; + +@Configuration +public class WebClientConfiguration { + @Bean + public WebClient webClient(WebClient.Builder webClientBuilder) { + return webClientBuilder.build(); //contentType(MediaType.APPLICATION_FORM_URLENCODED) + } +} diff --git a/src/main/java/ru/ulstu/http/HttpService.java b/src/main/java/ru/ulstu/http/HttpService.java new file mode 100644 index 0000000..093f58a --- /dev/null +++ b/src/main/java/ru/ulstu/http/HttpService.java @@ -0,0 +1,66 @@ +package ru.ulstu.http; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; +import org.springframework.web.reactive.function.BodyInserters; +import org.springframework.web.reactive.function.client.WebClient; + +import java.time.Duration; +import java.util.Optional; + +@Service +public class HttpService { + private final Logger log = LoggerFactory.getLogger(HttpService.class); + private final WebClient client; + + public HttpService(WebClient client) { + this.client = client; + } + + public JSONObject post(String url, JSONObject postData) { + log.debug("Service call: {}", url); + JSONObject response = null; + try { + response = new JSONObject(Optional.ofNullable(client + .post() + .uri(url) + .contentType(MediaType.APPLICATION_JSON) + .body(BodyInserters.fromValue(postData.toString())) + .accept(MediaType.APPLICATION_JSON) + .retrieve() + .bodyToMono(String.class) + .toFuture().get()).orElse("{response:\"empty\"}")); + } catch (Exception e) { + return new JSONObject("{response:\"empty\"}"); + } + log.debug("Service response: {}", response); + return response; + } + + public JSONArray get(String url) { + log.debug("Service call: {}", url); + try { + String response = client + .get() + .uri(url) + .accept(MediaType.APPLICATION_JSON) + .retrieve() + .bodyToMono(String.class) + .timeout(Duration.ofMinutes(1)) + .toFuture().get(); + if (response.startsWith("[")) { + return new JSONArray(response); + } else { + JSONArray jsonArray = new JSONArray(); + jsonArray.put(0, new JSONObject(response)); + return jsonArray; + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/ru/ulstu/service/PrometheusService.java b/src/main/java/ru/ulstu/service/PrometheusService.java new file mode 100644 index 0000000..35e9631 --- /dev/null +++ b/src/main/java/ru/ulstu/service/PrometheusService.java @@ -0,0 +1,66 @@ +package ru.ulstu.service; + +import org.json.JSONObject; +import org.springframework.stereotype.Service; +import ru.ulstu.datamodel.ts.TimeSeries; +import ru.ulstu.http.HttpService; + +@Service +public class PrometheusService { + + private static final String PROMETHEUS_API_URL = "http://prometheus.athene.tech/api/v1/"; + private final HttpService httpService; + + public PrometheusService(HttpService httpService) { + this.httpService = httpService; + } + + public TimeSeries executionQuery(String query){ // example: pve_cpu_usage_ratio{id="lxc/111"}[5h] + var timeSeries = new TimeSeries(); + var response = httpService.get(PROMETHEUS_API_URL + "query?query=" + query); + if (response.getJSONObject(0).getString("status").equals("success")){ + + } + return timeSeries; + } + + public TimeSeries executionQueryPost(String query){ // example: pve_cpu_usage_ratio{id="lxc/111"}[5h] + var timeSeries = new TimeSeries(); + var postData = new JSONObject(); + postData.put("query", query); + var response = httpService.post(PROMETHEUS_API_URL + "query", postData); +// if (response.getJSONObject(0).getString("status").equals("success")){ +// +// } + return timeSeries; + } + + public TimeSeries executionQueryRange(String query){ + var timeSeries = new TimeSeries(); + var response = httpService.get(PROMETHEUS_API_URL + "query_range?query=" + query + + "&start=2023-04-06T00:00:00.004Z&end=2023-04-07T00:00:00.004Z&step=15s"); + if (response.getJSONObject(0).getString("status").equals("success")){ + var array = response.getJSONObject(0).getJSONObject("data").getJSONArray("result"); + int a = 5; + } + return timeSeries; + } + + public TimeSeries executionQueryRangePost(String query){ + var timeSeries = new TimeSeries(); + var postData = new JSONObject(); + postData.put("query", query); + postData.put("start", "2023-04-06T00:00:00.004Z"); + postData.put("end", "2023-04-07T00:00:00.004Z"); + postData.put("step", "15s"); + + var response = httpService.post(PROMETHEUS_API_URL + "query_range", postData); +// response.getJSONObject(); +// if (response.getJSONObject(0).getString("status").equals("success")){ +// var array = response.getJSONObject(0).getJSONObject("data").getJSONArray("result"); +// int a = 5; +// } + return timeSeries; + } + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index bac6dd5..9fa1de2 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,6 +2,7 @@ spring.main.banner-mode=off logging.level.tech.athene=DEBUG server.port=8080 spring.jackson.serialization.WRITE_DATES_AS_TIMESTAMPS=false +spring.codec.max-in-memory-size=50MB joinfaces.primefaces.theme=afterwork joinfaces.primefaces.font-awesome=true time-series.db-path=time-series-db -- 2.34.1 From 3f23d32c87752a48c7fd66ac16b4695aee67ae14 Mon Sep 17 00:00:00 2001 From: Anton Skalkin Date: Tue, 10 Oct 2023 11:15:10 +0400 Subject: [PATCH 25/25] old some edit --- .../CompressionMetricController.java | 25 +++++++++++-- .../estimate/CompressionMetricService.java | 33 +++++++++++------ .../java/ru/ulstu/http/HttpClientService.java | 11 ++++++ ...HttpService.java => WebClientService.java} | 6 ++-- .../ulstu/score/AnomalyCompressionScore.java | 2 +- .../ru/ulstu/service/PrometheusService.java | 35 +++++++++++++++---- 6 files changed, 87 insertions(+), 25 deletions(-) create mode 100644 src/main/java/ru/ulstu/http/HttpClientService.java rename src/main/java/ru/ulstu/http/{HttpService.java => WebClientService.java} (92%) diff --git a/src/main/java/ru/ulstu/controller/CompressionMetricController.java b/src/main/java/ru/ulstu/controller/CompressionMetricController.java index f5265ee..5da1d80 100644 --- a/src/main/java/ru/ulstu/controller/CompressionMetricController.java +++ b/src/main/java/ru/ulstu/controller/CompressionMetricController.java @@ -10,6 +10,7 @@ import ru.ulstu.datamodel.ts.TimeSeries; import ru.ulstu.db.DbService; import ru.ulstu.db.model.TimeSeriesSet; import ru.ulstu.estimate.CompressionMetricService; +import ru.ulstu.service.PrometheusService; import ru.ulstu.service.UtilService; import ru.ulstu.target.AnomalyDifferenceSmoothed; import ru.ulstu.target.Target; @@ -24,21 +25,39 @@ public class CompressionMetricController { private final UtilService utilService; private final DbService dbService; + private final PrometheusService prometheusService; private final CompressionMetricService compressionMetricService; private final Target target; - public CompressionMetricController(UtilService utilService, DbService dbService, CompressionMetricService compressionMetricService, AnomalyDifferenceSmoothed anomalyDifferenceSmoothed) { + public CompressionMetricController(UtilService utilService, DbService dbService, PrometheusService prometheusService, CompressionMetricService compressionMetricService, AnomalyDifferenceSmoothed anomalyDifferenceSmoothed) { this.utilService = utilService; this.dbService = dbService; + this.prometheusService = prometheusService; this.compressionMetricService = compressionMetricService; this.target = anomalyDifferenceSmoothed; } - @GetMapping("getMetricOfTimeSeries") + @GetMapping("getMetricAtAnomaly") @Operation(description = "Получить метрику сжатия") - public ResponseEntity getRandomTimeSeries(@RequestParam("setKey") String setKey, @RequestParam("timeSeriesKey") String timeSeriesKey) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, IOException { + public ResponseEntity getMetricAtAnomaly(@RequestParam("setKey") String setKey, @RequestParam("timeSeriesKey") String timeSeriesKey) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException, IOException { var timeSeries = dbService.getTimeSeries(new TimeSeriesSet(setKey), timeSeriesKey); var timeSeriesResult = compressionMetricService.getCompressionTimeSeries(timeSeries); return new ResponseEntity<>(timeSeriesResult, HttpStatus.OK); } + + @GetMapping("getMetricAtAnomalyOfRandom") + @Operation(description = "Получить метрику сжатия") + public ResponseEntity getMetricAtAnomalyOfRandom(@RequestParam("lenght") int len) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + var timeSeries = utilService.getRandomTimeSeries(len); + var timeSeriesResult = compressionMetricService.getCompressionTimeSeries(timeSeries); + return new ResponseEntity<>(timeSeriesResult, HttpStatus.OK); + } + + @GetMapping("getMetricAtAnomalyOfPrometheus") + @Operation(description = "Получить метрику сжатия") + public ResponseEntity getMetricAtAnomalyOfPrometheus(@RequestParam("query") String query) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + var timeSeries = prometheusService.executionQueryRange(query); + var timeSeriesResult = compressionMetricService.getCompressionTimeSeries(timeSeries); + return new ResponseEntity<>(timeSeriesResult, HttpStatus.OK); + } } diff --git a/src/main/java/ru/ulstu/estimate/CompressionMetricService.java b/src/main/java/ru/ulstu/estimate/CompressionMetricService.java index 3d26eaa..0e9d0eb 100644 --- a/src/main/java/ru/ulstu/estimate/CompressionMetricService.java +++ b/src/main/java/ru/ulstu/estimate/CompressionMetricService.java @@ -12,6 +12,8 @@ import ru.ulstu.method.ftransform.FTransform; import ru.ulstu.score.ScoreMethod; import ru.ulstu.service.MethodParamBruteForce; import ru.ulstu.service.TimeSeriesService; +import ru.ulstu.service.UtilService; +import ru.ulstu.target.AnomalyDifferenceSmoothed; import java.lang.reflect.InvocationTargetException; import java.time.LocalDateTime; @@ -22,31 +24,40 @@ import java.util.stream.Collectors; @Service public class CompressionMetricService { - private final int DEFAULT_THREAD_COUNT = 50; + private final int DEFAULT_THREAD_COUNT = 10; private final TimeSeriesService timeSeriesService; private final FTransform fTransform; private final MethodParamBruteForce methodParamBruteForce; + private final UtilService utilService; private final ScoreMethod scoreMethod; + + private final AnomalyDifferenceSmoothed anomalyDifferenceSmoothed; private final ExecutorService executors = Executors.newFixedThreadPool(DEFAULT_THREAD_COUNT); - public CompressionMetricService(TimeSeriesService timeSeriesService, FTransform fTransform, MethodParamBruteForce methodParamBruteForce, ScoreMethod scoreMethod) { + public CompressionMetricService(TimeSeriesService timeSeriesService, FTransform fTransform, MethodParamBruteForce methodParamBruteForce, UtilService utilService, ScoreMethod scoreMethod, AnomalyDifferenceSmoothed anomalyDifferenceSmoothed) { this.timeSeriesService = timeSeriesService; this.fTransform = fTransform; this.methodParamBruteForce = methodParamBruteForce; + this.utilService = utilService; this.scoreMethod = scoreMethod; - } - - public void getCompressionMetric() { - - } - - public void getCompressionMetricForAnomaly() { - + this.anomalyDifferenceSmoothed = anomalyDifferenceSmoothed; } public TimeSeries getCompressionTimeSeries(TimeSeries ts) throws ModelingException, ExecutionException, InterruptedException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { - ModelingResult modelingResult = timeSeriesService.smoothTimeSeries(ts, "FTransform"); + //ModelingResult modelingResult = timeSeriesService.smoothTimeSeries(ts, "FTransform"); + ModelingResult modelingResult = getCompressionTimeSeriesWithFTransform(ts); //var ts1 = fTransform.getForecast(ts, ) + var timeSeriesAnomalyResult = anomalyDifferenceSmoothed.calculate(modelingResult.getTimeSeries()); + var timeSeriesAnomaly = anomalyDifferenceSmoothed.calculate(ts); + System.out.println(ts.getLength()+"\t" + +timeSeriesAnomaly.getLength()+"\t" + +Math.abs(timeSeriesAnomaly.getLength() - timeSeriesAnomalyResult.getLength())+"\t" + +(double)modelingResult.getTimeSeries().getLength()/ts.getLength()+"\t" + +modelingResult.getScore().getDoubleValue()); + System.out.println(timeSeriesAnomaly); + System.out.println(timeSeriesAnomalyResult); + System.out.println(utilService.getTimeSeriesToString(ts)); + System.out.println(utilService.getTimeSeriesToString(modelingResult.getTimeSeries())); return modelingResult.getTimeSeries(); } diff --git a/src/main/java/ru/ulstu/http/HttpClientService.java b/src/main/java/ru/ulstu/http/HttpClientService.java new file mode 100644 index 0000000..a564a43 --- /dev/null +++ b/src/main/java/ru/ulstu/http/HttpClientService.java @@ -0,0 +1,11 @@ +package ru.ulstu.http; + +import org.springframework.stereotype.Service; + +@Service +public class HttpClientService { + + public void get(){ + + } +} diff --git a/src/main/java/ru/ulstu/http/HttpService.java b/src/main/java/ru/ulstu/http/WebClientService.java similarity index 92% rename from src/main/java/ru/ulstu/http/HttpService.java rename to src/main/java/ru/ulstu/http/WebClientService.java index 093f58a..1d10093 100644 --- a/src/main/java/ru/ulstu/http/HttpService.java +++ b/src/main/java/ru/ulstu/http/WebClientService.java @@ -13,11 +13,11 @@ import java.time.Duration; import java.util.Optional; @Service -public class HttpService { - private final Logger log = LoggerFactory.getLogger(HttpService.class); +public class WebClientService { + private final Logger log = LoggerFactory.getLogger(WebClientService.class); private final WebClient client; - public HttpService(WebClient client) { + public WebClientService(WebClient client) { this.client = client; } diff --git a/src/main/java/ru/ulstu/score/AnomalyCompressionScore.java b/src/main/java/ru/ulstu/score/AnomalyCompressionScore.java index 883c989..9c3ad87 100644 --- a/src/main/java/ru/ulstu/score/AnomalyCompressionScore.java +++ b/src/main/java/ru/ulstu/score/AnomalyCompressionScore.java @@ -27,6 +27,6 @@ public class AnomalyCompressionScore extends ScoreMethod { var timeSeriesAnomalyResult = anomalyDifferenceSmoothed.calculate(model); var timeSeriesAnomaly = anomalyDifferenceSmoothed.calculate(tsValues); // туду: добавить сравнение аномальных точек - return model.getLength()/tsValues.size() + Math.abs(timeSeriesAnomaly.getLength() - timeSeriesAnomalyResult.getLength()); + return (double)model.getLength()/tsValues.size() + Math.abs(timeSeriesAnomaly.getLength() - timeSeriesAnomalyResult.getLength()); } } \ No newline at end of file diff --git a/src/main/java/ru/ulstu/service/PrometheusService.java b/src/main/java/ru/ulstu/service/PrometheusService.java index 35e9631..c17b45d 100644 --- a/src/main/java/ru/ulstu/service/PrometheusService.java +++ b/src/main/java/ru/ulstu/service/PrometheusService.java @@ -2,21 +2,34 @@ package ru.ulstu.service; import org.json.JSONObject; import org.springframework.stereotype.Service; +import org.springframework.web.util.UriComponentsBuilder; import ru.ulstu.datamodel.ts.TimeSeries; -import ru.ulstu.http.HttpService; +import ru.ulstu.datamodel.ts.TimeSeriesValue; +import ru.ulstu.http.WebClientService; + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; @Service public class PrometheusService { private static final String PROMETHEUS_API_URL = "http://prometheus.athene.tech/api/v1/"; - private final HttpService httpService; + private final WebClientService httpService; - public PrometheusService(HttpService httpService) { + public PrometheusService(WebClientService httpService) { this.httpService = httpService; } public TimeSeries executionQuery(String query){ // example: pve_cpu_usage_ratio{id="lxc/111"}[5h] var timeSeries = new TimeSeries(); + UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(PROMETHEUS_API_URL + "query") + .queryParam("query", + "pve_cpu_usage_ratio{id='lxc/111'}[1h]"); + var response = httpService.get(PROMETHEUS_API_URL + "query?query=" + query); if (response.getJSONObject(0).getString("status").equals("success")){ @@ -36,12 +49,20 @@ public class PrometheusService { } public TimeSeries executionQueryRange(String query){ + int numberTimeSeries = 14; + var nowDateTime = LocalDateTime.now().truncatedTo(ChronoUnit.SECONDS); var timeSeries = new TimeSeries(); var response = httpService.get(PROMETHEUS_API_URL + "query_range?query=" + query + - "&start=2023-04-06T00:00:00.004Z&end=2023-04-07T00:00:00.004Z&step=15s"); - if (response.getJSONObject(0).getString("status").equals("success")){ - var array = response.getJSONObject(0).getJSONObject("data").getJSONArray("result"); - int a = 5; + "&start=" + nowDateTime.minusHours(10).minusMinutes(100).toString()+".004Z" + + "&end=" + nowDateTime.minusHours(10).toString()+".004Z" + + "&step=15s"); + var array = response.getJSONObject(0).getJSONObject("data").getJSONArray("result").getJSONObject(numberTimeSeries).getJSONArray("values").toList(); + for (int i = 0; i < array.size(); i++) { + BigDecimal seconds = (BigDecimal)(((ArrayList) array.get(i)).get(0)); + Double value = Double.parseDouble((String)(((ArrayList) array.get(i)).get(1))); + LocalDateTime dateTime = + LocalDateTime.ofInstant(Instant.ofEpochSecond(seconds.longValue()), ZoneId.systemDefault()); + timeSeries.addValue(new TimeSeriesValue(dateTime, value)); } return timeSeries; } -- 2.34.1