Cascaded transformations in all fts models

benchmarks/benchmarks.py

@@ -108,7 +108,7 @@ def allIntervalForecasters(data_train, data_test, partitions, max_order=3,save=F
 
     lcolors = []
 
-    for count, model in enumerate(models, start=0):
+    for count, model in Util.enumerate2(models, start=0, step=2):
         mfts = model("")
         if not mfts.isHighOrder:
             if transformation is not None:
@@ -126,21 +126,20 @@ def allIntervalForecasters(data_train, data_test, partitions, max_order=3,save=F
                     objs.append(mfts)
                     lcolors.append(colors[count % ncol])
 
-
     print(getIntervalStatistics(data_test, objs))
 
     plotComparedSeries(data_test, objs, lcolors, typeonlegend=False, save=save, file=file, tam=tam,  intervals=True)
 
 
 def getIntervalStatistics(original, models):
-    ret = "Model	& Order     & Sharpness		& Resolution		& Coverage	\\ \n"
+    ret = "Model	& Order     & Sharpness		& Resolution		& Coverage	\\\\ \n"
     for fts in models:
         forecasts = fts.forecastInterval(original)
         ret += fts.shortname + "		& "
         ret += str(fts.order) + "		& "
         ret += str(round(Measures.sharpness(forecasts), 2)) + "		& "
         ret += str(round(Measures.resolution(forecasts), 2)) + "		& "
-        ret += str(round(Measures.coverage(original[fts.order:], forecasts[:-1]), 2)) + "	\\ \n"
+        ret += str(round(Measures.coverage(original[fts.order:], forecasts[:-1]), 2)) + "	\\\\ \n"
     return ret
 
 
@@ -213,11 +212,10 @@ def plotComparedIntervalsAhead(original, models, colors, distributions, time_fro
     mi = []
     ma = []
 
-    count = 0
+    for count, fts in enumerate(models, start=0):
-    for fts in models:
         if fts.hasDistributionForecasting and distributions[count]:
-            density = fts.forecastAheadDistribution(original[time_from - fts.order:time_from], time_to, resolution,
+            density = fts.forecastAheadDistribution(original[time_from - fts.order:time_from],
-                                                    parameters=None)
+                                                    time_to, resolution, parameters=True)
 
             y = density.columns
             t = len(y)
@@ -258,12 +256,22 @@ def plotComparedIntervalsAhead(original, models, colors, distributions, time_fro
                 forecasts.insert(0, None)
             ax.plot(forecasts, color=colors[count], label=fts.shortname)
 
-        count = count + 1
     ax.plot(original, color='black', label="Original")
     handles0, labels0 = ax.get_legend_handles_labels()
     ax.legend(handles0, labels0, loc=2)
     # ax.set_title(fts.name)
-    ax.set_ylim([min(mi), max(ma)])
+    _mi = min(mi)
+    if _mi < 0:
+        _mi *= 1.1
+    else:
+        _mi *= 0.9
+    _ma = max(ma)
+    if _ma < 0:
+        _ma *= 0.9
+    else:
+        _ma *= 1.1
+
+    ax.set_ylim([_mi, _ma])
     ax.set_ylabel('F(T)')
     ax.set_xlabel('T')
     ax.set_xlim([0, len(original)])
@@ -552,8 +560,8 @@ def compareModelsTable(original, models_fo, models_ho):
     return sup + header + body + "\\end{tabular}"
 
 
-def simpleSearch_RMSE(train, test, model, partitions, orders, save=False, file=None, tam=[10, 15], plotforecasts=False,
+def simpleSearch_RMSE(train, test, model, partitions, orders, save=False, file=None, tam=[10, 15],
-                      elev=30, azim=144):
+                      plotforecasts=False, elev=30, azim=144, intervals=False):
     ret = []
     errors = np.array([[0 for k in range(len(partitions))] for kk in range(len(orders))])
     forecasted_best = []
@@ -568,29 +576,28 @@ def simpleSearch_RMSE(train, test, model, partitions, orders, save=False, file=N
         ax0.set_xlabel('T')
     min_rmse = 1000000.0
     best = None
-    pc = 0
+
-    for p in partitions:
+    for pc, p in enumerate(partitions, start=0):
-        oc = 0
+
-        for o in orders:
         sets = Grid.GridPartitionerTrimf(train, p)
+        for oc, o in enumerate(orders, start=0):
             fts = model("q = " + str(p) + " n = " + str(o))
             fts.train(train, sets, o)
+            if not intervals:
                 forecasted = fts.forecast(test)
                 error = Measures.rmse(np.array(test[o:]), np.array(forecasted[:-1]))
-            mape = Measures.mape(np.array(test[o:]), np.array(forecasted[:-1]))
-            # print(train[o:])
-            # print(forecasted[-1])
                 for kk in range(o):
                     forecasted.insert(0, None)
                     if plotforecasts: ax0.plot(forecasted, label=fts.name)
-            # print(o, p, mape)
+            else:
+                forecasted = fts.forecastInterval(test)
+                error = 1.0 - Measures.rmse_interval(np.array(test[o:]), np.array(forecasted[:-1]))
             errors[oc, pc] = error
             if error < min_rmse:
                 min_rmse = error
                 best = fts
                 forecasted_best = forecasted
-            oc += 1
+
-        pc += 1
     # print(min_rmse)
     if plotforecasts:
         # handles0, labels0 = ax0.get_legend_handles_labels()

common/Transformations.py

@@ -30,14 +30,33 @@ class Differential(Transformation):
     def apply(self, data, param=None):
         if param is not None:
             self.lag = param
+
+        if not isinstance(data, (list, np.ndarray, np.generic)):
+            data = [data]
+
+        if isinstance(data, (np.ndarray, np.generic)):
+            data = data.tolist()
+
         n = len(data)
         diff = [data[t - self.lag] - data[t] for t in np.arange(self.lag, n)]
         for t in np.arange(0, self.lag): diff.insert(0, 0)
         return diff
 
     def inverse(self,data, param):
+
+        if isinstance(data, (np.ndarray, np.generic)):
+            data = data.tolist()
+
+        if not isinstance(data, list):
+            data = [data]
+
         n = len(data)
-        inc = [data[t] + param[t] for t in np.arange(1, n)]
+
+        inc = [data[t] + param[t] for t in np.arange(0, n)]
+
+        if n == 1:
+            return inc[0]
+        else:
             return inc
 
 

common/Util.py

@@ -21,3 +21,9 @@ def showAndSaveImage(fig,file,flag,lgd=None):
         else:
             fig.savefig(uniquefilename(file))
         plt.close(fig)
+
+
+def enumerate2(xs, start=0, step=1):
+    for x in xs:
+        yield (start, x)
+        start += step

fts.py

@@ -19,6 +19,8 @@ class FTS(object):
         self.dump = False
         self.transformations = []
         self.transformations_param = []
+        self.original_max = 0
+        self.original_min = 0
 
     def fuzzy(self, data):
         best = {"fuzzyset": "", "membership": 0.0}
@@ -59,8 +61,20 @@ class FTS(object):
     def appendTransformation(self, transformation):
         self.transformations.append(transformation)
 
-    def doTransformations(self,data,params=None):
+    def doTransformations(self,data,params=None,updateUoD=False):
         ndata = data
+
+        if updateUoD:
+            if min(data) < 0:
+                self.original_min = min(data) * 1.1
+            else:
+                self.original_min = min(data) * 0.9
+
+            if max(data) > 0:
+                self.original_max = max(data) * 1.1
+            else:
+                self.original_max = max(data) * 0.9
+
         if len(self.transformations) > 0:
             if params is None:
                 params = [ None for k in self.transformations]

hofts.py

@@ -62,7 +62,7 @@ class HighOrderFTS(fts.FTS):
 
     def train(self, data, sets, order=1,parameters=None):
 
-        data = self.doTransformations(data)
+        data = self.doTransformations(data, updateUoD=True)
 
         self.order = order
         self.sets = sets

ifts.py

@@ -49,9 +49,7 @@ class IntervalFTS(hofts.HighOrderFTS):
 
     def forecastInterval(self, data):
 
-        data = np.array(data)
+        ndata = np.array(self.doTransformations(data))
-
-        ndata = self.doTransformations(data)
 
         l = len(ndata)
 
@@ -115,8 +113,8 @@ class IntervalFTS(hofts.HighOrderFTS):
 
             # gerar o intervalo
             norm = sum(affected_flrgs_memberships)
-            lo_ = self.doInverseTransformations(sum(lo) / norm, param=[data[k - (self.order - 1): k + 1]])
+            lo_ = self.doInverseTransformations(sum(lo) / norm, params=[data[k - (self.order - 1): k + 1]])
-            up_ = self.doInverseTransformations(sum(up) / norm, param=[data[k - (self.order - 1): k + 1]])
+            up_ = self.doInverseTransformations(sum(up) / norm, params=[data[k - (self.order - 1): k + 1]])
             ret.append([lo_, up_])
 
         return ret

partitioners/Grid.py

@@ -9,20 +9,24 @@ from pyFTS.common import FuzzySet, Membership
 
 def GridPartitionerTrimf(data, npart, names=None, prefix="A"):
     sets = []
-    dmax = max(data)
+    if min(data) < 0:
-    dmax += dmax * 0.1
+        dmin = min(data) * 1.1
-    dmin = min(data)
+    else:
-    dmin -= dmin * 0.1
+        dmin = min(data) * 0.9
+
+    if max(data) > 0:
+        dmax = max(data) * 1.1
+    else:
+        dmax = max(data) * 0.9
+
     dlen = dmax - dmin
     partlen = math.ceil(dlen / npart)
-    #p2 = partlen / 2
+
-    #partition = dmin #+ partlen
     count = 0
     for c in np.arange(dmin, dmax, partlen):
         sets.append(
             FuzzySet.FuzzySet(prefix + str(count), Membership.trimf, [c - partlen, c, c + partlen],c))
         count += 1
-        #partition += partlen
 
     return sets
 

pfts.py

@@ -214,9 +214,7 @@ class ProbabilisticFTS(ifts.IntervalFTS):
 
     def forecastInterval(self, data):
 
-        data = np.array(data)
+        ndata = np.array(self.doTransformations(data))
-
-        ndata = self.doTransformations(data)
 
         l = len(ndata)
 
@@ -349,7 +347,15 @@ class ProbabilisticFTS(ifts.IntervalFTS):
 
     def getGridClean(self, resolution):
         grid = {}
-        for sbin in np.arange(self.sets[0].lower, self.sets[-1].upper, resolution):
+
+        if len(self.transformations) == 0:
+            _min = self.sets[0].lower
+            _max = self.sets[-1].upper
+        else:
+            _min = self.original_min
+            _max = self.original_max
+
+        for sbin in np.arange(_min,_max, resolution):
             grid[sbin] = 0
 
         return grid
@@ -388,6 +394,8 @@ class ProbabilisticFTS(ifts.IntervalFTS):
 
         index = SortedCollection.SortedCollection(iterable=grid.keys())
 
+        if parameters is None:
+
             grids = []
             for k in np.arange(0, steps):
                 grids.append(self.getGridClean(resolution))
@@ -426,12 +434,9 @@ class ProbabilisticFTS(ifts.IntervalFTS):
                 for p in root.paths():
                     path = list(reversed(list(filter(None.__ne__, p))))
 
-                if parameters is None:
                     qtle = self.forecastInterval(path)
+
                     grids[k - self.order] = self.gridCount(grids[k - self.order], resolution, index, np.ravel(qtle))
-                else:
-                    qtle = self.forecast(path)
-                    grids[k - self.order] = self.gridCountPoint(grids[k - self.order], resolution, index, np.ravel(qtle))
 
             for k in np.arange(0, steps):
                 tmp = np.array([grids[k][q] for q in sorted(grids[k])])
@@ -440,6 +445,38 @@ class ProbabilisticFTS(ifts.IntervalFTS):
             grid = self.getGridClean(resolution)
             df = pd.DataFrame(ret, columns=sorted(grid))
             return df
+        else:
+
+            print("novo")
+
+            ret = []
+
+            for k in np.arange(self.order, steps + self.order):
+
+                grid = self.getGridClean(resolution)
+                grid = self.gridCount(grid, resolution, index, intervals[k])
+
+                for qt in np.arange(0, 50, 1):
+                    # print(qt)
+                    qtle_lower = self.forecastInterval(
+                        [intervals[x][0] + qt * ((intervals[x][1] - intervals[x][0]) / 100) for x in
+                         np.arange(k - self.order, k)])
+                    grid = self.gridCount(grid, resolution, index, np.ravel(qtle_lower))
+                    qtle_upper = self.forecastInterval(
+                        [intervals[x][1] - qt * ((intervals[x][1] - intervals[x][0]) / 100) for x in
+                         np.arange(k - self.order, k)])
+                    grid = self.gridCount(grid, resolution, index, np.ravel(qtle_upper))
+                qtle_mid = self.forecastInterval(
+                    [intervals[x][0] + (intervals[x][1] - intervals[x][0]) / 2 for x in np.arange(k - self.order, k)])
+                grid = self.gridCount(grid, resolution, index, np.ravel(qtle_mid))
+
+                tmp = np.array([grid[k] for k in sorted(grid)])
+
+                ret.append(tmp / sum(tmp))
+
+            grid = self.getGridClean(resolution)
+            df = pd.DataFrame(ret, columns=sorted(grid))
+            return df
 
 
     def __str__(self):

tests/pfts.py

@@ -10,27 +10,26 @@ from mpl_toolkits.mplot3d import Axes3D
 
 import pandas as pd
 from pyFTS.partitioners import Grid
-from pyFTS.common import FLR,FuzzySet,Membership
+from pyFTS.common import FLR,FuzzySet,Membership,Transformations
 from pyFTS import fts,hofts,ifts,pfts,tree, chen
 from pyFTS.benchmarks import benchmarks as bchmk
 
 
 os.chdir("/home/petronio/dados/Dropbox/Doutorado/Disciplinas/AdvancedFuzzyTimeSeriesModels/")
 
-#enrollments = pd.read_csv("DataSets/Enrollments.csv", sep=";")
+enrollments = pd.read_csv("DataSets/Enrollments.csv", sep=";")
-#enrollments = np.array(enrollments["Enrollments"])
+enrollments = np.array(enrollments["Enrollments"])
 
-#enrollments_fs1 = Grid.GridPartitionerTrimf(enrollments,6)
+#diff = Transformations.Differential(1)
+
+fs = Grid.GridPartitionerTrimf(enrollments,6)
 
 #tmp = chen.ConventionalFTS("")
 
-pfts1_enrollments = pfts.ProbabilisticFTS("1")
+pfts1 = pfts.ProbabilisticFTS("1")
-#pfts1_enrollments.train(enrollments,enrollments_fs1,1)
+#pfts1.appendTransformation(diff)
-#pfts1_enrollments.shortname = "1st Order"
+pfts1.train(enrollments,fs,1)
-#pfts2_enrollments = pfts.ProbabilisticFTS("2")
-#pfts2_enrollments.dump = False
-#pfts2_enrollments.shortname = "2nd Order"
-#pfts2_enrollments.train(enrollments,enrollments_fs1,2)
 
+#bchmk.plotComparedIntervalsAhead(enrollments,[pfts1], ["blue"],[True],5,10)
 
-#pfts1_enrollments.forecastAheadDistribution2(enrollments[:15],5,100)
+pfts1.forecastAheadDistribution(enrollments,5,1, parameters=True)