Gradient descent training method for FCM_FTS

pyFTS/fcm/Activations.py

@@ -3,9 +3,9 @@ import numpy as np
 
 def step(x, deriv=False):
     if deriv:
-        1 * (x == 0)
+        return np.abs(1 * (x == 0))
     else:
-        return 1 * (x > 0)
+        return np.abs(1 * (x > 0))
 
 
 def sigmoid(x, deriv=False):
@@ -26,14 +26,14 @@ def softmax(x, deriv=False):
 
 def tanh(x, deriv=False):
     if deriv:
-        pass
+        return 1 - np.tanh(x)**2
     else:
         return np.tanh(x)
 
 
 def relu(x, deriv=False):
     if deriv:
-        return 1. * (x > 0)
+        return np.abs(1. * (x > 0))
     else:
-        return x * (x > 0)
+        return np.abs(x * (x > 0))
 

pyFTS/fcm/GD.py

@@ -1,20 +1,25 @@
 import  numpy as np
 
 
-def GD(data, model, alpha, momentum=0.5):
+def GD(data, model, **kwargs):
+    alpha = kwargs.get('alpha', 0.1)
+    momentum = kwargs.get('momentum', None)
+    iterations = kwargs.get('iterations', 1)
     num_concepts = model.partitioner.partitions
     weights = [np.random.normal(0,.01,(num_concepts,num_concepts)) for k in range(model.order)]
     last_gradient = [None for k in range(model.order) ]
+    for it in np.arange(iterations):
         for i in np.arange(model.order, len(data)):
+            #i = np.random.randint(model.order, len(data)-model.order)
             sample = data[i-model.order : i]
             target = data[i]
             model.fcm.weights = weights
             inputs = model.partitioner.fuzzyfy(sample, mode='vector')
-        activations = [model.fcm.activation_function(inputs[k]) for k in np.arange(model.order)]
+            activations = model.fcm.activate(inputs)
             forecast = model.predict(sample)[0]
             error = target - forecast #)**2
-        if error == np.nan:
+            if str(error) == 'nan' or error == np.nan or error == np.Inf:
-            pass
+                print('error')
             print(error)
             for k in np.arange(model.order):
                 deriv = error * model.fcm.activation_function(activations[k], deriv=True)
@@ -24,6 +29,7 @@ def GD(data, model, alpha, momentum=0.5):
 
                     tmp_grad = (momentum * last_gradient[k]) + alpha*deriv*inputs[k]
                     weights[k] -= tmp_grad
+                    last_gradient[k] = tmp_grad
                 else:
                     weights[k] -= alpha*deriv*inputs[k]
 

pyFTS/fcm/fts.py

@@ -11,14 +11,15 @@ class FCM_FTS(hofts.HighOrderFTS):
         self.fcm = common.FuzzyCognitiveMap(**kwargs)
 
     def train(self, data, **kwargs):
-        '''
+        method = kwargs.get('method','GA')
+        if method == 'GA':
             GA.parameters['num_concepts'] = self.partitioner.partitions
             GA.parameters['order'] = self.order
             GA.parameters['partitioner'] = self.partitioner
             ret = GA.execute(data, **kwargs)
             self.fcm.weights = ret['weights']
-        '''
+        elif method == 'GD':
-        self.fcm.weights = GD.GD(data, self, alpha=0.01)
+            self.fcm.weights = GD.GD(data, self, **kwargs)
 
     def forecast(self, ndata, **kwargs):
         ret = []
@@ -33,7 +34,10 @@ class FCM_FTS(hofts.HighOrderFTS):
 
             activation = self.fcm.activate(fuzzyfied)
 
-            final = np.dot(midpoints, activation)/np.sum(activation)
+            final = np.dot(midpoints, activation)/np.nanmax([1, np.sum(activation)])
+
+            if str(final) == 'nan' or final == np.nan or final == np.Inf:
+                print('error')
 
             ret.append(final)
 

pyFTS/tests/fcm_fts.py

@@ -18,27 +18,32 @@ data = df['glo_avg'].values[:]
 train = data[:7000]
 test = data[7000:7500]
 
-fs = Grid.GridPartitioner(data=train, npart=7)
+fs = Grid.GridPartitioner(data=train, npart=5)
 
 model = fcm_fts.FCM_FTS(partitioner=fs, order=2, activation_function = Activations.relu)
 
-model.fit(train,
+
-          ngen=30, #number of generations
+model.fit(train, method='GD', alpha=0.02, momentum=0.8, iteractions=3 )
-          mgen=7, # stop after mgen generations without improvement
-          npop=10, # number of individuals on population
-        pcruz=.5, # crossover percentual of population
-        pmut=.3, # mutation percentual of population
-        window_size = 7000,
-        train_rate = .8,
-        increment_rate =.2,
-        experiments=1
-        )
 
 Util.persist_obj(model, 'fcm_fts10c')
 '''
 model = Util.load_obj('fcm_fts05c')
 '''
+#forecasts = model.predict(test)
+
+#print(model)
+
+fig, ax = plt.subplots(nrows=1, ncols=1, figsize=[15,5])
+
+ax.plot(test,label='Original')
 
 forecasts = model.predict(test)
 
-print(model)
+for w in np.arange(model.order):
+  forecasts.insert(0,None)
+
+ax.plot(forecasts, label=model.shortname)
+
+plt.show()
+
+print("")