Gradient descent training method for FCM_FTS

pyFTS/fcm/Activations.py

@@ -1,18 +1,39 @@
 import numpy as np
 
 
-def step(x):
-    if x <= 0:
-        return 0
+def step(x, deriv=False):
+    if deriv:
+        1 * (x == 0)
     else:
-        return 1
+        return 1 * (x > 0)
 
 
-def sigmoid(x):
-    return 1 / (1 + np.exp(-x))
+def sigmoid(x, deriv=False):
+    if deriv:
+        #return sigmoid(x)*(1 - sigmoid(x))
+        return x * (1 - x)
+    else:
+        return 1 / (1 + np.exp(-x))
 
 
-def softmax(x):
-  mvs = sum([np.exp(k) for k in x.flatten()])
-  return np.array([np.exp(k)/mvs for k in x.flatten()])
+def softmax(x, deriv=False):
+    if deriv:
+        pass
+    else:
+        mvs = sum([np.exp(k) for k in x.flatten()])
+        return np.array([np.exp(k)/mvs for k in x.flatten()])
+
+
+def tanh(x, deriv=False):
+    if deriv:
+        pass
+    else:
+        return np.tanh(x)
+
+
+def relu(x, deriv=False):
+    if deriv:
+        return 1. * (x > 0)
+    else:
+        return x * (x > 0)
 

pyFTS/fcm/GA.py

@@ -20,7 +20,7 @@ parameters = {}
 
 #
 def genotype():
-    '''
+    """
     Create the individual genotype
 
     :param mf: membership function
@@ -32,30 +32,30 @@ def genotype():
     :param f1: accuracy fitness value
     :param f2: parsimony fitness value
     :return: the genotype, a dictionary with all hyperparameters
-    '''
+    """
     num_concepts = parameters['num_concepts']
     order = parameters['order']
-    ind = dict(weights=[np.random.normal(0,.5,(num_concepts,num_concepts)) for k in range(order)])
+    ind = dict(weights=[np.random.normal(0,1.,(num_concepts,num_concepts)) for k in range(order)])
     return ind
 
 
 def random_genotype():
-    '''
+    """
     Create random genotype
 
     :return: the genotype, a dictionary with all hyperparameters
-    '''
+    """
     return genotype()
 
 
 #
 def initial_population(n):
-    '''
+    """
     Create a random population of size n
 
     :param n: the size of the population
     :return: a list with n random individuals
-    '''
+    """
     pop = []
     for i in range(n):
         pop.append(random_genotype())
@@ -63,14 +63,14 @@ def initial_population(n):
 
 
 def phenotype(individual, train):
-    '''
+    """
     Instantiate the genotype, creating a fitted model with the genotype hyperparameters
 
     :param individual: a genotype
     :param train: the training dataset
     :param parameters: dict with model specific arguments for fit method.
     :return: a fitted FTS model
-    '''
+    """
     partitioner = parameters['partitioner']
     order = parameters['order']
 
@@ -81,9 +81,8 @@ def phenotype(individual, train):
     return model
 
 
-
 def evaluate(dataset, individual, **kwargs):
-    '''
+    """
     Evaluate an individual using a sliding window cross validation over the dataset.
 
     :param dataset: Evaluation dataset
@@ -93,7 +92,7 @@ def evaluate(dataset, individual, **kwargs):
     :param increment_rate: The increment of the scrolling window, relative to the window_size ([0,1])
     :param parameters: dict with model specific arguments for fit method.
     :return: a tuple (len_lags, rmse) with the parsimony fitness value and the accuracy fitness value
-    '''
+    """
     from pyFTS.common import Util
     from pyFTS.benchmarks import Measures
     from pyFTS.fcm.GA import phenotype
@@ -129,15 +128,14 @@ def evaluate(dataset, individual, **kwargs):
     return {'rmse': .6 * _rmse + .4 * _std}
 
 
-
 def tournament(population, objective):
-    '''
+    """
     Simple tournament selection strategy.
 
     :param population: the population
     :param objective: the objective to be considered on tournament
     :return:
-    '''
+    """
     n = len(population) - 1
 
     r1 = random.randint(0, n) if n > 2 else 0
@@ -146,14 +144,13 @@ def tournament(population, objective):
     return population[ix]
 
 
-
 def crossover(parents):
-    '''
+    """
     Crossover operation between two parents
 
     :param parents: a list with two genotypes
     :return: a genotype
-    '''
+    """
     import random
 
     descendent = genotype()
@@ -164,7 +161,7 @@ def crossover(parents):
         weights2 = parents[1]['weights'][k]
 
         for (row, col), a in np.ndenumerate(weights1):
-            new_weight.append(.7*weights1[row, col] + .3*weights2[row, col]  )
+            new_weight.append(.7*weights1[row, col] + .3*weights2[row, col] )
 
         descendent['weights'][k] = np.array(new_weight).reshape(weights1.shape)
 
@@ -172,12 +169,12 @@ def crossover(parents):
 
 
 def mutation(individual, pmut):
-    '''
+    """
     Mutation operator
 
     :param population:
     :return:
-    '''
+    """
     import numpy.random
 
     for k in range(parameters['order']):
@@ -197,18 +194,17 @@ def mutation(individual, pmut):
                 individual['weights'][k][row, col] += np.random.normal(0, .5, 1)
                 individual['weights'][k][row, col] = np.clip(individual['weights'][k][row, col], -1, 1)
 
-
     return individual
 
 
 def elitism(population, new_population):
-    '''
+    """
     Elitism operation, always select the best individual of the population and discard the worst
 
     :param population:
     :param new_population:
     :return:
-    '''
+    """
     population = sorted(population, key=itemgetter('rmse'))
     best = population[0]
 
@@ -220,7 +216,7 @@ def elitism(population, new_population):
 
 
 def GeneticAlgorithm(dataset, **kwargs):
-    '''
+    """
     Genetic algoritm for hyperparameter optimization
 
     :param dataset:
@@ -234,7 +230,7 @@ def GeneticAlgorithm(dataset, **kwargs):
     :param increment_rate: The increment of the scrolling window, relative to the window_size ([0,1])
     :param parameters: dict with model specific arguments for fit method.
     :return: the best genotype
-    '''
+    """
 
     statistics = []
 

pyFTS/fcm/GD.py

@@ -0,0 +1,30 @@
+import  numpy as np
+
+
+def GD(data, model, alpha, momentum=0.5):
+    num_concepts = model.partitioner.partitions
+    weights=[np.random.normal(0,.01,(num_concepts,num_concepts)) for k in range(model.order)]
+    last_gradient = [None for k in range(model.order) ]
+    for i in np.arange(model.order, len(data)):
+        sample = data[i-model.order : i]
+        target = data[i]
+        model.fcm.weights = weights
+        inputs = model.partitioner.fuzzyfy(sample, mode='vector')
+        activations = [model.fcm.activation_function(inputs[k]) for k in np.arange(model.order)]
+        forecast = model.predict(sample)[0]
+        error = target - forecast #)**2
+        if error == np.nan:
+            pass
+        print(error)
+        for k in np.arange(model.order):
+            deriv = error * model.fcm.activation_function(activations[k], deriv=True)
+            if momentum is not None:
+                if last_gradient[k] is None:
+                    last_gradient[k] = deriv*inputs[k]
+
+                tmp_grad = (momentum * last_gradient[k]) + alpha*deriv*inputs[k]
+                weights[k] -= tmp_grad
+            else:
+                weights[k] -= alpha*deriv*inputs[k]
+
+    return weights

pyFTS/fcm/common.py

@@ -8,7 +8,7 @@ class FuzzyCognitiveMap(object):
         self.order = kwargs.get('order',1)
         self.concepts = kwargs.get('partitioner',None)
         self.weights = []
-        self.activation_function = kwargs.get('func', Activations.sigmoid)
+        self.activation_function = kwargs.get('activation_function', Activations.sigmoid)
 
     def activate(self, concepts):
         dot_products = np.zeros(len(self.concepts))

pyFTS/fcm/fts.py

@@ -1,6 +1,6 @@
 from pyFTS.common import fts
 from pyFTS.models import hofts
-from pyFTS.fcm import common, GA, Activations
+from pyFTS.fcm import common, GA, Activations, GD
 import numpy as np
 
 
@@ -11,11 +11,14 @@ class FCM_FTS(hofts.HighOrderFTS):
         self.fcm = common.FuzzyCognitiveMap(**kwargs)
 
     def train(self, data, **kwargs):
+        '''
         GA.parameters['num_concepts'] = self.partitioner.partitions
         GA.parameters['order'] = self.order
         GA.parameters['partitioner'] = self.partitioner
         ret = GA.execute(data, **kwargs)
         self.fcm.weights = ret['weights']
+        '''
+        self.fcm.weights = GD.GD(data, self, alpha=0.01)
 
     def forecast(self, ndata, **kwargs):
         ret = []

pyFTS/tests/fcm_fts.py

@@ -0,0 +1,44 @@
+from pyFTS.fcm import Activations
+import numpy as np
+import os
+import matplotlib as plt
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+
+from pyFTS.fcm import fts as fcm_fts
+from pyFTS.partitioners import Grid
+from pyFTS.common import Util
+
+df = pd.read_csv('https://query.data.world/s/56i2vkijbvxhtv5gagn7ggk3zw3ksi', sep=';')
+
+data = df['glo_avg'].values[:]
+
+
+train = data[:7000]
+test = data[7000:7500]
+
+fs = Grid.GridPartitioner(data=train, npart=7)
+
+model = fcm_fts.FCM_FTS(partitioner=fs, order=2, activation_function = Activations.relu)
+
+model.fit(train,
+          ngen=30, #number of generations
+          mgen=7, # stop after mgen generations without improvement
+          npop=10, # number of individuals on population
+        pcruz=.5, # crossover percentual of population
+        pmut=.3, # mutation percentual of population
+        window_size = 7000,
+        train_rate = .8,
+        increment_rate =.2,
+        experiments=1
+        )
+
+Util.persist_obj(model, 'fcm_fts10c')
+'''
+model = Util.load_obj('fcm_fts05c')
+'''
+
+forecasts = model.predict(test)
+
+print(model)