From 2abc070f1dc50872a6c65f3a5965bfd73b75394b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=C3=B4nio=20C=C3=A2ndido?= Date: Wed, 29 Jan 2020 12:13:33 -0300 Subject: [PATCH] Gradient descent training method for FCM_FTS --- pyFTS/fcm/Activations.py | 39 ++++++++++++++++++++++++-------- pyFTS/fcm/GA.py | 48 ++++++++++++++++++---------------------- pyFTS/fcm/GD.py | 30 +++++++++++++++++++++++++ pyFTS/fcm/common.py | 2 +- pyFTS/fcm/fts.py | 5 ++++- pyFTS/tests/fcm_fts.py | 44 ++++++++++++++++++++++++++++++++++++ 6 files changed, 131 insertions(+), 37 deletions(-) create mode 100644 pyFTS/fcm/GD.py create mode 100644 pyFTS/tests/fcm_fts.py diff --git a/pyFTS/fcm/Activations.py b/pyFTS/fcm/Activations.py index b6b3722..43ac4ca 100644 --- a/pyFTS/fcm/Activations.py +++ b/pyFTS/fcm/Activations.py @@ -1,18 +1,39 @@ import numpy as np -def step(x): - if x <= 0: - return 0 +def step(x, deriv=False): + if deriv: + 1 * (x == 0) else: - return 1 + return 1 * (x > 0) -def sigmoid(x): - return 1 / (1 + np.exp(-x)) +def sigmoid(x, deriv=False): + if deriv: + #return sigmoid(x)*(1 - sigmoid(x)) + return x * (1 - x) + else: + return 1 / (1 + np.exp(-x)) -def softmax(x): - mvs = sum([np.exp(k) for k in x.flatten()]) - return np.array([np.exp(k)/mvs for k in x.flatten()]) +def softmax(x, deriv=False): + if deriv: + pass + else: + mvs = sum([np.exp(k) for k in x.flatten()]) + return np.array([np.exp(k)/mvs for k in x.flatten()]) + + +def tanh(x, deriv=False): + if deriv: + pass + else: + return np.tanh(x) + + +def relu(x, deriv=False): + if deriv: + return 1. * (x > 0) + else: + return x * (x > 0) diff --git a/pyFTS/fcm/GA.py b/pyFTS/fcm/GA.py index 55bd9c1..9e0f3bf 100644 --- a/pyFTS/fcm/GA.py +++ b/pyFTS/fcm/GA.py @@ -20,7 +20,7 @@ parameters = {} # def genotype(): - ''' + """ Create the individual genotype :param mf: membership function @@ -32,30 +32,30 @@ def genotype(): :param f1: accuracy fitness value :param f2: parsimony fitness value :return: the genotype, a dictionary with all hyperparameters - ''' + """ num_concepts = parameters['num_concepts'] order = parameters['order'] - ind = dict(weights=[np.random.normal(0,.5,(num_concepts,num_concepts)) for k in range(order)]) + ind = dict(weights=[np.random.normal(0,1.,(num_concepts,num_concepts)) for k in range(order)]) return ind def random_genotype(): - ''' + """ Create random genotype :return: the genotype, a dictionary with all hyperparameters - ''' + """ return genotype() # def initial_population(n): - ''' + """ Create a random population of size n :param n: the size of the population :return: a list with n random individuals - ''' + """ pop = [] for i in range(n): pop.append(random_genotype()) @@ -63,14 +63,14 @@ def initial_population(n): def phenotype(individual, train): - ''' + """ Instantiate the genotype, creating a fitted model with the genotype hyperparameters :param individual: a genotype :param train: the training dataset :param parameters: dict with model specific arguments for fit method. :return: a fitted FTS model - ''' + """ partitioner = parameters['partitioner'] order = parameters['order'] @@ -81,9 +81,8 @@ def phenotype(individual, train): return model - def evaluate(dataset, individual, **kwargs): - ''' + """ Evaluate an individual using a sliding window cross validation over the dataset. :param dataset: Evaluation dataset @@ -93,7 +92,7 @@ def evaluate(dataset, individual, **kwargs): :param increment_rate: The increment of the scrolling window, relative to the window_size ([0,1]) :param parameters: dict with model specific arguments for fit method. :return: a tuple (len_lags, rmse) with the parsimony fitness value and the accuracy fitness value - ''' + """ from pyFTS.common import Util from pyFTS.benchmarks import Measures from pyFTS.fcm.GA import phenotype @@ -129,15 +128,14 @@ def evaluate(dataset, individual, **kwargs): return {'rmse': .6 * _rmse + .4 * _std} - def tournament(population, objective): - ''' + """ Simple tournament selection strategy. :param population: the population :param objective: the objective to be considered on tournament :return: - ''' + """ n = len(population) - 1 r1 = random.randint(0, n) if n > 2 else 0 @@ -146,14 +144,13 @@ def tournament(population, objective): return population[ix] - def crossover(parents): - ''' + """ Crossover operation between two parents :param parents: a list with two genotypes :return: a genotype - ''' + """ import random descendent = genotype() @@ -164,7 +161,7 @@ def crossover(parents): weights2 = parents[1]['weights'][k] for (row, col), a in np.ndenumerate(weights1): - new_weight.append(.7*weights1[row, col] + .3*weights2[row, col] ) + new_weight.append(.7*weights1[row, col] + .3*weights2[row, col] ) descendent['weights'][k] = np.array(new_weight).reshape(weights1.shape) @@ -172,12 +169,12 @@ def crossover(parents): def mutation(individual, pmut): - ''' + """ Mutation operator :param population: :return: - ''' + """ import numpy.random for k in range(parameters['order']): @@ -197,18 +194,17 @@ def mutation(individual, pmut): individual['weights'][k][row, col] += np.random.normal(0, .5, 1) individual['weights'][k][row, col] = np.clip(individual['weights'][k][row, col], -1, 1) - return individual def elitism(population, new_population): - ''' + """ Elitism operation, always select the best individual of the population and discard the worst :param population: :param new_population: :return: - ''' + """ population = sorted(population, key=itemgetter('rmse')) best = population[0] @@ -220,7 +216,7 @@ def elitism(population, new_population): def GeneticAlgorithm(dataset, **kwargs): - ''' + """ Genetic algoritm for hyperparameter optimization :param dataset: @@ -234,7 +230,7 @@ def GeneticAlgorithm(dataset, **kwargs): :param increment_rate: The increment of the scrolling window, relative to the window_size ([0,1]) :param parameters: dict with model specific arguments for fit method. :return: the best genotype - ''' + """ statistics = [] diff --git a/pyFTS/fcm/GD.py b/pyFTS/fcm/GD.py new file mode 100644 index 0000000..98df383 --- /dev/null +++ b/pyFTS/fcm/GD.py @@ -0,0 +1,30 @@ +import numpy as np + + +def GD(data, model, alpha, momentum=0.5): + num_concepts = model.partitioner.partitions + weights=[np.random.normal(0,.01,(num_concepts,num_concepts)) for k in range(model.order)] + last_gradient = [None for k in range(model.order) ] + for i in np.arange(model.order, len(data)): + sample = data[i-model.order : i] + target = data[i] + model.fcm.weights = weights + inputs = model.partitioner.fuzzyfy(sample, mode='vector') + activations = [model.fcm.activation_function(inputs[k]) for k in np.arange(model.order)] + forecast = model.predict(sample)[0] + error = target - forecast #)**2 + if error == np.nan: + pass + print(error) + for k in np.arange(model.order): + deriv = error * model.fcm.activation_function(activations[k], deriv=True) + if momentum is not None: + if last_gradient[k] is None: + last_gradient[k] = deriv*inputs[k] + + tmp_grad = (momentum * last_gradient[k]) + alpha*deriv*inputs[k] + weights[k] -= tmp_grad + else: + weights[k] -= alpha*deriv*inputs[k] + + return weights \ No newline at end of file diff --git a/pyFTS/fcm/common.py b/pyFTS/fcm/common.py index 02db2de..89f8605 100644 --- a/pyFTS/fcm/common.py +++ b/pyFTS/fcm/common.py @@ -8,7 +8,7 @@ class FuzzyCognitiveMap(object): self.order = kwargs.get('order',1) self.concepts = kwargs.get('partitioner',None) self.weights = [] - self.activation_function = kwargs.get('func', Activations.sigmoid) + self.activation_function = kwargs.get('activation_function', Activations.sigmoid) def activate(self, concepts): dot_products = np.zeros(len(self.concepts)) diff --git a/pyFTS/fcm/fts.py b/pyFTS/fcm/fts.py index 5fe05ce..163aed2 100644 --- a/pyFTS/fcm/fts.py +++ b/pyFTS/fcm/fts.py @@ -1,6 +1,6 @@ from pyFTS.common import fts from pyFTS.models import hofts -from pyFTS.fcm import common, GA, Activations +from pyFTS.fcm import common, GA, Activations, GD import numpy as np @@ -11,11 +11,14 @@ class FCM_FTS(hofts.HighOrderFTS): self.fcm = common.FuzzyCognitiveMap(**kwargs) def train(self, data, **kwargs): + ''' GA.parameters['num_concepts'] = self.partitioner.partitions GA.parameters['order'] = self.order GA.parameters['partitioner'] = self.partitioner ret = GA.execute(data, **kwargs) self.fcm.weights = ret['weights'] + ''' + self.fcm.weights = GD.GD(data, self, alpha=0.01) def forecast(self, ndata, **kwargs): ret = [] diff --git a/pyFTS/tests/fcm_fts.py b/pyFTS/tests/fcm_fts.py new file mode 100644 index 0000000..664167c --- /dev/null +++ b/pyFTS/tests/fcm_fts.py @@ -0,0 +1,44 @@ +from pyFTS.fcm import Activations +import numpy as np +import os +import matplotlib as plt +import matplotlib.pyplot as plt +import seaborn as sns +import pandas as pd + +from pyFTS.fcm import fts as fcm_fts +from pyFTS.partitioners import Grid +from pyFTS.common import Util + +df = pd.read_csv('https://query.data.world/s/56i2vkijbvxhtv5gagn7ggk3zw3ksi', sep=';') + +data = df['glo_avg'].values[:] + + +train = data[:7000] +test = data[7000:7500] + +fs = Grid.GridPartitioner(data=train, npart=7) + +model = fcm_fts.FCM_FTS(partitioner=fs, order=2, activation_function = Activations.relu) + +model.fit(train, + ngen=30, #number of generations + mgen=7, # stop after mgen generations without improvement + npop=10, # number of individuals on population + pcruz=.5, # crossover percentual of population + pmut=.3, # mutation percentual of population + window_size = 7000, + train_rate = .8, + increment_rate =.2, + experiments=1 + ) + +Util.persist_obj(model, 'fcm_fts10c') +''' +model = Util.load_obj('fcm_fts05c') +''' + +forecasts = model.predict(test) + +print(model) \ No newline at end of file