Gradient descent training method for FCM_FTS

This commit is contained in:
Petrônio Cândido 2020-01-29 12:13:33 -03:00
parent e7d603015a
commit 2abc070f1d
6 changed files with 131 additions and 37 deletions

View File

@ -1,18 +1,39 @@
import numpy as np
def step(x):
if x <= 0:
return 0
def step(x, deriv=False):
if deriv:
1 * (x == 0)
else:
return 1
return 1 * (x > 0)
def sigmoid(x):
def sigmoid(x, deriv=False):
if deriv:
#return sigmoid(x)*(1 - sigmoid(x))
return x * (1 - x)
else:
return 1 / (1 + np.exp(-x))
def softmax(x):
def softmax(x, deriv=False):
if deriv:
pass
else:
mvs = sum([np.exp(k) for k in x.flatten()])
return np.array([np.exp(k)/mvs for k in x.flatten()])
def tanh(x, deriv=False):
if deriv:
pass
else:
return np.tanh(x)
def relu(x, deriv=False):
if deriv:
return 1. * (x > 0)
else:
return x * (x > 0)

View File

@ -20,7 +20,7 @@ parameters = {}
#
def genotype():
'''
"""
Create the individual genotype
:param mf: membership function
@ -32,30 +32,30 @@ def genotype():
:param f1: accuracy fitness value
:param f2: parsimony fitness value
:return: the genotype, a dictionary with all hyperparameters
'''
"""
num_concepts = parameters['num_concepts']
order = parameters['order']
ind = dict(weights=[np.random.normal(0,.5,(num_concepts,num_concepts)) for k in range(order)])
ind = dict(weights=[np.random.normal(0,1.,(num_concepts,num_concepts)) for k in range(order)])
return ind
def random_genotype():
'''
"""
Create random genotype
:return: the genotype, a dictionary with all hyperparameters
'''
"""
return genotype()
#
def initial_population(n):
'''
"""
Create a random population of size n
:param n: the size of the population
:return: a list with n random individuals
'''
"""
pop = []
for i in range(n):
pop.append(random_genotype())
@ -63,14 +63,14 @@ def initial_population(n):
def phenotype(individual, train):
'''
"""
Instantiate the genotype, creating a fitted model with the genotype hyperparameters
:param individual: a genotype
:param train: the training dataset
:param parameters: dict with model specific arguments for fit method.
:return: a fitted FTS model
'''
"""
partitioner = parameters['partitioner']
order = parameters['order']
@ -81,9 +81,8 @@ def phenotype(individual, train):
return model
def evaluate(dataset, individual, **kwargs):
'''
"""
Evaluate an individual using a sliding window cross validation over the dataset.
:param dataset: Evaluation dataset
@ -93,7 +92,7 @@ def evaluate(dataset, individual, **kwargs):
:param increment_rate: The increment of the scrolling window, relative to the window_size ([0,1])
:param parameters: dict with model specific arguments for fit method.
:return: a tuple (len_lags, rmse) with the parsimony fitness value and the accuracy fitness value
'''
"""
from pyFTS.common import Util
from pyFTS.benchmarks import Measures
from pyFTS.fcm.GA import phenotype
@ -129,15 +128,14 @@ def evaluate(dataset, individual, **kwargs):
return {'rmse': .6 * _rmse + .4 * _std}
def tournament(population, objective):
'''
"""
Simple tournament selection strategy.
:param population: the population
:param objective: the objective to be considered on tournament
:return:
'''
"""
n = len(population) - 1
r1 = random.randint(0, n) if n > 2 else 0
@ -146,14 +144,13 @@ def tournament(population, objective):
return population[ix]
def crossover(parents):
'''
"""
Crossover operation between two parents
:param parents: a list with two genotypes
:return: a genotype
'''
"""
import random
descendent = genotype()
@ -172,12 +169,12 @@ def crossover(parents):
def mutation(individual, pmut):
'''
"""
Mutation operator
:param population:
:return:
'''
"""
import numpy.random
for k in range(parameters['order']):
@ -197,18 +194,17 @@ def mutation(individual, pmut):
individual['weights'][k][row, col] += np.random.normal(0, .5, 1)
individual['weights'][k][row, col] = np.clip(individual['weights'][k][row, col], -1, 1)
return individual
def elitism(population, new_population):
'''
"""
Elitism operation, always select the best individual of the population and discard the worst
:param population:
:param new_population:
:return:
'''
"""
population = sorted(population, key=itemgetter('rmse'))
best = population[0]
@ -220,7 +216,7 @@ def elitism(population, new_population):
def GeneticAlgorithm(dataset, **kwargs):
'''
"""
Genetic algoritm for hyperparameter optimization
:param dataset:
@ -234,7 +230,7 @@ def GeneticAlgorithm(dataset, **kwargs):
:param increment_rate: The increment of the scrolling window, relative to the window_size ([0,1])
:param parameters: dict with model specific arguments for fit method.
:return: the best genotype
'''
"""
statistics = []

30
pyFTS/fcm/GD.py Normal file
View File

@ -0,0 +1,30 @@
import numpy as np
def GD(data, model, alpha, momentum=0.5):
num_concepts = model.partitioner.partitions
weights=[np.random.normal(0,.01,(num_concepts,num_concepts)) for k in range(model.order)]
last_gradient = [None for k in range(model.order) ]
for i in np.arange(model.order, len(data)):
sample = data[i-model.order : i]
target = data[i]
model.fcm.weights = weights
inputs = model.partitioner.fuzzyfy(sample, mode='vector')
activations = [model.fcm.activation_function(inputs[k]) for k in np.arange(model.order)]
forecast = model.predict(sample)[0]
error = target - forecast #)**2
if error == np.nan:
pass
print(error)
for k in np.arange(model.order):
deriv = error * model.fcm.activation_function(activations[k], deriv=True)
if momentum is not None:
if last_gradient[k] is None:
last_gradient[k] = deriv*inputs[k]
tmp_grad = (momentum * last_gradient[k]) + alpha*deriv*inputs[k]
weights[k] -= tmp_grad
else:
weights[k] -= alpha*deriv*inputs[k]
return weights

View File

@ -8,7 +8,7 @@ class FuzzyCognitiveMap(object):
self.order = kwargs.get('order',1)
self.concepts = kwargs.get('partitioner',None)
self.weights = []
self.activation_function = kwargs.get('func', Activations.sigmoid)
self.activation_function = kwargs.get('activation_function', Activations.sigmoid)
def activate(self, concepts):
dot_products = np.zeros(len(self.concepts))

View File

@ -1,6 +1,6 @@
from pyFTS.common import fts
from pyFTS.models import hofts
from pyFTS.fcm import common, GA, Activations
from pyFTS.fcm import common, GA, Activations, GD
import numpy as np
@ -11,11 +11,14 @@ class FCM_FTS(hofts.HighOrderFTS):
self.fcm = common.FuzzyCognitiveMap(**kwargs)
def train(self, data, **kwargs):
'''
GA.parameters['num_concepts'] = self.partitioner.partitions
GA.parameters['order'] = self.order
GA.parameters['partitioner'] = self.partitioner
ret = GA.execute(data, **kwargs)
self.fcm.weights = ret['weights']
'''
self.fcm.weights = GD.GD(data, self, alpha=0.01)
def forecast(self, ndata, **kwargs):
ret = []

44
pyFTS/tests/fcm_fts.py Normal file
View File

@ -0,0 +1,44 @@
from pyFTS.fcm import Activations
import numpy as np
import os
import matplotlib as plt
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from pyFTS.fcm import fts as fcm_fts
from pyFTS.partitioners import Grid
from pyFTS.common import Util
df = pd.read_csv('https://query.data.world/s/56i2vkijbvxhtv5gagn7ggk3zw3ksi', sep=';')
data = df['glo_avg'].values[:]
train = data[:7000]
test = data[7000:7500]
fs = Grid.GridPartitioner(data=train, npart=7)
model = fcm_fts.FCM_FTS(partitioner=fs, order=2, activation_function = Activations.relu)
model.fit(train,
ngen=30, #number of generations
mgen=7, # stop after mgen generations without improvement
npop=10, # number of individuals on population
pcruz=.5, # crossover percentual of population
pmut=.3, # mutation percentual of population
window_size = 7000,
train_rate = .8,
increment_rate =.2,
experiments=1
)
Util.persist_obj(model, 'fcm_fts10c')
'''
model = Util.load_obj('fcm_fts05c')
'''
forecasts = model.predict(test)
print(model)