From dd5ac3de2056087f3883a90141a26ce0343004f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=C3=B4nio=20C=C3=A2ndido?= Date: Wed, 29 Jan 2020 14:29:30 -0300 Subject: [PATCH] Gradient descent training method for FCM_FTS --- pyFTS/fcm/Activations.py | 10 ++++---- pyFTS/fcm/GD.py | 50 ++++++++++++++++++++++------------------ pyFTS/fcm/fts.py | 22 ++++++++++-------- pyFTS/tests/fcm_fts.py | 31 ++++++++++++++----------- 4 files changed, 64 insertions(+), 49 deletions(-) diff --git a/pyFTS/fcm/Activations.py b/pyFTS/fcm/Activations.py index 43ac4ca..12d574a 100644 --- a/pyFTS/fcm/Activations.py +++ b/pyFTS/fcm/Activations.py @@ -3,9 +3,9 @@ import numpy as np def step(x, deriv=False): if deriv: - 1 * (x == 0) + return np.abs(1 * (x == 0)) else: - return 1 * (x > 0) + return np.abs(1 * (x > 0)) def sigmoid(x, deriv=False): @@ -26,14 +26,14 @@ def softmax(x, deriv=False): def tanh(x, deriv=False): if deriv: - pass + return 1 - np.tanh(x)**2 else: return np.tanh(x) def relu(x, deriv=False): if deriv: - return 1. * (x > 0) + return np.abs(1. * (x > 0)) else: - return x * (x > 0) + return np.abs(x * (x > 0)) diff --git a/pyFTS/fcm/GD.py b/pyFTS/fcm/GD.py index 98df383..0694911 100644 --- a/pyFTS/fcm/GD.py +++ b/pyFTS/fcm/GD.py @@ -1,30 +1,36 @@ import numpy as np -def GD(data, model, alpha, momentum=0.5): +def GD(data, model, **kwargs): + alpha = kwargs.get('alpha', 0.1) + momentum = kwargs.get('momentum', None) + iterations = kwargs.get('iterations', 1) num_concepts = model.partitioner.partitions - weights=[np.random.normal(0,.01,(num_concepts,num_concepts)) for k in range(model.order)] + weights = [np.random.normal(0,.01,(num_concepts,num_concepts)) for k in range(model.order)] last_gradient = [None for k in range(model.order) ] - for i in np.arange(model.order, len(data)): - sample = data[i-model.order : i] - target = data[i] - model.fcm.weights = weights - inputs = model.partitioner.fuzzyfy(sample, mode='vector') - activations = [model.fcm.activation_function(inputs[k]) for k in np.arange(model.order)] - forecast = model.predict(sample)[0] - error = target - forecast #)**2 - if error == np.nan: - pass - print(error) - for k in np.arange(model.order): - deriv = error * model.fcm.activation_function(activations[k], deriv=True) - if momentum is not None: - if last_gradient[k] is None: - last_gradient[k] = deriv*inputs[k] + for it in np.arange(iterations): + for i in np.arange(model.order, len(data)): + #i = np.random.randint(model.order, len(data)-model.order) + sample = data[i-model.order : i] + target = data[i] + model.fcm.weights = weights + inputs = model.partitioner.fuzzyfy(sample, mode='vector') + activations = model.fcm.activate(inputs) + forecast = model.predict(sample)[0] + error = target - forecast #)**2 + if str(error) == 'nan' or error == np.nan or error == np.Inf: + print('error') + print(error) + for k in np.arange(model.order): + deriv = error * model.fcm.activation_function(activations[k], deriv=True) + if momentum is not None: + if last_gradient[k] is None: + last_gradient[k] = deriv*inputs[k] - tmp_grad = (momentum * last_gradient[k]) + alpha*deriv*inputs[k] - weights[k] -= tmp_grad - else: - weights[k] -= alpha*deriv*inputs[k] + tmp_grad = (momentum * last_gradient[k]) + alpha*deriv*inputs[k] + weights[k] -= tmp_grad + last_gradient[k] = tmp_grad + else: + weights[k] -= alpha*deriv*inputs[k] return weights \ No newline at end of file diff --git a/pyFTS/fcm/fts.py b/pyFTS/fcm/fts.py index 163aed2..e97eb25 100644 --- a/pyFTS/fcm/fts.py +++ b/pyFTS/fcm/fts.py @@ -11,14 +11,15 @@ class FCM_FTS(hofts.HighOrderFTS): self.fcm = common.FuzzyCognitiveMap(**kwargs) def train(self, data, **kwargs): - ''' - GA.parameters['num_concepts'] = self.partitioner.partitions - GA.parameters['order'] = self.order - GA.parameters['partitioner'] = self.partitioner - ret = GA.execute(data, **kwargs) - self.fcm.weights = ret['weights'] - ''' - self.fcm.weights = GD.GD(data, self, alpha=0.01) + method = kwargs.get('method','GA') + if method == 'GA': + GA.parameters['num_concepts'] = self.partitioner.partitions + GA.parameters['order'] = self.order + GA.parameters['partitioner'] = self.partitioner + ret = GA.execute(data, **kwargs) + self.fcm.weights = ret['weights'] + elif method == 'GD': + self.fcm.weights = GD.GD(data, self, **kwargs) def forecast(self, ndata, **kwargs): ret = [] @@ -33,7 +34,10 @@ class FCM_FTS(hofts.HighOrderFTS): activation = self.fcm.activate(fuzzyfied) - final = np.dot(midpoints, activation)/np.sum(activation) + final = np.dot(midpoints, activation)/np.nanmax([1, np.sum(activation)]) + + if str(final) == 'nan' or final == np.nan or final == np.Inf: + print('error') ret.append(final) diff --git a/pyFTS/tests/fcm_fts.py b/pyFTS/tests/fcm_fts.py index 664167c..04dceee 100644 --- a/pyFTS/tests/fcm_fts.py +++ b/pyFTS/tests/fcm_fts.py @@ -18,27 +18,32 @@ data = df['glo_avg'].values[:] train = data[:7000] test = data[7000:7500] -fs = Grid.GridPartitioner(data=train, npart=7) +fs = Grid.GridPartitioner(data=train, npart=5) model = fcm_fts.FCM_FTS(partitioner=fs, order=2, activation_function = Activations.relu) -model.fit(train, - ngen=30, #number of generations - mgen=7, # stop after mgen generations without improvement - npop=10, # number of individuals on population - pcruz=.5, # crossover percentual of population - pmut=.3, # mutation percentual of population - window_size = 7000, - train_rate = .8, - increment_rate =.2, - experiments=1 - ) + +model.fit(train, method='GD', alpha=0.02, momentum=0.8, iteractions=3 ) Util.persist_obj(model, 'fcm_fts10c') ''' model = Util.load_obj('fcm_fts05c') ''' +#forecasts = model.predict(test) + +#print(model) + +fig, ax = plt.subplots(nrows=1, ncols=1, figsize=[15,5]) + +ax.plot(test,label='Original') forecasts = model.predict(test) -print(model) \ No newline at end of file +for w in np.arange(model.order): + forecasts.insert(0,None) + +ax.plot(forecasts, label=model.shortname) + +plt.show() + +print("") \ No newline at end of file