sam/pyFTS
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

- common.Util.distributed_train

Browse Source 
- Big refactoring to change FTS.sets from list to dict. This refactoring allow to remove references to the fuzzy sets from the FLRG and save memory.
 - HOFTS and PWFTS train and forecasting simplification by using the method generate_lhs_flrg
 - Small others bugfixes/improvements
This commit is contained in:
Petrônio Cândido 2018-03-03 20:07:50 -03:00
parent 00db6a30ad
commit 9718f48b39
49 changed files with 1289 additions and 1166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
pyFTS/benchmarks/benchmarks.py
View File

@ -198,10 +198,10 @@ def point_sliding_window(data, windowsize, train=0.8, models=None, partitioners=
smape[tmp['key']] = []
u[tmp['key']] = []
times[tmp['key']] = []
rmse[tmp['key']].append(tmp['rmse'])
smape[tmp['key']].append(tmp['smape'])
u[tmp['key']].append(tmp['u'])
times[tmp['key']].append(tmp['time'])
rmse[tmp['key']].append_rhs(tmp['rmse'])
smape[tmp['key']].append_rhs(tmp['smape'])
u[tmp['key']].append_rhs(tmp['u'])
times[tmp['key']].append_rhs(tmp['time'])
print(tmp['key'], tmp['window'])
_process_end = time.time()
@ -393,10 +393,10 @@ def interval_sliding_window(data, windowsize, train=0.8, models=None, partitione
_sharp, _res, _cov = Measures.get_interval_statistics(test, mfts)
_end = time.time()
_tdiff += _end - _start
sharpness[_key].append(_sharp)
resolution[_key].append(_res)
coverage[_key].append(_cov)
times[_key].append(_tdiff)
sharpness[_key].append_rhs(_sharp)
resolution[_key].append_rhs(_res)
coverage[_key].append_rhs(_cov)
times[_key].append_rhs(_tdiff)
else:
for order in np.arange(1, max_order + 1):
@ -428,10 +428,10 @@ def interval_sliding_window(data, windowsize, train=0.8, models=None, partitione
_sharp, _res, _cov = Measures.get_interval_statistics(test, mfts)
_end = time.time()
_tdiff += _end - _start
sharpness[_key].append(_sharp)
resolution[_key].append(_res)
coverage[_key].append(_cov)
times[_key].append(_tdiff)
sharpness[_key].append_rhs(_sharp)
resolution[_key].append_rhs(_res)
coverage[_key].append_rhs(_cov)
times[_key].append_rhs(_tdiff)
return bUtil.save_dataframe_interval(coverage, experiments, file, objs, resolution, save, sharpness, synthetic, times)
@ -642,8 +642,8 @@ def ahead_sliding_window(data, windowsize, train, steps, models=None, resolution
_crps1, _crps2, _t1, _t2 = Measures.get_distribution_statistics(test,mfts,steps=steps,resolution=resolution)
crps_interval[_key].append(_crps1)
crps_distr[_key].append(_crps2)
crps_interval[_key].append_rhs(_crps1)
crps_distr[_key].append_rhs(_crps2)
times1[_key] = _tdiff + _t1
times2[_key] = _tdiff + _t2
@ -678,8 +678,8 @@ def ahead_sliding_window(data, windowsize, train, steps, models=None, resolution
_crps1, _crps2, _t1, _t2 = Measures.get_distribution_statistics(test, mfts, steps=steps,
resolution=resolution)
crps_interval[_key].append(_crps1)
crps_distr[_key].append(_crps2)
crps_interval[_key].append_rhs(_crps1)
crps_distr[_key].append_rhs(_crps2)
times1[_key] = _tdiff + _t1
times2[_key] = _tdiff + _t2

32
pyFTS/benchmarks/distributed_benchmarks.py
View File

@ -150,10 +150,10 @@ def point_sliding_window(data, windowsize, train=0.8, inc=0.1, models=None, part
smape[tmp['key']] = []
u[tmp['key']] = []
times[tmp['key']] = []
rmse[tmp['key']].append(tmp['rmse'])
smape[tmp['key']].append(tmp['smape'])
u[tmp['key']].append(tmp['u'])
times[tmp['key']].append(tmp['time'])
rmse[tmp['key']].append_rhs(tmp['rmse'])
smape[tmp['key']].append_rhs(tmp['smape'])
u[tmp['key']].append_rhs(tmp['u'])
times[tmp['key']].append_rhs(tmp['time'])
print(tmp['key'], tmp['window'])
else:
print(job.exception)
@ -377,14 +377,14 @@ def interval_sliding_window(data, windowsize, train=0.8, inc=0.1, models=None,
q75[tmp['key']] = []
q95[tmp['key']] = []
sharpness[tmp['key']].append(tmp['sharpness'])
resolution[tmp['key']].append(tmp['resolution'])
coverage[tmp['key']].append(tmp['coverage'])
times[tmp['key']].append(tmp['time'])
q05[tmp['key']].append(tmp['Q05'])
q25[tmp['key']].append(tmp['Q25'])
q75[tmp['key']].append(tmp['Q75'])
q95[tmp['key']].append(tmp['Q95'])
sharpness[tmp['key']].append_rhs(tmp['sharpness'])
resolution[tmp['key']].append_rhs(tmp['resolution'])
coverage[tmp['key']].append_rhs(tmp['coverage'])
times[tmp['key']].append_rhs(tmp['time'])
q05[tmp['key']].append_rhs(tmp['Q05'])
q25[tmp['key']].append_rhs(tmp['Q25'])
q75[tmp['key']].append_rhs(tmp['Q75'])
q95[tmp['key']].append_rhs(tmp['Q95'])
print(tmp['key'])
else:
print(job.exception)
@ -576,10 +576,10 @@ def ahead_sliding_window(data, windowsize, steps, resolution, train=0.8, inc=0.1
crps_distr[tmp['key']] = []
times1[tmp['key']] = []
times2[tmp['key']] = []
crps_interval[tmp['key']].append(tmp['CRPS_Interval'])
crps_distr[tmp['key']].append(tmp['CRPS_Distribution'])
times1[tmp['key']].append(tmp['TIME_Interval'])
times2[tmp['key']].append(tmp['TIME_Distribution'])
crps_interval[tmp['key']].append_rhs(tmp['CRPS_Interval'])
crps_distr[tmp['key']].append_rhs(tmp['CRPS_Distribution'])
times1[tmp['key']].append_rhs(tmp['TIME_Interval'])
times2[tmp['key']].append_rhs(tmp['TIME_Distribution'])
else:
print(job.exception)

24
pyFTS/benchmarks/parallel_benchmarks.py
View File

@ -127,10 +127,10 @@ def point_sliding_window(data, windowsize, train=0.8, models=None, partitioners=
smape[tmp['key']] = []
u[tmp['key']] = []
times[tmp['key']] = []
rmse[tmp['key']].append(tmp['rmse'])
smape[tmp['key']].append(tmp['smape'])
u[tmp['key']].append(tmp['u'])
times[tmp['key']].append(tmp['time'])
rmse[tmp['key']].append_rhs(tmp['rmse'])
smape[tmp['key']].append_rhs(tmp['smape'])
u[tmp['key']].append_rhs(tmp['u'])
times[tmp['key']].append_rhs(tmp['time'])
_process_end = time.time()
@ -254,10 +254,10 @@ def interval_sliding_window(data, windowsize, train=0.8, models=None, partitione
coverage[tmp['key']] = []
times[tmp['key']] = []
sharpness[tmp['key']].append(tmp['sharpness'])
resolution[tmp['key']].append(tmp['resolution'])
coverage[tmp['key']].append(tmp['coverage'])
times[tmp['key']].append(tmp['time'])
sharpness[tmp['key']].append_rhs(tmp['sharpness'])
resolution[tmp['key']].append_rhs(tmp['resolution'])
coverage[tmp['key']].append_rhs(tmp['coverage'])
times[tmp['key']].append_rhs(tmp['time'])
_process_end = time.time()
@ -384,10 +384,10 @@ def ahead_sliding_window(data, windowsize, train, steps,resolution, models=None,
times1[tmp['key']] = []
times2[tmp['key']] = []
crps_interval[tmp['key']].append(tmp['CRPS_Interval'])
crps_distr[tmp['key']].append(tmp['CRPS_Distribution'])
times1[tmp['key']].append(tmp['TIME_Interval'])
times2[tmp['key']].append(tmp['TIME_Distribution'])
crps_interval[tmp['key']].append_rhs(tmp['CRPS_Interval'])
crps_distr[tmp['key']].append_rhs(tmp['CRPS_Distribution'])
times1[tmp['key']].append_rhs(tmp['TIME_Interval'])
times2[tmp['key']].append_rhs(tmp['TIME_Distribution'])
_process_end = time.time()

4
pyFTS/common/FLR.py
View File

@ -17,7 +17,7 @@ class FLR(object):
self.RHS = RHS
def __str__(self):
return self.LHS.name + " -> " + self.RHS.name
return self.LHS + " -> " + self.RHS
class IndexedFLR(FLR):
@ -33,7 +33,7 @@ class IndexedFLR(FLR):
self.index = index
def __str__(self):
return str(self.index) + ": "+ self.LHS.name + " -> " + self.RHS.name
return str(self.index) + ": "+ self.LHS + " -> " + self.RHS
def generate_high_order_recurrent_flr(fuzzyData):

68
pyFTS/common/FuzzySet.py
View File

@ -58,47 +58,62 @@ class FuzzySet(object):
return self.name + ": " + str(self.mf.__name__) + "(" + str(self.parameters) + ")"
def fuzzyfy_instance(inst, fuzzySets):
def set_ordered(fuzzySets):
return [k for k in sorted(fuzzySets.keys())]
def fuzzyfy_instance(inst, fuzzySets, ordered_sets=None):
"""
Calculate the membership values for a data point given fuzzy sets
:param inst: data point
:param fuzzySets: list of fuzzy sets
:param fuzzySets: dict of fuzzy sets
:return: array of membership values
"""
mv = np.array([fs.membership(inst) for fs in fuzzySets])
return mv
if ordered_sets is None:
ordered_sets = set_ordered(fuzzySets)
mv = []
for key in ordered_sets:
mv.append( fuzzySets[key].membership(inst))
return np.array(mv)
def fuzzyfy_instances(data, fuzzySets):
def fuzzyfy_instances(data, fuzzySets, ordered_sets=None):
"""
Calculate the membership values for a data point given fuzzy sets
:param inst: data point
:param fuzzySets: list of fuzzy sets
:param fuzzySets: dict of fuzzy sets
:return: array of membership values
"""
ret = []
if ordered_sets is None:
ordered_sets = set_ordered(fuzzySets)
for inst in data:
mv = np.array([fs.membership(inst) for fs in fuzzySets])
mv = np.array([fuzzySets[key].membership(inst) for key in ordered_sets])
ret.append(mv)
return ret
def get_maximum_membership_fuzzyset(inst, fuzzySets):
def get_maximum_membership_fuzzyset(inst, fuzzySets, ordered_sets=None):
"""
Fuzzify a data point, returning the fuzzy set with maximum membership value
:param inst: data point
:param fuzzySets: list of fuzzy sets
:param fuzzySets: dict of fuzzy sets
:return: fuzzy set with maximum membership
"""
mv = fuzzyfy_instance(inst, fuzzySets)
return fuzzySets[np.argwhere(mv == max(mv))[0, 0]]
if ordered_sets is None:
ordered_sets = set_ordered(fuzzySets)
mv = np.array([fuzzySets[key].membership(inst) for key in ordered_sets])
key = ordered_sets[np.argwhere(mv == max(mv))[0, 0]]
return fuzzySets[key]
def get_maximum_membership_fuzzyset_index(inst, fuzzySets):
"""
Fuzzify a data point, returning the fuzzy set with maximum membership value
:param inst: data point
:param fuzzySets: list of fuzzy sets
:param fuzzySets: dict of fuzzy sets
:return: fuzzy set with maximum membership
"""
mv = fuzzyfy_instance(inst, fuzzySets)
@ -108,37 +123,38 @@ def get_maximum_membership_fuzzyset_index(inst, fuzzySets):
def fuzzyfy_series_old(data, fuzzySets, method='maximum'):
fts = []
for item in data:
fts.append(get_maximum_membership_fuzzyset(item, fuzzySets))
fts.append(get_maximum_membership_fuzzyset(item, fuzzySets).name)
return fts
def fuzzify_series(data, fuzzySets, method='maximum'):
def fuzzyfy_series(data, fuzzySets, method='maximum'):
fts = []
ordered_sets = set_ordered(fuzzySets)
for t, i in enumerate(data):
mv = np.array([fs.membership(i) for fs in fuzzySets])
mv = np.array([fuzzySets[key].membership(i) for key in ordered_sets])
if len(mv) == 0:
sets = check_bounds(i, fuzzySets)
sets = check_bounds(i, fuzzySets.items(), ordered_sets)
else:
if method == 'fuzzy':
ix = np.ravel(np.argwhere(mv > 0.0))
sets = [fuzzySets[i] for i in ix]
sets = [fuzzySets[ordered_sets[i]].name for i in ix]
elif method == 'maximum':
mx = max(mv)
ix = np.ravel(np.argwhere(mv == mx))
sets = fuzzySets[ix[0]]
sets = fuzzySets[ordered_sets[ix[0]]].name
fts.append(sets)
return fts
def check_bounds(data, sets):
if data < sets[0].get_lower():
return sets[0]
elif data > sets[-1].get_upper():
return sets[-1]
def check_bounds(data, sets, ordered_sets):
if data < sets[ordered_sets[0]].get_lower():
return sets[ordered_sets[0]]
elif data > sets[ordered_sets[-1]].get_upper():
return sets[ordered_sets[-1]]
def check_bounds_index(data, sets):
if data < sets[0].get_lower():
def check_bounds_index(data, sets, ordered_sets):
if data < sets[ordered_sets[0]].get_lower():
return 0
elif data > sets[-1].get_upper():
elif data > sets[ordered_sets[-1]].get_upper():
return len(sets) -1

70
pyFTS/common/Util.py
View File

@ -86,4 +86,72 @@ def persist_env(file):
dill.dump_session(file)
def load_env(file):
dill.load_session(file)
dill.load_session(file)
def simple_model_train(model, data, parameters):
model.train(data, **parameters)
return model
def distributed_train(model, train_method, nodes, fts_method, data, num_batches,
train_parameters, **kwargs):
import dispy, dispy.httpd, datetime
batch_save = kwargs.get('batch_save', True) # save model between batches
file_path = kwargs.get('file_path', None)
cluster = dispy.JobCluster(train_method, nodes=nodes) # , depends=dependencies)
http_server = dispy.httpd.DispyHTTPServer(cluster)
print("[{0: %H:%M:%S}] Distrituted Train Started".format(datetime.datetime.now()))
jobs = []
n = len(data)
batch_size = int(n / num_batches)
bcount = 1
for ct in range(model.order, n, batch_size):
if model.is_multivariate:
ndata = data.iloc[ct - model.order:ct + batch_size]
else:
ndata = data[ct - model.order: ct + batch_size]
#self.train(ndata, **kwargs)
tmp_model = fts_method(str(bcount))
tmp_model.clone_parameters(model)
job = cluster.submit(tmp_model, ndata, train_parameters)
job.id = bcount # associate an ID to identify jobs (if needed later)
jobs.append(job)
bcount += 1
for job in jobs:
print("[{0: %H:%M:%S}] Processing batch ".format(datetime.datetime.now()) + str(job.id))
tmp = job()
if job.status == dispy.DispyJob.Finished and tmp is not None:
model.merge(tmp)
if batch_save:
persist_obj(model, file_path)
else:
print(job.exception)
print(job.stdout)
print("[{0: %H:%M:%S}] Finished batch ".format(datetime.datetime.now()) + str(job.id))
print("[{0: %H:%M:%S}] Distrituted Train Finished".format(datetime.datetime.now()))
cluster.wait() # wait for all jobs to finish
cluster.print_status()
http_server.shutdown() # this waits until browser gets all updates
cluster.close()
return model

50
pyFTS/common/flrg.py
View File

@ -10,41 +10,63 @@ class FLRG(object):
self.midpoint = None
self.lower = None
self.upper = None
self.key = None
def get_membership(self, data):
def append_rhs(self, set, **kwargs):
pass
def get_key(self):
if self.key is None:
if isinstance(self.LHS, (list, set)):
names = [c for c in self.LHS]
elif isinstance(self.LHS, dict):
names = [self.LHS[k] for k in self.LHS.keys()]
else:
names = [self.LHS]
self.key = ""
for n in names:
if len(self.key) > 0:
self.key += ","
self.key = self.key + n
return self.key
def get_membership(self, data, sets):
ret = 0.0
if isinstance(self.LHS, (list, set)):
assert len(self.LHS) == len(data)
ret = np.nanmin([self.LHS[ct].membership(dat) for ct, dat in enumerate(data)])
ret = np.nanmin([sets[self.LHS[ct]].membership(dat) for ct, dat in enumerate(data)])
else:
ret = self.LHS.membership(data)
ret = sets[self.LHS].membership(data)
return ret
def get_midpoint(self):
def get_midpoint(self, sets):
if self.midpoint is None:
self.midpoint = np.nanmean(self.get_midpoints())
self.midpoint = np.nanmean(self.get_midpoints(sets))
return self.midpoint
def get_midpoints(self):
def get_midpoints(self,sets):
if isinstance(self.RHS, (list, set)):
return np.array([s.centroid for s in self.RHS])
return np.array([sets[s].centroid for s in self.RHS])
elif isinstance(self.RHS, dict):
return np.array([self.RHS[s].centroid for s in self.RHS.keys()])
return np.array([sets[self.RHS[s]].centroid for s in self.RHS.keys()])
def get_lower(self):
def get_lower(self,sets):
if self.lower is None:
if isinstance(self.RHS, list):
self.lower = min([rhs.lower for rhs in self.RHS])
self.lower = min([sets[rhs].lower for rhs in self.RHS])
elif isinstance(self.RHS, dict):
self.lower = min([self.RHS[s].lower for s in self.RHS.keys()])
self.lower = min([sets[self.RHS[s]].lower for s in self.RHS.keys()])
return self.lower
def get_upper(self, t):
def get_upper(self, t,sets):
if self.upper is None:
if isinstance(self.RHS, list):
self.upper = max([rhs.upper for rhs in self.RHS])
self.upper = max([sets[rhs].upper for rhs in self.RHS])
elif isinstance(self.RHS, dict):
self.upper = max([self.RHS[s].upper for s in self.RHS.keys()])
self.upper = max([sets[self.RHS[s]].upper for s in self.RHS.keys()])
return self.upper
def __len__(self):

130
pyFTS/common/fts.py
View File

@ -2,6 +2,11 @@ import numpy as np
import pandas as pd
from pyFTS.common import FuzzySet, SortedCollection, tree, Util
def parallel_train(data, method, **kwargs):
model = method(**kwargs)
model.train(data)
return model
class FTS(object):
"""
@ -124,7 +129,7 @@ class FTS(object):
tmp = tmp[0]
ret.append(tmp)
data.append(tmp)
data.append_rhs(tmp)
return ret
@ -173,6 +178,8 @@ class FTS(object):
:return:
"""
import datetime
num_batches = kwargs.get('num_batches', None)
save = kwargs.get('save_model', False) # save model on disk
@ -181,26 +188,83 @@ class FTS(object):
file_path = kwargs.get('file_path', None)
if num_batches is not None:
n = len(data)
batch_size = round(n / num_batches, 0)
for ct in range(self.order, n, batch_size):
if self.is_multivariate:
ndata = data.iloc[ct - self.order:ct + batch_size]
else:
ndata = data[ct - self.order : ct + batch_size]
self.train(ndata, **kwargs)
if batch_save:
Util.persist_obj(self,file_path)
distributed = kwargs.get('distributed', False)
if distributed:
nodes = kwargs.get('nodes', False)
train_method = kwargs.get('train_method', Util.simple_model_train)
Util.distributed_train(self, train_method, nodes, type(self), data, num_batches, {},
batch_save=batch_save, file_path=file_path)
else:
self.train(data, **kwargs)
print("[{0: %H:%M:%S}] Start training".format(datetime.datetime.now()))
if num_batches is not None:
n = len(data)
batch_size = int(n / num_batches)
bcount = 1
for ct in range(self.order, n, batch_size):
print("[{0: %H:%M:%S}] Starting batch ".format(datetime.datetime.now()) + str(bcount))
if self.is_multivariate:
ndata = data.iloc[ct - self.order:ct + batch_size]
else:
ndata = data[ct - self.order : ct + batch_size]
self.train(ndata, **kwargs)
if batch_save:
Util.persist_obj(self,file_path)
print("[{0: %H:%M:%S}] Finish batch ".format(datetime.datetime.now()) + str(bcount))
bcount += 1
else:
self.train(data, **kwargs)
print("[{0: %H:%M:%S}] Finish training".format(datetime.datetime.now()))
if save:
Util.persist_obj(self, file_path)
def clone_parameters(self, model):
self.order = model.order
self.shortname = model.shortname
self.name = model.name
self.detail = model.detail
self.is_high_order = model.is_high_order
self.min_order = model.min_order
self.has_seasonality = model.has_seasonality
self.has_point_forecasting = model.has_point_forecasting
self.has_interval_forecasting = model.has_interval_forecasting
self.has_probability_forecasting = model.has_probability_forecasting
self.is_multivariate = model.is_multivariate
self.dump = model.dump
self.transformations = model.transformations
self.transformations_param = model.transformations_param
self.original_max = model.original_max
self.original_min = model.original_min
self.partitioner = model.partitioner
self.sets = model.sets
self.auto_update = model.auto_update
self.benchmark_only = model.benchmark_only
self.indexer = model.indexer
def merge(self, model):
for key in model.flrgs.keys():
flrg = model.flrgs[key]
if flrg.get_key() not in self.flrgs:
self.flrgs[flrg.get_key()] = flrg
else:
if isinstance(flrg.RHS, (list, set)):
for k in flrg.RHS:
self.flrgs[flrg.get_key()].append_rhs(k)
elif isinstance(flrg.RHS, dict):
for k in flrg.RHS.keys():
self.flrgs[flrg.get_key()].append_rhs(flrg.RHS[k])
else:
self.flrgs[flrg.get_key()].append_rhs(flrg.RHS)
def append_transformation(self, transformation):
if transformation is not None:
self.transformations.append(transformation)
@ -251,42 +315,6 @@ class FTS(object):
def len_total(self):
return sum([len(k) for k in self.flrgs])
def get_empty_grid(self, _min, _max, resolution):
grid = {}
for sbin in np.arange(_min,_max, resolution):
grid[sbin] = 0
return grid
def getGridClean(self, resolution):
if len(self.transformations) == 0:
_min = self.sets[0].lower
_max = self.sets[-1].upper
else:
_min = self.original_min
_max = self.original_max
return self.get_empty_grid(_min, _max, resolution)
def gridCount(self, grid, resolution, index, interval):
#print(point_to_interval)
for k in index.inside(interval[0],interval[1]):
grid[k] += 1
return grid
def gridCountPoint(self, grid, resolution, index, point):
if not isinstance(point, (list, np.ndarray)):
point = [point]
for p in point:
k = index.find_ge(p)
grid[k] += 1
return grid
def get_UoD(self):
return [self.original_min, self.original_max]

23
pyFTS/models/chen.py
View File

@ -15,16 +15,19 @@ class ConventionalFLRG(flrg.FLRG):
self.LHS = LHS
self.RHS = set()
def append(self, c):
def get_key(self):
return sets[self.LHS].name
def append_rhs(self, c, **kwargs):
self.RHS.add(c)
def __str__(self):
tmp = self.LHS.name + " -> "
tmp = self.LHS + " -> "
tmp2 = ""
for c in sorted(self.RHS, key=lambda s: s.name):
for c in sorted(self.RHS, key=lambda s: s):
if len(tmp2) > 0:
tmp2 = tmp2 + ","
tmp2 = tmp2 + c.name
tmp2 = tmp2 + c
return tmp + tmp2
@ -38,11 +41,11 @@ class ConventionalFTS(fts.FTS):
def generate_flrg(self, flrs):
for flr in flrs:
if flr.LHS.name in self.flrgs:
self.flrgs[flr.LHS.name].append(flr.RHS)
if flr.LHS in self.flrgs:
self.flrgs[flr.LHS].append_rhs(flr.RHS)
else:
self.flrgs[flr.LHS.name] = ConventionalFLRG(flr.LHS)
self.flrgs[flr.LHS.name].append(flr.RHS)
self.flrgs[flr.LHS] = ConventionalFLRG(flr.LHS)
self.flrgs[flr.LHS].append_rhs(flr.RHS)
def train(self, data, **kwargs):
if kwargs.get('sets', None) is not None:
@ -64,14 +67,14 @@ class ConventionalFTS(fts.FTS):
mv = FuzzySet.fuzzyfy_instance(ndata[k], self.sets)
actual = self.sets[np.argwhere(mv == max(mv))[0, 0]]
actual = FuzzySet.get_maximum_membership_fuzzyset(ndata[k], self.sets) #self.sets[np.argwhere(mv == max(mv))[0, 0]]
if actual.name not in self.flrgs:
ret.append(actual.centroid)
else:
_flrg = self.flrgs[actual.name]
ret.append(_flrg.get_midpoint())
ret.append(_flrg.get_midpoint(self.sets))
ret = self.apply_inverse_transformations(ret, params=[data])

14
pyFTS/models/cheng.py
View File

@ -18,7 +18,7 @@ class TrendWeightedFLRG(yu.WeightedFLRG):
super(TrendWeightedFLRG, self).__init__(LHS, **kwargs)
self.w = None
def weights(self):
def weights(self, sets):
if self.w is None:
count_nochange = 0.0
count_up = 0.0
@ -27,10 +27,10 @@ class TrendWeightedFLRG(yu.WeightedFLRG):
for c in self.RHS:
tmp = 0
if self.LHS.centroid == c.centroid:
if sets[self.LHS].centroid == sets[c].centroid:
count_nochange += 1.0
tmp = count_nochange
elif self.LHS.centroid > c.centroid:
elif sets[self.LHS].centroid > sets[c].centroid:
count_down += 1.0
tmp = count_down
else:
@ -54,8 +54,8 @@ class TrendWeightedFTS(yu.WeightedFTS):
def generate_FLRG(self, flrs):
for flr in flrs:
if flr.LHS.name in self.flrgs:
self.flrgs[flr.LHS.name].append(flr.RHS)
if flr.LHS in self.flrgs:
self.flrgs[flr.LHS].append_rhs(flr.RHS)
else:
self.flrgs[flr.LHS.name] = TrendWeightedFLRG(flr.LHS)
self.flrgs[flr.LHS.name].append(flr.RHS)
self.flrgs[flr.LHS] = TrendWeightedFLRG(flr.LHS)
self.flrgs[flr.LHS].append_rhs(flr.RHS)

105
pyFTS/models/hofts.py
View File

@ -16,17 +16,9 @@ class HighOrderFLRG(flrg.FLRG):
self.RHS = {}
self.strlhs = ""
def append_rhs(self, c):
if c.name not in self.RHS:
self.RHS[c.name] = c
def str_lhs(self):
if len(self.strlhs) == 0:
for c in self.LHS:
if len(self.strlhs) > 0:
self.strlhs += ", "
self.strlhs = self.strlhs + str(c.name)
return self.strlhs
def append_rhs(self, c, **kwargs):
if c not in self.RHS:
self.RHS[c] = c
def append_lhs(self, c):
self.LHS.append(c)
@ -37,7 +29,7 @@ class HighOrderFLRG(flrg.FLRG):
if len(tmp) > 0:
tmp = tmp + ","
tmp = tmp + c
return self.str_lhs() + " -> " + tmp
return self.get_key() + " -> " + tmp
def __len__(self):
@ -55,40 +47,30 @@ class HighOrderFTS(fts.FTS):
self.setsDict = {}
self.is_high_order = True
def build_tree(self, node, lags, level):
if level >= self.order:
return
def generate_lhs_flrg(self, sample):
lags = {}
for s in lags[level]:
node.appendChild(tree.FLRGTreeNode(s))
flrgs = []
for child in node.getChildren():
self.build_tree(child, lags, level + 1)
for o in np.arange(0, self.order):
lhs = [key for key in self.partitioner.ordered_sets if self.sets[key].membership(sample[o]) > 0.0]
lags[o] = lhs
def build_tree_without_order(self, node, lags, level):
root = tree.FLRGTreeNode(None)
if level not in lags:
return
tree.build_tree_without_order(root, lags, 0)
for s in lags[level]:
node.appendChild(tree.FLRGTreeNode(s))
for child in node.getChildren():
self.build_tree_without_order(child, lags, level + 1)
def generateFLRG(self, flrs):
l = len(flrs)
for k in np.arange(self.order + 1, l):
# Trace the possible paths
for p in root.paths():
flrg = HighOrderFLRG(self.order)
path = list(reversed(list(filter(None.__ne__, p))))
for kk in np.arange(k - self.order, k):
flrg.append_lhs(flrs[kk].LHS)
for lhs in path:
flrg.append_lhs(lhs)
if flrg.str_lhs() in self.flrgs:
self.flrgs[flrg.str_lhs()].append_rhs(flrs[k].RHS)
else:
self.flrgs[flrg.str_lhs()] = flrg;
self.flrgs[flrg.str_lhs()].append_rhs(flrs[k].RHS)
flrgs.append(flrg)
return flrgs
def generate_flrg(self, data):
l = len(data)
@ -97,32 +79,16 @@ class HighOrderFTS(fts.FTS):
sample = data[k - self.order: k]
rhs = [set for set in self.sets if set.membership(data[k]) > 0.0]
rhs = [key for key in self.partitioner.ordered_sets if self.sets[key].membership(data[k]) > 0.0]
lags = {}
flrgs = self.generate_lhs_flrg(sample)
for o in np.arange(0, self.order):
lhs = [set for set in self.sets if set.membership(sample[o]) > 0.0]
lags[o] = lhs
root = tree.FLRGTreeNode(None)
self.build_tree_without_order(root, lags, 0)
# Trace the possible paths
for p in root.paths():
flrg = HighOrderFLRG(self.order)
path = list(reversed(list(filter(None.__ne__, p))))
for lhs in path:
flrg.append_lhs(lhs)
if flrg.str_lhs() not in self.flrgs:
self.flrgs[flrg.str_lhs()] = flrg;
for flrg in flrgs:
if flrg.get_key() not in self.flrgs:
self.flrgs[flrg.get_key()] = flrg;
for st in rhs:
self.flrgs[flrg.str_lhs()].append_rhs(st)
self.flrgs[flrg.get_key()].append_rhs(st)
def train(self, data, **kwargs):
@ -133,7 +99,7 @@ class HighOrderFTS(fts.FTS):
if kwargs.get('sets', None) is not None:
self.sets = kwargs.get('sets', None)
for s in self.sets: self.setsDict[s.name] = s
self.generate_flrg(data)
def forecast(self, data, **kwargs):
@ -148,16 +114,17 @@ class HighOrderFTS(fts.FTS):
ndata = self.apply_transformations(data)
for k in np.arange(self.order, l+1):
tmpdata = FuzzySet.fuzzyfy_series_old(ndata[k - self.order: k], self.sets)
tmpflrg = HighOrderFLRG(self.order)
flrgs = self.generate_lhs_flrg(ndata[k - self.order: k])
for s in tmpdata: tmpflrg.append_lhs(s)
for flrg in flrgs:
tmp = []
if flrg.get_key() not in self.flrgs:
tmp.append(self.sets[flrg.LHS[-1]].centroid)
else:
flrg = self.flrgs[flrg.get_key()]
tmp.append(flrg.get_midpoint(self.sets))
if tmpflrg.str_lhs() not in self.flrgs:
ret.append(tmpdata[-1].centroid)
else:
flrg = self.flrgs[tmpflrg.str_lhs()]
ret.append(flrg.get_midpoint())
ret.append(np.nanmean(tmp))
ret = self.apply_inverse_transformations(ret, params=[data[self.order - 1:]])

28
pyFTS/models/hwang.py
View File

@ -20,28 +20,34 @@ class HighOrderFTS(fts.FTS):
def forecast(self, data, **kwargs):
ordered_sets = FuzzySet.set_ordered(self.sets)
ndata = self.apply_transformations(data)
cn = np.array([0.0 for k in range(len(self.sets))])
ow = np.array([[0.0 for k in range(len(self.sets))] for z in range(self.order - 1)])
rn = np.array([[0.0 for k in range(len(self.sets))] for z in range(self.order - 1)])
ft = np.array([0.0 for k in range(len(self.sets))])
l = len(self.sets)
cn = np.array([0.0 for k in range(l)])
ow = np.array([[0.0 for k in range(l)] for z in range(self.order - 1)])
rn = np.array([[0.0 for k in range(l)] for z in range(self.order - 1)])
ft = np.array([0.0 for k in range(l)])
ret = []
for t in np.arange(self.order-1, len(ndata)):
for s in range(len(self.sets)):
cn[s] = self.sets[s].membership(ndata[t])
for ix in range(l):
s = ordered_sets[ix]
cn[ix] = self.sets[s].membership(ndata[t])
for w in range(self.order - 1):
ow[w, s] = self.sets[s].membership(ndata[t - w])
rn[w, s] = ow[w, s] * cn[s]
ft[s] = max(ft[s], rn[w, s])
ow[w, ix] = self.sets[s].membership(ndata[t - w])
rn[w, ix] = ow[w, ix] * cn[ix]
ft[ix] = max(ft[ix], rn[w, ix])
mft = max(ft)
out = 0.0
count = 0.0
for s in range(len(self.sets)):
if ft[s] == mft:
for ix in range(l):
s = ordered_sets[ix]
if ft[ix] == mft:
out = out + self.sets[s].centroid
count += 1.0
ret.append(out / count)

8
pyFTS/models/ifts.py
View File

@ -24,16 +24,16 @@ class IntervalFTS(hofts.HighOrderFTS):
self.is_high_order = True
def get_upper(self, flrg):
if flrg.str_lhs() in self.flrgs:
tmp = self.flrgs[flrg.str_lhs()]
if flrg.get_key() in self.flrgs:
tmp = self.flrgs[flrg.get_key()]
ret = tmp.get_upper()
else:
ret = flrg.LHS[-1].upper
return ret
def get_lower(self, flrg):
if flrg.str_lhs() in self.flrgs:
tmp = self.flrgs[flrg.str_lhs()]
if flrg.get_key() in self.flrgs:
tmp = self.flrgs[flrg.get_key()]
ret = tmp.get_lower()
else:
ret = flrg.LHS[-1].lower

33
pyFTS/models/ismailefendi.py
View File

@ -19,12 +19,12 @@ class ImprovedWeightedFLRG(flrg.FLRG):
self.count = 0.0
self.w = None
def append(self, c):
if c.name not in self.RHS:
self.RHS[c.name] = c
self.rhs_counts[c.name] = 1.0
def append_rhs(self, c, **kwargs):
if c not in self.RHS:
self.RHS[c] = c
self.rhs_counts[c] = 1.0
else:
self.rhs_counts[c.name] += 1.0
self.rhs_counts[c] += 1.0
self.count += 1.0
def weights(self):
@ -33,7 +33,7 @@ class ImprovedWeightedFLRG(flrg.FLRG):
return self.w
def __str__(self):
tmp = self.LHS.name + " -> "
tmp = self.LHS + " -> "
tmp2 = ""
for c in sorted(self.RHS.keys()):
if len(tmp2) > 0:
@ -51,31 +51,30 @@ class ImprovedWeightedFTS(fts.FTS):
super(ImprovedWeightedFTS, self).__init__(1, "IWFTS " + name, **kwargs)
self.name = "Improved Weighted FTS"
self.detail = "Ismail & Efendi"
self.setsDict = {}
def generate_flrg(self, flrs):
for flr in flrs:
if flr.LHS.name in self.flrgs:
self.flrgs[flr.LHS.name].append(flr.RHS)
if flr.LHS in self.flrgs:
self.flrgs[flr.LHS].append_rhs(flr.RHS)
else:
self.flrgs[flr.LHS.name] = ImprovedWeightedFLRG(flr.LHS);
self.flrgs[flr.LHS.name].append(flr.RHS)
self.flrgs[flr.LHS] = ImprovedWeightedFLRG(flr.LHS);
self.flrgs[flr.LHS].append_rhs(flr.RHS)
def train(self, data, **kwargs):
if kwargs.get('sets', None) is not None:
self.sets = kwargs.get('sets', None)
for s in self.sets: self.setsDict[s.name] = s
ndata = self.apply_transformations(data)
tmpdata = FuzzySet.fuzzyfy_series_old(ndata, self.sets)
tmpdata = FuzzySet.fuzzyfy_series(ndata, self.sets, method="maximum")
flrs = FLR.generate_recurrent_flrs(tmpdata)
self.generate_flrg(flrs)
def forecast(self, data, **kwargs):
l = 1
ordered_sets = FuzzySet.set_ordered(self.sets)
data = np.array(data)
ndata = self.apply_transformations(data)
@ -85,15 +84,13 @@ class ImprovedWeightedFTS(fts.FTS):
for k in np.arange(0, l):
mv = FuzzySet.fuzzyfy_instance(ndata[k], self.sets)
actual = self.sets[np.argwhere(mv == max(mv))[0, 0]]
actual = FuzzySet.get_maximum_membership_fuzzyset(ndata[k], self.sets, ordered_sets)
if actual.name not in self.flrgs:
ret.append(actual.centroid)
else:
flrg = self.flrgs[actual.name]
mp = flrg.get_midpoints()
mp = flrg.get_midpoints(self.sets)
ret.append(mp.dot(flrg.weights()))

2
pyFTS/models/multivariate/FLR.py
View File

@ -19,7 +19,7 @@ class FLR(object):
self.RHS = set
def __str__(self):
return str([self.LHS[k].name for k in self.LHS.keys()]) + " -> " + self.RHS.name
return str([self.LHS[k] for k in self.LHS.keys()]) + " -> " + self.RHS

19
pyFTS/models/multivariate/flrg.py
View File

@ -12,31 +12,20 @@ class FLRG(flg.FLRG):
super(FLRG,self).__init__(0,**kwargs)
self.LHS = kwargs.get('lhs', {})
self.RHS = set()
self.key = None
def set_lhs(self, var, set):
self.LHS[var] = set
def append_rhs(self, set):
def append_rhs(self, set, **kwargs):
self.RHS.add(set)
def get_key(self):
if self.key is None:
_str = ""
for k in self.LHS.keys():
_str += "," if len(_str) > 0 else ""
_str += self.LHS[k].name
self.key = _str
return self.key
def get_membership(self, data):
return np.nanmin([self.LHS[k].membership(data[k]) for k in self.LHS.keys()])
def get_membership(self, data, sets):
return np.nanmin([sets[self.LHS[k]].membership(data[k]) for k in self.LHS.keys()])
def __str__(self):
_str = ""
for k in self.RHS:
_str += "," if len(_str) > 0 else ""
_str += k.name
_str += k
return self.get_key() + " -> " + _str

10
pyFTS/models/multivariate/mvfts.py
View File

@ -51,7 +51,7 @@ class MVFTS(fts.FTS):
flr = MVFLR.FLR()
for c, e in enumerate(path, start=0):
flr.set_lhs(e.variable, e)
flr.set_lhs(e.variable, e.name)
flrs.append(flr)
@ -71,7 +71,7 @@ class MVFTS(fts.FTS):
for flr in tmp_flrs:
for t in target:
flr.set_rhs(t)
flr.set_rhs(t.name)
flrs.append(flr)
return flrs
@ -122,6 +122,12 @@ class MVFTS(fts.FTS):
params=data[self.target_variable.data_label].values)
return ret
def clone_parameters(self, model):
super(MVFTS, self).clone_parameters(model)
self.explanatory_variables = model.explanatory_variables
self.target_variable = model.target_variable
def __str__(self):
_str = self.name + ":\n"
for k in self.flrgs.keys():

4
pyFTS/models/nonstationary/cvfts.py
View File

@ -38,10 +38,10 @@ class ConditionalVarianceFTS(chen.ConventionalFTS):
def generate_flrg(self, flrs, **kwargs):
for flr in flrs:
if flr.LHS.name in self.flrgs:
self.flrgs[flr.LHS.name].append(flr.RHS)
self.flrgs[flr.LHS.name].append_rhs(flr.RHS)
else:
self.flrgs[flr.LHS.name] = nsfts.ConventionalNonStationaryFLRG(flr.LHS)
self.flrgs[flr.LHS.name].append(flr.RHS)
self.flrgs[flr.LHS.name].append_rhs(flr.RHS)
def _smooth(self, a):
return .1 * a[0] + .3 * a[1] + .6 * a[2]

3
pyFTS/models/nonstationary/flrg.py
View File

@ -10,6 +10,9 @@ class NonStationaryFLRG(flrg.FLRG):
self.LHS = LHS
self.RHS = set()
def get_key(self):
return self.LHS.name
def get_membership(self, data, t, window_size=1):
ret = 0.0
if isinstance(self.LHS, (list, set)):

49
pyFTS/models/nonstationary/honsfts.py
View File

@ -11,21 +11,12 @@ class HighOrderNonStationaryFLRG(flrg.NonStationaryFLRG):
self.LHS = []
self.RHS = {}
self.strlhs = ""
def appendRHS(self, c):
def append_rhs(self, c, **kwargs):
if c.name not in self.RHS:
self.RHS[c.name] = c
def strLHS(self):
if len(self.strlhs) == 0:
for c in self.LHS:
if len(self.strlhs) > 0:
self.strlhs += ", "
self.strlhs = self.strlhs + c.name
return self.strlhs
def appendLHS(self, c):
def append_lhs(self, c):
self.LHS.append(c)
def __str__(self):
@ -34,7 +25,7 @@ class HighOrderNonStationaryFLRG(flrg.NonStationaryFLRG):
if len(tmp) > 0:
tmp = tmp + ","
tmp = tmp + c
return self.strLHS() + " -> " + tmp
return self.get_key() + " -> " + tmp
class HighOrderNonStationaryFTS(hofts.HighOrderFTS):
@ -81,13 +72,13 @@ class HighOrderNonStationaryFTS(hofts.HighOrderFTS):
path = list(reversed(list(filter(None.__ne__, p))))
for c, e in enumerate(path, start=0):
flrg.appendLHS(e)
flrg.append_lhs(e)
if flrg.strLHS() not in self.flrgs:
self.flrgs[flrg.strLHS()] = flrg;
if flrg.get_key() not in self.flrgs:
self.flrgs[flrg.get_key()] = flrg;
for st in rhs:
self.flrgs[flrg.strLHS()].append_rhs(st)
self.flrgs[flrg.get_key()].append_rhs(st)
# flrgs = sorted(flrgs, key=lambda flrg: flrg.get_midpoint(0, window_size=1))
@ -135,12 +126,12 @@ class HighOrderNonStationaryFTS(hofts.HighOrderFTS):
flrg = HighOrderNonStationaryFLRG(self.order)
for kk in path:
flrg.appendLHS(self.sets[kk])
flrg.append_lhs(self.sets[kk])
affected_flrgs.append(flrg)
# affected_flrgs_memberships.append(flrg.get_membership(sample, disp))
# affected_flrgs_memberships.append_rhs(flrg.get_membership(sample, disp))
# print(flrg.str_lhs())
# print(flrg.get_key())
# the FLRG is here because of the bounds verification
mv = []
@ -192,14 +183,14 @@ class HighOrderNonStationaryFTS(hofts.HighOrderFTS):
tmp.append(common.check_bounds(sample[-1], self.sets, tdisp))
elif len(affected_flrgs) == 1:
flrg = affected_flrgs[0]
if flrg.str_lhs() in self.flrgs:
tmp.append(self.flrgs[flrg.str_lhs()].get_midpoint(tdisp))
if flrg.get_key() in self.flrgs:
tmp.append(self.flrgs[flrg.get_key()].get_midpoint(tdisp))
else:
tmp.append(flrg.LHS[-1].get_midpoint(tdisp))
else:
for ct, aset in enumerate(affected_flrgs):
if aset.str_lhs() in self.flrgs:
tmp.append(self.flrgs[aset.str_lhs()].get_midpoint(tdisp) *
if aset.get_key() in self.flrgs:
tmp.append(self.flrgs[aset.get_key()].get_midpoint(tdisp) *
affected_flrgs_memberships[ct])
else:
tmp.append(aset.LHS[-1].get_midpoint(tdisp)*
@ -246,18 +237,18 @@ class HighOrderNonStationaryFTS(hofts.HighOrderFTS):
upper.append(aset.get_upper(tdisp))
elif len(affected_flrgs) == 1:
_flrg = affected_flrgs[0]
if _flrg.str_lhs() in self.flrgs:
lower.append(self.flrgs[_flrg.str_lhs()].get_lower(tdisp))
upper.append(self.flrgs[_flrg.str_lhs()].get_upper(tdisp))
if _flrg.get_key() in self.flrgs:
lower.append(self.flrgs[_flrg.get_key()].get_lower(tdisp))
upper.append(self.flrgs[_flrg.get_key()].get_upper(tdisp))
else:
lower.append(_flrg.LHS[-1].get_lower(tdisp))
upper.append(_flrg.LHS[-1].get_upper(tdisp))
else:
for ct, aset in enumerate(affected_flrgs):
if aset.str_lhs() in self.flrgs:
lower.append(self.flrgs[aset.str_lhs()].get_lower(tdisp) *
if aset.get_key() in self.flrgs:
lower.append(self.flrgs[aset.get_key()].get_lower(tdisp) *
affected_flrgs_memberships[ct])
upper.append(self.flrgs[aset.str_lhs()].get_upper(tdisp) *
upper.append(self.flrgs[aset.get_key()].get_upper(tdisp) *
affected_flrgs_memberships[ct])
else:
lower.append(aset.LHS[-1].get_lower(tdisp) *

9
pyFTS/models/nonstationary/nsfts.py
View File

@ -11,7 +11,10 @@ class ConventionalNonStationaryFLRG(flrg.NonStationaryFLRG):
self.LHS = LHS
self.RHS = set()
def append(self, c):
def get_key(self):
return self.LHS.name
def append_rhs(self, c, **kwargs):
self.RHS.add(c)
def __str__(self):
@ -36,10 +39,10 @@ class NonStationaryFTS(fts.FTS):
def generate_flrg(self, flrs, **kwargs):
for flr in flrs:
if flr.LHS.name in self.flrgs:
self.flrgs[flr.LHS.name].append(flr.RHS)
self.flrgs[flr.LHS.name].append_rhs(flr.RHS)
else:
self.flrgs[flr.LHS.name] = ConventionalNonStationaryFLRG(flr.LHS)
self.flrgs[flr.LHS.name].append(flr.RHS)
self.flrgs[flr.LHS.name].append_rhs(flr.RHS)
def train(self, data, **kwargs):

14
pyFTS/models/nonstationary/partitioners.py
View File

@ -15,11 +15,12 @@ class PolynomialNonStationaryPartitioner(partitioner.Partitioner):
prefix=part.prefix, transformation=part.transformation,
indexer=part.indexer)
self.sets = []
self.sets = {}
loc_params, wid_params = self.get_polynomial_perturbations(data, **kwargs)
for ct, set in enumerate(part.sets):
for ct, key in enumerate(part.sets.keys()):
set = part.sets[key]
loc_roots = np.roots(loc_params[ct])[0]
wid_roots = np.roots(wid_params[ct])[0]
tmp = common.FuzzySet(set.name, set.mf, set.parameters,
@ -30,7 +31,7 @@ class PolynomialNonStationaryPartitioner(partitioner.Partitioner):
width_params=wid_params[ct],
width_roots=wid_roots, **kwargs)
self.sets.append(tmp)
self.sets[set.name] = tmp
def poly_width(self, par1, par2, rng, deg):
a = np.polyval(par1, rng)
@ -114,9 +115,10 @@ class ConstantNonStationaryPartitioner(partitioner.Partitioner):
prefix=part.prefix, transformation=part.transformation,
indexer=part.indexer)
self.sets = []
self.sets = {}
for set in part.sets:
for key in part.sets.keys():
set = part.sets[key]
tmp = common.FuzzySet(set.name, set.mf, set.parameters, **kwargs)
self.sets.append(tmp)
self.sets[key] =tmp

332
pyFTS/models/pwfts.py
View File

@ -19,79 +19,74 @@ class ProbabilisticWeightedFLRG(hofts.HighOrderFLRG):
self.frequency_count = 0.0
self.Z = None
def append_rhs(self, c):
self.frequency_count += 1.0
if c.name in self.RHS:
self.rhs_count[c.name] += 1.0
else:
self.RHS[c.name] = c
self.rhs_count[c.name] = 1.0
def get_membership(self, data, sets):
return np.nanprod([sets[k].membership(data[k]) for k in self.LHS])
def appendRHSFuzzy(self, c, mv):
def append_rhs(self, c, **kwargs):
mv = kwargs.get('mv', 1.0)
self.frequency_count += mv
if c.name in self.RHS:
self.rhs_count[c.name] += mv
if c in self.RHS:
self.rhs_count[c] += mv
else:
self.RHS[c.name] = c
self.rhs_count[c.name] = mv
self.RHS[c] = c
self.rhs_count[c] = mv
def get_RHSprobability(self, c):
return self.rhs_count[c] / self.frequency_count
def lhs_probability(self, x, norm, uod, nbins):
def lhs_conditional_probability(self, x, sets, norm, uod, nbins):
pk = self.frequency_count / norm
tmp = pk * (self.lhs_membership(x) / self.partition_function(uod, nbins=nbins))
tmp = pk * (self.get_membership(x, sets) / self.partition_function(uod, nbins=nbins))
return tmp
def rhs_unconditional_probability(self, c):
return self.rhs_count[c] / self.frequency_count
def rhs_conditional_probability(self, x, sets, uod, nbins):
total = 0.0
for rhs in self.RHS.keys():
set = self.RHS[rhs]
wi = self.get_RHSprobability(rhs)
for rhs in self.RHS:
set = sets[rhs]
wi = self.rhs_unconditional_probability(rhs)
mv = set.membership(x) / set.partition_function(uod, nbins=nbins)
total += wi * mv
return total
def lhs_membership(self,x):
mv = []
for count, set in enumerate(self.LHS):
mv.append(set.membership(x[count]))
min_mv = np.min(mv)
return min_mv
def partition_function(self, uod, nbins=100):
def partition_function(self, sets, uod, nbins=100):
if self.Z is None:
self.Z = 0.0
for k in np.linspace(uod[0], uod[1], nbins):
for set in self.LHS:
self.Z += set.membership(k)
for key in self.LHS:
self.Z += sets[key].membership(k)
return self.Z
def get_midpoint(self):
def get_midpoint(self, sets):
'''Return the expectation of the PWFLRG, the weighted sum'''
return sum(np.array([self.get_RHSprobability(s) * self.RHS[s].centroid
for s in self.RHS.keys()]))
if self.midpoint is None:
self.midpoint = np.sum(np.array([self.rhs_unconditional_probability(s) * sets[s].centroid
for s in self.RHS]))
def get_upper(self):
return sum(np.array([self.get_RHSprobability(s) * self.RHS[s].upper
for s in self.RHS.keys()]))
return self.midpoint
def get_lower(self):
return sum(np.array([self.get_RHSprobability(s) * self.RHS[s].lower
for s in self.RHS.keys()]))
def get_upper(self, sets):
if self.upper is None:
self.upper = np.sum(np.array([self.rhs_unconditional_probability(s) * sets[s].upper for s in self.RHS]))
return self.upper
def get_lower(self, sets):
if self.lower is None:
self.lower = np.sum(np.array([self.rhs_unconditional_probability(s) * sets[s].lower for s in self.RHS]))
return self.lower
def __str__(self):
tmp2 = ""
for c in sorted(self.RHS.keys()):
for c in sorted(self.RHS):
if len(tmp2) > 0:
tmp2 = tmp2 + ", "
tmp2 = tmp2 + "(" + str(round(self.rhs_count[c] / self.frequency_count, 3)) + ")" + c
return self.str_lhs() + " -> " + tmp2
return self.get_key() + " -> " + tmp2
class ProbabilisticWeightedFTS(ifts.IntervalFTS):
@ -132,6 +127,31 @@ class ProbabilisticWeightedFTS(ifts.IntervalFTS):
else:
self.generate_flrg(data)
def generate_lhs_flrg(self, sample):
lags = {}
flrgs = []
for o in np.arange(0, self.order):
lhs = [key for key in self.partitioner.ordered_sets if self.sets[key].membership(sample[o]) > 0.0]
lags[o] = lhs
root = tree.FLRGTreeNode(None)
tree.build_tree_without_order(root, lags, 0)
# Trace the possible paths
for p in root.paths():
flrg = ProbabilisticWeightedFLRG(self.order)
path = list(reversed(list(filter(None.__ne__, p))))
for lhs in path:
flrg.append_lhs(lhs)
flrgs.append(flrg)
return flrgs
def generate_flrg(self, data):
l = len(data)
for k in np.arange(self.order, l):
@ -139,111 +159,69 @@ class ProbabilisticWeightedFTS(ifts.IntervalFTS):
sample = data[k - self.order: k]
mvs = FuzzySet.fuzzyfy_instances(sample, self.sets)
lags = {}
flrgs = self.generate_lhs_flrg(sample)
mv = FuzzySet.fuzzyfy_instance(data[k], self.sets)
tmp = np.argwhere(mv)
idx = np.ravel(tmp) # flatten the array
for flrg in flrgs:
for o in np.arange(0, self.order):
_sets = [self.sets[kk] for kk in np.arange(0, len(self.sets)) if mvs[o][kk] > 0]
lhs_mv = flrg.get_membership(sample, self.sets)
lags[o] = _sets
if flrg.get_key() not in self.flrgs:
self.flrgs[flrg.get_key()] = flrg;
root = tree.FLRGTreeNode(None)
fuzzyfied = [(s, self.sets[s].membership(data[k]))
for s in self.sets.keys() if self.sets[s].membership(data[k]) > 0]
self.build_tree_without_order(root, lags, 0)
mvs = []
for set, mv in fuzzyfied:
self.flrgs[flrg.get_key()].append_rhs(set, mv=lhs_mv * mv)
mvs.append(mv)
# Trace the possible paths
for p in root.paths():
flrg = ProbabilisticWeightedFLRG(self.order)
path = list(reversed(list(filter(None.__ne__, p))))
tmp_fq = sum([lhs_mv*kk for kk in mvs if kk > 0])
tmp_path = []
for c, e in enumerate(path, start=0):
tmp_path.append( e.membership( sample[c] ) )
flrg.append_lhs(e)
self.global_frequency_count += tmp_fq
lhs_mv = np.prod(tmp_path)
if flrg.str_lhs() not in self.flrgs:
self.flrgs[flrg.str_lhs()] = flrg;
for st in idx:
self.flrgs[flrg.str_lhs()].appendRHSFuzzy(self.sets[st], lhs_mv * mv[st])
tmp_fq = sum([lhs_mv*kk for kk in mv if kk > 0])
self.global_frequency_count = self.global_frequency_count + tmp_fq
def generateFLRG(self, flrs):
l = len(flrs)
for k in np.arange(self.order, l+1):
if self.dump: print("FLR: " + str(k))
flrg = ProbabilisticWeightedFLRG(self.order)
for kk in np.arange(k - self.order, k):
flrg.append_lhs(flrs[kk].LHS)
if self.dump: print("LHS: " + str(flrs[kk]))
if flrg.str_lhs() in self.flrgs:
self.flrgs[flrg.str_lhs()].append_rhs(flrs[k - 1].RHS)
else:
self.flrgs[flrg.str_lhs()] = flrg
self.flrgs[flrg.str_lhs()].append_rhs(flrs[k - 1].RHS)
if self.dump: print("RHS: " + str(flrs[k-1]))
self.global_frequency_count += 1
def update_model(self,data):
pass
fzzy = FuzzySet.fuzzyfy_series_old(data, self.sets)
flrg = ProbabilisticWeightedFLRG(self.order)
for k in np.arange(0, self.order): flrg.append_lhs(fzzy[k])
if flrg.str_lhs() in self.flrgs:
self.flrgs[flrg.str_lhs()].append_rhs(fzzy[self.order])
else:
self.flrgs[flrg.str_lhs()] = flrg
self.flrgs[flrg.str_lhs()].append_rhs(fzzy[self.order])
self.global_frequency_count += 1
def add_new_PWFLGR(self, flrg):
if flrg.str_lhs() not in self.flrgs:
if flrg.get_key() not in self.flrgs:
tmp = ProbabilisticWeightedFLRG(self.order)
for fs in flrg.LHS: tmp.append_lhs(fs)
tmp.append_rhs(flrg.LHS[-1])
self.flrgs[tmp.str_lhs()] = tmp;
self.flrgs[tmp.get_key()] = tmp;
self.global_frequency_count += 1
def get_flrg_global_probability(self, flrg):
if flrg.str_lhs() in self.flrgs:
return self.flrgs[flrg.str_lhs()].frequency_count / self.global_frequency_count
def flrg_lhs_unconditional_probability(self, flrg):
if flrg.get_key() in self.flrgs:
return self.flrgs[flrg.get_key()].frequency_count / self.global_frequency_count
else:
self.add_new_PWFLGR(flrg)
return self.get_flrg_global_probability(flrg)
return self.flrg_lhs_unconditional_probability(flrg)
def flrg_lhs_conditional_probability(self, x, flrg):
mv = flrg.get_membership(x, self.sets)
pb = self.flrg_lhs_unconditional_probability(flrg)
return mv * pb
def get_midpoint(self, flrg):
if flrg.str_lhs() in self.flrgs:
tmp = self.flrgs[flrg.str_lhs()]
ret = tmp.get_midpoint() #sum(np.array([tmp.get_RHSprobability(s) * self.setsDict[s].centroid for s in tmp.RHS]))
if flrg.get_key() in self.flrgs:
tmp = self.flrgs[flrg.get_key()]
ret = tmp.get_midpoint(self.sets) #sum(np.array([tmp.rhs_unconditional_probability(s) * self.setsDict[s].centroid for s in tmp.RHS]))
else:
pi = 1 / len(flrg.LHS)
ret = sum(np.array([pi * s.centroid for s in flrg.LHS]))
ret = sum(np.array([pi * self.sets[s].centroid for s in flrg.LHS]))
return ret
def get_conditional_probability(self, x, flrg):
def flrg_rhs_conditional_probability(self, x, flrg):
if flrg.str_lhs() in self.flrgs:
_flrg = self.flrgs[flrg.str_lhs()]
if flrg.get_key() in self.flrgs:
_flrg = self.flrgs[flrg.get_key()]
cond = []
for s in _flrg.RHS:
_set = self.setsDict[s]
tmp = _flrg.get_RHSprobability(s) * (_set.membership(x) / _set.partition_function(uod=self.get_UoD()))
for s in _flrg.RHS.keys():
_set = self.sets[s]
tmp = _flrg.rhs_unconditional_probability(s) * (_set.membership(x) / _set.partition_function(uod=self.get_UoD()))
cond.append(tmp)
ret = sum(np.array(cond))
else:
@ -255,21 +233,21 @@ class ProbabilisticWeightedFTS(ifts.IntervalFTS):
return ret
def get_upper(self, flrg):
if flrg.str_lhs() in self.flrgs:
tmp = self.flrgs[flrg.str_lhs()]
ret = tmp.get_upper()
if flrg.get_key() in self.flrgs:
tmp = self.flrgs[flrg.get_key()]
ret = tmp.get_upper(self.sets)
else:
pi = 1 / len(flrg.LHS)
ret = sum(np.array([pi * s.upper for s in flrg.LHS]))
ret = sum(np.array([pi * self.sets[s].upper for s in flrg.LHS]))
return ret
def get_lower(self, flrg):
if flrg.str_lhs() in self.flrgs:
tmp = self.flrgs[flrg.str_lhs()]
ret = tmp.get_lower()
if flrg.get_key() in self.flrgs:
tmp = self.flrgs[flrg.get_key()]
ret = tmp.get_lower(self.sets)
else:
pi = 1 / len(flrg.LHS)
ret = sum(np.array([pi * s.lower for s in flrg.LHS]))
ret = sum(np.array([pi * self.sets[s].lower for s in flrg.LHS]))
return ret
def forecast(self, data, **kwargs):
@ -282,84 +260,20 @@ class ProbabilisticWeightedFTS(ifts.IntervalFTS):
for k in np.arange(self.order - 1, l):
# print(k)
sample = ndata[k - (self.order - 1): k + 1]
affected_flrgs = []
affected_flrgs_memberships = []
norms = []
flrgs = self.generate_lhs_flrg(sample)
mp = []
# Find the sets which membership > 0 for each lag
lags = {}
if self.order > 1:
subset = ndata[k - (self.order - 1): k + 1]
for count, instance in enumerate(subset):
mb = FuzzySet.fuzzyfy_instance(instance, self.sets)
tmp = np.argwhere(mb)
idx = np.ravel(tmp) # flatten the array
if idx.size == 0: # the element is out of the bounds of the Universe of Discourse
if instance <= self.sets[0].lower:
idx = [0]
elif instance >= self.sets[-1].upper:
idx = [len(self.sets) - 1]
else:
raise Exception(instance)
lags[count] = idx
# Build the tree with all possible paths
root = tree.FLRGTreeNode(None)
self.build_tree(root, lags, 0)
# Trace the possible paths and build the PFLRG's
for p in root.paths():
path = list(reversed(list(filter(None.__ne__, p))))
flrg = hofts.HighOrderFLRG(self.order)
for kk in path: flrg.append_lhs(self.sets[kk])
assert len(flrg.LHS) == subset.size, str(subset) + " -> " + str([s.name for s in flrg.LHS])
##
affected_flrgs.append(flrg)
# Find the general membership of FLRG
affected_flrgs_memberships.append(flrg.get_membership(subset))
else:
mv = FuzzySet.fuzzyfy_instance(ndata[k], self.sets) # get all membership values
tmp = np.argwhere(mv) # get the indices of values > 0
idx = np.ravel(tmp) # flatten the array
if idx.size == 0: # the element is out of the bounds of the Universe of Discourse
if ndata[k] <= self.sets[0].lower:
idx = [0]
elif ndata[k] >= self.sets[-1].upper:
idx = [len(self.sets) - 1]
else:
raise Exception(ndata[k])
for kk in idx:
flrg = hofts.HighOrderFLRG(self.order)
flrg.append_lhs(self.sets[kk])
affected_flrgs.append(flrg)
affected_flrgs_memberships.append(mv[kk])
for count, flrg in enumerate(affected_flrgs):
# achar o os bounds de cada FLRG, ponderados pela probabilidade e pertinência
norm = self.get_flrg_global_probability(flrg) * affected_flrgs_memberships[count]
norms = []
for flrg in flrgs:
norm = self.flrg_lhs_conditional_probability(sample, flrg)
if norm == 0:
norm = self.get_flrg_global_probability(flrg) # * 0.001
norm = self.flrg_lhs_unconditional_probability(flrg) # * 0.001
mp.append(norm * self.get_midpoint(flrg))
norms.append(norm)
# gerar o intervalo
# gerar o intervalo
norm = sum(norms)
if norm == 0:
ret.append(0)
@ -401,7 +315,7 @@ class ProbabilisticWeightedFTS(ifts.IntervalFTS):
dist = self.forecast_distribution(ndata)
lo_qt = dist[0].quantile(self.alpha)
up_qt = dist[0].quantile(1.0 - self.alpha)
ret.append([lo_qt, up_qt])
ret.append_rhs([lo_qt, up_qt])
def interval_extremum(self, k, ndata, ret):
affected_flrgs = []
@ -475,9 +389,9 @@ class ProbabilisticWeightedFTS(ifts.IntervalFTS):
affected_flrgs_memberships.append(mv[kk])
for count, flrg in enumerate(affected_flrgs):
# achar o os bounds de cada FLRG, ponderados pela probabilidade e pertinência
norm = self.get_flrg_global_probability(flrg) * affected_flrgs_memberships[count]
norm = self.flrg_lhs_unconditional_probability(flrg) * affected_flrgs_memberships[count]
if norm == 0:
norm = self.get_flrg_global_probability(flrg) # * 0.001
norm = self.flrg_lhs_unconditional_probability(flrg) # * 0.001
up.append(norm * self.get_upper(flrg))
lo.append(norm * self.get_lower(flrg))
norms.append(norm)
@ -485,11 +399,11 @@ class ProbabilisticWeightedFTS(ifts.IntervalFTS):
# gerar o intervalo
norm = sum(norms)
if norm == 0:
ret.append([0, 0])
ret.append_rhs([0, 0])
else:
lo_ = sum(lo) / norm
up_ = sum(up) / norm
ret.append([lo_, up_])
ret.append_rhs([lo_, up_])
def forecast_distribution(self, data, **kwargs):
@ -505,20 +419,22 @@ class ProbabilisticWeightedFTS(ifts.IntervalFTS):
ret = []
uod = self.get_UoD()
_keys = sorted(self.flrgs.keys())
_bins = np.linspace(uod[0], uod[1], nbins)
for k in np.arange(self.order - 1, l):
sample = ndata[k - (self.order - 1): k + 1]
flrgs = self.generate_lhs_flrg(sample)
dist = ProbabilityDistribution.ProbabilityDistribution(smooth, uod=uod, bins=_bins, **kwargs)
for bin in _bins:
num = []
den = []
for s in _keys:
flrg = self.flrgs[s]
pk = flrg.lhs_probability(sample, self.global_frequency_count, uod, nbins)
wi = flrg.rhs_conditional_probability(bin, self.setsDict, uod, nbins)
for s in flrgs:
flrg = self.flrgs[s.get_key()]
pk = flrg.lhs_conditional_probability(sample, self.global_frequency_count, uod, nbins)
wi = flrg.rhs_conditional_probability(bin, self.sets, uod, nbins)
num.append(wi * pk)
den.append(pk)
pf = sum(num) / sum(den)

26
pyFTS/models/sadaei.py
View File

@ -21,7 +21,7 @@ class ExponentialyWeightedFLRG(flrg.FLRG):
self.c = kwargs.get("c",default_c)
self.w = None
def append(self, c):
def append_rhs(self, c, **kwargs):
self.RHS.append(c)
self.count = self.count + 1.0
@ -33,15 +33,15 @@ class ExponentialyWeightedFLRG(flrg.FLRG):
return self.w
def __str__(self):
tmp = self.LHS.name + " -> "
tmp = self.LHS + " -> "
tmp2 = ""
cc = 0
wei = [self.c ** k for k in np.arange(0.0, self.count, 1.0)]
tot = sum(wei)
for c in sorted(self.RHS, key=lambda s: s.name):
for c in sorted(self.RHS):
if len(tmp2) > 0:
tmp2 = tmp2 + ","
tmp2 = tmp2 + c.name + "(" + str(wei[cc] / tot) + ")"
tmp2 = tmp2 + c + "(" + str(wei[cc] / tot) + ")"
cc = cc + 1
return tmp + tmp2
@ -59,24 +59,26 @@ class ExponentialyWeightedFTS(fts.FTS):
def generate_flrg(self, flrs, c):
for flr in flrs:
if flr.LHS.name in self.flrgs:
self.flrgs[flr.LHS.name].append(flr.RHS)
if flr.LHS in self.flrgs:
self.flrgs[flr.LHS].append_rhs(flr.RHS)
else:
self.flrgs[flr.LHS.name] = ExponentialyWeightedFLRG(flr.LHS, c=c);
self.flrgs[flr.LHS.name].append(flr.RHS)
self.flrgs[flr.LHS] = ExponentialyWeightedFLRG(flr.LHS, c=c);
self.flrgs[flr.LHS].append_rhs(flr.RHS)
def train(self, data, **kwargs):
self.c = kwargs.get('parameters', default_c)
if kwargs.get('sets', None) is not None:
self.sets = kwargs.get('sets', None)
ndata = self.apply_transformations(data)
tmpdata = FuzzySet.fuzzyfy_series_old(ndata, self.sets)
tmpdata = FuzzySet.fuzzyfy_series(ndata, self.sets, method='maximum')
flrs = FLR.generate_recurrent_flrs(tmpdata)
self.generate_flrg(flrs, self.c)
def forecast(self, data, **kwargs):
l = 1
ordered_sets = FuzzySet.set_ordered(self.sets)
data = np.array(data)
ndata = self.apply_transformations(data)
@ -87,15 +89,13 @@ class ExponentialyWeightedFTS(fts.FTS):
for k in np.arange(0, l):
mv = FuzzySet.fuzzyfy_instance(ndata[k], self.sets)
actual = self.sets[np.argwhere(mv == max(mv))[0, 0]]
actual = FuzzySet.get_maximum_membership_fuzzyset(ndata[k], self.sets, ordered_sets)
if actual.name not in self.flrgs:
ret.append(actual.centroid)
else:
flrg = self.flrgs[actual.name]
mp = flrg.get_midpoints()
mp = flrg.get_midpoints(self.sets)
ret.append(mp.dot(flrg.weights()))

33
pyFTS/models/seasonal/cmsfts.py
View File

@ -4,26 +4,26 @@ from pyFTS.models.seasonal import sfts
from pyFTS.models import chen
class ContextualSeasonalFLRG(object):
class ContextualSeasonalFLRG(sfts.SeasonalFLRG):
"""
Contextual Seasonal Fuzzy Logical Relationship Group
"""
def __init__(self, seasonality):
self.season = seasonality
self.flrgs = {}
super(ContextualSeasonalFLRG, self).__init__(seasonality)
self.RHS = {}
def append(self, flr):
if flr.LHS.name in self.flrgs:
self.flrgs[flr.LHS.name].append(flr.RHS)
def append_rhs(self, flr, **kwargs):
if flr.LHS in self.RHS:
self.RHS[flr.LHS].append_rhs(flr.RHS)
else:
self.flrgs[flr.LHS.name] = chen.ConventionalFLRG(flr.LHS)
self.flrgs[flr.LHS.name].append(flr.RHS)
self.RHS[flr.LHS] = chen.ConventionalFLRG(flr.LHS)
self.RHS[flr.LHS].append_rhs(flr.RHS)
def __str__(self):
tmp = str(self.season) + ": \n "
tmp = str(self.LHS) + ": \n "
tmp2 = "\t"
for r in sorted(self.flrgs):
tmp2 += str(self.flrgs[r]) + "\n\t"
for r in sorted(self.RHS):
tmp2 += str(self.RHS[r]) + "\n\t"
return tmp + tmp2 + "\n"
@ -50,7 +50,7 @@ class ContextualMultiSeasonalFTS(sfts.SeasonalFTS):
if str(flr.index) not in self.flrgs:
self.flrgs[str(flr.index)] = ContextualSeasonalFLRG(flr.index)
self.flrgs[str(flr.index)].append(flr)
self.flrgs[str(flr.index)].append_rhs(flr)
def train(self, data, **kwargs):
if kwargs.get('sets', None) is not None:
@ -61,13 +61,14 @@ class ContextualMultiSeasonalFTS(sfts.SeasonalFTS):
self.generate_flrg(flrs)
def get_midpoints(self, flrg, data):
if data.name in flrg.flrgs:
ret = np.array([s.centroid for s in flrg.flrgs[data.name].RHS])
if data in flrg.flrgs:
ret = np.array([self.sets[s].centroid for s in flrg.flrgs[data.name].RHS])
return ret
else:
return np.array([data.centroid])
return np.array([self.sets[data].centroid])
def forecast(self, data, **kwargs):
ordered_sets = FuzzySet.set_ordered(self.sets)
ret = []
@ -78,7 +79,7 @@ class ContextualMultiSeasonalFTS(sfts.SeasonalFTS):
flrg = self.flrgs[str(index[k])]
d = FuzzySet.get_maximum_membership_fuzzyset(ndata[k], self.sets)
d = FuzzySet.get_maximum_membership_fuzzyset(ndata[k], self.sets, ordered_sets)
mp = self.get_midpoints(flrg, d)

2
pyFTS/models/seasonal/msfts.py
View File

@ -26,7 +26,7 @@ class MultiSeasonalFTS(sfts.SeasonalFTS):
if str(flr.index) not in self.flrgs:
self.flrgs[str(flr.index)] = sfts.SeasonalFLRG(flr.index)
self.flrgs[str(flr.index)].append(flr.RHS)
self.flrgs[str(flr.index)].append_rhs(flr.RHS)
def train(self, data, **kwargs):
if kwargs.get('sets', None) is not None:

21
pyFTS/models/seasonal/partitioner.py
View File

@ -33,7 +33,7 @@ class TimeGridPartitioner(partitioner.Partitioner):
self.sets = self.build(None)
def build(self, data):
sets = []
sets = {}
kwargs = {'variable': self.variable}
@ -46,7 +46,7 @@ class TimeGridPartitioner(partitioner.Partitioner):
count = 0
for c in np.arange(self.min, self.max, partlen):
set_name = self.prefix + str(count) if self.setnames is None else self.setnames[count]
set_name = self.get_name(count)
if self.membership_function == Membership.trimf:
if c == self.min:
tmp = Composite(set_name, superset=True)
@ -58,14 +58,14 @@ class TimeGridPartitioner(partitioner.Partitioner):
[c - partlen, c, c + partlen], c,
**kwargs))
tmp.centroid = c
sets.append(tmp)
sets[set_name] = tmp
else:
sets.append(FuzzySet(self.season, set_name, Membership.trimf,
sets[set_name] = FuzzySet(self.season, set_name, Membership.trimf,
[c - partlen, c, c + partlen], c,
**kwargs))
**kwargs)
elif self.membership_function == Membership.gaussmf:
sets.append(FuzzySet(self.season, set_name, Membership.gaussmf, [c, partlen / 3], c,
**kwargs))
sets[set_name] = FuzzySet(self.season, set_name, Membership.gaussmf, [c, partlen / 3], c,
**kwargs)
elif self.membership_function == Membership.trapmf:
q = partlen / 4
if c == self.min:
@ -78,17 +78,18 @@ class TimeGridPartitioner(partitioner.Partitioner):
[c - partlen, c - q, c + q, c + partlen], c,
**kwargs))
tmp.centroid = c
sets.append(tmp)
sets[set_name] = tmp
else:
sets.append(FuzzySet(self.season, set_name, Membership.trapmf,
sets[set_name] = FuzzySet(self.season, set_name, Membership.trapmf,
[c - partlen, c - q, c + q, c + partlen], c,
**kwargs))
**kwargs)
count += 1
self.min = 0
return sets
def plot(self, ax):
"""
Plot the

9
pyFTS/models/seasonal/sfts.py
View File

@ -10,14 +10,17 @@ import numpy as np
from pyFTS.common import FuzzySet, FLR, fts
class SeasonalFLRG(FLR.FLR):
class SeasonalFLRG(FLR.FLRG):
"""First Order Seasonal Fuzzy Logical Relationship Group"""
def __init__(self, seasonality):
super(SeasonalFLRG, self).__init__(None,None)
self.LHS = seasonality
self.RHS = []
def append(self, c):
def get_key(self):
return self.LHS
def append_rhs(self, c, **kwargs):
self.RHS.append(c)
def __str__(self):
@ -57,7 +60,7 @@ class SeasonalFTS(fts.FTS):
self.flrgs[ss] = SeasonalFLRG(season)
#print(season)
self.flrgs[ss].append(flr.RHS)
self.flrgs[ss].append_rhs(flr.RHS)
def train(self, data, **kwargs):
if kwargs.get('sets', None) is not None:

36
pyFTS/models/song.py
View File

@ -20,40 +20,46 @@ class ConventionalFTS(fts.FTS):
self.R = None
if self.sets is not None:
self.R = np.zeros((len(self.sets),len(self.sets)))
l = len(self.sets)
self.R = np.zeros((l,l))
def flr_membership_matrix(self, flr):
lm = [flr.LHS.membership(k.centroid) for k in self.sets]
rm = [flr.RHS.membership(k.centroid) for k in self.sets]
ordered_set = FuzzySet.set_ordered(self.sets)
centroids = [self.sets[k].centroid for k in ordered_set]
lm = [self.sets[flr.LHS].membership(k) for k in centroids]
rm = [self.sets[flr.RHS].membership(k) for k in centroids]
r = np.zeros((len(self.sets), len(self.sets)))
for k in range(0,len(self.sets)):
for l in range(0, len(self.sets)):
r[k][l] = min(lm[k],rm[l])
l = len(ordered_set)
r = np.zeros((l, l))
for k in range(0,l):
for l in range(0, l):
r[k][l] = min(lm[k], rm[l])
return r
def operation_matrix(self, flrs):
l = len(self.sets)
if self.R is None:
self.R = np.zeros((len(self.sets), len(self.sets)))
self.R = np.zeros((l, l))
for k in flrs:
mm = self.flr_membership_matrix(k)
for k in range(0, len(self.sets)):
for l in range(0, len(self.sets)):
self.R[k][l] = max(r[k][l], mm[k][l])
for k in range(0, l):
for l in range(0, l):
self.R[k][l] = max(self.R[k][l], mm[k][l])
def train(self, data, **kwargs):
if kwargs.get('sets', None) is not None:
self.sets = kwargs.get('sets', None)
ndata = self.apply_transformations(data)
tmpdata = FuzzySet.fuzzyfy_series_old(ndata, self.sets)
tmpdata = FuzzySet.fuzzyfy_series(ndata, self.sets, method='maximum')
flrs = FLR.generate_non_recurrent_flrs(tmpdata)
self.operation_matrix(flrs)
def forecast(self, data, **kwargs):
ordered_set = FuzzySet.set_ordered(self.sets)
ndata = np.array(self.apply_transformations(data))
l = len(ndata)
@ -69,9 +75,9 @@ class ConventionalFTS(fts.FTS):
fs = np.ravel(np.argwhere(r == max(r)))
if len(fs) == 1:
ret.append(self.sets[fs[0]].centroid)
ret.append(self.sets[ordered_set[fs[0]]].centroid)
else:
mp = [self.sets[s].centroid for s in fs]
mp = [self.sets[ordered_set[s]].centroid for s in fs]
ret.append( sum(mp)/len(mp))

36
pyFTS/models/yu.py
View File

@ -17,24 +17,27 @@ class WeightedFLRG(flrg.FLRG):
self.LHS = LHS
self.RHS = []
self.count = 1.0
self.w = None
def append(self, c):
def append_rhs(self, c, **kwargs):
self.RHS.append(c)
self.count = self.count + 1.0
def weights(self):
tot = sum(np.arange(1.0, self.count, 1.0))
return np.array([k / tot for k in np.arange(1.0, self.count, 1.0)])
def weights(self, sets):
if self.w is None:
tot = sum(np.arange(1.0, self.count, 1.0))
self.w = np.array([k / tot for k in np.arange(1.0, self.count, 1.0)])
return self.w
def __str__(self):
tmp = self.LHS.name + " -> "
tmp = self.LHS + " -> "
tmp2 = ""
cc = 1.0
tot = sum(np.arange(1.0, self.count, 1.0))
for c in sorted(self.RHS, key=lambda s: s.name):
for c in sorted(self.RHS):
if len(tmp2) > 0:
tmp2 = tmp2 + ","
tmp2 = tmp2 + c.name + "(" + str(round(cc / tot, 3)) + ")"
tmp2 = tmp2 + c + "(" + str(round(cc / tot, 3)) + ")"
cc = cc + 1.0
return tmp + tmp2
@ -48,11 +51,11 @@ class WeightedFTS(fts.FTS):
def generate_FLRG(self, flrs):
for flr in flrs:
if flr.LHS.name in self.flrgs:
self.flrgs[flr.LHS.name].append(flr.RHS)
if flr.LHS in self.flrgs:
self.flrgs[flr.LHS].append_rhs(flr.RHS)
else:
self.flrgs[flr.LHS.name] = WeightedFLRG(flr.LHS);
self.flrgs[flr.LHS.name].append(flr.RHS)
self.flrgs[flr.LHS] = WeightedFLRG(flr.LHS);
self.flrgs[flr.LHS].append_rhs(flr.RHS)
def train(self, data, **kwargs):
if kwargs.get('sets', None) is not None:
@ -63,6 +66,9 @@ class WeightedFTS(fts.FTS):
self.generate_FLRG(flrs)
def forecast(self, data, **kwargs):
ordered_sets = FuzzySet.set_ordered(self.sets)
l = 1
data = np.array(data)
@ -75,17 +81,15 @@ class WeightedFTS(fts.FTS):
for k in np.arange(0, l):
mv = FuzzySet.fuzzyfy_instance(ndata[k], self.sets)
actual = self.sets[np.argwhere(mv == max(mv))[0, 0]]
actual = FuzzySet.get_maximum_membership_fuzzyset(ndata[k], self.sets, ordered_sets)
if actual.name not in self.flrgs:
ret.append(actual.centroid)
else:
flrg = self.flrgs[actual.name]
mp = flrg.get_midpoints()
mp = flrg.get_midpoints(self.sets)
ret.append(mp.dot(flrg.weights()))
ret.append(mp.dot(flrg.weights(self.sets)))
ret = self.apply_inverse_transformations(ret, params=[data])

47
pyFTS/notebooks/Chen - ConventionalFTS.ipynb
View File

File diff suppressed because one or more lines are too long

44
pyFTS/notebooks/Cheng - TrendWeightedFTS.ipynb
View File

File diff suppressed because one or more lines are too long

414
pyFTS/notebooks/HighOrderFTS.ipynb
View File

File diff suppressed because one or more lines are too long

143
pyFTS/notebooks/Hwang - HighOrderFTS.ipynb
View File

File diff suppressed because one or more lines are too long

42
pyFTS/notebooks/Ismail & Efendi - ImprovedWeightedFTS.ipynb
View File

File diff suppressed because one or more lines are too long

62
pyFTS/notebooks/Sadaei et Al - ExponentialyWeightedFTS.ipynb
View File

File diff suppressed because one or more lines are too long

188
pyFTS/notebooks/Song - ConventionalFTS.ipynb
View File

File diff suppressed because one or more lines are too long

52
pyFTS/notebooks/Yu - WeightedFTS.ipynb
View File

File diff suppressed because one or more lines are too long

41
pyFTS/notebooks/multivariate.ipynb
View File

File diff suppressed because one or more lines are too long

7
pyFTS/partitioners/CMeans.py
View File

@ -82,15 +82,16 @@ class CMeansPartitioner(partitioner.Partitioner):
super(CMeansPartitioner, self).__init__(name="CMeans", **kwargs)
def build(self, data):
sets = []
sets = {}
centroides = c_means(self.partitions, data, 1)
centroides.append(self.max)
centroides.append(self.min)
centroides = list(set(centroides))
centroides.sort()
for c in np.arange(1, len(centroides) - 1):
sets.append(FuzzySet.FuzzySet(self.prefix + str(c), Membership.trimf,
_name = self.get_name(c)
sets[_name] = FuzzySet.FuzzySet(_name, Membership.trimf,
[round(centroides[c - 1], 3), round(centroides[c], 3), round(centroides[c + 1], 3)],
round(centroides[c], 3)))
round(centroides[c], 3))
return sets

11
pyFTS/partitioners/Entropy.py
View File

@ -83,7 +83,7 @@ class EntropyPartitioner(partitioner.Partitioner):
super(EntropyPartitioner, self).__init__(name="Entropy", **kwargs)
def build(self, data):
sets = []
sets = {}
partitions = bestSplit(data, self.partitions)
partitions.append(self.min)
@ -91,15 +91,16 @@ class EntropyPartitioner(partitioner.Partitioner):
partitions = list(set(partitions))
partitions.sort()
for c in np.arange(1, len(partitions) - 1):
_name = self.get_name(c)
if self.membership_function == Membership.trimf:
sets.append(FuzzySet.FuzzySet(self.prefix + str(c), Membership.trimf,
[partitions[c - 1], partitions[c], partitions[c + 1]],partitions[c]))
sets[_name] = FuzzySet.FuzzySet(_name, Membership.trimf,
[partitions[c - 1], partitions[c], partitions[c + 1]],partitions[c])
elif self.membership_function == Membership.trapmf:
b1 = (partitions[c] - partitions[c - 1])/2
b2 = (partitions[c + 1] - partitions[c]) / 2
sets.append(FuzzySet.FuzzySet(self.prefix + str(c), Membership.trapmf,
sets[_name] = FuzzySet.FuzzySet(_name, Membership.trapmf,
[partitions[c - 1], partitions[c] - b1,
partitions[c] + b2, partitions[c + 1]],
partitions[c]))
partitions[c])
return sets

44
pyFTS/partitioners/FCM.py
View File

@ -1,21 +1,24 @@
import numpy as np
import math
import random as rnd
import functools,operator
from pyFTS.common import FuzzySet,Membership
import functools, operator
from pyFTS.common import FuzzySet, Membership
from pyFTS.partitioners import partitioner
#import CMeans
# import CMeans
# S. T. Li, Y. C. Cheng, and S. Y. Lin, “A FCM-based deterministic forecasting model for fuzzy time series,”
# Comput. Math. Appl., vol. 56, no. 12, pp. 3052–3063, Dec. 2008. DOI: 10.1016/j.camwa.2008.07.033.
def fuzzy_distance(x, y):
if isinstance(x, list):
tmp = functools.reduce(operator.add, [(x[k] - y[k])**2 for k in range(0,len(x))])
tmp = functools.reduce(operator.add, [(x[k] - y[k]) ** 2 for k in range(0, len(x))])
else:
tmp = (x - y) ** 2
return math.sqrt(tmp)
def membership(val, vals):
soma = 0
for k in vals:
@ -30,7 +33,7 @@ def fuzzy_cmeans(k, dados, tam, m, deltadist=0.001):
tam_dados = len(dados)
# Inicializa as centróides escolhendo elementos aleatórios dos conjuntos
centroides = [dados[rnd.randint(0, tam_dados-1)] for kk in range(0, k)]
centroides = [dados[rnd.randint(0, tam_dados - 1)] for kk in range(0, k)]
# Tabela de pertinência das instâncias aos grupos
grupos = [[0 for kk in range(0, k)] for xx in range(0, tam_dados)]
@ -81,7 +84,7 @@ def fuzzy_cmeans(k, dados, tam, m, deltadist=0.001):
oldgrp = [xx for xx in grupo]
for atr in range(0, tam):
soma = functools.reduce(operator.add,
[grupos[xk][grupo_count] * dados[xk][atr] for xk in range(0, tam_dados)])
[grupos[xk][grupo_count] * dados[xk][atr] for xk in range(0, tam_dados)])
norm = functools.reduce(operator.add, [grupos[xk][grupo_count] for xk in range(0, tam_dados)])
centroides[grupo_count][atr] = soma / norm
else:
@ -104,28 +107,31 @@ class FCMPartitioner(partitioner.Partitioner):
"""
"""
def __init__(self, **kwargs):
super(FCMPartitioner, self).__init__(name="FCM", **kwargs)
def build(self,data):
sets = []
def build(self, data):
sets = {}
centroids = fuzzy_cmeans(self.partitions, data, 1, 2)
centroids.append(self.max)
centroids.append(self.min)
centroids = list(set(centroids))
centroids.sort()
for c in np.arange(1,len(centroids)-1):
for c in np.arange(1, len(centroids) - 1):
_name = self.get_name(c)
if self.membership_function == Membership.trimf:
sets.append(FuzzySet.FuzzySet(self.prefix+str(c),Membership.trimf,
[round(centroids[c-1],3), round(centroids[c],3), round(centroids[c+1],3)],
round(centroids[c],3) ) )
sets[_name] = FuzzySet.FuzzySet(_name, Membership.trimf,
[round(centroids[c - 1], 3), round(centroids[c], 3),
round(centroids[c + 1], 3)],
round(centroids[c], 3))
elif self.membership_function == Membership.trapmf:
q1 = (round(centroids[c], 3) - round(centroids[c - 1], 3))/2
q2 = (round(centroids[c+1], 3) - round(centroids[c], 3)) / 2
sets.append(FuzzySet.FuzzySet(self.prefix + str(c), Membership.trimf,
[round(centroids[c - 1], 3), round(centroids[c], 3) - q1,
round(centroids[c], 3) + q2, round(centroids[c + 1], 3)],
round(centroids[c], 3)))
q1 = (round(centroids[c], 3) - round(centroids[c - 1], 3)) / 2
q2 = (round(centroids[c + 1], 3) - round(centroids[c], 3)) / 2
sets[_name] = FuzzySet.FuzzySet(_name, Membership.trimf,
[round(centroids[c - 1], 3), round(centroids[c], 3) - q1,
round(centroids[c], 3) + q2, round(centroids[c + 1], 3)],
round(centroids[c], 3))
return sets
return sets

12
pyFTS/partitioners/Grid.py
View File

@ -21,7 +21,7 @@ class GridPartitioner(partitioner.Partitioner):
super(GridPartitioner, self).__init__(name="Grid", **kwargs)
def build(self, data):
sets = []
sets = {}
kwargs = {'type': self.type, 'variable': self.variable}
@ -30,16 +30,14 @@ class GridPartitioner(partitioner.Partitioner):
count = 0
for c in np.arange(self.min, self.max, partlen):
_name = self.get_name(count)
if self.membership_function == Membership.trimf:
sets.append(
FuzzySet.FuzzySet(self.prefix + str(count), Membership.trimf, [c - partlen, c, c + partlen],c,**kwargs))
sets[_name] = FuzzySet.FuzzySet(_name, Membership.trimf, [c - partlen, c, c + partlen],c,**kwargs)
elif self.membership_function == Membership.gaussmf:
sets.append(
FuzzySet.FuzzySet(self.prefix + str(count), Membership.gaussmf, [c, partlen / 3], c,**kwargs))
sets[_name] = FuzzySet.FuzzySet(_name, Membership.gaussmf, [c, partlen / 3], c,**kwargs)
elif self.membership_function == Membership.trapmf:
q = partlen / 2
sets.append(
FuzzySet.FuzzySet(self.prefix + str(count), Membership.trapmf, [c - partlen, c - q, c + q, c + partlen], c,**kwargs))
sets[_name] = FuzzySet.FuzzySet(_name, Membership.trapmf, [c - partlen, c - q, c + q, c + partlen], c,**kwargs)
count += 1
self.min = self.min - partlen

15
pyFTS/partitioners/Huarng.py
View File

@ -29,22 +29,23 @@ class HuarngPartitioner(partitioner.Partitioner):
else:
base = 100
sets = []
sets = {}
dlen = self.max - self.min
npart = math.ceil(dlen / base)
partition = math.ceil(self.min)
for c in range(npart):
_name = self.get_name(c)
if self.membership_function == Membership.trimf:
sets.append( FuzzySet.FuzzySet(self.prefix + str(c), Membership.trimf,
[partition - base, partition, partition + base], partition))
sets[_name] = FuzzySet.FuzzySet(_name, Membership.trimf,
[partition - base, partition, partition + base], partition)
elif self.membership_function == Membership.gaussmf:
sets.append(FuzzySet.FuzzySet(self.prefix + str(c), Membership.gaussmf,
[partition, base/2], partition))
sets[_name] = FuzzySet.FuzzySet(_name, Membership.gaussmf,
[partition, base/2], partition)
elif self.membership_function == Membership.trapmf:
sets.append(FuzzySet.FuzzySet(self.prefix + str(c), Membership.trapmf,
sets[_name] = FuzzySet.FuzzySet(_name, Membership.trapmf,
[partition - base, partition - (base/2),
partition + (base / 2), partition + base], partition))
partition + (base / 2), partition + base], partition)
partition += base

3
pyFTS/partitioners/Util.py
View File

@ -26,7 +26,8 @@ def plot_sets(data, sets, titles, tam=[12, 10], save=False, file=None):
ax = axes[k]
ax.set_title(titles[k])
ax.set_ylim([0, 1.1])
for s in sets[k]:
for key in sets[k].keys():
s = sets[k][key]
if s.mf == Membership.trimf:
ax.plot(s.parameters,[0,1,0])
elif s.mf == Membership.gaussmf:

30
pyFTS/partitioners/partitioner.py
View File

@ -20,15 +20,16 @@ class Partitioner(object):
:param transformation: data transformation to be applied on data
"""
self.name = kwargs.get('name',"")
self.partitions = kwargs.get('npart',10)
self.sets = []
self.membership_function = kwargs.get('func',Membership.trimf)
self.setnames = kwargs.get('names',None)
self.prefix = kwargs.get('prefix','A')
self.transformation = kwargs.get('transformation',None)
self.indexer = kwargs.get('indexer',None)
self.partitions = kwargs.get('npart', 10)
self.sets = {}
self.membership_function = kwargs.get('func', Membership.trimf)
self.setnames = kwargs.get('names', None)
self.prefix = kwargs.get('prefix', 'A')
self.transformation = kwargs.get('transformation', None)
self.indexer = kwargs.get('indexer', None)
self.variable = kwargs.get('variable', None)
self.type = kwargs.get('type', 'common')
self.ordered_sets = None
if kwargs.get('preprocess',True):
@ -58,6 +59,11 @@ class Partitioner(object):
self.sets = self.build(ndata)
if self.ordered_sets is None and self.setnames is not None:
self.ordered_sets = self.setnames
else:
self.ordered_sets = FuzzySet.set_ordered(self.sets)
del(ndata)
def build(self, data):
@ -68,6 +74,9 @@ class Partitioner(object):
"""
pass
def get_name(self, counter):
return self.prefix + str(counter) if self.setnames is None else self.setnames[counter]
def plot(self, ax):
"""
Plot the
@ -79,7 +88,8 @@ class Partitioner(object):
ax.set_xlim([self.min, self.max])
ticks = []
x = []
for s in self.sets:
for key in self.sets.keys():
s = self.sets[key]
if s.type == 'common':
self.plot_set(ax, s)
elif s.type == 'composite':
@ -103,6 +113,6 @@ class Partitioner(object):
def __str__(self):
tmp = self.name + ":\n"
for a in self.sets:
tmp += str(a)+ "\n"
for key in self.sets.keys():
tmp += str(self.sets[key])+ "\n"
return tmp

2
pyFTS/probabilistic/ProbabilityDistribution.py
View File

@ -107,7 +107,7 @@ class ProbabilityDistribution(object):
if str(ret) not in self.qtl:
self.qtl[str(ret)] = []
self.qtl[str(ret)].append(k)
self.qtl[str(ret)].append_rhs(k)
_keys = [float(k) for k in sorted(self.qtl.keys())]

6
pyFTS/tests/distributed.py
View File

@ -90,9 +90,9 @@ for job in jobs:
results[_m][_o][_p]['mape'] = []
results[_m][_o][_p]['u'] = []
results[_m][_o][_p]['rmse'].append(tmp['rmse'])
results[_m][_o][_p]['mape'].append(tmp['mape'])
results[_m][_o][_p]['u'].append(tmp['u'])
results[_m][_o][_p]['rmse'].append_rhs(tmp['rmse'])
results[_m][_o][_p]['mape'].append_rhs(tmp['mape'])
results[_m][_o][_p]['u'].append_rhs(tmp['u'])
cluster.wait() # wait for all jobs to finish

66
pyFTS/tests/multivariate.py
View File

@ -4,45 +4,49 @@ from pyFTS.data import TAIEX as tx
from pyFTS.common import Transformations
from pyFTS.data import SONDA
df = SONDA.get_dataframe()
train = df.iloc[0:1572480] #three years
test = df.iloc[1572480:2096640] #ears
del df
from pyFTS.partitioners import Grid, Util as pUtil
from pyFTS.common import Transformations
from pyFTS.models.multivariate import common, variable, mvfts
from pyFTS.models.seasonal import partitioner as seasonal
from pyFTS.models.seasonal.common import DateTime
bc = Transformations.BoxCox(0)
diff = Transformations.Differential(1)
df = tx.get_dataframe()
df = df.dropna()
#df.loc[2209]
train = df.iloc[2000:2500]
test = df.iloc[2500:3000]
np = 10
from pyFTS.partitioners import Grid, Util as pUtil
from pyFTS.models.multivariate import common, variable
model = mvfts.MVFTS("")
model = common.MVFTS("")
fig, axes = plt.subplots(nrows=5, ncols=1,figsize=[15,10])
#fig, axes = plt.subplots(nrows=5, ncols=1,figsize=[10,10])
sp = {'seasonality': DateTime.day_of_year , 'names': ['Jan','Feb','Mar','Apr','May','Jun','Jul', 'Aug','Sep','Oct','Nov','Dec']}
vopen = variable.Variable("Open", data_label="Openly", partitioner=Grid.GridPartitioner, npart=40, data=df)
model.append_variable(vopen)
#vopen.partitioner.plot(axes[0])
vhigh = variable.Variable("High", data_label="Highest", partitioner=Grid.GridPartitioner, npart=40, data=df)#train)
model.append_variable(vhigh)
#vhigh.partitioner.plot(axes[1])
vlow = variable.Variable("Low", data_label="Lowermost", partitioner=Grid.GridPartitioner, npart=40, data=df)#train)
model.append_variable(vlow)
#vlow.partitioner.plot(axes[2])
vclose = variable.Variable("Close", data_label="Close", partitioner=Grid.GridPartitioner, npart=40, data=df)#train)
model.append_variable(vclose)
#vclose.partitioner.plot(axes[3])
vvol = variable.Variable("Volume", data_label="Volume", partitioner=Grid.GridPartitioner, npart=100, data=df,
transformation=bc)#train)
model.append_variable(vvol)
#vvol.partitioner.plot(axes[4])
vmonth = variable.Variable("Month", data_label="datahora", partitioner=seasonal.TimeGridPartitioner, npart=12,
data=train, partitioner_specific=sp)
model.append_variable(vmonth)
model.target_variable = vvol
sp = {'seasonality': DateTime.minute_of_day}
#plt.tight_layout()
model.train(train)
vhour = variable.Variable("Hour", data_label="datahora", partitioner=seasonal.TimeGridPartitioner, npart=24,
data=train, partitioner_specific=sp)
model.append_variable(vhour)
forecasted = model.forecast(test)
vhumid = variable.Variable("Humidity", data_label="humid", partitioner=Grid.GridPartitioner, npart=np, data=train)
model.append_variable(vhumid)
print([round(k,0) for k in test['Volume'].values.tolist()])
print([round(k,0) for k in forecasted])
vpress = variable.Variable("AtmPress", data_label="press", partitioner=Grid.GridPartitioner, npart=np, data=train)
model.append_variable(vpress)
vrain = variable.Variable("Rain", data_label="rain", partitioner=Grid.GridPartitioner, npart=20, data=train)#train)
model.append_variable(vrain)
model.target_variable = vrain
model.fit(train, num_batches=20, save=True, batch_save=True, file_path='mvfts_sonda3', distributed=True,
nodes=['192.168.0.110','192.168.0.106'])