import torch
from pylissom.math import normalize
from pylissom.nn.functional.weights import apply_circular_mask_to_weights
[docs]def linear_decay(w, start, epoch, final_epoch):
radius = start + epoch * (1.0 - start) / final_epoch
normalize(apply_circular_mask_to_weights(w.data.t_(), radius=radius))
w.data.t_()
return
[docs]def kill_neurons(w, threshold):
return w.masked_fill_(mask=torch.lt(w, threshold), value=0)
# # TODO: test
# def afferent_normalize(radius, strength, afferent_input, activation):
# reshaped_input = afferent_input.data.repeat(afferent_input.data.size()[1], 1)
# masked_input = apply_circular_mask_to_weights(reshaped_input, radius)
# sums = masked_input.sum(1).unsqueeze(1).t()
# den = 1 + strength * sums
# activation = activation / den
# return activation
[docs]def piecewise_sigmoid(min_theta, max_theta, inp):
mask_zeros = torch.le(inp, min_theta)
mask_ones = torch.ge(inp, max_theta)
output = (inp - min_theta).div(max_theta - min_theta)
output.masked_fill_(mask=mask_zeros, value=0)
output.masked_fill_(mask=mask_ones, value=1)
return output
[docs]def check_compatible_mul(module_one, module_two):
r"""Checks that two modules have correct sizes for matrix multiplication"""
if module_one.out_features != module_two.in_features:
raise ValueError(
"Matmul: {}.out_features doesn't match {}.in_features".format(str(module_one), str(module_two)))
[docs]def check_compatible_add(module_one, module_two):
r"""Checks that two modules have correct sizes for matrix addition"""
if module_one.out_features != module_two.out_features:
raise ValueError(
"Add: {}.out_features doesn't match {}.out_features".format(str(module_one), str(module_two)))