import numpy as np


def sigmoid(x):
    """
    S(x) = 1 / ( 1 + e^(-x) )
    :param x: input
    :return: S(x)
    """
    return 1 / (1 + np.exp(-x))


def relu(x):
    """
    f(x) = max(0, x)
    :param x:
    :return:
    """
    return max(0, x)


def tanh(x):
    """
    tanh x = (sinh x)/(cosh x) = (e^x - e(-x)) / (e^x + e(-x))
    :param x:
    :return:
    """
    return (np.exp(x) - np.exp(-x)) / (np.exp(x) + np.exp(-x))


def softmax(inputs):
    """
    y_k = exp(x_k) / sum(exp(x_i))
    :param inputs:
    :return:
    """
    _max = np.max(inputs)
    _exp = np.exp(inputs - _max)  # avoid overflow
    _sum_exp = np.sum(_exp)
    return _exp / _sum_exp
