102 lines
3.9 KiB
C++
102 lines
3.9 KiB
C++
|
/********************************************************
|
||
|
|
* ██████╗ ██████╗████████╗██╗
|
||
|
|
* ██╔════╝ ██╔════╝╚══██╔══╝██║
|
||
|
|
* ██║ ███╗██║ ██║ ██║
|
||
|
|
* ██║ ██║██║ ██║ ██║
|
||
|
|
* ╚██████╔╝╚██████╗ ██║ ███████╗
|
||
|
|
* ╚═════╝ ╚═════╝ ╚═╝ ╚══════╝
|
||
|
|
* Geophysical Computational Tools & Library (GCTL)
|
||
|
|
*
|
||
|
|
* Copyright (c) 2022 Yi Zhang (yizhang-geo@zju.edu.cn)
|
||
|
|
*
|
||
|
|
* GCTL is distributed under a dual licensing scheme. You can redistribute
|
||
|
|
* it and/or modify it under the terms of the GNU Lesser General Public
|
||
|
|
* License as published by the Free Software Foundation, either version 2
|
||
|
|
* of the License, or (at your option) any later version. You should have
|
||
|
|
* received a copy of the GNU Lesser General Public License along with this
|
||
|
|
* program. If not, see <http://www.gnu.org/licenses/>.
|
||
|
|
*
|
||
|
|
* If the terms and conditions of the LGPL v.2. would prevent you from using
|
||
|
|
* the GCTL, please consider the option to obtain a commercial license for a
|
||
|
|
* fee. These licenses are offered by the GCTL's original author. As a rule,
|
||
|
|
* licenses are provided "as-is", unlimited in time for a one time fee. Please
|
||
|
|
* send corresponding requests to: yizhang-geo@zju.edu.cn. Please do not forget
|
||
|
|
* to include some description of your company and the realm of its activities.
|
||
|
|
* Also add information on how to contact you by electronic and paper mail.
|
||
|
|
******************************************************/
|
||
|
|
|
||
|
|
#include "activation_mish.h"
|
||
|
|
|
||
|
|
gctl::mish::mish() {}
|
||
|
|
|
||
|
|
gctl::mish::~mish() {}
|
||
|
|
|
||
|
|
void gctl::mish::activate(const matrix<double> &z, matrix<double> &a)
|
||
|
|
{
|
||
|
|
// Mish(x) = x * tanh(softplus(x))
|
||
|
|
// softplus(x) = log(1 + exp(x))
|
||
|
|
// a = activation(z) = Mish(z)
|
||
|
|
// Z = [z1, ..., zn], A = [a1, ..., an], n observations
|
||
|
|
// h(x) = tanh(softplus(x)) = (1 + exp(x))^2 - 1
|
||
|
|
// ------------------
|
||
|
|
// (1 + exp(x))^2 + 1
|
||
|
|
// Let s = exp(-abs(x)), t = 1 + s
|
||
|
|
// If x >= 0, then h(x) = (t^2 - s^2) / (t^2 + s^2)
|
||
|
|
// If x <= 0, then h(x) = (t^2 - 1) / (t^2 + 1)
|
||
|
|
a.resize(z.row_size(), z.col_size());
|
||
|
|
|
||
|
|
int i, j;
|
||
|
|
#pragma omp parallel for private (i, j) schedule(guided)
|
||
|
|
for (i = 0; i < a.row_size(); i++)
|
||
|
|
{
|
||
|
|
for (j = 0; j < a.col_size(); j++)
|
||
|
|
{
|
||
|
|
a[i][j] = z[i][j]*std::tanh(log(1.0 + exp(z[i][j])));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
void gctl::mish::apply_jacobian(const matrix<double> &z,
|
||
|
|
const matrix<double> &a, const matrix<double> &f, matrix<double> &g)
|
||
|
|
{
|
||
|
|
// Apply the Jacobian matrix J to a vector f
|
||
|
|
// J = d_a / d_z = diag(Mish'(z))
|
||
|
|
// g = J * f = Mish'(z) .* f
|
||
|
|
// Z = [z1, ..., zn], G = [g1, ..., gn], F = [f1, ..., fn]
|
||
|
|
// Note: When entering this function, Z and G may point to the same matrix
|
||
|
|
// Let h(x) = tanh(softplus(x))
|
||
|
|
// Mish'(x) = h(x) + x * h'(x)
|
||
|
|
// h'(x) = tanh'(softplus(x)) * softplus'(x)
|
||
|
|
// = [1 - h(x)^2] * exp(x) / (1 + exp(x))
|
||
|
|
// = [1 - h(x)^2] / (1 + exp(-x))
|
||
|
|
// Mish'(x) = h(x) + [x - Mish(x) * h(x)] / (1 + exp(-x))
|
||
|
|
// A = Mish(Z) = Z .* h(Z) => h(Z) = A ./ Z, h(0) = 0.6
|
||
|
|
g.resize(a.row_size(), a.col_size());
|
||
|
|
|
||
|
|
int i, j;
|
||
|
|
#pragma omp parallel for private (i, j) schedule(guided)
|
||
|
|
for (i = 0; i < g.row_size(); i++)
|
||
|
|
{
|
||
|
|
for (j = 0; j < g.col_size(); j++)
|
||
|
|
{
|
||
|
|
g[i][j] = std::tanh(log(1.0 + exp(z[i][j])));
|
||
|
|
}
|
||
|
|
|
||
|
|
for (j = 0; j < g.col_size(); j++)
|
||
|
|
{
|
||
|
|
g[i][j] = f[i][j]*(g[i][j] + (z[i][j] - a[i][j]*g[i][j])/(1.0 + exp(-1.0*z[i][j])));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string gctl::mish::activation_name() const
|
||
|
|
{
|
||
|
|
return "Mish";
|
||
|
|
}
|
||
|
|
|
||
|
|
gctl::activation_type_e gctl::mish::activation_type() const
|
||
|
|
{
|
||
|
|
return Mish;
|
||
|
|
}
|