Neurons.h
Go to the documentation of this file.
1 /*!
2  *
3  *
4  * \brief -
5  *
6  * \author O.Krause
7  * \date 2011
8  *
9  *
10  * \par Copyright 1995-2017 Shark Development Team
11  *
12  * <BR><HR>
13  * This file is part of Shark.
14  * <http://shark-ml.org/>
15  *
16  * Shark is free software: you can redistribute it and/or modify
17  * it under the terms of the GNU Lesser General Public License as published
18  * by the Free Software Foundation, either version 3 of the License, or
19  * (at your option) any later version.
20  *
21  * Shark is distributed in the hope that it will be useful,
22  * but WITHOUT ANY WARRANTY; without even the implied warranty of
23  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
24  * GNU Lesser General Public License for more details.
25  *
26  * You should have received a copy of the GNU Lesser General Public License
27  * along with Shark. If not, see <http://www.gnu.org/licenses/>.
28  *
29  */
30 #ifndef MODELS_NEURONS_H
31 #define MODELS_NEURONS_H
32 
33 #include <shark/LinAlg/Base.h>
34 
35 
36 namespace shark{
37 namespace detail{
38  ///\brief Baseclass for all Neurons. it defines y=operator(x) for evaluation and derivative(y) for the derivative of the sigmoid.
39  ///
40  ///You need to provide a public member function function() and functionDerivative() in the derived class.
41  ///Those functions calculate value and derivative for a single input.
42  ///Due to template magic, the neurons can either use vectors or matrices as input.
43  ///Additionally, they avoid temporary values completely using ublas magic.
44  ///Usage:
45  ///struct Neuron:public NeuronBase<Neuron> {
46  /// double function(double x)const{return ...}
47  /// double functionDerivative(double y)const{return ...}
48  ///};
49  template<class Derived>
50  //again, one step ahead using templates!
51  class NeuronBase{
52  private:
53  template<class T>
54  struct Function{
55  typedef T argument_type;
56  typedef argument_type result_type;
57  static const bool zero_identity = false;
58 
59  Function(NeuronBase<Derived> const* self):m_self(static_cast<Derived const*>(self)){}
60 
61  result_type operator()(argument_type x)const{
62  return m_self->function(x);
63  }
64  Derived const* m_self;
65  };
66  template<class T>
67  struct FunctionDerivative{
68  typedef T argument_type;
69  typedef argument_type result_type;
70  static const bool zero_identity = false;
71 
72  FunctionDerivative(NeuronBase<Derived> const* self):m_self(static_cast<Derived const*>(self)){}
73 
74  result_type operator()(argument_type x)const{
75  return m_self->functionDerivative(x);
76  }
77  Derived const* m_self;
78  };
79  public:
80 
81  ///for a given input vector, calculates the elementwise application of the sigmoid function defined by Derived.
82  template<class E, class Device>
83  blas::vector_unary<E, Function<typename E::value_type> > operator()(blas::vector_expression<E, Device> const& x)const{
84  typedef Function<typename E::value_type> functor_type;
85  return blas::vector_unary<E, functor_type >(x(),functor_type(this));
86  }
87  ///for a given batch of input vectors, calculates the elementwise application of the sigmoid function defined by Derived.
88  template<class E, class Device>
89  blas::matrix_unary<E, Function<typename E::value_type> > operator()(blas::matrix_expression<E, Device> const& x)const{
90  typedef Function<typename E::value_type> functor_type;
91  return blas::matrix_unary<E, functor_type >(x(),functor_type(this));
92  }
93  ///Calculates the elementwise application of the sigmoid function derivative defined by Derived.
94  ///It's input is a matrix or vector of previously calculated neuron responses generated by operator()
95  template<class E, class Device>
96  blas::vector_unary<E, FunctionDerivative<typename E::value_type> > derivative(blas::vector_expression<E, Device> const& x)const{
97  typedef FunctionDerivative<typename E::value_type> functor_type;
98  return blas::vector_unary<E, functor_type >(x(),functor_type(this));
99  }
100  ///Calculates the elementwise application of the sigmoid function derivative defined by Derived.
101  ///It's input is a matrix or vector of previously calculated neuron responses generated by operator()
102  template<class E, class Device>
103  blas::matrix_unary<E, FunctionDerivative<typename E::value_type> > derivative(blas::matrix_expression<E, Device> const& x)const{
104  typedef FunctionDerivative<typename E::value_type> functor_type;
105  return blas::matrix_unary<E, functor_type >(x(),functor_type(this));
106  }
107  };
108 }
109 
110 ///\brief Neuron which computes the Logistic (logistic) function with range [0,1].
111 ///
112 ///The Logistic function is
113 ///\f[ f(x)=\frac 1 {1+exp^(-x)}\f]
114 ///it's derivative can be computed as
115 ///\f[ f'(x)= 1-f(x)^2 \f]
116 struct LogisticNeuron : public detail::NeuronBase<LogisticNeuron>{
117  template<class T>
118  T function(T x)const{
119  return sigmoid(x);
120  }
121  template<class T>
122  T functionDerivative(T y)const{
123  return y * (1 - y);
124  }
125 };
126 ///\brief Neuron which computes the hyperbolic tangenst with range [-1,1].
127 ///
128 ///The Tanh function is
129 ///\f[ f(x)=\tanh(x) = \frac 2 {1+exp^(-2x)}-1 \f]
130 ///it's derivative can be computed as
131 ///\f[ f'(x)= f(x)(1-f(x)) \f]
132 struct TanhNeuron: public detail::NeuronBase<TanhNeuron>{
133  template<class T>
134  T function(T x)const{
135  return std::tanh(x);
136  }
137  template<class T>
138  T functionDerivative(T y)const{
139  return 1.0 - y*y;
140  }
141 };
142 ///\brief Linear activation Neuron.
143 struct LinearNeuron: public detail::NeuronBase<LinearNeuron>{
144  template<class T>
145  T function(T x)const{
146  return x;
147  }
148  template<class T>
149  T functionDerivative(T y)const{
150  return 1.0;
151  }
152 };
153 
154 ///\brief Rectifier Neuron f(x) = max(0,x)
155 struct RectifierNeuron: public detail::NeuronBase<RectifierNeuron>{
156  template<class T>
157  T function(T x)const{
158  return std::max<T>(0,x);
159  }
160  template<class T>
161  T functionDerivative(T y)const{
162  if(y == 0)
163  return T(0);
164  return T(1);
165  }
166 };
167 
168 ///\brief Fast sigmoidal function, which does not need to compute an exponential function.
169 ///
170 ///It is defined as
171 ///\f[ f(x)=\frac x {1+|x|}\f]
172 ///it's derivative can be computed as
173 ///\f[ f'(x)= (1 - |f(x)|)^2 \f]
174 struct FastSigmoidNeuron: public detail::NeuronBase<FastSigmoidNeuron>{
175  template<class T>
176  T function(T x)const{
177  return x/(1+std::abs(x));
178  }
179  template<class T>
180  T functionDerivative(T y)const{
181  return sqr(1.0 - std::abs(y));
182  }
183 };
184 
185 
186 /// \brief Wraps a given neuron type and implements dropout for it
187 ///
188 /// The function works by setting the output randomly to 0 with a 50% chance.
189 /// The function assumes for the wrapped neuron type that the derivative
190 /// for all points for which the output is 0, is 0. This is true for the LogisticNeuron,
191 /// FastSigmoidNeuron and RectifierNeuron.
192 template<class Neuron>
193 struct DropoutNeuron: public detail::NeuronBase<DropoutNeuron<Neuron> >{
194  DropoutNeuron():m_probability(0.5),m_stochastic(true){}
195  template<class T>
196  T function(T x)const{
197  if(m_stochastic && random::coinToss(random::globalRng, m_probability)){
198  return T(0);
199  }
200  else if(!m_stochastic){
201  return (1-m_probability)*m_neuron.function(x);
202  }else{
203  return m_neuron.function(x);
204  }
205  }
206  template<class T>
207  T functionDerivative(T y)const{
208  if(!m_stochastic){
209  return (1-m_probability)*m_neuron.functionDerivative(y/ (1-m_probability));
210  }else{
211  return m_neuron.functionDerivative(y);
212  }
213  }
214 
215  void setProbability(double probability){m_probability = probability;}
216  void setStochastic(bool stochastic){m_stochastic = stochastic;}
217 
218 private:
219  double m_probability;
220  bool m_stochastic;
221  Neuron m_neuron;
222 };
223 
224 }
225 
226 #endif