GeneralizationLoss.h
Go to the documentation of this file.
1 /*!
2  *
3  *
4  * \brief Stopping Criterion which stops, when the generalization of the solution gets worse
5  *
6  *
7  *
8  * \author O. Krause
9  * \date 2010
10  *
11  *
12  * \par Copyright 1995-2017 Shark Development Team
13  *
14  * <BR><HR>
15  * This file is part of Shark.
16  * <http://shark-ml.org/>
17  *
18  * Shark is free software: you can redistribute it and/or modify
19  * it under the terms of the GNU Lesser General Public License as published
20  * by the Free Software Foundation, either version 3 of the License, or
21  * (at your option) any later version.
22  *
23  * Shark is distributed in the hope that it will be useful,
24  * but WITHOUT ANY WARRANTY; without even the implied warranty of
25  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26  * GNU Lesser General Public License for more details.
27  *
28  * You should have received a copy of the GNU Lesser General Public License
29  * along with Shark. If not, see <http://www.gnu.org/licenses/>.
30  *
31  */
32 
33 #ifndef SHARK_TRAINERS_STOPPINGCRITERA_GENERALIZATIONLOSS_H
34 #define SHARK_TRAINERS_STOPPINGCRITERA_GENERALIZATIONLOSS_H
35 
37 #include <shark/Core/ResultSets.h>
38 #include <queue>
39 #include <numeric>
40 #include <algorithm>
41 #include <shark/LinAlg/Base.h>
42 namespace shark{
43 
44 /// \brief The generalization loss calculates the relative increase of the validation error compared to the minimum training error.
45 ///
46 /// The generalization loss at iteration t is calculated as
47 /// \f$ GL(t) 100 \left( \frac {E_v(t)} {\min_{t'} E_l(t')} -1 \right) \f$
48 /// where \f$ E_v \f$ is the validation error and \f$ E_l \f$ the training error.
49 /// This is a good indicator for overfitting, since it measures directly the gap between the two values. This method
50 /// stops when the generalization error is bigger than some predefined value. The disadvantage is, that
51 /// when the training error is still changing much a big generalization loss might be repaired later on. So this method
52 /// might stop to soon.
53 ///
54 /// Terminology for this and other stopping criteria is taken from (and also see):
55 ///
56 /// Lutz Prechelt. Early Stopping - but when? In Genevieve B. Orr and
57 /// Klaus-Robert Müller: Neural Networks: Tricks of the Trade, volume
58 /// 1524 of LNCS, Springer, 1997.
59 ///
60 template<class PointType = RealVector>
61 class GeneralizationLoss: public AbstractStoppingCriterion< ValidatedSingleObjectiveResultSet<PointType> >{
62 public:
64  ///constructs a generaliazationLoss which stops, when the GL > maxLoss
65  ///@param maxLoss maximum loss allowed before stopping
66  GeneralizationLoss(double maxLoss){
67  m_maxLoss = maxLoss;
68  reset();
69  }
70  /// returns true if the training should stop. The generalization
71  /// loss orders the optimizer to stop as soon as the validation
72  /// error grows larger than a certain factor of the minimum
73  /// validation error encountered so far.
74  bool stop(const ResultSet& set){
75  m_minTraining = std::min(m_minTraining, set.value);
76  m_gl = set.validation/m_minTraining - 1;
77 
78  return m_gl > m_maxLoss;
79  }
80  ///resets the internal state
81  void reset(){
82  m_minTraining = std::numeric_limits<double>::max();
83  }
84  ///returns the current generalization loss
85  double value() const{
86  return m_gl;
87  }
88 protected:
89  ///minimum training error
90  double m_minTraining;
91  ///minimum validation error
93 
94  ///maximum loss allowed
95  double m_maxLoss;
96  ///current generalization loss
97  double m_gl;
98 
99 
100 };
101 }
102 
103 
104 #endif