Autoencoder.h

Go to the documentation of this file.

/*!
 * \brief   Implements the autoencoder
 * 
 * \author      O. Krause
 * \date        2010-2014
 *
 *
 * \par Copyright 1995-2017 Shark Development Team
 * 
 * <BR><HR>
 * This file is part of Shark.
 * <http://shark-ml.org/>
 * 
 * Shark is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published 
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * Shark is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with Shark.  If not, see <http://www.gnu.org/licenses/>.
 *
 */
#ifndef SHARK_MODELS_AUTOENCODER_H
#define SHARK_MODELS_AUTOENCODER_H

#include <shark/Models/AbstractModel.h>
#include <shark/Models/Neurons.h>
#include <boost/serialization/vector.hpp>

namespace shark{

/// \brief implements the autoencoder
///
/// The formula is
///  \f[ f(x) = \sigma_2(W^T\sigma_1(Wx+b_1)+b_2)\f]
/// Where \f$ W, W_2, b_1 \f$ and \f$b_2 \f$ are the weights and
///  \f$\sigma_1\f$ and \f$ \sigma_2\f$ are the activation functions for hidden and output units.
///
/// see TiedAutoencoder for the tied weights version where \f$ W_2=W_1^T \f$.
template<class HiddenNeuron,class OutputNeuron>
class Autoencoder :public AbstractModel<RealVector,RealVector>
{
    struct InternalState: public State{
        RealMatrix hiddenResponses;
        RealMatrix outputResponses;
    };
    

public:
    Autoencoder(){
        m_features|=HAS_FIRST_PARAMETER_DERIVATIVE;
        m_features|=HAS_FIRST_INPUT_DERIVATIVE;
    }

    //! \brief From INameable: return the class name.
    std::string name() const{ 
        return "Autoencoder";
    }

    //! \brief Number of input neurons.
    std::size_t inputSize()const{
        return outputSize();
    }
    //! \brief Number of output neurons.
    std::size_t outputSize()const{
        return outputBias().size();
    }
    
    //! \brief Total number of hidden neurons.
    std::size_t numberOfHiddenNeurons()const{
        return encoderMatrix().size1();
    }
    
    /// \brief Returns the hidden bias weight vector.
    RealVector const& hiddenBias()const{
        return m_hiddenBias;
    }
    
    /// \brief Returns the hidden bias weight vector.
    RealVector& hiddenBias(){
        return m_hiddenBias;
    }
    
    /// \brief Returns the output bias weight vector.
    RealVector const& outputBias()const{
        return m_outputBias;
    }
    /// \brief Returns the output bias weight vector.
    RealVector& outputBias(){
        return m_outputBias;
    }
    
    /// \brief Weight matrix for the direction input->hidden.
    RealMatrix const& encoderMatrix()const{
        return m_encoderMatrix;
    }
    /// \brief Weight matrix for the direction input->hidden.
    RealMatrix& encoderMatrix(){
        return m_encoderMatrix;
    }
    /// \brief Weight matrix for the direction hidden->output
    ///
    RealMatrix const& decoderMatrix()const{
        return m_decoderMatrix;
    }
    /// \brief Weight matrix for the direction hidden->output
    RealMatrix& decoderMatrix(){
        return m_decoderMatrix;
    }
    
    //! \brief Returns the total number of parameters of the network. 
    std::size_t numberOfParameters()const{
        return 2*inputSize()*numberOfHiddenNeurons()+inputSize()+numberOfHiddenNeurons();
    }

    //! returns the vector of used parameters inside the weight matrix
    RealVector parameterVector() const{
        return to_vector(m_encoderMatrix) | to_vector(m_decoderMatrix) | m_hiddenBias | m_outputBias;
    }
    //! uses the values inside the parametervector to set the used values inside the weight matrix
    void setParameterVector(RealVector const& newParameters){
        SIZE_CHECK(newParameters.size() == numberOfParameters());
        
        std::size_t endWeights1 = m_encoderMatrix.size1() * m_encoderMatrix.size2();
        std::size_t endWeights2 = endWeights1 +  m_decoderMatrix.size1() * m_decoderMatrix.size2();
        std::size_t endBias = endWeights2 + m_hiddenBias.size();
        noalias(to_vector(m_encoderMatrix)) = subrange(newParameters,0,endWeights1);
        noalias(to_vector(m_decoderMatrix)) = subrange(newParameters,endWeights1,endWeights2);
        noalias(m_hiddenBias) = subrange(newParameters,endWeights2,endBias);
        noalias(m_outputBias) = subrange(newParameters,endBias,endBias+m_outputBias.size());
    }

    //! \brief Returns the output of all neurons after the last call of eval
    //!
    //!     \param  state last result of eval
    //!     \return Output value of the neurons.
    RealMatrix const& hiddenResponses(State const& state)const{
        InternalState const& s = state.toState<InternalState>();
        return s.hiddenResponses;
    }
    
    /// \brief Returns the activation function of the hidden units.
    HiddenNeuron const& hiddenActivationFunction()const{
        return m_hiddenNeuron;
    }
    /// \brief Returns the activation function of the output units.
    OutputNeuron const& outputActivationFunction()const{
        return m_outputNeuron;
    }
    
    /// \brief Returns the activation function of the hidden units.
    HiddenNeuron& hiddenActivationFunction(){
        return m_hiddenNeuron;
    }
    /// \brief Returns the activation function of the output units.
    OutputNeuron& outputActivationFunction(){
        return m_outputNeuron;
    }
    
    boost::shared_ptr<State> createState()const{
        return boost::shared_ptr<State>(new InternalState());
    }

    void evalLayer(std::size_t layer,RealMatrix const& patterns,RealMatrix& outputs)const{
        SIZE_CHECK(layer < 2);
        std::size_t numPatterns = patterns.size1();
        
        if(layer == 0){//input->hidden
            SIZE_CHECK(patterns.size2() == encoderMatrix().size2());
            std::size_t numOutputs = encoderMatrix().size1();
            outputs.resize(numPatterns,numOutputs);
            outputs.clear();
            noalias(outputs) = prod(patterns,trans(encoderMatrix())) + repeat(hiddenBias(),numPatterns);
            noalias(outputs) = m_hiddenNeuron(outputs);
        }
        else{//hidden->output
            SIZE_CHECK(patterns.size2() == decoderMatrix().size2());
            std::size_t numOutputs = decoderMatrix().size1();
            outputs.resize(numPatterns,numOutputs);
            outputs.clear();
            noalias(outputs) = prod(patterns,trans(decoderMatrix())) + repeat(outputBias(),numPatterns);
            noalias(outputs) = m_outputNeuron(outputs);
        }
    }
    
    ///\brief Returns the response of the i-th layer given the input of that layer.
    ///
    /// this is usefull if only a portion of the network needs to be evaluated
    /// be aware that this only works without shortcuts in the network
    Data<RealVector> evalLayer(std::size_t layer, Data<RealVector> const& patterns)const{
        SIZE_CHECK(layer < 2);
        int batches = (int) patterns.numberOfBatches();
        Data<RealVector> result(batches);
        SHARK_PARALLEL_FOR(int i = 0; i < batches; ++i){
            evalLayer(layer,patterns.batch(i),result.batch(i));
        }
        return result;
    }
    
    Data<RealVector> encode(Data<RealVector> const& patterns)const{
        return evalLayer(0,patterns);
    }
    
    Data<RealVector> decode(Data<RealVector> const& patterns)const{
        return evalLayer(1,patterns);
    }
    
    template<class Label>
    LabeledData<RealVector,Label> encode(
        LabeledData<RealVector,Label> const& data
    )const{
        return LabeledData<RealVector,Label>(encode(data.inputs()),data.labels());
    }
    
    template<class Label>
    LabeledData<RealVector,Label> decode(
        LabeledData<RealVector,Label> const& data
    )const{
        return LabeledData<RealVector,Label>(decode(data.inputs()),data.labels());
    }
    
    
    void eval(RealMatrix const& patterns,RealMatrix& output, State& state)const{
        InternalState& s = state.toState<InternalState>();
        evalLayer(0,patterns,s.hiddenResponses);//propagate input->hidden
        evalLayer(1,s.hiddenResponses,s.outputResponses);//propagate hidden->output
        output = s.outputResponses;
    }
    using AbstractModel<RealVector,RealVector>::eval;

    void weightedParameterDerivative(
        BatchInputType const& patterns, RealMatrix const& coefficients, State const& state, RealVector& gradient
    )const{
        SIZE_CHECK(coefficients.size2() == outputSize());
        SIZE_CHECK(coefficients.size1() == patterns.size1());
        
        RealMatrix outputDelta = coefficients;
        RealMatrix hiddenDelta;
        computeDelta(state,outputDelta,hiddenDelta);
        computeParameterDerivative(patterns,outputDelta,hiddenDelta,state,gradient);
    }
    
    void weightedInputDerivative(
        BatchInputType const& patterns, RealMatrix const& coefficients, State const& state, BatchInputType& inputDerivative
    )const{
        SIZE_CHECK(coefficients.size2() == outputSize());
        SIZE_CHECK(coefficients.size1() == patterns.size1());
        
        RealMatrix outputDelta = coefficients;
        RealMatrix hiddenDelta;
        computeDelta(state,outputDelta,hiddenDelta,inputDerivative);
    }
    
    virtual void weightedDerivatives(
        BatchInputType const & patterns,
        BatchOutputType const & coefficients,
        State const& state,
        RealVector& parameterDerivative,
        BatchInputType& inputDerivative
    )const{
        SIZE_CHECK(coefficients.size2() == outputSize());
        SIZE_CHECK(coefficients.size1() == patterns.size1());

        RealMatrix outputDelta = coefficients;
        RealMatrix hiddenDelta;
        computeDelta(state,outputDelta,hiddenDelta,inputDerivative);
        computeParameterDerivative(patterns,outputDelta,hiddenDelta,state,parameterDerivative);
    }
    
    void setStructure(
        std::size_t in,std::size_t hidden
    ){
        m_encoderMatrix.resize(hidden,in);
        m_decoderMatrix.resize(in,hidden);
        m_hiddenBias.resize(hidden);
        m_outputBias.resize(in);
    }
    
    //! From ISerializable, reads a model from an archive
    void read( InArchive & archive ){
        archive>>m_encoderMatrix;
        archive>>m_decoderMatrix;
        archive>>m_hiddenBias;
        archive>>m_outputBias;
    }

    //! From ISerializable, writes a model to an archive
    void write( OutArchive & archive ) const{
        archive<<m_encoderMatrix;
        archive<<m_decoderMatrix;
        archive<<m_hiddenBias;
        archive<<m_outputBias;
    }


private:
    
    void computeDelta(
        State const& state, RealMatrix& outputDelta, RealMatrix& hiddenDelta
    )const{
        InternalState const& s = state.toState<InternalState>();

        noalias(outputDelta) *= m_outputNeuron.derivative(s.outputResponses);
        hiddenDelta.resize(outputDelta.size1(),numberOfHiddenNeurons());
        noalias(hiddenDelta) = prod(outputDelta,decoderMatrix());
        noalias(hiddenDelta) *= m_hiddenNeuron.derivative(s.hiddenResponses);
    }
    
    void computeDelta(
        State const& state, RealMatrix& outputDelta, RealMatrix& hiddenDelta, RealMatrix& inputDelta
    )const{
        computeDelta(state,outputDelta,hiddenDelta);
        inputDelta.resize(outputDelta.size1(),inputSize());
        noalias(inputDelta) = prod(hiddenDelta,encoderMatrix());
    }
    
    void computeParameterDerivative(
        RealMatrix const& patterns, RealMatrix const& outputDelta, RealMatrix const& hiddenDelta,
        State const& state, RealVector& gradient
    )const{
        InternalState const& s = state.toState<InternalState>();
        std::size_t hiddenParams = inputSize()*numberOfHiddenNeurons();
        std::size_t numHidden = numberOfHiddenNeurons();
        gradient.resize(numberOfParameters());
        auto  gradEncoder  = to_matrix(subrange(gradient,0,hiddenParams),numHidden,inputSize());
        auto  gradDecoder  = to_matrix(subrange(gradient,hiddenParams,2*hiddenParams),outputSize(),numHidden);
        noalias(gradDecoder) = prod(trans(outputDelta),s.hiddenResponses);
        noalias(gradEncoder) = prod(trans(hiddenDelta),patterns);
        
        std::size_t hiddenBiasPos = 2*hiddenParams;
        std::size_t outputBiasPos = hiddenBiasPos+numHidden;
        subrange(gradient,hiddenBiasPos,outputBiasPos) = sum_rows(hiddenDelta);
        subrange(gradient,outputBiasPos,outputBiasPos+inputSize()) = sum_rows(outputDelta);
    }

    //! weight matrix between input and hidden layer. 
    RealMatrix m_encoderMatrix;
    //! weight matrix between hiddenand output layer. 
    RealMatrix m_decoderMatrix;
    //! bias weights of the hidden neurons
    RealVector m_hiddenBias;
    //! bias weights of the visible neurons
    RealVector m_outputBias;

    //!Type of hidden neuron. See Models/Neurons.h for a few choices
    HiddenNeuron m_hiddenNeuron;
    //! Type of output neuron. See Models/Neurons.h for a few choices
    OutputNeuron m_outputNeuron;
};


}
#endif