diff --git a/tmva/tmva/inc/TMVA/MethodSVM.h b/tmva/tmva/inc/TMVA/MethodSVM.h index 1b59d327eb2e8..cc7181f3dbcb0 100644 --- a/tmva/tmva/inc/TMVA/MethodSVM.h +++ b/tmva/tmva/inc/TMVA/MethodSVM.h @@ -53,6 +53,9 @@ #ifndef ROOT_TVectorD #include "TVectorD.h" #endif +#ifndef ROOT_TMVA_SVKernelFunction +#include "TMVA/SVKernelFunction.h" +#endif #endif namespace TMVA @@ -74,9 +77,18 @@ namespace TMVA virtual Bool_t HasAnalysisType( Types::EAnalysisType type, UInt_t numberClasses, UInt_t numberTargets ); + // optimise tuning parameters + virtual std::map OptimizeTuningParameters(TString fomType="ROCIntegral", TString fitType="Minuit"); + virtual void SetTuneParameters(std::map tuneParameters); + std::vector MakeKernelList(std::string multiKernels, TString kernel); + std::map< TString,std::vector > GetTuningOptions(); + // training method void Train( void ); + // revoke training (required for optimise tuning parameters) + void Reset( void ); + using MethodBase::ReadWeightsFromStream; // write weights to file @@ -97,6 +109,17 @@ namespace TMVA // ranking of input variables const Ranking* CreateRanking() { return 0; } + // for SVM optimisation + void SetGamma(Double_t g){fGamma = g;} + void SetCost(Double_t c){fCost = c;} + void SetMGamma(std::string & mg); + void SetOrder(Double_t o){fOrder = o;} + void SetTheta(Double_t t){fTheta = t;} + void SetKappa(Double_t k){fKappa = k;} + void SetMult(Double_t m){fMult = m;} + + void GetMGamma(const std::vector & gammas); + protected: // make ROOT-independent C++ class for classifier response (classifier-specific implementation) @@ -111,6 +134,7 @@ namespace TMVA void DeclareOptions(); void DeclareCompatibilityOptions(); void ProcessOptions(); + Double_t getLoss( TString lossFunction ); Float_t fCost; // cost value Float_t fTolerance; // tolerance parameter @@ -126,12 +150,23 @@ namespace TMVA TVectorD* fMinVars; // for normalization //is it still needed?? TVectorD* fMaxVars; // for normalization //is it still needed?? - // for backward compatibility + // for kernel functions TString fTheKernel; // kernel name Float_t fDoubleSigmaSquared; // for RBF Kernel Int_t fOrder; // for Polynomial Kernel ( polynomial order ) Float_t fTheta; // for Sigmoidal Kernel Float_t fKappa; // for Sigmoidal Kernel + Float_t fMult; + std::vector fmGamma; // vector of gammas for multi-gaussian kernel + Float_t fNumVars; // number of input variables for multi-gaussian + std::vector fVarNames; + std::string fGammas; + std::string fGammaList; + std::string fTune; // Specify parameters to be tuned + std::string fMultiKernels; + + Int_t fDataSize; + TString fLoss; ClassDef(MethodSVM,0) // Support Vector Machine }; diff --git a/tmva/tmva/inc/TMVA/SVKernelFunction.h b/tmva/tmva/inc/TMVA/SVKernelFunction.h index 0207f72132ffe..1fb22f37e227c 100644 --- a/tmva/tmva/inc/TMVA/SVKernelFunction.h +++ b/tmva/tmva/inc/TMVA/SVKernelFunction.h @@ -39,25 +39,33 @@ namespace TMVA { public: + enum EKernelType { kLinear , kRBF, kPolynomial, kSigmoidal, kMultiGauss, kProd, kSum}; + SVKernelFunction(); SVKernelFunction( Float_t ); + SVKernelFunction( EKernelType, Float_t, Float_t=0); + SVKernelFunction( std::vector params ); + SVKernelFunction(EKernelType k, std::vector kernels, std::vector gammas, Float_t gamma, Float_t order, Float_t theta); ~SVKernelFunction(); Float_t Evaluate( SVEvent* ev1, SVEvent* ev2 ); - enum EKernelType { kLinear , kRBF, kPolynomial, kSigmoidal }; - void setCompatibilityParams(EKernelType k, UInt_t order, Float_t theta, Float_t kappa); private: Float_t fGamma; // documentation + // vector of gammas for multidimensional gaussian + std::vector fmGamma; + // kernel, order, theta, and kappa are for backward compatibility EKernelType fKernel; UInt_t fOrder; Float_t fTheta; Float_t fKappa; + + std::vector fKernelsList; }; } diff --git a/tmva/tmva/src/MethodSVM.cxx b/tmva/tmva/src/MethodSVM.cxx index 3da9365288ac5..d33f48fd3c403 100644 --- a/tmva/tmva/src/MethodSVM.cxx +++ b/tmva/tmva/src/MethodSVM.cxx @@ -20,6 +20,12 @@ * Kamil Kraszewski - IFJ PAN & UJ, Krakow, Poland * * Maciej Kruk - IFJ PAN & AGH, Krakow, Poland * * * + * Introduction of kernel parameter optimisation * + * and additional kernel functions by: * + * Adrian Bevan - Queen Mary * + * University of London, UK * + * Tom Stevenson - Queen Mary * + * University of London, UK * * * * Copyright (c) 2005: * * CERN, Switzerland * @@ -70,10 +76,15 @@ #include "TMVA/MethodBase.h" #include "TMVA/MsgLogger.h" #include "TMVA/Types.h" +#include "TMVA/Interval.h" +#include "TMVA/OptimizeConfigParameters.h" +#include "TMVA/ResultsClassification.h" #include using std::vector; +using std::string; +using std::stringstream; //const Int_t basketsize__ = 1280000; REGISTER_METHOD(SVM) @@ -102,7 +113,18 @@ TMVA::MethodSVM::MethodSVM( const TString& jobName, const TString& methodTitle, , fOrder(0) , fTheta(0) , fKappa(0) + , fMult(0) + ,fNumVars(0) + , fGammas("") + , fGammaList("") + , fDataSize(0) + , fLoss(0) { + fVarNames.clear(); + fNumVars = theData.GetVariableInfos().size(); + for( int i=0; iclear(); + for (UInt_t i=0; isize(); i++) { + delete fInputData->at(i); + } if (fWgSet !=0) { delete fWgSet; fWgSet=0; } if (fSVKernelFunction !=0 ) { delete fSVKernelFunction; fSVKernelFunction = 0; } } +//////////////////////////////////////////////////////////////////////////////// +// reset the method, as if it had just been instantiated (forget all training etc.) + +void TMVA::MethodSVM::Reset( void ) +{ + // reset the method, as if it had just been instantiated (forget all training etc.) + fSupportVectors->clear(); + for (UInt_t i=0; isize(); i++){ + delete fInputData->at(i); + fInputData->at(i)=0; + } + fInputData->clear(); + if (fWgSet !=0) { fWgSet=0; } + if (fSVKernelFunction !=0 ) { fSVKernelFunction = 0; } + if (Data()){ + Data()->DeleteResults(GetMethodName(), Types::kTraining, GetAnalysisType()); + } + + Log() << kDEBUG << " successfully(?) reset the method " << Endl; +} + //////////////////////////////////////////////////////////////////////////////// /// SVM can handle classification with 2 classes and regression with one regression-target @@ -170,8 +226,20 @@ void TMVA::MethodSVM::Init() void TMVA::MethodSVM::DeclareOptions() { + DeclareOptionRef( fTheKernel = "RBF", "Kernel", "Pick which kernel ( RBF or MultiGauss )"); // for gaussian kernel parameter(s) DeclareOptionRef( fGamma = 1., "Gamma", "RBF kernel parameter: Gamma (size of the Kernel)"); + // for polynomial kernel parameter(s) + DeclareOptionRef( fOrder = 3, "Order", "Polynomial Kernel parameter: polynomial order"); + DeclareOptionRef( fTheta = 1., "Theta", "Polynomial Kernel parameter: polynomial theta"); + // for multi-gaussian kernel parameter(s) + DeclareOptionRef( fGammas = "", "GammaList", "MultiGauss parameters" ); + + // for range and step number for kernel paramter optimisation + DeclareOptionRef( fTune = "All", "Tune", "Tune Parameters"); + // for list of kernels to be used with product or sum kernel + DeclareOptionRef( fMultiKernels = "None", "KernelList", "Sum or product of kernels"); + DeclareOptionRef( fLoss = "hinge", "Loss", "Loss function"); DeclareOptionRef( fCost, "C", "Cost parameter" ); if (DoRegression()) { @@ -222,12 +290,88 @@ void TMVA::MethodSVM::Train() Data()->SetCurrentType(Types::kTraining); Log() << kDEBUG << "Create event vector"<< Endl; - for (Int_t ievt=0; ievtGetNEvents(); ievt++){ - if (GetEvent(ievt)->GetWeight() != 0) - fInputData->push_back(new SVEvent(GetEvent(ievt), fCost, DataInfo().IsSignal(GetEvent(ievt)))); + + fDataSize = Data()->GetNEvents(); + Int_t nSignal = Data()->GetNEvtSigTrain(); + Int_t nBackground = Data()->GetNEvtBkgdTrain(); + Double_t CSig; + Double_t CBkg; + + // Use number of signal and background from above to weight the cost parameter + // so that the training is not biased towards the larger dataset when the signal + // and background samples are significantly different sizes. + if(nSignal < nBackground){ + CSig = fCost; + CBkg = CSig*((double)nSignal/nBackground); + } + else{ + CBkg = fCost; + CSig = CBkg*((double)nSignal/nBackground); } - fSVKernelFunction = new SVKernelFunction(fGamma); + // Loop over events and assign the correct cost parameter. + for (Int_t ievnt=0; ievntGetNEvents(); ievnt++){ + if (GetEvent(ievnt)->GetWeight() != 0){ + if(DataInfo().IsSignal(GetEvent(ievnt))){ + fInputData->push_back(new SVEvent(GetEvent(ievnt), CSig, DataInfo().IsSignal\ + (GetEvent(ievnt)))); + } + else{ + fInputData->push_back(new SVEvent(GetEvent(ievnt), CBkg, DataInfo().IsSignal\ + (GetEvent(ievnt)))); + } + } + } + + // Set the correct kernel function. + // Here we only use valid Mercer kernels. In the literature some people have reported reasonable + // results using Sigmoid kernel function however that is not a valid Mercer kernel and is not used here. + if( fTheKernel == "RBF"){ + fSVKernelFunction = new SVKernelFunction( SVKernelFunction::kRBF, fGamma); + } + else if( fTheKernel == "MultiGauss" ){ + if(fGammas!=""){ + SetMGamma(fGammas); + fGammaList=fGammas; + } + else{ + if(fmGamma.size()!=0){ GetMGamma(fmGamma); } // Set fGammas if empty to write to XML file + else{ + for(Int_t ngammas=0; ngammassize()<<" event instances"<< Endl; Timer bldwstime( GetName()); @@ -245,14 +389,8 @@ void TMVA::MethodSVM::Train() fBparm = fWgSet->GetBpar(); fSupportVectors = fWgSet->GetSupportVectors(); - - delete fWgSet; fWgSet=0; - - // for (UInt_t i=0; isize();i++) delete fInputData->at(i); - delete fInputData; - fInputData=0; } //////////////////////////////////////////////////////////////////////////////// @@ -263,6 +401,9 @@ void TMVA::MethodSVM::AddWeightsXMLTo( void* parent ) const void* wght = gTools().AddChild(parent, "Weights"); gTools().AddAttr(wght,"fBparm",fBparm); gTools().AddAttr(wght,"fGamma",fGamma); + gTools().AddAttr(wght,"fGammaList",fGammaList); + gTools().AddAttr(wght,"fTheta",fTheta); + gTools().AddAttr(wght,"fOrder",fOrder); gTools().AddAttr(wght,"NSupVec",fSupportVectors->size()); for (std::vector::iterator veciter=fSupportVectors->begin(); @@ -291,6 +432,9 @@ void TMVA::MethodSVM::ReadWeightsFromXML( void* wghtnode ) { gTools().ReadAttr( wghtnode, "fBparm",fBparm ); gTools().ReadAttr( wghtnode, "fGamma",fGamma); + gTools().ReadAttr( wghtnode, "fGammaList",fGammaList); + gTools().ReadAttr( wghtnode, "fOrder",fOrder); + gTools().ReadAttr( wghtnode, "fTheta",fTheta); UInt_t fNsupv=0; gTools().ReadAttr( wghtnode, "NSupVec",fNsupv ); @@ -332,7 +476,28 @@ void TMVA::MethodSVM::ReadWeightsFromXML( void* wghtnode ) for (UInt_t ivar = 0; ivar < GetNvar(); ivar++) gTools().ReadAttr( maxminnode,"Var"+gTools().StringFromInt(ivar),(*fMinVars)[ivar]); if (fSVKernelFunction!=0) delete fSVKernelFunction; - fSVKernelFunction = new SVKernelFunction(fGamma); + if( fTheKernel == "RBF" ){ + fSVKernelFunction = new SVKernelFunction(SVKernelFunction::kRBF, fGamma); + } + else if( fTheKernel == "MultiGauss" ){ + SetMGamma(fGammaList); + fSVKernelFunction = new SVKernelFunction(fmGamma); + } + else if( fTheKernel == "Polynomial" ){ + fSVKernelFunction = new SVKernelFunction(SVKernelFunction::kPolynomial, fOrder, fTheta); + } + else if( fTheKernel == "Prod" ){ + SetMGamma(fGammaList); + fSVKernelFunction = new SVKernelFunction(SVKernelFunction::kSum, MakeKernelList(fMultiKernels,fTheKernel), fmGamma, fGamma, fOrder, fTheta); + } + else if( fTheKernel == "Sum" ){ + SetMGamma(fGammaList); + fSVKernelFunction = new SVKernelFunction(SVKernelFunction::kSum, MakeKernelList(fMultiKernels,fTheKernel), fmGamma, fGamma, fOrder, fTheta); + } + else { + Log() << kWARNING << fTheKernel << " is not a recognised kernel function." << Endl; + exit(1); + } delete svector; } @@ -581,3 +746,453 @@ void TMVA::MethodSVM::GetHelpMessage() const Log() << "events so that a coarse preliminary tuning should be performed on " << Endl; Log() << "reduced data sets." << Endl; } + +//////////////////////////////////////////////////////////////////////////////// +/// Optimize Tuning Parameters +/// This is used to optimise the kernel function parameters and cost. All kernel parameters +/// are optimised by default with default ranges, however the parameters to be optimised can +/// be set when booking the method with the option Tune. +/// Example: "Tune=Gamma[0.01;1.0;100]" would only tune the RBF Gamma between 0.01 and 1.0 +/// with 100 steps. +std::map TMVA::MethodSVM::OptimizeTuningParameters(TString fomType, TString fitType) +{ + // Call the Optimizer with the set of kernel parameters and ranges that are meant to be tuned. + std::map< TString,std::vector > optVars; + // Get parameters and options specified in booking of method. + if(fTune != "All"){ + optVars= GetTuningOptions(); + } + std::map< TString,std::vector >::iterator iter; + // Fill all the tuning parameters that should be optimized into a map + std::map tuneParameters; + std::map tunedParameters; + // Note: the 3rd parameter in the interval is the "number of bins", NOT the stepsize!! + // The actual values are always read from the middle of the bins. + Log() << kINFO << "Using the " << fTheKernel << " kernel." << Endl; + // Setup map of parameters based on the specified options or defaults. + if( fTheKernel == "RBF" ){ + if(fTune == "All"){ + tuneParameters.insert(std::pair("Gamma",new Interval(0.01,1.,100))); + tuneParameters.insert(std::pair("C",new Interval(0.01,1.,100))); + } + else{ + for(iter=optVars.begin(); iter!=optVars.end(); iter++){ + if( iter->first == "Gamma" || iter->first == "C"){ + tuneParameters.insert(std::pair(iter->first, new Interval(iter->second.at(0),iter->second.at(1),iter->second.at(2)))); + } + else{ + Log() << kWARNING << iter->first << " is not a recognised tuneable parameter." << Endl; + exit(1); + } + } + } + } + else if( fTheKernel == "Polynomial" ){ + if (fTune == "All"){ + tuneParameters.insert(std::pair("Order", new Interval(1,10,10))); + tuneParameters.insert(std::pair("Theta", new Interval(0.01,1.,100))); + tuneParameters.insert(std::pair("C", new Interval(0.01,1.,100))); + } + else{ + for(iter=optVars.begin(); iter!=optVars.end(); iter++){ + if( iter->first == "Theta" || iter->first == "C"){ + tuneParameters.insert(std::pair(iter->first, new Interval(iter->second.at(0),iter->second.at(1),iter->second.at(2)))); + } + else if( iter->first == "Order"){ + tuneParameters.insert(std::pair(iter->first, new Interval(iter->second.at(0),iter->second.at(1),iter->second.at(2)))); + } + else{ + Log() << kWARNING << iter->first << " is not a recognised tuneable parameter." << Endl; + exit(1); + } + } + } + } + else if( fTheKernel == "MultiGauss" ){ + if (fTune == "All"){ + for(int i=0; i(str,new Interval(0.01,1.,100))); + } + tuneParameters.insert(std::pair("C",new Interval(0.01,1.,100))); + } else { + for(iter=optVars.begin(); iter!=optVars.end(); iter++){ + if( iter->first == "GammaList"){ + for(int j=0; j(str, new Interval(iter->second.at(0),iter->second.at(1),iter->second.at(2)))); + } + } + else if( iter->first == "C"){ + tuneParameters.insert(std::pair(iter->first, new Interval(iter->second.at(0),iter->second.at(1),iter->second.at(2)))); + } + else{ + Log() << kWARNING << iter->first << " is not a recognised tuneable parameter." << Endl; + exit(1); + } + } + } + } + else if( fTheKernel == "Prod" ){ + std::stringstream tempstring(fMultiKernels); + std::string value; + while (std::getline(tempstring,value,'*')){ + if(value == "RBF"){ + tuneParameters.insert(std::pair("Gamma",new Interval(0.01,1.,100))); + } + else if(value == "MultiGauss"){ + for(int i=0; i(str,new Interval(0.01,1.,100))); + } + } + else if(value == "Polynomial"){ + tuneParameters.insert(std::pair("Order",new Interval(1,10,10))); + tuneParameters.insert(std::pair("Theta",new Interval(0.0,1.0,101))); + } + else { + Log() << kWARNING << value << " is not a recognised kernel function." << Endl; + exit(1); + } + } + tuneParameters.insert(std::pair("C",new Interval(0.01,1.,100))); + } + else if( fTheKernel == "Sum" ){ + std::stringstream tempstring(fMultiKernels); + std::string value; + while (std::getline(tempstring,value,'+')){ + if(value == "RBF"){ + tuneParameters.insert(std::pair("Gamma",new Interval(0.01,1.,100))); + } + else if(value == "MultiGauss"){ + for(int i=0; i(str,new Interval(0.01,1.,100))); + } + } + else if(value == "Polynomial"){ + tuneParameters.insert(std::pair("Order",new Interval(1,10,10))); + tuneParameters.insert(std::pair("Theta",new Interval(0.0,1.0,101))); + } + else { + Log() << kWARNING << value << " is not a recognised kernel function." << Endl; + exit(1); + } + } + tuneParameters.insert(std::pair("C",new Interval(0.01,1.,100))); + } + else { + Log() << kWARNING << fTheKernel << " is not a recognised kernel function." << Endl; + exit(1); + } + Log() << kINFO << " the following SVM parameters will be tuned on the respective *grid*\n" << Endl; + std::map::iterator it; + for(it=tuneParameters.begin(); it!=tuneParameters.end(); it++){ + Log() << kWARNING << it->first <second)->Print(Log()); + Log()< tuneParameters) +{ + std::map::iterator it; + if( fTheKernel == "RBF" ){ + for(it=tuneParameters.begin(); it!=tuneParameters.end(); it++){ + Log() << kWARNING << it->first << " = " << it->second << Endl; + if (it->first == "Gamma"){ + SetGamma (it->second); + } + else if(it->first == "C"){ + SetCost (it->second); + } + else { + Log() << kFATAL << " SetParameter for " << it->first << " not implemented " << Endl; + } + } + } + else if( fTheKernel == "MultiGauss" ){ + fmGamma.clear(); + for(int i=0; ifirst << " = " << tuneParameters.find(str)->second << Endl; + fmGamma.push_back(tuneParameters.find(str)->second); + } + for(it=tuneParameters.begin(); it!=tuneParameters.end(); it++){ + if (it->first == "C"){ + Log() << kWARNING << it->first << " = " << it->second << Endl; + SetCost(it->second); + break; + } + } + } + else if( fTheKernel == "Polynomial" ){ + for(it=tuneParameters.begin(); it!=tuneParameters.end(); it++){ + Log() << kWARNING << it->first << " = " << it->second << Endl; + if (it->first == "Order"){ + SetOrder(it->second); + } + else if (it->first == "Theta"){ + SetTheta(it->second); + } + else if(it->first == "C"){ SetCost (it->second); + } + else if(it->first == "Mult"){ + SetMult(it->second); + } + else{ + Log() << kFATAL << " SetParameter for " << it->first << " not implemented " << Endl; + } + } + } + else if( fTheKernel == "Prod" || fTheKernel == "Sum"){ + fmGamma.clear(); + for(it=tuneParameters.begin(); it!=tuneParameters.end(); it++){ + bool foundParam = false; + Log() << kWARNING << it->first << " = " << it->second << Endl; + for(int i=0; ifirst == str){ + fmGamma.push_back(it->second); + foundParam = true; + } + } + if (it->first == "Gamma"){ + SetGamma (it->second); + foundParam = true; + } + else if (it->first == "Order"){ + SetOrder (it->second); + foundParam = true; + } + else if (it->first == "Theta"){ + SetTheta (it->second); + foundParam = true; + } + else if (it->first == "C"){ SetCost (it->second); + SetCost (it->second); + foundParam = true; + } + else{ + if(!foundParam){ + Log() << kFATAL << " SetParameter for " << it->first << " not implemented " << Endl; + } + } + } + } + else { + Log() << kWARNING << fTheKernel << " is not a recognised kernel function." << Endl; + exit(1); + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// Takes as input a string of values for multigaussian gammas and splits it, filling the +/// gamma vector required by the SVKernelFunction. Example: "GammaList=0.1,0.2,0.3" would +/// make a vector with Gammas of 0.1,0.2 & 0.3 corresponding to input variables 1,2 & 3 +/// respectively. +void TMVA::MethodSVM::SetMGamma(std::string & mg){ + std::stringstream tempstring(mg); + Float_t value; + while (tempstring >> value){ + fmGamma.push_back(value); + + if (tempstring.peek() == ','){ + tempstring.ignore(); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// Produces GammaList string for multigaussian kernel to be written to xml file +void TMVA::MethodSVM::GetMGamma(const std::vector & gammas){ + std::ostringstream tempstring; + for(UInt_t i = 0; i TMVA::MethodSVM::MakeKernelList(std::string multiKernels, TString kernel) +{ + std::vector kernelsList; + std::stringstream tempstring(multiKernels); + std::string value; + if(kernel=="Prod"){ + while (std::getline(tempstring,value,'*')){ + if(value == "RBF"){ kernelsList.push_back(SVKernelFunction::kRBF);} + else if(value == "MultiGauss"){ + kernelsList.push_back(SVKernelFunction::kMultiGauss); + if(fGammas!=""){ + SetMGamma(fGammas); + } + } + else if(value == "Polynomial"){ kernelsList.push_back(SVKernelFunction::kPolynomial);} + else { + Log() << kWARNING << value << " is not a recognised kernel function." << Endl; + exit(1); + } + } + } + else if(kernel=="Sum"){ + while (std::getline(tempstring,value,'+')){ + if(value == "RBF"){ kernelsList.push_back(SVKernelFunction::kRBF);} + else if(value == "MultiGauss"){ + kernelsList.push_back(SVKernelFunction::kMultiGauss); + if(fGammas!=""){ + SetMGamma(fGammas); + } + } + else if(value == "Polynomial"){ kernelsList.push_back(SVKernelFunction::kPolynomial);} + else { + Log() << kWARNING << value << " is not a recognised kernel function." << Endl; + exit(1); + } + } + } + else { + Log() << kWARNING << "Unable to split MultiKernels. Delimiters */+ required." << Endl; + exit(1); + } + return kernelsList; +} + +//////////////////////////////////////////////////////////////////////////////// +/// GetTuningOptions +/// Function to allow for ranges and number of steps (for scan) when optimising kernel +/// function parameters. Specified when booking the method after the parameter to be +/// optimised between square brackets with each value separated by ;, the first value +/// is the lower limit, the second the upper limit and the third is the number of steps. +/// Example: "Tune=Gamma[0.01;1.0;100]" would only tune the RBF Gamma between 0.01 and +/// 100 steps. +std::map< TString,std::vector > TMVA::MethodSVM::GetTuningOptions() +{ + std::map< TString,std::vector > optVars; + std::stringstream tempstring(fTune); + std::string value; + while (std::getline(tempstring,value,',')){ + unsigned first = value.find('[')+1; + unsigned last = value.find_last_of(']'); + std::string optParam = value.substr(0,first-1); + std::stringstream strNew (value.substr(first,last-first)); + Double_t optInterval; + std::vector tempVec; + UInt_t i = 0; + while (strNew >> optInterval){ + tempVec.push_back(optInterval); + if (strNew.peek() == ';'){ + strNew.ignore(); + } + ++i; + } + if(i != 3 && i == tempVec.size()){ + if(optParam == "C" || optParam == "Gamma" || optParam == "GammaList" || optParam == "Theta"){ + switch(i){ + case 0: + tempVec.push_back(0.01); + case 1: + tempVec.push_back(1.); + case 2: + tempVec.push_back(100); + } + } + else if(optParam == "Order"){ + switch(i){ + case 0: + tempVec.push_back(1); + case 1: + tempVec.push_back(10); + case 2: + tempVec.push_back(10); + } + } + else{ + Log() << kWARNING << optParam << " is not a recognised tuneable parameter." << Endl; + exit(1); + } + } + optVars.insert(std::pair >(optParam,tempVec)); + } + return optVars; +} + +//////////////////////////////////////////////////////////////////////////////// +/// getLoss +/// Calculates loss for testing dataset. The loss function can be specified when +/// booking the method, otherwise defaults to hinge loss. Currently not used however +/// is accesible if required. +Double_t TMVA::MethodSVM::getLoss(TString lossFunction){ + Double_t loss = 0.0; + Double_t sumW = 0.0; + Double_t temp = 0.0; + Data()->SetCurrentType(Types::kTesting); + ResultsClassification* mvaRes = dynamic_cast ( Data()->GetResults(GetMethodName(),Types::kTesting, Types::kClassification) ); + for (Long64_t ievt=0; ievtGetWeight(); + if(DataInfo().IsSignal(ev)){ + if(lossFunction == "hinge"){ + temp += w*(1-v); + } + else if(lossFunction == "exp"){ + temp += w*TMath::Exp(-v); + } + else if(lossFunction == "binomial"){ + temp += w*TMath::Log(1+TMath::Exp(-2*v)); + } + else{ + Log() << kWARNING << lossFunction << " is not a recognised loss function." << Endl; + exit(1); + } + } + else{ + if(lossFunction == "hinge"){ + temp += w*v; + } + else if(lossFunction == "exp"){ + temp += w*TMath::Exp(-(1-v)); + } + else if(lossFunction == "binomial"){ + temp += w*TMath::Log(1+TMath::Exp(-2*(1-v))); + } + else{ + Log() << kWARNING << lossFunction << " is not a recognised loss function." << Endl; + exit(1); + } + } + sumW += w; + } + loss = temp/sumW; + + return loss; +} diff --git a/tmva/tmva/src/OptimizeConfigParameters.cxx b/tmva/tmva/src/OptimizeConfigParameters.cxx index 535a163410a2e..e23b382ee4b59 100644 --- a/tmva/tmva/src/OptimizeConfigParameters.cxx +++ b/tmva/tmva/src/OptimizeConfigParameters.cxx @@ -246,6 +246,9 @@ void TMVA::OptimizeConfigParameters::optimizeFit() // iterates through the tuneParameters !!!! } + // added to allow for transformation on input variables i.e. norm + GetMethod()->GetTransformationHandler().CalcTransformations(GetMethod()->Data()->GetEventCollection()); + // create the fitter FitterBase* fitter = NULL; diff --git a/tmva/tmva/src/SVKernelFunction.cxx b/tmva/tmva/src/SVKernelFunction.cxx index 0479d8e47464d..c19e15142a54e 100644 --- a/tmva/tmva/src/SVKernelFunction.cxx +++ b/tmva/tmva/src/SVKernelFunction.cxx @@ -15,6 +15,12 @@ * Andrzej Zemla - IFJ PAN, Krakow, Poland * * (IFJ PAN: Henryk Niewodniczanski Inst. Nucl. Physics, Krakow, Poland) * * * + * MultiGaussian, Product and Sum kernels included: * + * Adrian Bevan - Queen Mary * + * University of London, UK * + * Tom Stevenson - Queen Mary * + * University of London, UK * + * * * Copyright (c) 2005: * * CERN, Switzerland * * MPI-K Heidelberg, Germany * @@ -52,6 +58,50 @@ TMVA::SVKernelFunction::SVKernelFunction( Float_t gamma ) fTheta(0), fKappa(0) { + fmGamma.clear(); +} + +//////////////////////////////////////////////////////////////////////////////// +/// constructor + +TMVA::SVKernelFunction::SVKernelFunction( EKernelType k, Float_t param1, Float_t param2) + : fKernel(k) +{ + if (k==kRBF) { fGamma = param1; } + else if (k==kPolynomial){ + fOrder = param1; + fTheta = param2; + } + fKernelsList.clear(); +} + +//////////////////////////////////////////////////////////////////////////////// +/// constructor + +TMVA::SVKernelFunction::SVKernelFunction( std::vector params ) : + fKernel(kMultiGauss) +{ + fmGamma.clear(); + for( std::vector::const_iterator iter = params.begin(); iter != params.end()\ + ; ++iter ){ + fmGamma.push_back(*iter); + } + //fKernelsList.clear(); +} + +//////////////////////////////////////////////////////////////////////////////// +/// constructor + +TMVA::SVKernelFunction::SVKernelFunction(EKernelType k, std::vector kernels, std::vector gammas, Float_t gamma, Float_t order, Float_t theta) : + fGamma(gamma), + fKernel(k), + fOrder(order), + fTheta(theta) +{ + fmGamma.clear(); + fKernelsList.clear(); + fKernelsList = kernels; + fmGamma = gammas; } //////////////////////////////////////////////////////////////////////////////// @@ -59,6 +109,8 @@ TMVA::SVKernelFunction::SVKernelFunction( Float_t gamma ) TMVA::SVKernelFunction::~SVKernelFunction() { + fmGamma.clear(); + fKernelsList.clear(); } //////////////////////////////////////////////////////////////////////////////// @@ -86,8 +138,30 @@ Float_t TMVA::SVKernelFunction::Evaluate( SVEvent* ev1, SVEvent* ev2 ) return TMath::Exp(-norm*fGamma); } + case kMultiGauss: + { + // Kernel function with a kernel parameter gamma for + // each input variable. Described in "An Introduction to + // Support Vector Machines and Other Kernel-based Learning + // Methods" by Cristianini and Shawe-Taylor, Section 3.5 + std::vector *v1 = ev1->GetDataVector(); + std::vector *v2 = ev2->GetDataVector(); + if(fmGamma.size() != v1->size()){ + std::cout << "Fewer gammas than input variables! #Gammas= " << fmGamma.size() << " #Input variables= " << v1->size() << std::endl; + std::cout << "***> abort program execution" << std::endl; + exit(1); + } + + Float_t result = 1.; + for (UInt_t i = 0; i < v1->size(); i++) { + result *= TMath::Exp( -((*v1)[i] -(*v2)[i])*((*v1)[i] -(*v2)[i])*fmGamma[i] ); + } + return result; + } case kPolynomial: { + // Polynomial kernel of form (z.x + theta)^n + // it should be noted that the power is currently only integer std::vector *v1 = ev1->GetDataVector(); std::vector *v2 = ev2->GetDataVector(); Float_t prod = fTheta; @@ -95,14 +169,13 @@ Float_t TMVA::SVKernelFunction::Evaluate( SVEvent* ev1, SVEvent* ev2 ) Float_t result = 1.; Int_t i = fOrder; - for (; i > 0; i /= 2) { - if (i%2) result = prod; - prod *= prod; - } + result = TMath::Power(prod,i); return result; } case kLinear: { + // This is legacy code. The linear polynomial is a special case + // of the polynomial with order=1 and theta=0. std::vector *v1 = ev1->GetDataVector(); std::vector *v2 = ev2->GetDataVector(); Float_t prod = 0; @@ -110,7 +183,10 @@ Float_t TMVA::SVKernelFunction::Evaluate( SVEvent* ev1, SVEvent* ev2 ) return prod; } case kSigmoidal: - { + { + // This kernel doesn't always result in a positive-semidefinite Gram + // matrix so should be used with caution and therefore not + // currently accessible. This is not a valid Mercer kernel std::vector *v1 = ev1->GetDataVector(); std::vector *v2 = ev2->GetDataVector(); Float_t prod = 0; @@ -119,6 +195,37 @@ Float_t TMVA::SVKernelFunction::Evaluate( SVEvent* ev1, SVEvent* ev2 ) prod += fTheta; return TMath::TanH( prod ); } + case kProd: + { + // Calculate product of kernels by looping over list of kernels + // and evaluating the value for each, setting kernel back to + // kProd before returning so it can be used again. Described in "An Introduction to // Support Vector Machines and Other Kernel-based Learning + // Methods" by Cristianini and Shawe-Taylor, Section 3.3.2 + Float_t kernelVal; + kernelVal = 1; + for(UInt_t i = 0; i - IFJ PAN, Krakow, Poland * * (IFJ PAN: Henryk Niewodniczanski Inst. Nucl. Physics, Krakow, Poland) * * * +* Minor modification to improve optimisation of kernel values: * +* Adrian Bevan - Queen Mary * +* University of London, UK * +* Tom Stevenson - Queen Mary * +* University of London, UK * +* * * Copyright (c) 2005: * * CERN, Switzerland * * MPI-K Heidelberg, Germany * @@ -62,8 +68,9 @@ TMVA::SVKernelMatrix::SVKernelMatrix( std::vector* inputVectors, }catch(...){ Log() << kFATAL << "Input data too large. Not enough memory to allocate memory for Support Vector Kernel Matrix. Please reduce the number of input events or use a different method."<Evaluate((*inputVectors)[i], (*inputVectors)[i]); for (UInt_t j = 0; j <=i; j++) { fSVKernelMatrix[i][j] = fKernelFunction->Evaluate((*inputVectors)[i], (*inputVectors)[j]); }