#include <YMLL.h>
using namespace YMLL;

int main(int argc, char* argv[])
{
	// Get training and testing sets.
	// If there is no testing set, just use the training set as testing set.
	string trainFilename;
	cout << "Training set file (CSV): ";
	cin >> trainFilename;

	string testFilename;
	cout << "Testing set file (CSV): ";
	cin >> testFilename;

	TDSpecialDataLoad loadTrain(trainFilename);
	TDSpecialDataLoad loadTest(testFilename);
	Dataset train;
	Dataset test;
	train.Load(loadTrain);
	test.Load(loadTest);

	int choice;
	// Type of machine
	Machine *machine;
	bool isClassification;
	do 
	{
		cout << "(1) FFBPNN classification\n";
		cout << "(2) SVMStar\n";
		cout << "(3) SVMLight classification\n";
		cout << "(4) LibSVM classification\n";
		cout << "(5) KNN\n";
		cout << "(6) PNN\n";
		cout << "(7) PLS classification\n";
		cout << "(8) PCR\n";
		cout << "(9) PLS\n";
		cout << "(10) PLS (no intercept)\n";
		cout << "(11) Continuum power regression\n";
		cout << "(12) Continuum regression\n";
		cout << "(13) GRNN\n";
		cout << "(14) FFBPNN\n";
		cout << "(15) SVMLight regression\n";
		cout << "(16) LibSVM regression\n";
		cout << "(17) KNN regression\n";
		cin >> choice;
	} while (choice < 1 || choice > 17);
	switch (choice)
	{
		case 1:
		{	
			isClassification = true;

			size_t maxIteration = 1000;
			double noiseFactor = 0.0;
			double tolerance = 0.0001;
			double learningRate = 0.5;
			double momentum = 0.9;
			long seed = 1;

			int selection;
			do 
			{
				cout << "(1) Maximum iterations = " << maxIteration << "\n";
				cout << "(2) Noise factor = " << noiseFactor << "\n";
				cout << "(3) Tolerance = " << tolerance << "\n";
				cout << "(4) Learning rate = " << learningRate << "\n";
				cout << "(5) Momentum = " << momentum << "\n";
				cout << "(6) Random seed = " << seed << "\n";
				cout << "Parameter to change (0 to end)";
				cin >> selection;

				switch (selection)
				{
					case 1:
						cout << "Maximum iterations: ";
						cin >> maxIteration;
						break;

					case 2:
						cout << "Noise factor: ";
						cin >> noiseFactor;
						break;

					case 3:
						cout << "Tolerance: ";
						cin >> tolerance;
						break;

					case 4:
						cout << "Learning rate: ";
						cin >> learningRate;
						break;

					case 5:
						cout << "Momentum: ";
						cin >> momentum;
						break;

					case 6:
						cout << "Random seed: ";
						cin >> seed;
						break;
				}
			} while (selection != 0);
			machine = new FFBPNNClassificationMachine;
			machine->SetVariable("maximum iterations", maxIteration);
			machine->SetVariable("noise factor", noiseFactor);
			machine->SetVariable("tolerance", tolerance);
			machine->SetVariable("learning rate", learningRate);
			machine->SetVariable("momentum", momentum);
			machine->SetVariable("random seed", seed);		
			break;
		}

		case 2:
		{	
			isClassification = true;

			bool bias = false;
			size_t maxIterations = 30000;
			double maxGamma = 0.999999;
			double Cplus = numeric_limits<double>::max();
			double Cminus = numeric_limits<double>::max();
			int selection;
			do 
			{
				cout << "(1) Add bias = " << bias << "\n";
				cout << "(2) Max. iterations = " << maxIterations << "\n";
				cout << "(3) Max. gamma = " << maxGamma << "\n";
				cout << "(4) C plus = " << Cplus << "\n";
				cout << "(5) C minus = " << Cminus << "\n";
				cout << "Parameter to change (0 to end)";
				cin >> selection;

				switch (selection)
				{
					case 1:
						cout << "Add bias: ";
						cin >> bias;
						break;

					case 2:
						cout << "Max. iterations: ";
						cin >> maxIterations;
						break;

					case 3:
						cout << "Max. gamma: ";
						cin >> maxGamma;
						break;

					case 4:
						cout << "C plus: ";
						cin >> Cplus;
						break;

					case 5:
						cout << "C minus: ";
						cin >> Cminus;
						break;
				}
			} while (selection != 0);
			machine = new SVMStarMachine;
			machine->SetVariable("add bias", bias);
			machine->SetVariable("max iterations", maxIterations);
			machine->SetVariable("max gamma", maxGamma);
			machine->SetVariable("C plus", Cplus);
			machine->SetVariable("C minus", Cminus);
			break;
		}

		case 3:
		{	
			isClassification = true;

			long biasHyperplane = 1;
			long removeInconsistent= 0;
			long skipFinalOptCheck = 0;
			long maxQPSize = 10;
			long newVarsInQP = 0;
			long iterToShrink = -9999;
			long kernelCacheSize = 40;
			double C = 0.0;
			double EPS = 0.1;
			double transductionPosRatio = -1.0;
			double cost = 1.0;
			double epsilonCrit = 0.001;
			double rho = 1.0;
			long XADepth = 0;
			long computeLOO = 0;
			long kernel = 2;
			bool useSigma = true;

			int selection;
			do 
			{
				cout << "(1) Biased hyperplane = " << biasHyperplane << "\n";
				cout << "(2) Remove inconsistent = " << removeInconsistent << "\n";
				cout << "(3) Skip final opt check = " << skipFinalOptCheck << "\n";
				cout << "(4) Max QP size = " << maxQPSize << "\n";
				cout << "(5) New vars in QP = " << newVarsInQP << "\n";
				cout << "(6) Iter to shrink = " << iterToShrink << "\n";
				cout << "(7) Kernel cache size = " << kernelCacheSize << "\n";
				cout << "(8) C = " << C << "\n";
				cout << "(9) EPS = " << EPS << "\n";
				cout << "(10) Transduction pos ratio = " << transductionPosRatio << "\n";
				cout << "(11) Cost ratio = " << cost << "\n";
				cout << "(12) Epsilon crit = " << epsilonCrit << "\n";
				cout << "(13) Rho = " << rho << "\n";
				cout << "(14) XA depth = " << XADepth << "\n";
				cout << "(15) Compute LOO = " << computeLOO << "\n";
				cout << "(16) Kernel type = " << kernel << "\n";
				cout << "(17) Use sigma instead of gamma for Gaussian kernel? = " << useSigma << "\n";
				cout << "Parameter to change (0 to end)";
				cin >> selection;

				switch (selection)
				{
					case 1:
						cout << "Biased hyperplane: ";
						cin >> biasHyperplane;
						break;

					case 2:
						cout << "Remove inconsistent: ";
						cin >> removeInconsistent;
						break;

					case 3:
						cout << "Skip final opt check: ";
						cin >> skipFinalOptCheck;
						break;

					case 4:
						cout << "Max QP size: ";
						cin >> maxQPSize;
						break;

					case 5:
						cout << "New vars in QP: ";
						cin >> newVarsInQP;
						break;

					case 6:
						cout << "Iter to shrink: ";
						cin >> iterToShrink;
						break;

					case 7:
						cout << "Kernel cache size: ";
						cin >> kernelCacheSize;
						break;

					case 8:
						cout << "C: ";
						cin >> C;
						break;

					case 9:
						cout << "EPS: ";
						cin >> EPS;
						break;

					case 10:
						cout << "Transduction pos ratio: ";
						cin >> transductionPosRatio;
						break;

					case 11:
						cout << "Cost ratio: ";
						cin >> cost;
						break;

					case 12:
						cout << "Epsilon crit: ";
						cin >> epsilonCrit;
						break;

					case 13:
						cout << "Rho: ";
						cin >> rho;
						break;

					case 14:
						cout << "XA depth: ";
						cin >> XADepth;
						break;

					case 15:
						cout << "Compute LOO: ";
						cin >> computeLOO;
						break;

					case 16:
						cout << "Kernel type: ";
						cin >> kernel;
						break;

					case 17:
						cout << "Use sigma instead of gamma for Gaussian kernel?: ";
						cin >> useSigma;
						break;
				}
			} while (selection != 0);
			SVMLightMachine tempMachine;
			tempMachine.SetVariable("svm type", (long)1);
			tempMachine.SetVariable("biased hyperplane", biasHyperplane);
			tempMachine.SetVariable("remove inconsistent", removeInconsistent);
			tempMachine.SetVariable("skip final opt check", skipFinalOptCheck);
			tempMachine.SetVariable("max qp size", maxQPSize);
			tempMachine.SetVariable("new vars in qp", newVarsInQP);
			tempMachine.SetVariable("iter to shrink", iterToShrink);
			tempMachine.SetVariable("kernel cache size", kernelCacheSize);
			tempMachine.SetVariable("c", C);
			tempMachine.SetVariable("eps", EPS);
			tempMachine.SetVariable("transduction pos ratio", transductionPosRatio);
			tempMachine.SetVariable("cost ratio", cost);
			tempMachine.SetVariable("epsilon crit", epsilonCrit);
			tempMachine.SetVariable("rho", rho);
			tempMachine.SetVariable("xa depth", XADepth);
			tempMachine.SetVariable("compute loo", computeLOO);
			tempMachine.SetVariable("kernel type", kernel);
			tempMachine.ChangeGaussianParameter_ = useSigma;
			machine = tempMachine.ClonePtr();
			break;
		}

		case 4:
		{	
			isClassification = true;

			int kernel = 2;
			double cost = 1e38;
			double P = 0.001;
			double cacheSize = 40.0;
			double epsilon = 0.1;
			int shrink = 1;
			bool useSigma = true;
			double posWeight = 1.0;
			double negWeight = 1.0;

			int selection;
			do 
			{
				cout << "(1) Kernel type = " << kernel << "\n";
				cout << "(2) Cost = " << cost << "\n";
				cout << "(3) P = " << P << "\n";
				cout << "(4) Cache size = " << cacheSize << "\n";
				cout << "(5) Epsilon = " << epsilon << "\n";
				cout << "(6) Shrinking = " << shrink << "\n";
				cout << "(7) Use sigma instead of gamma for Gaussian kernel? = " << useSigma << "\n";
				cout << "(8) Class 1 weight = " << posWeight << "\n";
				cout << "(9) Class -1 weight = " << negWeight << "\n";
				cout << "Parameter to change (0 to end)";
				cin >> selection;

				switch (selection)
				{
					case 1:
						cout << "Kernel type: ";
						cin >> kernel;
						break;

					case 2:
						cout << "Cost: ";
						cin >> cost;
						break;

					case 3:
						cout << "P: ";
						cin >> P;
						break;

					case 4:
						cout << "Cache size: ";
						cin >> cacheSize;
						break;

					case 5:
						cout << "Epsilon: ";
						cin >> epsilon;
						break;

					case 6:
						cout << "Shrinking: ";
						cin >> shrink;
						break;

					case 7:
						cout << "Use sigma instead of gamma for Gaussian kernel?: ";
						cin >> useSigma;
						break;

					case 8:
						cout << "Class 1 weight: ";
						cin >> posWeight;
						break;

					case 9:
						cout << "Class -1 weight: ";
						cin >> negWeight;
						break;
				}
			} while (selection != 0);		
			LibSVMMachine tempMachine;
			tempMachine.SetVariable("svm type", 0);
			tempMachine.SetVariable("kernel type", kernel);
			tempMachine.SetVariable("cost", cost);
			tempMachine.SetVariable("p", P);
			tempMachine.SetVariable("cache size", cacheSize);
			tempMachine.SetVariable("epsilon", epsilon);
			tempMachine.SetVariable("shrinking", shrink);
			tempMachine.SetVariable("pos weight", posWeight);
			tempMachine.SetVariable("neg weight", negWeight);
			tempMachine.ChangeGaussianParameter_ = useSigma;
			machine = tempMachine.ClonePtr();
			break;
		}

		case 5:
		{	
			isClassification = true;
			machine = new KNNMachine;
			break;
		}

		case 6:
		{	
			isClassification = true;
			machine = new PNNMachine;	
			break;
		}

		case 7:
		{	
			isClassification = true;
			machine = new PLSClassifyMachine;
			break;
		}

		case 8:
		{	
			isClassification = false;
			machine = new PCRMachine;
			break;
		}

		case 9:
		{	
			isClassification = false;
			machine = new PLSMachine;
			break;
		}

		case 10:
		{	
			isClassification = false;
			machine = new PLSNoInterceptMachine;
			break;
		}

		case 11:
		{	
			isClassification = false;
			machine = new ContinuumPowerRegressionMachine;
			
			double power = 1.0;
			int selection;
			do 
			{
				cout << "(1) Power = " << power << "\n";
				cout << "Parameter to change (0 to end)";
				cin >> selection;

				switch (selection)
				{
					case 1:
						cout << "Power: ";
						cin >> power;
						break;
			}
			} while (selection != 0);
			machine->SetVariable("power", power);
			break;
		}

		case 12:
		{	
			isClassification = false;
			machine = new ContinuumRegressionMachine;

			double power = 0.5;
			int selection;
			do 
			{
				cout << "(1) Power = " << power << "\n";
				cout << "Parameter to change (0 to end)";
				cin >> selection;

				switch (selection)
				{
					case 1:
						cout << "Power: ";
						cin >> power;
						break;
				}
			} while (selection != 0);
			machine->SetVariable("power", power);
			break;
		}

		case 13:
		{	
			isClassification = false;
			machine = new GRNNMachine;
			break;
		}

		case 14:
		{	
			isClassification = false;

			size_t maxIteration = 1000;
			double noiseFactor = 0.0;
			double tolerance = 0.0001;
			double learningRate = 0.5;
			double momentum = 0.9;
			long seed = 1;

			int selection;
			do 
			{
				cout << "(1) Maximum iterations = " << maxIteration << "\n";
				cout << "(2) Noise factor = " << noiseFactor << "\n";
				cout << "(3) Tolerance = " << tolerance << "\n";
				cout << "(4) Learning rate = " << learningRate << "\n";
				cout << "(5) Momentum = " << momentum << "\n";
				cout << "(6) Random seed = " << seed << "\n";
				cout << "Parameter to change (0 to end)";
				cin >> selection;

				switch (selection)
				{
					case 1:
						cout << "Maximum iterations: ";
						cin >> maxIteration;
						break;

					case 2:
						cout << "Noise factor: ";
						cin >> noiseFactor;
						break;

					case 3:
						cout << "Tolerance: ";
						cin >> tolerance;
						break;

					case 4:
						cout << "Learning rate: ";
						cin >> learningRate;
						break;

					case 5:
						cout << "Momentum: ";
						cin >> momentum;
						break;

					case 6:
						cout << "Random seed: ";
						cin >> seed;
						break;
				}
			} while (selection != 0);				
			machine = new FFBPNNMachine;
			machine->SetVariable("maximum iterations", maxIteration);
			machine->SetVariable("noise factor", noiseFactor);
			machine->SetVariable("tolerance", tolerance);
			machine->SetVariable("learning rate", learningRate);
			machine->SetVariable("momentum", momentum);
			machine->SetVariable("random seed", seed);		
			break;
		}

		case 15:
		{	
			isClassification = false;

			long biasHyperplane = 1;
			long removeInconsistent= 0;
			long skipFinalOptCheck = 0;
			long maxQPSize = 10;
			long newVarsInQP = 0;
			long iterToShrink = -9999;
			long kernelCacheSize = 40;
			double C = 0.0;
			double EPS = 0.1;
			double transductionPosRatio = -1.0;
			double cost = 1.0;
			double epsilonCrit = 0.001;
			double rho = 1.0;
			long XADepth = 0;
			long computeLOO = 0;
			long kernel = 2;
			bool useSigma = true;

			int selection;
			do 
			{
				cout << "(1) Biased hyperplane = " << biasHyperplane << "\n";
				cout << "(2) Remove inconsistent = " << removeInconsistent << "\n";
				cout << "(3) Skip final opt check = " << skipFinalOptCheck << "\n";
				cout << "(4) Max QP size = " << maxQPSize << "\n";
				cout << "(5) New vars in QP = " << newVarsInQP << "\n";
				cout << "(6) Iter to shrink = " << iterToShrink << "\n";
				cout << "(7) Kernel cache size = " << kernelCacheSize << "\n";
				cout << "(8) C = " << C << "\n";
				cout << "(9) EPS = " << EPS << "\n";
				cout << "(10) Transduction pos ratio = " << transductionPosRatio << "\n";
				cout << "(11) Cost ratio = " << cost << "\n";
				cout << "(12) Epsilon crit = " << epsilonCrit << "\n";
				cout << "(13) Rho = " << rho << "\n";
				cout << "(14) XA depth = " << XADepth << "\n";
				cout << "(15) Compute LOO = " << computeLOO << "\n";
				cout << "(16) Kernel type = " << kernel << "\n";
				cout << "(17) Use sigma instead of gamma for Gaussian kernel? = " << useSigma << "\n";
				cout << "Parameter to change (0 to end)";
				cin >> selection;

				switch (selection)
				{
					case 1:
						cout << "Biased hyperplane: ";
						cin >> biasHyperplane;
						break;

					case 2:
						cout << "Remove inconsistent: ";
						cin >> removeInconsistent;
						break;

					case 3:
						cout << "Skip final opt check: ";
						cin >> skipFinalOptCheck;
						break;

					case 4:
						cout << "Max QP size: ";
						cin >> maxQPSize;
						break;

					case 5:
						cout << "New vars in QP: ";
						cin >> newVarsInQP;
						break;

					case 6:
						cout << "Iter to shrink: ";
						cin >> iterToShrink;
						break;

					case 7:
						cout << "Kernel cache size: ";
						cin >> kernelCacheSize;
						break;

					case 8:
						cout << "C: ";
						cin >> C;
						break;

					case 9:
						cout << "EPS: ";
						cin >> EPS;
						break;

					case 10:
						cout << "Transduction pos ratio: ";
						cin >> transductionPosRatio;
						break;

					case 11:
						cout << "Cost ratio: ";
						cin >> cost;
						break;

					case 12:
						cout << "Epsilon crit: ";
						cin >> epsilonCrit;
						break;

					case 13:
						cout << "Rho: ";
						cin >> rho;
						break;

					case 14:
						cout << "XA depth: ";
						cin >> XADepth;
						break;

					case 15:
						cout << "Compute LOO: ";
						cin >> computeLOO;
						break;

					case 16:
						cout << "Kernel type: ";
						cin >> kernel;
						break;

					case 17:
						cout << "Use sigma instead of gamma for Gaussian kernel?: ";
						cin >> useSigma;
						break;
				}
			} while (selection != 0);
			SVMLightMachine tempMachine;
			tempMachine.SetVariable("svm type", (long)2);
			tempMachine.SetVariable("biased hyperplane", biasHyperplane);
			tempMachine.SetVariable("remove inconsistent", removeInconsistent);
			tempMachine.SetVariable("skip final opt check", skipFinalOptCheck);
			tempMachine.SetVariable("max qp size", maxQPSize);
			tempMachine.SetVariable("new vars in qp", newVarsInQP);
			tempMachine.SetVariable("iter to shrink", iterToShrink);
			tempMachine.SetVariable("kernel cache size", kernelCacheSize);
			tempMachine.SetVariable("c", C);
			tempMachine.SetVariable("eps", EPS);
			tempMachine.SetVariable("transduction pos ratio", transductionPosRatio);
			tempMachine.SetVariable("cost ratio", cost);
			tempMachine.SetVariable("epsilon crit", epsilonCrit);
			tempMachine.SetVariable("rho", rho);
			tempMachine.SetVariable("xa depth", XADepth);
			tempMachine.SetVariable("compute loo", computeLOO);
			tempMachine.SetVariable("kernel type", kernel);
			tempMachine.ChangeGaussianParameter_ = useSigma;
			machine = tempMachine.ClonePtr();
			break;
		}

		case 16:
		{	
			isClassification = false;
			
			int kernel = 2;
			double cost = 1e38;
			double P = 0.001;
			double cacheSize = 40.0;
			double epsilon = 0.1;
			int shrink = 1;
			bool useSigma = true;

			int selection;
			do 
			{
				cout << "(1) Kernel type = " << kernel << "\n";
				cout << "(2) Cost = " << cost << "\n";
				cout << "(3) P = " << P << "\n";
				cout << "(4) Cache size = " << cacheSize << "\n";
				cout << "(5) Epsilon = " << epsilon << "\n";
				cout << "(6) Shrinking = " << shrink << "\n";
				cout << "(7) Use sigma instead of gamma for Gaussian kernel? = " << useSigma << "\n";
				cout << "Parameter to change (0 to end)";
				cin >> selection;

				switch (selection)
				{
					case 1:
						cout << "Kernel type: ";
						cin >> kernel;
						break;

					case 2:
						cout << "Cost: ";
						cin >> cost;
						break;

					case 3:
						cout << "P: ";
						cin >> P;
						break;

					case 4:
						cout << "Cache size: ";
						cin >> cacheSize;
						break;

					case 5:
						cout << "Epsilon: ";
						cin >> epsilon;
						break;

					case 6:
						cout << "Shrinking: ";
						cin >> shrink;
						break;

					case 7:
						cout << "Use sigma instead of gamma for Gaussian kernel?: ";
						cin >> useSigma;
						break;
				}
			} while (selection != 0);		
			LibSVMMachine tempMachine;
			tempMachine.SetVariable("svm type", 3);
			tempMachine.SetVariable("kernel type", kernel);
			tempMachine.SetVariable("cost", cost);
			tempMachine.SetVariable("p", P);
			tempMachine.SetVariable("cache size", cacheSize);
			tempMachine.SetVariable("epsilon", epsilon);
			tempMachine.SetVariable("shrinking", shrink);
			tempMachine.ChangeGaussianParameter_ = useSigma;
			machine = tempMachine.ClonePtr();
			break;
		}

		case 17:
		{	
			isClassification = false;
			machine = new KNNRegressionMachine;
			break;
		}
	}
	int machineChoice = choice;

	// Set up trainer.
	TrainerSingleParameter trainer;
	trainer.SetDataset(train);
	
	typedef TrainerSingleParameter::FunctionT FunctionT;
	OptimizerSingleParameter<FunctionT> *optimizer;

	do 
	{
		cout << "(1) Increment\n";
		cout << "(2) Golden section\n";
		cin >> choice;
	} while (choice < 1 || choice > 2);

	switch (choice)
	{
		case 1:
		{
			double parameterStart = 1;
			double parameterEnd = 30;
			double increment = 1;
			int selection;
			do 
			{
				cout << "(1) Start value for parameter = " << parameterStart << "\n";
				cout << "(2) End value for parameter = " << parameterEnd << "\n";
				cout << "(3) Increment per iteration = " << increment << "\n";
				cout << "Parameter to change (0 to end)";
				cin >> selection;

				switch (selection)
				{
					case 1:
						cout << "Start value for parameter: ";
						cin >> parameterStart;
						break;

					case 2:
						cout << "End value for parameter: ";
						cin >> parameterEnd;
						break;

					case 3:
						cout << "Increment per iteration: ";
						cin >> increment;
						break;
				}
			} while (selection != 0);
			optimizer = new Increment<FunctionT>(*(trainer.f_));
			static_cast<Increment<FunctionT>*>(optimizer)->SetStepSize(increment);
			optimizer->SetBounds(parameterStart, parameterEnd);
			break;
		}

		case 2:
		{
			double parameterStart = 1e-6;
			double parameterEnd = 30;
			int selection;
			do 
			{
				cout << "(1) Start value for parameter = " << parameterStart << "\n";
				cout << "(2) End value for parameter = " << parameterEnd << "\n";
				cout << "Parameter to change (0 to end)";
				cin >> selection;

				switch (selection)
				{
					case 1:
						cout << "Start value for parameter: ";
						cin >> parameterStart;
						break;

					case 2:
						cout << "End value for parameter: ";
						cin >> parameterEnd;
						break;
				}
			} while (selection != 0);
			optimizer = new GoldenSection<FunctionT>(*(trainer.f_));
			optimizer->SetBounds(parameterStart, parameterEnd);
			break;
		}
	}

	optimizer->Initialize();
	trainer.SetOptimizer(*optimizer);
	delete optimizer;

	// Type of performance measurement	
	PerformanceMeasurer *performancemeasurer;
	string performanceMeasurerType;
	if (isClassification)
	{
		do 
		{
			cout << "(1) Absolute error\n";
			cout << "(2) Relative error\n";
			cout << "(3) Matthews correlation coefficient\n";
			cout << "(4) Cohen Kappa Coefficient\n";
			cin >> choice;
		} while (choice < 1 || choice > 4);

		if (choice == 1)
		{
			performancemeasurer = new AbsoluteErrorRatePerformanceMeasurer;
			performanceMeasurerType = "Absolute error";
		}
		else if (choice == 2)
		{
			performancemeasurer = new RelativeErrorRatePerformanceMeasurer;
			performanceMeasurerType = "Relative error";
		}
		else if (choice == 3)
		{
			performancemeasurer = new MatthewsCorrelationCoefficientPerformanceMeasurer;
			performanceMeasurerType = "Matthews cc";
		}
		else if (choice == 4)
		{
			performancemeasurer = new CohenKappaCoefficientPerformanceMeasurer;
			performanceMeasurerType = "Cohen kappa";
		}
	}
	else
	{
		do 
		{
			cout << "(1) Coefficient of determination\n";
			cout << "(2) Adjusted coefficient of determination\n";
			cout << "(3) Pearson r square\n";
			cout << "(4) Spearman rho\n";
			cout << "(5) Mean square error\n";
			cout << "(6) Root mean square error\n";
			cout << "(7) Standard deviation\n";
			cin >> choice;
		} while (choice < 1 || choice > 7);
		switch (choice)
		{
			case 1:
				performancemeasurer = new CoefficientOfDeterminationPerformanceMeasurer;
				performanceMeasurerType = "R^2";
				break;

			case 2:
				performancemeasurer = new AdjustedCoefficientOfDeterminationPerformanceMeasurer;
				performanceMeasurerType = "AdjR^2";
				break;

			case 3:
				performancemeasurer = new PearsonR2PerformanceMeasurer;
				performanceMeasurerType = "Pearson R^2";
				break;

			case 4:
				performancemeasurer = new SpearmanRhoPerformanceMeasurer;
				performanceMeasurerType = "Spearman rho";
				break;

			case 5:
				performancemeasurer = new MeanSquareErrorPerformanceMeasurer;
				performanceMeasurerType = "MSE";
				break;

			case 6:
				performancemeasurer = new RootMeanSquareErrorPerformanceMeasurer;
				performanceMeasurerType = "RMSE";
				break;

			case 7:
				performancemeasurer = new StandardDeviationPerformanceMeasurer;
				performanceMeasurerType = "Standard error";
				break;
		}
	}
	bool findMin = performancemeasurer->lessIsBetter_;

	// Type of validation method
	do 
	{
		cout << "(1) Training\n";
		cout << "(2) Testing\n";
		cout << "(3) LOO\n";
		cout << "(4) Boostrap\n";
		cout << "(5) NFoldCV\n";
		cout << "(6) StratifiedNFoldCV\n";
		cout << "(7) Random subsampling\n";
		cin >> choice;
	} while (choice < 1 || choice > 7);
	ObjectiveFunction *OF;
	switch (choice)
	{
		case 1: // Training
			OF = new FixedTestingSetObjectiveFunction;
			static_cast<FixedTestingSetObjectiveFunction*>(&*OF)->test_ = train;
			break;

		case 2: // Testing
			OF = new FixedTestingSetObjectiveFunction;
			static_cast<FixedTestingSetObjectiveFunction*>(&*OF)->test_ = test;
			break;

		case 3: // LOO
		{
			OF = new ObjectiveFunction;
			LOODatasetSplit loo;
			OF->SetOuterLoopDatasetSplit(loo);
			break;
		}
		case 4: // Bootstrap
		{
			OF = new ObjectiveFunction;
			BootStrapDatasetSplit bootstrap;
			OF->SetOuterLoopDatasetSplit(bootstrap);
			size_t repeat;
			cout << "No. of times to repeat validation: ";
			cin >> repeat;
			OF->SetVariable("outer loop repeat", (size_t)repeat);
			break;
		}
		case 5: // N Fold CV
		{	
			OF = new ObjectiveFunction;
			NFoldCVDatasetSplit cv;
			OF->SetOuterLoopDatasetSplit(cv);
			size_t folds;
			cout << "No. of folds: ";
			cin >> folds;
			size_t repeat;
			cout << "No. of times to repeat validation: ";
			cin >> repeat;
			OF->SetVariable("outer loop repeat", (size_t)repeat);
			OF->SetVariable("outer loop parameter", (double)folds);
			break;
		}
		case 6: // Stratified N Fold CV
		{
			OF = new ObjectiveFunction;
			StratifiedNFoldCVDatasetSplit scv;
			OF->SetOuterLoopDatasetSplit(scv);
			size_t folds;
			cout << "No. of folds: ";
			cin >> folds;
			size_t repeat;
			cout << "No. of times to repeat validation: ";
			cin >> repeat;
			OF->SetVariable("outer loop repeat", (size_t)repeat);
			OF->SetVariable("outer loop parameter", (double)folds);
			break;
		}	
		case 7: // Random sampling
		{
			OF = new ObjectiveFunction;
			RandomDatasetSplit random;
			OF->SetOuterLoopDatasetSplit(random);
			double proportion;
			cout << "Proportion of testing set: ";
			cin >> proportion;
			size_t repeat;
			cout << "No. of times to repeat validation: ";
			cin >> repeat;
			OF->SetVariable("outer loop repeat", (size_t)repeat);
			OF->SetVariable("outer loop parameter", (double)(size_t(proportion*train.size())));
			break;
		}
	}
	OF->SetMachine(*machine);
	delete machine;
	if (machineChoice == 1 || machineChoice == 14) // FFBPNN
	{
		CADEXDatasetSplit cadex;
		OF->SetInnerLoopDatasetSplit(cadex);
		OF->SetVariable("inner loop parameter", (double)(size_t(0.2*train.size())));
	}

	// Type of objective function
	do 
	{
		cout << "(1) Validation\n";
		cout << "(2) TSET +/- X*|TSET-CVSET|\n";
		cout << "(3) TSET +/- X*(|TSET-CVSET|+|TSET-VSET|)\n";
		cout << "(4) (CVSET + X*TSET)/(1+X)\n";
		cin >> choice;
	} while (choice < 1 || choice > 3);
	ObjectiveFunction *objectiveFunction;
	switch (choice)
	{
		case 1:
			objectiveFunction = OF->ClonePtr();
			objectiveFunction->SetPerformanceMeasurer(*performancemeasurer);
			break;

		case 2: // TSET +/- X*|TSET - CVSET|
		{
			CombinedObjectiveFunction of;
			of.SetPerformanceMeasurer(*performancemeasurer);

			FixedTestingSetObjectiveFunction tset;
			tset.SetMachine(*machine);
			tset.test_ = train;
			if (machineChoice == 1 || machineChoice == 14) //FFBPNN
			{
				CADEXDatasetSplit cadex;
				tset.SetInnerLoopDatasetSplit(cadex);
				tset.SetVariable("inner loop parameter", (double)(size_t(0.2*train.size())));
			}

			double weight;
			cout << "Weight: ";
			cin >> weight;

			of.SetObjectiveFunction(tset, 1.0);
			of.SetObjectiveFunction(*OF, findMin ? weight : -weight);
			of.SetFunction(CombinedObjectiveFunction::FUNCTION1);

			objectiveFunction = of.ClonePtr();
			break;
		}
		case 3: // TSET +/- X1*|TSET - VSET| +/- X2|TSET - CVSET|
		{	
			CombinedObjectiveFunction of;
			of.SetPerformanceMeasurer(*performancemeasurer);

			FixedTestingSetObjectiveFunction tset;
			tset.SetMachine(*machine);
			tset.test_ = train;

			FixedTestingSetObjectiveFunction vset;
			vset.SetMachine(*machine);
			vset.test_ = test;

			if (machineChoice == 1 || machineChoice == 14) // FFBPNN
			{
				CADEXDatasetSplit cadex;
				tset.SetInnerLoopDatasetSplit(cadex);
				tset.SetVariable("inner loop parameter", (double)(size_t(0.2*train.size())));
				vset.SetInnerLoopDatasetSplit(cadex);
				vset.SetVariable("inner loop parameter", (double)(size_t(0.2*train.size())));
			}

			double weight;
			cout << "Weight: ";
			cin >> weight;

			of.SetObjectiveFunction(tset, 1.0);
			of.SetObjectiveFunction(vset, findMin ? weight : -weight);
			of.SetObjectiveFunction(*OF, findMin ? weight : -weight);
			of.SetFunction(CombinedObjectiveFunction::FUNCTION1);

			objectiveFunction = of.ClonePtr();
			break;
		}
		case 4: // (CVSET + X*TSET)/(1+X)
		{
			CombinedObjectiveFunction of;
			of.SetPerformanceMeasurer(*performancemeasurer);

			FixedTestingSetObjectiveFunction tset;
			tset.SetMachine(*machine);
			tset.test_ = train;
			if (machineChoice == 3 || machineChoice == 22)
			{
				CADEXDatasetSplit cadex;
				tset.SetInnerLoopDatasetSplit(cadex);
				tset.SetVariable("inner loop parameter", (double)(size_t(0.2*train.size())));
			}

			double weight;
			cout << "Weight: ";
			cin >> weight;

			of.SetObjectiveFunction(*OF, 1.0);
			of.SetObjectiveFunction(tset, weight);
			of.SetFunction(CombinedObjectiveFunction::FUNCTION3);

			objectiveFunction = of.ClonePtr();
			break;
		}
	}
	trainer.findMin_ = findMin ? 1.0 : -1.0;
	trainer.SetObjectiveFunction(*objectiveFunction);
	delete performancemeasurer;
	delete OF;
	delete objectiveFunction;

	// Start trainer.
	size_t iter = 1;
	string output = "Iteration " + ntos<string>(iter) + ", " + 
					"Parameter: " + 
					ntos<string>(trainer.GetCurrentParameter()) + ", " +
					performanceMeasurerType + ": ";
	while (trainer.Iteration())
	{
		output += ntos<string>(trainer.error_) + "\n";
		cout << output;
		output = "Iteration " + ntos<string>(++iter) + ", " + 
				 "Parameter: " + 
				 ntos<string>(trainer.GetCurrentParameter()) + ", " +
				 performanceMeasurerType + ": ";
	}

	string remarks = "Best parameter: " + ntos<string>(trainer.GetOptimalParameter()) + ", " +
					 performanceMeasurerType + ": " + ntos<string>(trainer.minError_) + "\n";
	cout << remarks;

	return 0;
}