Commit 609d5791 authored by Antoine PIGEAU's avatar Antoine PIGEAU
Browse files

update experiment + export latex done

parent 88d2aaa4
......@@ -152,7 +152,7 @@ class Classifier(ClassifierInterface):
typeFeatureGroup = ConstantFeature.TYPE_FEATURE_GROUP,
features = None,
featuresByCourse= None,
featuresByPeriod = ConstantFeature.IMPLEMENTED_FEATURES_FORWARD_BEST_MERGED_LR)
featuresByPeriod = ConstantFeature.BEST_LR_FEATURE_MERGED_ALL_PERIODS)
@staticmethod
def getClassifierAdaBoostMerge():
......
......@@ -29,6 +29,7 @@ from featureManager.oulad.constant import Constant as ConstantFeature
from featureManager.oulad.featureMultiGroup import FeatureMultiGroup
from classifierManager.classifier import Classifier as ClassifierInterface
from classifierManager.constant import Constant as ConstantClassifier
from project.projectParameters import ProjectParameters
......@@ -81,3 +82,80 @@ class Classifier(ClassifierInterface):
def getFeatureMultiGroup(self):
return FeatureMultiGroup()
@staticmethod
def getClassifierLR():
return Classifier.getClassifier(model = ConstantClassifier.LogisticRegression,
hiddenLayers = None,
typeFeatureGroup = ConstantFeature.TYPE_FEATURE_GROUP,
features = None,
featuresByCourse = ConstantFeature.BEST_LR_FEATURE_ALL_COURSES,
featuresByPeriod = None)
@staticmethod
def getClassifierAdaBoost():
return Classifier.getClassifier(model = ConstantClassifier.AdaBoost,
hiddenLayers = None,
typeFeatureGroup = ConstantFeature.TYPE_FEATURE_GROUP,
features = ConstantFeature.IMPLEMENTED_FEATURES,
featuresByCourse = None,
featuresByPeriod = None)
@staticmethod
def getClassifierRandomForest():
return Classifier.getClassifier(model = ConstantClassifier.RandomForest,
hiddenLayers = None,
typeFeatureGroup = ConstantFeature.TYPE_FEATURE_GROUP,
features = ConstantFeature.IMPLEMENTED_FEATURES,
featuresByCourse = None,
featuresByPeriod = None)
@staticmethod
def getClassifierPerceptron():
return Classifier.getClassifier(model = ConstantClassifier.Perceptron,
hiddenLayers = ConstantFeature.HIDDEN_LAYERS_BEST_NN_PERCEPTRON,
typeFeatureGroup = ConstantFeature.TYPE_FEATURE_GROUP,
features = ConstantFeature.IMPLEMENTED_FEATURES,
featuresByCourse = None,
featuresByPeriod = None)
@staticmethod
def getClassifierLRMerge():
return Classifier.getClassifier(model = ConstantClassifier.LogisticRegression,
directory = Classifier.DIRECTORY_EXPERIMENT_MERGED,
hiddenLayers = None,
typeFeatureGroup = ConstantFeature.TYPE_FEATURE_GROUP,
features = None,
featuresByCourse= None,
featuresByPeriod = ConstantFeature.BEST_LR_FEATURE_MERGED_ALL_PERIODS)
@staticmethod
def getClassifierAdaBoostMerge():
return Classifier.getClassifier(model = ConstantClassifier.AdaBoost,
directory = Classifier.DIRECTORY_EXPERIMENT_MERGED,
hiddenLayers = None,
typeFeatureGroup = ConstantFeature.TYPE_FEATURE_GROUP,
features = ConstantFeature.IMPLEMENTED_FEATURES,
featuresByCourse= None,
featuresByPeriod = None)
@staticmethod
def getClassifierRandomForestMerge():
return Classifier.getClassifier(model = ConstantClassifier.RandomForest,
directory = Classifier.DIRECTORY_EXPERIMENT_MERGED,
hiddenLayers = None,
typeFeatureGroup = ConstantFeature.TYPE_FEATURE_GROUP,
features = ConstantFeature.IMPLEMENTED_FEATURES,
featuresByCourse= None,
featuresByPeriod = None)
@staticmethod
def getClassifierPerceptronMerge():
return Classifier.getClassifier(model = ConstantClassifier.Perceptron,
hiddenLayers = ConstantFeature.HIDDEN_LAYERS_BEST_NN_PERCEPTRON_MERGED,
typeFeatureGroup = ConstantFeature.TYPE_FEATURE_GROUP,
features = ConstantFeature.IMPLEMENTED_FEATURES,
featuresByCourse = None,
featuresByPeriod = None)
......@@ -230,8 +230,32 @@ def exportTransfertLearningResult(fileName, dictResult):
Export the result for the transfert learning experiments
@param fileName: file to export the result
@param dictResult: result to print dictResult[idCourse][timeToCut] = [avgAuc, AvgAccuracy, #courseImproved]
@param dictResult: result to print dictResult[algo][timeToCut] = [avgAuc, AvgAccuracy, #courseImproved]
'''
pass
with open(fileName, 'w') as fileResult:
for model in dictResult.keys():
print(model)
resultModel = dictResult[model]
fileResult.write(model+ " && ")
for t in [25, 50, 75, 100]:
auc, accuracy, number = resultModel[t]
auc = round(auc,2)
accuracy = round(accuracy, 2)
fileResult.write(str(auc)+" & "+str(accuracy)+" & "+str(number))
if(t != 100):
fileResult.write(" && " )
else:
fileResult.write("\\\\" )
fileResult.write("\n\\myClineAUC \n")
......@@ -73,15 +73,25 @@ class Constant(ConstantInterface):
IMPLEMENTED_FEATURES_BEST_BACKWARD_LR_MERGED_ALL_COURSES_25 = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 30, 32, 33, 34, 35, 36]
IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_25 = [30, 35, 34, 11, 33]
IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_50 = [27, 32, 9, 22]
IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_75 = [27, 28, 32, 9, 10, 35, 5, 15]
IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_100 = [27, 1, 7, 23]
IMPLEMENTED_FEATURES_FORWARD_BEST_MERGED_LR = { 25 : IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_25,
50 : IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_50,
75 : IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_75,
100 : IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_100}
# IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_25 = [30, 35, 34, 11, 33]
# IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_50 = [27, 32, 9, 22]
# IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_75 = [27, 28, 32, 9, 10, 35, 5, 15]
# IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_100 = [27, 1, 7, 23]
#
# IMPLEMENTED_FEATURES_FORWARD_BEST_MERGED_LR = { 25 : IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_25,
# 50 : IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_50,
# 75 : IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_75,
# 100 : IMPLEMENTED_FEATURES_BEST_FORWARD_LR_MERGED_ALL_COURSES_100}
#
BEST_LR_FEATURE_MERGED_25 = [30, 35, 34, 11, 33]
BEST_LR_FEATURE_MERGED_50 = [27, 32, 9, 22]
BEST_LR_FEATURE_MERGED_75 = [27, 28, 32, 9, 10, 35, 5, 15]
BEST_LR_FEATURE_MERGED_100 = [27, 1, 7, 23]
BEST_LR_FEATURE_MERGED_ALL_PERIODS = {25 : BEST_LR_FEATURE_MERGED_25,
50 : BEST_LR_FEATURE_MERGED_50,
75 : BEST_LR_FEATURE_MERGED_75,
100 : BEST_LR_FEATURE_MERGED_100}
......
......@@ -46,7 +46,7 @@ if __name__ == "__main__":
startTime = datetime.now()
dictClassifiers = {}
dictResult = {}
dictOutput = {}
dictClassifiers[ConstantClassifier.AdaBoost] = (Classifier.getClassifierAdaBoostMerge(), Classifier.getClassifierAdaBoost())
dictClassifiers[ConstantClassifier.RandomForest] = (Classifier.getClassifierRandomForestMerge(), Classifier.getClassifierRandomForest())
......@@ -121,7 +121,7 @@ if __name__ == "__main__":
#print('aucs: ', aucs)
dictResult[key] = {}
dictOutput[key] = {}
for t in [25, 50, 75, 100]:
......@@ -144,14 +144,14 @@ if __name__ == "__main__":
print(key+" Course Improved with transfert learning #[", len(coursesImproved),", ", np.mean(howBetterAucs), ", ", np.mean(howBetterAccuracys) ,"] : " , coursesImproved)
dictResult[key][t] = [np.mean(howBetterAucs), np.mean(howBetterAccuracys), len(coursesImproved)]
dictOutput[key][t] = [np.mean(howBetterAucs), np.mean(howBetterAccuracys), len(coursesImproved)]
fileNameResult = os.path.join(ProjectParameters.DIRECTORY_EXPERIMENT,
ProjectParameters.DIRECTORY_OPEN_CLASS_ROOM,
"LatexTransfertLeanrning.txt")
"LatexTransfertLearning.txt")
exportLatex.exportTransfertLearningResult(fileNameResult, dictResult)
exportLatex.exportTransfertLearningResult(fileNameResult, dictOutput)
stopTime = datetime.now()
print(("end of the classification task ("+str(stopTime-startTime)+")"))
......
# -*- coding: UTF-8 -*-
'''
Created on 29-Apr-2014
This file is part of Hubble-UserProfile.
Hubble-UserProfile is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Hubble-UserProfile is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with Hubble-UserProfile. If not, see <http://www.gnu.org/licenses/>.
@author: Antoine Pigeau
'''
import os
import numpy as np
import scipy.stats
from datetime import datetime
from project.projectParameters import ProjectParameters
from classifierManager.oulad.classifier import Classifier
from classifierManager.constant import Constant as ConstantClassifier
from exportManager import exportLatex
from featureManager.ocr.constant import Constant as ConstantFeature
from model.oulad.constant import Constant as ConstantModel
from model.oulad.course import Course
from model.mergedCourse import MergedCourse
if __name__ == "__main__":
print("start of the process boosting")
startTime = datetime.now()
dictClassifiers = {}
dictOutput = {}
dictClassifiers[ConstantClassifier.AdaBoost] = (Classifier.getClassifierAdaBoostMerge(), Classifier.getClassifierAdaBoost())
dictClassifiers[ConstantClassifier.RandomForest] = (Classifier.getClassifierRandomForestMerge(), Classifier.getClassifierRandomForest())
dictClassifiers[ConstantClassifier.Perceptron] = (Classifier.getClassifierPerceptronMerge(), Classifier.getClassifierPerceptron())
dictClassifiers[ConstantClassifier.LogisticRegression] = (Classifier.getClassifierLRMerge(), Classifier.getClassifierLR())
for key in dictClassifiers.keys():
classifierMerged, classifier = dictClassifiers[key]
'''
Prediction Merge
'''
courses = []
dictCourses = {}
for idCourse in ConstantModel.ID_COURSES:
course = Course(idCourse, granularity=ConstantModel.CHAPTER)
courses.append(course)
dictCourses[idCourse] = course
mergedCourse = MergedCourse(courses)
dictResultMerged = classifierMerged.predictionTaskForMergedForAllPeriods(mergedCourse, ntime=10, cache=True)
'''
Prediction
'''
dictResult = classifier.predictionTaskForAllPeriods(ntime=10, cache=True)
'''
Statistic test
'''
pvalues = {}
for t in [25, 50, 75, 100]:
pvalues[t] = {}
for idCourse in ConstantModel.ID_COURSES:
#print('dictResultMerged[t]: ', dictResultMerged[t])
resultAllCoursesMerged = dictResultMerged[t].dictAllCourses
resultsAllCourses = dictResult[t].dictAllCourses
aucsMerged = resultAllCoursesMerged[idCourse].aucs
aucs = resultsAllCourses[idCourse].aucs
accuraciesMerged = resultAllCoursesMerged[idCourse].accuracies
accuracies = resultAllCoursesMerged[idCourse].accuracies
pvalue = scipy.stats.ttest_ind(aucsMerged, aucs, equal_var=True)
avgAucs = np.nanmean(aucs)
avgAucsMerged = np.nanmean(aucsMerged)
avgAccuracies = np.nanmean(accuracies)
avgAccuraciesMerged = np.nanmean(aucsMerged)
pvalues[t][idCourse] = (pvalue.pvalue, avgAucsMerged > avgAucs, np.abs(avgAucsMerged - avgAucs), np.abs(avgAccuraciesMerged - avgAccuracies))
#print('aucsMerged: ', aucsMerged)
#print('aucs: ', aucs)
dictOutput[key] = {}
for t in [25, 50, 75, 100]:
print("---- traces cut at ", t)
coursesImproved = []
howBetterAucs = []
howBetterAccuracys = []
for idCourse in ConstantModel.ID_COURSES:
(pvalue, isBetter, howBetterAuc, howBetterAccuracy) = pvalues[t][idCourse]
print(idCourse, " : ", pvalue)
if(pvalue < 0.05 and isBetter):
coursesImproved.append((idCourse, howBetterAuc, howBetterAccuracy))
howBetterAucs.append(howBetterAuc)
howBetterAccuracys.append(howBetterAccuracy)
print(key+" Course Improved with transfert learning #[", len(coursesImproved),", ", np.mean(howBetterAucs), ", ", np.mean(howBetterAccuracys) ,"] : " , coursesImproved)
dictOutput[key][t] = [np.mean(howBetterAucs), np.mean(howBetterAccuracys), len(coursesImproved)]
fileNameResult = os.path.join(ProjectParameters.DIRECTORY_EXPERIMENT,
ProjectParameters.DIRECTORY_OPEN_CLASS_ROOM,
"LatexTransfertLearning.txt")
exportLatex.exportTransfertLearningResult(fileNameResult, dictOutput)
stopTime = datetime.now()
print(("end of the classification task ("+str(stopTime-startTime)+")"))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment