Merge branch '2018-improvement' of...

Merge branch '2018-improvement' of https://gitlab.univ-nantes.fr/pigeau-a/2015-Hubble-UserProfiles.git into 2018-improvement

2015-Hubble-UserProfile/neuralNetworkManager/main.py

@@ -50,6 +50,7 @@ if __name__ == "__main__":
     #print("accuracy:"+str(accuracy))
     #print("confusion matrix:"+str(confusionMatrix))
     
+    #script.launchTestNTimes(Course.COURSE_ID_JAVA, groups, features=range(0,16), hiddenLayers=[32], ntime=10, whereToCut=None, whereToCutUnity=Sequence.PERIOD_TO_CUT_UNITY_PERCENT)
     
     script.launchTestForAllCoursesPeriod(groups)
     #script.exportLastResult()

2015-Hubble-UserProfile/neuralNetworkManager/scriptNeuralNetwork.py

@@ -182,7 +182,11 @@ class ScriptNeuralNetwork:
         print("Shape train X:"+str(train_x.shape))
         print("Shape test X:"+str(test_x.shape))
         print("course vocabulary:"+str(len(course.getSequenceId())))
-            
+        
+        ''' free the memory '''
+        dataLoader = None
+        
+         
         if len(groups) == 2:
             nn = DenseBinaryClass(numberOfClass=len(groups), filters=hiddenLayers, typeNN=typeNN)
         else:
@@ -248,6 +252,8 @@ class ScriptNeuralNetwork:
         nameCourse = Course.getCourseName(idCourse, summary=True)
         fileNameSavedResult = os.path.join(ScriptNeuralNetwork.DIRECTORY_EXPERIMENT, ScriptNeuralNetwork.NAME_FILE_RESULT+str(nameCourse)+'Session'+str(whereToCut)+"hiddenLayers="+str(hiddenLayers)+'Features'+featureStr+'.p')
         
+        print("fileName: "+str(fileNameSavedResult))
+        
         if os.path.exists(fileNameSavedResult):
             with open(fileNameSavedResult, "rb") as fileResult:
                 (accuracy, confusionMatrice, nbEpoch) = pickle.load(fileResult)
@@ -257,6 +263,8 @@ class ScriptNeuralNetwork:
 
             return (accuracy, confusionMatrice, nbEpoch)
         
+        #raise Exception('launchTestNTimes: not possible !!!!!')
+        
         for _ in range(0, ntime):
             (accuracy, confusionMatrix, nbEpoch) = self.launchTest(idCourse, groups, features, hiddenLayers, whereToCut=whereToCut, whereToCutUnity=whereToCutUnity)
             
@@ -308,8 +316,7 @@ class ScriptNeuralNetwork:
         
             for idCourse in Course.ID_COURSES:
                 if (idCourse != Course.COURSE_ID_API_REST and
-                    idCourse != Course.COURSE_ID_HTML_CSS and
-                    idCourse != Course.COURSE_ID_JAVA) : 
+                    idCourse != Course.COURSE_ID_HTML_CSS) : 
                     #and 
                     #idCourse != Course.COURSE_ID_JAVA_SCRIPT):
                     
@@ -357,7 +364,8 @@ class ScriptNeuralNetwork:
         
         features = range(0,16)
         hiddenLayers = [32]
-        whereToCuts = [75, 25, 50, None]
+        whereToCuts = [25, 50, 75, None]
+
         whereToCutUnity = Sequence.PERIOD_TO_CUT_UNITY_PERCENT#, (14, features), (21, features), (28, features), (None, features)]
         
         ''' one entry per course. Each value is a list of tuple [(accuracyGroup1, accuracyGroup2, ...), (...)]. One tuple per period'''
@@ -370,31 +378,32 @@ class ScriptNeuralNetwork:
             
             print("accuracies:"+str(accuracies))
             print("listScore:"+str(listScore))
-            
-#             for i, idCourse in enumerate(Course.ID_COURSES):
-#                 scoresCourse = dictAccuracyPerGroup.get(idCourse, [])
-#                 
-#                 if not scoresCourse:
-#                     dictAccuracyPerGroup[idCourse] = scoresCourse
-#                 
-#                 #(idCourse, accuracy, accuracyPerGroup) = accuraciesPerCourse[i]
-#                 scoresCourse.append((accuracies[i], listScore[i]))
-#         
-#         fileNameSavedResult = os.path.join(ScriptNeuralNetwork.DIRECTORY_EXPERIMENT, (ScriptNeuralNetwork.NAME_FILE_RESULT+
-#                                                                                       "whereTocut"+str(whereToCuts)+
-#                                                                                       "features="+str(features)+
-#                                                                                       "layers="+str(hiddenLayers)+
-#                                                                                       ".p"))
-#         with open(fileNameSavedResult, "wb") as fileResult:
-#             pickle.dump((accuracies, dictAccuracyPerGroup, whereToCuts, groups), fileResult) 
-#             
-#         fileNameResult = os.path.join(ScriptNeuralNetwork.DIRECTORY_EXPERIMENT, ("Latex"+
-#                                                                                  ScriptNeuralNetwork.NAME_FILE_RESULT+
-#                                                                                  "-whereTocut"+str(whereToCuts)+
-#                                                                                  "features="+str(features)+
-#                                                                                  "layers="+str(hiddenLayers)+
-#                                                                                  ".txt"))
-#         exportLatex.exportResult(fileNameResult, dictAccuracyPerGroup)
+            
+            for i, idCourse in enumerate(Course.ID_COURSES):
+                scoresCourse = dictAccuracyPerGroup.get(idCourse, [])
+                 
+                if not scoresCourse:
+                    dictAccuracyPerGroup[idCourse] = scoresCourse
+                 
+                #(idCourse, accuracy, accuracyPerGroup) = accuraciesPerCourse[i]
+                scoresCourse.append((accuracies[i], listScore[i]))
+         
+        fileNameSavedResult = os.path.join(ScriptNeuralNetwork.DIRECTORY_EXPERIMENT, (ScriptNeuralNetwork.NAME_FILE_RESULT+
+                                                                                      "whereTocut"+str(whereToCuts)+
+                                                                                      "features="+str(features)+
+                                                                                      "layers="+str(hiddenLayers)+
+                                                                                      ".p"))
+        with open(fileNameSavedResult, "wb") as fileResult:
+            pickle.dump((accuracies, dictAccuracyPerGroup, w, groups), fileResult) 
+             
+        fileNameResult = os.path.join(ScriptNeuralNetwork.DIRECTORY_EXPERIMENT, ("Latex"+
+                                                                                 ScriptNeuralNetwork.NAME_FILE_RESULT+
+                                                                                 "-whereTocut"+str(whereToCuts)+
+                                                                                 "features="+str(features)+
+                                                                                 "layers="+str(hiddenLayers)+
+                                                                                 ".txt"))
+        exportLatex.exportResult(fileNameResult, dictAccuracyPerGroup)
+
     
     def exportLastResult(self):
         
@@ -420,11 +429,12 @@ class ScriptNeuralNetwork:
                         [32, 16], [32, 32]]
 
         hiddenLayers = [[2], [4], [8], [16], [32], [4, 8], [8, 16], [16, 16], [16, 32], [32, 64], [64, 128]]
-        hiddenLayers = [[64], [32, 64], [32, 32], [64, 64], [64, 128], [128, 128]]
+        hiddenLayers = [[4], [8], [4, 4], [8, 4], [8, 8], [16, 8], [16, 16], [32, 16]]
+        hiddenLayers = [[32, 2], [32, 8]]
         #hiddenLayers = [[4]]
         
         for hidden in hiddenLayers:
-            self.launchTestForAllCourses(groups, features, hidden, whereToCut=25, whereToCutUnity=Sequence.PERIOD_TO_CUT_UNITY_PERCENT)
+            self.launchTestForAllCourses(groups, features, hidden, whereToCut=None, whereToCutUnity=Sequence.PERIOD_TO_CUT_UNITY_PERCENT)
         
     
 #     def testOnAllcourse(self, features, period = None, hiddenLayers = []):