Adding evaluation scripts to simplify the evaluation process

evaluation-scripts/generate-all

+#!/bin/bash
+
+DIR=`dirname $0`
+
+BENCHTOOL="$DIR/../fr.inria.diverse.cloning.benchmark.tool.product/target/products/fr.inria.diverse.cloning.benchmark.tool.product/linux/gtk/x86_64/cloning-benchmark"
+MMGENERATOR="$BENCHTOOL mmgen"
+MATERIALGENERATOR="$BENCHTOOL matgen"
+
+OUTPUTFOLDER=./benchmarktool-workingdir
+
+
+for i in {1..10}
+do
+    $MMGENERATOR --maxClasses 50  --maxProp 250 --maxMutProp 50 --nbMetamodels 10 --outputFolder=$OUTPUTFOLDER/$i
+    $MATERIALGENERATOR  --metamodelsFolder=$OUTPUTFOLDER/$i --outputFile=$OUTPUTFOLDER/bundle$i.jar # --outputProjectFolder=./$i/generated-project    
+done

evaluation-scripts/plot-all

+#!/bin/bash
+
+RARGS=""
+for i in "$@"
+do
+    RARGS="$RARGS $i"
+done
+
+R --no-save --args $RARGS < ./process-data.R
+
+ls *.pdf | xargs -I '{}' pdfcrop '{}' '{}'
+
+mv *.pdf benchmarktool-workingdir

evaluation-scripts/process-data.R

+library(reshape2)
+library(scales)
+
+
+
+###
+# To prepare the ploting with a file of a name and a format (svg of pdf)
+###
+initImage <- function(name, extension) {
+
+    #par(mar=c(0,0,0,0), mgp=c(0,0,0,0))
+
+    if(extension=="svg")
+        svg(filename=paste(name,".svg", sep=""))
+    else
+        pdf(file=paste(name,".pdf", sep=""))
+}
+
+
+###
+# Retrieves in a df the memory used in a given run
+###
+getValue <- function(data, mmname, cloningname, nbclones, column) {
+        value = data[data$MetamodelName==mmname & data$CloningTechnique==cloningname & data$NumberOfClones==nbclones, c(column)]
+        return(value)
+}
+
+
+
+###  
+# Computes and return an operation that, for a given row, 
+# computes memory gain with regards to another cloning technique
+###
+computeGainCreator <- function(rawdata, cloningName, column) {
+
+    numberClonesColumn = which(colnames(rawdata)=="NumberOfClones")
+    totalMemoryColumn = which(colnames(rawdata)==column)
+    nameColumn = which(colnames(rawdata)=="MetamodelName")
+    computeMemoryRatio <- function(row) {
+        totalMemory <- as.integer(row[totalMemoryColumn])
+        name <- as.character(row[nameColumn])
+        nbClones <- as.integer(row[numberClonesColumn])
+        otherCloningMemory = getValue(rawdata, name, cloningName,nbClones,column)
+        result = ((otherCloningMemory - totalMemory) / otherCloningMemory)
+        return(result)
+    }
+    
+    # We return the function that can process each row
+    return(computeMemoryRatio)
+}
+
+
+
+###
+# Preprocessing: new columns, enforce order, etc.
+###
+preprocessData <- function (rawdata) {
+
+	# We merge the columns cloningtechnique and light (to create "LightDeep" for instance)
+	#rawdata$CloningTechnique = paste(rawdata$Light, rawdata$CloningTechnique, sep="")
+
+	# For each line, we compute the gain compared to three out of four techniques, and we put it in 
+	# Note that the "1" indicates "compute for each row" as opposed to "2"
+	rawdata$MemoryGainDeep =         apply(rawdata,1,computeGainCreator(rawdata, 'DeepCloning', "MemoryUsed"))
+    rawdata$MemoryGainLightDeep =    apply(rawdata,1,computeGainCreator(rawdata, 'ShareFieldsOnly',"MemoryUsed"))
+	rawdata$MemoryGainMutClassOnly = apply(rawdata,1,computeGainCreator(rawdata, 'ShareObjOnly',"MemoryUsed"))
+	rawdata$TimeGainDeep =           apply(rawdata,1,computeGainCreator(rawdata, 'DeepCloning', "TimeSpent"))
+    rawdata$TimeGainLightDeep =      apply(rawdata,1,computeGainCreator(rawdata, 'ShareFieldsOnly',"TimeSpent"))
+	rawdata$TimeGainMutClassOnly =   apply(rawdata,1,computeGainCreator(rawdata, 'ShareObjOnly',"TimeSpent"))
+
+	# Performance gain as well
+	rawdata$PerfGain = ((rawdata$OriginalModelPerformance - rawdata$CloneModelPerformance)/rawdata$OriginalModelPerformance)
+	
+	print ("###################################### Computed media for performance gain:")
+	thousand = rawdata[rawdata$NumberOfClones==1000 & rawdata$CloningTechnique=="ShareFieldsOnly",]
+	print(median(thousand$PerfGain))
+
+	thousand = rawdata[rawdata$NumberOfClones==1000 & rawdata$CloningTechnique=="ShareAll",]
+	print(median(thousand$PerfGain))
+    print ("######################################")
+	
+	# Getting the list of cloning techniques, in the right order
+	techniques = unique(rawdata$CloningTechnique)
+
+	# Enforcing the order of the cloning techniques column
+	# https://stackoverflow.com/questions/19710986/how-to-make-acast-preserve-order-of-variable-in-reshape2
+	rawdata$CloningTechnique = factor(rawdata$CloningTechnique, techniques)
+
+    return(rawdata)
+
+}
+
+
+
+###
+# Creates a barplot that compares a given metamodel on one aspect (memory or time) for numerous "number of clones" values
+###
+plotNumberOfClones <- function(rawdata, metamodelname, minvalue, maxvalue, measure, title){
+
+    prefiltered = rawdata
+
+    filtered = prefiltered[prefiltered$MetamodelName==metamodelname & prefiltered$NumberOfClones<=maxvalue & prefiltered$NumberOfClones>=minvalue & prefiltered$CloningTechnique!="Deep",
+        c(measure,'CloningTechnique', 'NumberOfClones')]
+
+	# Transform the table nicely: NumberOfClones becomes lines, and CloningTechniques columns
+	# Means that we want to compare "cloningtechniques" regarding "measure" for each "numberofclones"
+	result = dcast(filtered,CloningTechnique~NumberOfClones,value.var=measure)
+
+	# We store the column with the name of the cloning techniques, for the legend, and we delete it
+	legendc = c("ShareFieldsOnly","ShareObjectsOnly","ShareAll")#result$CloningTechnique
+	colors = gray.colors(length(legendc), start = 0.9, end = 0.3, gamma = 2.2)
+	result$CloningTechnique = NULL
+
+	# We plot with bars, and we put a legend
+	name = paste(metamodelname,"_",minvalue,"-",maxvalue,"_clones_",measure, sep="")
+	formats = c("pdf")
+	
+	    
+    myylab=measure
+    if (measure == "MemoryGainDeep") {
+        myylab = "memory gain over DeepClone"
+    }
+	
+	for (format in formats) {
+	    initImage(name,format)
+        barplot(as.matrix(result), main=title, xlab= "number of clones", ylab=myylab,beside=TRUE, col=colors, yaxt="n")
+        axis(2, at=pretty(filtered[,c(measure)]), lab=paste0(pretty(filtered[,c(measure)]) * 100,"%"), las=TRUE,pos=0)
+	    legend("topleft", as.vector(legendc), cex=1, bty="n", fill=colors);
+	}
+	
+	
+	return(result)
+}
+
+
+
+###
+# Creates a plot... mostly used for memory
+###
+plotStuff <- function(rawdata, measure, variable, numberOfClones, cloning, style, maxX, maxY, percentX) {
+    myxlim = NULL
+    if (maxX > 0) {
+        myxlim = c(0,maxX)
+    }
+    myylim = NULL
+    if (maxY > 0) {
+        myylim = c(0,maxY)
+    }
+    
+    myxlab=variable
+    if (variable == "ObjShareShareableClassesRatio") {
+        myxlab = "proportion of shareable classes (case ObjShare)"
+    }
+    if (variable == "NoObjShareShareablePropertiesInPartShareableClassesDensity") {
+        myxlab = "density of shareable properties in part shareable classes\n (ShareFieldsOnly strategy)"
+    }
+    if (variable == "ObjShareShareablePropertiesInPartShareableClassesDensity") {
+        myxlab = "density of shareable properties in part shareable classes\n (ShareBoth strategy)"
+    }
+    if (variable == "ObjShareShareableClassesRatio") {
+        myxlab = "proportion of shareable classes (ShareObjectsOnly strategy)"
+    }
+    if (variable == "NoObjShareShareablePropertiesInPartShareableClassesDensity") {
+        myxlab = "density of shareable properties in part shareable classes\n (ShareFieldsOnly strategy)"
+    }
+    if (variable == "ObjShareShareablePropertiesInPartShareableClassesDensity") {
+        myxlab = "density of shareable properties in part shareable classes\n (ShareBoth strategy)"
+    }
+    
+    myylab=measure
+    if (measure == "MemoryGainDeep") {
+        myylab = "memory gain over DeepClone"
+    }
+    if (measure == "MemoryGainLightDeep") {
+        myylab = "memory gain over ShareFieldsOnly"
+    }
+    if (measure == "MemoryGainMutClassOnly") {
+        myylab = "memory gain over ShareObjectsOnly"
+    }
+    if (measure == "PerfGain") {
+        myylab = "Model manipulation efficiency gain over the original model"
+    }
+
+
+
+    data = rawdata
+    
+    mutclassonly = data[data$CloningTechnique==cloning & data$NumberOfClones==numberOfClones,]
+
+    initImage(paste(cloning,measure,variable,sep=""),'pdf')
+    
+    
+    # Type=h pour mettre des petits batons, lwd pour l'épaisseur des traits
+    plot(mutclassonly[,c(variable)], mutclassonly[,c(measure)], pch=style, xlab=myxlab, ylab=myylab, yaxt="n", xaxt="n",frame=F, xlim=myxlim, ylim=myylim)#, type="h", lwd="3")#, main=title)
+    text(mutclassonly[,c(variable)], mutclassonly[,c(measure)], mutclassonly[,c("MetamodelName")], cex=0.6, pos=1, col="red")
+    axis(2, at=pretty(mutclassonly[,c(measure)]), lab=paste0(pretty(mutclassonly[,c(measure)]) * 100,"%"), las=TRUE,pos=0)
+    
+    if (percentX) {
+        axis(1, at=pretty(mutclassonly[,c(measure)]), lab=paste0(pretty(mutclassonly[,c(measure)]) * 100,"%"), las=TRUE,pos=0)
+    } else {
+        axis(1, at=pretty(mutclassonly[,c(variable)]), pos=0)
+    }
+ 
+}
+
+###
+# Adds points to a plot created with "plotStuff"
+###
+addPointsPlotStuff <- function(rawdata, measure,variable, numberOfClones, cloning, style) {
+    data = rawdata
+    mutclassonly = data[data$CloningTechnique==cloning & data$NumberOfClones==numberOfClones,]
+    points(mutclassonly[,c(variable)], mutclassonly[,c(measure)], pch=style)
+    
+    text(mutclassonly[,c(variable)], mutclassonly[,c(measure)], mutclassonly[,c("MetamodelName")], cex=0.6, pos=1, col="red") 
+}
+    
+   
+
+###################################################################################################################################
+###################################################################################################################################
+###################################################################################################################################
+
+
+# Getting arguments (either single, or multiple ones)
+args = commandArgs(TRUE)
+myrawdata = c()
+if (length(args)==1) {
+    myrawdata = read.csv(args)
+} else {
+    for (arg in args) {
+        # Parsing CSV file
+        newdata = read.csv(arg)
+        if (length(myrawdata) == 0) {
+            myrawdata = newdata
+        } else {
+            myrawdata = rbind(myrawdata,newdata)
+        }
+    }
+}
+
+
+# Preprocessing (new columns, etc)
+data = preprocessData(myrawdata)
+
+
+# 1:ShareObjects, 2:ShareFields, 3:ShareAll, 4:NoShare
+pointsStyles=c(15,4,2,1)
+
+# 
+# Plotting performance /!\ FILTERING + ONLY PROXY TECHNIQUES
+#
+perfdata=data[(data$CloneModelPerformance>-1.0) & data$NoObjShareShareablePropertiesInPartShareableClassesDensity<30,]
+plotStuff         (perfdata,'PerfGain','NoObjShareShareablePropertiesInPartShareableClassesDensity', 1, 'ShareFieldsOnly',      pointsStyles[2], -1, -1,FALSE)
+addPointsPlotStuff(perfdata,'PerfGain','NoObjShareShareablePropertiesInPartShareableClassesDensity', 1, 'ShareAll',pointsStyles[3])
+legend("topright", c("ShareFieldsOnly","ShareAll"), cex=1, bty="n", pch=pointsStyles[2:3])
+
+#
+# Plotting the memory gain graph (all metamodels) for shareobjects
+#
+plotStuff         (data, 'MemoryGainDeep', 'ObjShareShareableClassesRatio', 1000, 'ShareAll',      pointsStyles[3], 1, 1, TRUE)
+#addPointsPlotStuff(data, 'MemoryGainDeep', 'ObjShareShareableClassesRatio', 1000, 'LightDeep',         pointsStyles[2])
+addPointsPlotStuff(data, 'MemoryGainDeep', 'ObjShareShareableClassesRatio', 1000, 'ShareObjOnly', pointsStyles[1])
+legend("right", c("ShareObjectsOnly","ShareAll"), cex=1, bty="n", pch=c(pointsStyles[1],pointsStyles[3]))
+
+#
+# Plotting the memory gain (sharefields)
+#
+sharefieldsdata=data[data$NoObjShareShareablePropertiesInPartShareableClassesDensity<30,]
+plotStuff(sharefieldsdata, 'MemoryGainDeep', 'NoObjShareShareablePropertiesInPartShareableClassesDensity', 1000, 'ShareFieldsOnly', pointsStyles[2], -1, 1, FALSE)
+legend("topleft", c("ShareFieldsOnly"), cex=1, bty="n", pch=c(pointsStyles[2]))
+
+
+#
+# Plotting the Time gain graph (all metamodels)  for shareobjects
+#
+#plotStuff         (data, 'TimeGainDeep', 'ObjShareShareableClassesRatio', 100, 'LightMutClassOnly',      pointsStyles[3], 1, 1, "")
+#addPointsPlotStuff(data, 'TimeGainDeep', 'ObjShareShareableClassesRatio', 1000, 'LightDeep',         pointsStyles[2])
+#addPointsPlotStuff(data, 'TimeGainDeep', 'ObjShareShareableClassesRatio', 100, 'MutClassOnly', pointsStyles[1])
+#legend("topleft", c("ShareObjectsOnly","ShareAll"), cex=1, bty="n", pch=c(pointsStyles[1],pointsStyles[3]))
+
+
+#
+# Plotting the time gain (sharefields)
+#
+#plotStuff(sharefieldsdata, 'TimeGainDeep', 'NoObjShareShareablePropertiesInPartShareableClassesDensity', 100, 'LightDeep', pointsStyles[2], -1, 1, "")
+
+
+#
+# Plotting the nbclones-not-relevant graph (note : RANDOMLY CHOSEN METAMODEL)
+#
+#plotNumberOfClones(data, "bundle1-mm9", 0, 10000, 'MemoryGainDeep', "")
+
+############################################################ tests
+
+
+
+
+
+
+
+
+
+
+
+
+#plotStuff(data, 'MemoryGainLightDeep', 'ObjShareShareableClassesRatio', 1000, 'LightMutClassOnly',  15, 1, 1, "Memory gain of ShareBoth compared to ShareFields")
+
+
+#plotStuff(data, 'MemoryGainDeep', 'NoObjShareShareablePropertiesInPartShareableClassesDensity', 1000, 'LightDeep', 15, -1, 1, "Memory gain of ShareFields compared to NoShare")
+
+#plotStuff(myrawdata, 'MemoryGainDeep', 'ObjShareShareableClassesRatio', 100, 'LightDeep', 15, -1, -1, "Memory gain of ShareFields compared to NoShare")
+
+#plotStuff(data, 'TimeGainDeep', 'ObjShareShareableClassesRatio', 10000, 'MutClassOnly', 15, -1, -1, "Time gain of ShareFields compared to NoShare")
+#plotStuff(data[data$NoObjShareShareablePropertiesInPartShareableClassesDensity<100,], 'TimeGainDeep', 'NoObjShareShareablePropertiesInPartShareableClassesDensity', 100, 'LightDeep', 15, -1, -1, "Time gain of ShareFields compared to NoShare")
+#plotStuff(data, 'TimeGainDeep', 'ObjShareShareableClassesRatio', 10000, 'LightMutClassOnly', 15, -1, -1, "Time gain of ShareFields compared to NoShare")
+
+#plotStuff(data, 'MemoryGainMutClassOnly', 'ObjShareShareablePropertiesInPartShareableClassesDensity', 1000, 'LightMutClassOnly', 15, -1, 1, "Memory gain of ShareBoth compared to ShareObjects")
+
+
+#plotStuff(myrawdata, 'MemoryGainDeep', 'ObjShareShareableClassesRatio', 100, 'LightMutClassOnly',15, -1, -1, "Memory gain of ShareBoth compared to NoShare")
+
+
+

evaluation-scripts/run-all

+#!/bin/bash
+
+DIR=`dirname $0`
+
+BENCHTOOLDIR="$DIR/../fr.inria.diverse.cloning.benchmark.tool.product/target/products/fr.inria.diverse.cloning.benchmark.tool.product/linux/gtk/x86_64/"
+BENCHTOOL="$BENCHTOOLDIR/cloning-benchmark"
+TOOL="$BENCHTOOL run"
+
+OUTPUTFOLDER=./benchmarktool-workingdir
+
+# Parsing arguments
+BUNDLES=""
+for arg in "$@"
+do
+    BUNDLES="$BUNDLES$arg,"
+done
+
+# To reset registered OSGi bundles
+rm -rf "$BENCHTOOLDIR/configuration/org.eclipse.osgi"
+
+$TOOL --benchmarkMetamodels=$BUNDLES --nbClones 1 --nbtries 5 --outputFile $OUTPUTFOLDER/resultsFinal1.csv --dumpsFolder=/tmp2/cloningDumpsAll
+#$TOOL --benchmarkMetamodels=$BUNDLES --nbClones 10 --nbtries 5 --outputFile $OUTPUTFOLDER/resultsFinal10.csv --dumpsFolder=/tmp2/cloningDumpsAll
+#$TOOL --benchmarkMetamodels=$BUNDLES --nbClones 100 --nbtries 5 --outputFile $OUTPUTFOLDER/resultsFinal100.csv --dumpsFolder=/tmp2/cloningDumpsAll
+$TOOL --benchmarkMetamodels=$BUNDLES --nbClones 1000 --nbtries 5 --outputFile $OUTPUTFOLDER/resultsFinal1000.csv --dumpsFolder=/tmp2/cloningDumpsAll --keepDumps
+#$TOOL --benchmarkMetamodels=$BUNDLES --nbClones 10000 --nbtries 5 --outputFile $OUTPUTFOLDER/resultsFinal10000.csv --dumpsFolder=/tmp2/cloningDumpsAll