Commit 1da19274 authored by Erwan Bousse's avatar Erwan Bousse
Browse files

Adding evaluation scripts to simplify the evaluation process

parent 1f279736
#!/bin/bash
DIR=`dirname $0`
BENCHTOOL="$DIR/../fr.inria.diverse.cloning.benchmark.tool.product/target/products/fr.inria.diverse.cloning.benchmark.tool.product/linux/gtk/x86_64/cloning-benchmark"
MMGENERATOR="$BENCHTOOL mmgen"
MATERIALGENERATOR="$BENCHTOOL matgen"
OUTPUTFOLDER=./benchmarktool-workingdir
for i in {1..10}
do
$MMGENERATOR --maxClasses 50 --maxProp 250 --maxMutProp 50 --nbMetamodels 10 --outputFolder=$OUTPUTFOLDER/$i
$MATERIALGENERATOR --metamodelsFolder=$OUTPUTFOLDER/$i --outputFile=$OUTPUTFOLDER/bundle$i.jar # --outputProjectFolder=./$i/generated-project
done
#!/bin/bash
RARGS=""
for i in "$@"
do
RARGS="$RARGS $i"
done
R --no-save --args $RARGS < ./process-data.R
ls *.pdf | xargs -I '{}' pdfcrop '{}' '{}'
mv *.pdf benchmarktool-workingdir
library(reshape2)
library(scales)
###
# To prepare the ploting with a file of a name and a format (svg of pdf)
###
initImage <- function(name, extension) {
#par(mar=c(0,0,0,0), mgp=c(0,0,0,0))
if(extension=="svg")
svg(filename=paste(name,".svg", sep=""))
else
pdf(file=paste(name,".pdf", sep=""))
}
###
# Retrieves in a df the memory used in a given run
###
getValue <- function(data, mmname, cloningname, nbclones, column) {
value = data[data$MetamodelName==mmname & data$CloningTechnique==cloningname & data$NumberOfClones==nbclones, c(column)]
return(value)
}
###
# Computes and return an operation that, for a given row,
# computes memory gain with regards to another cloning technique
###
computeGainCreator <- function(rawdata, cloningName, column) {
numberClonesColumn = which(colnames(rawdata)=="NumberOfClones")
totalMemoryColumn = which(colnames(rawdata)==column)
nameColumn = which(colnames(rawdata)=="MetamodelName")
computeMemoryRatio <- function(row) {
totalMemory <- as.integer(row[totalMemoryColumn])
name <- as.character(row[nameColumn])
nbClones <- as.integer(row[numberClonesColumn])
otherCloningMemory = getValue(rawdata, name, cloningName,nbClones,column)
result = ((otherCloningMemory - totalMemory) / otherCloningMemory)
return(result)
}
# We return the function that can process each row
return(computeMemoryRatio)
}
###
# Preprocessing: new columns, enforce order, etc.
###
preprocessData <- function (rawdata) {
# We merge the columns cloningtechnique and light (to create "LightDeep" for instance)
#rawdata$CloningTechnique = paste(rawdata$Light, rawdata$CloningTechnique, sep="")
# For each line, we compute the gain compared to three out of four techniques, and we put it in
# Note that the "1" indicates "compute for each row" as opposed to "2"
rawdata$MemoryGainDeep = apply(rawdata,1,computeGainCreator(rawdata, 'DeepCloning', "MemoryUsed"))
rawdata$MemoryGainLightDeep = apply(rawdata,1,computeGainCreator(rawdata, 'ShareFieldsOnly',"MemoryUsed"))
rawdata$MemoryGainMutClassOnly = apply(rawdata,1,computeGainCreator(rawdata, 'ShareObjOnly',"MemoryUsed"))
rawdata$TimeGainDeep = apply(rawdata,1,computeGainCreator(rawdata, 'DeepCloning', "TimeSpent"))
rawdata$TimeGainLightDeep = apply(rawdata,1,computeGainCreator(rawdata, 'ShareFieldsOnly',"TimeSpent"))
rawdata$TimeGainMutClassOnly = apply(rawdata,1,computeGainCreator(rawdata, 'ShareObjOnly',"TimeSpent"))
# Performance gain as well
rawdata$PerfGain = ((rawdata$OriginalModelPerformance - rawdata$CloneModelPerformance)/rawdata$OriginalModelPerformance)
print ("###################################### Computed media for performance gain:")
thousand = rawdata[rawdata$NumberOfClones==1000 & rawdata$CloningTechnique=="ShareFieldsOnly",]
print(median(thousand$PerfGain))
thousand = rawdata[rawdata$NumberOfClones==1000 & rawdata$CloningTechnique=="ShareAll",]
print(median(thousand$PerfGain))
print ("######################################")
# Getting the list of cloning techniques, in the right order
techniques = unique(rawdata$CloningTechnique)
# Enforcing the order of the cloning techniques column
# https://stackoverflow.com/questions/19710986/how-to-make-acast-preserve-order-of-variable-in-reshape2
rawdata$CloningTechnique = factor(rawdata$CloningTechnique, techniques)
return(rawdata)
}
###
# Creates a barplot that compares a given metamodel on one aspect (memory or time) for numerous "number of clones" values
###
plotNumberOfClones <- function(rawdata, metamodelname, minvalue, maxvalue, measure, title){
prefiltered = rawdata
filtered = prefiltered[prefiltered$MetamodelName==metamodelname & prefiltered$NumberOfClones<=maxvalue & prefiltered$NumberOfClones>=minvalue & prefiltered$CloningTechnique!="Deep",
c(measure,'CloningTechnique', 'NumberOfClones')]
# Transform the table nicely: NumberOfClones becomes lines, and CloningTechniques columns
# Means that we want to compare "cloningtechniques" regarding "measure" for each "numberofclones"
result = dcast(filtered,CloningTechnique~NumberOfClones,value.var=measure)
# We store the column with the name of the cloning techniques, for the legend, and we delete it
legendc = c("ShareFieldsOnly","ShareObjectsOnly","ShareAll")#result$CloningTechnique
colors = gray.colors(length(legendc), start = 0.9, end = 0.3, gamma = 2.2)
result$CloningTechnique = NULL
# We plot with bars, and we put a legend
name = paste(metamodelname,"_",minvalue,"-",maxvalue,"_clones_",measure, sep="")
formats = c("pdf")
myylab=measure
if (measure == "MemoryGainDeep") {
myylab = "memory gain over DeepClone"
}
for (format in formats) {
initImage(name,format)
barplot(as.matrix(result), main=title, xlab= "number of clones", ylab=myylab,beside=TRUE, col=colors, yaxt="n")
axis(2, at=pretty(filtered[,c(measure)]), lab=paste0(pretty(filtered[,c(measure)]) * 100,"%"), las=TRUE,pos=0)
legend("topleft", as.vector(legendc), cex=1, bty="n", fill=colors);
}
return(result)
}
###
# Creates a plot... mostly used for memory
###
plotStuff <- function(rawdata, measure, variable, numberOfClones, cloning, style, maxX, maxY, percentX) {
myxlim = NULL
if (maxX > 0) {
myxlim = c(0,maxX)
}
myylim = NULL
if (maxY > 0) {
myylim = c(0,maxY)
}
myxlab=variable
if (variable == "ObjShareShareableClassesRatio") {
myxlab = "proportion of shareable classes (case ObjShare)"
}
if (variable == "NoObjShareShareablePropertiesInPartShareableClassesDensity") {
myxlab = "density of shareable properties in part shareable classes\n (ShareFieldsOnly strategy)"
}
if (variable == "ObjShareShareablePropertiesInPartShareableClassesDensity") {
myxlab = "density of shareable properties in part shareable classes\n (ShareBoth strategy)"
}
if (variable == "ObjShareShareableClassesRatio") {
myxlab = "proportion of shareable classes (ShareObjectsOnly strategy)"
}
if (variable == "NoObjShareShareablePropertiesInPartShareableClassesDensity") {
myxlab = "density of shareable properties in part shareable classes\n (ShareFieldsOnly strategy)"
}
if (variable == "ObjShareShareablePropertiesInPartShareableClassesDensity") {
myxlab = "density of shareable properties in part shareable classes\n (ShareBoth strategy)"
}
myylab=measure
if (measure == "MemoryGainDeep") {
myylab = "memory gain over DeepClone"
}
if (measure == "MemoryGainLightDeep") {
myylab = "memory gain over ShareFieldsOnly"
}
if (measure == "MemoryGainMutClassOnly") {
myylab = "memory gain over ShareObjectsOnly"
}
if (measure == "PerfGain") {
myylab = "Model manipulation efficiency gain over the original model"
}
data = rawdata
mutclassonly = data[data$CloningTechnique==cloning & data$NumberOfClones==numberOfClones,]
initImage(paste(cloning,measure,variable,sep=""),'pdf')
# Type=h pour mettre des petits batons, lwd pour l'épaisseur des traits
plot(mutclassonly[,c(variable)], mutclassonly[,c(measure)], pch=style, xlab=myxlab, ylab=myylab, yaxt="n", xaxt="n",frame=F, xlim=myxlim, ylim=myylim)#, type="h", lwd="3")#, main=title)
text(mutclassonly[,c(variable)], mutclassonly[,c(measure)], mutclassonly[,c("MetamodelName")], cex=0.6, pos=1, col="red")
axis(2, at=pretty(mutclassonly[,c(measure)]), lab=paste0(pretty(mutclassonly[,c(measure)]) * 100,"%"), las=TRUE,pos=0)
if (percentX) {
axis(1, at=pretty(mutclassonly[,c(measure)]), lab=paste0(pretty(mutclassonly[,c(measure)]) * 100,"%"), las=TRUE,pos=0)
} else {
axis(1, at=pretty(mutclassonly[,c(variable)]), pos=0)
}
}
###
# Adds points to a plot created with "plotStuff"
###
addPointsPlotStuff <- function(rawdata, measure,variable, numberOfClones, cloning, style) {
data = rawdata
mutclassonly = data[data$CloningTechnique==cloning & data$NumberOfClones==numberOfClones,]
points(mutclassonly[,c(variable)], mutclassonly[,c(measure)], pch=style)
text(mutclassonly[,c(variable)], mutclassonly[,c(measure)], mutclassonly[,c("MetamodelName")], cex=0.6, pos=1, col="red")
}
###################################################################################################################################
###################################################################################################################################
###################################################################################################################################
# Getting arguments (either single, or multiple ones)
args = commandArgs(TRUE)
myrawdata = c()
if (length(args)==1) {
myrawdata = read.csv(args)
} else {
for (arg in args) {
# Parsing CSV file
newdata = read.csv(arg)
if (length(myrawdata) == 0) {
myrawdata = newdata
} else {
myrawdata = rbind(myrawdata,newdata)
}
}
}
# Preprocessing (new columns, etc)
data = preprocessData(myrawdata)
# 1:ShareObjects, 2:ShareFields, 3:ShareAll, 4:NoShare
pointsStyles=c(15,4,2,1)
#
# Plotting performance /!\ FILTERING + ONLY PROXY TECHNIQUES
#
perfdata=data[(data$CloneModelPerformance>-1.0) & data$NoObjShareShareablePropertiesInPartShareableClassesDensity<30,]
plotStuff (perfdata,'PerfGain','NoObjShareShareablePropertiesInPartShareableClassesDensity', 1, 'ShareFieldsOnly', pointsStyles[2], -1, -1,FALSE)
addPointsPlotStuff(perfdata,'PerfGain','NoObjShareShareablePropertiesInPartShareableClassesDensity', 1, 'ShareAll',pointsStyles[3])
legend("topright", c("ShareFieldsOnly","ShareAll"), cex=1, bty="n", pch=pointsStyles[2:3])
#
# Plotting the memory gain graph (all metamodels) for shareobjects
#
plotStuff (data, 'MemoryGainDeep', 'ObjShareShareableClassesRatio', 1000, 'ShareAll', pointsStyles[3], 1, 1, TRUE)
#addPointsPlotStuff(data, 'MemoryGainDeep', 'ObjShareShareableClassesRatio', 1000, 'LightDeep', pointsStyles[2])
addPointsPlotStuff(data, 'MemoryGainDeep', 'ObjShareShareableClassesRatio', 1000, 'ShareObjOnly', pointsStyles[1])
legend("right", c("ShareObjectsOnly","ShareAll"), cex=1, bty="n", pch=c(pointsStyles[1],pointsStyles[3]))
#
# Plotting the memory gain (sharefields)
#
sharefieldsdata=data[data$NoObjShareShareablePropertiesInPartShareableClassesDensity<30,]
plotStuff(sharefieldsdata, 'MemoryGainDeep', 'NoObjShareShareablePropertiesInPartShareableClassesDensity', 1000, 'ShareFieldsOnly', pointsStyles[2], -1, 1, FALSE)
legend("topleft", c("ShareFieldsOnly"), cex=1, bty="n", pch=c(pointsStyles[2]))
#
# Plotting the Time gain graph (all metamodels) for shareobjects
#
#plotStuff (data, 'TimeGainDeep', 'ObjShareShareableClassesRatio', 100, 'LightMutClassOnly', pointsStyles[3], 1, 1, "")
#addPointsPlotStuff(data, 'TimeGainDeep', 'ObjShareShareableClassesRatio', 1000, 'LightDeep', pointsStyles[2])
#addPointsPlotStuff(data, 'TimeGainDeep', 'ObjShareShareableClassesRatio', 100, 'MutClassOnly', pointsStyles[1])
#legend("topleft", c("ShareObjectsOnly","ShareAll"), cex=1, bty="n", pch=c(pointsStyles[1],pointsStyles[3]))
#
# Plotting the time gain (sharefields)
#
#plotStuff(sharefieldsdata, 'TimeGainDeep', 'NoObjShareShareablePropertiesInPartShareableClassesDensity', 100, 'LightDeep', pointsStyles[2], -1, 1, "")
#
# Plotting the nbclones-not-relevant graph (note : RANDOMLY CHOSEN METAMODEL)
#
#plotNumberOfClones(data, "bundle1-mm9", 0, 10000, 'MemoryGainDeep', "")
############################################################ tests
#plotStuff(data, 'MemoryGainLightDeep', 'ObjShareShareableClassesRatio', 1000, 'LightMutClassOnly', 15, 1, 1, "Memory gain of ShareBoth compared to ShareFields")
#plotStuff(data, 'MemoryGainDeep', 'NoObjShareShareablePropertiesInPartShareableClassesDensity', 1000, 'LightDeep', 15, -1, 1, "Memory gain of ShareFields compared to NoShare")
#plotStuff(myrawdata, 'MemoryGainDeep', 'ObjShareShareableClassesRatio', 100, 'LightDeep', 15, -1, -1, "Memory gain of ShareFields compared to NoShare")
#plotStuff(data, 'TimeGainDeep', 'ObjShareShareableClassesRatio', 10000, 'MutClassOnly', 15, -1, -1, "Time gain of ShareFields compared to NoShare")
#plotStuff(data[data$NoObjShareShareablePropertiesInPartShareableClassesDensity<100,], 'TimeGainDeep', 'NoObjShareShareablePropertiesInPartShareableClassesDensity', 100, 'LightDeep', 15, -1, -1, "Time gain of ShareFields compared to NoShare")
#plotStuff(data, 'TimeGainDeep', 'ObjShareShareableClassesRatio', 10000, 'LightMutClassOnly', 15, -1, -1, "Time gain of ShareFields compared to NoShare")
#plotStuff(data, 'MemoryGainMutClassOnly', 'ObjShareShareablePropertiesInPartShareableClassesDensity', 1000, 'LightMutClassOnly', 15, -1, 1, "Memory gain of ShareBoth compared to ShareObjects")
#plotStuff(myrawdata, 'MemoryGainDeep', 'ObjShareShareableClassesRatio', 100, 'LightMutClassOnly',15, -1, -1, "Memory gain of ShareBoth compared to NoShare")
#!/bin/bash
DIR=`dirname $0`
BENCHTOOLDIR="$DIR/../fr.inria.diverse.cloning.benchmark.tool.product/target/products/fr.inria.diverse.cloning.benchmark.tool.product/linux/gtk/x86_64/"
BENCHTOOL="$BENCHTOOLDIR/cloning-benchmark"
TOOL="$BENCHTOOL run"
OUTPUTFOLDER=./benchmarktool-workingdir
# Parsing arguments
BUNDLES=""
for arg in "$@"
do
BUNDLES="$BUNDLES$arg,"
done
# To reset registered OSGi bundles
rm -rf "$BENCHTOOLDIR/configuration/org.eclipse.osgi"
$TOOL --benchmarkMetamodels=$BUNDLES --nbClones 1 --nbtries 5 --outputFile $OUTPUTFOLDER/resultsFinal1.csv --dumpsFolder=/tmp2/cloningDumpsAll
#$TOOL --benchmarkMetamodels=$BUNDLES --nbClones 10 --nbtries 5 --outputFile $OUTPUTFOLDER/resultsFinal10.csv --dumpsFolder=/tmp2/cloningDumpsAll
#$TOOL --benchmarkMetamodels=$BUNDLES --nbClones 100 --nbtries 5 --outputFile $OUTPUTFOLDER/resultsFinal100.csv --dumpsFolder=/tmp2/cloningDumpsAll
$TOOL --benchmarkMetamodels=$BUNDLES --nbClones 1000 --nbtries 5 --outputFile $OUTPUTFOLDER/resultsFinal1000.csv --dumpsFolder=/tmp2/cloningDumpsAll --keepDumps
#$TOOL --benchmarkMetamodels=$BUNDLES --nbClones 10000 --nbtries 5 --outputFile $OUTPUTFOLDER/resultsFinal10000.csv --dumpsFolder=/tmp2/cloningDumpsAll
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment