patrick/tplp-planning-benchmark
1
0
Fork 0
Code Issues Releases Activity

Add option to print evaluation as plain text

Browse Source
This commit is contained in:
Patrick Lühne 2017-12-14 16:48:04 +01:00
parent 519338232c
commit 0ef8b69486
Signed by: patrick
GPG Key ID: 05F3611E97A70ABF
1 changed files with 56 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
evaluate.py
View File

@ -263,32 +263,58 @@ def renderResultsTable(configurations, instanceSetID, instanceSet, instances, re
print("</tr></thead></table>") print("</tr></thead></table>")
def renderAggregatedResultsTable(type, configurations, instanceSetID, instanceSet, instances, results, config): def renderAggregatedResultsTable(type, configurations, instanceSetID, instanceSet, instances, results, config, format = "html"):
aggregatedResults = aggregateResults(configurations, instanceSetID, instanceSet, instances, results) aggregatedResults = aggregateResults(configurations, instanceSetID, instanceSet, instances, results)
if not aggregatedResults: if not aggregatedResults:
print("<!-- error -->") print("<!-- error -->")
return return
print("<h2>" + instanceSetID + " (" + type + ")</h2><table><thead><tr><th>IPC</th><th>domain</th>") def compareConfigurations(configuration):
hasResults = instanceSetID in configuration["instanceSets"] and len(aggregatedResults[("total", "")][configuration["id"]]["results"]) > 0
for configuration in configurations: if not hasResults:
return (0, config["limits"]["time"])
return (-aggregatedResults[("total", "")][configuration["id"]]["instances solved"], aggregatedResults[("total", "")][configuration["id"]]["average runtime"])
sortedConfigurations = sorted(configurations, key = compareConfigurations)
if format == "html":
print("<h2>" + instanceSetID + " (" + type + ")</h2><table><thead><tr><th>IPC</th><th>domain</th>")
else:
print("# " + instanceSetID)
print("domain", end = "")
for configuration in sortedConfigurations:
if not instanceSetID in configuration["instanceSets"]: if not instanceSetID in configuration["instanceSets"]:
continue continue
if format == "html":
print("<th><div title=\"" + str(configuration["options"]) + "\">" + configuration["id"] + "</div></th>") print("<th><div title=\"" + str(configuration["options"]) + "\">" + configuration["id"] + "</div></th>")
else:
print("\t" + configuration["id"], end="")
if format == "html":
print("</tr></thead><tbody>") print("</tr></thead><tbody>")
else:
print("")
for ipcDomain, results in sorted(aggregatedResults.items()): for ipcDomain, results in sorted(aggregatedResults.items()):
if format == "html":
print("<tr><td class=\"col-header\">" + ipcDomain[0] + "</td><td class=\"col-header\">" + ipcDomain[1] + "</td>") print("<tr><td class=\"col-header\">" + ipcDomain[0] + "</td><td class=\"col-header\">" + ipcDomain[1] + "</td>")
else:
print((ipcDomain[0][4:] if ipcDomain[0][0:4] == "ipc-" else ipcDomain[0]) + ("/" + ipcDomain[1] if ipcDomain[1] else ""), end = "")
for configuration in configurations: for configuration in sortedConfigurations:
if not instanceSetID in configuration["instanceSets"]: if not instanceSetID in configuration["instanceSets"]:
continue continue
if len(results[configuration["id"]]["results"]) == 0: if len(results[configuration["id"]]["results"]) == 0:
if format == "html":
print("<td class=\"tbd\"></td>") print("<td class=\"tbd\"></td>")
else:
print("\t", end = "")
continue continue
r = results[configuration["id"]][type] r = results[configuration["id"]][type]
@ -297,9 +323,18 @@ def renderAggregatedResultsTable(type, configurations, instanceSetID, instanceSe
value = (r / config["limits"]["time"]) ** 0.2 if type == "average runtime" else 1.0 - r / max(1, len(results[configuration["id"]]["results"])) value = (r / config["limits"]["time"]) ** 0.2 if type == "average runtime" else 1.0 - r / max(1, len(results[configuration["id"]]["results"]))
classes = " result-" + str(int(value * 100)) classes = " result-" + str(int(value * 100))
if format == "html":
print("<td class=\"result" + classes + "\">" + numberFormat % r + "</td>") print("<td class=\"result" + classes + "\">" + numberFormat % r + "</td>")
else:
print("\t" + numberFormat % r, end = "")
if format != "html":
print("")
if format == "html":
print("</tr></tbody></thead></table>") print("</tr></tbody></thead></table>")
else:
print()
def main(): def main():
with open("config.yml", "r") as stream: with open("config.yml", "r") as stream:
@ -307,6 +342,11 @@ def main():
configurations, instances, results = collectResults(config) configurations, instances, results = collectResults(config)
if len(sys.argv) > 1 and sys.argv[1] == "--format=plain-text":
for instanceSetID, instanceSet in instances.items():
renderAggregatedResultsTable("instances solved", configurations, instanceSetID, instanceSet, instances, results, config, format = "plain")
renderAggregatedResultsTable("average runtime", configurations, instanceSetID, instanceSet, instances, results, config, format = "plain")
else:
print("<!DOCTYPE html><html lang=\"en\"><head><title>TPLP benchmark results</title><meta charset=\"UTF-8\"><link rel=\"stylesheet\" href=\"style.css?v=2\" type=\"text/css\"></head><body><main><h1>TPLP Benchmark Results</h1><div class=\"footnote\">last updated at " + time.strftime("%Y-%m-%d %H:%M:%S %z") + "</div>") print("<!DOCTYPE html><html lang=\"en\"><head><title>TPLP benchmark results</title><meta charset=\"UTF-8\"><link rel=\"stylesheet\" href=\"style.css?v=2\" type=\"text/css\"></head><body><main><h1>TPLP Benchmark Results</h1><div class=\"footnote\">last updated at " + time.strftime("%Y-%m-%d %H:%M:%S %z") + "</div>")
for instanceSetID, instanceSet in instances.items(): for instanceSetID, instanceSet in instances.items():