diff --git a/evaluate.py b/evaluate.py index 37a8b2a75..d2cd2d14d 100755 --- a/evaluate.py +++ b/evaluate.py @@ -263,32 +263,58 @@ def renderResultsTable(configurations, instanceSetID, instanceSet, instances, re print("") -def renderAggregatedResultsTable(type, configurations, instanceSetID, instanceSet, instances, results, config): +def renderAggregatedResultsTable(type, configurations, instanceSetID, instanceSet, instances, results, config, format = "html"): aggregatedResults = aggregateResults(configurations, instanceSetID, instanceSet, instances, results) if not aggregatedResults: print("") return - print("

" + instanceSetID + " (" + type + ")

") + def compareConfigurations(configuration): + hasResults = instanceSetID in configuration["instanceSets"] and len(aggregatedResults[("total", "")][configuration["id"]]["results"]) > 0 - for configuration in configurations: + if not hasResults: + return (0, config["limits"]["time"]) + + return (-aggregatedResults[("total", "")][configuration["id"]]["instances solved"], aggregatedResults[("total", "")][configuration["id"]]["average runtime"]) + + sortedConfigurations = sorted(configurations, key = compareConfigurations) + + if format == "html": + print("

" + instanceSetID + " (" + type + ")

IPCdomain
") + else: + print("# " + instanceSetID) + print("domain", end = "") + + for configuration in sortedConfigurations: if not instanceSetID in configuration["instanceSets"]: continue - print("") + if format == "html": + print("") + else: + print("\t" + configuration["id"], end="") - print("") + if format == "html": + print("") + else: + print("") for ipcDomain, results in sorted(aggregatedResults.items()): - print("") + if format == "html": + print("") + else: + print((ipcDomain[0][4:] if ipcDomain[0][0:4] == "ipc-" else ipcDomain[0]) + ("/" + ipcDomain[1] if ipcDomain[1] else ""), end = "") - for configuration in configurations: + for configuration in sortedConfigurations: if not instanceSetID in configuration["instanceSets"]: continue if len(results[configuration["id"]]["results"]) == 0: - print("") + if format == "html": + print("") + else: + print("\t", end = "") continue r = results[configuration["id"]][type] @@ -297,9 +323,18 @@ def renderAggregatedResultsTable(type, configurations, instanceSetID, instanceSe value = (r / config["limits"]["time"]) ** 0.2 if type == "average runtime" else 1.0 - r / max(1, len(results[configuration["id"]]["results"])) classes = " result-" + str(int(value * 100)) - print("") + if format == "html": + print("") + else: + print("\t" + numberFormat % r, end = "") - print("
IPCdomain
" + configuration["id"] + "
" + configuration["id"] + "
" + ipcDomain[0] + "" + ipcDomain[1] + "
" + ipcDomain[0] + "" + ipcDomain[1] + "" + numberFormat % r + "" + numberFormat % r + "
") + if format != "html": + print("") + + if format == "html": + print("") + else: + print() def main(): with open("config.yml", "r") as stream: @@ -307,13 +342,18 @@ def main(): configurations, instances, results = collectResults(config) - print("TPLP benchmark results

TPLP Benchmark Results

last updated at " + time.strftime("%Y-%m-%d %H:%M:%S %z") + "
") + if len(sys.argv) > 1 and sys.argv[1] == "--format=plain-text": + for instanceSetID, instanceSet in instances.items(): + renderAggregatedResultsTable("instances solved", configurations, instanceSetID, instanceSet, instances, results, config, format = "plain") + renderAggregatedResultsTable("average runtime", configurations, instanceSetID, instanceSet, instances, results, config, format = "plain") + else: + print("TPLP benchmark results

TPLP Benchmark Results

last updated at " + time.strftime("%Y-%m-%d %H:%M:%S %z") + "
") - for instanceSetID, instanceSet in instances.items(): - renderAggregatedResultsTable("instances solved", configurations, instanceSetID, instanceSet, instances, results, config) - renderAggregatedResultsTable("average runtime", configurations, instanceSetID, instanceSet, instances, results, config) - renderResultsTable(configurations, instanceSetID, instanceSet, instances, results) + for instanceSetID, instanceSet in instances.items(): + renderAggregatedResultsTable("instances solved", configurations, instanceSetID, instanceSet, instances, results, config) + renderAggregatedResultsTable("average runtime", configurations, instanceSetID, instanceSet, instances, results, config) + renderResultsTable(configurations, instanceSetID, instanceSet, instances, results) - print("
") + print("
") main()