diff --git a/benchmark.py b/benchmark.py index eb70839b7..cf438b70b 100644 --- a/benchmark.py +++ b/benchmark.py @@ -61,47 +61,72 @@ def git(command, cwd, enforce = False): if enforce: raise RuntimeError("git error") -def initRepo(config): - dataDir = config["storage"]["local"] +def getResultsDir(config): + return os.path.join(config["storage"]["local"], "results") - # clone repo if not existing - if not os.path.isdir(config["storage"]["local"]): - git(["clone", config["storage"]["remote"], dataDir], None, enforce = True) +def getConfigDir(config): + return os.path.join(config["storage"]["local"], "config") +def getStatusDir(config): + return os.path.join(config["storage"]["local"], "status") + +def configureGit(dir, config): # default settings - git(["config", "--local", "user.name", config["storage"]["userName"]], dataDir, enforce = True) - git(["config", "--local", "user.email", config["storage"]["userEMail"]], dataDir, enforce = True) + git(["config", "--local", "user.name", config["storage"]["userName"]], dir, enforce = True) + git(["config", "--local", "user.email", config["storage"]["userEMail"]], dir, enforce = True) if "userSigningKey" in config["storage"]: - git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], dataDir, enforce = True) - git(["config", "--local", "commit.gpgsign", "true"], dataDir, enforce = True) + git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], dir, enforce = True) + git(["config", "--local", "commit.gpgsign", "true"], dir, enforce = True) else: - git(["config", "--local", "commit.gpgsign", "false"], dataDir, enforce = True) + git(["config", "--local", "commit.gpgsign", "false"], dir, enforce = True) + +def initRepo(config): + resultsDir = getResultsDir(config) + configDir = getConfigDir(config) + statusDir = getStatusDir(config) + + # clone repo if not existing + if not os.path.isdir(statusDir): + os.makedirs(statusDir) + git(["clone", config["storage"]["remote"], "--branch=" + config["storage"]["branches"]["status"], statusDir], None, enforce = True) + git(["worktree", "add", os.path.join("..", "results"), "-b" + config["storage"]["branches"]["results"], "origin/" + config["storage"]["branches"]["results"]], statusDir, enforce = True) + git(["branch", "--set-upstream-to=" + "origin/" + config["storage"]["branches"]["results"]], resultsDir, enforce = True) + git(["worktree", "add", os.path.join("..", "config"), "-b" + config["storage"]["branches"]["config"], "origin/" + config["storage"]["branches"]["config"]], statusDir, enforce = True) + git(["branch", "--set-upstream-to=" + "origin/" + config["storage"]["branches"]["config"]], configDir, enforce = True) + + # default settings + configureGit(resultsDir, config) + configureGit(configDir, config) + configureGit(statusDir, config) + + if "userSigningKey" in config["storage"]: + git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], statusDir, enforce = True) + git(["config", "--local", "commit.gpgsign", "true"], statusDir, enforce = True) + else: + git(["config", "--local", "commit.gpgsign", "false"], statusDir, enforce = True) # fetch origin - git(["fetch"], cwd = dataDir) + git(["fetch"], cwd = statusDir) # pull all branches - for key, branch in config["storage"]["branches"].items(): - git(["checkout", branch], cwd = dataDir, enforce = True) - git(["pull"], cwd = dataDir) + git(["pull"], cwd = configDir) + git(["pull"], cwd = statusDir) + git(["pull"], cwd = resultsDir) def readBenchmarkConfig(config): initRepo(config) - dataDir = config["storage"]["local"] - - # checkout config branch - git(["checkout", config["storage"]["branches"]["config"]], cwd = dataDir, enforce = True) + configDir = getConfigDir(config) # read instance list - instancesFile = os.path.join(config["storage"]["local"], "instances.yml") + instancesFile = os.path.join(configDir, "instances.yml") with open(instancesFile, "r") as stream: instances = yaml.load(stream, Loader = yaml.CLoader) # read configurations to test - configurationsFile = os.path.join(config["storage"]["local"], "configurations.yml") + configurationsFile = os.path.join(configDir, "configurations.yml") with open(configurationsFile, "r") as stream: configurations = yaml.load(stream, Loader = yaml.CLoader) @@ -130,11 +155,7 @@ def outputFilenames(configuration, instance, config): def nextJob(config): benchmarkConfig = readBenchmarkConfig(config) - - dataDir = config["storage"]["local"] - - # checkout results branch - git(["checkout", config["storage"]["branches"]["results"]], cwd = dataDir, enforce = True) + resultsDir = getResultsDir(config) configurations = benchmarkConfig["configurations"]["configurations"] instances = benchmarkConfig["instances"] @@ -143,9 +164,9 @@ def nextJob(config): for instance in instanceSet: for configuration in configurations: filenames = outputFilenames(configuration, instance, config) - outputFile = os.path.join(config["storage"]["local"], filenames["outputFile"]) - errorFile = os.path.join(config["storage"]["local"], filenames["errorFile"]) - environmentFile = os.path.join(config["storage"]["local"], filenames["environmentFile"]) + outputFile = os.path.join(resultsDir, filenames["outputFile"]) + errorFile = os.path.join(resultsDir, filenames["errorFile"]) + environmentFile = os.path.join(resultsDir, filenames["environmentFile"]) if not instanceSetID in configuration["instanceSets"]: continue @@ -156,12 +177,8 @@ def nextJob(config): return None def writeStatus(message, config): - dataDir = config["storage"]["local"] - - # checkout status branch - git(["checkout", config["storage"]["branches"]["status"]], cwd = dataDir, enforce = True) - - statusFilename = os.path.join(dataDir, "status.log") + statusDir = getStatusDir(config) + statusFilename = os.path.join(statusDir, "status.log") if os.path.exists(statusFilename): with open(statusFilename, "r") as statusFile: @@ -173,22 +190,18 @@ def writeStatus(message, config): with open(statusFilename, "w") as statusFile: print(time.strftime("%Y-%m-%d %H:%M:%S %z") + "\t" + message + "\n" + "".join(content), file = statusFile, end = "") - git(["add", "status.log"], dataDir, enforce = True) - git(["commit", "-m Update status: " + message], dataDir, enforce = True) - git(["push"], dataDir) + git(["add", "status.log"], statusDir, enforce = True) + git(["commit", "-m Update status: " + message], statusDir, enforce = True) + git(["push", "origin", config["storage"]["branches"]["status"]], statusDir) def runJob(configuration, instance, config): jobName = "[" + str(configuration["id"]) + " | " + instance["ipc"] + " | " + instance["domain"] + " | " + str(instance["instance"]) + "]" writeStatus("started benchmark job " + jobName, config) - dataDir = config["storage"]["local"] - + resultsDir = getResultsDir(config) inputFiles = inputFilenames(instance, config) - # checkout results branch - git(["checkout", config["storage"]["branches"]["results"]], cwd = dataDir, enforce = True) - command = \ [ config["executables"]["timeout"]["binary"], @@ -207,14 +220,14 @@ def runJob(configuration, instance, config): stdout, stderr, exitCode = executeCommand(command, cwd = plannerDir) outputFiles = outputFilenames(configuration, instance, config) - outputDir = os.path.dirname(os.path.join(config["storage"]["local"], outputFiles["outputFile"])) + outputDir = os.path.dirname(os.path.join(resultsDir, outputFiles["outputFile"])) if not os.path.isdir(outputDir): os.makedirs(outputDir) - with open(os.path.join(config["storage"]["local"], outputFiles["outputFile"]), "w") as outputFile, \ - open(os.path.join(config["storage"]["local"], outputFiles["errorFile"]), "w") as errorFile, \ - open(os.path.join(config["storage"]["local"], outputFiles["environmentFile"]), "w") as environmentFile: + with open(os.path.join(resultsDir, outputFiles["outputFile"]), "w") as outputFile, \ + open(os.path.join(resultsDir, outputFiles["errorFile"]), "w") as errorFile, \ + open(os.path.join(resultsDir, outputFiles["environmentFile"]), "w") as environmentFile: print(stdout, file = outputFile) print("# configuration: " + str(configuration), file = errorFile) print("# instance: " + str(instance), file = errorFile) @@ -245,15 +258,15 @@ def runJob(configuration, instance, config): print(yaml.dump(environment, default_flow_style = False), file = environmentFile) - git(["add", outputFiles["outputFile"], outputFiles["errorFile"], outputFiles["environmentFile"]], dataDir, enforce = True) + git(["add", outputFiles["outputFile"], outputFiles["errorFile"], outputFiles["environmentFile"]], resultsDir, enforce = True) if exitCode == 0: message = "Add benchmark result" else: message = "Add corrupt benchmark result" - git(["commit", "-m " + message + " " + jobName], dataDir, enforce = True) - git(["push"], dataDir) + git(["commit", "-m " + message + " " + jobName], resultsDir, enforce = True) + git(["push", "origin", config["storage"]["branches"]["results"]], resultsDir) if exitCode != 0: writeStatus("errors reported for benchmark job " + jobName, config)