From 24b1bf655bed8b2df90317bcd7acee4284150421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Fri, 24 Nov 2017 15:11:57 +0100 Subject: [PATCH] Use Git worktrees instead of branches Checking out branches with many files consumes increasing amounts of time. For this reason, this commit introduces worktrees that represent the individual branches in different directories, while still maintaining only one copy of the .git directory. --- benchmark.py | 111 ++++++++++++++++++++++++++++----------------------- 1 file changed, 62 insertions(+), 49 deletions(-) diff --git a/benchmark.py b/benchmark.py index eb70839b7..cf438b70b 100644 --- a/benchmark.py +++ b/benchmark.py @@ -61,47 +61,72 @@ def git(command, cwd, enforce = False): if enforce: raise RuntimeError("git error") -def initRepo(config): - dataDir = config["storage"]["local"] +def getResultsDir(config): + return os.path.join(config["storage"]["local"], "results") - # clone repo if not existing - if not os.path.isdir(config["storage"]["local"]): - git(["clone", config["storage"]["remote"], dataDir], None, enforce = True) +def getConfigDir(config): + return os.path.join(config["storage"]["local"], "config") +def getStatusDir(config): + return os.path.join(config["storage"]["local"], "status") + +def configureGit(dir, config): # default settings - git(["config", "--local", "user.name", config["storage"]["userName"]], dataDir, enforce = True) - git(["config", "--local", "user.email", config["storage"]["userEMail"]], dataDir, enforce = True) + git(["config", "--local", "user.name", config["storage"]["userName"]], dir, enforce = True) + git(["config", "--local", "user.email", config["storage"]["userEMail"]], dir, enforce = True) if "userSigningKey" in config["storage"]: - git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], dataDir, enforce = True) - git(["config", "--local", "commit.gpgsign", "true"], dataDir, enforce = True) + git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], dir, enforce = True) + git(["config", "--local", "commit.gpgsign", "true"], dir, enforce = True) else: - git(["config", "--local", "commit.gpgsign", "false"], dataDir, enforce = True) + git(["config", "--local", "commit.gpgsign", "false"], dir, enforce = True) + +def initRepo(config): + resultsDir = getResultsDir(config) + configDir = getConfigDir(config) + statusDir = getStatusDir(config) + + # clone repo if not existing + if not os.path.isdir(statusDir): + os.makedirs(statusDir) + git(["clone", config["storage"]["remote"], "--branch=" + config["storage"]["branches"]["status"], statusDir], None, enforce = True) + git(["worktree", "add", os.path.join("..", "results"), "-b" + config["storage"]["branches"]["results"], "origin/" + config["storage"]["branches"]["results"]], statusDir, enforce = True) + git(["branch", "--set-upstream-to=" + "origin/" + config["storage"]["branches"]["results"]], resultsDir, enforce = True) + git(["worktree", "add", os.path.join("..", "config"), "-b" + config["storage"]["branches"]["config"], "origin/" + config["storage"]["branches"]["config"]], statusDir, enforce = True) + git(["branch", "--set-upstream-to=" + "origin/" + config["storage"]["branches"]["config"]], configDir, enforce = True) + + # default settings + configureGit(resultsDir, config) + configureGit(configDir, config) + configureGit(statusDir, config) + + if "userSigningKey" in config["storage"]: + git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], statusDir, enforce = True) + git(["config", "--local", "commit.gpgsign", "true"], statusDir, enforce = True) + else: + git(["config", "--local", "commit.gpgsign", "false"], statusDir, enforce = True) # fetch origin - git(["fetch"], cwd = dataDir) + git(["fetch"], cwd = statusDir) # pull all branches - for key, branch in config["storage"]["branches"].items(): - git(["checkout", branch], cwd = dataDir, enforce = True) - git(["pull"], cwd = dataDir) + git(["pull"], cwd = configDir) + git(["pull"], cwd = statusDir) + git(["pull"], cwd = resultsDir) def readBenchmarkConfig(config): initRepo(config) - dataDir = config["storage"]["local"] - - # checkout config branch - git(["checkout", config["storage"]["branches"]["config"]], cwd = dataDir, enforce = True) + configDir = getConfigDir(config) # read instance list - instancesFile = os.path.join(config["storage"]["local"], "instances.yml") + instancesFile = os.path.join(configDir, "instances.yml") with open(instancesFile, "r") as stream: instances = yaml.load(stream, Loader = yaml.CLoader) # read configurations to test - configurationsFile = os.path.join(config["storage"]["local"], "configurations.yml") + configurationsFile = os.path.join(configDir, "configurations.yml") with open(configurationsFile, "r") as stream: configurations = yaml.load(stream, Loader = yaml.CLoader) @@ -130,11 +155,7 @@ def outputFilenames(configuration, instance, config): def nextJob(config): benchmarkConfig = readBenchmarkConfig(config) - - dataDir = config["storage"]["local"] - - # checkout results branch - git(["checkout", config["storage"]["branches"]["results"]], cwd = dataDir, enforce = True) + resultsDir = getResultsDir(config) configurations = benchmarkConfig["configurations"]["configurations"] instances = benchmarkConfig["instances"] @@ -143,9 +164,9 @@ def nextJob(config): for instance in instanceSet: for configuration in configurations: filenames = outputFilenames(configuration, instance, config) - outputFile = os.path.join(config["storage"]["local"], filenames["outputFile"]) - errorFile = os.path.join(config["storage"]["local"], filenames["errorFile"]) - environmentFile = os.path.join(config["storage"]["local"], filenames["environmentFile"]) + outputFile = os.path.join(resultsDir, filenames["outputFile"]) + errorFile = os.path.join(resultsDir, filenames["errorFile"]) + environmentFile = os.path.join(resultsDir, filenames["environmentFile"]) if not instanceSetID in configuration["instanceSets"]: continue @@ -156,12 +177,8 @@ def nextJob(config): return None def writeStatus(message, config): - dataDir = config["storage"]["local"] - - # checkout status branch - git(["checkout", config["storage"]["branches"]["status"]], cwd = dataDir, enforce = True) - - statusFilename = os.path.join(dataDir, "status.log") + statusDir = getStatusDir(config) + statusFilename = os.path.join(statusDir, "status.log") if os.path.exists(statusFilename): with open(statusFilename, "r") as statusFile: @@ -173,22 +190,18 @@ def writeStatus(message, config): with open(statusFilename, "w") as statusFile: print(time.strftime("%Y-%m-%d %H:%M:%S %z") + "\t" + message + "\n" + "".join(content), file = statusFile, end = "") - git(["add", "status.log"], dataDir, enforce = True) - git(["commit", "-m Update status: " + message], dataDir, enforce = True) - git(["push"], dataDir) + git(["add", "status.log"], statusDir, enforce = True) + git(["commit", "-m Update status: " + message], statusDir, enforce = True) + git(["push", "origin", config["storage"]["branches"]["status"]], statusDir) def runJob(configuration, instance, config): jobName = "[" + str(configuration["id"]) + " | " + instance["ipc"] + " | " + instance["domain"] + " | " + str(instance["instance"]) + "]" writeStatus("started benchmark job " + jobName, config) - dataDir = config["storage"]["local"] - + resultsDir = getResultsDir(config) inputFiles = inputFilenames(instance, config) - # checkout results branch - git(["checkout", config["storage"]["branches"]["results"]], cwd = dataDir, enforce = True) - command = \ [ config["executables"]["timeout"]["binary"], @@ -207,14 +220,14 @@ def runJob(configuration, instance, config): stdout, stderr, exitCode = executeCommand(command, cwd = plannerDir) outputFiles = outputFilenames(configuration, instance, config) - outputDir = os.path.dirname(os.path.join(config["storage"]["local"], outputFiles["outputFile"])) + outputDir = os.path.dirname(os.path.join(resultsDir, outputFiles["outputFile"])) if not os.path.isdir(outputDir): os.makedirs(outputDir) - with open(os.path.join(config["storage"]["local"], outputFiles["outputFile"]), "w") as outputFile, \ - open(os.path.join(config["storage"]["local"], outputFiles["errorFile"]), "w") as errorFile, \ - open(os.path.join(config["storage"]["local"], outputFiles["environmentFile"]), "w") as environmentFile: + with open(os.path.join(resultsDir, outputFiles["outputFile"]), "w") as outputFile, \ + open(os.path.join(resultsDir, outputFiles["errorFile"]), "w") as errorFile, \ + open(os.path.join(resultsDir, outputFiles["environmentFile"]), "w") as environmentFile: print(stdout, file = outputFile) print("# configuration: " + str(configuration), file = errorFile) print("# instance: " + str(instance), file = errorFile) @@ -245,15 +258,15 @@ def runJob(configuration, instance, config): print(yaml.dump(environment, default_flow_style = False), file = environmentFile) - git(["add", outputFiles["outputFile"], outputFiles["errorFile"], outputFiles["environmentFile"]], dataDir, enforce = True) + git(["add", outputFiles["outputFile"], outputFiles["errorFile"], outputFiles["environmentFile"]], resultsDir, enforce = True) if exitCode == 0: message = "Add benchmark result" else: message = "Add corrupt benchmark result" - git(["commit", "-m " + message + " " + jobName], dataDir, enforce = True) - git(["push"], dataDir) + git(["commit", "-m " + message + " " + jobName], resultsDir, enforce = True) + git(["push", "origin", config["storage"]["branches"]["results"]], resultsDir) if exitCode != 0: writeStatus("errors reported for benchmark job " + jobName, config)