Use Git worktrees instead of branches
Checking out branches with many files consumes increasing amounts of time. For this reason, this commit introduces worktrees that represent the individual branches in different directories, while still maintaining only one copy of the .git directory.
This commit is contained in:
parent
966746e4f9
commit
24b1bf655b
GPG Key ID:
05F3611E97A70ABF
111
benchmark.py
111
benchmark.py
@ -61,47 +61,72 @@ def git(command, cwd, enforce = False):
|
||||
if enforce:
|
||||
raise RuntimeError("git error")
|
||||
|
||||
def initRepo(config):
|
||||
dataDir = config["storage"]["local"]
|
||||
def getResultsDir(config):
|
||||
return os.path.join(config["storage"]["local"], "results")
|
||||
|
||||
# clone repo if not existing
|
||||
if not os.path.isdir(config["storage"]["local"]):
|
||||
git(["clone", config["storage"]["remote"], dataDir], None, enforce = True)
|
||||
def getConfigDir(config):
|
||||
return os.path.join(config["storage"]["local"], "config")
|
||||
|
||||
def getStatusDir(config):
|
||||
return os.path.join(config["storage"]["local"], "status")
|
||||
|
||||
def configureGit(dir, config):
|
||||
# default settings
|
||||
git(["config", "--local", "user.name", config["storage"]["userName"]], dataDir, enforce = True)
|
||||
git(["config", "--local", "user.email", config["storage"]["userEMail"]], dataDir, enforce = True)
|
||||
git(["config", "--local", "user.name", config["storage"]["userName"]], dir, enforce = True)
|
||||
git(["config", "--local", "user.email", config["storage"]["userEMail"]], dir, enforce = True)
|
||||
|
||||
if "userSigningKey" in config["storage"]:
|
||||
git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], dataDir, enforce = True)
|
||||
git(["config", "--local", "commit.gpgsign", "true"], dataDir, enforce = True)
|
||||
git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], dir, enforce = True)
|
||||
git(["config", "--local", "commit.gpgsign", "true"], dir, enforce = True)
|
||||
else:
|
||||
git(["config", "--local", "commit.gpgsign", "false"], dataDir, enforce = True)
|
||||
git(["config", "--local", "commit.gpgsign", "false"], dir, enforce = True)
|
||||
|
||||
def initRepo(config):
|
||||
resultsDir = getResultsDir(config)
|
||||
configDir = getConfigDir(config)
|
||||
statusDir = getStatusDir(config)
|
||||
|
||||
# clone repo if not existing
|
||||
if not os.path.isdir(statusDir):
|
||||
os.makedirs(statusDir)
|
||||
git(["clone", config["storage"]["remote"], "--branch=" + config["storage"]["branches"]["status"], statusDir], None, enforce = True)
|
||||
git(["worktree", "add", os.path.join("..", "results"), "-b" + config["storage"]["branches"]["results"], "origin/" + config["storage"]["branches"]["results"]], statusDir, enforce = True)
|
||||
git(["branch", "--set-upstream-to=" + "origin/" + config["storage"]["branches"]["results"]], resultsDir, enforce = True)
|
||||
git(["worktree", "add", os.path.join("..", "config"), "-b" + config["storage"]["branches"]["config"], "origin/" + config["storage"]["branches"]["config"]], statusDir, enforce = True)
|
||||
git(["branch", "--set-upstream-to=" + "origin/" + config["storage"]["branches"]["config"]], configDir, enforce = True)
|
||||
|
||||
# default settings
|
||||
configureGit(resultsDir, config)
|
||||
configureGit(configDir, config)
|
||||
configureGit(statusDir, config)
|
||||
|
||||
if "userSigningKey" in config["storage"]:
|
||||
git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], statusDir, enforce = True)
|
||||
git(["config", "--local", "commit.gpgsign", "true"], statusDir, enforce = True)
|
||||
else:
|
||||
git(["config", "--local", "commit.gpgsign", "false"], statusDir, enforce = True)
|
||||
|
||||
# fetch origin
|
||||
git(["fetch"], cwd = dataDir)
|
||||
git(["fetch"], cwd = statusDir)
|
||||
|
||||
# pull all branches
|
||||
for key, branch in config["storage"]["branches"].items():
|
||||
git(["checkout", branch], cwd = dataDir, enforce = True)
|
||||
git(["pull"], cwd = dataDir)
|
||||
git(["pull"], cwd = configDir)
|
||||
git(["pull"], cwd = statusDir)
|
||||
git(["pull"], cwd = resultsDir)
|
||||
|
||||
def readBenchmarkConfig(config):
|
||||
initRepo(config)
|
||||
|
||||
dataDir = config["storage"]["local"]
|
||||
|
||||
# checkout config branch
|
||||
git(["checkout", config["storage"]["branches"]["config"]], cwd = dataDir, enforce = True)
|
||||
configDir = getConfigDir(config)
|
||||
|
||||
# read instance list
|
||||
instancesFile = os.path.join(config["storage"]["local"], "instances.yml")
|
||||
instancesFile = os.path.join(configDir, "instances.yml")
|
||||
|
||||
with open(instancesFile, "r") as stream:
|
||||
instances = yaml.load(stream, Loader = yaml.CLoader)
|
||||
|
||||
# read configurations to test
|
||||
configurationsFile = os.path.join(config["storage"]["local"], "configurations.yml")
|
||||
configurationsFile = os.path.join(configDir, "configurations.yml")
|
||||
|
||||
with open(configurationsFile, "r") as stream:
|
||||
configurations = yaml.load(stream, Loader = yaml.CLoader)
|
||||
@ -130,11 +155,7 @@ def outputFilenames(configuration, instance, config):
|
||||
|
||||
def nextJob(config):
|
||||
benchmarkConfig = readBenchmarkConfig(config)
|
||||
|
||||
dataDir = config["storage"]["local"]
|
||||
|
||||
# checkout results branch
|
||||
git(["checkout", config["storage"]["branches"]["results"]], cwd = dataDir, enforce = True)
|
||||
resultsDir = getResultsDir(config)
|
||||
|
||||
configurations = benchmarkConfig["configurations"]["configurations"]
|
||||
instances = benchmarkConfig["instances"]
|
||||
@ -143,9 +164,9 @@ def nextJob(config):
|
||||
for instance in instanceSet:
|
||||
for configuration in configurations:
|
||||
filenames = outputFilenames(configuration, instance, config)
|
||||
outputFile = os.path.join(config["storage"]["local"], filenames["outputFile"])
|
||||
errorFile = os.path.join(config["storage"]["local"], filenames["errorFile"])
|
||||
environmentFile = os.path.join(config["storage"]["local"], filenames["environmentFile"])
|
||||
outputFile = os.path.join(resultsDir, filenames["outputFile"])
|
||||
errorFile = os.path.join(resultsDir, filenames["errorFile"])
|
||||
environmentFile = os.path.join(resultsDir, filenames["environmentFile"])
|
||||
|
||||
if not instanceSetID in configuration["instanceSets"]:
|
||||
continue
|
||||
@ -156,12 +177,8 @@ def nextJob(config):
|
||||
return None
|
||||
|
||||
def writeStatus(message, config):
|
||||
dataDir = config["storage"]["local"]
|
||||
|
||||
# checkout status branch
|
||||
git(["checkout", config["storage"]["branches"]["status"]], cwd = dataDir, enforce = True)
|
||||
|
||||
statusFilename = os.path.join(dataDir, "status.log")
|
||||
statusDir = getStatusDir(config)
|
||||
statusFilename = os.path.join(statusDir, "status.log")
|
||||
|
||||
if os.path.exists(statusFilename):
|
||||
with open(statusFilename, "r") as statusFile:
|
||||
@ -173,22 +190,18 @@ def writeStatus(message, config):
|
||||
with open(statusFilename, "w") as statusFile:
|
||||
print(time.strftime("%Y-%m-%d %H:%M:%S %z") + "\t" + message + "\n" + "".join(content), file = statusFile, end = "")
|
||||
|
||||
git(["add", "status.log"], dataDir, enforce = True)
|
||||
git(["commit", "-m Update status: " + message], dataDir, enforce = True)
|
||||
git(["push"], dataDir)
|
||||
git(["add", "status.log"], statusDir, enforce = True)
|
||||
git(["commit", "-m Update status: " + message], statusDir, enforce = True)
|
||||
git(["push", "origin", config["storage"]["branches"]["status"]], statusDir)
|
||||
|
||||
def runJob(configuration, instance, config):
|
||||
jobName = "[" + str(configuration["id"]) + " | " + instance["ipc"] + " | " + instance["domain"] + " | " + str(instance["instance"]) + "]"
|
||||
|
||||
writeStatus("started benchmark job " + jobName, config)
|
||||
|
||||
dataDir = config["storage"]["local"]
|
||||
|
||||
resultsDir = getResultsDir(config)
|
||||
inputFiles = inputFilenames(instance, config)
|
||||
|
||||
# checkout results branch
|
||||
git(["checkout", config["storage"]["branches"]["results"]], cwd = dataDir, enforce = True)
|
||||
|
||||
command = \
|
||||
[
|
||||
config["executables"]["timeout"]["binary"],
|
||||
@ -207,14 +220,14 @@ def runJob(configuration, instance, config):
|
||||
stdout, stderr, exitCode = executeCommand(command, cwd = plannerDir)
|
||||
|
||||
outputFiles = outputFilenames(configuration, instance, config)
|
||||
outputDir = os.path.dirname(os.path.join(config["storage"]["local"], outputFiles["outputFile"]))
|
||||
outputDir = os.path.dirname(os.path.join(resultsDir, outputFiles["outputFile"]))
|
||||
|
||||
if not os.path.isdir(outputDir):
|
||||
os.makedirs(outputDir)
|
||||
|
||||
with open(os.path.join(config["storage"]["local"], outputFiles["outputFile"]), "w") as outputFile, \
|
||||
open(os.path.join(config["storage"]["local"], outputFiles["errorFile"]), "w") as errorFile, \
|
||||
open(os.path.join(config["storage"]["local"], outputFiles["environmentFile"]), "w") as environmentFile:
|
||||
with open(os.path.join(resultsDir, outputFiles["outputFile"]), "w") as outputFile, \
|
||||
open(os.path.join(resultsDir, outputFiles["errorFile"]), "w") as errorFile, \
|
||||
open(os.path.join(resultsDir, outputFiles["environmentFile"]), "w") as environmentFile:
|
||||
print(stdout, file = outputFile)
|
||||
print("# configuration: " + str(configuration), file = errorFile)
|
||||
print("# instance: " + str(instance), file = errorFile)
|
||||
@ -245,15 +258,15 @@ def runJob(configuration, instance, config):
|
||||
|
||||
print(yaml.dump(environment, default_flow_style = False), file = environmentFile)
|
||||
|
||||
git(["add", outputFiles["outputFile"], outputFiles["errorFile"], outputFiles["environmentFile"]], dataDir, enforce = True)
|
||||
git(["add", outputFiles["outputFile"], outputFiles["errorFile"], outputFiles["environmentFile"]], resultsDir, enforce = True)
|
||||
|
||||
if exitCode == 0:
|
||||
message = "Add benchmark result"
|
||||
else:
|
||||
message = "Add corrupt benchmark result"
|
||||
|
||||
git(["commit", "-m " + message + " " + jobName], dataDir, enforce = True)
|
||||
git(["push"], dataDir)
|
||||
git(["commit", "-m " + message + " " + jobName], resultsDir, enforce = True)
|
||||
git(["push", "origin", config["storage"]["branches"]["results"]], resultsDir)
|
||||
|
||||
if exitCode != 0:
|
||||
writeStatus("errors reported for benchmark job " + jobName, config)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user