1
0
Fork 0

Use Git worktrees instead of branches

Checking out branches with many files consumes increasing amounts of
time. For this reason, this commit introduces worktrees that represent
the individual branches in different directories, while still
maintaining only one copy of the .git directory.
This commit is contained in:
Patrick Lühne 2017-11-24 15:11:57 +01:00
parent 966746e4f9
commit 24b1bf655b
Signed by: patrick
GPG Key ID: 05F3611E97A70ABF
1 changed files with 62 additions and 49 deletions

View File

@ -61,47 +61,72 @@ def git(command, cwd, enforce = False):
if enforce:
raise RuntimeError("git error")
def initRepo(config):
dataDir = config["storage"]["local"]
def getResultsDir(config):
return os.path.join(config["storage"]["local"], "results")
# clone repo if not existing
if not os.path.isdir(config["storage"]["local"]):
git(["clone", config["storage"]["remote"], dataDir], None, enforce = True)
def getConfigDir(config):
return os.path.join(config["storage"]["local"], "config")
def getStatusDir(config):
return os.path.join(config["storage"]["local"], "status")
def configureGit(dir, config):
# default settings
git(["config", "--local", "user.name", config["storage"]["userName"]], dataDir, enforce = True)
git(["config", "--local", "user.email", config["storage"]["userEMail"]], dataDir, enforce = True)
git(["config", "--local", "user.name", config["storage"]["userName"]], dir, enforce = True)
git(["config", "--local", "user.email", config["storage"]["userEMail"]], dir, enforce = True)
if "userSigningKey" in config["storage"]:
git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], dataDir, enforce = True)
git(["config", "--local", "commit.gpgsign", "true"], dataDir, enforce = True)
git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], dir, enforce = True)
git(["config", "--local", "commit.gpgsign", "true"], dir, enforce = True)
else:
git(["config", "--local", "commit.gpgsign", "false"], dataDir, enforce = True)
git(["config", "--local", "commit.gpgsign", "false"], dir, enforce = True)
def initRepo(config):
resultsDir = getResultsDir(config)
configDir = getConfigDir(config)
statusDir = getStatusDir(config)
# clone repo if not existing
if not os.path.isdir(statusDir):
os.makedirs(statusDir)
git(["clone", config["storage"]["remote"], "--branch=" + config["storage"]["branches"]["status"], statusDir], None, enforce = True)
git(["worktree", "add", os.path.join("..", "results"), "-b" + config["storage"]["branches"]["results"], "origin/" + config["storage"]["branches"]["results"]], statusDir, enforce = True)
git(["branch", "--set-upstream-to=" + "origin/" + config["storage"]["branches"]["results"]], resultsDir, enforce = True)
git(["worktree", "add", os.path.join("..", "config"), "-b" + config["storage"]["branches"]["config"], "origin/" + config["storage"]["branches"]["config"]], statusDir, enforce = True)
git(["branch", "--set-upstream-to=" + "origin/" + config["storage"]["branches"]["config"]], configDir, enforce = True)
# default settings
configureGit(resultsDir, config)
configureGit(configDir, config)
configureGit(statusDir, config)
if "userSigningKey" in config["storage"]:
git(["config", "--local", "user.signingkey", config["storage"]["userSigningKey"]], statusDir, enforce = True)
git(["config", "--local", "commit.gpgsign", "true"], statusDir, enforce = True)
else:
git(["config", "--local", "commit.gpgsign", "false"], statusDir, enforce = True)
# fetch origin
git(["fetch"], cwd = dataDir)
git(["fetch"], cwd = statusDir)
# pull all branches
for key, branch in config["storage"]["branches"].items():
git(["checkout", branch], cwd = dataDir, enforce = True)
git(["pull"], cwd = dataDir)
git(["pull"], cwd = configDir)
git(["pull"], cwd = statusDir)
git(["pull"], cwd = resultsDir)
def readBenchmarkConfig(config):
initRepo(config)
dataDir = config["storage"]["local"]
# checkout config branch
git(["checkout", config["storage"]["branches"]["config"]], cwd = dataDir, enforce = True)
configDir = getConfigDir(config)
# read instance list
instancesFile = os.path.join(config["storage"]["local"], "instances.yml")
instancesFile = os.path.join(configDir, "instances.yml")
with open(instancesFile, "r") as stream:
instances = yaml.load(stream, Loader = yaml.CLoader)
# read configurations to test
configurationsFile = os.path.join(config["storage"]["local"], "configurations.yml")
configurationsFile = os.path.join(configDir, "configurations.yml")
with open(configurationsFile, "r") as stream:
configurations = yaml.load(stream, Loader = yaml.CLoader)
@ -130,11 +155,7 @@ def outputFilenames(configuration, instance, config):
def nextJob(config):
benchmarkConfig = readBenchmarkConfig(config)
dataDir = config["storage"]["local"]
# checkout results branch
git(["checkout", config["storage"]["branches"]["results"]], cwd = dataDir, enforce = True)
resultsDir = getResultsDir(config)
configurations = benchmarkConfig["configurations"]["configurations"]
instances = benchmarkConfig["instances"]
@ -143,9 +164,9 @@ def nextJob(config):
for instance in instanceSet:
for configuration in configurations:
filenames = outputFilenames(configuration, instance, config)
outputFile = os.path.join(config["storage"]["local"], filenames["outputFile"])
errorFile = os.path.join(config["storage"]["local"], filenames["errorFile"])
environmentFile = os.path.join(config["storage"]["local"], filenames["environmentFile"])
outputFile = os.path.join(resultsDir, filenames["outputFile"])
errorFile = os.path.join(resultsDir, filenames["errorFile"])
environmentFile = os.path.join(resultsDir, filenames["environmentFile"])
if not instanceSetID in configuration["instanceSets"]:
continue
@ -156,12 +177,8 @@ def nextJob(config):
return None
def writeStatus(message, config):
dataDir = config["storage"]["local"]
# checkout status branch
git(["checkout", config["storage"]["branches"]["status"]], cwd = dataDir, enforce = True)
statusFilename = os.path.join(dataDir, "status.log")
statusDir = getStatusDir(config)
statusFilename = os.path.join(statusDir, "status.log")
if os.path.exists(statusFilename):
with open(statusFilename, "r") as statusFile:
@ -173,22 +190,18 @@ def writeStatus(message, config):
with open(statusFilename, "w") as statusFile:
print(time.strftime("%Y-%m-%d %H:%M:%S %z") + "\t" + message + "\n" + "".join(content), file = statusFile, end = "")
git(["add", "status.log"], dataDir, enforce = True)
git(["commit", "-m Update status: " + message], dataDir, enforce = True)
git(["push"], dataDir)
git(["add", "status.log"], statusDir, enforce = True)
git(["commit", "-m Update status: " + message], statusDir, enforce = True)
git(["push", "origin", config["storage"]["branches"]["status"]], statusDir)
def runJob(configuration, instance, config):
jobName = "[" + str(configuration["id"]) + " | " + instance["ipc"] + " | " + instance["domain"] + " | " + str(instance["instance"]) + "]"
writeStatus("started benchmark job " + jobName, config)
dataDir = config["storage"]["local"]
resultsDir = getResultsDir(config)
inputFiles = inputFilenames(instance, config)
# checkout results branch
git(["checkout", config["storage"]["branches"]["results"]], cwd = dataDir, enforce = True)
command = \
[
config["executables"]["timeout"]["binary"],
@ -207,14 +220,14 @@ def runJob(configuration, instance, config):
stdout, stderr, exitCode = executeCommand(command, cwd = plannerDir)
outputFiles = outputFilenames(configuration, instance, config)
outputDir = os.path.dirname(os.path.join(config["storage"]["local"], outputFiles["outputFile"]))
outputDir = os.path.dirname(os.path.join(resultsDir, outputFiles["outputFile"]))
if not os.path.isdir(outputDir):
os.makedirs(outputDir)
with open(os.path.join(config["storage"]["local"], outputFiles["outputFile"]), "w") as outputFile, \
open(os.path.join(config["storage"]["local"], outputFiles["errorFile"]), "w") as errorFile, \
open(os.path.join(config["storage"]["local"], outputFiles["environmentFile"]), "w") as environmentFile:
with open(os.path.join(resultsDir, outputFiles["outputFile"]), "w") as outputFile, \
open(os.path.join(resultsDir, outputFiles["errorFile"]), "w") as errorFile, \
open(os.path.join(resultsDir, outputFiles["environmentFile"]), "w") as environmentFile:
print(stdout, file = outputFile)
print("# configuration: " + str(configuration), file = errorFile)
print("# instance: " + str(instance), file = errorFile)
@ -245,15 +258,15 @@ def runJob(configuration, instance, config):
print(yaml.dump(environment, default_flow_style = False), file = environmentFile)
git(["add", outputFiles["outputFile"], outputFiles["errorFile"], outputFiles["environmentFile"]], dataDir, enforce = True)
git(["add", outputFiles["outputFile"], outputFiles["errorFile"], outputFiles["environmentFile"]], resultsDir, enforce = True)
if exitCode == 0:
message = "Add benchmark result"
else:
message = "Add corrupt benchmark result"
git(["commit", "-m " + message + " " + jobName], dataDir, enforce = True)
git(["push"], dataDir)
git(["commit", "-m " + message + " " + jobName], resultsDir, enforce = True)
git(["push", "origin", config["storage"]["branches"]["results"]], resultsDir)
if exitCode != 0:
writeStatus("errors reported for benchmark job " + jobName, config)