diff --git a/benchmark.py b/benchmark.py
index 962cd9d33..40228cbad 100644
--- a/benchmark.py
+++ b/benchmark.py
@@ -136,9 +136,10 @@ def nextJob(config):
 			for configuration in configurations:
 				filenames = outputFilenames(configuration, instance, config)
 				outputFile = os.path.join(config["storage"]["local"], filenames["outputFile"])
+				errorFile = os.path.join(config["storage"]["local"], filenames["errorFile"])
 				environmentFile = os.path.join(config["storage"]["local"], filenames["environmentFile"])
 
-				if not os.path.exists(outputFile) or not os.path.exists(environmentFile):
+				if not os.path.exists(outputFile) or not os.path.exists(errorFile) or not os.path.exists(environmentFile):
 					return {"configuration": configuration, "instance": instance}
 
 	return None