diff --git a/benchmark.py b/benchmark.py
index ef2c045fa..914df8f8c 100644
--- a/benchmark.py
+++ b/benchmark.py
@@ -139,7 +139,7 @@ def nextJob(config):
 	configurations = benchmarkConfig["configurations"]["configurations"]
 	instances = benchmarkConfig["instances"]
 
-	for instanceSetName, instanceSet in instances.items():
+	for instanceSetID, instanceSet in instances.items():
 		for instance in instanceSet:
 			for configuration in configurations:
 				filenames = outputFilenames(configuration, instance, config)
@@ -147,6 +147,9 @@ def nextJob(config):
 				errorFile = os.path.join(config["storage"]["local"], filenames["errorFile"])
 				environmentFile = os.path.join(config["storage"]["local"], filenames["environmentFile"])
 
+				if not instanceSetID in configuration["instanceSets"]:
+					continue
+
 				if not os.path.exists(outputFile) or not os.path.exists(errorFile) or not os.path.exists(environmentFile):
 					return {"configuration": configuration, "instance": instance}