Update to latest revision

Cargo.lock

@@ -35,6 +35,7 @@ dependencies = [
  "notify 4.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_env_logger 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "walkdir 2.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "yaml-rust 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -248,8 +249,8 @@ dependencies = [
  "inotify-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
  "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -715,7 +716,7 @@ dependencies = [
 
 [[package]]
 name = "tokio-executor"
-version = "0.1.5"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "futures 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -723,7 +724,7 @@ dependencies = [
 
 [[package]]
 name = "tokio-io"
-version = "0.1.9"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -733,7 +734,7 @@ dependencies = [
 
 [[package]]
 name = "tokio-reactor"
-version = "0.1.6"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -744,8 +745,8 @@ dependencies = [
  "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -964,9 +965,9 @@ dependencies = [
 "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
 "checksum termios 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "70226acdf12d182df757d9fb07c0257a1558ec48c8059f607d6b38145ce4e2fa"
 "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
-"checksum tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c117b6cf86bb730aab4834f10df96e4dd586eff2c3c27d3781348da49e255bde"
+"checksum tokio-executor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "84823b932d566bc3c6aa644df4ca36cb38593c50b7db06011fd4e12e31e4047e"
-"checksum tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "8b8a85fffbec3c5ab1ab62324570230dcd37ee5996a7859da5caf7b9d45e3e8c"
+"checksum tokio-io 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8d6cc2de7725863c86ac71b0b9068476fec50834f055a243558ef1655bbd34cb"
-"checksum tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4b26fd37f1125738b2170c80b551f69ff6fecb277e6e5ca885e53eec2b005018"
+"checksum tokio-reactor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4bfbaf9f260635649ec26b6fb4aded03887295ffcd999f6e43fd2c4758f758ea"
 "checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d"
 "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
 "checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25"

Cargo.toml

@@ -11,6 +11,7 @@ indicatif = "0.9"
 yaml-rust = "0.4"
 notify = "4"
 tempfile = "3"
+walkdir = "2"
 
 [dependencies.cpython]
 git = "https://github.com/dgrunwald/rust-cpython/"

examples/test.rs

@@ -16,10 +16,18 @@ fn main()
 
 	let mut benchmark_repository = BenchmarkRepository::new("git@git.luehne.de:patrick/tplp-planning-benchmark.git", Path::new("storage"), "git", "Potassco Bot", "bot@potassco.org");
 
-	let content = benchmark_repository.read_file(Path::new("configurations.yml"), "config").unwrap();
+	/*let files = vec!
+		[
+			TargetPath{source: &Path::new("/tmp/test"), destination: &Path::new("foobar/test")},
+			TargetPath{source: &Path::new("/tmp/test2"), destination: &Path::new("foobar/test-2")},
+		];
+
+	benchmark_repository.commit_files(&files[..], "test-results");*/
+
+	let content = benchmark_repository.read_file(Path::new("configurations.yml"), "test-config").unwrap();
 	let configurations = &YamlLoader::load_from_str(&content).unwrap()[0]["configurations"];
 
-	let content = benchmark_repository.read_file(Path::new("instances.yml"), "config").unwrap();
+	let content = benchmark_repository.read_file(Path::new("instances.yml"), "test-config").unwrap();
 	let instances = &YamlLoader::load_from_str(&content).unwrap()[0];
 
 	for configuration in configurations.as_vec().unwrap()
@@ -37,13 +45,13 @@ fn main()
 				let file_name_base = format!("{}/{}/{}/{}", configuration_id, instance_ipc, instance_domain, instance_number);
 				let file_name_output = format!("{}.out", file_name_base);
 
-				if (benchmark_repository.file_exists(Path::new(&file_name_output), "results"))
+				if (benchmark_repository.file_exists(Path::new(&file_name_output), "test-results"))
 				{
 					println!("done: [{}, {}, {}/{}/{}]", configuration_id, instance_set_id, instance_ipc, instance_domain, instance_number);
 					continue;
 				}
 
-				let test = benchmark_repository.create_autocommit_directory(Path::new(&file_name_base), "results").unwrap();
+				let test = benchmark_repository.create_autocommit_directory(Path::new(&file_name_base), "test-results").unwrap();
 				//println!("{}", test.display());
 
 				//println!("to do: [{}, {}, {}/{}/{}]", configuration_id, instance_set_id, instance_ipc, instance_domain, instance_number);
@@ -51,8 +59,5 @@ fn main()
 		}
 	}
 
-	let duration = time::Duration::from_secs(20);
-	thread::sleep(duration);
-
 	benchmark_repository.wait_for_autocommit_thread();
 }

src/lib.rs

@@ -5,36 +5,45 @@ extern crate log;
 extern crate indicatif;
 extern crate notify;
 extern crate tempfile;
+extern crate walkdir;
 
 use git2::{Cred, Error, FetchOptions, Index, IndexEntry, IndexTime, Progress, PushOptions, RemoteCallbacks, ResetType, Repository, Signature, TreeBuilder};
 use git2::build::{CheckoutBuilder, RepoBuilder};
-use std::fs::{File, create_dir_all, read};
+use std::fs::{File, create_dir_all, read, remove_file};
 use std::io::Cursor;
 use std::path::{Path, PathBuf};
 use std::string::String;
 use std::str;
+use std::sync::{Arc, Mutex};
 use std::sync::mpsc::{channel, Sender, Receiver};
 use std::thread::{spawn, JoinHandle};
 use indicatif::{ProgressBar, ProgressStyle};
-use notify::{raw_watcher, RecommendedWatcher, RawEvent, Op};
+use notify::{raw_watcher, RecommendedWatcher, RawEvent, Op, RecursiveMode, Watcher};
 use tempfile::TempDir;
+use walkdir::WalkDir;
 
-pub struct BenchmarkRepository
+pub struct BenchmarkRepositoryInner
 {
 	repository: Repository,
 	ssh_user: String,
 	user_name: String,
 	user_email: String,
+}
+
+pub struct BenchmarkRepository
+{
+	inner: Arc<Mutex<BenchmarkRepositoryInner>>,
 	autocommit_channel: (Sender<RawEvent>, Receiver<RawEvent>),
 	autocommit_directory: Option<TempDir>,
 	autocommit_thread: Option<JoinHandle<()>>,
 	autocommit_watcher: Option<RecommendedWatcher>,
+	autocommit_locks: Arc<Mutex<usize>>,
 }
 
-pub struct TargetPath<'a>
+pub struct TargetPath
 {
-	pub source: &'a Path,
+	pub source: PathBuf,
-	pub destination: &'a Path,
+	pub destination: PathBuf,
 }
 
 impl BenchmarkRepository
@@ -74,11 +83,13 @@ impl BenchmarkRepository
 
 	fn reset_origin(&self, remote_url: &str) -> &Self
 	{
-		let remote = match self.repository.find_remote("origin")
+		let inner = self.inner.lock().unwrap();
+
+		let remote = match inner.repository.find_remote("origin")
 		{
 			Ok(remote) => remote,
 			Err(_) =>
-				match self.repository.remote("origin", remote_url)
+				match inner.repository.remote("origin", remote_url)
 				{
 					Ok(remote) => remote,
 					Err(error) => panic!("Could not reset remote “origin”: {}", error),
@@ -92,6 +103,8 @@ impl BenchmarkRepository
 
 	fn fetch_branch(&self, branch_name: &str) -> &Self
 	{
+		let inner = self.inner.lock().unwrap();
+
 		let mut progress_bar = ProgressBar::new(0);
 		progress_bar.set_style(BenchmarkRepository::progress_bar_style());
 
@@ -100,10 +113,10 @@ impl BenchmarkRepository
 			remote_callbacks.credentials(
 				|_, _, _|
 				{
-					match Cred::ssh_key_from_agent(&self.ssh_user)
+					match Cred::ssh_key_from_agent(&inner.ssh_user)
 					{
 						Ok(credentials) => Ok(credentials),
-						Err(error) => panic!("could not retrieve key pair for SSH authentication as user “{}”: {}", self.ssh_user, error),
+						Err(error) => panic!("could not retrieve key pair for SSH authentication as user “{}”: {}", inner.ssh_user, error),
 					}
 				});
 
@@ -113,7 +126,7 @@ impl BenchmarkRepository
 			let mut fetch_options = FetchOptions::new();
 			fetch_options.remote_callbacks(remote_callbacks);
 
-			let mut origin = self.repository.find_remote("origin").expect("could not find remote “origin”");
+			let mut origin = inner.repository.find_remote("origin").expect("could not find remote “origin”");
 
 			info!("Updating branch “{}”", branch_name);
 
@@ -156,17 +169,24 @@ impl BenchmarkRepository
 			Err(_) => BenchmarkRepository::init(base_path),
 		};
 
-		let benchmark_repository =
+		let benchmark_repository_inner =
-			BenchmarkRepository
+			BenchmarkRepositoryInner
 			{
 				repository: repository,
 				ssh_user: ssh_user.to_string(),
 				user_name: user_name.to_string(),
 				user_email: user_email.to_string(),
+			};
+
+		let benchmark_repository =
+			BenchmarkRepository
+			{
+				inner: Arc::new(Mutex::new(benchmark_repository_inner)),
 				autocommit_channel: channel(),
 				autocommit_directory: None,
 				autocommit_thread: None,
 				autocommit_watcher: None,
+				autocommit_locks: Arc::new(Mutex::new(0)),
 			};
 
 		benchmark_repository
@@ -178,10 +198,10 @@ impl BenchmarkRepository
 		benchmark_repository
 	}
 
-	pub fn read_file_as_index_entry(&self, file_path: &Path, result_file_path: &Path) -> IndexEntry
+	pub fn read_file_as_index_entry(inner: &BenchmarkRepositoryInner, file_path: &Path, result_file_path: &Path) -> IndexEntry
 	{
 		// create a new blob with the file contents
-		let object_id = match self.repository.blob_path(file_path)
+		let object_id = match inner.repository.blob_path(file_path)
 		{
 			Ok(object_id) => object_id,
 			Err(error) => panic!("Could not write blob for “{}”: {}", file_path.display(), error),
@@ -206,10 +226,33 @@ impl BenchmarkRepository
 		}
 	}
 
-	pub fn commit_files(&self, file_paths: &[TargetPath], branch_name: &str)
+	pub fn commit_directory(inner: &BenchmarkRepositoryInner, directory_path: &TargetPath, branch_name: &str)
+	{
+		let mut file_paths = vec![];
+
+		for entry in WalkDir::new(&directory_path.source)
+		{
+			let entry = entry.unwrap();
+
+			if entry.path().file_name().unwrap() == ".lock" || entry.file_type().is_dir()
+			{
+				continue;
+			}
+
+			let relative_path = entry.path().strip_prefix(&directory_path.source).unwrap();
+
+			trace!("Adding “{}” (from “{}”) to autocommit list", relative_path.display(), entry.path().display());
+
+			file_paths.push(TargetPath{source: entry.path().to_owned(), destination: directory_path.destination.join(&relative_path)});
+		}
+
+		BenchmarkRepository::commit_files(&inner, &file_paths, branch_name);
+	}
+
+	pub fn commit_files(inner: &BenchmarkRepositoryInner, file_paths: &[TargetPath], branch_name: &str)
 	{
 		let tip_reference_name = format!("refs/remotes/origin/{}", branch_name);
-		let tip_reference = match self.repository.find_reference(&tip_reference_name)
+		let tip_reference = match inner.repository.find_reference(&tip_reference_name)
 		{
 			Ok(value) => value,
 			Err(error) => panic!("Could not find reference “{}”: {}", tip_reference_name, error),
@@ -235,7 +278,7 @@ impl BenchmarkRepository
 
 		for target_path in file_paths
 		{
-			let index_entry = self.read_file_as_index_entry(target_path.source, target_path.destination);
+			let index_entry = BenchmarkRepository::read_file_as_index_entry(inner, &target_path.source, &target_path.destination);
 
 			if let Err(error) = index.add(&index_entry)
 			{
@@ -243,13 +286,13 @@ impl BenchmarkRepository
 			}
 		}
 
-		let tree_object_id = match index.write_tree_to(&self.repository)
+		let tree_object_id = match index.write_tree_to(&inner.repository)
 		{
 			Ok(value) => value,
 			Err(error) => panic!("Could not write index to tree: {}", error),
 		};
 
-		let tree = match self.repository.find_tree(tree_object_id)
+		let tree = match inner.repository.find_tree(tree_object_id)
 		{
 			Ok(value) => value,
 			Err(error) => panic!("Could obtain tree: {}", error),
@@ -257,7 +300,7 @@ impl BenchmarkRepository
 
 		info!("Created tree object “{}”", tree_object_id);
 
-		let signature = Signature::now(&self.user_name, &self.user_email).expect("Could not create signature");
+		let signature = Signature::now(&inner.user_name, &inner.user_email).expect("Could not create signature");
 		let message = format!("Add files");
 
 		let parent = match tip_reference.peel_to_commit()
@@ -266,7 +309,7 @@ impl BenchmarkRepository
 			Err(error) => panic!("Could not peel reference: {}", error),
 		};
 
-		let commit_id = match self.repository.commit(Some(&tip_reference_name), &signature, &signature, &message, &tree, &[&parent])
+		let commit_id = match inner.repository.commit(Some(&tip_reference_name), &signature, &signature, &message, &tree, &[&parent])
 		{
 			Ok(value) => value,
 			Err(error) => panic!("Could not write commit: {}", error),
@@ -280,10 +323,10 @@ impl BenchmarkRepository
 		remote_callbacks.credentials(
 			|_, _, _|
 			{
-				match Cred::ssh_key_from_agent(&self.ssh_user)
+				match Cred::ssh_key_from_agent(&inner.ssh_user)
 				{
 					Ok(value) => Ok(value),
-					Err(error) => panic!("could not retrieve key pair for SSH authentication as user “{}”: {}", self.ssh_user, error),
+					Err(error) => panic!("could not retrieve key pair for SSH authentication as user “{}”: {}", inner.ssh_user, error),
 				}
 			});
 
@@ -293,14 +336,16 @@ impl BenchmarkRepository
 		let mut push_options = PushOptions::new();
 		push_options.remote_callbacks(remote_callbacks);
 
-		let mut remote = self.repository.find_remote("origin").expect("");
+		let mut remote = inner.repository.find_remote("origin").expect("");
 		remote.push(&[&push_refspec], Some(&mut push_options)).expect("couldn’t push");
 	}
 
 	pub fn file_exists(&self, file_path: &Path, branch_name: &str) -> bool
 	{
+		let inner = self.inner.lock().unwrap();
+
 		let tip_reference_name = format!("refs/remotes/origin/{}", branch_name);
-		let tip_reference = match self.repository.find_reference(&tip_reference_name)
+		let tip_reference = match inner.repository.find_reference(&tip_reference_name)
 		{
 			Ok(value) => value,
 			Err(error) => panic!("Could not find reference “{}”: {}", tip_reference_name, error),
@@ -318,7 +363,7 @@ impl BenchmarkRepository
 			Err(error) => return false,
 		};
 
-		let blob = match self.repository.find_blob(object_id)
+		let blob = match inner.repository.find_blob(object_id)
 		{
 			Ok(blob) => blob,
 			Err(error) => return false,
@@ -329,8 +374,10 @@ impl BenchmarkRepository
 
 	pub fn read_file(&self, file_path: &Path, branch_name: &str) -> Option<String>
 	{
+		let inner = self.inner.lock().unwrap();
+
 		let tip_reference_name = format!("refs/remotes/origin/{}", branch_name);
-		let tip_reference = match self.repository.find_reference(&tip_reference_name)
+		let tip_reference = match inner.repository.find_reference(&tip_reference_name)
 		{
 			Ok(value) => value,
 			Err(error) => panic!("Could not find reference “{}”: {}", tip_reference_name, error),
@@ -348,7 +395,7 @@ impl BenchmarkRepository
 			Err(error) => return None,
 		};
 
-		let blob = match self.repository.find_blob(object_id)
+		let blob = match inner.repository.find_blob(object_id)
 		{
 			Ok(blob) => blob,
 			Err(error) => return None,
@@ -376,6 +423,8 @@ impl BenchmarkRepository
 					Err(error) => panic!("Could not create autocommit directory: {}", error),
 				};
 
+				trace!("Created temporary autocommit directory in {}", tmp_dir.path().display());
+
 				let lock_file_path = tmp_dir.path().join(".lock");
 
 				if let Err(error) = File::create(&lock_file_path)
@@ -385,22 +434,27 @@ impl BenchmarkRepository
 
 				let (sender, receiver) = channel();
 
-				let watcher = match raw_watcher(sender)
+				let mut watcher = match raw_watcher(sender)
 				{
 					Ok(value) => value,
 					Err(error) => panic!("Could not create filesystem watcher: {}", error),
 				};
 
+				watcher.watch(&tmp_dir, RecursiveMode::Recursive);
+
 				self.autocommit_watcher = Some(watcher);
 				self.autocommit_directory = Some(tmp_dir);
 
-				trace!("{:#?}", receiver);
+				let autocommit_base_path = self.autocommit_directory.as_ref().unwrap().path().to_owned();
+				let inner = self.inner.clone();
+				let autocommit_locks = self.autocommit_locks.clone();
 
 				let thread = spawn(
 					move ||
 					{
 						trace!("Autocommit thread running");
-						trace!("{:#?}", receiver);
+
+						let mut active = true;
 
 						loop
 						{
@@ -408,12 +462,48 @@ impl BenchmarkRepository
 							{
 								Ok(RawEvent{path: Some(path), op: Ok(Op::REMOVE), cookie: _}) =>
 								{
-									trace!("Received “remove” event for path “{}”", path.display());
+									if path == lock_file_path
+									{
+										trace!("Waiting for remaining autocommit locks to be released");
+										active = false;
+									}
+									else
+									{
+										trace!("Received “remove” event for path “{}”", path.display());
+
+										let stripped_path = match path.strip_prefix(&autocommit_base_path)
+										{
+											Ok(value) => value,
+											Err(error) => panic!("Cannot handle “remove” event for path “{}”", path.display()),
+										};
+
+										let mut components = stripped_path.components();
+										let branch_name = components.next().unwrap();
+										let result_path = stripped_path.strip_prefix(&branch_name).unwrap().parent().unwrap();
+
+										info!("Autocommiting “{}”", result_path.display());
+
+										let inner = inner.lock().unwrap();
+
+										BenchmarkRepository::commit_directory(&inner, &TargetPath{source: path.parent().unwrap().to_owned(), destination: result_path.to_owned()}, &branch_name.as_os_str().to_str().unwrap());
+
+										let mut autocommit_locks = autocommit_locks.lock().unwrap();
+										*autocommit_locks -= 1;
+									}
 								},
 								Err(error) => panic!("Error handling notify event: {}", error),
 								_ => (),
 							}
+
+							let mut autocommit_locks = autocommit_locks.lock().unwrap();
+
+							if *autocommit_locks == 0
+							{
+								break;
+							}
 						}
+
+						trace!("Autocommit thread finished");
 					});
 
 				self.autocommit_thread = Some(thread);
@@ -447,6 +537,9 @@ impl BenchmarkRepository
 			}
 		}
 
+		let mut autocommit_locks = self.autocommit_locks.lock().unwrap();
+		*autocommit_locks += 1;
+
 		Ok(result_directory_path)
 	}
 
@@ -457,6 +550,15 @@ impl BenchmarkRepository
 			panic!("No autocommit thread started");
 		}
 
+		let lock_file_path = self.autocommit_directory.as_ref().unwrap().path().join(".lock");
+
+		if let Err(error) = remove_file(&lock_file_path)
+		{
+			panic!("Could not remove lock file “{}”: {}", lock_file_path.display(), error);
+		}
+
+		info!("Removed lock file");
+
 		let autocommit_thread = self.autocommit_thread;
 
 		if let Err(RecvError) = autocommit_thread.unwrap().join()