PR curve

src/descriptors/mod.rs

@@ -76,6 +76,6 @@ impl Descriptor for Median {
 pub fn get_all_descriptors() -> Vec<Box<dyn Descriptor>> {
 	let mut descriptors: Vec<Box<dyn Descriptor>> = Vec::with_capacity(2);
 	descriptors.push(Box::new(DCT::new()));
-	descriptors.push(Box::new(Median));
+	//descriptors.push(Box::new(Median));
 	descriptors
 }

src/store.rs

@@ -19,9 +19,9 @@ pub enum SaveError {
 
 /// Struct for calculating Recall-Precision per mutation
 struct PRStats {
-	true_positives: u64,
-	false_positives: u64,
-	false_negatives: u64,
+	true_positives: [u64; 64],
+	false_positives: [u64; 64],
+	false_negatives: [u64; 64],
 	tag: String,
 	name: String
 }
@@ -131,6 +131,15 @@ impl DescriptorStore {
 		}
 	}
 
+	pub fn get(&self, value: &String) -> Option<u64> {
+		for (key, bucket) in self.map.iter() {
+			if bucket.contains(&value) {
+				return Some(*key);
+			}
+		}
+		None
+	}
+
 	/// Inserts a single value into the store
 	pub fn insert(&mut self, key: &u64, value: String) {
 		let bucket = match self.map.get(&key) {
@@ -196,6 +205,19 @@ impl DescriptorStore {
 		neighbours
 	}
 
+
+	/// Returns a flat vector with all the filenames that are within a given distance
+	pub fn nn_flat_results(&self, from: u64, max_distance: isize) -> Vec<String> {
+		let nn = self.nn(from, max_distance);
+		let mut results: Vec<String> = Vec::new();
+		for (key, _) in nn {
+			// We know the key exists, so getting the result should never be none
+			let mut bucket = self.map.get(key).unwrap().clone();
+			results.append(&mut bucket);
+		}
+		results
+	}
+
 	pub fn print_nn(&self, from: u64, max_distance: isize, show_images: bool) {
 		//println!("{from:064b}");
 		let neighbours = self.nn(from, max_distance);
@@ -263,9 +285,9 @@ impl DescriptorStore {
 		for mutator in get_all_mutators() {
 			mutator_stats.push(
 				PRStats {
-					true_positives: 0,
-					false_positives: 0,
-					false_negatives: 0,
+					true_positives: [0; 64],
+					false_positives: [0; 64],
+					false_negatives: [0; 64],
 					tag: "mut".to_string() + &mutator.tag(),
 					name: mutator.info(),
 				}
@@ -273,54 +295,65 @@ impl DescriptorStore {
 		}
 
 		let set = self.seen.clone().unwrap();
-
 		for image in set.iter() {
-			debug!("Checking {}", image);
+			let phash = self.get(image).unwrap();
+			debug!("Checking {}, with phash: {}", image, phash);
+			for threshold in 0..64 {
+				debug!("Threshold: {}", threshold);
+				//Assume miss, therefore a false negative
+				//Undo the miss when there is a true positive
+				for mutator in &mut mutator_stats {
+					mutator.false_negatives[threshold as usize] += 1;
+				}
 
-			// find the bucket this image is in
-			for (key, bucket) in self.map.iter() {
-				if bucket.contains(&image) {
-					debug!("{} found in bucket {}", image, key);
-					for value in bucket {
-						if value != image && value.ends_with(image) {
-							// Mutated version of image in same bucket
+				for found in self.nn_flat_results(phash, threshold) {
+					if found.eq(image) || found.ends_with(&format!(".{}", image)) { // True positive
 						for mutator in &mut mutator_stats {
-								mutator.false_negatives += 1;
-								if value.starts_with(&mutator.tag) {
-									mutator.true_positives += 1;
-									mutator.false_negatives -= 1;
+							if found.starts_with(&mutator.tag) {
+								debug!("{} is hit for {}", found, mutator.name);
+								mutator.true_positives[threshold as usize] += 1;
+								mutator.false_negatives[threshold as usize] -= 1;
 							}
 						}
-							debug!("We matched {} to {}!", value, image);
-						} else if value != image {
-							// Other image in same bucket
-							//false_positives += 1;
+					} else { // False positive!
+						// Mutated misses only count for the mutator
+						// Unmutated misses count for everyone
 						for mutator in &mut mutator_stats {
-								if value.starts_with(&mutator.tag) {
-									mutator.false_positives += 1;
+							if found.starts_with(&mutator.tag) || !found.starts_with("mut") {
+								debug!("{} is false positive for {}", found, mutator.name);
+								mutator.false_positives[threshold as usize] += 1;
 							}
 						}
 					}
 				}
-					break;
 			}
 		}
-		}
-		// 
+
 		for mutator in &mut mutator_stats {
+			for threshold in 0..64 {
+				let tpos = mutator.true_positives[threshold] as f64;
+				let fpos = mutator.false_positives[threshold] as f64;
+				let fneg = mutator.false_negatives[threshold] as f64;
+				let p = tpos / (tpos + fpos);
+				let r = tpos / (tpos + fneg);
+				let f1 = 2.0*tpos / (2.0*tpos + fpos + fneg);
+				println!("{} / {}:", mutator.name, threshold);
+				println!("Precision: {}, Recall: {}, F1 score: {}", p, r, f1);
+				println!("Tp: {}, Fp: {}, Fn: {}", tpos, fpos, fneg);
+			}
 			// println!("\t{} true positives: {}", mutator.name, mutator.true_positives);
 			// println!("\t{} false positives: {}", mutator.name, mutator.false_positives);
-			println!(
-				"\t{} Precision: {:.3}", mutator.name, 
-				mutator.true_positives as f64 / 
-				(mutator.true_positives + mutator.false_positives) as f64
-			);
-			println!(
-				"\t{} Recall: {:.3}", mutator.name, 
-				mutator.true_positives as f64 / 
-				//(mutator.true_positives + mutator.false_negatives) as f64
-				set.len() as f64
-			);
+			// println!(
+			// 	"\t{} Precision: {:.3}", mutator.name, 
+			// 	mutator.true_positives as f64 / 
+			// 	(mutator.true_positives + mutator.false_positives) as f64
+			// );
+			// println!(
+			// 	"\t{} Recall: {:.3}", mutator.name, 
+			// 	mutator.true_positives as f64 / 
+			// 	//(mutator.true_positives + mutator.false_negatives) as f64
+			// 	set.len() as f64
+			// );
 		}
 		//println!("Images processed: {}", set.len())
 		// println!("{} true positives", true_positives);