From 3bb28a7e5bc6832f459443df5eebd6e472a57819 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Mon, 10 Jun 2024 23:41:23 +0200 Subject: [PATCH] PR curve --- src/descriptors/mod.rs | 2 +- src/store.rs | 117 ++++++++++++++++++++++++++--------------- 2 files changed, 76 insertions(+), 43 deletions(-) diff --git a/src/descriptors/mod.rs b/src/descriptors/mod.rs index 5e0e747..ee01e49 100644 --- a/src/descriptors/mod.rs +++ b/src/descriptors/mod.rs @@ -76,6 +76,6 @@ impl Descriptor for Median { pub fn get_all_descriptors() -> Vec> { let mut descriptors: Vec> = Vec::with_capacity(2); descriptors.push(Box::new(DCT::new())); - descriptors.push(Box::new(Median)); + //descriptors.push(Box::new(Median)); descriptors } \ No newline at end of file diff --git a/src/store.rs b/src/store.rs index 3756abb..3db0bb3 100644 --- a/src/store.rs +++ b/src/store.rs @@ -19,9 +19,9 @@ pub enum SaveError { /// Struct for calculating Recall-Precision per mutation struct PRStats { - true_positives: u64, - false_positives: u64, - false_negatives: u64, + true_positives: [u64; 64], + false_positives: [u64; 64], + false_negatives: [u64; 64], tag: String, name: String } @@ -131,6 +131,15 @@ impl DescriptorStore { } } + pub fn get(&self, value: &String) -> Option { + for (key, bucket) in self.map.iter() { + if bucket.contains(&value) { + return Some(*key); + } + } + None + } + /// Inserts a single value into the store pub fn insert(&mut self, key: &u64, value: String) { let bucket = match self.map.get(&key) { @@ -196,6 +205,19 @@ impl DescriptorStore { neighbours } + + /// Returns a flat vector with all the filenames that are within a given distance + pub fn nn_flat_results(&self, from: u64, max_distance: isize) -> Vec { + let nn = self.nn(from, max_distance); + let mut results: Vec = Vec::new(); + for (key, _) in nn { + // We know the key exists, so getting the result should never be none + let mut bucket = self.map.get(key).unwrap().clone(); + results.append(&mut bucket); + } + results + } + pub fn print_nn(&self, from: u64, max_distance: isize, show_images: bool) { //println!("{from:064b}"); let neighbours = self.nn(from, max_distance); @@ -263,9 +285,9 @@ impl DescriptorStore { for mutator in get_all_mutators() { mutator_stats.push( PRStats { - true_positives: 0, - false_positives: 0, - false_negatives: 0, + true_positives: [0; 64], + false_positives: [0; 64], + false_negatives: [0; 64], tag: "mut".to_string() + &mutator.tag(), name: mutator.info(), } @@ -273,54 +295,65 @@ impl DescriptorStore { } let set = self.seen.clone().unwrap(); - for image in set.iter() { - debug!("Checking {}", image); + let phash = self.get(image).unwrap(); + debug!("Checking {}, with phash: {}", image, phash); + for threshold in 0..64 { + debug!("Threshold: {}", threshold); + //Assume miss, therefore a false negative + //Undo the miss when there is a true positive + for mutator in &mut mutator_stats { + mutator.false_negatives[threshold as usize] += 1; + } - // find the bucket this image is in - for (key, bucket) in self.map.iter() { - if bucket.contains(&image) { - debug!("{} found in bucket {}", image, key); - for value in bucket { - if value != image && value.ends_with(image) { - // Mutated version of image in same bucket - for mutator in &mut mutator_stats { - mutator.false_negatives += 1; - if value.starts_with(&mutator.tag) { - mutator.true_positives += 1; - mutator.false_negatives -= 1; - } + for found in self.nn_flat_results(phash, threshold) { + if found.eq(image) || found.ends_with(&format!(".{}", image)) { // True positive + for mutator in &mut mutator_stats { + if found.starts_with(&mutator.tag) { + debug!("{} is hit for {}", found, mutator.name); + mutator.true_positives[threshold as usize] += 1; + mutator.false_negatives[threshold as usize] -= 1; } - debug!("We matched {} to {}!", value, image); - } else if value != image { - // Other image in same bucket - //false_positives += 1; - for mutator in &mut mutator_stats { - if value.starts_with(&mutator.tag) { - mutator.false_positives += 1; - } + } + } else { // False positive! + // Mutated misses only count for the mutator + // Unmutated misses count for everyone + for mutator in &mut mutator_stats { + if found.starts_with(&mutator.tag) || !found.starts_with("mut") { + debug!("{} is false positive for {}", found, mutator.name); + mutator.false_positives[threshold as usize] += 1; } } } - break; } } } - // + for mutator in &mut mutator_stats { + for threshold in 0..64 { + let tpos = mutator.true_positives[threshold] as f64; + let fpos = mutator.false_positives[threshold] as f64; + let fneg = mutator.false_negatives[threshold] as f64; + let p = tpos / (tpos + fpos); + let r = tpos / (tpos + fneg); + let f1 = 2.0*tpos / (2.0*tpos + fpos + fneg); + println!("{} / {}:", mutator.name, threshold); + println!("Precision: {}, Recall: {}, F1 score: {}", p, r, f1); + println!("Tp: {}, Fp: {}, Fn: {}", tpos, fpos, fneg); + } // println!("\t{} true positives: {}", mutator.name, mutator.true_positives); // println!("\t{} false positives: {}", mutator.name, mutator.false_positives); - println!( - "\t{} Precision: {:.3}", mutator.name, - mutator.true_positives as f64 / - (mutator.true_positives + mutator.false_positives) as f64 - ); - println!( - "\t{} Recall: {:.3}", mutator.name, - mutator.true_positives as f64 / - //(mutator.true_positives + mutator.false_negatives) as f64 - set.len() as f64 - ); + // println!( + // "\t{} Precision: {:.3}", mutator.name, + // mutator.true_positives as f64 / + // (mutator.true_positives + mutator.false_positives) as f64 + // ); + // println!( + // "\t{} Recall: {:.3}", mutator.name, + // mutator.true_positives as f64 / + // //(mutator.true_positives + mutator.false_negatives) as f64 + // set.len() as f64 + // ); } //println!("Images processed: {}", set.len()) // println!("{} true positives", true_positives);