From 6276ca1f77f20b9b815e142b5c72714069c286d5 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Fri, 3 May 2024 13:34:16 +0200 Subject: [PATCH 01/32] adds dct? --- src/descriptors.rs | 50 +++++++++++++++++++++++++++++++++++++++++++--- src/main.rs | 6 +++--- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/descriptors.rs b/src/descriptors.rs index d4b0e02..39815c1 100644 --- a/src/descriptors.rs +++ b/src/descriptors.rs @@ -1,4 +1,5 @@ use image::GenericImageView; +use std::f64::consts::PI; pub trait Descriptor { fn info(&self) -> String { @@ -10,16 +11,25 @@ pub trait Descriptor { } } +fn median64(values: &[T]) -> T { + let mut sorted_values = values.to_vec(); + sorted_values.sort(); + let len = sorted_values.len(); + sorted_values[len/2] +} + pub struct Median; impl Descriptor for Median { fn describe(&self, img: image::DynamicImage) -> u64 { - let mut median: u64 = 0; //Average for now while i figure out sorting in rust let img = img.grayscale().thumbnail_exact(8, 8); + let mut values: [u8; 64] = [0; 64]; + let mut i: usize = 0; for (_, _, pix) in img.pixels() { - median += pix[0] as u64; + values[i] = pix[0]; + i = i+1; } - let median: u8 = (median/64) as u8; + let median = median64(&values); let mut mask: u64 = 0; img.save("debug.png").unwrap(); for (_, _, pix) in img.pixels() { @@ -32,3 +42,37 @@ impl Descriptor for Median { } } +pub struct DCT; + +impl Descriptor for DCT { + fn describe(&self, img: image::DynamicImage) -> u64 { + let mut dct_values: [f64; 64] = [0.0; 64]; + let img = img.grayscale().thumbnail_exact(8, 8); + let mut total_sum: f64 = 0.0; + + for j in 0..7 { + for i in 0..7 { + let mut sum: f64 = 0.0; + let k = (i*8)+j; + for (x, y, pix) in img.pixels() { + let n: f64 = ((y*8)+x).into(); + let pixel: i16 = pix[0] as i16 - 127; + sum += pixel as f64 * (PI/64.0 * (n + 0.5) * k as f64).cos(); + } + dct_values[k] = sum; + total_sum += sum; + //println!("{sum}"); + } + } + total_sum /= 64.0; + + let mut mask: u64 = 0; + for dct in dct_values { + if dct > 0.0 { + mask += 1 + } + mask = mask << 1; + } + mask + } +} diff --git a/src/main.rs b/src/main.rs index 5bd9dc0..b8883fb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,4 @@ -use descriptors::{Descriptor, Median}; +use descriptors::{Descriptor, Median, DCT}; use std::collections::HashMap; use std::fs; @@ -28,10 +28,10 @@ fn main() { } for (key, val) in map.iter() { - println!("{key}: {val}"); + println!("{key: >15}: {val}\t {val:b}"); } //Serialize hashmap with MessagePack: let serialized: Vec = rmp_serde::to_vec(&map).unwrap(); - fs::write("map.messagepack",&serialized).unwrap(); + //fs::write("map.messagepack",&serialized).unwrap(); } From 16d57f9c57554e13e8e8eebef8a99c63248fd3d0 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sat, 4 May 2024 10:55:48 +0200 Subject: [PATCH 02/32] rename crate --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 05b1879..60ffb63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "image-similarity" +name = "image_similarity" version = "0.1.0" edition = "2021" From 08dd17f292dd73eb9970f8a39c552821e2ff6d72 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sat, 4 May 2024 10:56:04 +0200 Subject: [PATCH 03/32] rename crate --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 0effe9b..ec10b68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -231,7 +231,7 @@ dependencies = [ ] [[package]] -name = "image-similarity" +name = "image_similarity" version = "0.1.0" dependencies = [ "image", From 3c861cb796cc9fa96c524c9dba14a5b0aa6e5f29 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sat, 4 May 2024 10:57:22 +0200 Subject: [PATCH 04/32] move experiment binary to folder --- src/bin/experiment.rs | 35 +++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/main.rs | 36 +----------------------------------- 3 files changed, 37 insertions(+), 35 deletions(-) create mode 100644 src/bin/experiment.rs create mode 100644 src/lib.rs diff --git a/src/bin/experiment.rs b/src/bin/experiment.rs new file mode 100644 index 0000000..a1ffec5 --- /dev/null +++ b/src/bin/experiment.rs @@ -0,0 +1,35 @@ +use image_similarity::descriptors::{Descriptor, Median, DCT}; +use std::collections::HashMap; +use std::fs; + +fn main() { + //Init all descriptors: + let desc = Median; + + //Initialize hashmap, empty or from cached file: + let count = fs::read_dir("img").unwrap().count(); + let map_file = fs::read("map.messagepack"); + let mut map: HashMap = match map_file { + Ok(f) => rmp_serde::from_slice(&f).unwrap(), + Err(_e) => HashMap::with_capacity(count), + }; + + //Calculate phashes for all images + for node in fs::read_dir("img").unwrap() { + let file = node.expect("Error walking directory"); + let name = file.file_name().into_string().expect("Issue with filename"); + if !map.contains_key(&name) { + let img = image::open(file.path()).expect("Unable to open file"); + let phash = desc.describe(img); + map.insert(name, phash); + } + } + + for (key, val) in map.iter() { + println!("{key: >15}: {val}\t {val:b}"); + } + + //Serialize hashmap with MessagePack: + let serialized: Vec = rmp_serde::to_vec(&map).unwrap(); + //fs::write("map.messagepack",&serialized).unwrap(); +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..e27c649 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1 @@ +pub mod descriptors; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index b8883fb..f8ed123 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,37 +1,3 @@ -use descriptors::{Descriptor, Median, DCT}; -use std::collections::HashMap; -use std::fs; - -pub mod descriptors; - fn main() { - //Init all descriptors: - let desc = Median; - - //Initialize hashmap, empty or from cached file: - let count = fs::read_dir("img").unwrap().count(); - let map_file = fs::read("map.messagepack"); - let mut map: HashMap = match map_file { - Ok(f) => rmp_serde::from_slice(&f).unwrap(), - Err(_e) => HashMap::with_capacity(count), - }; - - //Calculate phashes for all images - for node in fs::read_dir("img").unwrap() { - let file = node.expect("Error walking directory"); - let name = file.file_name().into_string().expect("Issue with filename"); - if !map.contains_key(&name) { - let img = image::open(file.path()).expect("Unable to open file"); - let phash = desc.describe(img); - map.insert(name, phash); - } - } - - for (key, val) in map.iter() { - println!("{key: >15}: {val}\t {val:b}"); - } - - //Serialize hashmap with MessagePack: - let serialized: Vec = rmp_serde::to_vec(&map).unwrap(); - //fs::write("map.messagepack",&serialized).unwrap(); + print!("Hello world!") } From a9ee164b4f9c2cb1c3848113e26d224c7dae5322 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sat, 4 May 2024 10:57:42 +0200 Subject: [PATCH 05/32] resize function refactor --- src/descriptors.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/descriptors.rs b/src/descriptors.rs index 39815c1..5c187d3 100644 --- a/src/descriptors.rs +++ b/src/descriptors.rs @@ -5,6 +5,9 @@ pub trait Descriptor { fn info(&self) -> String { "Descriptor".to_string() } + fn resize(&self, img: image::DynamicImage) -> image::DynamicImage { + img.grayscale().thumbnail_exact(8, 8) + } fn describe(&self, img: image::DynamicImage) -> u64; fn distance(&self, a: u64, b: u64) -> u64 { (a^b).count_ones().into() @@ -22,7 +25,7 @@ pub struct Median; impl Descriptor for Median { fn describe(&self, img: image::DynamicImage) -> u64 { - let img = img.grayscale().thumbnail_exact(8, 8); + let img = self.resize(img); let mut values: [u8; 64] = [0; 64]; let mut i: usize = 0; for (_, _, pix) in img.pixels() { @@ -47,7 +50,7 @@ pub struct DCT; impl Descriptor for DCT { fn describe(&self, img: image::DynamicImage) -> u64 { let mut dct_values: [f64; 64] = [0.0; 64]; - let img = img.grayscale().thumbnail_exact(8, 8); + let img = self.resize(img); let mut total_sum: f64 = 0.0; for j in 0..7 { From f25a03660d51a94d4c449b0024b429b1ded1dce5 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sat, 4 May 2024 15:51:48 +0200 Subject: [PATCH 06/32] unused prefix --- src/descriptors.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/descriptors.rs b/src/descriptors.rs index 5c187d3..58a70ba 100644 --- a/src/descriptors.rs +++ b/src/descriptors.rs @@ -51,7 +51,7 @@ impl Descriptor for DCT { fn describe(&self, img: image::DynamicImage) -> u64 { let mut dct_values: [f64; 64] = [0.0; 64]; let img = self.resize(img); - let mut total_sum: f64 = 0.0; + let mut _total_sum: f64 = 0.0; for j in 0..7 { for i in 0..7 { @@ -63,11 +63,11 @@ impl Descriptor for DCT { sum += pixel as f64 * (PI/64.0 * (n + 0.5) * k as f64).cos(); } dct_values[k] = sum; - total_sum += sum; + _total_sum += sum; //println!("{sum}"); } } - total_sum /= 64.0; + _total_sum /= 64.0; let mut mask: u64 = 0; for dct in dct_values { From 3e03c947275b8ab3f96ddfbb7f57359aa44acc3e Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sat, 4 May 2024 15:52:14 +0200 Subject: [PATCH 07/32] remove unused --- src/bin/experiment.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/experiment.rs b/src/bin/experiment.rs index a1ffec5..76d2ad2 100644 --- a/src/bin/experiment.rs +++ b/src/bin/experiment.rs @@ -1,4 +1,4 @@ -use image_similarity::descriptors::{Descriptor, Median, DCT}; +use image_similarity::descriptors::{Descriptor, Median}; use std::collections::HashMap; use std::fs; @@ -30,6 +30,6 @@ fn main() { } //Serialize hashmap with MessagePack: - let serialized: Vec = rmp_serde::to_vec(&map).unwrap(); + let _serialized: Vec = rmp_serde::to_vec(&map).unwrap(); //fs::write("map.messagepack",&serialized).unwrap(); } From 401d6122595a7c74eb6b629f65dcc69cbd52cf4a Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sat, 4 May 2024 15:52:36 +0200 Subject: [PATCH 08/32] dctfilename binary in base64 --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + src/bin/dctfilename.rs | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 src/bin/dctfilename.rs diff --git a/Cargo.lock b/Cargo.lock index ec10b68..b24af28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,6 +14,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bit_field" version = "0.10.1" @@ -234,6 +240,7 @@ dependencies = [ name = "image_similarity" version = "0.1.0" dependencies = [ + "base64", "image", "rmp-serde", "serde", diff --git a/Cargo.toml b/Cargo.toml index 60ffb63..f3a3851 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,3 +9,4 @@ edition = "2021" image = "0.24.4" serde = "1.0.147" rmp-serde = "1.1.1" +base64 = "0.22.1" diff --git a/src/bin/dctfilename.rs b/src/bin/dctfilename.rs new file mode 100644 index 0000000..43392fe --- /dev/null +++ b/src/bin/dctfilename.rs @@ -0,0 +1,36 @@ +use image_similarity::descriptors::{Descriptor, DCT}; +use std::env; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; + +struct Cfg { + path: String +} + +impl Cfg{ + fn load(args: &[String]) -> Result { + if args.len() < 2 { + return Err("Filename argument required"); + } + let path = args[1].clone(); + Ok(Cfg { path }) + } +} + +fn main() { + //Init all descriptors: + let desc = DCT; + + let args: Vec = env::args().collect(); + //dbg!(args); + + let cfg = Cfg::load(&args) + .expect("Error loading config"); + + let img = image::open(cfg.path) + .expect("Unable to open file"); + let phash: u64 = desc.describe(img); + println!("{phash}\n{phash:b}"); + let output = URL_SAFE_NO_PAD.encode(phash.to_be_bytes()); + println!("{output}") + +} From c50fb4d7856a30ef2a3502f5a2f8ea244999f367 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sat, 4 May 2024 23:26:13 +0200 Subject: [PATCH 09/32] better dct maybe --- src/descriptors.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/descriptors.rs b/src/descriptors.rs index 58a70ba..7141319 100644 --- a/src/descriptors.rs +++ b/src/descriptors.rs @@ -51,24 +51,24 @@ impl Descriptor for DCT { fn describe(&self, img: image::DynamicImage) -> u64 { let mut dct_values: [f64; 64] = [0.0; 64]; let img = self.resize(img); - let mut _total_sum: f64 = 0.0; - - for j in 0..7 { - for i in 0..7 { + for v in 0..7 { + for u in 0..7 { + let k = (u*8)+v; + let v: f64 = v as f64; + let u: f64 = u as f64; let mut sum: f64 = 0.0; - let k = (i*8)+j; for (x, y, pix) in img.pixels() { - let n: f64 = ((y*8)+x).into(); - let pixel: i16 = pix[0] as i16 - 127; - sum += pixel as f64 * (PI/64.0 * (n + 0.5) * k as f64).cos(); + let x: f64 = 1.0 + 2.0 * x as f64; + let y: f64 = 1.0 + 2.0 * y as f64; + let pixel = (pix[0] as i16 - 127) as f64; + sum += + pixel * + (x*u*PI/16.0).cos() * + (y*v*PI/16.0).cos() } dct_values[k] = sum; - _total_sum += sum; - //println!("{sum}"); } } - _total_sum /= 64.0; - let mut mask: u64 = 0; for dct in dct_values { if dct > 0.0 { From c61f87bdc3506be4cf2fbe062c681103635d2d5a Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 5 May 2024 14:36:49 +0200 Subject: [PATCH 10/32] inverse dct --- src/descriptors.rs | 57 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/src/descriptors.rs b/src/descriptors.rs index 7141319..344b287 100644 --- a/src/descriptors.rs +++ b/src/descriptors.rs @@ -1,5 +1,5 @@ -use image::GenericImageView; -use std::f64::consts::PI; +use image::{save_buffer, GenericImageView}; +use std::f64::consts::{PI, SQRT_2}; pub trait Descriptor { fn info(&self) -> String { @@ -50,10 +50,19 @@ pub struct DCT; impl Descriptor for DCT { fn describe(&self, img: image::DynamicImage) -> u64 { let mut dct_values: [f64; 64] = [0.0; 64]; + let mut reconstructed: [u8; 64] = [0; 64]; let img = self.resize(img); - for v in 0..7 { - for u in 0..7 { - let k = (u*8)+v; + let _ = img.save("resize.png").expect("Error saving file"); + for u in 0..8 { + for v in 0..8 { + let k = (v*8)+u; + let mut alpha = 0.25; + if u == 0 { + alpha = alpha / SQRT_2 + } + if v == 0 { + alpha = alpha / SQRT_2 + } let v: f64 = v as f64; let u: f64 = u as f64; let mut sum: f64 = 0.0; @@ -66,9 +75,45 @@ impl Descriptor for DCT { (x*u*PI/16.0).cos() * (y*v*PI/16.0).cos() } - dct_values[k] = sum; + dct_values[k] = alpha * sum; + //println!("For {u},{v} the output is {}", dct_values[k]); + //println!{"{}", dct_values[k]} } } + for k in 0..64 { + let x = (k%8) as f64; + let y = (k/8) as f64; + let mut sum = 0.0; + for u in 0..8 { + for v in 0..8 { + let mut alpha = 1.0; + if u == 0 { + alpha = alpha / SQRT_2 + } + if v == 0 { + alpha = alpha / SQRT_2 + } + let uv = (v*8)+u; + let v = v as f64; + let u = u as f64; + sum += + alpha * + dct_values[uv] * + ((2.0 * x + 1.0) * u * PI / 16.0).cos() * + ((2.0 * y + 1.0) * v * PI / 16.0).cos() + } + } + println!("Reconstructed pixel: {}", 127 + (0.25 * sum).round() as u16); + reconstructed[k] = std::cmp::min(255_u16, 127 + (0.25 * sum).round() as u16) as u8; + //println!("{x},{y}: {}", dct_values[k]) + } + save_buffer( + "reconstructed.png", + &reconstructed, + 8, + 8, + image::ColorType::L8 + ).expect("Error saving buffer"); let mut mask: u64 = 0; for dct in dct_values { if dct > 0.0 { From 75dff82f0f9dc9c02952b72a9175a0bb595972f4 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 5 May 2024 22:45:31 +0200 Subject: [PATCH 11/32] dct with working inverse --- src/descriptors.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/descriptors.rs b/src/descriptors.rs index 344b287..1ecb029 100644 --- a/src/descriptors.rs +++ b/src/descriptors.rs @@ -76,7 +76,6 @@ impl Descriptor for DCT { (y*v*PI/16.0).cos() } dct_values[k] = alpha * sum; - //println!("For {u},{v} the output is {}", dct_values[k]); //println!{"{}", dct_values[k]} } } @@ -103,9 +102,9 @@ impl Descriptor for DCT { ((2.0 * y + 1.0) * v * PI / 16.0).cos() } } - println!("Reconstructed pixel: {}", 127 + (0.25 * sum).round() as u16); - reconstructed[k] = std::cmp::min(255_u16, 127 + (0.25 * sum).round() as u16) as u8; - //println!("{x},{y}: {}", dct_values[k]) + sum = 127.0 + (0.25 * sum).round(); + //println!("Reconstructed pixel: {}", sum); + reconstructed[k] = std::cmp::min(255_u8, sum as u8); } save_buffer( "reconstructed.png", From dcf4be921ce35bc09393b6a7506da43b8fbd05cd Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Mon, 6 May 2024 01:14:53 +0200 Subject: [PATCH 12/32] quantization matrix wip --- src/bin/dctfilename.rs | 2 +- src/descriptors.rs | 78 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/bin/dctfilename.rs b/src/bin/dctfilename.rs index 43392fe..2c59bb7 100644 --- a/src/bin/dctfilename.rs +++ b/src/bin/dctfilename.rs @@ -18,7 +18,7 @@ impl Cfg{ fn main() { //Init all descriptors: - let desc = DCT; + let desc = DCT::new(); let args: Vec = env::args().collect(); //dbg!(args); diff --git a/src/descriptors.rs b/src/descriptors.rs index 1ecb029..b7f419f 100644 --- a/src/descriptors.rs +++ b/src/descriptors.rs @@ -45,14 +45,76 @@ impl Descriptor for Median { } } -pub struct DCT; +/// A struct representing a Discrete Cosine Transform descriptor +/// Transforms an image into an 8x8 grayscale version, and transforms it +/// into the frequency domain +pub struct DCT { + quantization_matrix: [u8; 64], +} + + +/// Interprets a 64-element array as an 8x8 matrix +/// returns a nicely printable string +fn print_matrix(array: [T; 64]) -> String { + let mut output = String::new(); + for x in 0..8 { + for y in 0..8 { + output.push_str(&array[x*8+y].to_string()); + output.push_str(",\t"); + } + output.push('\n'); + } + output +} + + +// Define S such that if (Q < 50), then S = 5000/Q, else S = 200 – 2*Q. +// The output quantization matrix Ts[i,j] at each location of row i and column j is such that +// Ts[i,j] = floor((S * Tb[i,j] + 50) / 100) + + +impl DCT { + pub fn new() -> DCT { + let base_quantization_matrix: [u8; 64] = [ + 16, 11, 10, 16, 24, 40, 51, 61, + 12, 12, 14, 19, 26, 58, 60, 55, + 14, 13, 16, 24, 40, 57, 69, 56, + 14, 17, 22, 29, 51, 87, 80, 62, + 18, 22, 37, 56, 6, 10, 103, 77, + 24, 35, 55, 64, 8, 10, 113, 92, + 49, 64, 78, 8, 10, 12, 12, 101, + 72, 92, 95, 9, 11, 10, 103, 99, + ]; + DCT { quantization_matrix: base_quantization_matrix } + } + pub fn quantization_matrix(&self, quality: u8) -> [u8; 64] { + let mut quantization_matrix = self.quantization_matrix.clone(); + let scalar: f32 = match quality { + 1..=49 => 5000.0/quality as f32, + 50..=100 => 200.0 - 2.0*quality as f32, + _ => 1.0 //TODO: error + }; + for i in 0..64 { + quantization_matrix[i] = ((scalar * self.quantization_matrix[i] as f32 + 50.0) / 100.0).floor() as u8; + if quantization_matrix[i] == 0 { + quantization_matrix[i] = 1; + } + } + quantization_matrix + } +} impl Descriptor for DCT { fn describe(&self, img: image::DynamicImage) -> u64 { + let quality: u8 = 15; + let qmatrix = self.quantization_matrix(quality); + + println!("Base quantization matrix:\n{}", print_matrix(self.quantization_matrix)); + println!("Q-{} quantization matrix:\n{}", quality, print_matrix(qmatrix)); let mut dct_values: [f64; 64] = [0.0; 64]; let mut reconstructed: [u8; 64] = [0; 64]; let img = self.resize(img); - let _ = img.save("resize.png").expect("Error saving file"); + img.save("resize.png").expect("Error saving file"); for u in 0..8 { for v in 0..8 { let k = (v*8)+u; @@ -79,6 +141,17 @@ impl Descriptor for DCT { //println!{"{}", dct_values[k]} } } + println!("DCT-coefficients:\n {}", print_matrix(dct_values)); + // Quantization: + for i in 0..64 { + dct_values[i] = (dct_values[i] / qmatrix[i] as f64).round(); + } + println!("DCT-coefficients, quantized:\n {}", print_matrix(dct_values)); + // De-quantization: + for i in 0..64 { + dct_values[i] = dct_values[i] * qmatrix[i] as f64; + } + println!("DCT-coefficients, de-quantized:\n {}", print_matrix(dct_values)); for k in 0..64 { let x = (k%8) as f64; let y = (k/8) as f64; @@ -106,6 +179,7 @@ impl Descriptor for DCT { //println!("Reconstructed pixel: {}", sum); reconstructed[k] = std::cmp::min(255_u8, sum as u8); } + //println!("{}", print_matrix(reconstructed)); save_buffer( "reconstructed.png", &reconstructed, From 4c55afa1f494cf7cef06f4fe9cdb52677ac73bdd Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Mon, 6 May 2024 23:12:14 +0200 Subject: [PATCH 13/32] refactor into modules --- src/bin/dctfilename.rs | 3 +- src/bin/experiment.rs | 5 +- src/{descriptors.rs => descriptors/dct.rs} | 74 ++-------------------- src/descriptors/mod.rs | 64 +++++++++++++++++++ 4 files changed, 74 insertions(+), 72 deletions(-) rename src/{descriptors.rs => descriptors/dct.rs} (72%) create mode 100644 src/descriptors/mod.rs diff --git a/src/bin/dctfilename.rs b/src/bin/dctfilename.rs index 2c59bb7..21a4ce4 100644 --- a/src/bin/dctfilename.rs +++ b/src/bin/dctfilename.rs @@ -1,4 +1,5 @@ -use image_similarity::descriptors::{Descriptor, DCT}; +//use image_similarity::descriptors::dct::DCT; +use image_similarity::descriptors::{DCT, Descriptor}; use std::env; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; diff --git a/src/bin/experiment.rs b/src/bin/experiment.rs index 76d2ad2..a17d45e 100644 --- a/src/bin/experiment.rs +++ b/src/bin/experiment.rs @@ -1,10 +1,11 @@ -use image_similarity::descriptors::{Descriptor, Median}; +use image_similarity::descriptors; +use image_similarity::descriptors::Descriptor; use std::collections::HashMap; use std::fs; fn main() { //Init all descriptors: - let desc = Median; + let desc = descriptors::dct::DCT::new(); //Initialize hashmap, empty or from cached file: let count = fs::read_dir("img").unwrap().count(); diff --git a/src/descriptors.rs b/src/descriptors/dct.rs similarity index 72% rename from src/descriptors.rs rename to src/descriptors/dct.rs index b7f419f..39b33e7 100644 --- a/src/descriptors.rs +++ b/src/descriptors/dct.rs @@ -1,77 +1,13 @@ use image::{save_buffer, GenericImageView}; use std::f64::consts::{PI, SQRT_2}; - -pub trait Descriptor { - fn info(&self) -> String { - "Descriptor".to_string() - } - fn resize(&self, img: image::DynamicImage) -> image::DynamicImage { - img.grayscale().thumbnail_exact(8, 8) - } - fn describe(&self, img: image::DynamicImage) -> u64; - fn distance(&self, a: u64, b: u64) -> u64 { - (a^b).count_ones().into() - } -} - -fn median64(values: &[T]) -> T { - let mut sorted_values = values.to_vec(); - sorted_values.sort(); - let len = sorted_values.len(); - sorted_values[len/2] -} - -pub struct Median; - -impl Descriptor for Median { - fn describe(&self, img: image::DynamicImage) -> u64 { - let img = self.resize(img); - let mut values: [u8; 64] = [0; 64]; - let mut i: usize = 0; - for (_, _, pix) in img.pixels() { - values[i] = pix[0]; - i = i+1; - } - let median = median64(&values); - let mut mask: u64 = 0; - img.save("debug.png").unwrap(); - for (_, _, pix) in img.pixels() { - if pix[0] > median { - mask += 1; - } - mask = mask << 1; - } - mask - } -} +use crate::descriptors::{Descriptor, print_matrix, DCT}; /// A struct representing a Discrete Cosine Transform descriptor /// Transforms an image into an 8x8 grayscale version, and transforms it /// into the frequency domain -pub struct DCT { - quantization_matrix: [u8; 64], -} - - -/// Interprets a 64-element array as an 8x8 matrix -/// returns a nicely printable string -fn print_matrix(array: [T; 64]) -> String { - let mut output = String::new(); - for x in 0..8 { - for y in 0..8 { - output.push_str(&array[x*8+y].to_string()); - output.push_str(",\t"); - } - output.push('\n'); - } - output -} - - -// Define S such that if (Q < 50), then S = 5000/Q, else S = 200 – 2*Q. -// The output quantization matrix Ts[i,j] at each location of row i and column j is such that -// Ts[i,j] = floor((S * Tb[i,j] + 50) / 100) - +// pub struct DCT { +// quantization_matrix: [u8; 64], +// } impl DCT { pub fn new() -> DCT { @@ -196,4 +132,4 @@ impl Descriptor for DCT { } mask } -} +} \ No newline at end of file diff --git a/src/descriptors/mod.rs b/src/descriptors/mod.rs new file mode 100644 index 0000000..16e70f8 --- /dev/null +++ b/src/descriptors/mod.rs @@ -0,0 +1,64 @@ +pub trait Descriptor { + fn info(&self) -> String { + "Descriptor".to_string() + } + fn resize(&self, img: image::DynamicImage) -> image::DynamicImage { + img.grayscale().thumbnail_exact(8, 8) + } + fn describe(&self, img: image::DynamicImage) -> u64; + fn distance(&self, a: u64, b: u64) -> u64 { + (a^b).count_ones().into() + } +} + +/// Interprets a 64-element array as an 8x8 matrix +/// returns a nicely printable string +fn print_matrix(array: [T; 64]) -> String { + let mut output = String::new(); + for x in 0..8 { + for y in 0..8 { + output.push_str(&array[x*8+y].to_string()); + output.push_str(",\t"); + } + output.push('\n'); + } + output +} + +pub struct DCT { + quantization_matrix: [u8; 64], +} + +pub mod dct; + +// fn median64(values: &[T]) -> T { +// let mut sorted_values = values.to_vec(); +// sorted_values.sort(); +// let len = sorted_values.len(); +// sorted_values[len/2] +// } + + +// pub struct Median; + +// impl Descriptor for Median { +// fn describe(&self, img: image::DynamicImage) -> u64 { +// let img = self.resize(img); +// let mut values: [u8; 64] = [0; 64]; +// let mut i: usize = 0; +// for (_, _, pix) in img.pixels() { +// values[i] = pix[0]; +// i = i+1; +// } +// let median = median64(&values); +// let mut mask: u64 = 0; +// img.save("debug.png").unwrap(); +// for (_, _, pix) in img.pixels() { +// if pix[0] > median { +// mask += 1; +// } +// mask = mask << 1; +// } +// mask +// } +// } From 31351ea91ddeab7df2cddfd4545f4c1c6e1ad3ba Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Tue, 7 May 2024 01:31:26 +0200 Subject: [PATCH 14/32] refactor idct --- src/descriptors/dct.rs | 66 +++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index 39b33e7..de0d91f 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -38,6 +38,41 @@ impl DCT { } quantization_matrix } + + /// TODO + // pub fn dct(&self, img: image::DynamicImage) -> [f64; 64] { + // } + + pub fn idct(&self, dct_values: [f64; 64]) -> [u8; 64] { + let mut reconstructed: [u8; 64] = [0; 64]; + for k in 0..64 { + let x = (k%8) as f64; + let y = (k/8) as f64; + let mut sum = 0.0; + for u in 0..8 { + for v in 0..8 { + let mut alpha = 1.0; + if u == 0 { + alpha = alpha / SQRT_2 + } + if v == 0 { + alpha = alpha / SQRT_2 + } + let uv = (v*8)+u; + let v = v as f64; + let u = u as f64; + sum += + alpha * + dct_values[uv] * + ((2.0 * x + 1.0) * u * PI / 16.0).cos() * + ((2.0 * y + 1.0) * v * PI / 16.0).cos() + } + } + sum = 127.0 + (0.25 * sum).round(); + reconstructed[k] = std::cmp::min(255_u8, sum as u8); + } + reconstructed + } } impl Descriptor for DCT { @@ -48,7 +83,6 @@ impl Descriptor for DCT { println!("Base quantization matrix:\n{}", print_matrix(self.quantization_matrix)); println!("Q-{} quantization matrix:\n{}", quality, print_matrix(qmatrix)); let mut dct_values: [f64; 64] = [0.0; 64]; - let mut reconstructed: [u8; 64] = [0; 64]; let img = self.resize(img); img.save("resize.png").expect("Error saving file"); for u in 0..8 { @@ -88,34 +122,8 @@ impl Descriptor for DCT { dct_values[i] = dct_values[i] * qmatrix[i] as f64; } println!("DCT-coefficients, de-quantized:\n {}", print_matrix(dct_values)); - for k in 0..64 { - let x = (k%8) as f64; - let y = (k/8) as f64; - let mut sum = 0.0; - for u in 0..8 { - for v in 0..8 { - let mut alpha = 1.0; - if u == 0 { - alpha = alpha / SQRT_2 - } - if v == 0 { - alpha = alpha / SQRT_2 - } - let uv = (v*8)+u; - let v = v as f64; - let u = u as f64; - sum += - alpha * - dct_values[uv] * - ((2.0 * x + 1.0) * u * PI / 16.0).cos() * - ((2.0 * y + 1.0) * v * PI / 16.0).cos() - } - } - sum = 127.0 + (0.25 * sum).round(); - //println!("Reconstructed pixel: {}", sum); - reconstructed[k] = std::cmp::min(255_u8, sum as u8); - } - //println!("{}", print_matrix(reconstructed)); + let reconstructed = self.idct(dct_values); + println!("Reconstructed pixel values:\n{}", print_matrix(reconstructed)); save_buffer( "reconstructed.png", &reconstructed, From 1aa32a3788dd525164ab0617c35795f6c9ef23bf Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Wed, 8 May 2024 10:12:53 +0200 Subject: [PATCH 15/32] refactor dct and quality --- src/bin/dctfilename.rs | 3 +- src/descriptors/dct.rs | 114 +++++++++++++++++++++-------------------- src/descriptors/mod.rs | 13 +++-- 3 files changed, 68 insertions(+), 62 deletions(-) diff --git a/src/bin/dctfilename.rs b/src/bin/dctfilename.rs index 21a4ce4..ef9f440 100644 --- a/src/bin/dctfilename.rs +++ b/src/bin/dctfilename.rs @@ -19,10 +19,9 @@ impl Cfg{ fn main() { //Init all descriptors: - let desc = DCT::new(); + let desc = DCT::new().with_quality(50); let args: Vec = env::args().collect(); - //dbg!(args); let cfg = Cfg::load(&args) .expect("Error loading config"); diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index de0d91f..a285a78 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -2,14 +2,9 @@ use image::{save_buffer, GenericImageView}; use std::f64::consts::{PI, SQRT_2}; use crate::descriptors::{Descriptor, print_matrix, DCT}; -/// A struct representing a Discrete Cosine Transform descriptor -/// Transforms an image into an 8x8 grayscale version, and transforms it -/// into the frequency domain -// pub struct DCT { -// quantization_matrix: [u8; 64], -// } impl DCT { + /// Returns a new DCT instance with a calculated DCT matrix. pub fn new() -> DCT { let base_quantization_matrix: [u8; 64] = [ 16, 11, 10, 16, 24, 40, 51, 61, @@ -23,8 +18,10 @@ impl DCT { ]; DCT { quantization_matrix: base_quantization_matrix } } - pub fn quantization_matrix(&self, quality: u8) -> [u8; 64] { - let mut quantization_matrix = self.quantization_matrix.clone(); + + // Builds DCT with given quality value + pub fn with_quality(mut self, quality: u8) -> Self { + let mut quantization_matrix: [u8; 64] = [0; 64]; let scalar: f32 = match quality { 1..=49 => 5000.0/quality as f32, 50..=100 => 200.0 - 2.0*quality as f32, @@ -36,54 +33,14 @@ impl DCT { quantization_matrix[i] = 1; } } - quantization_matrix + self.quantization_matrix = quantization_matrix; + self } - /// TODO - // pub fn dct(&self, img: image::DynamicImage) -> [f64; 64] { - // } - - pub fn idct(&self, dct_values: [f64; 64]) -> [u8; 64] { - let mut reconstructed: [u8; 64] = [0; 64]; - for k in 0..64 { - let x = (k%8) as f64; - let y = (k/8) as f64; - let mut sum = 0.0; - for u in 0..8 { - for v in 0..8 { - let mut alpha = 1.0; - if u == 0 { - alpha = alpha / SQRT_2 - } - if v == 0 { - alpha = alpha / SQRT_2 - } - let uv = (v*8)+u; - let v = v as f64; - let u = u as f64; - sum += - alpha * - dct_values[uv] * - ((2.0 * x + 1.0) * u * PI / 16.0).cos() * - ((2.0 * y + 1.0) * v * PI / 16.0).cos() - } - } - sum = 127.0 + (0.25 * sum).round(); - reconstructed[k] = std::cmp::min(255_u8, sum as u8); - } - reconstructed - } -} - -impl Descriptor for DCT { - fn describe(&self, img: image::DynamicImage) -> u64 { - let quality: u8 = 15; - let qmatrix = self.quantization_matrix(quality); - - println!("Base quantization matrix:\n{}", print_matrix(self.quantization_matrix)); - println!("Q-{} quantization matrix:\n{}", quality, print_matrix(qmatrix)); + pub fn dct(&self, img: image::DynamicImage) -> [f64; 64] { + //let qmatrix = self.quantization_matrix(self.quality); + //println!("Q-{} quantization matrix:\n{}", self.quality, print_matrix(qmatrix)); let mut dct_values: [f64; 64] = [0.0; 64]; - let img = self.resize(img); img.save("resize.png").expect("Error saving file"); for u in 0..8 { for v in 0..8 { @@ -111,19 +68,64 @@ impl Descriptor for DCT { //println!{"{}", dct_values[k]} } } + dct_values + } + + pub fn idct(&self, dct_values: [f64; 64]) -> [u8; 64] { + let mut reconstructed: [u8; 64] = [0; 64]; + for k in 0..64 { + let x = (k%8) as f64; + let y = (k/8) as f64; + let mut sum = 0.0; + for u in 0..8 { + for v in 0..8 { + let mut alpha = 1.0; + if u == 0 { + alpha = alpha / SQRT_2 + } + if v == 0 { + alpha = alpha / SQRT_2 + } + let uv = (v*8)+u; + let v = v as f64; + let u = u as f64; + sum += + alpha * + dct_values[uv] * + ((2.0 * x + 1.0) * u * PI / 16.0).cos() * + ((2.0 * y + 1.0) * v * PI / 16.0).cos(); + } + } + sum = 127.0 + (0.25 * sum).round(); + reconstructed[k] = std::cmp::min(255_u8, sum as u8); + } + reconstructed + } +} + +impl Descriptor for DCT { + fn describe(&self, img: image::DynamicImage) -> u64 { + let img = self.resize(img); + let mut dct_values = self.dct(img); println!("DCT-coefficients:\n {}", print_matrix(dct_values)); + // Quantization: + println!("Using quantization matrix:\n{}", print_matrix(self.quantization_matrix)); for i in 0..64 { - dct_values[i] = (dct_values[i] / qmatrix[i] as f64).round(); + dct_values[i] = (dct_values[i] / self.quantization_matrix[i] as f64).round(); } println!("DCT-coefficients, quantized:\n {}", print_matrix(dct_values)); + // De-quantization: for i in 0..64 { - dct_values[i] = dct_values[i] * qmatrix[i] as f64; + dct_values[i] = dct_values[i] * self.quantization_matrix[i] as f64; } println!("DCT-coefficients, de-quantized:\n {}", print_matrix(dct_values)); + + // Reconstruction original pixel values: let reconstructed = self.idct(dct_values); println!("Reconstructed pixel values:\n{}", print_matrix(reconstructed)); + save_buffer( "reconstructed.png", &reconstructed, @@ -131,6 +133,8 @@ impl Descriptor for DCT { 8, image::ColorType::L8 ).expect("Error saving buffer"); + + // Calculating descriptor from dct values: let mut mask: u64 = 0; for dct in dct_values { if dct > 0.0 { diff --git a/src/descriptors/mod.rs b/src/descriptors/mod.rs index 16e70f8..0c54c9d 100644 --- a/src/descriptors/mod.rs +++ b/src/descriptors/mod.rs @@ -13,22 +13,25 @@ pub trait Descriptor { /// Interprets a 64-element array as an 8x8 matrix /// returns a nicely printable string -fn print_matrix(array: [T; 64]) -> String { +fn print_matrix(array: [T; 64]) -> String { let mut output = String::new(); for x in 0..8 { for y in 0..8 { - output.push_str(&array[x*8+y].to_string()); - output.push_str(",\t"); + let element = format!("{:.2}", &array[x*8+y]); + let element = format!("{:8}\t", element); + output.push_str(&element); } output.push('\n'); } output } +/// A struct representing a Discrete Cosine Transform descriptor +/// Transforms an image into an 8x8 grayscale version, and transforms it +/// into the frequency domain pub struct DCT { - quantization_matrix: [u8; 64], + quantization_matrix: [u8; 64] } - pub mod dct; // fn median64(values: &[T]) -> T { From 499f7f7bfd0c1663dc17d064ccbae73aa635f1d0 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Wed, 8 May 2024 12:01:33 +0200 Subject: [PATCH 16/32] fixes experiment binary --- src/bin/experiment.rs | 5 ++--- src/descriptors/dct.rs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/bin/experiment.rs b/src/bin/experiment.rs index a17d45e..e51f0e5 100644 --- a/src/bin/experiment.rs +++ b/src/bin/experiment.rs @@ -1,11 +1,10 @@ -use image_similarity::descriptors; -use image_similarity::descriptors::Descriptor; +use image_similarity::descriptors::{Descriptor, DCT}; use std::collections::HashMap; use std::fs; fn main() { //Init all descriptors: - let desc = descriptors::dct::DCT::new(); + let desc = DCT::new(); //Initialize hashmap, empty or from cached file: let count = fs::read_dir("img").unwrap().count(); diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index a285a78..597d003 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -4,7 +4,7 @@ use crate::descriptors::{Descriptor, print_matrix, DCT}; impl DCT { - /// Returns a new DCT instance with a calculated DCT matrix. + /// Returns a new DCT instance with a base quality DCT matrix. pub fn new() -> DCT { let base_quantization_matrix: [u8; 64] = [ 16, 11, 10, 16, 24, 40, 51, 61, From 89e7afca4ef8d9a6c684d94adc8495847f8552f0 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Fri, 10 May 2024 11:38:11 +0200 Subject: [PATCH 17/32] first version descriptorstore --- src/bin/dctfilename.rs | 2 +- src/lib.rs | 3 ++- src/main.rs | 8 +++++- src/store.rs | 59 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 src/store.rs diff --git a/src/bin/dctfilename.rs b/src/bin/dctfilename.rs index ef9f440..593f043 100644 --- a/src/bin/dctfilename.rs +++ b/src/bin/dctfilename.rs @@ -18,6 +18,7 @@ impl Cfg{ } fn main() { + //Init all descriptors: let desc = DCT::new().with_quality(50); @@ -32,5 +33,4 @@ fn main() { println!("{phash}\n{phash:b}"); let output = URL_SAFE_NO_PAD.encode(phash.to_be_bytes()); println!("{output}") - } diff --git a/src/lib.rs b/src/lib.rs index e27c649..c807575 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1 +1,2 @@ -pub mod descriptors; \ No newline at end of file +pub mod descriptors; +pub mod store; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index f8ed123..ccfd01b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,9 @@ +use image_similarity::store::DescriptorStore; +use image_similarity::descriptors::{DCT, Descriptor}; + fn main() { - print!("Hello world!") + let mut store = DescriptorStore::new(); + let desc = DCT::new().with_quality(30); + store.insert_directory("img", desc); + print!("{}", store); } diff --git a/src/store.rs b/src/store.rs new file mode 100644 index 0000000..a793382 --- /dev/null +++ b/src/store.rs @@ -0,0 +1,59 @@ +use std::collections::HashMap; +use crate::descriptors::Descriptor; +use std::fs; +use std::path::Path; +use std::fmt; + +pub struct DescriptorStore { + map: HashMap +} + +impl DescriptorStore { + /// Makes a new DescriptorStore. + /// Reads from file if possible, else empty + pub fn new() -> DescriptorStore { + //TODO: Set directory and file location from global config + let count = fs::read_dir("img").unwrap().count(); + let map_file = fs::read("map.messagepack"); + let map: HashMap = match map_file { + Ok(f) => rmp_serde::from_slice(&f).unwrap(), + Err(_e) => HashMap::with_capacity(count), + }; + DescriptorStore { map: map } + } + + /// Returns true iff the store already contains the key + pub fn contains(&self, key: String) -> bool { + self.map.contains_key(&key) + } + + /// Inserts a single value into the store + pub fn insert(mut self, key: String, value: u64) { + self.map.insert(key, value); + } + + + /// Calculates all descriptions with a given descriptor for a folder + pub fn insert_directory>(&mut self, dir: U, desc: T) { + for node in fs::read_dir(dir).unwrap() { + let file = node.expect("Error walking directory"); + let name = file.file_name().into_string().expect("Issue with filename"); + if !self.contains(name.to_string()) { + let img = image::open(file.path()).expect("Unable to open file"); + let phash = desc.describe(img); + self.map.insert(name, phash); + } + } + } +} + +impl fmt::Display for DescriptorStore { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut output = String::new(); + for (key, val) in self.map.iter() { + let entry = format!("{key: >15}: {val}\t {val:b}\n"); + output.push_str(&entry); + } + write!(f, "{}", output) + } +} \ No newline at end of file From dc57133825d400039d2d17fc85c2497c40192cbb Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Fri, 10 May 2024 17:52:02 +0200 Subject: [PATCH 18/32] updated logging --- Cargo.lock | 222 ++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 + src/bin/dctfilename.rs | 1 + src/main.rs | 6 +- src/store.rs | 6 +- 5 files changed, 228 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b24af28..54af603 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,64 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" + +[[package]] +name = "anstyle-parse" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +dependencies = [ + "anstyle", + "windows-sys", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -62,6 +120,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" + [[package]] name = "crc32fast" version = "1.3.2" @@ -126,6 +190,29 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "exr" version = "1.5.2" @@ -217,6 +304,12 @@ dependencies = [ "libc", ] +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "image" version = "0.24.4" @@ -241,11 +334,19 @@ name = "image_similarity" version = "0.1.0" dependencies = [ "base64", + "env_logger", "image", + "log", "rmp-serde", "serde", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + [[package]] name = "jpeg-decoder" version = "0.2.6" @@ -288,12 +389,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "memchr" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memoffset" @@ -457,6 +561,35 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "regex" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" + [[package]] name = "rmp" version = "0.8.11" @@ -549,6 +682,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -614,3 +753,76 @@ name = "weezl" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" diff --git a/Cargo.toml b/Cargo.toml index f3a3851..ca7e200 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,3 +10,5 @@ image = "0.24.4" serde = "1.0.147" rmp-serde = "1.1.1" base64 = "0.22.1" +log = "0.4.21" +env_logger = "0.11.3" diff --git a/src/bin/dctfilename.rs b/src/bin/dctfilename.rs index 593f043..72eedf3 100644 --- a/src/bin/dctfilename.rs +++ b/src/bin/dctfilename.rs @@ -18,6 +18,7 @@ impl Cfg{ } fn main() { + env_logger::init(); //Init all descriptors: let desc = DCT::new().with_quality(50); diff --git a/src/main.rs b/src/main.rs index ccfd01b..3da260d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,11 @@ use image_similarity::store::DescriptorStore; -use image_similarity::descriptors::{DCT, Descriptor}; +use image_similarity::descriptors::DCT; +use log::info; fn main() { + env_logger::init(); let mut store = DescriptorStore::new(); let desc = DCT::new().with_quality(30); store.insert_directory("img", desc); - print!("{}", store); + info!("{}", store); } diff --git a/src/store.rs b/src/store.rs index a793382..2f86e4e 100644 --- a/src/store.rs +++ b/src/store.rs @@ -3,6 +3,7 @@ use crate::descriptors::Descriptor; use std::fs; use std::path::Path; use std::fmt; +use log::info; pub struct DescriptorStore { map: HashMap @@ -39,6 +40,7 @@ impl DescriptorStore { let file = node.expect("Error walking directory"); let name = file.file_name().into_string().expect("Issue with filename"); if !self.contains(name.to_string()) { + info!("Processing {}", name); let img = image::open(file.path()).expect("Unable to open file"); let phash = desc.describe(img); self.map.insert(name, phash); @@ -51,9 +53,9 @@ impl fmt::Display for DescriptorStore { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut output = String::new(); for (key, val) in self.map.iter() { - let entry = format!("{key: >15}: {val}\t {val:b}\n"); + let entry = format!("{key: >15}: {val}\t {val:064b}\n"); output.push_str(&entry); } - write!(f, "{}", output) + write!(f, "DescriptorStore:\n{}", output) } } \ No newline at end of file From 8838e41eb1769e99f7d48db319b94540fb82e2aa Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sat, 11 May 2024 01:22:58 +0200 Subject: [PATCH 19/32] output fixes --- src/bin/dctfilename.rs | 4 +++- src/bin/experiment.rs | 3 ++- src/descriptors/dct.rs | 24 +++++++++++------------- src/store.rs | 2 +- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/bin/dctfilename.rs b/src/bin/dctfilename.rs index 72eedf3..4c42190 100644 --- a/src/bin/dctfilename.rs +++ b/src/bin/dctfilename.rs @@ -2,6 +2,7 @@ use image_similarity::descriptors::{DCT, Descriptor}; use std::env; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use log::debug; struct Cfg { path: String @@ -31,7 +32,8 @@ fn main() { let img = image::open(cfg.path) .expect("Unable to open file"); let phash: u64 = desc.describe(img); - println!("{phash}\n{phash:b}"); + debug!("Phash integer:\n{phash}"); + debug!("Phash binary:\n{phash:b}"); let output = URL_SAFE_NO_PAD.encode(phash.to_be_bytes()); println!("{output}") } diff --git a/src/bin/experiment.rs b/src/bin/experiment.rs index e51f0e5..811b892 100644 --- a/src/bin/experiment.rs +++ b/src/bin/experiment.rs @@ -3,7 +3,8 @@ use std::collections::HashMap; use std::fs; fn main() { - //Init all descriptors: + env_logger::init(); + let desc = DCT::new(); //Initialize hashmap, empty or from cached file: diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index 597d003..2e04c8a 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -1,12 +1,13 @@ use image::{save_buffer, GenericImageView}; use std::f64::consts::{PI, SQRT_2}; -use crate::descriptors::{Descriptor, print_matrix, DCT}; +use crate::descriptors::{Descriptor, DCT, print_matrix}; +use log::debug; impl DCT { /// Returns a new DCT instance with a base quality DCT matrix. pub fn new() -> DCT { - let base_quantization_matrix: [u8; 64] = [ + let quantization_matrix: [u8; 64] = [ 16, 11, 10, 16, 24, 40, 51, 61, 12, 12, 14, 19, 26, 58, 60, 55, 14, 13, 16, 24, 40, 57, 69, 56, @@ -16,7 +17,7 @@ impl DCT { 49, 64, 78, 8, 10, 12, 12, 101, 72, 92, 95, 9, 11, 10, 103, 99, ]; - DCT { quantization_matrix: base_quantization_matrix } + DCT { quantization_matrix } } // Builds DCT with given quality value @@ -37,9 +38,7 @@ impl DCT { self } - pub fn dct(&self, img: image::DynamicImage) -> [f64; 64] { - //let qmatrix = self.quantization_matrix(self.quality); - //println!("Q-{} quantization matrix:\n{}", self.quality, print_matrix(qmatrix)); + fn dct(&self, img: image::DynamicImage) -> [f64; 64] { let mut dct_values: [f64; 64] = [0.0; 64]; img.save("resize.png").expect("Error saving file"); for u in 0..8 { @@ -65,13 +64,12 @@ impl DCT { (y*v*PI/16.0).cos() } dct_values[k] = alpha * sum; - //println!{"{}", dct_values[k]} } } dct_values } - pub fn idct(&self, dct_values: [f64; 64]) -> [u8; 64] { + fn idct(&self, dct_values: [f64; 64]) -> [u8; 64] { let mut reconstructed: [u8; 64] = [0; 64]; for k in 0..64 { let x = (k%8) as f64; @@ -107,24 +105,24 @@ impl Descriptor for DCT { fn describe(&self, img: image::DynamicImage) -> u64 { let img = self.resize(img); let mut dct_values = self.dct(img); - println!("DCT-coefficients:\n {}", print_matrix(dct_values)); + debug!("DCT-coefficients:\n {}", print_matrix(dct_values)); // Quantization: - println!("Using quantization matrix:\n{}", print_matrix(self.quantization_matrix)); + debug!("Using quantization matrix:\n{}", print_matrix(self.quantization_matrix)); for i in 0..64 { dct_values[i] = (dct_values[i] / self.quantization_matrix[i] as f64).round(); } - println!("DCT-coefficients, quantized:\n {}", print_matrix(dct_values)); + debug!("DCT-coefficients, quantized:\n {}", print_matrix(dct_values)); // De-quantization: for i in 0..64 { dct_values[i] = dct_values[i] * self.quantization_matrix[i] as f64; } - println!("DCT-coefficients, de-quantized:\n {}", print_matrix(dct_values)); + debug!("DCT-coefficients, de-quantized:\n {}", print_matrix(dct_values)); // Reconstruction original pixel values: let reconstructed = self.idct(dct_values); - println!("Reconstructed pixel values:\n{}", print_matrix(reconstructed)); + debug!("Reconstructed pixel values:\n{}", print_matrix(reconstructed)); save_buffer( "reconstructed.png", diff --git a/src/store.rs b/src/store.rs index 2f86e4e..20c6bcd 100644 --- a/src/store.rs +++ b/src/store.rs @@ -20,7 +20,7 @@ impl DescriptorStore { Ok(f) => rmp_serde::from_slice(&f).unwrap(), Err(_e) => HashMap::with_capacity(count), }; - DescriptorStore { map: map } + DescriptorStore { map } } /// Returns true iff the store already contains the key From b8ed4c0c7eca94d7b5b1e76b83f456ea6ae811b6 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 12 May 2024 00:16:38 +0200 Subject: [PATCH 20/32] store uitgewerkt --- src/main.rs | 12 ++++++--- src/store.rs | 73 +++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 64 insertions(+), 21 deletions(-) diff --git a/src/main.rs b/src/main.rs index 3da260d..09d360b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,8 +4,14 @@ use log::info; fn main() { env_logger::init(); - let mut store = DescriptorStore::new(); - let desc = DCT::new().with_quality(30); - store.insert_directory("img", desc); + let mut store = + DescriptorStore::new() + .with_file("store.messagepack"); info!("{}", store); + let desc = + DCT::new() + .with_quality(30); + store.insert_directory("data", desc); + info!("{}", store); + store.save().expect("Error saving"); } diff --git a/src/store.rs b/src/store.rs index 20c6bcd..23e3fd6 100644 --- a/src/store.rs +++ b/src/store.rs @@ -3,24 +3,46 @@ use crate::descriptors::Descriptor; use std::fs; use std::path::Path; use std::fmt; -use log::info; +use log::{info, debug, error}; + +#[derive(Debug)] +pub enum SaveError { + Serialization, + File, +} pub struct DescriptorStore { - map: HashMap + map: HashMap, + save_location: std::path::PathBuf, } impl DescriptorStore { - /// Makes a new DescriptorStore. - /// Reads from file if possible, else empty - pub fn new() -> DescriptorStore { - //TODO: Set directory and file location from global config - let count = fs::read_dir("img").unwrap().count(); - let map_file = fs::read("map.messagepack"); - let map: HashMap = match map_file { - Ok(f) => rmp_serde::from_slice(&f).unwrap(), - Err(_e) => HashMap::with_capacity(count), + /// Makes a new empty DescriptorStore with default settings + pub fn new() -> Self { + let map: HashMap = HashMap::new(); + DescriptorStore { map, save_location: std::path::PathBuf::from("store.messagepack")} + } + + /// Sets the file location and loads data from file (if available) + pub fn with_file>(mut self, path: P) -> Self { + self.save_location = std::path::PathBuf::from(path.as_ref()); + let map_file = fs::read(&self.save_location); + match map_file { + Ok(f) => self.map = rmp_serde::from_slice(&f).unwrap(), + Err(_) => info!("{} not found, starting from empty store.", self.save_location.display()), }; - DescriptorStore { map } + self + } + + pub fn save(&self) -> Result<(), SaveError> { + let serialized: Vec = match rmp_serde::to_vec(&self.map) { + Ok(value) => value, + Err(_e) => return Err(SaveError::Serialization), + }; + match fs::write(&self.save_location, &serialized) { + Ok(_) => Ok(()), + Err(_e) => Err(SaveError::File), + } } /// Returns true iff the store already contains the key @@ -29,22 +51,37 @@ impl DescriptorStore { } /// Inserts a single value into the store - pub fn insert(mut self, key: String, value: u64) { + pub fn insert(&mut self, key: String, value: u64) { self.map.insert(key, value); } - /// Calculates all descriptions with a given descriptor for a folder + /// Calculates all descriptions with a given descriptor pub fn insert_directory>(&mut self, dir: U, desc: T) { for node in fs::read_dir(dir).unwrap() { let file = node.expect("Error walking directory"); - let name = file.file_name().into_string().expect("Issue with filename"); + let name = match file.file_name().into_string() { + Ok(v) => v, + Err(e) => { + error!("Error reading {}: {:?}:", file.path().to_string_lossy(), e); + continue + } + }; if !self.contains(name.to_string()) { info!("Processing {}", name); - let img = image::open(file.path()).expect("Unable to open file"); + let img = match image::open(file.path()) { + Ok(v) => v, + Err(e) => { + error!("Failed to process {}: {}", name, e); + continue + } + }; let phash = desc.describe(img); self.map.insert(name, phash); + } else { + debug!("{} already known, skipping.", name); } + self.save().expect("error"); } } } @@ -53,9 +90,9 @@ impl fmt::Display for DescriptorStore { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut output = String::new(); for (key, val) in self.map.iter() { - let entry = format!("{key: >15}: {val}\t {val:064b}\n"); + let entry = format!("{key:020}: {val}\t {val:064b}\n"); output.push_str(&entry); } - write!(f, "DescriptorStore:\n{}", output) + write!(f, "DescriptorStore:\n{}Total: {}", output, self.map.len()) } } \ No newline at end of file From 3c657404f2dcb8a5da5728e867cbc313bc4fd7b9 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 12 May 2024 10:29:58 +0200 Subject: [PATCH 21/32] base quality on wrong q-factor --- src/descriptors/dct.rs | 2 +- src/store.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index 2e04c8a..a9144c7 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -26,7 +26,7 @@ impl DCT { let scalar: f32 = match quality { 1..=49 => 5000.0/quality as f32, 50..=100 => 200.0 - 2.0*quality as f32, - _ => 1.0 //TODO: error + _ => 100.0 // Invalid input: set to base quality }; for i in 0..64 { quantization_matrix[i] = ((scalar * self.quantization_matrix[i] as f32 + 50.0) / 100.0).floor() as u8; diff --git a/src/store.rs b/src/store.rs index 23e3fd6..63ae6dc 100644 --- a/src/store.rs +++ b/src/store.rs @@ -55,7 +55,6 @@ impl DescriptorStore { self.map.insert(key, value); } - /// Calculates all descriptions with a given descriptor pub fn insert_directory>(&mut self, dir: U, desc: T) { for node in fs::read_dir(dir).unwrap() { From a0c1a650b0c84d3bef17f3d3d39cc2cc27dd0f34 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 12 May 2024 10:44:15 +0200 Subject: [PATCH 22/32] Only reconstruct on debug --- src/descriptors/dct.rs | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index a9144c7..c453f33 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -1,7 +1,7 @@ use image::{save_buffer, GenericImageView}; use std::f64::consts::{PI, SQRT_2}; use crate::descriptors::{Descriptor, DCT, print_matrix}; -use log::debug; +use log::{debug, log_enabled, Level}; impl DCT { @@ -114,23 +114,25 @@ impl Descriptor for DCT { } debug!("DCT-coefficients, quantized:\n {}", print_matrix(dct_values)); - // De-quantization: - for i in 0..64 { - dct_values[i] = dct_values[i] * self.quantization_matrix[i] as f64; + if log_enabled!(Level::Debug) { + // De-quantization: + for i in 0..64 { + dct_values[i] = dct_values[i] * self.quantization_matrix[i] as f64; + } + debug!("DCT-coefficients, de-quantized:\n {}", print_matrix(dct_values)); + + // Reconstruction original pixel values: + let reconstructed = self.idct(dct_values); + debug!("Reconstructed pixel values:\n{}", print_matrix(reconstructed)); + + save_buffer( + "reconstructed.png", + &reconstructed, + 8, + 8, + image::ColorType::L8 + ).expect("Error saving buffer"); } - debug!("DCT-coefficients, de-quantized:\n {}", print_matrix(dct_values)); - - // Reconstruction original pixel values: - let reconstructed = self.idct(dct_values); - debug!("Reconstructed pixel values:\n{}", print_matrix(reconstructed)); - - save_buffer( - "reconstructed.png", - &reconstructed, - 8, - 8, - image::ColorType::L8 - ).expect("Error saving buffer"); // Calculating descriptor from dct values: let mut mask: u64 = 0; From 13fd6d06350630f128f6ed5567b8c17e3b73f73e Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 12 May 2024 13:11:52 +0200 Subject: [PATCH 23/32] only save when debugging --- src/descriptors/dct.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index c453f33..b399554 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -40,7 +40,6 @@ impl DCT { fn dct(&self, img: image::DynamicImage) -> [f64; 64] { let mut dct_values: [f64; 64] = [0.0; 64]; - img.save("resize.png").expect("Error saving file"); for u in 0..8 { for v in 0..8 { let k = (v*8)+u; @@ -115,6 +114,7 @@ impl Descriptor for DCT { debug!("DCT-coefficients, quantized:\n {}", print_matrix(dct_values)); if log_enabled!(Level::Debug) { + img.save("resize.png").expect("Error saving file"); // De-quantization: for i in 0..64 { dct_values[i] = dct_values[i] * self.quantization_matrix[i] as f64; From fcfdf4add216afe7c0f6b8bfe71f96c4d8b2eb5d Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 12 May 2024 13:15:13 +0200 Subject: [PATCH 24/32] renames resized image and fixes borrowing issue --- src/descriptors/dct.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index b399554..ab00797 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -38,7 +38,7 @@ impl DCT { self } - fn dct(&self, img: image::DynamicImage) -> [f64; 64] { + fn dct(&self, img: &image::DynamicImage) -> [f64; 64] { let mut dct_values: [f64; 64] = [0.0; 64]; for u in 0..8 { for v in 0..8 { @@ -102,8 +102,8 @@ impl DCT { impl Descriptor for DCT { fn describe(&self, img: image::DynamicImage) -> u64 { - let img = self.resize(img); - let mut dct_values = self.dct(img); + let resized = self.resize(img); + let mut dct_values = self.dct(&resized); debug!("DCT-coefficients:\n {}", print_matrix(dct_values)); // Quantization: @@ -114,7 +114,7 @@ impl Descriptor for DCT { debug!("DCT-coefficients, quantized:\n {}", print_matrix(dct_values)); if log_enabled!(Level::Debug) { - img.save("resize.png").expect("Error saving file"); + resized.save("resize.png").expect("Error saving file"); // De-quantization: for i in 0..64 { dct_values[i] = dct_values[i] * self.quantization_matrix[i] as f64; From cb4c5622145cb75abb43b433f3860fbff7b0ca39 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 12 May 2024 13:19:27 +0200 Subject: [PATCH 25/32] pass image as reference --- src/bin/dctfilename.rs | 2 +- src/descriptors/dct.rs | 2 +- src/descriptors/mod.rs | 4 ++-- src/store.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bin/dctfilename.rs b/src/bin/dctfilename.rs index 4c42190..931ced6 100644 --- a/src/bin/dctfilename.rs +++ b/src/bin/dctfilename.rs @@ -31,7 +31,7 @@ fn main() { let img = image::open(cfg.path) .expect("Unable to open file"); - let phash: u64 = desc.describe(img); + let phash: u64 = desc.describe(&img); debug!("Phash integer:\n{phash}"); debug!("Phash binary:\n{phash:b}"); let output = URL_SAFE_NO_PAD.encode(phash.to_be_bytes()); diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index ab00797..1244af9 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -101,7 +101,7 @@ impl DCT { } impl Descriptor for DCT { - fn describe(&self, img: image::DynamicImage) -> u64 { + fn describe(&self, img: &image::DynamicImage) -> u64 { let resized = self.resize(img); let mut dct_values = self.dct(&resized); debug!("DCT-coefficients:\n {}", print_matrix(dct_values)); diff --git a/src/descriptors/mod.rs b/src/descriptors/mod.rs index 0c54c9d..fe0690f 100644 --- a/src/descriptors/mod.rs +++ b/src/descriptors/mod.rs @@ -2,10 +2,10 @@ pub trait Descriptor { fn info(&self) -> String { "Descriptor".to_string() } - fn resize(&self, img: image::DynamicImage) -> image::DynamicImage { + fn resize(&self, img: &image::DynamicImage) -> image::DynamicImage { img.grayscale().thumbnail_exact(8, 8) } - fn describe(&self, img: image::DynamicImage) -> u64; + fn describe(&self, img: &image::DynamicImage) -> u64; fn distance(&self, a: u64, b: u64) -> u64 { (a^b).count_ones().into() } diff --git a/src/store.rs b/src/store.rs index 63ae6dc..04a6029 100644 --- a/src/store.rs +++ b/src/store.rs @@ -75,7 +75,7 @@ impl DescriptorStore { continue } }; - let phash = desc.describe(img); + let phash = desc.describe(&img); self.map.insert(name, phash); } else { debug!("{} already known, skipping.", name); From 659b53ed5cba9a17b7b1724fcba17031aa2a835a Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 12 May 2024 14:37:41 +0200 Subject: [PATCH 26/32] DCT mask calculation updated --- src/bin/dctfilename.rs | 2 +- src/descriptors/dct.rs | 66 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/src/bin/dctfilename.rs b/src/bin/dctfilename.rs index 931ced6..0641c45 100644 --- a/src/bin/dctfilename.rs +++ b/src/bin/dctfilename.rs @@ -33,7 +33,7 @@ fn main() { .expect("Unable to open file"); let phash: u64 = desc.describe(&img); debug!("Phash integer:\n{phash}"); - debug!("Phash binary:\n{phash:b}"); + debug!("Phash binary:\n{phash:064b}"); let output = URL_SAFE_NO_PAD.encode(phash.to_be_bytes()); println!("{output}") } diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index 1244af9..d6835fa 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -107,7 +107,7 @@ impl Descriptor for DCT { debug!("DCT-coefficients:\n {}", print_matrix(dct_values)); // Quantization: - debug!("Using quantization matrix:\n{}", print_matrix(self.quantization_matrix)); + //debug!("Using quantization matrix:\n{}", print_matrix(self.quantization_matrix)); for i in 0..64 { dct_values[i] = (dct_values[i] / self.quantization_matrix[i] as f64).round(); } @@ -119,11 +119,10 @@ impl Descriptor for DCT { for i in 0..64 { dct_values[i] = dct_values[i] * self.quantization_matrix[i] as f64; } - debug!("DCT-coefficients, de-quantized:\n {}", print_matrix(dct_values)); + //debug!("DCT-coefficients, de-quantized:\n {}", print_matrix(dct_values)); // Reconstruction original pixel values: let reconstructed = self.idct(dct_values); - debug!("Reconstructed pixel values:\n{}", print_matrix(reconstructed)); save_buffer( "reconstructed.png", @@ -135,13 +134,64 @@ impl Descriptor for DCT { } // Calculating descriptor from dct values: - let mut mask: u64 = 0; - for dct in dct_values { - if dct > 0.0 { - mask += 1 + + // Zigzag order for our flattened array, + // first 28 elements only + let zigzag: [usize; 28] = [ + 0, + 1, 8, + 16, 9, 2, + 3, 10, 17, 24, + 32, 25, 18, 11, 4, + 5, 12, 19, 26, 33, 40, + 48, 41, 34, 27, 20, 13, 6, + ]; + + // Mask that indicates if a dct coefficient is positive or negative + // By convention, when sign bit is 1, number is negative + let mut sign_mask: u64 = 0; + // If first horizontal AC coefficient is negative + // This might account for horizontal flips when applied to all horizontal coefficients. + let sign_mult = dct_values[1].signum(); + + // Mask that indicates if a coefficient is bigger or smaller than previous in order + let mut pearson_mask: u64 = 0; + let mut prev = dct_values[0]; + + + for i in zigzag { + let cur = dct_values[i]; + if cur > prev { + pearson_mask += 1; } - mask = mask << 1; + prev = cur; + + let signum = dct_values[i].signum(); + // Only multiply sign if dct-coefficient contains a horizontal component. + if + i % 8 != 0 && sign_mult * signum < 0.0 + || + signum < 0.0 + { + sign_mask += 1; + } + + // Shift masks + sign_mask = sign_mask << 1; + pearson_mask = pearson_mask << 1; } + debug!("Sign mask: {:028b}", sign_mask); + debug!("Pearson mask: {:028b}", pearson_mask); + + let mut mask = sign_mask; + debug!("Mask: {:064b}", mask); + mask = mask << 28; + debug!("Mask: {:064b}", mask); + mask += pearson_mask; + debug!("Mask: {:064b}", mask); + mask = mask << 8; + debug!("Mask: {:064b}", mask); + // TODO: Do something with these last 8 bits. mask } } \ No newline at end of file From 899ecbbee4ff5da06b374b5a063802681c25e7f1 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 12 May 2024 14:38:25 +0200 Subject: [PATCH 27/32] experiment binary fix --- src/bin/experiment.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/experiment.rs b/src/bin/experiment.rs index 811b892..ab2a847 100644 --- a/src/bin/experiment.rs +++ b/src/bin/experiment.rs @@ -21,7 +21,7 @@ fn main() { let name = file.file_name().into_string().expect("Issue with filename"); if !map.contains_key(&name) { let img = image::open(file.path()).expect("Unable to open file"); - let phash = desc.describe(img); + let phash = desc.describe(&img); map.insert(name, phash); } } From 2bc050bb4426136085a89d187059add27732622c Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 12 May 2024 15:46:31 +0200 Subject: [PATCH 28/32] docstrings --- src/descriptors/dct.rs | 4 +++- src/descriptors/mod.rs | 3 +++ src/lib.rs | 4 ++++ src/store.rs | 3 +++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index d6835fa..15cb98a 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -20,7 +20,8 @@ impl DCT { DCT { quantization_matrix } } - // Builds DCT with given quality value + /// Builds DCT with given quality value + /// quality pub fn with_quality(mut self, quality: u8) -> Self { let mut quantization_matrix: [u8; 64] = [0; 64]; let scalar: f32 = match quality { @@ -38,6 +39,7 @@ impl DCT { self } + /// fn dct(&self, img: &image::DynamicImage) -> [f64; 64] { let mut dct_values: [f64; 64] = [0.0; 64]; for u in 0..8 { diff --git a/src/descriptors/mod.rs b/src/descriptors/mod.rs index fe0690f..88ccf30 100644 --- a/src/descriptors/mod.rs +++ b/src/descriptors/mod.rs @@ -1,3 +1,6 @@ +//! Descriptors that can be used to describe an image. +//! If two images are (almost) the same, their descriptions will be the same. + pub trait Descriptor { fn info(&self) -> String { "Descriptor".to_string() diff --git a/src/lib.rs b/src/lib.rs index c807575..ea0f982 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1,6 @@ +//! Near-copy image similarity detection. + +//! Descriptors that can be used to describe an image. +//! If two images are (almost) the same, their descriptions will be the same. pub mod descriptors; pub mod store; \ No newline at end of file diff --git a/src/store.rs b/src/store.rs index 04a6029..99f9940 100644 --- a/src/store.rs +++ b/src/store.rs @@ -1,3 +1,6 @@ +//! Persistent store of all calculated image descriptions. +//! Can be queried to find identical or similar images. + use std::collections::HashMap; use crate::descriptors::Descriptor; use std::fs; From 335f7e7d8c3e91975078a3474fe10dfd3cf37a12 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Sun, 12 May 2024 16:07:43 +0200 Subject: [PATCH 29/32] removes default implementation of info --- src/descriptors/dct.rs | 3 +++ src/descriptors/mod.rs | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index 15cb98a..fbcc9d9 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -103,6 +103,9 @@ impl DCT { } impl Descriptor for DCT { + fn info(&self) -> String { + "DCT".to_string() + } fn describe(&self, img: &image::DynamicImage) -> u64 { let resized = self.resize(img); let mut dct_values = self.dct(&resized); diff --git a/src/descriptors/mod.rs b/src/descriptors/mod.rs index 88ccf30..8d5806c 100644 --- a/src/descriptors/mod.rs +++ b/src/descriptors/mod.rs @@ -2,9 +2,8 @@ //! If two images are (almost) the same, their descriptions will be the same. pub trait Descriptor { - fn info(&self) -> String { - "Descriptor".to_string() - } + /// Print the name of the descriptor, + fn info(&self) -> String; fn resize(&self, img: &image::DynamicImage) -> image::DynamicImage { img.grayscale().thumbnail_exact(8, 8) } From 4327fe8c7ca1905c9321ac88f12864d18d8ee535 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Tue, 14 May 2024 14:37:33 +0200 Subject: [PATCH 30/32] inverted hashmap, added bktree --- Cargo.lock | 73 ++++++++++++++++++++++++++++++++++++------ Cargo.toml | 1 + src/bin/dctquery.rs | 42 ++++++++++++++++++++++++ src/descriptors/dct.rs | 4 +-- src/store.rs | 40 ++++++++++++++++++----- 5 files changed, 140 insertions(+), 20 deletions(-) create mode 100644 src/bin/dctquery.rs diff --git a/Cargo.lock b/Cargo.lock index 54af603..af7c90e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,6 +90,15 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bktree" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb1e744816f6a3b9e962186091867f3e5959d4dac995777ec254631cb00b21c" +dependencies = [ + "num", +] + [[package]] name = "bumpalo" version = "3.11.1" @@ -334,6 +343,7 @@ name = "image_similarity" version = "0.1.0" dependencies = [ "base64", + "bktree", "env_logger", "image", "log", @@ -436,20 +446,52 @@ dependencies = [ ] [[package]] -name = "num-integer" -version = "0.1.45" +name = "num" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ - "autocfg", + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", "num-traits", ] [[package]] -name = "num-rational" -version = "0.4.1" +name = "num-bigint" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -457,10 +499,21 @@ dependencies = [ ] [[package]] -name = "num-traits" -version = "0.2.15" +name = "num-rational" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] diff --git a/Cargo.toml b/Cargo.toml index ca7e200..305ba96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,3 +12,4 @@ rmp-serde = "1.1.1" base64 = "0.22.1" log = "0.4.21" env_logger = "0.11.3" +bktree = "1.0.1" diff --git a/src/bin/dctquery.rs b/src/bin/dctquery.rs new file mode 100644 index 0000000..65bec74 --- /dev/null +++ b/src/bin/dctquery.rs @@ -0,0 +1,42 @@ +//use image_similarity::descriptors::dct::DCT; +use image_similarity::descriptors::{DCT, Descriptor}; +use image_similarity::store::DescriptorStore; +use std::env; +use log::{info, debug, error}; + +struct Cfg { + path: String +} + +impl Cfg{ + fn load(args: &[String]) -> Result { + if args.len() < 2 { + return Err("Filename argument required"); + } + let path = args[1].clone(); + Ok(Cfg { path }) + } +} + +fn main() { + env_logger::init(); + + //Init all descriptors: + let desc = DCT::new().with_quality(50); + + let args: Vec = env::args().collect(); + + let cfg = Cfg::load(&args) + .expect("Error loading config"); + + let img = image::open(cfg.path) + .expect("Unable to open file"); + let store = + DescriptorStore::new() + .with_file("store.messagepack"); + //info!("{}", store); + let phash: u64 = desc.describe(&img); + info!("Phash integer:\n{phash}"); + info!("Phash binary:\n{phash:064b}"); + store.knn(phash, 4); +} \ No newline at end of file diff --git a/src/descriptors/dct.rs b/src/descriptors/dct.rs index fbcc9d9..62df718 100644 --- a/src/descriptors/dct.rs +++ b/src/descriptors/dct.rs @@ -174,8 +174,8 @@ impl Descriptor for DCT { let signum = dct_values[i].signum(); // Only multiply sign if dct-coefficient contains a horizontal component. if - i % 8 != 0 && sign_mult * signum < 0.0 - || + //i % 8 != 0 && sign_mult * signum < 0.0 + // || signum < 0.0 { sign_mask += 1; diff --git a/src/store.rs b/src/store.rs index 99f9940..4461f1f 100644 --- a/src/store.rs +++ b/src/store.rs @@ -7,6 +7,7 @@ use std::fs; use std::path::Path; use std::fmt; use log::{info, debug, error}; +use bktree::*; #[derive(Debug)] pub enum SaveError { @@ -15,15 +16,20 @@ pub enum SaveError { } pub struct DescriptorStore { - map: HashMap, + map: HashMap, save_location: std::path::PathBuf, + bktree: BkTree, } impl DescriptorStore { /// Makes a new empty DescriptorStore with default settings pub fn new() -> Self { - let map: HashMap = HashMap::new(); - DescriptorStore { map, save_location: std::path::PathBuf::from("store.messagepack")} + let map: HashMap = HashMap::new(); + DescriptorStore { + map, + save_location: std::path::PathBuf::from("store.messagepack"), + bktree: BkTree::new(hamming_distance) + } } /// Sets the file location and loads data from file (if available) @@ -34,6 +40,9 @@ impl DescriptorStore { Ok(f) => self.map = rmp_serde::from_slice(&f).unwrap(), Err(_) => info!("{} not found, starting from empty store.", self.save_location.display()), }; + for i in self.map.keys() { + self.bktree.insert(*i); + } self } @@ -49,12 +58,12 @@ impl DescriptorStore { } /// Returns true iff the store already contains the key - pub fn contains(&self, key: String) -> bool { + pub fn contains(&self, key: u64) -> bool { self.map.contains_key(&key) } /// Inserts a single value into the store - pub fn insert(&mut self, key: String, value: u64) { + pub fn insert(&mut self, key: u64, value: String) { self.map.insert(key, value); } @@ -69,7 +78,7 @@ impl DescriptorStore { continue } }; - if !self.contains(name.to_string()) { + if true { // !self.contains(name.to_string()) { info!("Processing {}", name); let img = match image::open(file.path()) { Ok(v) => v, @@ -79,20 +88,35 @@ impl DescriptorStore { } }; let phash = desc.describe(&img); - self.map.insert(name, phash); + if self.contains(phash) { + println!("{} duplicate of {:?}", name, self.map.get(&phash)); + } + self.map.insert(phash, name); } else { debug!("{} already known, skipping.", name); } self.save().expect("error"); } } + + /// K-nearest neighbors + pub fn knn(&self, from: u64, k: isize) { + println!("{from:b}"); + for (element, distance) in self.bktree.find(from, k) { + let name = self.map.get(element); + match name { + Some(n) => println!("{element:b}: {n} (distance: {distance})"), + None => () + }; + } + } } impl fmt::Display for DescriptorStore { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut output = String::new(); for (key, val) in self.map.iter() { - let entry = format!("{key:020}: {val}\t {val:064b}\n"); + let entry = format!("{key:064b}: {val}\t {val:020}\n"); output.push_str(&entry); } write!(f, "DescriptorStore:\n{}Total: {}", output, self.map.len()) From 8c3112f9f2b8d7a1f3e1c360df1752183773ab83 Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Tue, 14 May 2024 23:18:18 +0200 Subject: [PATCH 31/32] viuer terminal images --- Cargo.lock | 351 ++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 1 + src/bin/dctquery.rs | 11 +- src/store.rs | 24 ++- 4 files changed, 371 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index af7c90e..b1b58e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ansi_colours" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1558bd2075d341b9ca698ec8eb6fcc55a746b1fc4255585aad5b141d918a80" +dependencies = [ + "rgb", +] + [[package]] name = "anstream" version = "0.6.14" @@ -53,7 +62,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -63,7 +72,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -72,6 +81,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -90,6 +105,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + [[package]] name = "bktree" version = "1.0.1" @@ -135,6 +156,18 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -187,6 +220,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossterm" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +dependencies = [ + "bitflags 2.5.0", + "crossterm_winapi", + "libc", + "mio", + "parking_lot", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -199,6 +257,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "env_filter" version = "0.1.0" @@ -222,6 +286,16 @@ dependencies = [ "log", ] +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "exr" version = "1.5.2" @@ -237,6 +311,12 @@ dependencies = [ "threadpool", ] +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + [[package]] name = "flate2" version = "1.0.24" @@ -342,13 +422,14 @@ dependencies = [ name = "image_similarity" version = "0.1.0" dependencies = [ - "base64", + "base64 0.22.1", "bktree", "env_logger", "image", "log", "rmp-serde", "serde", + "viuer", ] [[package]] @@ -375,6 +456,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "lebe" version = "0.5.2" @@ -383,9 +470,15 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.137" +version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" + +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -436,6 +529,18 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + [[package]] name = "nanorand" version = "0.7.0" @@ -534,6 +639,29 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +[[package]] +name = "parking_lot" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.5", +] + [[package]] name = "paste" version = "1.0.9" @@ -566,7 +694,7 @@ version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d708eaf860a19b19ce538740d2b4bdeeb8337fa53f7738455e706623ad5c638" dependencies = [ - "bitflags", + "bitflags 1.3.2", "crc32fast", "flate2", "miniz_oxide 0.6.2", @@ -614,6 +742,15 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "regex" version = "1.10.4" @@ -643,6 +780,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +[[package]] +name = "rgb" +version = "0.8.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8" +dependencies = [ + "bytemuck", +] + [[package]] name = "rmp" version = "0.8.11" @@ -665,6 +811,19 @@ dependencies = [ "serde", ] +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "scoped_threadpool" version = "0.1.9" @@ -683,6 +842,36 @@ version = "1.0.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -709,6 +898,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "threadpool" version = "1.8.1" @@ -741,6 +951,22 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "viuer" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec2ede5c8814363f92f862892dfe71a266f6816b649ca435aed1ff5e2cf3454e" +dependencies = [ + "ansi_colours", + "base64 0.21.7", + "console", + "crossterm", + "image", + "lazy_static", + "tempfile", + "termcolor", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -807,13 +1033,68 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -822,28 +1103,46 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.5" @@ -856,24 +1155,48 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.5" diff --git a/Cargo.toml b/Cargo.toml index 305ba96..4375ae2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,4 @@ base64 = "0.22.1" log = "0.4.21" env_logger = "0.11.3" bktree = "1.0.1" +viuer = "0.7.1" diff --git a/src/bin/dctquery.rs b/src/bin/dctquery.rs index 65bec74..4a998e1 100644 --- a/src/bin/dctquery.rs +++ b/src/bin/dctquery.rs @@ -3,6 +3,7 @@ use image_similarity::descriptors::{DCT, Descriptor}; use image_similarity::store::DescriptorStore; use std::env; use log::{info, debug, error}; +use viuer::{Config, print}; struct Cfg { path: String @@ -31,6 +32,14 @@ fn main() { let img = image::open(cfg.path) .expect("Unable to open file"); + println!("Iamge"); + let conf = viuer::Config { + width: Some(16), + height: Some(8), + ..Default::default() + }; + viuer::print(&img, &conf).expect("Image printing failed."); + println!("Iamge"); let store = DescriptorStore::new() .with_file("store.messagepack"); @@ -38,5 +47,5 @@ fn main() { let phash: u64 = desc.describe(&img); info!("Phash integer:\n{phash}"); info!("Phash binary:\n{phash:064b}"); - store.knn(phash, 4); + store.knn(phash, 10); } \ No newline at end of file diff --git a/src/store.rs b/src/store.rs index 4461f1f..d13840a 100644 --- a/src/store.rs +++ b/src/store.rs @@ -8,6 +8,7 @@ use std::path::Path; use std::fmt; use log::{info, debug, error}; use bktree::*; +use viuer::{Config, print}; #[derive(Debug)] pub enum SaveError { @@ -102,10 +103,31 @@ impl DescriptorStore { /// K-nearest neighbors pub fn knn(&self, from: u64, k: isize) { println!("{from:b}"); + let mut x = 0; + let mut y = 8; + let (term_width, _) = viuer::terminal_size(); + println!("\t\t\t\t\tTERMWIDTH {term_width}"); for (element, distance) in self.bktree.find(from, k) { let name = self.map.get(element); match name { - Some(n) => println!("{element:b}: {n} (distance: {distance})"), + Some(n) => { + println!("{element:b}: {n} (distance: {distance})"); + if x+16 >= term_width { + x = 0; + y += 8; + } + let conf = viuer::Config { + width: Some(16), + height: Some(8), + x: x, + y: y, + ..Default::default() + }; + x += 16; + let path = "data/".to_string() + &n; + let img = image::open(&path).unwrap(); + viuer::print(&img, &conf).expect("Image printing failed."); + }, None => () }; } From 1b2536f1e0832c1a7f09c18a31d06c0ed009af8e Mon Sep 17 00:00:00 2001 From: Mark Hoekveen Date: Fri, 17 May 2024 13:43:18 +0200 Subject: [PATCH 32/32] refactors and new dct binary --- Cargo.lock | 78 ++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 1 + src/bin/dct.rs | 66 +++++++++++++++++++++++++++++++++++ src/bin/dctfilename.rs | 24 ++++--------- src/bin/dctquery.rs | 1 - src/main.rs | 58 ++++++++++++++++++++++++++----- src/store.rs | 4 +-- 7 files changed, 196 insertions(+), 36 deletions(-) create mode 100644 src/bin/dct.rs diff --git a/Cargo.lock b/Cargo.lock index b1b58e8..66a23d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,6 +144,46 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.64", +] + +[[package]] +name = "clap_lex" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" + [[package]] name = "color_quant" version = "1.1.0" @@ -384,6 +424,12 @@ dependencies = [ "crunchy", ] +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -424,6 +470,7 @@ version = "0.1.0" dependencies = [ "base64 0.22.1", "bktree", + "clap", "env_logger", "image", "log", @@ -685,7 +732,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.103", ] [[package]] @@ -702,18 +749,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.21" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -887,6 +934,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" version = "1.0.103" @@ -898,6 +951,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ad3dee41f36859875573074334c200d1add8e4a87bb37113ebd31d926b7b11f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.10.1" @@ -994,7 +1058,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.103", "wasm-bindgen-shared", ] @@ -1016,7 +1080,7 @@ checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.103", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/Cargo.toml b/Cargo.toml index 4375ae2..9d7e515 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,3 +14,4 @@ log = "0.4.21" env_logger = "0.11.3" bktree = "1.0.1" viuer = "0.7.1" +clap = { version = "4.5.4", features = ["derive"] } diff --git a/src/bin/dct.rs b/src/bin/dct.rs new file mode 100644 index 0000000..266f324 --- /dev/null +++ b/src/bin/dct.rs @@ -0,0 +1,66 @@ +use std::env; +use std::f64::consts::{PI, SQRT_2}; +use image::{save_buffer, GenericImageView, GrayImage}; + +fn dct(img: &image::DynamicImage) -> GrayImage { + //let mut dct_values: = [0.0; img.width() * img.height()]; + let w = img.width(); + let h = img.height(); + let mut dct_img = GrayImage::new(w, h); + + //let mut dct_values: Vec = Vec::new(); + for u in 0..w { + for v in 0..h { + let w = w as f64; + let h = h as f64; + //let k = (v*img.width())+u; + let mut alpha = 0.25; + if u == 0 { + alpha = alpha / SQRT_2 + } + if v == 0 { + alpha = alpha / SQRT_2 + } + let v: f64 = v as f64; + let u: f64 = u as f64; + let mut sum: f64 = 0.0; + for (x, y, pix) in img.pixels() { + let x: f64 = x as f64; + let y: f64 = y as f64; + let pixel = pix[0] as f64; + // sum += + // pixel * + // (x*u*PI/16.0).cos() * + // (y*v*PI/16.0).cos() + sum += + pixel * + ( + PI/w* + (x+0.5)*u + ).cos() + * + ( + PI/h* + (y+0.5)*v + ).cos() + } + //dct_values[k] = alpha * sum; + //dct_values.push(alpha * sum); + print!("{sum}\t"); + //sum = (sum + 200000.0)/784.0; + print!("{}\n", sum * alpha); + let pixel = (sum).abs() as u8; + println!("{u}, {v}:\t {sum}"); + dct_img.put_pixel(u as u32, v as u32, image::Luma([pixel])); + } + } + dct_img +} + +fn main() { + let args: Vec = env::args().collect(); + let path = args[1].clone(); + let img = image::open(&path).unwrap(); + let dct = dct(&img); + dct.save("dct.png").unwrap(); +} \ No newline at end of file diff --git a/src/bin/dctfilename.rs b/src/bin/dctfilename.rs index 0641c45..b81c50e 100644 --- a/src/bin/dctfilename.rs +++ b/src/bin/dctfilename.rs @@ -1,34 +1,22 @@ //use image_similarity::descriptors::dct::DCT; use image_similarity::descriptors::{DCT, Descriptor}; -use std::env; +use std::path::PathBuf; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use log::debug; +use clap::Parser; +#[derive(Parser)] +#[command(version, about, long_about = None)] struct Cfg { - path: String -} - -impl Cfg{ - fn load(args: &[String]) -> Result { - if args.len() < 2 { - return Err("Filename argument required"); - } - let path = args[1].clone(); - Ok(Cfg { path }) - } + path: PathBuf } fn main() { env_logger::init(); + let cfg = Cfg::parse(); - //Init all descriptors: let desc = DCT::new().with_quality(50); - let args: Vec = env::args().collect(); - - let cfg = Cfg::load(&args) - .expect("Error loading config"); - let img = image::open(cfg.path) .expect("Unable to open file"); let phash: u64 = desc.describe(&img); diff --git a/src/bin/dctquery.rs b/src/bin/dctquery.rs index 4a998e1..d8a74f8 100644 --- a/src/bin/dctquery.rs +++ b/src/bin/dctquery.rs @@ -3,7 +3,6 @@ use image_similarity::descriptors::{DCT, Descriptor}; use image_similarity::store::DescriptorStore; use std::env; use log::{info, debug, error}; -use viuer::{Config, print}; struct Cfg { path: String diff --git a/src/main.rs b/src/main.rs index 09d360b..5b6f311 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,17 +1,59 @@ use image_similarity::store::DescriptorStore; -use image_similarity::descriptors::DCT; +use image_similarity::descriptors::{Descriptor, DCT}; use log::info; +use std::path::PathBuf; +use std::thread; +use clap::Parser; -fn main() { - env_logger::init(); +#[derive(Parser)] +#[command(version, about, long_about = None)] +struct Cfg { + /// Path that contains the input images. Will not traverse directories. + path: PathBuf +} + +fn make_store(descriptor: T, input_path: PathBuf, save_path: PathBuf) -> () { let mut store = DescriptorStore::new() - .with_file("store.messagepack"); + .with_file(save_path); info!("{}", store); - let desc = - DCT::new() - .with_quality(30); - store.insert_directory("data", desc); + store.insert_directory(input_path, descriptor); info!("{}", store); store.save().expect("Error saving"); } + +fn main() { + env_logger::init(); + let cfg = Cfg::parse(); + + let mut threads = vec![]; + + let dct_50 = + DCT::new() + .with_quality(50); + let dct_30 = + DCT::new() + .with_quality(30); + let dct_10 = + DCT::new() + .with_quality(10); + + threads.push( + thread::spawn(move || { + make_store(dct_50, "data".into(), "dct50.messagepack".into()); + }) + ); + threads.push( + thread::spawn(move || { + make_store(dct_30, "data".into(), "dct30.messagepack".into()); + }) + ); + threads.push( + thread::spawn(move || { + make_store(dct_10, "data".into(), "dct10.messagepack".into()); + }) + ); + for thread in threads { + let _ = thread.join(); + } +} diff --git a/src/store.rs b/src/store.rs index d13840a..c5c7ef9 100644 --- a/src/store.rs +++ b/src/store.rs @@ -8,7 +8,6 @@ use std::path::Path; use std::fmt; use log::{info, debug, error}; use bktree::*; -use viuer::{Config, print}; #[derive(Debug)] pub enum SaveError { @@ -66,6 +65,7 @@ impl DescriptorStore { /// Inserts a single value into the store pub fn insert(&mut self, key: u64, value: String) { self.map.insert(key, value); + self.bktree.insert(key); } /// Calculates all descriptions with a given descriptor @@ -112,7 +112,7 @@ impl DescriptorStore { match name { Some(n) => { println!("{element:b}: {n} (distance: {distance})"); - if x+16 >= term_width { + if x+16 >= term_width { x = 0; y += 8; }