backprop harness

Cargo.lock

@@ -1128,7 +1128,7 @@ dependencies = [
  "hercules_rt",
  "juno_build",
  "nom 6.2.2",
- "rand 0.8.5",
+ "rand 0.9.0",
  "with_builtin_macros",
 ]
 

juno_samples/rodinia/backprop/Cargo.toml

@@ -21,4 +21,4 @@ async-std = "*"
 clap = { version = "*", features = ["derive"] }
 with_builtin_macros = "0.1.0"
 nom = "*"
-rand = "*"
+rand = "0.9.0"

juno_samples/rodinia/backprop/src/backprop.jn

@@ -81,7 +81,7 @@ fn backprop<input_n, hidden_n, output_n: usize>(
 //) -> (f32[input_n + 1, hidden_n + 1], f32[input_n + 1, hidden_n + 1],
 //      f32[hidden_n + 1, output_n + 1], f32[hidden_n + 1, output_n + 1]) {
 ) -> f32 {
-
+  // TODO: We may need to set some first elements to 1.0 in some places
   let hidden_vals = layer_forward::<input_n, hidden_n>(input_vals, input_weights);
   let output_vals = layer_forward::<hidden_n, output_n>(hidden_vals, hidden_weights);
   let (out_err, out_delta) = output_error::<output_n>(target, output_vals);

juno_samples/rodinia/backprop/src/main.rs

@@ -2,65 +2,124 @@
 
 juno_build::juno!("backprop");
 
+mod rust_backprop;
+
 use hercules_rt::{runner, HerculesMutBox, HerculesImmBox, HerculesImmBoxTo, HerculesMutBoxTo};
 
-use rand::Rng;
+use rand::{Rng, SeedableRng};
+use rand::rngs::StdRng;
+
+use clap::Parser;
+
+#[derive(Parser)]
+#[clap(author, version, about, long_about = None)]
+struct BackpropInputs {
+    layer_size: usize,
+}
+
+fn run_backprop(
+    input_n: u64,
+    hidden_n: u64,
+    output_n: u64,
+    input_vals: &[f32],
+    input_weights: &[f32],
+    hidden_weights: &[f32],
+    target: &[f32],
+    input_prev_weights: &[f32],
+    hidden_prev_weights: &[f32],
+) -> (Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>) {
+    let input_vals = HerculesImmBox::from(input_vals);
+    let target = HerculesImmBox::from(target);
+
+    let mut input_weights = HerculesMutBox::from(input_weights.to_vec());
+    let mut hidden_weights = HerculesMutBox::from(hidden_weights.to_vec());
+    let mut input_prev_weights = HerculesMutBox::from(input_prev_weights.to_vec());
+    let mut hidden_prev_weights = HerculesMutBox::from(hidden_prev_weights.to_vec());
+
+    let mut runner = runner!(backprop);
+    let _ = async_std::task::block_on(async {
+        runner.run(input_n,
+                   hidden_n,
+                   output_n,
+                   input_vals.to(),
+                   input_weights.to(),
+                   hidden_weights.to(),
+                   target.to(),
+                   input_prev_weights.to(),
+                   hidden_prev_weights.to(),
+        )
+        .await
+    });
+
+    (input_weights.as_slice().to_vec(), hidden_weights.as_slice().to_vec(),
+     input_prev_weights.as_slice().to_vec(), hidden_prev_weights.as_slice().to_vec())
+}
+
+fn backprop_harness(args: BackpropInputs) {
+    let BackpropInputs {
+        layer_size,
+    } = args;
+
+    let mut rng = StdRng::seed_from_u64(7);
+
+    let input_n = layer_size;
+    let hidden_n = 16;
+    let output_n = 1;
+
+    let mut input_vals = vec![0.0; input_n + 1];
+    input_vals[0] = 1.0;
+
+    // For some reason the bpnn_randomize_row function used on target just sets it to 0.1
+    let target = vec![0.1; output_n + 1];
+
+    let input_weights = (0..(input_n+1)*(hidden_n+1)).map(|_| rng.random::<f32>()).collect::<Vec<_>>();
+    let hidden_weights = (0..(hidden_n+1)*(output_n+1)).map(|_| rng.random::<f32>()).collect::<Vec<_>>();
+
+    let input_prev_weights = vec![0.0; (input_n + 1) * (hidden_n + 1)];
+    let hidden_prev_weights = vec![0.0; (hidden_n + 1) * (output_n + 1)];
+
+    let (juno_input_weights, juno_hidden_weights,
+         juno_input_prev_weights, juno_hidden_prev_weights)
+    = run_backprop(
+        input_n as u64,
+        hidden_n as u64,
+        output_n as u64,
+        &input_vals,
+        &input_weights,
+        &hidden_weights,
+        &target,
+        &input_prev_weights,
+        &hidden_prev_weights,
+    );
+
+    let (rust_input_weights, rust_hidden_weights,
+         rust_input_prev_weights, rust_hidden_prev_weights)
+    = rust_backprop::backprop(
+        input_n,
+        hidden_n,
+        output_n,
+        &input_vals,
+        input_weights,
+        hidden_weights,
+        &target,
+        input_prev_weights,
+        hidden_prev_weights,
+    );
+
+    assert_eq!(juno_input_weights, rust_input_weights);
+    assert_eq!(juno_hidden_weights, rust_hidden_weights);
+    assert_eq!(juno_input_prev_weights, rust_input_prev_weights);
+    assert_eq!(juno_hidden_prev_weights, rust_hidden_prev_weights);
+}
 
 fn main() {
-    let n_in  = 2;
-    let n_hid = 4;
-    let n_out = 1;
-
-    let mut rng = rand::thread_rng();
-
-    let mut in_weights = (0..(n_in+1)*(n_hid+1)).map(|_| rng.gen::<f32>()).collect::<Vec<_>>();
-    let mut in_prev_weights = (0..(n_in+1)*(n_hid+1)).map(|_| rng.gen::<f32>()).collect::<Vec<_>>();
-    let mut hid_weights = (0..(n_hid+1)*(n_out+1)).map(|_| rng.gen::<f32>()).collect::<Vec<_>>();
-    let mut hid_prev_weights = (0..(n_hid+1)*(n_out+1)).map(|_| rng.gen::<f32>()).collect::<Vec<_>>();
-
-    println!("{:?}", in_weights);
-    println!("{:?}", in_prev_weights);
-    println!("{:?}", hid_weights);
-    println!("{:?}", hid_prev_weights);
-    println!("");
-
-    let input = vec![1.0, 0.0, 1.0];
-    let target = vec![1.0, 1.0];
-
-    {
-        let mut in_weights = HerculesMutBox::from(in_weights.as_mut_slice());
-        let mut in_prev_weights = HerculesMutBox::from(in_prev_weights.as_mut_slice());
-        let mut hid_weights = HerculesMutBox::from(hid_weights.as_mut_slice());
-        let mut hid_prev_weights = HerculesMutBox::from(hid_prev_weights.as_mut_slice());
-
-        let input = HerculesImmBox::from(input.as_slice());
-        let target = HerculesImmBox::from(target.as_slice());
-
-        let mut runner = runner!(backprop);
-        // Drop the result, we don't care about it
-        async_std::task::block_on(async {
-            let _ = runner.run(
-                n_in as u64,
-                n_hid as u64,
-                n_out as u64,
-                input.to(),
-                in_weights.to(),
-                hid_weights.to(),
-                target.to(),
-                in_prev_weights.to(),
-                hid_prev_weights.to(),
-            ).await;
-
-            println!("{:?}", in_weights.as_slice());
-            println!("{:?}", in_prev_weights.as_slice());
-            println!("{:?}", hid_weights.as_slice());
-            println!("{:?}", hid_prev_weights.as_slice());
-            println!("");
-        });
-    }
-
-    println!("{:?}", in_weights);
-    println!("{:?}", in_prev_weights);
-    println!("{:?}", hid_weights);
-    println!("{:?}", hid_prev_weights);
+    let args = BackpropInputs::parse();
+    backprop_harness(args);
+}
+
+#[test]
+fn backprop_test() {
+    backprop_harness(BackpropInputs {
+        layer_size: 65536,
+    });
 }

juno_samples/rodinia/backprop/src/rust_backprop.rs

+fn layer_forward(n: usize, m: usize, vals: &[f32], weights: &[f32]) -> Vec<f32> {
+    let mut result = vec![0.0; m + 1];
+    result[0] = 1.0;
+    
+    for j in 1..=m {
+        let mut sum = 0.0;
+        for k in 0..=n {
+            sum += weights[k * (m + 1) + j] * vals[k];
+        }
+        result[j] = 1.0 / (1.0 + (-sum).exp());
+    }
+
+    result
+}
+
+fn output_error(n: usize, target: &[f32], actual: &[f32]) -> Vec<f32> {
+    let mut result = vec![0.0; n + 1];
+    for j in 1..=n {
+        let o = actual[j];
+        let t = target[j];
+        result[j] = o * (1.0 - o) * (t - o);
+    }
+    result
+}
+
+fn hidden_error(n: usize, m: usize, delta: &[f32], weights: &[f32], actual: &[f32]) -> Vec<f32> {
+    let mut result = vec![0.0; n + 1];
+    for j in 1..=n {
+        let h = actual[j];
+        let mut sum = 0.0;
+        for k in 1..=m {
+            sum += delta[k] * weights[j * (m + 1) + k];
+        }
+        result[j] = h * (1.0 - h) * sum;
+    }
+    result
+}
+
+fn adjust_weights(
+    n: usize,
+    m: usize,
+    delta: &[f32],
+    vals: &[f32],
+    mut weights: Vec<f32>,
+    mut prev_weights: Vec<f32>,
+) -> (Vec<f32>, Vec<f32>) {
+    for j in 1..=m {
+        for k in 0..=n {
+            let new_dw = (0.3 * delta[j] * vals[k]) + (0.3 * weights[k * (m + 1) + j]);
+            weights[k * (m + 1) + j] += new_dw;
+            prev_weights[k * (m + 1) + j] = new_dw;
+        }
+    }
+
+    (weights, prev_weights)
+}
+
+pub fn backprop(
+    input_n: usize,
+    hidden_n: usize,
+    output_n: usize,
+    input_vals: &[f32],
+    input_weights: Vec<f32>,
+    hidden_weights: Vec<f32>,
+    target: &[f32],
+    input_prev_weights: Vec<f32>,
+    hidden_prev_weights: Vec<f32>,
+) -> (Vec<f32>, Vec<f32>, Vec<f32>, Vec<f32>) {
+    let hidden_vals = layer_forward(input_n, hidden_n, input_vals, &input_weights);
+    let output_vals = layer_forward(hidden_n, output_n, &hidden_vals, &hidden_weights);
+
+    let out_delta = output_error(output_n, target, &output_vals);
+    let hid_delta = hidden_error(hidden_n, output_n, &out_delta, &hidden_weights, &hidden_vals);
+
+    let (hidden_weights, hidden_prev_weights)
+        = adjust_weights(hidden_n, output_n, &out_delta, &hidden_vals, hidden_weights, hidden_prev_weights);
+    let (input_weights, input_prev_weights)
+        = adjust_weights(input_n, hidden_n, &hid_delta, &input_vals, input_weights, input_prev_weights);
+
+    (input_weights, hidden_weights, input_prev_weights, hidden_prev_weights)
+}