feat/solvers: reintroduce solvers for Layers

Author: hobofan

Cargo.toml

@@ -20,6 +20,7 @@ collenchyma-nn = { version = "0.3.2", default-features = false }
 
 log = "0.3.2"
 rand = "0.3.0"
+num = "0.1"
 
 clippy = { version = "0.0.41", optional = true }
 

examples/benchmarks.rs

@@ -157,7 +157,6 @@ fn bench_alexnet() {
         let func = || {
             let forward_time = timeit_loops!(1, {
                 {
-                    let loss = &mut 0f32;
                     let inp = SharedTensor::<f32>::new(backend.device(), &vec![128, 3, 224, 224]).unwrap();
 
                     let inp_lock = Arc::new(RwLock::new(inp));
@@ -239,7 +238,6 @@ fn bench_overfeat() {
         let func = || {
             let forward_time = timeit_loops!(1, {
                 {
-                    let loss = &mut 0f32;
                     let inp = SharedTensor::<f32>::new(backend.device(), &vec![128, 3, 231, 231]).unwrap();
 
                     let inp_lock = Arc::new(RwLock::new(inp));
@@ -324,7 +322,6 @@ fn bench_vgg_a() {
     cfg.add_layer(LayerConfig::new("conv8/relu", LayerType::ReLU));
     let pool5_layer_cfg = PoolingConfig { mode: PoolingMode::Max, filter_shape: vec![2], stride: vec![2], padding: vec![0] };
     cfg.add_layer(LayerConfig::new("pool5", pool5_layer_cfg));
-    let fc1_layer_cfg = LinearConfig { output_size: 4096 };
     cfg.add_layer(LayerConfig::new("fc1", LinearConfig { output_size: 4096 }));
     cfg.add_layer(LayerConfig::new("fc2", LinearConfig { output_size: 4096 }));
     cfg.add_layer(LayerConfig::new("fc3", LinearConfig { output_size: 1000 }));
@@ -337,7 +334,6 @@ fn bench_vgg_a() {
         let func = || {
             let forward_time = timeit_loops!(1, {
                 {
-                    let loss = &mut 0f32;
                     let inp = SharedTensor::<f32>::new(backend.device(), &vec![64, 3, 224, 224]).unwrap();
 
                     let inp_lock = Arc::new(RwLock::new(inp));

src/layer.rs

@@ -1,6 +1,6 @@
 //! Provides the generics and interfaces for the specific [Layers][layers].
 //! [layers]: ../layers/index.html
-use co::{IBackend, ITensorDesc, SharedTensor};
+use co::prelude::*;
 use layers::*;
 use weight::WeightConfig;
 use util::{ArcLock, native_backend, LayerOps};
@@ -610,13 +610,12 @@ impl<B: IBackend + LayerOps<f32> + 'static> Layer<B> {
     /// [3]: ../solver/enum.LRPolicy.html
     pub fn update_weights<SolverB: IBackend + ::util::SolverOps<f32>>(&mut self, backend: &SolverB) {
         let mut shared_a = ::util::native_scalar(-1f32);
-        let _ = shared_a.add_device(backend.device());
-        shared_a.sync(backend.device()).unwrap();
+        let _ = shared_a.add_device(IBackend::device(backend));
+        shared_a.sync(IBackend::device(backend)).unwrap();
         for (weight_gradient, weight_data) in self.learnable_weights_gradients().iter().zip(&mut self.learnable_weights_data()) {
-            weight_gradient.write().unwrap().sync(backend.device()).unwrap();
-            weight_data.write().unwrap().sync(backend.device()).unwrap();
+            weight_gradient.write().unwrap().sync(IBackend::device(backend)).unwrap();
+            weight_data.write().unwrap().sync(IBackend::device(backend)).unwrap();
             backend.axpy_plain(&shared_a, &weight_gradient.read().unwrap(), &mut weight_data.write().unwrap()).unwrap();
-            // weight_blob.write().unwrap().apply_diff(backend) // TODO: solver
         }
     }
 
@@ -690,6 +689,17 @@ impl<B: IBackend + LayerOps<f32> + 'static> Layer<B> {
         if let Some(gradients) = self.worker.learnable_weights_gradients() { gradients }
         else { self.weights_gradient.clone() }
     }
+
+    /// Returns the learning rate for all the learnable weights in the layer.
+    ///
+    /// If the layer is a container layer it will return all learning rates of the
+    /// layers inside it.
+    pub fn learnable_weights_lr(&self) -> Vec<Option<f32>> {
+        if let Some(lr) = self.worker.learnable_weights_lr() { lr }
+        // else { self.weights_lr.clone() }
+        else {
+            self.learnable_weights_data().iter().map(|_| Some(1f32)).collect::<Vec<_>>() }
+    }
 }
 
 /// A Layer in a [Neural Network][1] that can handle forward and backward of a computation step.
@@ -1024,6 +1034,14 @@ pub trait ILayer<B: IBackend> : ComputeOutput<f32, B> + ComputeInputGradient<f32
     fn learnable_weights_gradients(&self) -> Option<Vec<ArcLock<SharedTensor<f32>>>> {
         None
     }
+
+    /// Return the learning rates for the learnable weights inside the layer.
+    ///
+    /// This should only be overridden by container layers,
+    /// where the weights are not easily exposable.
+    fn learnable_weights_lr(&self) -> Option<Vec<Option<f32>>> {
+        None
+    }
 }
 
 /// A Layer that can compute the output for a given input.
@@ -1061,7 +1079,7 @@ pub trait ComputeParametersGradient<T, B: IBackend> {
 
 impl<B: IBackend> fmt::Debug for ILayer<B> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "({}, {})", "foo", "bar")
+        write!(f, "({})", "ILayer")
     }
 }
 

src/lib.rs

@@ -124,15 +124,16 @@ extern crate timeit;
 #[macro_use]
 extern crate log;
 extern crate rand;
+extern crate num;
 extern crate collenchyma as co;
 extern crate collenchyma_blas as coblas;
 extern crate collenchyma_nn as conn;
 pub mod layer;
 pub mod layers;
-// #[cfg(feature="cuda")]
-// pub mod solver;
-// #[cfg(feature="cuda")]
-// pub mod solvers;
+#[cfg(feature="cuda")]
+pub mod solver;
+#[cfg(feature="cuda")]
+pub mod solvers;
 pub mod weight;
 
 pub mod util;