option for velocity sensitive notes!

Author: carsonswope

installer/build_all.ps1

@@ -26,5 +26,5 @@ $fatbin_dir = 'cuda_fatbin'
   /Dmodel_cfg=$model_cfg `
   /Dfatbin_dir=$fatbin_dir `
   /DAPP_NAME=3d-beats `
-  /DAPP_VERSION=2.1 `
+  /DAPP_VERSION=2.2 `
   /Oinstaller

readme.md

@@ -92,8 +92,6 @@ Now try the full 3d-beats app.
 
 # TODOs:
 
-- velocity:
-  - option to vary midi notes' velocity with velocity of tap
 - ignore fingertips that are not featured prominently / or have high variance from mean shift
 - handle when mean shift has identified a pixel which is not part of the depth image (0 depth!)
 - auto-tuning of plane / z threshold. should be able to determine best z threshold automatically

src/3d_bz.py

@@ -58,7 +58,10 @@ def __init__(self):
 
         self.gauss_sigma = 2.5
         self.z_thresh_offset = 25.
-        self.min_velocity = 5.
+        self.min_velocity = 10.
+
+        self.velocity_sensitive = True
+        self.max_velocity = 120. # anything w/ higher velocity is clipped at this point
 
         self.group_min_size = 0.06
 
@@ -313,22 +316,34 @@ def tick(self, _):
         # right hand on right
         self.hand_states[0].draw_imgui(self.z_thresh_offset, self.dpi_scale, (pos_start_x + (self.width * self.dpi_scale / 2), pos_start_y))
 
-        # per finger calibration
         imgui.new_line()
+        pos_start_x, pos_start_y = imgui.get_cursor_pos()
+
+        imgui.push_item_width(200)
+        _, self.min_velocity = imgui.slider_float('min velocity', self.min_velocity, 0., 50.)
+        _, self.velocity_sensitive = imgui.checkbox('velocity sensitive', self.velocity_sensitive)
+        if self.velocity_sensitive:
+            _, self.max_velocity = imgui.slider_float('max velocity', self.max_velocity, 50., 200.)
+        imgui.pop_item_width()
+
+        imgui.set_cursor_pos((pos_start_x + 400, pos_start_y))
+
+        # per finger calibration
         if imgui.button('reset fingers'):
             for h in self.hand_states:
                 for f, t in zip(h.fingertips, self.DEFAULT_FINGERTIP_THRESHOLDS):
                     f.z_thresh = t
 
-        imgui.same_line()
+        imgui.set_cursor_pos((pos_start_x + 400, imgui.get_cursor_pos()[1]))
         self.calibrate_next_frame = imgui.button('recalibrate plane')
 
-        imgui.new_line()
+        imgui.set_cursor_pos((pos_start_x + 400, imgui.get_cursor_pos()[1]))
+        imgui.push_item_width(200)
         self.midi.draw_imgui()
+        imgui.pop_item_width()
 
         imgui.end()
 
-
         imgui.set_next_window_position(0, (self.DIM_Y + 220) * self.dpi_scale)
         imgui.set_next_window_size(400 * self.dpi_scale, 124 * self.dpi_scale)
         imgui.set_next_window_bg_alpha(0.3)
@@ -337,7 +352,6 @@ def tick(self, _):
         imgui.push_item_width(150. * self.dpi_scale)
         _, self.PLANE_Z_OUTLIER_THRESHOLD = imgui.slider_float('plane threshold', self.PLANE_Z_OUTLIER_THRESHOLD, 0., 100.)
         _, self.z_thresh_offset = imgui.slider_float('finger threshold offset', self.z_thresh_offset, 0., 100.)
-        _, self.min_velocity = imgui.slider_float('min velocity', self.min_velocity, 0., 100.)
         imgui.pop_item_width()
 
 
@@ -491,6 +505,10 @@ def run_per_hand_pipeline(self, g_id, flip_x):
 
         for i, f_idx in zip(range(len(self.fingertip_idxes)), self.fingertip_idxes):
 
+            hand_state.fingertips[i].velocity_sensitive = self.velocity_sensitive
+            hand_state.fingertips[i].min_velocity = self.min_velocity
+            hand_state.fingertips[i].max_velocity = self.max_velocity
+
             px, py = label_means[f_idx-1].astype(np.int32)
             px *= self.LABELS_REDUCE
             py *= self.LABELS_REDUCE
@@ -504,7 +522,7 @@ def run_per_hand_pipeline(self, g_id, flip_x):
                 pt.append(1.)
                 pt = self.calibrated_plane.plane @ pt
                 pt_z = -pt[2]
-                hand_state.fingertips[i].next_z_pos(pt_z, self.z_thresh_offset, self.min_velocity)
+                hand_state.fingertips[i].next_z_pos(pt_z, self.z_thresh_offset)
 
 if __name__ == '__main__':
     run_app(App_3d_bz)

src/hand_state.py

@@ -15,15 +15,21 @@ def __init__(self, on_fn, off_fn, num_positions = 40, z_thresh = 150, midi_note
         self.midi_note = midi_note
         self.note_on = False
 
-        self.calibrate_alpha = 0.05
+        self.calibrate_alpha = 0.1
+
+        self.min_velocity = 15.
+        self.velocity_sensitive = True
+        self.max_velocity = 150.
+
+        self.min_midi_velocity = 0.4 # out of 1
 
     def reset_positions(self):
         # turn off note if on
         self.positions = [0 for _ in range(self.num_positions)]
         self.set_midi_state(False)
         pass
 
-    def next_z_pos(self, z_pos, z_thresh_offset, min_velocity):
+    def next_z_pos(self, z_pos, z_thresh_offset):
 
         self.positions.append(z_pos)
         while len(self.positions) > self.num_positions:
@@ -33,18 +39,26 @@ def next_z_pos(self, z_pos, z_thresh_offset, min_velocity):
             # z_pos = self.positions[-1]
             # must be located below 'on surface' threshold, and have downward velocity above threshold to be a note
             if z_pos < (self.z_thresh + z_thresh_offset):
-                if np.all(-np.diff(self.positions)[-2:] > min_velocity):
-                    self.set_midi_state(True)
+                last_2_velocities = -np.diff(self.positions)[-2:]
+                if np.all(last_2_velocities > self.min_velocity):
+                    if self.velocity_sensitive:
+                        v = (np.sum(last_2_velocities) / 2) / (self.max_velocity - self.min_velocity)
+                        v = self.min_midi_velocity + (v * (1 - self.min_midi_velocity))
+                        if v > 1:
+                            v = 1
+                    else:
+                        v = 1.
+                    self.set_midi_state(True, v)
             else:
-                self.set_midi_state(False)
+                self.set_midi_state(False, 0)
 
         if self.note_on:
             self.on_positions.append(z_pos)
 
-    def set_midi_state(self, s):
+    def set_midi_state(self, s, vel):
         if s and not self.note_on:
             self.note_on = True
-            self.on_fn(self.midi_note, 127) # todo: velocity!
+            self.on_fn(self.midi_note, int(vel * 127))
             self.on_positions.clear()
 
         elif not s and self.note_on: