Audio replicator with noise reduction 16khz

README.md

@@ -36,6 +36,21 @@ Additionally:
 - The OMX ALSA library reads audio in 16-bit 16000Hz stereo, one channel is just empty. To reduce streaming bandwidth the TinyAlsa replacement library converts the stereo data to mono.
 - To reduce streaming bandwidth even further, the 16-bit PCM data is converted to 8-bit uLaw and finally results in 128kbit/s.
 
+### RTSP audio support:
+The datapath of the audio is as follows:
+Mic -> ADC -> Kernel sound driver -> TinyAlsa lib -> OMX ALSA plugin -> Camera application (rmm)
+
+To maintain audio support for the original Yi application, the audio should be cloned at one of the steps with the following in mind:
+- Kernel driver can be used by 1 "sink"
+- TinyAlsa can only openend by one "sink"
+- OMX libs are closed source and are not compatible with the "available" I1 SDK.
+
+Audio support is implemented by replacing the original TinyAlsa library with a version that copies the read audio frames to a pipe. This pipe is read by the RTSP server. The RTSP server uses a patched WAVFileSource to read the audio data from the pipe. (Since it tries to read the WAV header 2x I saw no (quick) other way than to hardcode the PCM format into the WAVFileSource code.)
+
+Additionally:
+- The OMX ALSA library reads audio in 16-bit 16000Hz stereo, one channel is just empty. To reduce streaming bandwidth the TinyAlsa replacement library converts the stereo data to mono.
+- To reduce streaming bandwidth even further, the 16-bit PCM data is converted to 8-bit uLaw and finally results in 128kbit/s.
+
 ## Table of Contents
 
 - [Features](#features)

src/mkfifo/Makefile

@@ -0,0 +1,10 @@
+CC=/opt/yi/arm-linux-gnueabihf-4.8.3-201404/bin/arm-linux-gnueabihf-gcc
+
+all: fifo install
+
+fifo:
+	$(CC) mkfifo.c -o mkfifo
+
+install:
+	mkdir -p _install/bin
+	cp mkfifo _install/bin

src/mkfifo/mkfifo.c

@@ -0,0 +1,61 @@
+/* vi: set sw=4 ts=4: */
+/*
+ * Mini mkfifo implementation for busybox
+ *
+ * Copyright (C) 1999 by Randolph Chung <tausq@debian.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ *
+ */
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <errno.h>
+#include <stdlib.h>
+//#include "busybox.h"
+
+
+void show_usage(void)
+{
+	printf("mkfifo fifo_name\n");
+}
+
+int main(int argc, char **argv)
+{
+	char *thisarg;
+	mode_t mode = 0755;
+
+	argc--;
+	argv++;
+
+	/* Parse any options */
+	while (argc > 1) {
+		if (**argv != '-')
+			show_usage();
+		thisarg = *argv;
+		thisarg++;
+		switch (*thisarg) {
+		default:
+			show_usage();
+		}
+		argc--;
+		argv++;
+	}
+	if (argc < 1 || *argv[0] == '-')
+		show_usage();
+	if (mkfifo(*argv, mode) < 0)
+		printf("mkfifo failed\n");
+	return EXIT_SUCCESS;
+}

src/rRTSPServer_audio/Makefile.rRTSPServer_audio

@@ -70,7 +70,8 @@ livemedia:
 	$(CPLUSPLUS_COMPILER) -c $(CPLUSPLUS_FLAGS) -o $@ $<
 
 rRTSPServer_OBJS	= src/rRTSPServer.$(OBJ) \
-				src/WAVAudioFifoSource.$(OBJ) src/WAVAudioFifoServerMediaSubsession.$(OBJ)
+				src/WAVAudioFifoSource.$(OBJ) src/WAVAudioFifoServerMediaSubsession.$(OBJ) \
+				src/YiNoiseReduction.$(OBJ) src/DummySink.$(OBJ)
 
 rRTSPServer$(EXE):	$(rRTSPServer_OBJS) $(LOCAL_LIBS)
 	$(LINK)$@ $(CONSOLE_LINK_OPTS) $(rRTSPServer_OBJS) $(LIBS) -lpthread

src/rRTSPServer_audio/compile.rRTSPServer_audio

@@ -14,10 +14,13 @@ export CC=${CROSSPREFIX}gcc
 export LD=${CROSSPREFIX}ld
 export AS=${CROSSPREFIX}as
 export AR=${CROSSPREFIX}ar
+export LDFLAGS="-L`pwd`/dummylib -lOMX_AVQE_A"
 
 SCRIPT_DIR=$(cd `dirname $0` && pwd)
 cd $SCRIPT_DIR
 
+make -C dummylib
+
 cd live || exit 1
 
 make clean

src/rRTSPServer_audio/dummylib/.gitignore

@@ -0,0 +1,2 @@
+libOMX_AVQE_A.so
+

src/rRTSPServer_audio/dummylib/Makefile

@@ -0,0 +1,5 @@
+all:
+	$(CC) -fPIC -shared -o libOMX_AVQE_A.so YiAudioLibFuncs.c
+
+clean:
+	rm -rf libOMX_AVQE_A.so

src/rRTSPServer_audio/dummylib/README.md

@@ -0,0 +1,3 @@
+This directory contains dummy libraries with empty implementations of
+noise reduction required functions. This is done to prevent adding
+Yi firmware components to the repositories.

src/rRTSPServer_audio/dummylib/YiAudioLibFuncs.c

@@ -0,0 +1,41 @@
+#include "YiAudioLibFuncs.h"
+
+unsigned int IaaApc_GetBufferSize(void)
+{
+  return 0;
+}
+
+int IaaApc_Init(char* WorkBuffer, ApcStruct* apc_struct)
+{
+  return 0;
+}
+
+int IaaApc_Run(short* SampleBuffer)
+{
+  (void)SampleBuffer;
+  return 0;
+}
+
+void IaaApc_Free(void)
+{
+  int a = 1 + 1;
+}
+
+int IaaApc_SetNrEnable(int on)
+{
+  (void)on;
+  return 0;
+}
+
+int IaaApc_SetNrSmoothLevel(unsigned int lvl)
+{
+  (void) lvl;
+  return 0;
+}
+
+int IaaApc_SetNrMode(int lvl)
+{
+  (void) lvl;
+  return 0;
+}
+

src/rRTSPServer_audio/dummylib/YiAudioLibFuncs.h

@@ -0,0 +1,26 @@
+#ifndef YI_AUDIO_LIB_FUNCS_H_
+#define YI_AUDIO_LIB_FUNCS_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+	unsigned int point_number;
+	unsigned int channels;
+	unsigned int rate;
+} ApcStruct;
+
+unsigned int IaaApc_GetBufferSize(void);
+int IaaApc_Init(char* WorkBuffer, ApcStruct* apc_struct);
+int IaaApc_Run(short* SampleBuffer);
+void IaaApc_Free(void);
+int IaaApc_SetNrEnable(int on);
+int IaaApc_SetNrSmoothLevel(unsigned int lvl);
+int IaaApc_SetNrMode(int lvl);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/rRTSPServer_audio/include/DummySink.hh

@@ -0,0 +1,29 @@
+#include "liveMedia.hh"
+#include "BasicUsageEnvironment.hh"
+
+class DummySink: public MediaSink {
+public:
+  static DummySink* createNew(UsageEnvironment& env,
+			      char const* streamId = NULL); // identifies the stream itself (optional)
+
+private:
+  DummySink(UsageEnvironment& env, char const* streamId);
+    // called only by "createNew()"
+  virtual ~DummySink();
+
+  static void afterGettingFrame(void* clientData, unsigned frameSize,
+                                unsigned numTruncatedBytes,
+				struct timeval presentationTime,
+                                unsigned durationInMicroseconds);
+  void afterGettingFrame(unsigned frameSize, unsigned numTruncatedBytes,
+			 struct timeval presentationTime, unsigned durationInMicroseconds);
+
+private:
+  // redefined virtual functions:
+  virtual Boolean continuePlaying();
+
+private:
+  u_int8_t* fReceiveBuffer;
+  char* fStreamId;
+};
+

src/rRTSPServer_audio/include/WAVAudioFifoServerMediaSubsession.hh

@@ -25,17 +25,15 @@ along with this library; if not, write to the Free Software Foundation, Inc.,
 #ifndef _FILE_SERVER_MEDIA_SUBSESSION_HH
 #include "FileServerMediaSubsession.hh"
 #endif
+#include "StreamReplicator.hh"
 
-class WAVAudioFifoServerMediaSubsession: public FileServerMediaSubsession{
+class WAVAudioFifoServerMediaSubsession: public OnDemandServerMediaSubsession {
 public:
   static WAVAudioFifoServerMediaSubsession*
-  createNew(UsageEnvironment& env, char const* fileName, Boolean reuseFirstSource,
-	    Boolean convertToULaw = False);
-      // If "convertToULaw" is True, 16-bit audio streams are converted to
-      // 8-bit u-law audio prior to streaming.
+  createNew(UsageEnvironment& env, StreamReplicator* replicator, Boolean reuseFirstSource, Boolean convertToULaw);
 
 protected:
-  WAVAudioFifoServerMediaSubsession(UsageEnvironment& env, char const* fileName,
+  WAVAudioFifoServerMediaSubsession(UsageEnvironment& env, StreamReplicator* replicator,
 				    Boolean reuseFirstSource, Boolean convertToULaw);
       // called only by createNew();
   virtual ~WAVAudioFifoServerMediaSubsession();
@@ -54,7 +52,6 @@ protected: // redefined virtual functions
   virtual float duration() const;
 
 protected:
-  Boolean fConvertToULaw;
 
   // The following parameters of the input stream are set after
   // "createNewStreamSource" is called:
@@ -63,6 +60,8 @@ protected:
   unsigned fSamplingFrequency;
   unsigned fNumChannels;
   float fFileDuration;
+  StreamReplicator* fReplicator;
+  Boolean fConvertToULaw;
 };
 
 #endif

src/rRTSPServer_audio/include/WAVAudioFifoSource.hh

@@ -27,14 +27,11 @@ along with this library; if not, write to the Free Software Foundation, Inc.,
 #include "AudioInputDevice.hh"
 #endif
 
-typedef enum {
-  WA_PCM = 0x01,
-  WA_PCMA = 0x06,
-  WA_PCMU = 0x07,
-  WA_IMA_ADPCM = 0x11,
-  WA_UNKNOWN
-} WAV_AUDIO_FORMAT;
-
+#define  WA_PCM 0x01
+#define  WA_PCMA 0x06
+#define  WA_PCMU 0x07
+#define  WA_IMA_ADPCM 0x11
+#define  WA_UNKNOWN 0x12
 
 class WAVAudioFifoSource: public AudioInputDevice {
 public:

src/rRTSPServer_audio/include/YiNoiseReduction.hh

@@ -0,0 +1,60 @@
+/**********
+This library is free software; you can redistribute it and/or modify it under
+the terms of the GNU Lesser General Public License as published by the
+Free Software Foundation; either version 3 of the License, or (at your
+option) any later version. (See <http://www.gnu.org/copyleft/lesser.html>.)
+
+This library is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for
+more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this library; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+**********/
+//
+// Source template used to create this filter:
+// Copyright (c) 1996-2020 Live Networks, Inc.  All rights reserved.
+// Filters for converting between raw PCM audio and uLaw
+// C++ header
+
+#ifndef _AUDIO_FILTER_HH
+#define _AUDIO_FILTER_HH
+
+#ifndef _FRAMED_FILTER_HH
+#include "FramedFilter.hh"
+#endif
+
+#include "../../../dummylib/YiAudioLibFuncs.h"
+
+#define ALSA_BUF_SIZE 512
+
+class YiNoiseReduction: public FramedFilter {
+public:
+  static YiNoiseReduction* createNew(UsageEnvironment& env, FramedSource* inputSource, unsigned int level);
+
+protected:
+  YiNoiseReduction(UsageEnvironment& env, FramedSource* inputSource, unsigned int level);
+  // called only by createNew()
+  virtual ~YiNoiseReduction();
+
+private:
+  // Redefined virtual functions:
+  virtual void doGetNextFrame();
+
+private:
+  ApcStruct apStruct;
+  void* apBuf;
+  int apHandle;
+  static void afterGettingFrame(void* clientData, unsigned frameSize,
+				unsigned numTruncatedBytes,
+				struct timeval presentationTime,
+				unsigned durationInMicroseconds);
+  void afterGettingFrame1(unsigned frameSize,
+			  unsigned numTruncatedBytes,
+			  struct timeval presentationTime,
+			  unsigned durationInMicroseconds);
+};
+
+#endif

src/rRTSPServer_audio/scripts/h264_grabber_h.sh

@@ -0,0 +1,3 @@
+#!/bin/ash
+
+while true; do h264grabber -r high > /tmp/h264_high_fifo ; done

src/rRTSPServer_audio/scripts/h264_grabber_l.sh

@@ -0,0 +1,3 @@
+#!/bin/ash
+
+while true; do h264grabber -r low > /tmp/h264_low_fifo ; done

src/rRTSPServer_audio/scripts/rtsp_restart.sh

@@ -0,0 +1,34 @@
+/home/yi-hack/script # cat rtsp_restart.sh
+#!/bin/ash
+
+echo "To run this in the background, do this:"
+echo "cd /tmp"
+echo "nohup 2>&1 rtsp_restart.sh &"
+
+echo "Killing all RTSP related processes"
+killall wd_rtsp.sh
+killall h264grabber_l
+killall h264grabber_h
+killall rRTSPServer_l
+killall rRTSPServer_h
+killall rRTSPServer_audio_h
+killall rRTSPServer_audio_l
+killall h264_grabber_h.sh
+killall h264_grabber_l.sh
+
+
+echo "Creating H264 fifos"
+rm -rf /tmp/h264_low_fifo /tmp/h264_high_fifo
+./mkfifo /tmp/h264_low_fifo
+./mkfifo /tmp/h264_high_fifo
+chmod 755 /tmp/h264_low_fifo
+chmod 755 /tmp/h264_high_fifo
+
+
+echo "Starting H264 grabbers"
+./h264_grabber_h.sh &
+./h264_grabber_l.sh &
+
+echo "Starting RTSP server"
+NR_LEVEL=25 ./rRTSPServer_audio
+