diff --git a/README.md b/README.md index 4c23ca8..dfbf670 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,21 @@ Additionally: - The OMX ALSA library reads audio in 16-bit 16000Hz stereo, one channel is just empty. To reduce streaming bandwidth the TinyAlsa replacement library converts the stereo data to mono. - To reduce streaming bandwidth even further, the 16-bit PCM data is converted to 8-bit uLaw and finally results in 128kbit/s. +### RTSP audio support: +The datapath of the audio is as follows: +Mic -> ADC -> Kernel sound driver -> TinyAlsa lib -> OMX ALSA plugin -> Camera application (rmm) + +To maintain audio support for the original Yi application, the audio should be cloned at one of the steps with the following in mind: +- Kernel driver can be used by 1 "sink" +- TinyAlsa can only openend by one "sink" +- OMX libs are closed source and are not compatible with the "available" I1 SDK. + +Audio support is implemented by replacing the original TinyAlsa library with a version that copies the read audio frames to a pipe. This pipe is read by the RTSP server. The RTSP server uses a patched WAVFileSource to read the audio data from the pipe. (Since it tries to read the WAV header 2x I saw no (quick) other way than to hardcode the PCM format into the WAVFileSource code.) + +Additionally: +- The OMX ALSA library reads audio in 16-bit 16000Hz stereo, one channel is just empty. To reduce streaming bandwidth the TinyAlsa replacement library converts the stereo data to mono. +- To reduce streaming bandwidth even further, the 16-bit PCM data is converted to 8-bit uLaw and finally results in 128kbit/s. + ## Table of Contents - [Features](#features) diff --git a/src/mkfifo/Makefile b/src/mkfifo/Makefile new file mode 100644 index 0000000..eb781f4 --- /dev/null +++ b/src/mkfifo/Makefile @@ -0,0 +1,10 @@ +CC=/opt/yi/arm-linux-gnueabihf-4.8.3-201404/bin/arm-linux-gnueabihf-gcc + +all: fifo install + +fifo: + $(CC) mkfifo.c -o mkfifo + +install: + mkdir -p _install/bin + cp mkfifo _install/bin diff --git a/src/mkfifo/mkfifo.c b/src/mkfifo/mkfifo.c new file mode 100644 index 0000000..1258462 --- /dev/null +++ b/src/mkfifo/mkfifo.c @@ -0,0 +1,61 @@ +/* vi: set sw=4 ts=4: */ +/* + * Mini mkfifo implementation for busybox + * + * Copyright (C) 1999 by Randolph Chung + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include +#include +#include +//#include "busybox.h" + + +void show_usage(void) +{ + printf("mkfifo fifo_name\n"); +} + +int main(int argc, char **argv) +{ + char *thisarg; + mode_t mode = 0755; + + argc--; + argv++; + + /* Parse any options */ + while (argc > 1) { + if (**argv != '-') + show_usage(); + thisarg = *argv; + thisarg++; + switch (*thisarg) { + default: + show_usage(); + } + argc--; + argv++; + } + if (argc < 1 || *argv[0] == '-') + show_usage(); + if (mkfifo(*argv, mode) < 0) + printf("mkfifo failed\n"); + return EXIT_SUCCESS; +} diff --git a/src/rRTSPServer_audio/Makefile.rRTSPServer_audio b/src/rRTSPServer_audio/Makefile.rRTSPServer_audio index 024212b..972c064 100644 --- a/src/rRTSPServer_audio/Makefile.rRTSPServer_audio +++ b/src/rRTSPServer_audio/Makefile.rRTSPServer_audio @@ -70,7 +70,8 @@ livemedia: $(CPLUSPLUS_COMPILER) -c $(CPLUSPLUS_FLAGS) -o $@ $< rRTSPServer_OBJS = src/rRTSPServer.$(OBJ) \ - src/WAVAudioFifoSource.$(OBJ) src/WAVAudioFifoServerMediaSubsession.$(OBJ) + src/WAVAudioFifoSource.$(OBJ) src/WAVAudioFifoServerMediaSubsession.$(OBJ) \ + src/YiNoiseReduction.$(OBJ) src/DummySink.$(OBJ) rRTSPServer$(EXE): $(rRTSPServer_OBJS) $(LOCAL_LIBS) $(LINK)$@ $(CONSOLE_LINK_OPTS) $(rRTSPServer_OBJS) $(LIBS) -lpthread diff --git a/src/rRTSPServer_audio/compile.rRTSPServer_audio b/src/rRTSPServer_audio/compile.rRTSPServer_audio index 398c945..7598c3e 100755 --- a/src/rRTSPServer_audio/compile.rRTSPServer_audio +++ b/src/rRTSPServer_audio/compile.rRTSPServer_audio @@ -14,10 +14,13 @@ export CC=${CROSSPREFIX}gcc export LD=${CROSSPREFIX}ld export AS=${CROSSPREFIX}as export AR=${CROSSPREFIX}ar +export LDFLAGS="-L`pwd`/dummylib -lOMX_AVQE_A" SCRIPT_DIR=$(cd `dirname $0` && pwd) cd $SCRIPT_DIR +make -C dummylib + cd live || exit 1 make clean diff --git a/src/rRTSPServer_audio/dummylib/.gitignore b/src/rRTSPServer_audio/dummylib/.gitignore new file mode 100644 index 0000000..c86322a --- /dev/null +++ b/src/rRTSPServer_audio/dummylib/.gitignore @@ -0,0 +1,2 @@ +libOMX_AVQE_A.so + diff --git a/src/rRTSPServer_audio/dummylib/Makefile b/src/rRTSPServer_audio/dummylib/Makefile new file mode 100644 index 0000000..ddb09d9 --- /dev/null +++ b/src/rRTSPServer_audio/dummylib/Makefile @@ -0,0 +1,5 @@ +all: + $(CC) -fPIC -shared -o libOMX_AVQE_A.so YiAudioLibFuncs.c + +clean: + rm -rf libOMX_AVQE_A.so diff --git a/src/rRTSPServer_audio/dummylib/README.md b/src/rRTSPServer_audio/dummylib/README.md new file mode 100644 index 0000000..a594af5 --- /dev/null +++ b/src/rRTSPServer_audio/dummylib/README.md @@ -0,0 +1,3 @@ +This directory contains dummy libraries with empty implementations of +noise reduction required functions. This is done to prevent adding +Yi firmware components to the repositories. diff --git a/src/rRTSPServer_audio/dummylib/YiAudioLibFuncs.c b/src/rRTSPServer_audio/dummylib/YiAudioLibFuncs.c new file mode 100644 index 0000000..2763915 --- /dev/null +++ b/src/rRTSPServer_audio/dummylib/YiAudioLibFuncs.c @@ -0,0 +1,41 @@ +#include "YiAudioLibFuncs.h" + +unsigned int IaaApc_GetBufferSize(void) +{ + return 0; +} + +int IaaApc_Init(char* WorkBuffer, ApcStruct* apc_struct) +{ + return 0; +} + +int IaaApc_Run(short* SampleBuffer) +{ + (void)SampleBuffer; + return 0; +} + +void IaaApc_Free(void) +{ + int a = 1 + 1; +} + +int IaaApc_SetNrEnable(int on) +{ + (void)on; + return 0; +} + +int IaaApc_SetNrSmoothLevel(unsigned int lvl) +{ + (void) lvl; + return 0; +} + +int IaaApc_SetNrMode(int lvl) +{ + (void) lvl; + return 0; +} + diff --git a/src/rRTSPServer_audio/dummylib/YiAudioLibFuncs.h b/src/rRTSPServer_audio/dummylib/YiAudioLibFuncs.h new file mode 100644 index 0000000..cee2d8e --- /dev/null +++ b/src/rRTSPServer_audio/dummylib/YiAudioLibFuncs.h @@ -0,0 +1,26 @@ +#ifndef YI_AUDIO_LIB_FUNCS_H_ +#define YI_AUDIO_LIB_FUNCS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + unsigned int point_number; + unsigned int channels; + unsigned int rate; +} ApcStruct; + +unsigned int IaaApc_GetBufferSize(void); +int IaaApc_Init(char* WorkBuffer, ApcStruct* apc_struct); +int IaaApc_Run(short* SampleBuffer); +void IaaApc_Free(void); +int IaaApc_SetNrEnable(int on); +int IaaApc_SetNrSmoothLevel(unsigned int lvl); +int IaaApc_SetNrMode(int lvl); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/rRTSPServer_audio/include/DummySink.hh b/src/rRTSPServer_audio/include/DummySink.hh new file mode 100644 index 0000000..9ee4de7 --- /dev/null +++ b/src/rRTSPServer_audio/include/DummySink.hh @@ -0,0 +1,29 @@ +#include "liveMedia.hh" +#include "BasicUsageEnvironment.hh" + +class DummySink: public MediaSink { +public: + static DummySink* createNew(UsageEnvironment& env, + char const* streamId = NULL); // identifies the stream itself (optional) + +private: + DummySink(UsageEnvironment& env, char const* streamId); + // called only by "createNew()" + virtual ~DummySink(); + + static void afterGettingFrame(void* clientData, unsigned frameSize, + unsigned numTruncatedBytes, + struct timeval presentationTime, + unsigned durationInMicroseconds); + void afterGettingFrame(unsigned frameSize, unsigned numTruncatedBytes, + struct timeval presentationTime, unsigned durationInMicroseconds); + +private: + // redefined virtual functions: + virtual Boolean continuePlaying(); + +private: + u_int8_t* fReceiveBuffer; + char* fStreamId; +}; + diff --git a/src/rRTSPServer_audio/include/WAVAudioFifoServerMediaSubsession.hh b/src/rRTSPServer_audio/include/WAVAudioFifoServerMediaSubsession.hh index f561b47..6ad9438 100644 --- a/src/rRTSPServer_audio/include/WAVAudioFifoServerMediaSubsession.hh +++ b/src/rRTSPServer_audio/include/WAVAudioFifoServerMediaSubsession.hh @@ -25,17 +25,15 @@ along with this library; if not, write to the Free Software Foundation, Inc., #ifndef _FILE_SERVER_MEDIA_SUBSESSION_HH #include "FileServerMediaSubsession.hh" #endif +#include "StreamReplicator.hh" -class WAVAudioFifoServerMediaSubsession: public FileServerMediaSubsession{ +class WAVAudioFifoServerMediaSubsession: public OnDemandServerMediaSubsession { public: static WAVAudioFifoServerMediaSubsession* - createNew(UsageEnvironment& env, char const* fileName, Boolean reuseFirstSource, - Boolean convertToULaw = False); - // If "convertToULaw" is True, 16-bit audio streams are converted to - // 8-bit u-law audio prior to streaming. + createNew(UsageEnvironment& env, StreamReplicator* replicator, Boolean reuseFirstSource, Boolean convertToULaw); protected: - WAVAudioFifoServerMediaSubsession(UsageEnvironment& env, char const* fileName, + WAVAudioFifoServerMediaSubsession(UsageEnvironment& env, StreamReplicator* replicator, Boolean reuseFirstSource, Boolean convertToULaw); // called only by createNew(); virtual ~WAVAudioFifoServerMediaSubsession(); @@ -54,7 +52,6 @@ protected: // redefined virtual functions virtual float duration() const; protected: - Boolean fConvertToULaw; // The following parameters of the input stream are set after // "createNewStreamSource" is called: @@ -63,6 +60,8 @@ protected: unsigned fSamplingFrequency; unsigned fNumChannels; float fFileDuration; + StreamReplicator* fReplicator; + Boolean fConvertToULaw; }; #endif diff --git a/src/rRTSPServer_audio/include/WAVAudioFifoSource.hh b/src/rRTSPServer_audio/include/WAVAudioFifoSource.hh index 3118b61..3bc9e92 100644 --- a/src/rRTSPServer_audio/include/WAVAudioFifoSource.hh +++ b/src/rRTSPServer_audio/include/WAVAudioFifoSource.hh @@ -27,14 +27,11 @@ along with this library; if not, write to the Free Software Foundation, Inc., #include "AudioInputDevice.hh" #endif -typedef enum { - WA_PCM = 0x01, - WA_PCMA = 0x06, - WA_PCMU = 0x07, - WA_IMA_ADPCM = 0x11, - WA_UNKNOWN -} WAV_AUDIO_FORMAT; - +#define WA_PCM 0x01 +#define WA_PCMA 0x06 +#define WA_PCMU 0x07 +#define WA_IMA_ADPCM 0x11 +#define WA_UNKNOWN 0x12 class WAVAudioFifoSource: public AudioInputDevice { public: diff --git a/src/rRTSPServer_audio/include/YiNoiseReduction.hh b/src/rRTSPServer_audio/include/YiNoiseReduction.hh new file mode 100644 index 0000000..d4399b9 --- /dev/null +++ b/src/rRTSPServer_audio/include/YiNoiseReduction.hh @@ -0,0 +1,60 @@ +/********** +This library is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License as published by the +Free Software Foundation; either version 3 of the License, or (at your +option) any later version. (See .) + +This library is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for +more details. + +You should have received a copy of the GNU Lesser General Public License +along with this library; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +**********/ +// +// Source template used to create this filter: +// Copyright (c) 1996-2020 Live Networks, Inc. All rights reserved. +// Filters for converting between raw PCM audio and uLaw +// C++ header + +#ifndef _AUDIO_FILTER_HH +#define _AUDIO_FILTER_HH + +#ifndef _FRAMED_FILTER_HH +#include "FramedFilter.hh" +#endif + +#include "../../../dummylib/YiAudioLibFuncs.h" + +#define ALSA_BUF_SIZE 512 + +class YiNoiseReduction: public FramedFilter { +public: + static YiNoiseReduction* createNew(UsageEnvironment& env, FramedSource* inputSource, unsigned int level); + +protected: + YiNoiseReduction(UsageEnvironment& env, FramedSource* inputSource, unsigned int level); + // called only by createNew() + virtual ~YiNoiseReduction(); + +private: + // Redefined virtual functions: + virtual void doGetNextFrame(); + +private: + ApcStruct apStruct; + void* apBuf; + int apHandle; + static void afterGettingFrame(void* clientData, unsigned frameSize, + unsigned numTruncatedBytes, + struct timeval presentationTime, + unsigned durationInMicroseconds); + void afterGettingFrame1(unsigned frameSize, + unsigned numTruncatedBytes, + struct timeval presentationTime, + unsigned durationInMicroseconds); +}; + +#endif diff --git a/src/rRTSPServer_audio/scripts/h264_grabber_h.sh b/src/rRTSPServer_audio/scripts/h264_grabber_h.sh new file mode 100755 index 0000000..ad97b2c --- /dev/null +++ b/src/rRTSPServer_audio/scripts/h264_grabber_h.sh @@ -0,0 +1,3 @@ +#!/bin/ash + +while true; do h264grabber -r high > /tmp/h264_high_fifo ; done diff --git a/src/rRTSPServer_audio/scripts/h264_grabber_l.sh b/src/rRTSPServer_audio/scripts/h264_grabber_l.sh new file mode 100755 index 0000000..320d87d --- /dev/null +++ b/src/rRTSPServer_audio/scripts/h264_grabber_l.sh @@ -0,0 +1,3 @@ +#!/bin/ash + +while true; do h264grabber -r low > /tmp/h264_low_fifo ; done diff --git a/src/rRTSPServer_audio/scripts/rtsp_restart.sh b/src/rRTSPServer_audio/scripts/rtsp_restart.sh new file mode 100755 index 0000000..cf22aa0 --- /dev/null +++ b/src/rRTSPServer_audio/scripts/rtsp_restart.sh @@ -0,0 +1,34 @@ +/home/yi-hack/script # cat rtsp_restart.sh +#!/bin/ash + +echo "To run this in the background, do this:" +echo "cd /tmp" +echo "nohup 2>&1 rtsp_restart.sh &" + +echo "Killing all RTSP related processes" +killall wd_rtsp.sh +killall h264grabber_l +killall h264grabber_h +killall rRTSPServer_l +killall rRTSPServer_h +killall rRTSPServer_audio_h +killall rRTSPServer_audio_l +killall h264_grabber_h.sh +killall h264_grabber_l.sh + + +echo "Creating H264 fifos" +rm -rf /tmp/h264_low_fifo /tmp/h264_high_fifo +./mkfifo /tmp/h264_low_fifo +./mkfifo /tmp/h264_high_fifo +chmod 755 /tmp/h264_low_fifo +chmod 755 /tmp/h264_high_fifo + + +echo "Starting H264 grabbers" +./h264_grabber_h.sh & +./h264_grabber_l.sh & + +echo "Starting RTSP server" +NR_LEVEL=25 ./rRTSPServer_audio + diff --git a/src/rRTSPServer_audio/src/DummySink.cpp b/src/rRTSPServer_audio/src/DummySink.cpp new file mode 100644 index 0000000..9adaee4 --- /dev/null +++ b/src/rRTSPServer_audio/src/DummySink.cpp @@ -0,0 +1,58 @@ +#include "liveMedia.hh" +#include "BasicUsageEnvironment.hh" +#include "DummySink.hh" + +// Implementation of "DummySink": + +// Even though we're not going to be doing anything with the incoming data, we still need to receive it. +// Define the size of the buffer that we'll use: +#define DUMMY_SINK_RECEIVE_BUFFER_SIZE 100000 + +DummySink* DummySink::createNew(UsageEnvironment& env, char const* streamId) { + return new DummySink(env, streamId); +} + +DummySink::DummySink(UsageEnvironment& env, char const* streamId) + : MediaSink(env) { + fStreamId = strDup(streamId); + fReceiveBuffer = new u_int8_t[DUMMY_SINK_RECEIVE_BUFFER_SIZE]; +} + +DummySink::~DummySink() { + delete[] fReceiveBuffer; + delete[] fStreamId; +} + +void DummySink::afterGettingFrame(void* clientData, unsigned frameSize, unsigned numTruncatedBytes, + struct timeval presentationTime, unsigned durationInMicroseconds) { + DummySink* sink = (DummySink*)clientData; + sink->afterGettingFrame(frameSize, numTruncatedBytes, presentationTime, durationInMicroseconds); +} + +// If you don't want to see debugging output for each received frame, then comment out the following line: +#define DEBUG_PRINT_EACH_RECEIVED_FRAME 1 + +void DummySink::afterGettingFrame(unsigned frameSize, unsigned numTruncatedBytes, + struct timeval presentationTime, unsigned /*durationInMicroseconds*/) { + // We've just received a frame of data. (Optionally) print out information about it: +#ifdef DEBUG + if ((int)presentationTime.tv_sec % 10 == 0) { + char uSecsStr[6+1]; // used to output the 'microseconds' part of the presentation time + sprintf(uSecsStr, "%06u", (unsigned)presentationTime.tv_usec); + envir() << "DummySink Presentation time: " << (int)presentationTime.tv_sec << "." << uSecsStr; + } +#endif + + // Then continue, to request the next frame of data: + continuePlaying(); +} + +Boolean DummySink::continuePlaying() { + if (fSource == NULL) return False; // sanity check (should not happen) + + // Request the next frame of data from our input source. "afterGettingFrame()" will get called later, when it arrives: + fSource->getNextFrame(fReceiveBuffer, DUMMY_SINK_RECEIVE_BUFFER_SIZE, + afterGettingFrame, this, + onSourceClosure, this); + return True; +} diff --git a/src/rRTSPServer_audio/src/WAVAudioFifoServerMediaSubsession.cpp b/src/rRTSPServer_audio/src/WAVAudioFifoServerMediaSubsession.cpp index 94ed6a8..4c2c762 100644 --- a/src/rRTSPServer_audio/src/WAVAudioFifoServerMediaSubsession.cpp +++ b/src/rRTSPServer_audio/src/WAVAudioFifoServerMediaSubsession.cpp @@ -23,18 +23,17 @@ along with this library; if not, write to the Free Software Foundation, Inc., #include "WAVAudioFifoSource.hh" #include "uLawAudioFilter.hh" #include "SimpleRTPSink.hh" +#include "YiNoiseReduction.hh" WAVAudioFifoServerMediaSubsession* WAVAudioFifoServerMediaSubsession -::createNew(UsageEnvironment& env, char const* fileName, Boolean reuseFirstSource, - Boolean convertToULaw) { - return new WAVAudioFifoServerMediaSubsession(env, fileName, - reuseFirstSource, convertToULaw); +::createNew(UsageEnvironment& env, StreamReplicator* replicator, Boolean reuseFirstSource, Boolean convertToULaw) { + return new WAVAudioFifoServerMediaSubsession(env, replicator, reuseFirstSource, convertToULaw); } WAVAudioFifoServerMediaSubsession -::WAVAudioFifoServerMediaSubsession(UsageEnvironment& env, char const* fileName, - Boolean reuseFirstSource, Boolean convertToULaw) - : FileServerMediaSubsession(env, fileName, reuseFirstSource), +::WAVAudioFifoServerMediaSubsession(UsageEnvironment& env, StreamReplicator* replicator, Boolean reuseFirstSource, Boolean convertToULaw) + : OnDemandServerMediaSubsession(env, reuseFirstSource), + fReplicator(replicator), fConvertToULaw(convertToULaw) { } @@ -97,51 +96,49 @@ ::setStreamSourceScale(FramedSource* inputSource, float scale) { FramedSource* WAVAudioFifoServerMediaSubsession ::createNewStreamSource(unsigned /*clientSessionId*/, unsigned& estBitrate) { FramedSource* resultSource = NULL; - do { - WAVAudioFifoSource* wavSource = WAVAudioFifoSource::createNew(envir(), fFileName); - if (wavSource == NULL) break; - - // Get attributes of the audio source: - - fAudioFormat = wavSource->getAudioFormat(); - fBitsPerSample = wavSource->bitsPerSample(); - // We handle only 4,8,16,20,24 bits-per-sample audio: - if (fBitsPerSample%4 != 0 || fBitsPerSample < 4 || fBitsPerSample > 24 || fBitsPerSample == 12) { - envir() << "The input file contains " << fBitsPerSample << " bit-per-sample audio, which we don't handle\n"; - break; + WAVAudioFifoSource* originalSource = NULL; + FramedFilter* previousSource = (FramedFilter*)fReplicator->inputSource(); + + // Iterate back into the filter chain until a source is found that + // has a sample frequency and expected to be a WAVAudioFifoSource. + for (int x = 0; x < 10; x++) { + if (((WAVAudioFifoSource*)(previousSource))->bitsPerSample() != 0) { +#ifdef DEBUG + printf("WAVAudioFifoSource found at x = %d\n", x); +#endif + originalSource = (WAVAudioFifoSource*)(previousSource); + break; } - fSamplingFrequency = wavSource->samplingFrequency(); - fNumChannels = wavSource->numChannels(); + previousSource = (FramedFilter*)previousSource->inputSource(); + } +#ifdef DEBUG + printf("fReplicator->inputSource() = %p\n", originalSource); +#endif + resultSource = fReplicator->createStreamReplica(); + if (resultSource == NULL) { + printf("Failed to create stream replica\n"); + Medium::close(resultSource); + return NULL; + } + else { + fAudioFormat = originalSource->getAudioFormat(); + fBitsPerSample = originalSource->bitsPerSample(); + fSamplingFrequency = originalSource->samplingFrequency(); + fConvertToULaw = True; + fNumChannels = originalSource->numChannels(); unsigned bitsPerSecond = fSamplingFrequency*fBitsPerSample*fNumChannels; +#ifdef DEBUG + printf("Original source FMT: %d bps: %d freq: %d\n", fAudioFormat, fBitsPerSample, fSamplingFrequency); +#endif + fFileDuration = ~0;//(float)((8.0*originalSource->numPCMBytes())/(fSamplingFrequency*fNumChannels*fBitsPerSample)); - fFileDuration = (float)((8.0*wavSource->numPCMBytes())/(fSamplingFrequency*fNumChannels*fBitsPerSample)); + estBitrate = (bitsPerSecond+500)/1000; // kbps - // Add in any filter necessary to transform the data prior to streaming: - resultSource = wavSource; // by default - if (fAudioFormat == WA_PCM) { - if (fBitsPerSample == 16) { - // Note that samples in the WAV audio file are in little-endian order. - if (fConvertToULaw) { - // Add a filter that converts from raw 16-bit PCM audio to 8-bit u-law audio: - resultSource = uLawFromPCMAudioSource::createNew(envir(), wavSource, 1/*little-endian*/); - bitsPerSecond /= 2; - } else { - // Add a filter that converts from little-endian to network (big-endian) order: - resultSource = EndianSwap16::createNew(envir(), wavSource); - } - } else if (fBitsPerSample == 20 || fBitsPerSample == 24) { - // Add a filter that converts from little-endian to network (big-endian) order: - resultSource = EndianSwap24::createNew(envir(), wavSource); - } - } + if (fConvertToULaw) + estBitrate /= 2; - estBitrate = (bitsPerSecond+500)/1000; // kbps return resultSource; - } while (0); - - // An error occurred: - Medium::close(resultSource); - return NULL; + } } RTPSink* WAVAudioFifoServerMediaSubsession @@ -200,7 +197,9 @@ ::createNewRTPSink(Groupsock* rtpGroupsock, } else { //unknown format break; } - +#ifdef DEBUG + printf("Create SimpleRTPSink: %s, freq: %d, channels %d\n", mimeType, fSamplingFrequency, fNumChannels); +#endif return SimpleRTPSink::createNew(envir(), rtpGroupsock, payloadFormatCode, fSamplingFrequency, "audio", mimeType, fNumChannels); diff --git a/src/rRTSPServer_audio/src/WAVAudioFifoSource.cpp b/src/rRTSPServer_audio/src/WAVAudioFifoSource.cpp index b345129..61fd46e 100644 --- a/src/rRTSPServer_audio/src/WAVAudioFifoSource.cpp +++ b/src/rRTSPServer_audio/src/WAVAudioFifoSource.cpp @@ -127,7 +127,7 @@ WAVAudioFifoSource::WAVAudioFifoSource(UsageEnvironment& env, FILE* fid) // Header vaules: 8 Khz, 16 bit, mono fWAVHeaderSize = 0; fBitsPerSample = 16; - fSamplingFrequency = 8000; + fSamplingFrequency = 16000; fNumChannels = 1; fAudioFormat = (unsigned char)WA_PCM; @@ -142,6 +142,9 @@ WAVAudioFifoSource::WAVAudioFifoSource(UsageEnvironment& env, FILE* fid) unsigned samplesPerFrame = desiredSamplesPerFrame < maxSamplesPerFrame ? desiredSamplesPerFrame : maxSamplesPerFrame; fPreferredFrameSize = (samplesPerFrame*fNumChannels*fBitsPerSample)/8; + // Yi Mstar noise reduction requires a framesize of 512 + fPreferredFrameSize = 512; + fFidIsSeekable = FileIsSeekable(fFid); // Now that we've finished reading the WAV header, all future reads (of audio samples) from the file will be asynchronous: makeSocketNonBlocking(fileno(fFid)); diff --git a/src/rRTSPServer_audio/src/YiNoiseReduction.cpp b/src/rRTSPServer_audio/src/YiNoiseReduction.cpp new file mode 100644 index 0000000..2f3d61c --- /dev/null +++ b/src/rRTSPServer_audio/src/YiNoiseReduction.cpp @@ -0,0 +1,112 @@ +/********** +This library is free software; you can redistribute it and/or modify it under +the terms of the GNU Lesser General Public License as published by the +Free Software Foundation; either version 3 of the License, or (at your +option) any later version. (See .) + +This library is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for +more details. + +You should have received a copy of the GNU Lesser General Public License +along with this library; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +**********/ +// Source template used to create this filter: +// Copyright (c) 1996-2020 Live Networks, Inc. All rights reserved. +// Filters for converting between raw PCM audio and uLaw +// Implementation + +#include "YiNoiseReduction.hh" +#include "WAVAudioFifoSource.hh" + +// Level defines the ammount of noise reduction. On YiHome1080 6FUS is was set to 30. +YiNoiseReduction* +YiNoiseReduction::createNew(UsageEnvironment& env, FramedSource* inputSource, unsigned int level) { + return new YiNoiseReduction(env, inputSource, level); +} + +YiNoiseReduction::YiNoiseReduction(UsageEnvironment& env, FramedSource* inputSource, unsigned int level) + : FramedFilter(env, inputSource) { + // Allocate a working buffer for the noise reduction functions. + apBuf = malloc(IaaApc_GetBufferSize()); + if (apBuf == NULL) { + printf("Failed to allocate memory for buffer\n"); + return; + } + + // Setup audio processing struct + apStruct.point_number = 256; // Magic + apStruct.channels = 1; // Mono + apStruct.rate = ((WAVAudioFifoSource*)(inputSource))->samplingFrequency(); +#ifdef DEBUG + printf("Sampling freq input source: %d Hz\n", apStruct.rate); +#endif + // Initialize noise reduction. + int res = IaaApc_Init((char* const)apBuf, &apStruct); + if (res != 0) { + printf("Audio processing init failed\n"); + return; + } + + // Enable noise reduction + res = IaaApc_SetNrEnable(1); + if (res != 0) { + printf("Failed setting level for noise reduction\n"); + return; + } + + // Set level + res = IaaApc_SetNrMode(level); + if (res != 0) { + printf("Failed setting level for noise reduction\n"); + return; + } + printf("Noise reduction enabled, level: %d\n", level); +} + +YiNoiseReduction::~YiNoiseReduction() { + // Free working buffer and noise reduction. + IaaApc_Free(); + if (apBuf != NULL) { + free(apBuf); + } +} + +void YiNoiseReduction::doGetNextFrame() { + // Arrange to read data directly into the client's buffer: + fInputSource->getNextFrame(fTo, fMaxSize, + afterGettingFrame, this, + FramedSource::handleClosure, this); +} + +void YiNoiseReduction::afterGettingFrame(void* clientData, unsigned frameSize, + unsigned numTruncatedBytes, + struct timeval presentationTime, + unsigned durationInMicroseconds) { + YiNoiseReduction* source = (YiNoiseReduction*)clientData; + source->afterGettingFrame1(frameSize, numTruncatedBytes, + presentationTime, durationInMicroseconds); +} + +void YiNoiseReduction::afterGettingFrame1(unsigned frameSize, unsigned numTruncatedBytes, + struct timeval presentationTime, + unsigned durationInMicroseconds) { + // Provide the sample buffer to IaaApc_Run to execute noise reduction. + // - The IaaApc_Run function replaces the original data. + // - IaaApc_Run requires 512 samples to work on + // - This was the previously hardcoded preferred framesize. + int res = IaaApc_Run((int16_t*)fTo); + if (res != 0) { + printf("Failure during executing IaaApc_Run(): %d\n", res); + } + + // Complete delivery to the client: + fFrameSize = frameSize; + fNumTruncatedBytes = numTruncatedBytes + (frameSize - fFrameSize); + fPresentationTime = presentationTime; + fDurationInMicroseconds = durationInMicroseconds; + afterGetting(this); +} + diff --git a/src/rRTSPServer_audio/src/rRTSPServer.cpp b/src/rRTSPServer_audio/src/rRTSPServer.cpp index 3bc478b..44315d7 100644 --- a/src/rRTSPServer_audio/src/rRTSPServer.cpp +++ b/src/rRTSPServer_audio/src/rRTSPServer.cpp @@ -23,6 +23,10 @@ along with this library; if not, write to the Free Software Foundation, Inc., #include "liveMedia.hh" #include "BasicUsageEnvironment.hh" #include "WAVAudioFifoServerMediaSubsession.hh" +#include "WAVAudioFifoSource.hh" +#include "StreamReplicator.hh" +#include "YiNoiseReduction.hh" +#include "DummySink.hh" UsageEnvironment* env; @@ -36,6 +40,45 @@ Boolean reuseFirstSource = True; // change the following "False" to "True": Boolean iFramesOnly = False; +StreamReplicator* startReplicatorStream(const char* inputAudioFileName, Boolean convertToULaw, unsigned int nr_level) { + // Create a single WAVAudioFifo source that will be replicated for mutliple streams + WAVAudioFifoSource* wavSource = WAVAudioFifoSource::createNew(*env, inputAudioFileName); + if (wavSource == NULL) { + *env << "Failed to create Fifo Source \n"; + } + + // Optionally enable the noise reduction filter + FramedSource* intermediateSource; + if (nr_level > 0) { + intermediateSource = YiNoiseReduction::createNew(*env, wavSource, nr_level); + } else { + intermediateSource = wavSource; + } + + // Optionally convert to uLaw pcm + FramedSource* resultSource; + if (convertToULaw) { + resultSource = uLawFromPCMAudioSource::createNew(*env, intermediateSource, 1/*little-endian*/); + } else { + resultSource = EndianSwap16::createNew(*env, intermediateSource); + } + + // Create and start the replicator that will be given to each subsession + StreamReplicator* replicator = StreamReplicator::createNew(*env, resultSource); + + // Begin by creating an input stream from our replicator: + FramedSource* source = replicator->createStreamReplica(); + + // Then create a 'dummy sink' object to receive thie replica stream: + MediaSink* sink = DummySink::createNew(*env, "dummy"); + + // Now, start playing, feeding the sink object from the source: + sink->startPlaying(*source, NULL, NULL); + + return replicator; +} + + static void announceStream(RTSPServer* rtspServer, ServerMediaSession* sms, char const* streamName, char const* inputFileName, int audio) { char* url = rtspServer->rtspURL(sms); @@ -57,6 +100,7 @@ int main(int argc, char** argv) { int nm; char user[65]; char pwd[65]; + int nr_level = 0; Boolean convertToULaw = True; char const* inputAudioFileName = "/tmp/audio_fifo"; @@ -86,8 +130,14 @@ int main(int argc, char** argv) { strcpy(pwd, str); } + str = getenv("NR_LEVEL"); + if (str && sscanf (str, "%i", &nm) == 1 && nm >= 0 && nm <= 30) { + nr_level = nm; + } + // If fifo doesn't exist, disable audio if (stat (inputAudioFileName, &stat_buffer) != 0) { + *env << "Audio fifo does not exist, disabling audio.n"; audio = 0; } @@ -105,6 +155,9 @@ int main(int argc, char** argv) { // access to the server. } + // Create and start the replicator that will be given to each subsession + StreamReplicator* replicator = startReplicatorStream(inputAudioFileName, convertToULaw, nr_level); + // Create the RTSP server: RTSPServer* rtspServer = RTSPServer::createNew(*env, port, authDB); if (rtspServer == NULL) { @@ -124,7 +177,7 @@ int main(int argc, char** argv) { if (res == 0) { char const* streamName = "ch0_0.h264"; - char const* inputFileName = "stdin"; + char const* inputFileName = "/tmp/h264_high_fifo"; // First, make sure that the RTPSinks' buffers will be large enough to handle the huge size of DV frames (as big as 288000). OutPacketBuffer::maxSize = 300000; @@ -136,7 +189,7 @@ int main(int argc, char** argv) { ::createNew(*env, inputFileName, reuseFirstSource)); if (audio == 1) { sms_high->addSubsession(WAVAudioFifoServerMediaSubsession - ::createNew(*env, inputAudioFileName, reuseFirstSource, convertToULaw)); + ::createNew(*env, replicator, reuseFirstSource, convertToULaw)); } rtspServer->addServerMediaSession(sms_high); @@ -144,10 +197,10 @@ int main(int argc, char** argv) { } // A H.264 video elementary stream: - if (res == 1) + if (res == 0) { char const* streamName = "ch0_1.h264"; - char const* inputFileName = "stdin"; + char const* inputFileName = "/tmp/h264_low_fifo"; // First, make sure that the RTPSinks' buffers will be large enough to handle the huge size of DV frames (as big as 288000). OutPacketBuffer::maxSize = 300000; @@ -159,13 +212,31 @@ int main(int argc, char** argv) { ::createNew(*env, inputFileName, reuseFirstSource)); if (audio == 1) { sms_low->addSubsession(WAVAudioFifoServerMediaSubsession - ::createNew(*env, inputAudioFileName, reuseFirstSource, convertToULaw)); + ::createNew(*env, replicator, reuseFirstSource, convertToULaw)); } rtspServer->addServerMediaSession(sms_low); announceStream(rtspServer, sms_low, streamName, inputFileName, audio); } + // A PCM audio elementary stream: + if (audio) + { + char const* streamName = "ch0_2.h264"; + + // First, make sure that the RTPSinks' buffers will be large enough to handle the huge size of DV frames (as big as 288000). + OutPacketBuffer::maxSize = 300000; + + ServerMediaSession* sms_audio + = ServerMediaSession::createNew(*env, streamName, streamName, + descriptionString); + sms_audio->addSubsession(WAVAudioFifoServerMediaSubsession + ::createNew(*env, replicator, reuseFirstSource, convertToULaw)); + rtspServer->addServerMediaSession(sms_audio); + + announceStream(rtspServer, sms_audio, streamName, inputAudioFileName, audio); + } + // Also, attempt to create a HTTP server for RTSP-over-HTTP tunneling. // Try first with the default HTTP port (80), and then with the alternative HTTP // port numbers (8000 and 8080). diff --git a/src/tinyalsa/Yihack_tinyalsa.patch b/src/tinyalsa/Yihack_tinyalsa.patch index dc0714c..20af774 100644 --- a/src/tinyalsa/Yihack_tinyalsa.patch +++ b/src/tinyalsa/Yihack_tinyalsa.patch @@ -110,7 +110,7 @@ diff -Naur tinyalsa-1.0.0.ori/src/pcm.c tinyalsa-1.0.0/src/pcm.c #define SNDRV_PCM_HW_PARAMS_NO_PERIOD_WAKEUP (1<<2) +#define CONVERT_TO_MONO -+#define CONVERT_TO_8KHZ ++//#define CONVERT_TO_8KHZ +#ifdef CONVERT_TO_8KHZ +#define CONVERT_TO_MONO +#endif