FLAC and Opus support

This adds the MP4-registered codec entry "fLaC", as well as the
non-registered codec entries "flac" and "FLAC".

Browsers added FLAC support before the MP4 Registration Authority listed
the codec entry "fLaC", and went with "flac". Bugs have been opened
with browsers to address this, but in order to deal with legacy
playlist manifests (they may list "flac" or even "FLAC"), this adds a
new interface and utility method to generate fallback names, and replace
them as they're encountered.

Author: jprjr

src/controller/level-controller.ts

@@ -18,9 +18,11 @@ import { isCodecSupportedInMp4 } from '../utils/codecs';
 import { addGroupId, assignTrackIdsByGroup } from './level-helper';
 import BasePlaylistController from './base-playlist-controller';
 import { PlaylistContextType, PlaylistLevelType } from '../types/loader';
+import { getCodecCompatNames } from '../utils/codecs';
 import type Hls from '../hls';
 import type { HlsUrlParameters, LevelParsed } from '../types/level';
 import type { MediaPlaylist } from '../types/media-playlist';
+import type { CodecCompatNames } from '../utils/codecs';
 
 const chromeOrFirefox: boolean = /chrome|firefox/.test(
   navigator.userAgent.toLowerCase()
@@ -89,6 +91,7 @@ export default class LevelController extends BasePlaylistController {
     let resolutionFound = false;
     let videoCodecFound = false;
     let audioCodecFound = false;
+    const codecCompatNames: CodecCompatNames = getCodecCompatNames();
 
     // regroup redundant levels together
     data.levels.forEach((levelParsed: LevelParsed) => {
@@ -109,6 +112,22 @@ export default class LevelController extends BasePlaylistController {
         levelParsed.audioCodec = undefined;
       }
 
+      // replace the video codec name if required
+      if (
+        levelParsed.videoCodec &&
+        levelParsed.videoCodec in codecCompatNames.video
+      ) {
+        levelParsed.videoCodec = codecCompatNames.video[levelParsed.videoCodec];
+      }
+
+      // replace the audio codec name if required
+      if (
+        levelParsed.audioCodec &&
+        levelParsed.audioCodec in codecCompatNames.audio
+      ) {
+        levelParsed.audioCodec = codecCompatNames.audio[levelParsed.audioCodec];
+      }
+
       const levelKey = `${levelParsed.bitrate}-${levelParsed.attrs.RESOLUTION}-${levelParsed.attrs.CODECS}`;
       levelFromSet = levelSet[levelKey];
 

src/controller/stream-controller.ts

@@ -1210,7 +1210,12 @@ export default class StreamController
         }
       }
       // HE-AAC is broken on Android, always signal audio codec as AAC even if variant manifest states otherwise
-      if (ua.indexOf('android') !== -1 && audio.container !== 'audio/mpeg') {
+      if (
+        audioCodec &&
+        audioCodec.indexOf('mp4a.40.5') !== -1 &&
+        ua.indexOf('android') !== -1 &&
+        audio.container !== 'audio/mpeg'
+      ) {
         // Exclude mpeg audio
         audioCodec = 'mp4a.40.2';
         this.log(`Android: force audio codec to ${audioCodec}`);

src/demux/transmuxer-interface.ts

@@ -10,11 +10,13 @@ import { ErrorTypes, ErrorDetails } from '../errors';
 import { getMediaSource } from '../utils/mediasource-helper';
 import { EventEmitter } from 'eventemitter3';
 import { Fragment, Part } from '../loader/fragment';
+import { getCodecCompatNames } from '../utils/codecs';
 import type { ChunkMetadata, TransmuxerResult } from '../types/transmuxer';
 import type Hls from '../hls';
 import type { HlsEventEmitter } from '../events';
 import type { PlaylistLevelType } from '../types/loader';
 import type { TypeSupported } from './tsdemuxer';
+import type { CodecCompatNames } from '../utils/codecs';
 
 const MediaSource = getMediaSource() || { isTypeSupported: () => false };
 
@@ -61,6 +63,8 @@ export default class TransmuxerInterface {
       mpeg: MediaSource.isTypeSupported('audio/mpeg'),
       mp3: MediaSource.isTypeSupported('audio/mp4; codecs="mp3"'),
     };
+    const codecCompatNames: CodecCompatNames = getCodecCompatNames();
+
     // navigator.vendor is not always available in Web Worker
     // refer to https://developer.mozilla.org/en-US/docs/Web/API/WorkerGlobalScope/navigator
     const vendor = navigator.vendor;
@@ -89,6 +93,7 @@ export default class TransmuxerInterface {
         worker.postMessage({
           cmd: 'init',
           typeSupported: typeSupported,
+          codecCompatNames: codecCompatNames,
           vendor: vendor,
           id: id,
           config: JSON.stringify(config),
@@ -105,6 +110,7 @@ export default class TransmuxerInterface {
         this.transmuxer = new Transmuxer(
           this.observer,
           typeSupported,
+          codecCompatNames,
           config,
           vendor,
           id
@@ -115,6 +121,7 @@ export default class TransmuxerInterface {
       this.transmuxer = new Transmuxer(
         this.observer,
         typeSupported,
+        codecCompatNames,
         config,
         vendor,
         id

src/demux/transmuxer-worker.ts

@@ -37,6 +37,7 @@ export default function TransmuxerWorker(self) {
         self.transmuxer = new Transmuxer(
           observer,
           data.typeSupported,
+          data.codecCompatNames,
           config,
           data.vendor,
           data.id

src/demux/transmuxer.ts

@@ -15,6 +15,7 @@ import type { TransmuxerResult, ChunkMetadata } from '../types/transmuxer';
 import type { HlsConfig } from '../config';
 import type { LevelKey } from '../loader/level-key';
 import type { PlaylistLevelType } from '../types/loader';
+import type { CodecCompatNames } from '../utils/codecs';
 
 let now;
 // performance.now() not available on WebWorker, at least on Safari Desktop
@@ -41,6 +42,7 @@ const muxConfig: MuxConfig[] = [
 export default class Transmuxer {
   private observer: HlsEventEmitter;
   private typeSupported: TypeSupported;
+  private codecCompatNames: CodecCompatNames;
   private config: HlsConfig;
   private vendor: string;
   private id: PlaylistLevelType;
@@ -55,12 +57,14 @@ export default class Transmuxer {
   constructor(
     observer: HlsEventEmitter,
     typeSupported: TypeSupported,
+    codecCompatNames: CodecCompatNames,
     config: HlsConfig,
     vendor: string,
     id: PlaylistLevelType
   ) {
     this.observer = observer;
     this.typeSupported = typeSupported;
+    this.codecCompatNames = codecCompatNames;
     this.config = config;
     this.vendor = vendor;
     this.id = id;
@@ -392,7 +396,7 @@ export default class Transmuxer {
     data: Uint8Array,
     transmuxConfig: TransmuxConfig
   ) {
-    const { config, observer, typeSupported, vendor } = this;
+    const { config, observer, typeSupported, codecCompatNames, vendor } = this;
     const {
       audioCodec,
       defaultInitPts,
@@ -421,7 +425,13 @@ export default class Transmuxer {
     const Remuxer: MuxConfig['remux'] = mux.remux;
     const Demuxer: MuxConfig['demux'] = mux.demux;
     if (!remuxer || !(remuxer instanceof Remuxer)) {
-      this.remuxer = new Remuxer(observer, config, typeSupported, vendor);
+      this.remuxer = new Remuxer(
+        observer,
+        config,
+        typeSupported,
+        codecCompatNames,
+        vendor
+      );
     }
     if (!demuxer || !(demuxer instanceof Demuxer)) {
       this.demuxer = new Demuxer(observer, config, typeSupported);

src/remux/mp4-remuxer.ts

@@ -51,6 +51,7 @@ export default class MP4Remuxer implements Remuxer {
     observer: HlsEventEmitter,
     config: HlsConfig,
     typeSupported,
+    codecCompatNames,
     vendor = ''
   ) {
     this.observer = observer;

src/remux/passthrough-remuxer.ts

@@ -24,6 +24,7 @@ import type {
   DemuxedUserdataTrack,
   PassthroughTrack,
 } from '../types/demuxer';
+import type { CodecCompatNames } from '../utils/codecs';
 
 class PassThroughRemuxer implements Remuxer {
   private emitInitSegment: boolean = false;
@@ -33,6 +34,17 @@ class PassThroughRemuxer implements Remuxer {
   private initPTS?: number;
   private initTracks?: TrackSet;
   private lastEndTime: number | null = null;
+  private codecCompatNames: CodecCompatNames;
+
+  constructor(
+    observer,
+    config,
+    typeSupported,
+    codecCompatNames: CodecCompatNames,
+    vendor
+  ) {
+    this.codecCompatNames = codecCompatNames;
+  }
 
   public destroy() {}
 
@@ -71,13 +83,19 @@ class PassThroughRemuxer implements Remuxer {
         initData.audio,
         ElementaryStreamTypes.AUDIO
       );
+      if (audioCodec in this.codecCompatNames.audio) {
+        audioCodec = this.codecCompatNames.audio[audioCodec];
+      }
     }
 
     if (!videoCodec) {
       videoCodec = getParsedTrackCodec(
         initData.video,
         ElementaryStreamTypes.VIDEO
       );
+      if (videoCodec in this.codecCompatNames.video) {
+        videoCodec = this.codecCompatNames.video[videoCodec];
+      }
     }
 
     const tracks: TrackSet = {};
@@ -246,6 +264,8 @@ function getParsedTrackCodec(
   if (parsedCodec === 'avc1' || type === ElementaryStreamTypes.VIDEO) {
     return 'avc1.42e01e';
   }
+  if (parsedCodec === 'fLaC') return parsedCodec;
+  if (parsedCodec === 'Opus') return parsedCodec;
   return 'mp4a.40.5';
 }
 export default PassThroughRemuxer;

src/utils/codecs.ts

@@ -14,6 +14,9 @@ const sampleEntryCodesISO = {
     dtsh: true,
     'ec-3': true,
     enca: true,
+    fLaC: true,
+    flac: true, // non-compliant: browsers may expect this to be lowercase instead of MP4RA 'fLaC'
+    FLAC: true, // non-compliant: apple tools recommended string
     g719: true,
     g726: true,
     m4ae: true,
@@ -83,3 +86,65 @@ export function isCodecSupportedInMp4(codec: string, type: CodecType): boolean {
     `${type || 'video'}/mp4;codecs="${codec}"`
   );
 }
+
+interface AudioCodecCompatNames {
+  flac?: string;
+  fLaC?: string;
+  FLAC?: string;
+  Opus?: string;
+  opus?: string;
+}
+
+export interface CodecCompatNames {
+  audio: AudioCodecCompatNames;
+  video: {};
+  text: {};
+}
+
+// Some browsers expect the codec strings for FLAC and Opus
+// to be lowercase. The MP4RA defines the codec values as
+// "fLaC" and "Opus", but most browsers supported the lowercase
+// strings before the MP4RA published these values.
+//
+// Additionally, some Apple tools recommended the string
+// "FLAC" in manifest files, which is also non-compliant. We use
+// this to re-map codec names as necessary.
+
+export function getCodecCompatNames(): CodecCompatNames {
+  const audioCompatNames: AudioCodecCompatNames = {};
+  const videoCompatNames = {};
+  const textCompatNames = {};
+
+  // if the MP4RA string is available, opt for that
+  // in all cases
+  if (isCodecSupportedInMp4('fLaC', 'audio')) {
+    audioCompatNames.FLAC = 'fLaC';
+    audioCompatNames.flac = 'fLaC';
+  } else {
+    if (isCodecSupportedInMp4('flac', 'audio')) {
+      audioCompatNames.fLaC = 'flac';
+      audioCompatNames.FLAC = 'flac';
+    } else {
+      if (isCodecSupportedInMp4('FLAC', 'audio')) {
+        audioCompatNames.flac = 'FLAC';
+        audioCompatNames.fLaC = 'FLAC';
+      }
+    }
+  }
+
+  if (isCodecSupportedInMp4('Opus', 'audio')) {
+    audioCompatNames.opus = 'Opus';
+  } else {
+    if (isCodecSupportedInMp4('opus', 'audio')) {
+      audioCompatNames.Opus = 'opus';
+    }
+  }
+
+  const codecCompatNames: CodecCompatNames = {
+    audio: audioCompatNames,
+    video: videoCompatNames,
+    text: textCompatNames,
+  };
+
+  return codecCompatNames;
+}