Look for SPS instead of keyframe with H264 packetization_mode=0 (#220)

Author: sgfn

lib/ex_webrtc/peer_connection/configuration.ex

@@ -27,31 +27,7 @@ defmodule ExWebRTC.PeerConnection.Configuration do
     clock_rate: 90_000
   }
 
-  # Ensure we are using H264 with packetization_mode=1 by default
-  # (packetization_mode=0 has issues when switching layers)
-  @default_codec_params_h264 %RTPCodecParameters{
-    payload_type: 99,
-    mime_type: "video/H264",
-    clock_rate: 90_000,
-    sdp_fmtp_line: %FMTP{
-      pt: 99,
-      level_asymmetry_allowed: true,
-      packetization_mode: 1,
-      profile_level_id: 0x42E01F
-    }
-  }
-
-  @default_codec_params_av1 %RTPCodecParameters{
-    payload_type: 45,
-    mime_type: "video/AV1",
-    clock_rate: 90_000,
-    sdp_fmtp_line: %FMTP{pt: 45, level_idx: 5, profile: 0, tier: 0}
-  }
-
-  @default_audio_codecs [@default_codec_params_opus]
-
-  @default_video_codecs [
-    @default_codec_params_vp8,
+  @default_codec_params_h264 [
     %RTPCodecParameters{
       payload_type: 98,
       mime_type: "video/H264",
@@ -63,10 +39,34 @@ defmodule ExWebRTC.PeerConnection.Configuration do
         profile_level_id: 0x42E01F
       }
     },
-    @default_codec_params_h264,
-    @default_codec_params_av1
+    %RTPCodecParameters{
+      payload_type: 99,
+      mime_type: "video/H264",
+      clock_rate: 90_000,
+      sdp_fmtp_line: %FMTP{
+        pt: 99,
+        level_asymmetry_allowed: true,
+        packetization_mode: 1,
+        profile_level_id: 0x42E01F
+      }
+    }
   ]
 
+  @default_codec_params_av1 %RTPCodecParameters{
+    payload_type: 45,
+    mime_type: "video/AV1",
+    clock_rate: 90_000,
+    sdp_fmtp_line: %FMTP{pt: 45, level_idx: 5, profile: 0, tier: 0}
+  }
+
+  @default_audio_codecs [@default_codec_params_opus]
+  @default_video_codecs [
+                          @default_codec_params_vp8,
+                          @default_codec_params_h264,
+                          @default_codec_params_av1
+                        ]
+                        |> List.flatten()
+
   @typedoc """
   Allowed audio codec names which will get expanded to the relevant default `t:ExWebRTC.RTPCodecParameters.t/0`
   """
@@ -718,7 +718,7 @@ defmodule ExWebRTC.PeerConnection.Configuration do
         options
 
       Enum.all?(codecs, &is_atom/1) ->
-        expanded_codecs = Enum.map(codecs, &expand_default_codec/1)
+        expanded_codecs = codecs |> Enum.map(&expand_default_codec/1) |> List.flatten()
         Keyword.put(options, key, expanded_codecs)
 
       true ->

lib/ex_webrtc/rtp/h264.ex

@@ -7,10 +7,19 @@ defmodule ExWebRTC.RTP.H264 do
 
   # Copied nearly 1-to-1 from https://github.com/membraneframework/membrane_rtp_h264_plugin/blob/master/lib/rtp_h264/utils.ex
   # originally based on galene's implementation https://github.com/jech/galene/blob/6fbdf0eab2c9640e673d9f9ec0331da24cbf2c4c/codecs/codecs.go#L119
-  # but only looks for SPS
-  # it is also unclear why we sometimes check against nalu type == 7
-  # and sometimes against nalu type == 5 but galene does it this way
-  # and it works
+  # but only looks for SPS, in packetization_mode=0 as well as packetization_mode=1.
+  #
+  # It's been empirically tested with simulated packet loss that for packetization_mode=0 (`nalu_type in 1..23` clause):
+  # * if we're checking against `nalu_type == 5`, the stream breaks regularly when switching layers,
+  # * if we're checking against `nalu_type == 5 or nalu_type == 7`, the stream breaks occasionally when switching layers,
+  #   this happens when we've lost the packet containing SPS, but received the following one containing the keyframe,
+  # * if we're checking against `nalu_type == 7`, no issues were encountered.
+  #
+  # Janus also does it this way.
+  # https://github.com/meetecho/janus-gateway/blob/3367f41de9225daed812ca0991c259f1458fe49f/src/utils.h#L352
+  #
+  # For more info, refer to the H264 spec and RFC 6184, sections 5.4 and 6
+  # https://datatracker.ietf.org/doc/html/rfc6184#section-5.4
 
   @doc """
   Returns a boolean telling if the packets contains a beginning of a H264 intra-frame.
@@ -21,14 +30,21 @@ defmodule ExWebRTC.RTP.H264 do
 
   def keyframe?(%Packet{}), do: false
 
+  # Reserved
   defp do_keyframe?(0, _), do: false
-  defp do_keyframe?(nalu_type, _) when nalu_type in 1..23, do: nalu_type == 5
+
+  # Single NAL Unit packets: check if NALU contains SPS (type 7)
+  defp do_keyframe?(nalu_type, _) when nalu_type in 1..23, do: nalu_type == 7
+
+  # STAP-A
   defp do_keyframe?(24, aus), do: check_aggr_units(24, aus)
 
+  # STAP-B, MTAP16, MTAP24
   defp do_keyframe?(nalu_type, <<_don::16, aus::binary>>)
        when nalu_type in 25..27,
        do: check_aggr_units(nalu_type, aus)
 
+  # FU-A, FU-B
   defp do_keyframe?(nalu_type, <<s::1, _e::1, _r::1, type::5, _fu_payload::binary>>)
        when nalu_type in 28..29,
        do: s == 1 and type == 7

test/ex_webrtc/peer_connection/configuration_test.exs

@@ -515,11 +515,13 @@ defmodule ExWebRTC.PeerConnection.ConfigurationTest do
 
     assert Configuration.expand_default_codecs(og_options) == og_options
 
-    [video_codecs: [vp8_params, h264_params]] =
+    # 2 packetization_modes for H264
+    [video_codecs: [vp8_params, h264_params_0, h264_params_1]] =
       Configuration.expand_default_codecs(video_codecs: [:vp8, :h264])
 
     assert vp8_params.mime_type == "video/VP8"
-    assert h264_params.mime_type == "video/H264"
+    assert h264_params_0.mime_type == "video/H264"
+    assert h264_params_1.mime_type == "video/H264"
 
     assert_raise RuntimeError, fn -> Configuration.expand_default_codecs(video_codecs: [:av2]) end