Add set_sender_codec (#190)

Author: mickel8

lib/ex_webrtc/peer_connection.ex

@@ -9,6 +9,7 @@ defmodule ExWebRTC.PeerConnection do
 
   require Logger
 
+  alias ExWebRTC.RTPCodecParameters
   alias __MODULE__.{Configuration, Demuxer, TWCCRecorder}
 
   alias ExWebRTC.{
@@ -152,9 +153,67 @@ defmodule ExWebRTC.PeerConnection do
     GenServer.call(peer_connection, :get_configuration)
   end
 
+  @doc """
+  Sets the codec that will be used for sending RTP packets.
+
+  `send_rtp/4` overrides some of the RTP packet fields.
+  In particular, when multiple codecs are negotiated, `send_rtp/4` will use
+  payload type of the most preffered by the remote side codec (i.e. the first
+  one from the list of codecs in the remote description).
+
+  Use this function if you want to select, which codec (hence payload type)
+  should be used for sending.
+
+  Once the first RTP packet is sent (via `send_rtp/4`), `set_sender_codec/3`
+  can only be called with a codec with the same clock rate.
+
+  Although very unlikely, keep in mind that after renegotiation,
+  the selected codec may no longer be supported by the remote side and you might
+  need to call this function again, passing a new codec.
+
+  To check available codecs you can use `get_transceivers/1`:
+
+  ```
+  {:ok, pc} = PeerConnection.start_link()
+  {:ok, rtp_sender} = PeerConnection.add_track(MediaStreamTrack.new(:video))
+
+  # exchange SDP with the remote side
+  # {:ok, offer} = PeerConnection.create_offer(pc)
+  # ...
+
+  tr =
+    pc
+    |> PeerConnection.get_transceivers()
+    |> Enum.find(fn tr -> tr.sender.id == rtp_sender.id end)
+
+  dbg(tr.codecs) # list of supported codecs both for sending and receiving
+
+  # e.g. always prefer h264 over vp8
+  h264 = Enum.find(tr.codecs, fn codec -> codec.mime_type == "video/H264" end)
+  vp8 = Enum.find(tr.codecs, fn codec -> codec.mime_type == "video/VP8" end)
+
+  :ok = PeerConnection.set_sender_codec(pc, rtp_sender.id, h264 || vp8)
+  ```
+
+  This function can only be called once the first negotiation passes.
+  """
+  @spec set_sender_codec(peer_connection(), RTPSender.id(), RTPCodecParameters.t()) ::
+          :ok | {:error, term()}
+  def set_sender_codec(peer_connection, sender_id, codec) do
+    GenServer.call(peer_connection, {:set_sender_codec, sender_id, codec})
+  end
+
   @doc """
   Sends an RTP packet to the remote peer using the track specified by the `track_id`.
 
+  The following fields of the RTP packet will be overwritten by this function:
+  * payload type
+  * ssrc
+  * rtp header extensions
+
+  If you negotiated multiple codecs (hence payload types) and you want to choose,
+  which one should be used, see `set_sender_codec/3`.
+
   Options:
     * `rtx?` - send the packet as if it was retransmitted (use SSRC and payload type specific to RTX)
   """
@@ -576,6 +635,31 @@ defmodule ExWebRTC.PeerConnection do
     {:reply, state.config, state}
   end
 
+  @impl true
+  def handle_call({:set_sender_codec, sender_id, codec}, _from, state) do
+    state.transceivers
+    |> Enum.with_index()
+    |> Enum.find(fn {tr, _idx} -> tr.sender.id == sender_id end)
+    |> case do
+      {tr, idx} when tr.direction in [:sendrecv, :sendonly] ->
+        case RTPTransceiver.set_sender_codec(tr, codec) do
+          {:ok, tr} ->
+            transceivers = List.replace_at(state.transceivers, idx, tr)
+            state = %{state | transceivers: transceivers}
+            {:reply, :ok, state}
+
+          {:error, _reason} = error ->
+            {:reply, error, state}
+        end
+
+      {_tr, _idx} ->
+        {:reply, {:error, :invalid_transceiver_direction}, state}
+
+      nil ->
+        {:reply, {:error, :invalid_sender_id}, state}
+    end
+  end
+
   @impl true
   def handle_call(:get_connection_state, _from, state) do
     {:reply, state.conn_state, state}
@@ -1135,7 +1219,8 @@ defmodule ExWebRTC.PeerConnection do
           end
 
         {packet, tr} = RTPTransceiver.send_packet(tr, packet, rtx?)
-        :ok = DTLSTransport.send_rtp(state.dtls_transport, packet)
+
+        if packet != <<>>, do: :ok = DTLSTransport.send_rtp(state.dtls_transport, packet)
 
         transceivers = List.replace_at(state.transceivers, idx, tr)
         state = %{state | transceivers: transceivers}

lib/ex_webrtc/peer_connection/configuration.ex

@@ -35,11 +35,22 @@ defmodule ExWebRTC.PeerConnection.Configuration do
       clock_rate: 90_000,
       sdp_fmtp_line: %FMTP{
         pt: 98,
-        level_asymmetry_allowed: 1,
+        level_asymmetry_allowed: true,
         packetization_mode: 0,
         profile_level_id: 0x42E01F
       }
     },
+    %RTPCodecParameters{
+      payload_type: 99,
+      mime_type: "video/H264",
+      clock_rate: 90_000,
+      sdp_fmtp_line: %FMTP{
+        pt: 99,
+        level_asymmetry_allowed: true,
+        packetization_mode: 1,
+        profile_level_id: 0x42E01F
+      }
+    },
     %RTPCodecParameters{
       payload_type: 45,
       mime_type: "video/AV1",
@@ -252,10 +263,27 @@ defmodule ExWebRTC.PeerConnection.Configuration do
     |> Keyword.put(:audio_extensions, Enum.map(audio_extensions, fn {_, ext} -> ext end))
     |> Keyword.put(:video_extensions, Enum.map(video_extensions, fn {_, ext} -> ext end))
     |> then(&struct(__MODULE__, &1))
+    |> ensure_unique_payload_types()
     |> populate_feedbacks(feedbacks)
     |> add_features()
   end
 
+  defp ensure_unique_payload_types(config) do
+    audio_pt = Enum.map(config.audio_codecs, fn codec -> codec.payload_type end)
+
+    if length(audio_pt) != length(Enum.uniq(audio_pt)) do
+      raise "Payload types in audio codecs are not unique."
+    end
+
+    video_pt = Enum.map(config.video_codecs, fn codec -> codec.payload_type end)
+
+    if length(video_pt) != length(Enum.uniq(video_pt)) do
+      raise "Payload types in video codecs are not unique."
+    end
+
+    config
+  end
+
   defp add_features(config) do
     %__MODULE__{features: features} = config
 
@@ -405,6 +433,9 @@ defmodule ExWebRTC.PeerConnection.Configuration do
     config
     |> update_extensions(sdp)
     |> update_codecs(sdp)
+    # if update went wrong (there are duplicates in payload types),
+    # we should never continue as this may lead to hard to debug errors
+    |> ensure_unique_payload_types()
   end
 
   defp update_extensions(config, sdp) do
@@ -425,25 +456,41 @@ defmodule ExWebRTC.PeerConnection.Configuration do
   defp do_update_extensions(extensions, sdp_extensions, free_ids) do
     # we replace extension ids in config to ids from the SDP
     # in case we have an extension in config but not in SDP, we replace
-    # its id to some free (not present in SDP) id, so it doesn't conflict
+    # its id only when it's occupied to some free (not present in SDP) id, so it doesn't conflict
     Enum.map_reduce(extensions, free_ids, fn ext, free_ids ->
-      sdp_extensions
-      |> Enum.find(&(&1.uri == ext.uri))
-      |> case do
-        nil ->
+      case find_in_sdp_rtp_extensions(sdp_extensions, ext) do
+        {nil, false} ->
+          {ext, free_ids}
+
+        {nil, true} ->
           [id | rest] = free_ids
           {%Extmap{ext | id: id}, rest}
 
-        other ->
+        {other, _id_used} ->
           {%Extmap{ext | id: other.id}, free_ids}
       end
     end)
   end
 
+  # Searches for rtp extension in sdp rtp extensions.
+  # If ext is not found, id_used determines whether ext's id
+  # is already present in sdp_extensions.
+  # Otherwise, id_used can have any value.
+  defp find_in_sdp_rtp_extensions(sdp_extensions, ext, id_used \\ false)
+  defp find_in_sdp_rtp_extensions([], _ext, id_used), do: {nil, id_used}
+
+  defp find_in_sdp_rtp_extensions([sdp_ext | sdp_extensions], ext, id_used) do
+    if sdp_ext.uri == ext.uri do
+      {sdp_ext, id_used}
+    else
+      find_in_sdp_rtp_extensions(sdp_extensions, ext, id_used || sdp_ext.id == ext.id)
+    end
+  end
+
   defp update_codecs(config, sdp) do
     %__MODULE__{audio_codecs: audio_codecs, video_codecs: video_codecs} = config
     sdp_codecs = SDPUtils.get_rtp_codec_parameters(sdp)
-    free_pts = get_free_payload_types(sdp_codecs)
+    free_pts = get_free_payload_types(audio_codecs ++ video_codecs ++ sdp_codecs)
 
     {audio_codecs, free_pts} = do_update_codecs(audio_codecs, sdp_codecs, free_pts)
     {video_codecs, _free_pts} = do_update_codecs(video_codecs, sdp_codecs, free_pts)
@@ -452,29 +499,27 @@ defmodule ExWebRTC.PeerConnection.Configuration do
   end
 
   defp do_update_codecs(codecs, sdp_codecs, free_pts) do
-    # we replace codec payload types in config to payload types from SDP
-    # both normal codecs and rtx (we also update apt FMTP attribute in rtxs)
-    # other codecs that are present in config but not in SDP
-    # are also updated with values from a pool of free payload types (not present in SDP)
-    # to make sure they don't conflict
-    {sdp_rtxs, sdp_codecs} = Enum.split_with(sdp_codecs, &rtx?/1)
+    # We replace codec payload types in config to payload types from SDP
+    # both for normal codecs and rtx (we also update apt FMTP attribute in rtxs).
+    # Other codecs that are present in config but not in SDP, and their
+    # payload type is already present in SDP, are also updated with values
+    # from a pool of free payload types (not present in SDP) to make sure they don't conflict
     {rtxs, codecs} = Enum.split_with(codecs, &rtx?/1)
 
     {codecs, {free_pts, mapping}} =
       Enum.map_reduce(codecs, {free_pts, %{}}, fn codec, {free_pts, mapping} ->
-        sdp_codecs
-        |> Enum.find(
-          &(String.downcase(&1.mime_type) == String.downcase(codec.mime_type) and
-              &1.clock_rate == codec.clock_rate and
-              &1.channels == codec.channels)
-        )
-        |> case do
-          nil ->
+        case find_in_sdp_codecs(sdp_codecs, codec) do
+          # there is no such codec and its payload type is not used
+          {nil, false} ->
+            {codec, {free_pts, Map.put(mapping, codec.payload_type, codec.payload_type)}}
+
+          # there is no such codec, but its payload type is used
+          {nil, true} ->
             [pt | rest] = free_pts
             new_codec = do_update_codec(codec, pt)
             {new_codec, {rest, Map.put(mapping, codec.payload_type, pt)}}
 
-          other ->
+          {other, _pt_used} ->
             new_codec = do_update_codec(codec, other.payload_type)
             {new_codec, {free_pts, Map.put(mapping, codec.payload_type, other.payload_type)}}
         end
@@ -486,15 +531,18 @@ defmodule ExWebRTC.PeerConnection.Configuration do
         %RTPCodecParameters{rtx | sdp_fmtp_line: %FMTP{fmtp | apt: Map.fetch!(mapping, apt)}}
       end)
       |> Enum.map_reduce(free_pts, fn rtx, free_pts ->
-        sdp_rtxs
-        |> Enum.find(&(&1.sdp_fmtp_line.apt == rtx.sdp_fmtp_line.apt))
-        |> case do
-          nil ->
+        case find_in_sdp_codecs(sdp_codecs, rtx) do
+          # there is no such codec and its payload type is not used
+          {nil, false} ->
+            {rtx, free_pts}
+
+          # there is no such codec, but its payload type is used
+          {nil, true} ->
             [pt | rest] = free_pts
             rtx = do_update_codec(rtx, pt)
             {rtx, rest}
 
-          other ->
+          {other, _pt_used} ->
             rtx = do_update_codec(rtx, other.payload_type)
             {rtx, free_pts}
         end
@@ -503,6 +551,38 @@ defmodule ExWebRTC.PeerConnection.Configuration do
     {codecs ++ rtxs, free_pts}
   end
 
+  # Searches for codec in sdp_codecs.
+  # If codec is not found, pt_used determines whether
+  # codec's payload type is already present in sdp_codecs.
+  # Otherwise, pt_used can have any value.
+  defp find_in_sdp_codecs(sdp_codecs, codec, pt_used \\ false)
+
+  defp find_in_sdp_codecs([], _codec, pt_used), do: {nil, pt_used}
+
+  defp find_in_sdp_codecs([sdp_codec | sdp_codecs], codec, pt_used) do
+    if String.ends_with?(codec.mime_type, "/rtx") do
+      if sdp_codec.sdp_fmtp_line != nil && sdp_codec.sdp_fmtp_line.apt == codec.sdp_fmtp_line.apt do
+        {sdp_codec, pt_used}
+      else
+        find_in_sdp_codecs(
+          sdp_codecs,
+          codec,
+          pt_used || sdp_codec.payload_type == codec.payload_type
+        )
+      end
+    else
+      if codec_equal_soft?(sdp_codec, codec) do
+        {sdp_codec, pt_used}
+      else
+        find_in_sdp_codecs(
+          sdp_codecs,
+          codec,
+          pt_used || sdp_codec.payload_type == codec.payload_type
+        )
+      end
+    end
+  end
+
   defp do_update_codec(codec, new_pt) do
     %RTPCodecParameters{rtcp_fbs: fbs, sdp_fmtp_line: fmtp} = codec
     new_fbs = Enum.map(fbs, &%RTCPFeedback{&1 | pt: new_pt})
@@ -515,7 +595,7 @@ defmodule ExWebRTC.PeerConnection.Configuration do
   def intersect_codecs(config, mline) do
     # we assume that this function is called after
     # the config was updated based on the remote SDP
-    # so the payload types should match
+    # so the payload types (in codec_equal?) should match
     codecs =
       case mline.type do
         :audio -> config.audio_codecs
@@ -526,13 +606,7 @@ defmodule ExWebRTC.PeerConnection.Configuration do
     |> SDPUtils.get_rtp_codec_parameters()
     |> Enum.flat_map(fn sdp_codec ->
       codecs
-      |> Enum.find(
-        # as of now, we ignore sdp_fmtp_line
-        &(String.downcase(&1.mime_type) == String.downcase(sdp_codec.mime_type) and
-            &1.payload_type == sdp_codec.payload_type and
-            &1.clock_rate == sdp_codec.clock_rate and
-            &1.channels == sdp_codec.channels)
-      )
+      |> Enum.find(&codec_equal?(&1, sdp_codec))
       |> case do
         nil ->
           []
@@ -544,6 +618,36 @@ defmodule ExWebRTC.PeerConnection.Configuration do
     end)
   end
 
+  # soft functions does not compare payload types
+  @doc false
+  @spec codec_equal?(RTPCodecParameters.t(), RTPCodecParameters.t()) :: boolean()
+  def codec_equal?(c1, c2) do
+    String.downcase(c1.mime_type) == String.downcase(c2.mime_type) and
+      c1.payload_type == c2.payload_type and
+      c1.clock_rate == c2.clock_rate and
+      c1.channels == c2.channels and fmtp_equal?(c1, c2)
+  end
+
+  defp codec_equal_soft?(c1, c2) do
+    String.downcase(c1.mime_type) == String.downcase(c2.mime_type) and
+      c1.clock_rate == c2.clock_rate and
+      c1.channels == c2.channels and fmtp_equal_soft?(c1, c2)
+  end
+
+  defp fmtp_equal?(%{sdp_fmtp_line: nil}, _c2), do: true
+  defp fmtp_equal?(_c1, %{sdp_fmtp_line: nil}), do: true
+  defp fmtp_equal?(c1, c2), do: c1.sdp_fmtp_line == c2.sdp_fmtp_line
+
+  defp fmtp_equal_soft?(%{sdp_fmtp_line: nil}, _c2), do: true
+  defp fmtp_equal_soft?(_c1, %{sdp_fmtp_line: nil}), do: true
+
+  defp fmtp_equal_soft?(c1, c2) do
+    fmtp1 = %{c1.sdp_fmtp_line | pt: nil}
+    fmtp2 = %{c2.sdp_fmtp_line | pt: nil}
+
+    fmtp1 == fmtp2
+  end
+
   @doc false
   @spec intersect_extensions(t(), ExSDP.Media.t()) :: [Extmap.t()]
   def intersect_extensions(config, mline) do