diff --git a/lib/ex_webrtc/rtp/av1/leb128.ex b/lib/ex_webrtc/rtp/av1/leb128.ex new file mode 100644 index 00000000..c123113b --- /dev/null +++ b/lib/ex_webrtc/rtp/av1/leb128.ex @@ -0,0 +1,36 @@ +defmodule ExWebRTC.RTP.AV1.LEB128 do + @moduledoc false + # Utilities for handling unsigned Little Endian Base 128 integers + + import Bitwise + + # see https://chromium.googlesource.com/external/webrtc/+/HEAD/modules/rtp_rtcp/source/rtp_packetizer_av1.cc#61 + @spec encode(non_neg_integer(), [bitstring()]) :: binary() + def encode(value, acc \\ []) + + def encode(value, acc) when value < 0x80 do + for group <- Enum.reverse([value | acc]), into: <<>> do + <> + end + end + + def encode(value, acc) do + group = 0x80 ||| (value &&& 0x7F) + encode(value >>> 7, [group | acc]) + end + + # see https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/rtc_base/byte_buffer.cc;drc=8e78783dc1f7007bad46d657c9f332614e240fd8;l=107 + @spec read(binary(), non_neg_integer(), non_neg_integer(), non_neg_integer()) :: + {:ok, pos_integer(), non_neg_integer()} | {:error, :invalid_leb128_data} + def read(data, read_bits \\ 0, leb128_size \\ 0, value \\ 0) + + def read(<<0::1, group::7, _rest::binary>>, read_bits, leb128_size, value) do + {:ok, leb128_size + 1, value ||| group <<< read_bits} + end + + def read(<<1::1, group::7, rest::binary>>, read_bits, leb128_size, value) do + read(rest, read_bits + 7, leb128_size + 1, value ||| group <<< read_bits) + end + + def read(_, _, _, _), do: {:error, :invalid_leb128_data} +end diff --git a/lib/ex_webrtc/rtp/av1/obu.ex b/lib/ex_webrtc/rtp/av1/obu.ex new file mode 100644 index 00000000..037ef570 --- /dev/null +++ b/lib/ex_webrtc/rtp/av1/obu.ex @@ -0,0 +1,144 @@ +defmodule ExWebRTC.RTP.AV1.OBU do + @moduledoc false + # Defines the Open Bitstream Unit, the base packetization unit of all structures present in the AV1 bitstream. + # + # Based on [the AV1 spec](https://aomediacodec.github.io/av1-spec/av1-spec.pdf). + # + # OBU syntax: + # 0 1 2 3 4 5 6 7 + # +-+-+-+-+-+-+-+-+ + # |0| type |X|S|-| (REQUIRED) + # +-+-+-+-+-+-+-+-+ + # X: | TID |SID|-|-|-| (OPTIONAL) + # +-+-+-+-+-+-+-+-+ + # |1| | + # +-+ OBU payload | + # S: |1| | (OPTIONAL, variable length leb128 encoded) + # +-+ size | + # |0| | + # +-+-+-+-+-+-+-+-+ + # | OBU payload | + # | ... | + + alias ExWebRTC.RTP.AV1.LEB128 + + @obu_sequence_header 1 + @obu_temporal_delimiter 2 + @obu_padding 15 + + @type t :: %__MODULE__{ + type: 0..15, + x: 0 | 1, + s: 0 | 1, + tid: 0..7 | nil, + sid: 0..3 | nil, + payload: binary() + } + + @enforce_keys [:type, :x, :s, :payload] + defstruct @enforce_keys ++ [:tid, :sid] + + @doc """ + Parses the low overhead bitstream format defined in AV1 spec section 5.2. + On success, returns the parsed OBU as well as the remainder of the AV1 bitstream. + """ + @spec parse(binary()) :: {:ok, t(), binary()} | {:error, :invalid_av1_bitstream} + def parse(av1_bitstream_binary) + + def parse(<<0::1, type::4, x::1, s::1, 0::1, rest::binary>>) do + with {:ok, tid, sid, rest} <- parse_extension_header(x, rest), + {:ok, payload, rest} <- parse_payload(s, rest), + :ok <- validate_payload(type, payload) do + {:ok, + %__MODULE__{ + type: type, + x: x, + s: s, + tid: tid, + sid: sid, + payload: payload + }, rest} + else + {:error, _} = err -> err + end + end + + def parse(_), do: {:error, :invalid_av1_bitstream} + + defp parse_extension_header(0, rest), do: {:ok, nil, nil, rest} + + defp parse_extension_header(1, <>), + do: {:ok, tid, sid, rest} + + defp parse_extension_header(_, _), do: {:error, :invalid_av1_bitstream} + + defp parse_payload(0, rest), do: {:ok, rest, <<>>} + + defp parse_payload(1, rest) do + with {:ok, leb128_size, payload_size} <- LEB128.read(rest), + <<_::binary-size(leb128_size), payload::binary-size(payload_size), rest::binary>> <- rest do + {:ok, payload, rest} + else + _ -> {:error, :invalid_av1_bitstream} + end + end + + defp validate_payload(@obu_padding, _), do: :ok + defp validate_payload(@obu_temporal_delimiter, <<>>), do: :ok + defp validate_payload(type, data) when type != @obu_temporal_delimiter and data != <<>>, do: :ok + defp validate_payload(_, _), do: {:error, :invalid_av1_bitstream} + + @spec serialize(t()) :: binary() + def serialize(%__MODULE__{type: type, x: x, s: s, payload: payload} = obu) do + obu_binary = + <<0::1, type::4, x::1, s::1, 0::1>> + |> add_extension_header(obu) + |> add_payload_size(obu) + + <> + end + + defp add_extension_header(obu_binary, %__MODULE__{x: 0, tid: nil, sid: nil}), do: obu_binary + + defp add_extension_header(obu_binary, %__MODULE__{x: 1, tid: tid, sid: sid}) + when tid != nil and sid != nil do + <> + end + + defp add_extension_header(_obu_binary, _invalid_obu), + do: raise("AV1 TID and SID must be set if, and only if X bit is set") + + defp add_payload_size(obu_binary, %__MODULE__{s: 0}), do: obu_binary + + defp add_payload_size(obu_binary, %__MODULE__{s: 1, payload: payload}) do + payload_size = payload |> byte_size() |> LEB128.encode() + <> + end + + @doc """ + Rewrites a specific case of the sequence header OBU to disable OBU dropping in the AV1 decoder + in accordance with av1-rtp-spec sec. 5. Leaves other OBUs unchanged. + """ + @spec disable_dropping_in_decoder_if_applicable(t()) :: t() + def disable_dropping_in_decoder_if_applicable(obu) + + # We're handling the following case: + # - still_picture = 0 + # - reduced_still_picture_header = 0 + # - timing_info_present_flag = 0 + # - operating_points_cnt_minus_1 = 0 + # - seq_level_idx[0] = 0 + # and setting operating_point_idc[0] = 0xFFF + # + # For the sequence header OBU syntax, refer to the AV1 spec sec. 5.5. + def disable_dropping_in_decoder_if_applicable( + %__MODULE__{ + type: @obu_sequence_header, + payload: <> + } = obu + ) do + %{obu | payload: <>} + end + + def disable_dropping_in_decoder_if_applicable(obu), do: obu +end diff --git a/lib/ex_webrtc/rtp/av1/payload.ex b/lib/ex_webrtc/rtp/av1/payload.ex new file mode 100644 index 00000000..af19867f --- /dev/null +++ b/lib/ex_webrtc/rtp/av1/payload.ex @@ -0,0 +1,96 @@ +defmodule ExWebRTC.RTP.AV1.Payload do + @moduledoc false + # Defines AV1 payload structure stored in RTP packet payload. + # + # Based on [RTP Payload Format for AV1](https://aomediacodec.github.io/av1-rtp-spec/v1.0.0.html). + # + # RTP payload syntax: + # 0 1 2 3 4 5 6 7 + # +-+-+-+-+-+-+-+-+ + # |Z|Y| W |N|-|-|-| (REQUIRED) + # +=+=+=+=+=+=+=+=+ (REPEATED W-1 times, or any times if W = 0) + # |1| | + # +-+ OBU fragment| + # |1| | (REQUIRED, leb128 encoded) + # +-+ size | + # |0| | + # +-+-+-+-+-+-+-+-+ + # | OBU fragment | + # | ... | + # +=+=+=+=+=+=+=+=+ + # | ... | + # +=+=+=+=+=+=+=+=+ if W > 0, last fragment MUST NOT have size field + # | OBU fragment | + # | ... | + # +=+=+=+=+=+=+=+=+ + + @type t :: %__MODULE__{ + z: 0 | 1, + y: 0 | 1, + w: 0 | 1 | 2 | 3, + n: 0 | 1, + payload: binary() + } + + @enforce_keys [:z, :y, :w, :n, :payload] + defstruct @enforce_keys ++ [] + + @doc """ + Parses RTP payload as AV1 payload. + """ + @spec parse(binary()) :: {:ok, t()} | {:error, :invalid_packet} + def parse(rtp_payload) + + def parse(<>) do + if payload == <<>> do + {:error, :invalid_packet} + else + {:ok, + %__MODULE__{ + z: z, + y: y, + w: w, + n: n, + payload: payload + }} + end + end + + def parse(_), do: {:error, :invalid_packet} + + @spec serialize(t()) :: binary() + def serialize(%__MODULE__{ + z: z, + y: y, + w: w, + n: n, + payload: payload + }) do + <> + end + + @doc """ + Payloads chunked fragments of single OBU and sets Z, Y bits. + """ + @spec payload_obu_fragments([binary()], 0 | 1) :: [t()] + def payload_obu_fragments(obu_fragments, n_bit \\ 0) + + def payload_obu_fragments([entire_obu], n_bit) do + [%__MODULE__{z: 0, y: 0, w: 1, n: n_bit, payload: entire_obu}] + end + + def payload_obu_fragments([first_obu_fragment | next_obu_fragments], n_bit) do + # First fragment of OBU: set Y bit + first_obu_payload = %__MODULE__{z: 0, y: 1, w: 1, n: n_bit, payload: first_obu_fragment} + + next_obu_payloads = + next_obu_fragments + # Middle fragments of OBU: set Z, Y bits + # av1-rtp-spec sec. 4.4: N is set only for the first packet of the coded video sequence. + |> Enum.map(&%__MODULE__{z: 1, y: 1, w: 1, n: 0, payload: &1}) + # Last fragment of OBU: set Z bit only (unset Y) + |> List.update_at(-1, &%{&1 | y: 0}) + + [first_obu_payload | next_obu_payloads] + end +end diff --git a/lib/ex_webrtc/rtp/av1/payloader.ex b/lib/ex_webrtc/rtp/av1/payloader.ex new file mode 100644 index 00000000..e386180d --- /dev/null +++ b/lib/ex_webrtc/rtp/av1/payloader.ex @@ -0,0 +1,78 @@ +defmodule ExWebRTC.RTP.Payloader.AV1 do + @moduledoc false + # Encapsulates AV1 video temporal units into RTP packets. + # + # Resources: + # * [RTP Payload Format for AV1 (av1-rtp-spec)](https://aomediacodec.github.io/av1-rtp-spec/v1.0.0.html) + # * [AV1 spec](https://aomediacodec.github.io/av1-spec/av1-spec.pdf). + # * https://norkin.org/research/av1_decoder_model/index.html + # * https://chromium.googlesource.com/external/webrtc/+/HEAD/modules/rtp_rtcp/source/video_rtp_depacketizer_av1.cc + + @behaviour ExWebRTC.RTP.Payloader.Behaviour + + alias ExWebRTC.RTP.AV1.{OBU, Payload} + alias ExWebRTC.Utils + + @obu_sequence_header 1 + @obu_temporal_delimiter 2 + + @aggregation_header_size_bytes 1 + + @type t :: %__MODULE__{ + max_payload_size: non_neg_integer() + } + + @enforce_keys [:max_payload_size] + defstruct @enforce_keys + + @impl true + def new(max_payload_size) when max_payload_size > 100 do + %__MODULE__{max_payload_size: max_payload_size} + end + + @impl true + def payload(payloader, temporal_unit) when temporal_unit != <<>> do + # In AV1, a temporal unit consists of all OBUs associated with a specific time instant. + # Temporal units always start with a temporal delimiter OBU. They may contain multiple AV1 frames. + # av1-rtp-spec sec. 5: The temporal delimiter OBU should be removed when transmitting. + obus = + case parse_obus(temporal_unit) do + [%OBU{type: @obu_temporal_delimiter} | next_obus] -> + next_obus + + _ -> + raise "Invalid AV1 temporal unit: does not start with temporal delimiter OBU" + end + + # With the current implementation, each RTP packet will contain one OBU element. + # This element can be an entire OBU, or a fragment of an OBU bigger than max_payload_size. + rtp_packets = + Stream.flat_map(obus, fn obu -> + n_bit = Utils.to_int(obu.type == @obu_sequence_header) + + obu + |> OBU.disable_dropping_in_decoder_if_applicable() + |> OBU.serialize() + |> Utils.chunk(payloader.max_payload_size - @aggregation_header_size_bytes) + |> Payload.payload_obu_fragments(n_bit) + end) + |> Stream.map(&Payload.serialize/1) + |> Enum.map(&ExRTP.Packet.new/1) + |> List.update_at(-1, &%{&1 | marker: true}) + + {rtp_packets, payloader} + end + + defp parse_obus(data, obus \\ []) + defp parse_obus(<<>>, obus), do: Enum.reverse(obus) + + defp parse_obus(data, obus) do + case OBU.parse(data) do + {:ok, obu, rest} -> + parse_obus(rest, [obu | obus]) + + {:error, :invalid_av1_bitstream} -> + raise "Invalid AV1 bitstream: unable to parse OBU" + end + end +end diff --git a/lib/ex_webrtc/rtp/payloader.ex b/lib/ex_webrtc/rtp/payloader.ex index 969ea3e2..fe7cf4d7 100644 --- a/lib/ex_webrtc/rtp/payloader.ex +++ b/lib/ex_webrtc/rtp/payloader.ex @@ -39,6 +39,7 @@ defmodule ExWebRTC.RTP.Payloader do defp to_payloader_module(mime_type) do case String.downcase(mime_type) do "video/vp8" -> {:ok, ExWebRTC.RTP.Payloader.VP8} + "video/av1" -> {:ok, ExWebRTC.RTP.Payloader.AV1} "audio/opus" -> {:ok, ExWebRTC.RTP.Payloader.Opus} "audio/pcma" -> {:ok, ExWebRTC.RTP.Payloader.G711} "audio/pcmu" -> {:ok, ExWebRTC.RTP.Payloader.G711} diff --git a/lib/ex_webrtc/rtp/vp8/payloader.ex b/lib/ex_webrtc/rtp/vp8/payloader.ex index bcf374ea..153516ff 100644 --- a/lib/ex_webrtc/rtp/vp8/payloader.ex +++ b/lib/ex_webrtc/rtp/vp8/payloader.ex @@ -9,6 +9,8 @@ defmodule ExWebRTC.RTP.Payloader.VP8 do @behaviour ExWebRTC.RTP.Payloader.Behaviour + alias ExWebRTC.Utils + @first_chunk_descriptor <<0::1, 0::1, 0::1, 1::1, 0::1, 0::3>> @next_chunk_descriptor <<0::1, 0::1, 0::1, 0::1, 0::1, 0::3>> @@ -29,7 +31,7 @@ defmodule ExWebRTC.RTP.Payloader.VP8 do @impl true def payload(%__MODULE__{} = payloader, frame) when frame != <<>> do - rtp_payloads = chunk(frame, payloader.max_payload_size - @desc_size_bytes) + rtp_payloads = Utils.chunk(frame, payloader.max_payload_size - @desc_size_bytes) [first_rtp_payload | next_rtp_payloads] = rtp_payloads @@ -45,17 +47,4 @@ defmodule ExWebRTC.RTP.Payloader.VP8 do {rtp_packets, payloader} end - - defp chunk(data, size, acc \\ []) - defp chunk(<<>>, _size, acc), do: Enum.reverse(acc) - - defp chunk(data, size, acc) do - case data do - <> -> - chunk(rest, size, [data | acc]) - - _other -> - chunk(<<>>, size, [data | acc]) - end - end end diff --git a/lib/ex_webrtc/utils.ex b/lib/ex_webrtc/utils.ex index 609e0cbb..ac86d542 100644 --- a/lib/ex_webrtc/utils.ex +++ b/lib/ex_webrtc/utils.ex @@ -24,4 +24,18 @@ defmodule ExWebRTC.Utils do def split_rtx_codecs(codecs) do Enum.split_with(codecs, &String.ends_with?(&1.mime_type, "/rtx")) end + + @spec chunk(binary(), pos_integer(), [binary()]) :: [binary()] + def chunk(data, size, acc \\ []) + def chunk(<<>>, _size, acc), do: Enum.reverse(acc) + + def chunk(data, size, acc) when size > 0 do + case data do + <> -> + chunk(rest, size, [data | acc]) + + _other -> + chunk(<<>>, size, [data | acc]) + end + end end diff --git a/test/ex_webrtc/rtp/av1/leb128_test.exs b/test/ex_webrtc/rtp/av1/leb128_test.exs new file mode 100644 index 00000000..e7f6ee16 --- /dev/null +++ b/test/ex_webrtc/rtp/av1/leb128_test.exs @@ -0,0 +1,22 @@ +defmodule ExWebrtc.RTP.AV1.LEB128Test do + use ExUnit.Case, async: true + + alias ExWebRTC.RTP.AV1.LEB128 + + test "encode" do + assert <<0>> == LEB128.encode(0) + assert <<5>> == LEB128.encode(5) + assert <<0xBF, 0x84, 0x3D>> == LEB128.encode(999_999) + end + + test "read" do + assert {:ok, 1, 0} == LEB128.read(<<0>>) + assert {:ok, 1, 5} == LEB128.read(<<5>>) + assert {:ok, 3, 999_999} == LEB128.read(<<0xBF, 0x84, 0x3D>>) + + assert {:ok, 3, 999_999} == LEB128.read(<<0xBF, 0x84, 0x3D, 0x00, 0x21, 0x37>>) + + assert {:error, :invalid_leb128_data} == LEB128.read(<<>>) + assert {:error, :invalid_leb128_data} == LEB128.read(<<255>>) + end +end diff --git a/test/ex_webrtc/rtp/av1/obu_test.exs b/test/ex_webrtc/rtp/av1/obu_test.exs new file mode 100644 index 00000000..27649aff --- /dev/null +++ b/test/ex_webrtc/rtp/av1/obu_test.exs @@ -0,0 +1,200 @@ +defmodule ExWebRTC.RTP.VP8.OBUTest do + use ExUnit.Case, async: true + + alias ExWebRTC.RTP.AV1.{LEB128, OBU} + + @obu_sequence_header 1 + @obu_temporal_delimiter 2 + @obu_padding 15 + + test "parse/1 and serialize/1" do + # test vectors are based on AV1 spec + + # random av1 data, not necessarily correct + obu_payload = <<0, 1, 2, 3, 42, 43, 44>> + + # Parse single OBU: type=3, X=0, S=0 + obu = obu_header(3, 0, 0) <> obu_payload + + parsed_obu = + %OBU{ + type: 3, + x: 0, + s: 0, + payload: obu_payload + } + + assert {:ok, parsed_obu, <<>>} == OBU.parse(obu) + assert obu == OBU.serialize(parsed_obu) + + # Parse single OBU: type=TD, X=1, TID=4, SID=1, S=0. TD must have empty payload + obu = obu_header(@obu_temporal_delimiter, 1, 0) <> <<4::3, 1::2, 0::3>> + + parsed_obu = + %OBU{ + type: @obu_temporal_delimiter, + x: 1, + s: 0, + tid: 4, + sid: 1, + payload: <<>> + } + + assert {:ok, parsed_obu, <<>>} == OBU.parse(obu) + assert obu == OBU.serialize(parsed_obu) + + # Parse AV1 low overhead bitstream: all OBUs must have S=1 + # OBU 1: type=TD, X=0 + obu_1 = obu_header(@obu_temporal_delimiter, 0, 1) <> LEB128.encode(0) + + parsed_obu_1 = + %OBU{ + type: @obu_temporal_delimiter, + x: 0, + s: 1, + payload: <<>> + } + + # OBU 2: type=SEQ_HDR, X=1, TID=0, SID=0 + obu_2 = + obu_header(@obu_sequence_header, 1, 1) <> + <<0::3, 0::2, 0::3>> <> LEB128.encode(byte_size(obu_payload)) <> obu_payload + + parsed_obu_2 = + %OBU{ + type: @obu_sequence_header, + x: 1, + s: 1, + tid: 0, + sid: 0, + payload: obu_payload + } + + # OBU 3: type=PAD, X=0 + obu_padding_payload = for _ <- 1..44, into: <<>>, do: <<0>> + + obu_3 = + obu_header(@obu_padding, 0, 1) <> + LEB128.encode(byte_size(obu_padding_payload)) <> obu_padding_payload + + parsed_obu_3 = + %OBU{ + type: @obu_padding, + x: 0, + s: 1, + payload: obu_padding_payload + } + + av1_bitstream = obu_1 <> obu_2 <> obu_3 + + assert {:ok, ^parsed_obu_1, av1_bitstream} = OBU.parse(av1_bitstream) + assert {:ok, ^parsed_obu_2, av1_bitstream} = OBU.parse(av1_bitstream) + assert {:ok, ^parsed_obu_3, <<>>} = OBU.parse(av1_bitstream) + + assert obu_1 == OBU.serialize(parsed_obu_1) + assert obu_2 == OBU.serialize(parsed_obu_2) + assert obu_3 == OBU.serialize(parsed_obu_3) + + # Errors + # Empty bitstream + assert {:error, :invalid_av1_bitstream} == OBU.parse(<<>>) + + # First bit set + assert {:error, :invalid_av1_bitstream} == OBU.parse(<<1::1, 0::7>>) + + # Last bit of first byte set + assert {:error, :invalid_av1_bitstream} == OBU.parse(<<0::7, 1::1>>) + + # X set but extension header absent + assert {:error, :invalid_av1_bitstream} == OBU.parse(obu_header(4, 1, 0)) + + # S set but no size + assert {:error, :invalid_av1_bitstream} == OBU.parse(obu_header(4, 0, 1)) + + # S set but invalid LEB128 data + assert {:error, :invalid_av1_bitstream} == OBU.parse(obu_header(4, 0, 1) <> <<255>>) + + # S set, size valid, but bitstream too short + assert {:error, :invalid_av1_bitstream} == + OBU.parse(obu_header(4, 0, 1) <> LEB128.encode(1234) <> <<9, 9, 7>>) + + # Temporal delimiter with payload + assert {:error, :invalid_av1_bitstream} == + OBU.parse(obu_header(@obu_temporal_delimiter, 0, 0) <> obu_payload) + + # OBU without payload that's neither TD nor padding + assert {:error, :invalid_av1_bitstream} == OBU.parse(obu_header(10, 0, 0)) + end + + test "disable_dropping_in_decoder_if_applicable/1" do + # Set op_idc_0 to 0xFFF in a specific case of the sequence header OBU + seq_profile = 5 + iddpf = 1 + op_idc_0 = 0xABC + rest = <<3::3, 42, 43, 44>> + + obu_to_modify = %OBU{ + type: @obu_sequence_header, + x: 0, + s: 0, + payload: dummy_sequence_header_obu(seq_profile, iddpf, 0, op_idc_0, 0, rest) + } + + modified_obu = %OBU{ + type: @obu_sequence_header, + x: 0, + s: 0, + payload: dummy_sequence_header_obu(seq_profile, iddpf, 0, 0xFFF, 0, rest) + } + + assert obu_to_modify != modified_obu + assert modified_obu == OBU.disable_dropping_in_decoder_if_applicable(obu_to_modify) + + # Don't touch other OBUa + obu_to_leave_unchanged = %OBU{ + type: @obu_sequence_header, + x: 0, + s: 0, + payload: dummy_sequence_header_obu(seq_profile, iddpf, 8, op_idc_0, 0, rest) + } + + assert obu_to_leave_unchanged == + OBU.disable_dropping_in_decoder_if_applicable(obu_to_leave_unchanged) + + obu_to_leave_unchanged = %OBU{ + type: @obu_sequence_header, + x: 0, + s: 0, + payload: dummy_sequence_header_obu(seq_profile, iddpf, 0, op_idc_0, 8, rest) + } + + assert obu_to_leave_unchanged == + OBU.disable_dropping_in_decoder_if_applicable(obu_to_leave_unchanged) + + obu_to_leave_unchanged = %OBU{ + type: @obu_temporal_delimiter, + x: 0, + s: 0, + payload: <<>> + } + + assert obu_to_leave_unchanged == + OBU.disable_dropping_in_decoder_if_applicable(obu_to_leave_unchanged) + end + + defp obu_header(type, x, s), do: <<0::1, type::4, x::1, s::1, 0::1>> + + defp dummy_sequence_header_obu( + seq_profile, + iddpf, + op_cnt_minus_1, + op_idc_0, + seq_level_idx_0, + rest + ) do + # still_picture=0, reduced_still_picture_header=0 (always 0 for video) + # timing_info_present_flag=0 (SEQ_HDR OBU has simpler structure without timing info) + <> + end +end diff --git a/test/ex_webrtc/rtp/av1/payload_test.exs b/test/ex_webrtc/rtp/av1/payload_test.exs new file mode 100644 index 00000000..62d227b6 --- /dev/null +++ b/test/ex_webrtc/rtp/av1/payload_test.exs @@ -0,0 +1,139 @@ +defmodule ExWebRTC.RTP.AV1.PayloadTest do + use ExUnit.Case, async: true + + alias ExWebRTC.RTP.AV1.Payload + alias ExWebRTC.Utils + + test "parse/1 and serialize/1" do + # test vectors are based on [RTP Payload Format for AV1](https://aomediacodec.github.io/av1-rtp-spec/v1.0.0.html) + + # random av1 data, not necessarily correct + av1_payload = <<0, 1, 2, 3>> + + # Z=0, Y=0, W=1, N=1 + rtp_payload = + <<0::1, 0::1, 1::2, 1::1, 0::3, av1_payload::binary>> + + parsed_payload = + %Payload{ + z: 0, + y: 0, + w: 1, + n: 1, + payload: av1_payload + } + + assert {:ok, parsed_payload} == Payload.parse(rtp_payload) + assert rtp_payload == Payload.serialize(parsed_payload) + + # Z=1, Y=0, W=3, N=0 + rtp_payload = + <<1::1, 0::1, 3::2, 0::1, 0::3, av1_payload::binary>> + + parsed_payload = + %Payload{ + z: 1, + y: 0, + w: 3, + n: 0, + payload: av1_payload + } + + assert {:ok, parsed_payload} == Payload.parse(rtp_payload) + assert rtp_payload == Payload.serialize(parsed_payload) + + assert {:error, :invalid_packet} = Payload.parse(<<>>) + + # No av1_payload + assert {:error, :invalid_packet} = Payload.parse(<<1::1, 1::1, 1::2, 0::1, 0::3>>) + end + + test "payload_obu_fragments/2" do + obu = + for i <- 0..9001, into: <<>> do + <> + end + + # Chunk size greater than OBU size, OBU not split + chunked_obu = Utils.chunk(obu, 10_000) + assert length(chunked_obu) == 1 + + # N=0. Expecting single RTP packet, Z=0, Y=0 + [obu_payload] = Payload.payload_obu_fragments(chunked_obu, 0) + + assert %Payload{ + z: 0, + y: 0, + w: 1, + n: 0, + payload: ^obu + } = obu_payload + + # OBU split in two + chunked_obu = Utils.chunk(obu, 5_000) + assert length(chunked_obu) == 2 + + # N=1. Expecting two RTP packets, first with Z=0, Y=1, N=1, second with Z=1, Y=0, N=0 + [obu_payload_1, obu_payload_2] = Payload.payload_obu_fragments(chunked_obu, 1) + + assert %Payload{ + z: 0, + y: 1, + w: 1, + n: 1, + payload: obu_chunk_1 + } = obu_payload_1 + + assert %Payload{ + z: 1, + y: 0, + w: 1, + n: 0, + payload: obu_chunk_2 + } = obu_payload_2 + + assert obu_chunk_1 <> obu_chunk_2 == obu + + # OBU split into more than two chunks + chunked_obu = Utils.chunk(obu, 100) + assert length(chunked_obu) > 2 + + # N=0. Expecting the RTP packets in the middle to have Z=1, Y=1 + obu_payloads = Payload.payload_obu_fragments(chunked_obu) + assert length(chunked_obu) == length(obu_payloads) + + [first_obu_payload | next_obu_payloads] = obu_payloads + {last_obu_payload, middle_obu_payloads} = List.pop_at(next_obu_payloads, -1) + + assert %Payload{ + z: 0, + y: 1, + w: 1, + n: 0, + payload: first_obu_chunk + } = first_obu_payload + + assert %Payload{ + z: 1, + y: 0, + w: 1, + n: 0, + payload: last_obu_chunk + } = last_obu_payload + + middle_obu_chunks = + for payload <- middle_obu_payloads, into: <<>> do + assert %Payload{ + z: 1, + y: 1, + w: 1, + n: 0, + payload: obu_chunk + } = payload + + obu_chunk + end + + assert first_obu_chunk <> middle_obu_chunks <> last_obu_chunk == obu + end +end diff --git a/test/ex_webrtc/rtp/av1/payloader_test.exs b/test/ex_webrtc/rtp/av1/payloader_test.exs new file mode 100644 index 00000000..5890fd68 --- /dev/null +++ b/test/ex_webrtc/rtp/av1/payloader_test.exs @@ -0,0 +1,29 @@ +defmodule ExWebRTC.RTP.AV1.PayloaderTest do + use ExUnit.Case, async: true + + alias ExWebRTC.Media.IVF.Reader + alias ExWebRTC.RTP.Payloader + + test "payload av1 video" do + # some OBUs in the fixture are bigger than 101 bytes + av1_payloader = Payloader.AV1.new(101) + {:ok, _header, ivf_reader} = Reader.open("test/fixtures/ivf/av1_correct.ivf") + + for _i <- 0..28, reduce: av1_payloader do + av1_payloader -> + {:ok, frame} = Reader.next_frame(ivf_reader) + {rtp_packets, av1_payloader} = Payloader.AV1.payload(av1_payloader, frame.data) + + # assert all packets are no bigger than 101 bytes + rtp_packets + |> Enum.each(fn rtp_packet -> + assert byte_size(rtp_packet.payload) <= 101 + end) + + last_rtp = List.last(rtp_packets) + assert last_rtp.marker == true + + av1_payloader + end + end +end diff --git a/test/ex_webrtc/rtp/payloader_test.exs b/test/ex_webrtc/rtp/payloader_test.exs index f41bbc01..6fafec3c 100644 --- a/test/ex_webrtc/rtp/payloader_test.exs +++ b/test/ex_webrtc/rtp/payloader_test.exs @@ -5,6 +5,7 @@ defmodule ExWebRTC.RTP.PayloaderTest do alias ExWebRTC.RTP.Payloader @frame <<0, 1, 2, 3>> + @av1_temporal_unit <<0::1, 2::4, 0::3>> test "creates a VP8 payloader and dispatches calls to its module" do assert {:ok, _payloader} = @@ -19,6 +20,18 @@ defmodule ExWebRTC.RTP.PayloaderTest do assert Payloader.payload(payloader, @frame) == Payloader.VP8.payload(payloader, @frame) end + test "creates an AV1 payloader and dispatches calls to its module" do + assert {:ok, payloader} = + %RTPCodecParameters{payload_type: 45, mime_type: "video/AV1", clock_rate: 90_000} + |> Payloader.new() + + assert Payloader.payload(payloader, @av1_temporal_unit) == + Payloader.AV1.payload(payloader, @av1_temporal_unit) + + # The sample frame is not a valid AV1 temporal unit + assert_raise RuntimeError, fn -> Payloader.payload(payloader, @frame) end + end + test "creates an Opus payloader and dispatches calls to its module" do assert {:ok, payloader} = %RTPCodecParameters{ diff --git a/test/ex_webrtc/utils_test.exs b/test/ex_webrtc/utils_test.exs new file mode 100644 index 00000000..ca6ae280 --- /dev/null +++ b/test/ex_webrtc/utils_test.exs @@ -0,0 +1,17 @@ +defmodule ExWebRTC.UtilsTest do + use ExUnit.Case, async: true + + alias ExWebRTC.Utils + + test "chunk/2" do + data = <<0, 1, 2, 3, 4, 5, 6, 7>> + assert [data] == Utils.chunk(data, 100) + assert [<<0, 1, 2>>, <<3, 4, 5>>, <<6, 7>>] = Utils.chunk(data, 3) + assert [<<0, 1>>, <<2, 3>>, <<4, 5>>, <<6, 7>>] == Utils.chunk(data, 2) + + assert_raise FunctionClauseError, fn -> Utils.chunk(data, 0) end + assert_raise FunctionClauseError, fn -> Utils.chunk(data, -22) end + + assert [] == Utils.chunk(<<>>, 100) + end +end diff --git a/test/fixtures/ivf/README.md b/test/fixtures/ivf/README.md index 50e162b3..8a8bc918 100644 --- a/test/fixtures/ivf/README.md +++ b/test/fixtures/ivf/README.md @@ -3,3 +3,4 @@ * empty - just an empty file * vp8_correct - https://chromium.googlesource.com/webm/vp8-test-vectors/+/refs/heads/main/vp80-00-comprehensive-001.ivf * vp8_invalid_last_frame - vp8_correct without last byte +* av1_correct - vp8_correct converted using ffmpeg (libsvtav1) diff --git a/test/fixtures/ivf/av1_correct.ivf b/test/fixtures/ivf/av1_correct.ivf new file mode 100644 index 00000000..6c1ee043 Binary files /dev/null and b/test/fixtures/ivf/av1_correct.ivf differ