odingrail Posts - Answer Overflow

odingrail

•Created by odingrail on 7/30/2024 in #membrane-help

Retransmit received RTP packets in secure way

Hello! We are working on SFU at the moment and we want to receive RTP packet from one peer, and broadcast it to multiple "listeners". We doing following in the code (we do not match to track_id because there is only video we experimenting with):

  @impl true
  def handle_info(
        {:ex_webrtc, pid, {:rtp, _track_id, _, rtp_packet}},
        state
      ) do
    Enum.each(state.peer_connections, fn
      {_peer_id, %ConnectionInfo{connected: true, track_id: nil}} ->
        Logger.warning("No track ID found for peer connection")

        :ok

      {_peer_id, %ConnectionInfo{connected: true, track_id: track_id} = connection_info} ->
        PeerConnection.send_rtp(
          connection_info.peer_connection,
          connection_info.track_id,
          rtp_packet
        )

      connection_info ->
        :ok
    end)

    {:noreply, state}
  end

  @impl true
  def handle_info(
        {:ex_webrtc, pid, {:rtp, _track_id, _, rtp_packet}},
        state
      ) do
    Enum.each(state.peer_connections, fn
      {_peer_id, %ConnectionInfo{connected: true, track_id: nil}} ->
        Logger.warning("No track ID found for peer connection")

        :ok

      {_peer_id, %ConnectionInfo{connected: true, track_id: track_id} = connection_info} ->
        PeerConnection.send_rtp(
          connection_info.peer_connection,
          connection_info.track_id,
          rtp_packet
        )

      connection_info ->
        :ok
    end)

    {:noreply, state}
  end

However, this approach is clearly break the cryptography part of the transmission. So what do we have to do? The first thought was to use the ExLibSRTP to unprotect and then protect the packet for each peer, but we don't have a direct access to DTLS transport as it's a private part of the PeerConnection.

15 replies

SMSoftware Mansion

•Created by odingrail on 7/23/2024 in #membrane-help

membrane_webrtc_plugin: %Membrane.Buffer with pts: nil, dts: nil received from audio track.

Hello, and thank you for the great ecosystem of libraries! 🙏 Currently, I'm working on SFU, which will utilize the membrane_webrtc_plugin to connect the streamer and the viewers. Everything is working pretty fine with video track. However, when I'm adding the audio one I'm starting to receive a bunch of errors, namely: 1) ArgumentError from membrane_realtimer_plugin, handle_buffer/4 function where it essentially tries to do subtraction from nil:

interval = Buffer.get_dts_or_pts(buffer) - state.previous_timestamp

interval = Buffer.get_dts_or_pts(buffer) - state.previous_timestamp

2) membrane_webrtc_plugin itself in sink.ex, handle_buffer fails for the exact reason, when the both dts and pts are nil. So my question is: is it a bug, or I'm doing something wrong with the pipeline? When I apply fixes to both problems the pipeline seems to be working fine. The example of the audio spec:

[
  child(:webrtc_sink, %WebRTC.Sink{
    signaling: %Membrane.WebRTC.SignalingChannel{pid: signaling_channel},
    tracks: [:audio, :video],
    video_codec: :h264
  }),
  get_child(:source_bin_audio_tee)
  |> child(:audio_parser, %Membrane.AAC.Parser{
    out_encapsulation: :ADTS
  })
  |> child(:aac_decoder, Membrane.AAC.FDK.Decoder)
  |> child(:converter, %Membrane.FFmpeg.SWResample.Converter{
    output_stream_format: %Membrane.RawAudio{
      sample_format: @audio_sample_format,
      sample_rate: @audio_sample_rate,
      channels: 2
    }
  })
  |> child(:encoder, %Membrane.Opus.Encoder{
    application: :audio,
    input_stream_format: %Membrane.RawAudio{
      channels: 2,
      sample_format: @audio_sample_format,
      sample_rate: @audio_sample_rate
    }
  })
  |> child(:parser, %Membrane.Opus.Parser{delimitation: :keep})
  |> get_child(:audio_tee)
]

# Then eventually
spec ++ [
  get_child(:audio_tee)
  |> via_in(Pad.ref(:input, :audio_track), options: [kind: :audio])
  |> get_child(:webrtc_sink)
]

[
  child(:webrtc_sink, %WebRTC.Sink{
    signaling: %Membrane.WebRTC.SignalingChannel{pid: signaling_channel},
    tracks: [:audio, :video],
    video_codec: :h264
  }),
  get_child(:source_bin_audio_tee)
  |> child(:audio_parser, %Membrane.AAC.Parser{
    out_encapsulation: :ADTS
  })
  |> child(:aac_decoder, Membrane.AAC.FDK.Decoder)
  |> child(:converter, %Membrane.FFmpeg.SWResample.Converter{
    output_stream_format: %Membrane.RawAudio{
      sample_format: @audio_sample_format,
      sample_rate: @audio_sample_rate,
      channels: 2
    }
  })
  |> child(:encoder, %Membrane.Opus.Encoder{
    application: :audio,
    input_stream_format: %Membrane.RawAudio{
      channels: 2,
      sample_format: @audio_sample_format,
      sample_rate: @audio_sample_rate
    }
  })
  |> child(:parser, %Membrane.Opus.Parser{delimitation: :keep})
  |> get_child(:audio_tee)
]

# Then eventually
spec ++ [
  get_child(:audio_tee)
  |> via_in(Pad.ref(:input, :audio_track), options: [kind: :audio])
  |> get_child(:webrtc_sink)
]

Looking forward to submit my local fixes if it's relevant. Otherwise hope to hear from you on that matter!

6 replies

Gaming

Programming