SM
Software Mansion•8mo ago
Aske

Dynamically starting children to Demux Mp4 tracks

I want to convert this to take arbitrary user uploaded Mp4 files where the tracks can have different indexes: 
    structure = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM)
      # Video
      |> via_out(Pad.ref(:output, 1))
      |> child(:vido_tmp, %Membrane.H264.Parser{
        output_stream_structure: :annexb
      })
      |> do_video()
      |> child(:video_out, %Membrane.H264.Parser{
        generate_best_effort_timestamps: %{framerate: {25, 1}},
        output_stream_structure: :avc1
      }),
      # Audio
      get_child(:demuxer)
      |> via_out(Pad.ref(:output, 2))
      |> do_audio(),
      # Mux
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})
      get_child(:video_out) |> get_child(:muxer),
      get_child(:audio_out) |> get_child(:muxer)
    ]
    structure = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM)
      # Video
      |> via_out(Pad.ref(:output, 1))
      |> child(:vido_tmp, %Membrane.H264.Parser{
        output_stream_structure: :annexb
      })
      |> do_video()
      |> child(:video_out, %Membrane.H264.Parser{
        generate_best_effort_timestamps: %{framerate: {25, 1}},
        output_stream_structure: :avc1
      }),
      # Audio
      get_child(:demuxer)
      |> via_out(Pad.ref(:output, 2))
      |> do_audio(),
      # Mux
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})
      get_child(:video_out) |> get_child(:muxer),
      get_child(:audio_out) |> get_child(:muxer)
    ]
I tried converting the above to: 
  @impl true
  def handle_init(_ctx, _path) do
    structure = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM),
      # Mux
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})
    ]

    {[spec: structure], %{}}
  end

  @impl true
  def handle_child_notification({:new_tracks, tracks}, :demuxer, _ctx, state) do
    spec =
      tracks
      |> Enum.map(fn
        {track_id, %Membrane.H264{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_video()

        {track_id, %Membrane.AAC{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_audio()
      end)

    {[spec: spec], state}
  end
  @impl true
  def handle_init(_ctx, _path) do
    structure = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM),
      # Mux
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})
    ]

    {[spec: structure], %{}}
  end

  @impl true
  def handle_child_notification({:new_tracks, tracks}, :demuxer, _ctx, state) do
    spec =
      tracks
      |> Enum.map(fn
        {track_id, %Membrane.H264{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_video()

        {track_id, %Membrane.AAC{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_audio()
      end)

    {[spec: spec], state}
  end
but now the demuxer doesn't run, because there are no outputs to pull data.. How do I add dynamic children, based on the track types?
1 Reply
Aske
AskeOP•8mo ago
I get this error: 
[error] GenServer #PID<0.2412.0> terminating
** (ArgumentError) errors were found at the given arguments:

  * 1st argument: not a nonempty list

    :erlang.hd([])
    (membrane_mp4_plugin 0.34.1) lib/membrane_mp4/muxer/isom.ex:130: Membrane.MP4.Muxer.ISOM.handle_demand/5
    (membrane_core 1.0.1) lib/membrane/core/callback_handler.ex:139: Membrane.Core.CallbackHandler.exec_callback/4
[error] GenServer #PID<0.2412.0> terminating
** (ArgumentError) errors were found at the given arguments:

  * 1st argument: not a nonempty list

    :erlang.hd([])
    (membrane_mp4_plugin 0.34.1) lib/membrane_mp4/muxer/isom.ex:130: Membrane.MP4.Muxer.ISOM.handle_demand/5
    (membrane_core 1.0.1) lib/membrane/core/callback_handler.ex:139: Membrane.Core.CallbackHandler.exec_callback/4
I guess I could attempt to pick apart the output of: 
iex> File.stream!(path, [], 2048) |> Enum.take(50) |> IO.iodata_to_binary() |> Membrane.MP4.Container.parse()
{:ok,
 [
   ftyp: %{
     children: [],
     fields: %{
       compatible_brands: ["isom", "iso2", "avc1", "mp41"],
       major_brand: "isom",
       major_brand_version: 512
     }
   },
   moov: %{
     children: [
       mvhd: %{
         children: [],
         fields: %{
           creation_time: 0,
           duration: 87872,
           flags: 0,
           matrix_value_A: {1, 0},
iex> File.stream!(path, [], 2048) |> Enum.take(50) |> IO.iodata_to_binary() |> Membrane.MP4.Container.parse()
{:ok,
 [
   ftyp: %{
     children: [],
     fields: %{
       compatible_brands: ["isom", "iso2", "avc1", "mp41"],
       major_brand: "isom",
       major_brand_version: 512
     }
   },
   moov: %{
     children: [
       mvhd: %{
         children: [],
         fields: %{
           creation_time: 0,
           duration: 87872,
           flags: 0,
           matrix_value_A: {1, 0},
but not sure that's a good approach. Ah.. I see another person posted a very similar question here, with no solution: https://discord.com/channels/464786597288738816/1088162780537954406/1088162780537954406 Okay figured it out now: 
  @impl true
  def handle_init(_ctx, _path) do
    spec = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM)
    ]

    {[spec: spec], %{}}
  end
  @impl true
  def handle_init(_ctx, _path) do
    spec = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM)
    ]

    {[spec: spec], %{}}
  end
  @impl true
  def handle_child_notification({:new_tracks, tracks}, :demuxer, _ctx, state) do
    spec =
      tracks
      |> Enum.flat_map(fn
        {track_id, %Membrane.H264{}} ->
          vid =
            get_child(:demuxer)
            |> via_out(Pad.ref(:output, track_id))
            |> do_video()

          [vid]

        {track_id, %Membrane.AAC{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_audio()
      end)

    # Mux to MP4
    muxer =
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})

    {[spec: [muxer | spec]], state}
  end
  @impl true
  def handle_child_notification({:new_tracks, tracks}, :demuxer, _ctx, state) do
    spec =
      tracks
      |> Enum.flat_map(fn
        {track_id, %Membrane.H264{}} ->
          vid =
            get_child(:demuxer)
            |> via_out(Pad.ref(:output, track_id))
            |> do_video()

          [vid]

        {track_id, %Membrane.AAC{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_audio()
      end)

    # Mux to MP4
    muxer =
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})

    {[spec: [muxer | spec]], state}
  end
Want results from more Discord servers?
Add your server