discord.js - Imagine an app•12mo ago

piping from udp stream into an audio resource

hello! i've been trying to write a bot to pipe desktop audio to a discord bot over udp (i use linux so this is my jank workaround to not having screen share audio), and i've settled on using dgram to open a udp socket and running ffmpeg locally to push to it (the full command is ffmpeg -f pulse -i default -f opus "udp://127.0.0.1:3756", and the local ip is fine for now since both the stream and bot are running on the same machine).

7 Replies

d.js toolkit•12mo ago

- What's your exact discord.js npm list discord.js and node node -v version? - Not a discord.js issue? Check out #other-js-ts. - Consider reading #how-to-get-help to improve your question! - Explain what exactly your issue is. - Post the full error stack trace, not just the top part! - Show your code! - Issue solved? Press the button!

boppleoppleOP•12mo ago

other than that, here's the relevant code: udpServer.js

import { createSocket } from "dgram";

const PORT_RANGE = [3700, 3799];

let serverLUT = {};

class ServerLUTEntry {

    constructor(server, port, buffer, stream){
        this.server = server;
        this.port = port;
        this.buffer = buffer;
        this.stream = stream;
    }
}
/**
 * 
 * @param {String} id 
 * @returns {ReadableStream | null}
 */
export function getStream(id) {
    if (Object.keys(serverLUT).includes(id)) {
        return serverLUT[id].stream;
    } else {
        return null;
    }
}

export function getServer(id, listening = defaultServerListening, message = defaultServerMessage, error = defaultServerError) {
    if (Object.keys(serverLUT).includes(id)) {
        return serverLUT[id].server;
    } else {
        if (serverLUT.length > PORT_RANGE[1] - PORT_RANGE[0]) {
            console.error("no empty ports");
            return null;
        }
        const server = createSocket("udp4");
        const port = pseudoHash(id);
        const buffer = Buffer.alloc(1476);

        serverLUT[id] = new ServerLUTEntry(server, port, buffer, new ReadableStream({
            pull: controller => {
                controller.enqueue(buffer);
            }
        }));

        server.bind(port);
        
        server.on("error", error);
        server.on("message", message);
        server.on("listening", listening);

        server.on("message", (msg, info) => {
            serverLUT[id].buffer.write(msg.toString());
        });
        
        return server;
    }
}

function pseudoHash(id) {
    let base_hash = Number.parseInt(id) % (1 + PORT_RANGE[1] - PORT_RANGE[0]) + PORT_RANGE[0];

    while (Object.values(serverLUT).includes(base_hash)) {
        base_hash = (base_hash + 1) % (1 + PORT_RANGE[1] - PORT_RANGE[0]);
    }

    return base_hash;
}

import { createSocket } from "dgram";

const PORT_RANGE = [3700, 3799];

let serverLUT = {};

class ServerLUTEntry {

    constructor(server, port, buffer, stream){
        this.server = server;
        this.port = port;
        this.buffer = buffer;
        this.stream = stream;
    }
}
/**
 * 
 * @param {String} id 
 * @returns {ReadableStream | null}
 */
export function getStream(id) {
    if (Object.keys(serverLUT).includes(id)) {
        return serverLUT[id].stream;
    } else {
        return null;
    }
}

export function getServer(id, listening = defaultServerListening, message = defaultServerMessage, error = defaultServerError) {
    if (Object.keys(serverLUT).includes(id)) {
        return serverLUT[id].server;
    } else {
        if (serverLUT.length > PORT_RANGE[1] - PORT_RANGE[0]) {
            console.error("no empty ports");
            return null;
        }
        const server = createSocket("udp4");
        const port = pseudoHash(id);
        const buffer = Buffer.alloc(1476);

        serverLUT[id] = new ServerLUTEntry(server, port, buffer, new ReadableStream({
            pull: controller => {
                controller.enqueue(buffer);
            }
        }));

        server.bind(port);
        
        server.on("error", error);
        server.on("message", message);
        server.on("listening", listening);

        server.on("message", (msg, info) => {
            serverLUT[id].buffer.write(msg.toString());
        });
        
        return server;
    }
}

function pseudoHash(id) {
    let base_hash = Number.parseInt(id) % (1 + PORT_RANGE[1] - PORT_RANGE[0]) + PORT_RANGE[0];

    while (Object.values(serverLUT).includes(base_hash)) {
        base_hash = (base_hash + 1) % (1 + PORT_RANGE[1] - PORT_RANGE[0]);
    }

    return base_hash;
}

app.js

async function share(interaction) {
  const { guild, member } = interaction;
  const connection = getVoiceConnection(guild.id);

  if (!connection) {
    interaction.reply("add tux to a vc first");
    return;
  }

  await interaction.deferReply();

  const server = getServer(guild.id, async () => {

    const stream = getStream(guild.id);

    const resource = createAudioResource(stream, {
      inputType: StreamType.Opus
    });

    const player = createAudioPlayer();

    connection.subscribe(player);
    server.once("message", () => {
      player.play(resource);
    });
    
    player.on("debug", console.log);

    await interaction.editReply(`server listening on udp://${process.env.HOST_IP}:${server.address().port}`);
  });
}

async function share(interaction) {
  const { guild, member } = interaction;
  const connection = getVoiceConnection(guild.id);

  if (!connection) {
    interaction.reply("add tux to a vc first");
    return;
  }

  await interaction.deferReply();

  const server = getServer(guild.id, async () => {

    const stream = getStream(guild.id);

    const resource = createAudioResource(stream, {
      inputType: StreamType.Opus
    });

    const player = createAudioPlayer();

    connection.subscribe(player);
    server.once("message", () => {
      player.play(resource);
    });
    
    player.on("debug", console.log);

    await interaction.editReply(`server listening on udp://${process.env.HOST_IP}:${server.address().port}`);
  });
}

sorry for multiple messages, discord doesn't like a lot of text :/ and if it would help to condense the code, i'll edit the above bits i should also probably mention that i've tried returning a promise containing the received udp packets directly to the ReadableStream's pull function instead of the buffer jank, but this was just the last iteration of code before i decided to ask here. the current problem with the code is that it throws a typeerror on a method in the audioresource constructor when i try to set the inputType to opus:

file:///home/bopple/Desktop/Code/linux-sharing-bot/node_modules/@discordjs/voice/dist/index.mjs:2319
    this.playStream.once("readable", () => this.started = true);
                    ^

TypeError: this.playStream.once is not a function
    at new AudioResource (file:///home/bopple/Desktop/Code/linux-sharing-bot/node_modules/@discordjs/voice/dist/index.mjs:2319:21)
    at createAudioResource (file:///home/bopple/Desktop/Code/linux-sharing-bot/node_modules/@discordjs/voice/dist/index.mjs:2397:12)
    at Socket.<anonymous> (file:///home/bopple/Desktop/Code/linux-sharing-bot/app.js:83:22)
    at Socket.emit (node:events:519:28)
    at startListening (node:dgram:183:10)
    at node:dgram:371:7
    at process.processTicksAndRejections (node:internal/process/task_queues:83:21)

file:///home/bopple/Desktop/Code/linux-sharing-bot/node_modules/@discordjs/voice/dist/index.mjs:2319
    this.playStream.once("readable", () => this.started = true);
                    ^

TypeError: this.playStream.once is not a function
    at new AudioResource (file:///home/bopple/Desktop/Code/linux-sharing-bot/node_modules/@discordjs/voice/dist/index.mjs:2319:21)
    at createAudioResource (file:///home/bopple/Desktop/Code/linux-sharing-bot/node_modules/@discordjs/voice/dist/index.mjs:2397:12)
    at Socket.<anonymous> (file:///home/bopple/Desktop/Code/linux-sharing-bot/app.js:83:22)
    at Socket.emit (node:events:519:28)
    at startListening (node:dgram:183:10)
    at node:dgram:371:7
    at process.processTicksAndRejections (node:internal/process/task_queues:83:21)

finally, my packages + version numbers: (Node v21.4.0)

"@discordjs/opus": "^0.9.0",
"@discordjs/voice": "^0.16.1",
"dgram": "^1.0.1",
"discord.js": "^14.14.1",
"dotenv": "^16.0.3",
"ffmpeg": "^0.0.4",
"libsodium-wrappers": "^0.7.13"

"@discordjs/opus": "^0.9.0",
"@discordjs/voice": "^0.16.1",
"dgram": "^1.0.1",
"discord.js": "^14.14.1",
"dotenv": "^16.0.3",
"ffmpeg": "^0.0.4",
"libsodium-wrappers": "^0.7.13"

yeah there's gonna be a lot of that, sorry :/ i don't need to worry about the hash right now (im refactoring that later, right now there is only one user so it doesn't matter) ~~i can fix the buffer issue, but i think that error will persist~~ if i change that, i think i get a type mismatch yeah, the error does in fact persist

d.js docs•12mo ago

To debug your voice connection and player: - Use debug: true when creating your VoiceConnection and AudioPlayer - Add an event listener to the <VoiceConnection> and the <AudioPlayer>:

// Add one for each class if applicable
<AudioPlayer | VoiceConnection>
    .on('debug', console.log)
    .on('error', console.error)

// Add one for each class if applicable
<AudioPlayer | VoiceConnection>
    .on('debug', console.log)
    .on('error', console.error)

- Add an error listener to the stream you are passing to the resource:

<Stream>.on('error', console.error)

<Stream>.on('error', console.error)

Note: The <> represents classes that need to be adapted to their respective name in your code

boppleoppleOP•12mo ago

I was still getting debug messages without that beforehand? and there are no debug messages before it crashes, even with the audioplayer option passed and even if i do make it past that error (for now just by switching audio formats) i do still get a typeerror if i change that line you mentioned earlier:

node:buffer:1086
    return this.utf8Write(string, 0, this.length);
                ^

TypeError: argument must be a string
    at Buffer.write (node:buffer:1086:17)
    at Socket.<anonymous> (file:///home/bopple/Desktop/Code/linux-sharing-bot/udpServer.js:68:25)
    at Socket.emit (node:events:531:35)
    at UDP.onMessage [as onmessage] (node:dgram:942:8) {
  code: 'ERR_INVALID_ARG_TYPE'
}

node:buffer:1086
    return this.utf8Write(string, 0, this.length);
                ^

TypeError: argument must be a string
    at Buffer.write (node:buffer:1086:17)
    at Socket.<anonymous> (file:///home/bopple/Desktop/Code/linux-sharing-bot/udpServer.js:68:25)
    at Socket.emit (node:events:531:35)
    at UDP.onMessage [as onmessage] (node:dgram:942:8) {
  code: 'ERR_INVALID_ARG_TYPE'
}

I think msg is a buffer? 1. I still get a garbled stream if I use raw (and switch to s16le audio format in ffmpeg) 2. opus streams still throw that random error from the audioplayer constructor i only ever get this far if i switch the streamtype to raw or rather just not opus so i still need to do that if the incoming data is opus as well? oooh maybe the acodec is correct, i think i'm sorry to keep bringing this up, but it was really my original question: do i need to change the type of stream i'm inputting or something to avoid this constructor error:

TypeError: this.playStream.once is not a function
    at new AudioResource (file:///home/bopple/Desktop/Code/linux-sharing-bot/node_modules/@discordjs/voice/dist/index.mjs:2319:21)
    at createAudioResource (file:///home/bopple/Desktop/Code/linux-sharing-bot/node_modules/@discordjs/voice/dist/index.mjs:2397:12)
    at Socket.<anonymous> (file:///home/bopple/Desktop/Code/linux-sharing-bot/app.js:82:22)
    at Socket.emit (node:events:519:28)
    at startListening (node:dgram:183:10)
    at node:dgram:371:7
    at process.processTicksAndRejections (node:internal/process/task_queues:83:21)

TypeError: this.playStream.once is not a function
    at new AudioResource (file:///home/bopple/Desktop/Code/linux-sharing-bot/node_modules/@discordjs/voice/dist/index.mjs:2319:21)
    at createAudioResource (file:///home/bopple/Desktop/Code/linux-sharing-bot/node_modules/@discordjs/voice/dist/index.mjs:2397:12)
    at Socket.<anonymous> (file:///home/bopple/Desktop/Code/linux-sharing-bot/app.js:82:22)
    at Socket.emit (node:events:519:28)
    at startListening (node:dgram:183:10)
    at node:dgram:371:7
    at process.processTicksAndRejections (node:internal/process/task_queues:83:21)

i dont think i can do anything with streamtype opus as long as i get this before any audio is processed update: i have switched to a readable instead of a readablestream to fix that issue, but now the readable stops getting called after 4 _read calls? i'm gonna try and just simplify this program on another branch tomorrow, do you know a good program i can reference for piping a custom readable into discordjs? don't worry about it if not, but i think i'm fundamentally misunderstanding something about node streams and that would help more than another documentation tab at this point

ThePedroo•12mo ago

@boppleopple Could you show current code?

boppleoppleOP•12mo ago

sorry, i got absolutely ruined by a storm in maine a day after posting that, i lost power for 4 days and internet still hasn’t come back (also i am a 6 hour drive away atm) once i have internet back home i’ll work on it again

ThePedroo•11mo ago

Sure

Gaming

Programming

piping from udp stream into an audio resource