dhe128 Posts - Answer Overflow

dhe128

Explore posts from servers

•Created by dhe128 on 10/30/2024 in #⚡｜serverless

How to get `/stream` serverless endpoint to "stream"?

Example from official documentation: https://docs.runpod.io/sdks/javascript/endpoints#stream

from time import sleep
import runpod


def handler(job):
    job_input = job["input"]["prompt"]

    for i in job_input:
        sleep(1)  # sleep for 1 second for effect
        yield i


runpod.serverless.start(
    {
        "handler": handler,
        "return_aggregate_stream": True,  # Ensures aggregated results are streamed back
    }
)

from time import sleep
import runpod


def handler(job):
    job_input = job["input"]["prompt"]

    for i in job_input:
        sleep(1)  # sleep for 1 second for effect
        yield i


runpod.serverless.start(
    {
        "handler": handler,
        "return_aggregate_stream": True,  # Ensures aggregated results are streamed back
    }
)

import runpodSdk from "runpod-sdk";

async function main() {
  const runpod = runpodSdk(RUNPOD_API_KEY);
  const endpoint = runpod.endpoint(ENDPOINT_ID);
  const result = await endpoint.run({
    input: {
      prompt: "Hello, World!",
    },
  });

  console.log(result);

  const { id } = result;
  for await (const result of endpoint.stream(id)) {
    console.log(`${JSON.stringify(result, null, 2)}`);
  }
  console.log("done streaming");
}

import runpodSdk from "runpod-sdk";

async function main() {
  const runpod = runpodSdk(RUNPOD_API_KEY);
  const endpoint = runpod.endpoint(ENDPOINT_ID);
  const result = await endpoint.run({
    input: {
      prompt: "Hello, World!",
    },
  });

  console.log(result);

  const { id } = result;
  for await (const result of endpoint.stream(id)) {
    console.log(`${JSON.stringify(result, null, 2)}`);
  }
  console.log("done streaming");
}

The example suggests that the /stream endpoint should return the entire input, one letter at a time. However, when I try it it only sends the first letter back, followed by the http connection closing. Is /stream intended to be streamed from start to finish, or polled repeatedly?

9 replies

Gaming

Programming