import logging

import runpod

from app.exllamav2_common import boot_engine, generate

logger = logging.getLogger(__name__)

async def handler(job: dict):
    request_dict: dict = job.pop("input", {})

    configs_dict = request_dict.copy()

    full_response = ""
    for full_response in generate(configs_dict):
        yield {"text": full_response, "finished": False}

    yield {"text": full_response, "finished": True}


boot_engine()

def concurrency_modifier(current_concurrency):
    max_concurrency = 1
    return max(0, max_concurrency - current_concurrency)

runpod.serverless.start({
    "handler": handler,
    "return_aggregate_stream": True,
})import logging

import runpod

from app.exllamav2_common import boot_engine, generate

logger = logging.getLogger(__name__)

async def handler(job: dict):
    request_dict: dict = job.pop("input", {})

    configs_dict = request_dict.copy()

    full_response = ""
    for full_response in generate(configs_dict):
        yield {"text": full_response, "finished": False}

    yield {"text": full_response, "finished": True}


boot_engine()

def concurrency_modifier(current_concurrency):
    max_concurrency = 1
    return max(0, max_concurrency - current_concurrency)

runpod.serverless.start({
    "handler": handler,
    "return_aggregate_stream": True,
})

import logging

import runpod

from app.exllamav2_common import boot_engine, generate

logger = logging.getLogger(__name__)

async def handler(job: dict):
    request_dict: dict = job.pop("input", {})

    configs_dict = request_dict.copy()

    full_response = ""
    for full_response in generate(configs_dict):
        yield {"text": full_response, "finished": False}

    yield {"text": full_response, "finished": True}


boot_engine()

def concurrency_modifier(current_concurrency):
    max_concurrency = 1
    return max(0, max_concurrency - current_concurrency)

runpod.serverless.start({
    "handler": handler,
    "return_aggregate_stream": True,
})import logging

import runpod

from app.exllamav2_common import boot_engine, generate

logger = logging.getLogger(__name__)

async def handler(job: dict):
    request_dict: dict = job.pop("input", {})

    configs_dict = request_dict.copy()

    full_response = ""
    for full_response in generate(configs_dict):
        yield {"text": full_response, "finished": False}

    yield {"text": full_response, "finished": True}


boot_engine()

def concurrency_modifier(current_concurrency):
    max_concurrency = 1
    return max(0, max_concurrency - current_concurrency)

runpod.serverless.start({
    "handler": handler,
    "return_aggregate_stream": True,
})