Apply LLM Gateway to Streaming

1
import logging
2
from typing import Type
3
4
import assemblyai as aai
5
from assemblyai.streaming.v3 import (
6
    BeginEvent,
7
    LLMGatewayResponseEvent,
8
    StreamingClient,
9
    StreamingClientOptions,
10
    StreamingError,
11
    StreamingEvents,
12
    StreamingParameters,
13
    TurnEvent,
14
    TerminationEvent,
15
)
16
from assemblyai.streaming.v3.models import LLMGatewayConfig, LLMGatewayMessage
17
18
api_key = "<YOUR_API_KEY>"
19
prompt = "Provide a brief summary of the transcript.\n\nTranscript: {{turn}}"
20
21
logging.basicConfig(level=logging.INFO)
22
logger = logging.getLogger(__name__)
23
24
def on_begin(self: Type[StreamingClient], event: BeginEvent):
25
    print(f"Session started: {event.id}")
26
27
def on_turn(self: Type[StreamingClient], event: TurnEvent):
28
    if event.end_of_turn:
29
        print(f"\nTranscript:\n{event.transcript}\n")
30
31
def on_llm_response(self: Type[StreamingClient], event: LLMGatewayResponseEvent):
32
    # Extract the actual LLM response content from the data
33
    llm_content = event.data.get("choices", [{}])[0].get("message", {}).get("content", "")
34
    print(f"LLM Response:\n{llm_content}\n")
35
36
def on_terminated(self: Type[StreamingClient], event: TerminationEvent):
37
    print(
38
        f"Session terminated: {event.audio_duration_seconds} seconds of audio processed"
39
    )
40
41
def on_error(self: Type[StreamingClient], error: StreamingError):
42
    print(f"Error occurred: {error}")
43
44
def main():
45
    client = StreamingClient(
46
        StreamingClientOptions(
47
            api_key=api_key,
48
            api_host="streaming.assemblyai.com",
49
        )
50
    )
51
52
    client.on(StreamingEvents.Begin, on_begin)
53
    client.on(StreamingEvents.Turn, on_turn)
54
    client.on(StreamingEvents.LLMGatewayResponse, on_llm_response)
55
    client.on(StreamingEvents.Termination, on_terminated)
56
    client.on(StreamingEvents.Error, on_error)
57
58
    client.connect(
59
        StreamingParameters(
60
            sample_rate=16000,
61
            speech_model="u3-rt-pro",
62
            format_turns=True,
63
            llm_gateway=LLMGatewayConfig(
64
                model="claude-sonnet-4-20250514",
65
                messages=[
66
                    LLMGatewayMessage(role="user", content=prompt)
67
                ],
68
                max_tokens=4000
69
            )
70
        )
71
    )
72
73
    try:
74
        client.stream(
75
            aai.extras.MicrophoneStream(sample_rate=16000)
76
        )
77
    finally:
78
        client.disconnect(terminate=True)
79
80
if __name__ == "__main__":
81
    main()

1
import logging
2
from typing import Type
3
4
import assemblyai as aai
5
from assemblyai.streaming.v3 import (
6
    BeginEvent,
7
    LLMGatewayResponseEvent,
8
    StreamingClient,
9
    StreamingClientOptions,
10
    StreamingError,
11
    StreamingEvents,
12
    StreamingParameters,
13
    TurnEvent,
14
    TerminationEvent,
15
)
16
from assemblyai.streaming.v3.models import LLMGatewayConfig, LLMGatewayMessage
17
18
api_key = "<YOUR_API_KEY>"
19
prompt = "Provide a brief summary of the transcript.\n\nTranscript: {{turn}}"
20
21
logging.basicConfig(level=logging.INFO)
22
logger = logging.getLogger(__name__)