Text Streaming for LLMs

Text Streaming Live Example

/src/modules/stream/StreamController.ts

import { get, prefix } from 'vovk';
 
type Token = { message: string };
 
@prefix('stream')
export default class StreamController {
  @get('tokens', { cors: true })
  static async *streamTokens() {
    const tokens: Token[] = [
      { message: 'Hello,' },
      { message: ' World' },
      { message: ' from' },
      { message: ' Stream' },
      { message: '!' },
    ];
 
    for (const token of tokens) {
      yield token;
      await new Promise((resolve) => setTimeout(resolve, 300));
    }
  }
}

Source code (opens in a new tab)

Async Iterators

Controller methods can implement generators that use * syntax and utilise yield keyword instead of regular return.

/src/modules/stream/StreamController.ts

import { get, prefix } from 'vovk';
 
type Token = { message: string };
 
@prefix('stream')
export default class StreamController {
  @get('tokens')
  static async *streamTokens() {
    const tokens: Token[] = [
      { message: 'Hello,' },
      { message: ' World' },
      { message: '!' },
    ];
 
    for (const token of tokens) {
      await new Promise((resolve) => setTimeout(resolve, 300));
      yield token;
    }
  }
}

In order to refactor this code and utilise Back-end Service you can move the streaming logic to StreamService static class.

/src/modules/stream/StreamService.ts

type Token = { message: string };
 
export default class StreamService {
  static async *streamTokens() {
    const tokens: Token[] = [
      { message: 'Hello,' },
      { message: ' World' },
      { message: '!' },
    ];
 
    for (const token of tokens) {
      await new Promise((resolve) => setTimeout(resolve, 300));
      yield token;
    }
  }
}

At the controller use yield* syntax to delegate iterable returned from StreamService.streamTokens.

/src/modules/stream/StreamController.ts

import { get, prefix } from 'vovk';
import StreamService from './StreamService';
 
@prefix('stream')
export default class StreamController {
  @get('tokens')
  static async *streamTokens() {
    yield* StreamService.streamTokens();
  }
}

Handling Stream Responses on the Client

Text response streaming (including usage of StreamResponse class) generate client method that returns a disposable async generator.

import { StreamController } from 'vovk-client';
 
{
    using stream = await StreamController.streamTokens();
 
    for await (const token of stream) {
        console.log(token);
    }
}

using keyword (that you can freely replace by let or const) indicates that when code block is reached the end (in case of early break or if the code block encountered an error) the stream is going to be closed by invoking stream.close() method automatically. stream.close() can also be called explicitly if needed.

To make sure that the stream is closed before moving to the next code block you can use await using syntax that disposes the stream asynchronous way.

import { StreamController } from 'vovk-client';
 
{
    await using stream = await StreamController.streamTokens();
    // ...
}
// on this line stream is already closed

Type Extraction Streaming with StreamResponse Class