WebRTC Video Streaming

Requirements

API key (obtain here)

Implementation

To prevent exposing the API key on the fronted, we recommend using a backend service to handle the API requests. Your frontend then makes calls to your backend service, which in turn makes the API calls to the Hub.

Obtain list of streams

Http

1===>
2GET /api/robot/$robotId/apps/$robotAppId/streams
3Authorization: Bearer <API_KEY>

Http

1<===
2200 OK
3Content-Type: application/json
4
5{
6  "streams": [
7    {
8      "streamId": "018f3ff8-519e-7b05-a63f-4c0e33036fdf",
9      "name": "Front camera,
10      "fps": 30,
11      "description": "Front camera"
12    }
13  ]
14}

Initiate WebRTC connection

Http

1===>
2GET /api/robot/$robotId/apps/$robotAppId/stream/$streamId
3Authorization: Bearer <API_KEY>

Http

1<===
2200 OK
3Content-Type: application/json
4
5{
6  "streamId": "018f3ff8-519e-7b05-a63f-4c0e33036fdf",
7  "sdp": "..."
8}

This endpoint can take up to 35 seconds to respond.

The returned SDP is the remote description of the WebRTC connection. Use it to obtain the local description and start the WebRTC connection.In case of failure, the response will be:

Http

1<===
2200 OK
3Content-Type: application/json
4
5{
6  "streamId": null,
7  "sdp": null,
8  "errorCode": "<ERROR_CODE>"
9}

<ERROR_CODE> can be one of the following:

unknown
timeout
stream_not_found
clients_limit
active_clients_limit
no_sdp
slow_connection
client_not_ready
not_initialized
invalid_metadata
video

Complete WebRTC connection

Once local description on frontend is obtained, submit the SDP to the remote with the following request:

Http

1===>
2POST /api/robot/$robotId/apps/$robotAppId/stream/$streamId
3Authorization: Bearer <API_KEY>
4Content-Type: application/json
5
6{
7  "streamId": "018f3ff8-519e-7b05-a63f-4c0e33036fdf",
8  "sdp": "..."
9}

Http

1<===
2200 OK
3Content-Type: application/json
4
5{
6  "success": true
7}

Example frontend code

Typescript

1export type SignalingParams = {
2  requestSdp: () => Promise<SdpRequestResult>;
3  submitSdp: (sdp: string, streamId: string) => Promise<{ success: boolean }>;
4};
5
6export type StreamMetadata = {
7  rtpTimestamp: number;
8  metadata: string;
9};
10
11export type SdpRequestResult = {
12  streamId: string | null;
13  sdp: string | null;
14  errorCode?: RtcErrorCode;
15  extra?: unknown;
16};
17
18const debugLog = (key: string, message: string, obj?: Record<string, any>) =>
19  console.log(obj ?? {}, `[WebRTC][${key}] ${message}`);
20
21function registerDebugListeners(connection: RTCPeerConnection) {
22  connection.addEventListener(
23    'icegatheringstatechange',
24    () => {
25      debugLog('iceGatheringLog', ` -> ${connection.iceGatheringState}`);
26    },
27    false,
28  );
29  connection.addEventListener(
30    'iceconnectionstatechange',
31    () => {
32      debugLog('iceConnectionLog', ` -> ${connection.iceConnectionState}`);
33    },
34    false,
35  );
36  connection.addEventListener(
37    'signalingstatechange',
38    () => {
39      debugLog('signalingLog', ` -> ${connection.signalingState}`);
40    },
41    false,
42  );
43}
44
45function createClosePeerConnection(args: { dataChannels: RTCDataChannel[]; connection: RTCPeerConnection }) {
46  return () => {
47    for (const dc of args.dataChannels) {
48      dc.close();
49    }
50
51    // close transceivers
52    if (args.connection.getTransceivers) {
53      for (const transceiver of args.connection.getTransceivers()) {
54        if (transceiver.stop) {
55          transceiver.stop();
56        }
57      }
58    }
59
60    // close local audio / video
61    for (const sender of args.connection.getSenders()) {
62      const track = sender.track;
63      if (track !== null) {
64        sender.track?.stop();
65      }
66    }
67
68    args.connection.close();
69  };
70}
71
72function createPeerConnection(args: {
73  servers?: RTCIceServer[];
74  onStream: (stream: MediaStream) => void;
75  onMetadata: (metadata: StreamMetadata) => void;
76}) {
77  const config: RTCConfiguration & { sdpSemantics: string } = {
78    sdpSemantics: 'unified-plan',
79    bundlePolicy: 'max-bundle',
80    iceServers: args.servers ?? [],
81    iceTransportPolicy: (window as any)['__overrideIceTransportPolicy'] ?? 'all',
82  };
83
84  const rtc = new RTCPeerConnection(config);
85
86  // register some listeners to help debugging
87  registerDebugListeners(rtc);
88
89  rtc.addEventListener('track', evt => args.onStream(evt.streams[0]));
90
91  let prevTime = 0;
92
93  const timeDiff = () => {
94    const now = performance.now();
95    if (!prevTime) {
96      prevTime = now;
97      return 0;
98    } else {
99      const diff = now - prevTime;
100      prevTime = now;
101      return diff;
102    }
103  };
104
105  const dcs: RTCDataChannel[] = [];
106  let pingTimeoutId: NodeJS.Timeout | null = null;
107  rtc.ondatachannel = event => {
108    const dc = event.channel;
109    dcs.push(dc);
110
111    if (dc.label === 'ping-pong') {
112      dc.onopen = () => {
113        debugLog('dataChannelLog', 'open');
114      };
115      dc.onmessage = evt => {
116        debugLog('dataChannelLog', `| ${dc.label} < ${evt.data}\n`);
117
118        pingTimeoutId = setTimeout(() => {
119          if (dc.readyState === 'open') {
120            const message = `Pong ${timeDiff()}`;
121            debugLog('dataChannelLog', `| ${dc.label} > ${message}\n`);
122            dc.send(message);
123          }
124        }, 1000);
125      };
126      dc.onclose = function () {
127        if (pingTimeoutId) clearTimeout(pingTimeoutId);
128        pingTimeoutId = null;
129        debugLog('dataChannelLog', '- close\n');
130      };
131    }
132  };
133
134  const close = createClosePeerConnection({
135    dataChannels: dcs,
136    connection: rtc,
137  });
138
139  return { rtc, close };
140}
141
142export class StreamingError extends Error {
143  readonly code: RtcErrorCode;
144  readonly extra?: unknown;
145  constructor(args: { code: RtcErrorCode; extra?: unknown }) {
146    super(args.code);
147    this.code = args.code;
148    this.extra = args.extra;
149  }
150}
151
152export async function watchWebRtcStream(
153  args: {
154    servers?: RTCIceServer[];
155    onStream: (stream: MediaStream) => void;
156    onMetadata: (metadata: StreamMetadata) => void;
157    signal: AbortSignal;
158  } & SignalingParams,
159): Promise<() => void> {
160  let closeConnection = () => noop();
161
162  try {
163    debugLog('watchWebRtcStream', 'requesting agent SDP');
164
165    const data = await args.requestSdp();
166
167    if (!data || data.errorCode) {
168      throw new StreamingError({
169        code: data?.errorCode ?? 'unknown',
170        extra: data?.extra,
171      });
172    }
173
174    const { sdp, streamId } = data;
175
176    if (!sdp || !streamId) {
177      throw new StreamingError({ code: 'no_sdp' });
178    }
179    if (args.signal.aborted) {
180      console.log('WebRtc request aborted!');
181      return closeConnection;
182    }
183
184    debugLog('watchWebRtcStream', 'received agent SDP', { sdp, streamId });
185
186    const { rtc, close } = createPeerConnection(args);
187    closeConnection = close;
188
189    await rtc.setRemoteDescription({ type: 'offer', sdp });
190
191    const answer = await rtc.createAnswer();
192    debugLog('watchWebRtcStream', 'created answer', { sdp: answer.sdp });
193    await rtc.setLocalDescription(answer);
194
195    while (!args.signal.aborted) {
196      if (rtc['iceGatheringState'] === 'complete') {
197        break;
198      }
199
200      await sleep(100);
201    }
202
203    if (args.signal.aborted) {
204      console.log('WebRtc request aborted!');
205      return closeConnection;
206    }
207
208    debugLog('watchWebRtcStream', 'sending SDP over');
209    await args.submitSdp(rtc.localDescription?.sdp ?? '', streamId);
210
211    if (args.signal.aborted) {
212      console.log('WebRtc request aborted!');
213      return closeConnection;
214    }
215  } catch (error) {
216    if (error instanceof StreamingError) {
217      throw error;
218    }
219    throw new StreamingError({
220      code: 'unknown',
221      extra: error,
222    });
223  } finally {
224    if (args.signal.aborted) {
225      closeConnection();
226    }
227  }
228
229  return closeConnection;
230}

ON THIS PAGE

WebRTC Video Streaming

Requirements

Implementation

Obtain list of streams

Initiate WebRTC connection

Complete WebRTC connection

Example frontend code