firmware/vendor/str0m/src/media/mod.rs

File

Blob: firmware/vendor/str0m/src/media/mod.rs

rust687 lines

1 //! Media (audio/video) related content.
2  
3 use std::collections::{HashMap, VecDeque};
4 use std::sync::Arc;
5 use std::time::Instant;
6  
7 use crate::RtcError;
8 use crate::change::AddMedia;
9 use crate::format::CodecConfig;
10  
11 use crate::packet::{CodecDepacketizer, DepacketizingBuffer, Payloader, RtpMeta};
12 use crate::rtp_::ExtensionMap;
13 use crate::rtp_::MidRid;
14 use crate::rtp_::SRTP_BLOCK_SIZE;
15 use crate::rtp_::SRTP_OVERHEAD;
16 use str0m_proto::Id;
17  
18 use crate::format::PayloadParams;
19 use crate::format::Vp9PacketizerMode;
20 use crate::sdp::Simulcast as SdpSimulcast;
21 use crate::sdp::{MediaLine, Msid};
22 use crate::streams::{RtpPacket, Streams};
23 use crate::util::already_happened;
24  
25 mod event;
26 pub use event::*;
27  
28 mod writer;
29 pub use writer::Writer;
30  
31 pub use crate::packet::MediaKind;
32 pub use crate::rtp_::{Direction, ExtensionValues, Frequency, MediaTime, Mid, Pt, Rid};
33  
34 /// Mid used for SSRC 0 non-media BWE probes.
35 ///
36 /// libwebrtc sends bandwidth estimation probes on SSRC 0 when:
37 /// - Video m-line with RTX is negotiated
38 /// - `allow_probe_without_media` is enabled (Chrome default)
39 /// - No video media packets have been sent yet
40 ///
41 /// These probes carry `transport_cc` for TWCC feedback but no real media.
42 pub(crate) const MID_PROBE: Mid = Mid::from_array(*b"~]probe\0\0\0\0\0\0\0\0\0");
43  
44 #[derive(Debug)]
45 /// Information about some configured media.
46 pub struct Media {
47     // ========================================= RTP level =========================================
48     //
49     /// Identifier of this media.
50     ///
51     /// RTP level.
52     mid: Mid,
53  
54     /// Canonical name.
55     ///
56     /// RTP level.
57     cname: String,
58  
59     /// Rid that we are expecting to see on incoming RTP packets that map to this mid.
60     /// Once discovered, we make an entry in `stream_rx`.
61     ///
62     /// RTP level.
63     rids_rx: Rids,
64  
65     /// Rid that we can send using the [`Writer`].
66     ///
67     /// RTP level.
68     rids_tx: Rids,
69  
70     // ========================================= SDP level =========================================
71     //
72     /// The index of this media line in the Session::media Vec.
73     ///
74     /// SDP property.
75     index: usize,
76  
77     /// "Stream and track" identifiers.
78     ///
79     /// This is for _outgoing_ SDP.
80     ///
81     /// SDP property.
82     msid: Msid,
83  
84     /// Audio or video.
85     kind: MediaKind,
86  
87     /// Current media direction.
88     ///
89     /// Can be altered via negotiation.
90     ///
91     /// SDP property.
92     dir: Direction,
93  
94     /// Remote PTs negotiated for this media.
95     ///
96     /// This tells us both the desired priority order of payload types
97     /// as well as which PT the remote side wants (in case they are narrowed).
98     ///
99     /// These must have corresponding entries in Session::codec_config.
100     ///
101     /// SDP property.
102     ///
103     /// If this is empty, the m-line is disabled/rejected (port=0 in SDP).
104     remote_pts: Vec<Pt>,
105  
106     /// Set when this m-line has been stopped via
107     /// [`SdpApi::stop_media`](crate::change::SdpApi::stop_media) or
108     /// rejected by the remote peer. Independent of `remote_pts` so that
109     /// an explicit stop preserves the negotiated PT list in SDP output
110     /// (the SDP grammar requires at least one fmt on a port=0 m-line).
111     stopped: bool,
112  
113     /// Remote extmaps negotiated for this media.
114     ///
115     /// The corresponding entries must exist in Session::codec_config.
116     ///
117     /// These are 1-indexed to be exactly like in the SDP.
118     remote_exts: ExtensionMap,
119  
120     /// [`true`] if this media was created by the remote peer, [`false`] if it was created by us.
121     remote_created: bool,
122  
123     /// Simulcast configuration, if set.
124     ///
125     /// SDP property.
126     simulcast: Option<SdpSimulcast>,
127  
128     // ========================================= Payloaders, etc =========================================
129     //
130     /// Buffers of incoming RTP packets. These do reordering/jitter buffer and also
131     /// depayload from RTP to frames.
132     depayloaders: HashMap<(Pt, Option<Rid>), DepacketizingBuffer>,
133  
134     /// Payloaders for outoing RTP packets.
135     payloaders: HashMap<(Pt, Option<Rid>), Payloader>,
136  
137     /// Frames to payload. Should typically only be 0 or 1.
138     to_payload: VecDeque<ToPayload>,
139  
140     pub(crate) need_open_event: bool,
141     pub(crate) need_changed_event: bool,
142  
143     /// When converting media lines to SDP, it's easier to represent the app m-line
144     /// as a Media. This field is true when we do that. No Session::medias will have
145     /// this set to true – they only exist temporarily.
146     pub(crate) app_tmp: bool,
147 }
148  
149 #[derive(Debug)]
150 /// Config value for [`Media::rids_rx()`] and [`Media::rids_tx()`]
151 pub enum Rids {
152     /// No rid is allowed.
153     None,
154     /// Any Rid is allowed.
155     ///
156     /// This is the default value for direct API.
157     Any,
158     /// These specific [`Rid`] are allowed.
159     ///
160     /// This is the default value for Simulcast configured via SDP.
161     Specific(Vec<Rid>),
162 }
163  
164 impl Rids {
165     pub(crate) fn contains(&self, rid: Rid) -> bool {
166         match self {
167             Rids::None => false,
168             Rids::Any => true,
169             Rids::Specific(v) => v.contains(&rid),
170         }
171     }
172  
173     pub(crate) fn is_specific(&self) -> bool {
174         matches!(self, Rids::Specific(_))
175     }
176  
177     fn add(&mut self, rid: Rid) {
178         match self {
179             Rids::None | Rids::Any => {
180                 *self = Rids::Specific(vec![rid]);
181             }
182             Rids::Specific(vec) if !vec.contains(&rid) => vec.push(rid),
183             Rids::Specific(_) => {}
184         }
185     }
186 }
187  
188 #[derive(Debug)]
189 pub(crate) struct ToPayload {
190     pub pt: Pt,
191     pub rid: Option<Rid>,
192     pub wallclock: Instant,
193     pub rtp_time: MediaTime,
194     pub start_of_talk_spurt: bool,
195     pub data: Arc<[u8]>,
196     pub ext_vals: ExtensionValues,
197 }
198  
199 impl Media {
200     /// Identifier of the media.
201     ///
202     /// RTP level.
203     pub fn mid(&self) -> Mid {
204         self.mid
205     }
206  
207     /// Canonical name.
208     ///
209     /// Persistent transport-level identifier for an RTP source.
210     ///
211     /// RTP level property. The value is sent in RTCP reports for `StreamTx`. Incoming
212     /// cnames can be found in [`StreamRx::cname`][crate::rtp::StreamRx::cname].
213     pub fn cname(&self) -> &str {
214         &self.cname
215     }
216  
217     /// Add rid as one we are expecting to receive for this mid.
218     ///
219     /// This is used for situations where we don't know the SSRC upfront, such as not having
220     /// a=ssrc lines in an SDP. Adding a rid means we are dynamically discovering the SSRC from
221     /// a mid/rid combination in the RTP header extensions.
222     ///
223     /// RTP level.
224     pub fn expect_rid_rx(&mut self, rid: Rid) {
225         self.rids_rx.add(rid);
226     }
227  
228     /// Rids we are expecting to see on incoming RTP packets that map to this mid.
229     ///
230     /// By default this is set to [`Rids::Any`], which changes to [`Rids::Specific`] via SDP negotiation
231     /// that configures Simulcast where specific rids are expected.
232     ///
233     /// RTP level.
234     pub fn rids_rx(&self) -> &Rids {
235         &self.rids_rx
236     }
237  
238     /// Rids we are can send via the [`Writer`].
239     ///
240     /// By default this is set to [`Rids::None`], which changes to [`Rids::Specific`] via SDP negotiation
241     /// that configures Simulcast where specific rids are expected.
242     ///
243     /// RTP level.
244     pub fn rids_tx(&self) -> &Rids {
245         &self.rids_tx
246     }
247  
248     pub(crate) fn index(&self) -> usize {
249         self.index
250     }
251  
252     pub(crate) fn msid(&self) -> &Msid {
253         &self.msid
254     }
255  
256     /// Identifier for the group this Media belongs to.
257     pub fn stream_id(&self) -> &str {
258         &self.msid().stream_id
259     }
260  
261     /// Identifier for this Media. Should be unique for the given stream id.
262     pub fn track_id(&self) -> &str {
263         &self.msid().track_id
264     }
265  
266     /// Whether this media is audio or video.
267     ///
268     /// SDP level property.
269     pub fn kind(&self) -> MediaKind {
270         self.kind
271     }
272  
273     /// Current direction. This can be changed using
274     /// [`SdpApi::set_direction()`][crate::SdpApi::set_direction()] followed by an SDP negotiation.
275     ///
276     /// To test whether it's possible to send media with the current direction, use
277     ///
278     /// ```no_run
279     /// # use str0m::media::Media;
280     /// let media: Media = todo!(); // Get hold of media row.
281     /// if media.direction().is_sending() {
282     ///     // media.write(...);
283     /// }
284     /// ```
285     ///
286     /// SDP level property.
287     pub fn direction(&self) -> Direction {
288         self.dir
289     }
290  
291     /// Whether this m-line is disabled/rejected (port=0 in SDP).
292     ///
293     /// An m-line is disabled if it has been stopped (via
294     /// [`SdpApi::stop_media`](crate::change::SdpApi::stop_media) or by the
295     /// remote peer), or if no codecs matched during negotiation.
296     ///
297     /// SDP level property.
298     pub fn disabled(&self) -> bool {
299         self.stopped || self.remote_pts.is_empty()
300     }
301  
302     /// Whether this m-line has been stopped.
303     ///
304     /// Unlike [`disabled`](Self::disabled) this does not include the "no
305     /// codecs matched" case - only explicit stop via
306     /// [`SdpApi::stop_media`](crate::change::SdpApi::stop_media) or a
307     /// port=0 m-line received from the remote peer. A stopped m-line
308     /// cannot be reactivated; its slot can however be recycled by a
309     /// subsequent new m-line (RFC 8829 §5.2.2).
310     pub fn stopped(&self) -> bool {
311         self.stopped
312     }
313  
314     pub(crate) fn mark_stopped(&mut self) {
315         self.stopped = true;
316     }
317  
318     pub(crate) fn simulcast(&self) -> Option<&SdpSimulcast> {
319         self.simulcast.as_ref()
320     }
321  
322     pub(crate) fn poll_sample(
323         &mut self,
324         params: &[PayloadParams],
325     ) -> Result<Option<MediaData>, RtcError> {
326         for ((pt, rid), buf) in &mut self.depayloaders {
327             if let Some(r) = buf.pop() {
328                 let dep = r.map_err(|e| RtcError::Packet(self.mid, *pt, e))?;
329                 let Some(codec) = params.iter().find(|c| c.pt() == *pt) else {
330                     return Ok(None);
331                 };
332                 return Ok(Some(MediaData {
333                     mid: self.mid,
334                     pt: *pt,
335                     rid: *rid,
336                     params: *codec,
337                     // The depacketized time is in the RTP wire clock rate. For the
338                     // media (samples/frame) API we present it in the codec's nominal
339                     // clock rate. These differ only for G722, which has a 16 kHz
340                     // nominal rate but an 8 kHz RTP clock rate (RFC 3551 §4.5.2). For
341                     // all other codecs this rebase is a no-op. See
342                     // https://en.wikipedia.org/wiki/RTP_payload_formats#cite_note-55
343                     time: dep.time.rebase(codec.spec().clock_rate),
344                     network_time: dep.first_network_time(),
345                     seq_range: dep.seq_range(),
346                     contiguous: dep.contiguous,
347                     ext_vals: dep.ext_vals(),
348                     codec_extra: dep.codec_extra,
349                     last_sender_info: dep.first_sender_info(),
350                     audio_start_of_talk_spurt: codec.spec().codec.is_audio()
351                         && dep.start_of_talkspurt(),
352                     data: dep.data.into(),
353                 }));
354             }
355         }
356         Ok(None)
357     }
358  
359     pub(crate) fn depayload(
360         &mut self,
361         rid: Option<Rid>,
362         packet: RtpPacket,
363         reordering_size_audio: usize,
364         reordering_size_video: usize,
365         params: &[PayloadParams],
366     ) {
367         if !self.dir.is_receiving() {
368             return;
369         }
370  
371         let pt = packet.header.payload_type;
372  
373         let key = (pt, rid);
374  
375         let exists = self.depayloaders.contains_key(&key);
376  
377         if !exists {
378             // This unwrap is ok, because the handle_input doesn't accept the RtpPacket for
379             // depayloading unless we have matched the PT to one in the session.
380             let params = params.iter().find(|p| p.pt == pt).unwrap();
381  
382             let codec = params.spec.codec;
383  
384             // How many packets to hold back in the jitter buffer.
385             let hold_back = if codec.is_audio() {
386                 reordering_size_audio
387             } else {
388                 reordering_size_video
389             };
390  
391             let mut depack: CodecDepacketizer = codec.into();
392  
393             // Enable DONL for H.265 when sprop-max-don-diff > 0 (RFC 7798 §7.1)
394             if let CodecDepacketizer::H265(ref mut h265) = depack {
395                 if params.spec.format.sprop_max_don_diff.unwrap_or(0) > 0 {
396                     h265.with_donl(true);
397                 }
398             }
399  
400             // Enable DONL for H.266 when sprop-max-don-diff > 0 (RFC 9328 §7.2)
401             if let CodecDepacketizer::H266(ref mut h266) = depack {
402                 if params.spec.format.sprop_max_don_diff.unwrap_or(0) > 0 {
403                     h266.with_donl(true);
404                 }
405             }
406  
407             let buffer = DepacketizingBuffer::new(depack, hold_back);
408  
409             self.depayloaders.insert((pt, rid), buffer);
410         }
411  
412         let meta = RtpMeta {
413             received: packet.timestamp,
414             time: packet.time,
415             seq_no: packet.seq_no,
416             header: packet.header.clone(),
417             last_sender_info: packet.last_sender_info,
418         };
419  
420         for ((other_pt, other_rid), buffer) in &mut self.depayloaders {
421             if *other_pt != pt && *other_rid == rid {
422                 buffer.push_padding(meta.clone());
423             }
424         }
425  
426         // The entry will be there by now.
427         let buffer = self.depayloaders.get_mut(&key).unwrap();
428  
429         buffer.push(meta, packet.payload);
430     }
431  
432     pub(crate) fn set_cname(&mut self, cname: String) {
433         self.cname = cname;
434     }
435  
436     pub(crate) fn set_msid(&mut self, msid: Msid) {
437         self.msid = msid;
438     }
439  
440     pub(crate) fn set_direction(&mut self, new_dir: Direction) {
441         self.need_changed_event = self.dir != new_dir;
442         self.dir = new_dir;
443     }
444  
445     pub(crate) fn set_simulcast(&mut self, s: SdpSimulcast) {
446         debug!("Set simulcast: {:?}", s);
447         self.simulcast = Some(s);
448     }
449  
450     fn payloader_for(
451         &mut self,
452         pt: Pt,
453         rid: Option<Rid>,
454         params: &[PayloadParams],
455         vp9_mode: Vp9PacketizerMode,
456     ) -> &mut Payloader {
457         self.payloaders.entry((pt, rid)).or_insert_with(|| {
458             // Unwrap is OK, the pt should be checked already when calling this function.
459             let params = params.iter().find(|p| p.pt == pt).unwrap();
460             Payloader::new(params.spec, vp9_mode)
461         })
462     }
463  
464     fn set_to_payload(&mut self, to_payload: ToPayload) -> Result<(), RtcError> {
465         if self.to_payload.len() > 100 {
466             return Err(RtcError::WriteWithoutPoll);
467         }
468  
469         self.to_payload.push_back(to_payload);
470  
471         Ok(())
472     }
473  
474     pub(crate) fn poll_timeout(&self) -> Option<Instant> {
475         if !self.to_payload.is_empty() {
476             Some(already_happened())
477         } else {
478             None
479         }
480     }
481  
482     pub(crate) fn do_payload(
483         &mut self,
484         streams: &mut Streams,
485         params: &[PayloadParams],
486         vp9_mode: Vp9PacketizerMode,
487         mtu: usize,
488     ) -> Result<(), RtcError> {
489         let Some(to_payload) = self.to_payload.pop_front() else {
490             return Ok(());
491         };
492  
493         let ToPayload { pt, rid, .. } = &to_payload;
494  
495         let is_audio = self.kind.is_audio();
496  
497         let midrid = MidRid(self.mid, *rid);
498  
499         let stream = streams.stream_tx_by_midrid(midrid);
500  
501         let Some(stream) = stream else {
502             return Err(RtcError::NoSenderSource);
503         };
504  
505         let pt = *pt;
506  
507         let payloader = self.payloader_for(pt, *rid, params, vp9_mode);
508  
509         let rtp_size: usize = mtu - SRTP_OVERHEAD;
510         // align to SRTP block size to minimize padding needs
511         let aligned_mtu: usize = rtp_size - rtp_size % SRTP_BLOCK_SIZE;
512  
513         payloader
514             .push_sample(to_payload, aligned_mtu, is_audio, stream)
515             .map_err(|e| RtcError::Packet(self.mid, pt, e))?;
516  
517         Ok(())
518     }
519  
520     pub(crate) fn set_remote_pts(&mut self, pts: Vec<Pt>) {
521         // Have we already set PTs?
522         if !self.remote_pts.is_empty() {
523             return;
524         }
525  
526         // TODO: We should verify the remote peer doesn't suddenly change the PT
527         // order or removes/adds PTs that weren't there from the start.
528         debug!("Mid ({}) remote PT order is: {:?}", self.mid, pts);
529         self.remote_pts = pts;
530     }
531  
532     pub(crate) fn set_remote_extmap(&mut self, exts: ExtensionMap) {
533         self.remote_exts = exts;
534     }
535  
536     /// The remote PT (payload types) configured for this Media.
537     ///
538     /// These are negotiated with the remote peer and is the order the remote prefer them.
539     ///
540     /// I.e. these can be fewer than the `PayloadParams` configured for the `Rtc` instance,
541     /// and in a different order.
542     pub fn remote_pts(&self) -> &[Pt] {
543         &self.remote_pts
544     }
545  
546     /// The remote, agreed on, extension map, configured for this Media.
547     ///
548     /// For the SDP API, these are negotiated with the remote peer.
549     ///
550     /// For the Direct API, these are a clone of the session configured values narrowed by media
551     /// kind (audio/video).
552     pub fn remote_extmap(&self) -> &ExtensionMap {
553         &self.remote_exts
554     }
555  
556     pub(crate) fn remote_created(&self) -> bool {
557         self.remote_created
558     }
559  
560     pub(crate) fn first_pt_with_rtx(&self, config: &CodecConfig) -> Option<Pt> {
561         config
562             .all_for_kind(self.kind)
563             // Only consider negotiated PTs
564             .filter(|p| self.remote_pts.contains(&p.pt))
565             // Map to the first PT found in payload params with RTX
566             .find_map(|p| p.resend().map(|_| p.pt))
567     }
568  
569     pub(crate) fn reset_depayloader(&mut self, payload_type: Pt, rid: Option<Rid>) {
570         // Simply remove the depayloader, it will be re-created on the next RTP packet.
571         self.depayloaders.remove(&(payload_type, rid));
572     }
573  
574     pub(crate) fn reset_depayloaders_for_rid(&mut self, rid: Option<Rid>) {
575         self.depayloaders
576             .retain(|(_, existing_rid), _| *existing_rid != rid);
577     }
578  
579     pub(crate) fn set_rid_rx(&mut self, rids: Rids) {
580         self.rids_rx = rids;
581     }
582  
583     pub(crate) fn set_rid_tx(&mut self, rids: Rids) {
584         self.rids_tx = rids;
585     }
586  
587     pub(crate) fn add_to_rid_tx(&mut self, rid: Rid) {
588         self.rids_tx.add(rid)
589     }
590 }
591  
592 impl Default for Media {
593     fn default() -> Self {
594         Self {
595             mid: Mid::new(),
596             index: 0,
597             app_tmp: false,
598             cname: Id::<20>::random().to_string(),
599             msid: Msid::random(),
600             kind: MediaKind::Video,
601             remote_pts: vec![],
602             stopped: false,
603             remote_exts: ExtensionMap::empty(),
604             remote_created: false,
605             dir: Direction::SendRecv,
606             simulcast: None,
607             rids_rx: Rids::Any,
608             rids_tx: Rids::None,
609             payloaders: HashMap::new(),
610             depayloaders: HashMap::new(),
611             to_payload: VecDeque::default(),
612             need_open_event: true,
613             need_changed_event: false,
614         }
615     }
616 }
617  
618 impl Media {
619     pub(crate) fn from_remote_media_line(
620         l: &MediaLine,
621         index: usize,
622         remote_created: bool,
623     ) -> Self {
624         Media {
625             mid: l.mid(),
626             index,
627             // This is not reflected back, and thus added by add_pending_changes().
628             // cname,
629             msid: l.msid().unwrap_or(Msid::random()),
630             kind: l.typ.clone().into(),
631             dir: if l.disabled {
632                 Direction::Inactive
633             } else {
634                 l.direction().invert() // remote direction is reverse.
635             },
636             remote_created,
637             ..Default::default()
638         }
639     }
640  
641     // Going from AddMedia to Media for pending in a Change and are sent
642     // in the offer to the other side.
643     //
644     // from_add_media is only used when creating temporary Media to be
645     // included in the SDP. We don't want to make an _actual_ changes with this.
646     pub(crate) fn from_add_media(a: AddMedia) -> Self {
647         Media {
648             mid: a.mid,
649             index: a.index,
650             cname: a.cname,
651             msid: a.msid,
652             kind: a.kind,
653             dir: a.dir,
654             remote_pts: a.pts,
655             remote_exts: a.exts,
656             remote_created: false,
657             simulcast: a.simulcast.map(|s| s.into_sdp()),
658             ..Default::default()
659         }
660     }
661  
662     pub(crate) fn from_app_tmp(mid: Mid, index: usize) -> Media {
663         Media {
664             mid,
665             index,
666             app_tmp: true,
667             ..Default::default()
668         }
669     }
670  
671     pub(crate) fn from_direct_api(
672         mid: Mid,
673         index: usize,
674         kind: MediaKind,
675         exts: ExtensionMap,
676     ) -> Media {
677         Media {
678             mid,
679             index,
680             kind,
681             dir: Direction::SendRecv,
682             remote_exts: exts,
683             ..Default::default()
684         }
685     }
686 }

1	//! Media (audio/video) related content.
2
3	use std::collections::{HashMap, VecDeque};
4	use std::sync::Arc;
5	use std::time::Instant;
6
7	use crate::RtcError;
8	use crate::change::AddMedia;
9	use crate::format::CodecConfig;
10
11	use crate::packet::{CodecDepacketizer, DepacketizingBuffer, Payloader, RtpMeta};
12	use crate::rtp_::ExtensionMap;
13	use crate::rtp_::MidRid;
14	use crate::rtp_::SRTP_BLOCK_SIZE;
15	use crate::rtp_::SRTP_OVERHEAD;
16	use str0m_proto::Id;
17
18	use crate::format::PayloadParams;
19	use crate::format::Vp9PacketizerMode;
20	use crate::sdp::Simulcast as SdpSimulcast;
21	use crate::sdp::{MediaLine, Msid};
22	use crate::streams::{RtpPacket, Streams};
23	use crate::util::already_happened;
24
25	mod event;
26	pub use event::*;
27
28	mod writer;
29	pub use writer::Writer;
30
31	pub use crate::packet::MediaKind;
32	pub use crate::rtp_::{Direction, ExtensionValues, Frequency, MediaTime, Mid, Pt, Rid};
33
34	/// Mid used for SSRC 0 non-media BWE probes.
35	///
36	/// libwebrtc sends bandwidth estimation probes on SSRC 0 when:
37	/// - Video m-line with RTX is negotiated
38	/// - `allow_probe_without_media` is enabled (Chrome default)
39	/// - No video media packets have been sent yet
40	///
41	/// These probes carry `transport_cc` for TWCC feedback but no real media.
42	pub(crate) const MID_PROBE: Mid = Mid::from_array(*b"~]probe\0\0\0\0\0\0\0\0\0");
43
44	#[derive(Debug)]
45	/// Information about some configured media.
46	pub struct Media {
47	// ========================================= RTP level =========================================
48	//
49	/// Identifier of this media.
50	///
51	/// RTP level.
52	mid: Mid,
53
54	/// Canonical name.
55	///
56	/// RTP level.
57	cname: String,
58
59	/// Rid that we are expecting to see on incoming RTP packets that map to this mid.
60	/// Once discovered, we make an entry in `stream_rx`.
61	///
62	/// RTP level.
63	rids_rx: Rids,
64
65	/// Rid that we can send using the [`Writer`].
66	///
67	/// RTP level.
68	rids_tx: Rids,
69
70	// ========================================= SDP level =========================================
71	//
72	/// The index of this media line in the Session::media Vec.
73	///
74	/// SDP property.
75	index: usize,
76
77	/// "Stream and track" identifiers.
78	///
79	/// This is for _outgoing_ SDP.
80	///
81	/// SDP property.
82	msid: Msid,
83
84	/// Audio or video.
85	kind: MediaKind,
86
87	/// Current media direction.
88	///
89	/// Can be altered via negotiation.
90	///
91	/// SDP property.
92	dir: Direction,
93
94	/// Remote PTs negotiated for this media.
95	///
96	/// This tells us both the desired priority order of payload types
97	/// as well as which PT the remote side wants (in case they are narrowed).
98	///
99	/// These must have corresponding entries in Session::codec_config.
100	///
101	/// SDP property.
102	///
103	/// If this is empty, the m-line is disabled/rejected (port=0 in SDP).
104	remote_pts: Vec<Pt>,
105
106	/// Set when this m-line has been stopped via
107	/// [`SdpApi::stop_media`](crate::change::SdpApi::stop_media) or
108	/// rejected by the remote peer. Independent of `remote_pts` so that
109	/// an explicit stop preserves the negotiated PT list in SDP output
110	/// (the SDP grammar requires at least one fmt on a port=0 m-line).
111	stopped: bool,
112
113	/// Remote extmaps negotiated for this media.
114	///
115	/// The corresponding entries must exist in Session::codec_config.
116	///
117	/// These are 1-indexed to be exactly like in the SDP.
118	remote_exts: ExtensionMap,
119
120	/// [`true`] if this media was created by the remote peer, [`false`] if it was created by us.
121	remote_created: bool,
122
123	/// Simulcast configuration, if set.
124	///
125	/// SDP property.
126	simulcast: Option<SdpSimulcast>,
127
128	// ========================================= Payloaders, etc =========================================
129	//
130	/// Buffers of incoming RTP packets. These do reordering/jitter buffer and also
131	/// depayload from RTP to frames.
132	depayloaders: HashMap<(Pt, Option<Rid>), DepacketizingBuffer>,
133
134	/// Payloaders for outoing RTP packets.
135	payloaders: HashMap<(Pt, Option<Rid>), Payloader>,
136
137	/// Frames to payload. Should typically only be 0 or 1.
138	to_payload: VecDeque<ToPayload>,
139
140	pub(crate) need_open_event: bool,
141	pub(crate) need_changed_event: bool,
142
143	/// When converting media lines to SDP, it's easier to represent the app m-line
144	/// as a Media. This field is true when we do that. No Session::medias will have
145	/// this set to true – they only exist temporarily.
146	pub(crate) app_tmp: bool,
147	}
148
149	#[derive(Debug)]
150	/// Config value for [`Media::rids_rx()`] and [`Media::rids_tx()`]
151	pub enum Rids {
152	/// No rid is allowed.
153	None,
154	/// Any Rid is allowed.
155	///
156	/// This is the default value for direct API.
157	Any,
158	/// These specific [`Rid`] are allowed.
159	///
160	/// This is the default value for Simulcast configured via SDP.
161	Specific(Vec<Rid>),
162	}
163
164	impl Rids {
165	pub(crate) fn contains(&self, rid: Rid) -> bool {
166	match self {
167	Rids::None => false,
168	Rids::Any => true,
169	Rids::Specific(v) => v.contains(&rid),
170	}
171	}
172
173	pub(crate) fn is_specific(&self) -> bool {
174	matches!(self, Rids::Specific(_))
175	}
176
177	fn add(&mut self, rid: Rid) {
178	match self {
179	Rids::None \| Rids::Any => {
180	*self = Rids::Specific(vec![rid]);
181	}
182	Rids::Specific(vec) if !vec.contains(&rid) => vec.push(rid),
183	Rids::Specific(_) => {}
184	}
185	}
186	}
187
188	#[derive(Debug)]
189	pub(crate) struct ToPayload {
190	pub pt: Pt,
191	pub rid: Option<Rid>,
192	pub wallclock: Instant,
193	pub rtp_time: MediaTime,
194	pub start_of_talk_spurt: bool,
195	pub data: Arc<[u8]>,
196	pub ext_vals: ExtensionValues,
197	}
198
199	impl Media {
200	/// Identifier of the media.
201	///
202	/// RTP level.
203	pub fn mid(&self) -> Mid {
204	self.mid
205	}
206
207	/// Canonical name.
208	///
209	/// Persistent transport-level identifier for an RTP source.
210	///
211	/// RTP level property. The value is sent in RTCP reports for `StreamTx`. Incoming
212	/// cnames can be found in [`StreamRx::cname`][crate::rtp::StreamRx::cname].
213	pub fn cname(&self) -> &str {
214	&self.cname
215	}
216
217	/// Add rid as one we are expecting to receive for this mid.
218	///
219	/// This is used for situations where we don't know the SSRC upfront, such as not having
220	/// a=ssrc lines in an SDP. Adding a rid means we are dynamically discovering the SSRC from
221	/// a mid/rid combination in the RTP header extensions.
222	///
223	/// RTP level.
224	pub fn expect_rid_rx(&mut self, rid: Rid) {
225	self.rids_rx.add(rid);
226	}
227
228	/// Rids we are expecting to see on incoming RTP packets that map to this mid.
229	///
230	/// By default this is set to [`Rids::Any`], which changes to [`Rids::Specific`] via SDP negotiation
231	/// that configures Simulcast where specific rids are expected.
232	///
233	/// RTP level.
234	pub fn rids_rx(&self) -> &Rids {
235	&self.rids_rx
236	}
237
238	/// Rids we are can send via the [`Writer`].
239	///
240	/// By default this is set to [`Rids::None`], which changes to [`Rids::Specific`] via SDP negotiation
241	/// that configures Simulcast where specific rids are expected.
242	///
243	/// RTP level.
244	pub fn rids_tx(&self) -> &Rids {
245	&self.rids_tx
246	}
247
248	pub(crate) fn index(&self) -> usize {
249	self.index
250	}
251
252	pub(crate) fn msid(&self) -> &Msid {
253	&self.msid
254	}
255
256	/// Identifier for the group this Media belongs to.
257	pub fn stream_id(&self) -> &str {
258	&self.msid().stream_id
259	}
260
261	/// Identifier for this Media. Should be unique for the given stream id.
262	pub fn track_id(&self) -> &str {
263	&self.msid().track_id
264	}
265
266	/// Whether this media is audio or video.
267	///
268	/// SDP level property.
269	pub fn kind(&self) -> MediaKind {
270	self.kind
271	}
272
273	/// Current direction. This can be changed using
274	/// [`SdpApi::set_direction()`][crate::SdpApi::set_direction()] followed by an SDP negotiation.
275	///
276	/// To test whether it's possible to send media with the current direction, use
277	///
278	/// ```no_run
279	/// # use str0m::media::Media;
280	/// let media: Media = todo!(); // Get hold of media row.
281	/// if media.direction().is_sending() {
282	/// // media.write(...);
283	/// }
284	/// ```
285	///
286	/// SDP level property.
287	pub fn direction(&self) -> Direction {
288	self.dir
289	}
290
291	/// Whether this m-line is disabled/rejected (port=0 in SDP).
292	///
293	/// An m-line is disabled if it has been stopped (via
294	/// [`SdpApi::stop_media`](crate::change::SdpApi::stop_media) or by the
295	/// remote peer), or if no codecs matched during negotiation.
296	///
297	/// SDP level property.
298	pub fn disabled(&self) -> bool {
299	self.stopped \|\| self.remote_pts.is_empty()
300	}
301
302	/// Whether this m-line has been stopped.
303	///
304	/// Unlike [`disabled`](Self::disabled) this does not include the "no
305	/// codecs matched" case - only explicit stop via
306	/// [`SdpApi::stop_media`](crate::change::SdpApi::stop_media) or a
307	/// port=0 m-line received from the remote peer. A stopped m-line
308	/// cannot be reactivated; its slot can however be recycled by a
309	/// subsequent new m-line (RFC 8829 §5.2.2).
310	pub fn stopped(&self) -> bool {
311	self.stopped
312	}
313
314	pub(crate) fn mark_stopped(&mut self) {
315	self.stopped = true;
316	}
317
318	pub(crate) fn simulcast(&self) -> Option<&SdpSimulcast> {
319	self.simulcast.as_ref()
320	}
321
322	pub(crate) fn poll_sample(
323	&mut self,
324	params: &[PayloadParams],
325	) -> Result<Option<MediaData>, RtcError> {
326	for ((pt, rid), buf) in &mut self.depayloaders {
327	if let Some(r) = buf.pop() {
328	let dep = r.map_err(\|e\| RtcError::Packet(self.mid, *pt, e))?;
329	let Some(codec) = params.iter().find(\|c\| c.pt() == *pt) else {
330	return Ok(None);
331	};
332	return Ok(Some(MediaData {
333	mid: self.mid,
334	pt: *pt,
335	rid: *rid,
336	params: *codec,
337	// The depacketized time is in the RTP wire clock rate. For the
338	// media (samples/frame) API we present it in the codec's nominal
339	// clock rate. These differ only for G722, which has a 16 kHz
340	// nominal rate but an 8 kHz RTP clock rate (RFC 3551 §4.5.2). For
341	// all other codecs this rebase is a no-op. See
342	// https://en.wikipedia.org/wiki/RTP_payload_formats#cite_note-55
343	time: dep.time.rebase(codec.spec().clock_rate),
344	network_time: dep.first_network_time(),
345	seq_range: dep.seq_range(),
346	contiguous: dep.contiguous,
347	ext_vals: dep.ext_vals(),
348	codec_extra: dep.codec_extra,
349	last_sender_info: dep.first_sender_info(),
350	audio_start_of_talk_spurt: codec.spec().codec.is_audio()
351	&& dep.start_of_talkspurt(),
352	data: dep.data.into(),
353	}));
354	}
355	}
356	Ok(None)
357	}
358
359	pub(crate) fn depayload(
360	&mut self,
361	rid: Option<Rid>,
362	packet: RtpPacket,
363	reordering_size_audio: usize,
364	reordering_size_video: usize,
365	params: &[PayloadParams],
366	) {
367	if !self.dir.is_receiving() {
368	return;
369	}
370
371	let pt = packet.header.payload_type;
372
373	let key = (pt, rid);
374
375	let exists = self.depayloaders.contains_key(&key);
376
377	if !exists {
378	// This unwrap is ok, because the handle_input doesn't accept the RtpPacket for
379	// depayloading unless we have matched the PT to one in the session.
380	let params = params.iter().find(\|p\| p.pt == pt).unwrap();
381
382	let codec = params.spec.codec;
383
384	// How many packets to hold back in the jitter buffer.
385	let hold_back = if codec.is_audio() {
386	reordering_size_audio
387	} else {
388	reordering_size_video
389	};
390
391	let mut depack: CodecDepacketizer = codec.into();
392
393	// Enable DONL for H.265 when sprop-max-don-diff > 0 (RFC 7798 §7.1)
394	if let CodecDepacketizer::H265(ref mut h265) = depack {
395	if params.spec.format.sprop_max_don_diff.unwrap_or(0) > 0 {
396	h265.with_donl(true);
397	}
398	}
399
400	// Enable DONL for H.266 when sprop-max-don-diff > 0 (RFC 9328 §7.2)
401	if let CodecDepacketizer::H266(ref mut h266) = depack {
402	if params.spec.format.sprop_max_don_diff.unwrap_or(0) > 0 {
403	h266.with_donl(true);
404	}
405	}
406
407	let buffer = DepacketizingBuffer::new(depack, hold_back);
408
409	self.depayloaders.insert((pt, rid), buffer);
410	}
411
412	let meta = RtpMeta {
413	received: packet.timestamp,
414	time: packet.time,
415	seq_no: packet.seq_no,
416	header: packet.header.clone(),
417	last_sender_info: packet.last_sender_info,
418	};
419
420	for ((other_pt, other_rid), buffer) in &mut self.depayloaders {
421	if other_pt != pt && other_rid == rid {
422	buffer.push_padding(meta.clone());
423	}
424	}
425
426	// The entry will be there by now.
427	let buffer = self.depayloaders.get_mut(&key).unwrap();
428
429	buffer.push(meta, packet.payload);
430	}
431
432	pub(crate) fn set_cname(&mut self, cname: String) {
433	self.cname = cname;
434	}
435
436	pub(crate) fn set_msid(&mut self, msid: Msid) {
437	self.msid = msid;
438	}
439
440	pub(crate) fn set_direction(&mut self, new_dir: Direction) {
441	self.need_changed_event = self.dir != new_dir;
442	self.dir = new_dir;
443	}
444
445	pub(crate) fn set_simulcast(&mut self, s: SdpSimulcast) {
446	debug!("Set simulcast: {:?}", s);
447	self.simulcast = Some(s);
448	}
449
450	fn payloader_for(
451	&mut self,
452	pt: Pt,
453	rid: Option<Rid>,
454	params: &[PayloadParams],
455	vp9_mode: Vp9PacketizerMode,
456	) -> &mut Payloader {
457	self.payloaders.entry((pt, rid)).or_insert_with(\|\| {
458	// Unwrap is OK, the pt should be checked already when calling this function.
459	let params = params.iter().find(\|p\| p.pt == pt).unwrap();
460	Payloader::new(params.spec, vp9_mode)
461	})
462	}
463
464	fn set_to_payload(&mut self, to_payload: ToPayload) -> Result<(), RtcError> {
465	if self.to_payload.len() > 100 {
466	return Err(RtcError::WriteWithoutPoll);
467	}
468
469	self.to_payload.push_back(to_payload);
470
471	Ok(())
472	}
473
474	pub(crate) fn poll_timeout(&self) -> Option<Instant> {
475	if !self.to_payload.is_empty() {
476	Some(already_happened())
477	} else {
478	None
479	}
480	}
481
482	pub(crate) fn do_payload(
483	&mut self,
484	streams: &mut Streams,
485	params: &[PayloadParams],
486	vp9_mode: Vp9PacketizerMode,
487	mtu: usize,
488	) -> Result<(), RtcError> {
489	let Some(to_payload) = self.to_payload.pop_front() else {
490	return Ok(());
491	};
492
493	let ToPayload { pt, rid, .. } = &to_payload;
494
495	let is_audio = self.kind.is_audio();
496
497	let midrid = MidRid(self.mid, *rid);
498
499	let stream = streams.stream_tx_by_midrid(midrid);
500
501	let Some(stream) = stream else {
502	return Err(RtcError::NoSenderSource);
503	};
504
505	let pt = *pt;
506
507	let payloader = self.payloader_for(pt, *rid, params, vp9_mode);
508
509	let rtp_size: usize = mtu - SRTP_OVERHEAD;
510	// align to SRTP block size to minimize padding needs
511	let aligned_mtu: usize = rtp_size - rtp_size % SRTP_BLOCK_SIZE;
512
513	payloader
514	.push_sample(to_payload, aligned_mtu, is_audio, stream)
515	.map_err(\|e\| RtcError::Packet(self.mid, pt, e))?;
516
517	Ok(())
518	}
519
520	pub(crate) fn set_remote_pts(&mut self, pts: Vec<Pt>) {
521	// Have we already set PTs?
522	if !self.remote_pts.is_empty() {
523	return;
524	}
525
526	// TODO: We should verify the remote peer doesn't suddenly change the PT
527	// order or removes/adds PTs that weren't there from the start.
528	debug!("Mid ({}) remote PT order is: {:?}", self.mid, pts);
529	self.remote_pts = pts;
530	}
531
532	pub(crate) fn set_remote_extmap(&mut self, exts: ExtensionMap) {
533	self.remote_exts = exts;
534	}
535
536	/// The remote PT (payload types) configured for this Media.
537	///
538	/// These are negotiated with the remote peer and is the order the remote prefer them.
539	///
540	/// I.e. these can be fewer than the `PayloadParams` configured for the `Rtc` instance,
541	/// and in a different order.
542	pub fn remote_pts(&self) -> &[Pt] {
543	&self.remote_pts
544	}
545
546	/// The remote, agreed on, extension map, configured for this Media.
547	///
548	/// For the SDP API, these are negotiated with the remote peer.
549	///
550	/// For the Direct API, these are a clone of the session configured values narrowed by media
551	/// kind (audio/video).
552	pub fn remote_extmap(&self) -> &ExtensionMap {
553	&self.remote_exts
554	}
555
556	pub(crate) fn remote_created(&self) -> bool {
557	self.remote_created
558	}
559
560	pub(crate) fn first_pt_with_rtx(&self, config: &CodecConfig) -> Option<Pt> {
561	config
562	.all_for_kind(self.kind)
563	// Only consider negotiated PTs
564	.filter(\|p\| self.remote_pts.contains(&p.pt))
565	// Map to the first PT found in payload params with RTX
566	.find_map(\|p\| p.resend().map(\|_\| p.pt))
567	}
568
569	pub(crate) fn reset_depayloader(&mut self, payload_type: Pt, rid: Option<Rid>) {
570	// Simply remove the depayloader, it will be re-created on the next RTP packet.
571	self.depayloaders.remove(&(payload_type, rid));
572	}
573
574	pub(crate) fn reset_depayloaders_for_rid(&mut self, rid: Option<Rid>) {
575	self.depayloaders
576	.retain(\|(_, existing_rid), _\| *existing_rid != rid);
577	}
578
579	pub(crate) fn set_rid_rx(&mut self, rids: Rids) {
580	self.rids_rx = rids;
581	}
582
583	pub(crate) fn set_rid_tx(&mut self, rids: Rids) {
584	self.rids_tx = rids;
585	}
586
587	pub(crate) fn add_to_rid_tx(&mut self, rid: Rid) {
588	self.rids_tx.add(rid)
589	}
590	}
591
592	impl Default for Media {
593	fn default() -> Self {
594	Self {
595	mid: Mid::new(),
596	index: 0,
597	app_tmp: false,
598	cname: Id::<20>::random().to_string(),
599	msid: Msid::random(),
600	kind: MediaKind::Video,
601	remote_pts: vec![],
602	stopped: false,
603	remote_exts: ExtensionMap::empty(),
604	remote_created: false,
605	dir: Direction::SendRecv,
606	simulcast: None,
607	rids_rx: Rids::Any,
608	rids_tx: Rids::None,
609	payloaders: HashMap::new(),
610	depayloaders: HashMap::new(),
611	to_payload: VecDeque::default(),
612	need_open_event: true,
613	need_changed_event: false,
614	}
615	}
616	}
617
618	impl Media {
619	pub(crate) fn from_remote_media_line(
620	l: &MediaLine,
621	index: usize,
622	remote_created: bool,
623	) -> Self {
624	Media {
625	mid: l.mid(),
626	index,
627	// This is not reflected back, and thus added by add_pending_changes().
628	// cname,
629	msid: l.msid().unwrap_or(Msid::random()),
630	kind: l.typ.clone().into(),
631	dir: if l.disabled {
632	Direction::Inactive
633	} else {
634	l.direction().invert() // remote direction is reverse.
635	},
636	remote_created,
637	..Default::default()
638	}
639	}
640
641	// Going from AddMedia to Media for pending in a Change and are sent
642	// in the offer to the other side.
643	//
644	// from_add_media is only used when creating temporary Media to be
645	// included in the SDP. We don't want to make an _actual_ changes with this.
646	pub(crate) fn from_add_media(a: AddMedia) -> Self {
647	Media {
648	mid: a.mid,
649	index: a.index,
650	cname: a.cname,
651	msid: a.msid,
652	kind: a.kind,
653	dir: a.dir,
654	remote_pts: a.pts,
655	remote_exts: a.exts,
656	remote_created: false,
657	simulcast: a.simulcast.map(\|s\| s.into_sdp()),
658	..Default::default()
659	}
660	}
661
662	pub(crate) fn from_app_tmp(mid: Mid, index: usize) -> Media {
663	Media {
664	mid,
665	index,
666	app_tmp: true,
667	..Default::default()
668	}
669	}
670
671	pub(crate) fn from_direct_api(
672	mid: Mid,
673	index: usize,
674	kind: MediaKind,
675	exts: ExtensionMap,
676	) -> Media {
677	Media {
678	mid,
679	index,
680	kind,
681	dir: Direction::SendRecv,
682	remote_exts: exts,
683	..Default::default()
684	}
685	}
686	}