screencapturekit/cm/
sample_buffer.rs

1//! `CMSampleBuffer` — re-exported from [`apple_cf::cm::CMSampleBuffer`] plus
2//! `ScreenCaptureKit`-specific extension traits for the `SCStreamFrameInfo`
3//! attachment readers and the few sample-buffer accessors that aren't
4//! framework-agnostic enough to live in `apple-cf` yet.
5//!
6//! Bring [`CMSampleBufferSCExt`] into scope to call `frame_status()`,
7//! `display_time()`, `frame_info()`, etc. on any `CMSampleBuffer` carrying
8//! `ScreenCaptureKit` attachments.
9//!
10//! Bring [`CMSampleBufferExt`] into scope for the
11//! `image_buffer()`/`audio_buffer_list()`/`make_data_ready()` accessors
12//! that are pending an apple-cf v0.2 API addition.
13
14use super::ffi;
15use super::{
16    AudioBuffer, AudioBufferList, AudioBufferListRaw, CMBlockBuffer, CMSampleTimingInfo, CMTime,
17    SCFrameStatus,
18};
19use crate::cv::CVPixelBuffer;
20
21/// Re-exported `CMSampleBuffer` — same opaque-pointer wrapper used across
22/// the doom-fish suite.
23pub use apple_cf::cm::CMSampleBuffer;
24
25// ------------------------------------------------------------------
26// FrameInfoFields — bit flags for the batched frame_info reader.
27// ------------------------------------------------------------------
28
29/// Bit flags marking which fields the batched [`CMSampleBufferSCExt::frame_info`]
30/// fetch managed to populate. Mirrors `FrameInfoFieldBits` in the Swift
31/// bridge — keep them in sync.
32struct FrameInfoFields;
33
34impl FrameInfoFields {
35    const STATUS: u32 = 1 << 0;
36    const DISPLAY_TIME: u32 = 1 << 1;
37    const SCALE_FACTOR: u32 = 1 << 2;
38    const CONTENT_SCALE: u32 = 1 << 3;
39    const CONTENT_RECT: u32 = 1 << 4;
40    const BOUNDING_RECT: u32 = 1 << 5;
41    const SCREEN_RECT: u32 = 1 << 6;
42    const PRESENTER_OVERLAY_RECT: u32 = 1 << 7;
43}
44
45/// Snapshot of every `SCStreamFrameInfo` attachment on a sample buffer.
46///
47/// Returned by [`CMSampleBufferSCExt::frame_info`]. Each field is `Some` when
48/// the underlying attachment was present (depends on macOS version, output
49/// type, and stream configuration); `None` indicates the attachment was
50/// missing.
51#[allow(clippy::derive_partial_eq_without_eq)]
52#[derive(Debug, Default, Clone, PartialEq)]
53pub struct FrameInfo {
54    /// `SCStreamFrameInfo.status` — frame completeness / idle state.
55    pub frame_status: Option<SCFrameStatus>,
56    /// `SCStreamFrameInfo.displayTime` — mach absolute time the frame was
57    /// composited.
58    pub display_time: Option<u64>,
59    /// `SCStreamFrameInfo.scaleFactor` — display scale (e.g. 2.0 for Retina).
60    pub scale_factor: Option<f64>,
61    /// `SCStreamFrameInfo.contentScale` — capture scale relative to the
62    /// source content.
63    pub content_scale: Option<f64>,
64    /// `SCStreamFrameInfo.contentRect` — captured content within the frame.
65    pub content_rect: Option<crate::cg::CGRect>,
66    /// `SCStreamFrameInfo.boundingRect` — bounding rect of all captured
67    /// windows (macOS 14.0+).
68    pub bounding_rect: Option<crate::cg::CGRect>,
69    /// `SCStreamFrameInfo.screenRect` — full screen rect (macOS 13.1+).
70    pub screen_rect: Option<crate::cg::CGRect>,
71    /// `SCStreamFrameInfo.presenterOverlayContentRect` — Presenter Overlay
72    /// bounding rect (macOS 14.2+).
73    pub presenter_overlay_content_rect: Option<crate::cg::CGRect>,
74}
75
76// ------------------------------------------------------------------
77// CMSampleBufferSCExt — ScreenCaptureKit-specific attachment readers.
78// ------------------------------------------------------------------
79
80/// Extension trait that exposes `SCStreamFrameInfo` attachment accessors on
81/// any [`CMSampleBuffer`] produced by `ScreenCaptureKit`.
82///
83/// These are SC-specific by design: they read attachment keys defined on
84/// `SCStreamFrameInfo` and are meaningless on sample buffers from other
85/// sources (videotoolbox, `AVFoundation` capture, etc.).
86pub trait CMSampleBufferSCExt {
87    /// `SCStreamFrameInfo.status` attachment.
88    fn frame_status(&self) -> Option<SCFrameStatus>;
89    /// `SCStreamFrameInfo.displayTime` attachment.
90    fn display_time(&self) -> Option<u64>;
91    /// `SCStreamFrameInfo.scaleFactor` attachment.
92    fn scale_factor(&self) -> Option<f64>;
93    /// `SCStreamFrameInfo.contentScale` attachment.
94    fn content_scale(&self) -> Option<f64>;
95    /// `SCStreamFrameInfo.contentRect` attachment.
96    fn content_rect(&self) -> Option<crate::cg::CGRect>;
97    /// `SCStreamFrameInfo.boundingRect` attachment.
98    fn bounding_rect(&self) -> Option<crate::cg::CGRect>;
99    /// `SCStreamFrameInfo.screenRect` attachment.
100    fn screen_rect(&self) -> Option<crate::cg::CGRect>;
101    /// `SCStreamFrameInfo.presenterOverlayContentRect` attachment.
102    fn presenter_overlay_content_rect(&self) -> Option<crate::cg::CGRect>;
103    /// `SCStreamFrameInfo.dirtyRects` attachment.
104    fn dirty_rects(&self) -> Option<Vec<crate::cg::CGRect>>;
105    /// Read every populated `SCStreamFrameInfo` attachment in a single
106    /// FFI round-trip.
107    fn frame_info(&self) -> Option<FrameInfo>;
108}
109
110impl CMSampleBufferSCExt for CMSampleBuffer {
111    fn frame_status(&self) -> Option<SCFrameStatus> {
112        unsafe {
113            let status = ffi::cm_sample_buffer_get_frame_status(self.as_ptr());
114            if status >= 0 {
115                SCFrameStatus::from_raw(status)
116            } else {
117                None
118            }
119        }
120    }
121
122    fn display_time(&self) -> Option<u64> {
123        unsafe {
124            let mut value: u64 = 0;
125            if ffi::cm_sample_buffer_get_display_time(self.as_ptr(), &mut value) {
126                Some(value)
127            } else {
128                None
129            }
130        }
131    }
132
133    fn scale_factor(&self) -> Option<f64> {
134        unsafe {
135            let mut value: f64 = 0.0;
136            if ffi::cm_sample_buffer_get_scale_factor(self.as_ptr(), &mut value) {
137                Some(value)
138            } else {
139                None
140            }
141        }
142    }
143
144    fn content_scale(&self) -> Option<f64> {
145        unsafe {
146            let mut value: f64 = 0.0;
147            if ffi::cm_sample_buffer_get_content_scale(self.as_ptr(), &mut value) {
148                Some(value)
149            } else {
150                None
151            }
152        }
153    }
154
155    fn content_rect(&self) -> Option<crate::cg::CGRect> {
156        unsafe {
157            let mut x = 0.0;
158            let mut y = 0.0;
159            let mut w = 0.0;
160            let mut h = 0.0;
161            if ffi::cm_sample_buffer_get_content_rect(self.as_ptr(), &mut x, &mut y, &mut w, &mut h)
162            {
163                Some(crate::cg::CGRect::new(x, y, w, h))
164            } else {
165                None
166            }
167        }
168    }
169
170    fn bounding_rect(&self) -> Option<crate::cg::CGRect> {
171        unsafe {
172            let mut x = 0.0;
173            let mut y = 0.0;
174            let mut w = 0.0;
175            let mut h = 0.0;
176            if ffi::cm_sample_buffer_get_bounding_rect(
177                self.as_ptr(),
178                &mut x,
179                &mut y,
180                &mut w,
181                &mut h,
182            ) {
183                Some(crate::cg::CGRect::new(x, y, w, h))
184            } else {
185                None
186            }
187        }
188    }
189
190    fn screen_rect(&self) -> Option<crate::cg::CGRect> {
191        unsafe {
192            let mut x = 0.0;
193            let mut y = 0.0;
194            let mut w = 0.0;
195            let mut h = 0.0;
196            if ffi::cm_sample_buffer_get_screen_rect(self.as_ptr(), &mut x, &mut y, &mut w, &mut h)
197            {
198                Some(crate::cg::CGRect::new(x, y, w, h))
199            } else {
200                None
201            }
202        }
203    }
204
205    fn presenter_overlay_content_rect(&self) -> Option<crate::cg::CGRect> {
206        #[cfg(feature = "macos_14_2")]
207        unsafe {
208            let mut x = 0.0;
209            let mut y = 0.0;
210            let mut w = 0.0;
211            let mut h = 0.0;
212            if ffi::cm_sample_buffer_get_presenter_overlay_content_rect(
213                self.as_ptr(),
214                &mut x,
215                &mut y,
216                &mut w,
217                &mut h,
218            ) {
219                Some(crate::cg::CGRect::new(x, y, w, h))
220            } else {
221                None
222            }
223        }
224        #[cfg(not(feature = "macos_14_2"))]
225        None
226    }
227
228    fn dirty_rects(&self) -> Option<Vec<crate::cg::CGRect>> {
229        unsafe {
230            let mut rects_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
231            let mut count: usize = 0;
232            if !ffi::cm_sample_buffer_get_dirty_rects(self.as_ptr(), &mut rects_ptr, &mut count) {
233                return None;
234            }
235            if rects_ptr.is_null() || count == 0 {
236                return None;
237            }
238            let rects_typed = rects_ptr.cast::<f64>();
239            let mut rects = Vec::with_capacity(count);
240            for i in 0..count {
241                let base = rects_typed.add(i * 4);
242                rects.push(crate::cg::CGRect::new(
243                    *base,
244                    *base.add(1),
245                    *base.add(2),
246                    *base.add(3),
247                ));
248            }
249            ffi::cm_sample_buffer_free_dirty_rects(rects_ptr);
250            Some(rects)
251        }
252    }
253
254    fn frame_info(&self) -> Option<FrameInfo> {
255        unsafe {
256            let mut fields: u32 = 0;
257            let mut status: i32 = 0;
258            let mut display_time: u64 = 0;
259            let mut scale_factor: f64 = 0.0;
260            let mut content_scale: f64 = 0.0;
261            let mut content_rect = [0.0_f64; 4];
262            let mut bounding_rect = [0.0_f64; 4];
263            let mut screen_rect = [0.0_f64; 4];
264            let mut presenter_overlay_rect = [0.0_f64; 4];
265            if !ffi::cm_sample_buffer_get_frame_info(
266                self.as_ptr(),
267                &mut fields,
268                &mut status,
269                &mut display_time,
270                &mut scale_factor,
271                &mut content_scale,
272                content_rect.as_mut_ptr(),
273                bounding_rect.as_mut_ptr(),
274                screen_rect.as_mut_ptr(),
275                presenter_overlay_rect.as_mut_ptr(),
276            ) {
277                return None;
278            }
279            let to_rect = |a: [f64; 4]| crate::cg::CGRect::new(a[0], a[1], a[2], a[3]);
280            Some(FrameInfo {
281                frame_status: ((fields & FrameInfoFields::STATUS) != 0)
282                    .then(|| SCFrameStatus::from_raw(status))
283                    .flatten(),
284                display_time: ((fields & FrameInfoFields::DISPLAY_TIME) != 0)
285                    .then_some(display_time),
286                scale_factor: ((fields & FrameInfoFields::SCALE_FACTOR) != 0)
287                    .then_some(scale_factor),
288                content_scale: ((fields & FrameInfoFields::CONTENT_SCALE) != 0)
289                    .then_some(content_scale),
290                content_rect: ((fields & FrameInfoFields::CONTENT_RECT) != 0)
291                    .then(|| to_rect(content_rect)),
292                bounding_rect: ((fields & FrameInfoFields::BOUNDING_RECT) != 0)
293                    .then(|| to_rect(bounding_rect)),
294                screen_rect: ((fields & FrameInfoFields::SCREEN_RECT) != 0)
295                    .then(|| to_rect(screen_rect)),
296                presenter_overlay_content_rect: ((fields
297                    & FrameInfoFields::PRESENTER_OVERLAY_RECT)
298                    != 0)
299                    .then(|| to_rect(presenter_overlay_rect)),
300            })
301        }
302    }
303}
304
305// ------------------------------------------------------------------
306// CMSampleBufferExt — generic accessors not yet in apple-cf.
307// ------------------------------------------------------------------
308
309/// Extension trait carrying generic `CMSampleBuffer` accessors that aren't
310/// available on [`apple_cf::cm::CMSampleBuffer`] yet (planned for an
311/// `apple-cf` v0.2 release).
312pub trait CMSampleBufferExt {
313    /// Construct a sample buffer wrapping a `CVPixelBuffer`.
314    ///
315    /// # Errors
316    ///
317    /// Returns the underlying `OSStatus` if `CoreMedia` fails to create the
318    /// sample buffer.
319    fn create_for_image_buffer(
320        image_buffer: &CVPixelBuffer,
321        presentation_time: CMTime,
322        duration: CMTime,
323    ) -> Result<Self, i32>
324    where
325        Self: Sized;
326
327    /// Borrow the attached `CVPixelBuffer`, if any.
328    fn image_buffer(&self) -> Option<CVPixelBuffer>;
329
330    /// Read the audio sample buffer's underlying `AudioBufferList`, if any.
331    fn audio_buffer_list(&self) -> Option<AudioBufferList>;
332
333    /// Output presentation timestamp (after timing adjustments).
334    fn output_presentation_timestamp(&self) -> CMTime;
335
336    /// Override the output presentation timestamp.
337    ///
338    /// # Errors
339    ///
340    /// Returns the underlying `OSStatus` if `CoreMedia` rejects the new value.
341    fn set_output_presentation_timestamp(&self, time: CMTime) -> Result<(), i32>;
342
343    /// Size of one sample at `index` in bytes.
344    fn sample_size(&self, index: usize) -> usize;
345
346    /// Sum of all sample sizes in this buffer.
347    fn total_sample_size(&self) -> usize;
348
349    /// Whether the underlying data is ready for reading.
350    fn is_data_ready(&self) -> bool;
351
352    /// Mark the underlying data as ready (flushes any pending make-data-ready
353    /// callbacks).
354    ///
355    /// # Errors
356    ///
357    /// Returns the underlying `OSStatus` if `CoreMedia` reports failure.
358    fn make_data_ready(&self) -> Result<(), i32>;
359
360    /// Read the timing info for the sample at `index`.
361    ///
362    /// # Errors
363    ///
364    /// Returns the underlying `OSStatus` if `index` is out of range.
365    fn sample_timing_info(&self, index: usize) -> Result<CMSampleTimingInfo, i32>;
366
367    /// Build an [`apple_cf::cg::CGImage`] from the buffer's attached
368    /// `CVImageBuffer`.
369    ///
370    /// Backed by `VTCreateCGImageFromCVPixelBuffer`, which understands every
371    /// pixel format `ScreenCaptureKit` (or any other `CoreMedia` producer) can
372    /// emit — BGRA, 420v YCbCr 8-bit bi-planar video range, l10r 10-bit ARGB,
373    /// etc. — and uses Apple's hardware path when one exists. The resulting
374    /// `CGImage` is `IOSurface`-backed when the source was, so passing it
375    /// straight into `ImageIO` (`CGImageDestinationAddImage` / `imageio-rs`
376    /// `ImageDestination::add_cg_image`) or into Metal sampling avoids any
377    /// host-side pixel copy.
378    ///
379    /// Returns the canonical `apple_cf::cg::CGImage` (the same type used by
380    /// `imageio-rs` and every other doom-fish suite crate that consumes
381    /// `CGImage`s), so the result flows straight into safe APIs with no
382    /// pointer juggling at the callsite.
383    ///
384    /// # Errors
385    ///
386    /// Returns the underlying `OSStatus` from `VTCreateCGImageFromCVPixelBuffer`
387    /// (or `-12731` `kCMSampleBufferError_NoSampleBufferContent` when the
388    /// buffer has no image buffer attached — typical for audio-only or
389    /// timing-metadata-only samples).
390    fn cg_image(&self) -> Result<apple_cf::cg::CGImage, i32>;
391}
392
393impl CMSampleBufferExt for CMSampleBuffer {
394    fn create_for_image_buffer(
395        image_buffer: &CVPixelBuffer,
396        presentation_time: CMTime,
397        duration: CMTime,
398    ) -> Result<Self, i32> {
399        unsafe {
400            let mut sample_buffer_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
401            let status = ffi::cm_sample_buffer_create_for_image_buffer(
402                image_buffer.as_ptr(),
403                presentation_time.value,
404                presentation_time.timescale,
405                duration.value,
406                duration.timescale,
407                &mut sample_buffer_ptr,
408            );
409            if status == 0 && !sample_buffer_ptr.is_null() {
410                Self::from_raw(sample_buffer_ptr).ok_or(status)
411            } else {
412                Err(status)
413            }
414        }
415    }
416
417    fn image_buffer(&self) -> Option<CVPixelBuffer> {
418        unsafe {
419            // SAFETY: cm_sample_buffer_get_image_buffer returns a +1
420            // (passRetained) CVImageBuffer; CVPixelBuffer::from_raw adopts that
421            // +1 reference, so ownership is balanced (released on drop).
422            let ptr = ffi::cm_sample_buffer_get_image_buffer(self.as_ptr());
423            CVPixelBuffer::from_raw(ptr)
424        }
425    }
426
427    fn audio_buffer_list(&self) -> Option<AudioBufferList> {
428        unsafe {
429            let mut num_buffers: u32 = 0;
430            let mut buffers_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
431            let mut buffers_len: usize = 0;
432            let mut block_buffer_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
433
434            ffi::cm_sample_buffer_get_audio_buffer_list(
435                self.as_ptr(),
436                &mut num_buffers,
437                &mut buffers_ptr,
438                &mut buffers_len,
439                &mut block_buffer_ptr,
440            );
441
442            if num_buffers == 0 {
443                None
444            } else {
445                Some(AudioBufferList {
446                    inner: AudioBufferListRaw {
447                        num_buffers,
448                        buffers_ptr: buffers_ptr.cast::<AudioBuffer>(),
449                        buffers_len,
450                    },
451                    block_buffer_ptr,
452                })
453            }
454        }
455    }
456
457    fn output_presentation_timestamp(&self) -> CMTime {
458        unsafe {
459            let mut value: i64 = 0;
460            let mut timescale: i32 = 0;
461            let mut flags: u32 = 0;
462            let mut epoch: i64 = 0;
463            ffi::cm_sample_buffer_get_output_presentation_timestamp(
464                self.as_ptr(),
465                &mut value,
466                &mut timescale,
467                &mut flags,
468                &mut epoch,
469            );
470            CMTime {
471                value,
472                timescale,
473                flags,
474                epoch,
475            }
476        }
477    }
478
479    fn set_output_presentation_timestamp(&self, time: CMTime) -> Result<(), i32> {
480        let status = unsafe {
481            ffi::cm_sample_buffer_set_output_presentation_timestamp(
482                self.as_ptr(),
483                time.value,
484                time.timescale,
485                time.flags,
486                time.epoch,
487            )
488        };
489        if status == 0 {
490            Ok(())
491        } else {
492            Err(status)
493        }
494    }
495
496    fn sample_size(&self, index: usize) -> usize {
497        unsafe { ffi::cm_sample_buffer_get_sample_size(self.as_ptr(), index) }
498    }
499
500    fn total_sample_size(&self) -> usize {
501        unsafe { ffi::cm_sample_buffer_get_total_sample_size(self.as_ptr()) }
502    }
503
504    fn is_data_ready(&self) -> bool {
505        unsafe { ffi::cm_sample_buffer_is_ready_for_data_access(self.as_ptr()) }
506    }
507
508    fn make_data_ready(&self) -> Result<(), i32> {
509        let status = unsafe { ffi::cm_sample_buffer_make_data_ready(self.as_ptr()) };
510        if status == 0 {
511            Ok(())
512        } else {
513            Err(status)
514        }
515    }
516
517    fn sample_timing_info(&self, index: usize) -> Result<CMSampleTimingInfo, i32> {
518        unsafe {
519            let mut dur_v: i64 = 0;
520            let mut dur_s: i32 = 0;
521            let mut dur_f: u32 = 0;
522            let mut dur_e: i64 = 0;
523            let mut pts_v: i64 = 0;
524            let mut pts_s: i32 = 0;
525            let mut pts_f: u32 = 0;
526            let mut pts_e: i64 = 0;
527            let mut dts_v: i64 = 0;
528            let mut dts_s: i32 = 0;
529            let mut dts_f: u32 = 0;
530            let mut dts_e: i64 = 0;
531            let status = ffi::cm_sample_buffer_get_sample_timing_info(
532                self.as_ptr(),
533                index,
534                &mut dur_v,
535                &mut dur_s,
536                &mut dur_f,
537                &mut dur_e,
538                &mut pts_v,
539                &mut pts_s,
540                &mut pts_f,
541                &mut pts_e,
542                &mut dts_v,
543                &mut dts_s,
544                &mut dts_f,
545                &mut dts_e,
546            );
547            if status == 0 {
548                Ok(CMSampleTimingInfo {
549                    duration: CMTime {
550                        value: dur_v,
551                        timescale: dur_s,
552                        flags: dur_f,
553                        epoch: dur_e,
554                    },
555                    presentation_time_stamp: CMTime {
556                        value: pts_v,
557                        timescale: pts_s,
558                        flags: pts_f,
559                        epoch: pts_e,
560                    },
561                    decode_time_stamp: CMTime {
562                        value: dts_v,
563                        timescale: dts_s,
564                        flags: dts_f,
565                        epoch: dts_e,
566                    },
567                })
568            } else {
569                Err(status)
570            }
571        }
572    }
573
574    fn cg_image(&self) -> Result<apple_cf::cg::CGImage, i32> {
575        unsafe {
576            let mut status: i32 = 0;
577            let ptr = ffi::cm_sample_buffer_create_cg_image(self.as_ptr(), &mut status);
578            if !ptr.is_null() && status == 0 {
579                // Safety: the Swift bridge returns a retained CGImage on
580                // success; passing it straight to CGImage::from_raw takes
581                // ownership of that refcount.
582                Ok(apple_cf::cg::CGImage::from_raw(ptr.cast_mut()))
583            } else {
584                Err(status)
585            }
586        }
587    }
588}
589
590// ------------------------------------------------------------------
591// data_buffer wrapper that returns the *local* CMBlockBuffer type
592// (for backward compat). apple_cf::cm::CMSampleBuffer also has its own
593// data_buffer() returning apple_cf::cm::CMBlockBuffer; the local one
594// here returns crate::cm::CMBlockBuffer which currently is its own
595// type. (Merging the two is Phase 4.)
596// ------------------------------------------------------------------
597
598/// Convenience: like [`apple_cf::cm::CMSampleBuffer::data_buffer`] but
599/// returns the local `crate::cm::CMBlockBuffer` (which is currently a
600/// different wrapper around the same underlying type).
601pub trait CMSampleBufferDataBufferExt {
602    fn data_buffer_local(&self) -> Option<CMBlockBuffer>;
603}
604
605impl CMSampleBufferDataBufferExt for CMSampleBuffer {
606    fn data_buffer_local(&self) -> Option<CMBlockBuffer> {
607        unsafe {
608            let ptr = ffi::cm_sample_buffer_get_data_buffer(self.as_ptr());
609            if ptr.is_null() {
610                return None;
611            }
612            // `CMSampleBufferGetDataBuffer` returns a +0 (unretained) reference.
613            // `CMBlockBuffer::from_raw` adopts a +1 reference and releases on
614            // drop, so we must retain first to keep the refcount balanced.
615            // (Mirrors apple-cf's own `CMSampleBuffer::data_buffer`.)
616            let retained = ffi::cm_block_buffer_retain(ptr);
617            CMBlockBuffer::from_raw(retained)
618        }
619    }
620}
screencapturekit/cm/sample_buffer.rs

screencapturekit/cm/
sample_buffer.rs
