1 /*****************************************************************************
2 * ios.m: iOS OpenGL ES provider
3 *****************************************************************************
4 * Copyright (C) 2001-2017 VLC authors and VideoLAN
7 * Authors: Pierre d'Herbemont <pdherbemont at videolan dot org>
8 * Felix Paul Kühne <fkuehne at videolan dot org>
9 * David Fuhrmann <david dot fuhrmann at googlemail dot com>
11 * Laurent Aimar <fenrir _AT_ videolan _DOT_ org>
12 * Eric Petit <titer@m0k.org>
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU Lesser General Public License as published by
16 * the Free Software Foundation; either version 2.1 of the License, or
17 * (at your option) any later version.
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU Lesser General Public License for more details.
24 * You should have received a copy of the GNU Lesser General Public License
25 * along with this program; if not, write to the Free Software Foundation,
26 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
27 *****************************************************************************/
29 /*****************************************************************************
31 *****************************************************************************/
33 #import <UIKit/UIKit.h>
34 #import <OpenGLES/EAGL.h>
35 #import <OpenGLES/ES2/gl.h>
36 #import <OpenGLES/ES2/glext.h>
37 #import <QuartzCore/QuartzCore.h>
44 #import <vlc_common.h>
45 #import <vlc_plugin.h>
46 #import <vlc_vout_display.h>
47 #import <vlc_opengl.h>
48 #import <vlc_dialog.h>
49 #import "opengl/vout_helper.h"
52 * Forward declarations
55 struct picture_sys_t {
56 CVPixelBufferRef pixelBuffer;
59 static int Open(vlc_object_t *);
60 static void Close(vlc_object_t *);
62 static picture_pool_t* PicturePool(vout_display_t *, unsigned);
63 static void PictureRender(vout_display_t *, picture_t *, subpicture_t *);
64 static void PictureDisplay(vout_display_t *, picture_t *, subpicture_t *);
65 static int Control(vout_display_t*, int, va_list);
67 static void *OurGetProcAddress(vlc_gl_t *, const char *);
69 static int OpenglESLock(vlc_gl_t *);
70 static void OpenglESSwap(vlc_gl_t *);
71 static void OpenglESUnlock(vlc_gl_t *);
77 set_shortname("iOS vout")
78 set_description("iOS OpenGL video output")
79 set_category(CAT_VIDEO)
80 set_subcategory(SUBCAT_VIDEO_VOUT)
81 set_capability("vout display", 300)
82 set_callbacks(Open, Close)
84 add_shortcut("vout_ios2", "vout_ios")
88 @interface VLCOpenGLES2VideoView : UIView {
89 vout_display_t *_voutDisplay;
90 EAGLContext *_eaglContext;
91 EAGLContext *_previousEaglContext;
95 BOOL _bufferNeedReset;
98 @property (readonly) GLuint renderBuffer;
99 @property (readonly) GLuint frameBuffer;
100 @property (readwrite) vout_display_t* voutDisplay;
101 @property (readonly) EAGLContext* eaglContext;
102 @property (readonly) BOOL isAppActive;
103 @property GLuint shaderProgram;
105 - (void)createBuffers;
106 - (void)destroyBuffers;
107 - (void)resetBuffers;
112 - (void)propagateDimensionsToVoutCore;
116 struct vout_display_sys_t
118 VLCOpenGLES2VideoView *glESView;
119 UIView *viewContainer;
120 UITapGestureRecognizer *tapRecognizer;
123 vout_display_opengl_t *vgl;
125 picture_pool_t *picturePool;
127 vout_display_place_t place;
132 VLCOpenGLES2VideoView *glESView;
135 static void *OurGetProcAddress(vlc_gl_t *gl, const char *name)
139 return dlsym(RTLD_DEFAULT, name);
142 static int Open(vlc_object_t *this)
144 vout_display_t *vd = (vout_display_t *)this;
146 if (vout_display_IsWindowed(vd))
149 vout_display_sys_t *sys = vlc_obj_calloc (this, 1, sizeof(*sys));
155 sys->picturePool = NULL;
158 var_Create(vd->obj.parent, "ios-eaglcontext", VLC_VAR_ADDRESS);
161 /* setup the actual OpenGL ES view */
162 [VLCOpenGLES2VideoView performSelectorOnMainThread:@selector(getNewView:)
163 withObject:[NSValue valueWithPointer:&sys->glESView]
165 [sys->glESView setVoutDisplay:vd];
167 if (!sys->glESView) {
168 msg_Err(vd, "Creating OpenGL ES 2 view failed");
172 [sys->glESView performSelectorOnMainThread:@selector(fetchViewContainer) withObject:nil waitUntilDone:YES];
173 if (!sys->viewContainer) {
174 msg_Err(vd, "Fetching view container failed");
178 const vlc_fourcc_t *subpicture_chromas;
179 video_format_t fmt = vd->fmt;
181 sys->gl = vlc_object_create(this, sizeof(*sys->gl));
185 struct gl_sys *glsys = sys->gl->sys =
186 vlc_obj_malloc(this, sizeof(struct gl_sys));
187 if (unlikely(!sys->gl->sys))
189 glsys->glESView = sys->glESView;
190 /* Initialize common OpenGL video display */
191 sys->gl->makeCurrent = OpenglESLock;
192 sys->gl->releaseCurrent = OpenglESUnlock;
193 sys->gl->swap = OpenglESSwap;
194 sys->gl->getProcAddress = OurGetProcAddress;
196 if (vlc_gl_MakeCurrent(sys->gl) != VLC_SUCCESS)
199 var_SetAddress(vd->obj.parent, "ios-eaglcontext", [sys->glESView eaglContext]);
201 sys->vgl = vout_display_opengl_New(&vd->fmt, &subpicture_chromas,
202 sys->gl, &vd->cfg->viewpoint);
203 vlc_gl_ReleaseCurrent(sys->gl);
208 vout_display_info_t info = vd->info;
209 info.has_pictures_invalid = false;
210 info.subpicture_chromas = subpicture_chromas;
212 /* Setup vout_display_t once everything is fine */
215 vd->pool = PicturePool;
216 vd->prepare = PictureRender;
217 vd->display = PictureDisplay;
218 vd->control = Control;
220 /* forward our dimensions to the vout core */
221 [sys->glESView performSelectorOnMainThread:@selector(propagateDimensionsToVoutCore) withObject:nil waitUntilDone:YES];
224 [[NSNotificationCenter defaultCenter] addObserver:sys->glESView
225 selector:@selector(applicationStateChanged:)
226 name:UIApplicationWillResignActiveNotification
228 [[NSNotificationCenter defaultCenter] addObserver:sys->glESView
229 selector:@selector(applicationStateChanged:)
230 name:UIApplicationDidBecomeActiveNotification
232 [sys->glESView reshape];
241 static void Close (vlc_object_t *this)
243 vout_display_t *vd = (vout_display_t *)this;
244 vout_display_sys_t *sys = vd->sys;
247 if (sys->tapRecognizer) {
248 [sys->tapRecognizer.view performSelectorOnMainThread:@selector(removeGestureRecognizer:) withObject:sys->tapRecognizer waitUntilDone:YES];
249 [sys->tapRecognizer release];
252 [sys->glESView setVoutDisplay:nil];
254 var_Destroy (vd, "drawable-nsobject");
255 @synchronized(sys->viewContainer) {
256 [sys->glESView performSelectorOnMainThread:@selector(removeFromSuperview) withObject:nil waitUntilDone:NO];
257 [sys->viewContainer performSelectorOnMainThread:@selector(release) withObject:nil waitUntilDone:NO];
259 sys->viewContainer = nil;
261 if (sys->gl != NULL) {
262 @synchronized (sys->glESView) {
263 msg_Dbg(this, "deleting display");
265 if (likely(sys->vgl))
267 vlc_gl_MakeCurrent(sys->gl);
268 vout_display_opengl_Delete(sys->vgl);
269 vlc_gl_ReleaseCurrent(sys->gl);
272 vlc_object_release(sys->gl);
275 [sys->glESView release];
277 var_Destroy(vd->obj.parent, "ios-eaglcontext");
280 /*****************************************************************************
281 * vout display callbacks
282 *****************************************************************************/
284 static int Control(vout_display_t *vd, int query, va_list ap)
286 vout_display_sys_t *sys = vd->sys;
289 case VOUT_DISPLAY_CHANGE_DISPLAY_FILLED:
290 case VOUT_DISPLAY_CHANGE_ZOOM:
291 case VOUT_DISPLAY_CHANGE_SOURCE_ASPECT:
292 case VOUT_DISPLAY_CHANGE_SOURCE_CROP:
293 case VOUT_DISPLAY_CHANGE_DISPLAY_SIZE:
299 const vout_display_cfg_t *cfg;
301 if (vlc_gl_MakeCurrent(sys->gl) != VLC_SUCCESS)
304 if (query == VOUT_DISPLAY_CHANGE_SOURCE_ASPECT ||
305 query == VOUT_DISPLAY_CHANGE_SOURCE_CROP) {
308 cfg = (const vout_display_cfg_t*)va_arg(ap, const vout_display_cfg_t *);
311 /* we don't adapt anything here regardless of what the vout core
312 * wants since we are not in a traditional desktop window */
316 vout_display_cfg_t cfg_tmp = *cfg;
318 viewSize = [sys->glESView viewSize];
320 /* on HiDPI displays, the point bounds don't equal the actual pixels */
321 CGFloat scaleFactor = sys->glESView.contentScaleFactor;
322 cfg_tmp.display.width = viewSize.width * scaleFactor;
323 cfg_tmp.display.height = viewSize.height * scaleFactor;
325 vout_display_place_t place;
326 vout_display_PlacePicture(&place, &vd->source, &cfg_tmp, false);
327 @synchronized (sys->glESView) {
331 vout_display_opengl_SetWindowAspectRatio(sys->vgl, (float)place.width / place.height);
333 // x / y are top left corner, but we need the lower left one
334 if (query != VOUT_DISPLAY_CHANGE_DISPLAY_SIZE)
335 vout_display_opengl_Viewport(sys->vgl, place.x,
336 cfg_tmp.display.height - (place.y + place.height),
337 place.width, place.height);
338 vlc_gl_ReleaseCurrent(sys->gl);
343 case VOUT_DISPLAY_CHANGE_VIEWPOINT:
344 return vout_display_opengl_SetViewpoint(sys->vgl,
345 &va_arg (ap, const vout_display_cfg_t* )->viewpoint);
347 case VOUT_DISPLAY_RESET_PICTURES:
348 vlc_assert_unreachable ();
350 msg_Err(vd, "Unknown request %d", query);
355 static void PictureDisplay(vout_display_t *vd, picture_t *pic, subpicture_t *subpicture)
357 vout_display_sys_t *sys = vd->sys;
358 @synchronized (sys->glESView) {
359 if (vlc_gl_MakeCurrent(sys->gl) == VLC_SUCCESS)
361 vout_display_opengl_Display(sys->vgl, &vd->source);
362 vlc_gl_ReleaseCurrent(sys->gl);
366 picture_Release(pic);
369 subpicture_Delete(subpicture);
372 static void PictureRender(vout_display_t *vd, picture_t *pic, subpicture_t *subpicture)
374 vout_display_sys_t *sys = vd->sys;
375 if (vlc_gl_MakeCurrent(sys->gl) == VLC_SUCCESS)
377 vout_display_opengl_Prepare(sys->vgl, pic, subpicture);
378 vlc_gl_ReleaseCurrent(sys->gl);
382 static picture_pool_t *PicturePool(vout_display_t *vd, unsigned requested_count)
384 vout_display_sys_t *sys = vd->sys;
386 if (!sys->picturePool && vlc_gl_MakeCurrent(sys->gl) == VLC_SUCCESS)
388 sys->picturePool = vout_display_opengl_GetPool(sys->vgl, requested_count);
389 vlc_gl_ReleaseCurrent(sys->gl);
391 return sys->picturePool;
394 /*****************************************************************************
395 * vout opengl callbacks
396 *****************************************************************************/
397 static int OpenglESLock(vlc_gl_t *gl)
399 struct gl_sys *sys = gl->sys;
401 if (unlikely(![sys->glESView isAppActive]))
404 [sys->glESView lock];
405 [sys->glESView resetBuffers];
409 static void OpenglESUnlock(vlc_gl_t *gl)
411 struct gl_sys *sys = gl->sys;
413 [sys->glESView unlock];
416 static void OpenglESSwap(vlc_gl_t *gl)
418 struct gl_sys *sys = gl->sys;
420 if (likely([sys->glESView isAppActive]))
421 [[sys->glESView eaglContext] presentRenderbuffer:GL_RENDERBUFFER];
425 /*****************************************************************************
427 *****************************************************************************/
428 @implementation VLCOpenGLES2VideoView
429 @synthesize voutDisplay = _voutDisplay, eaglContext = _eaglContext, isAppActive = _appActive;
433 return [CAEAGLLayer class];
436 + (void)getNewView:(NSValue *)value
438 id *ret = [value pointerValue];
439 *ret = [[self alloc] initWithFrame:CGRectMake(0.,0.,320.,240.)];
442 - (id)initWithFrame:(CGRect)frame
444 self = [super initWithFrame:frame];
449 _appActive = ([UIApplication sharedApplication].applicationState == UIApplicationStateActive);
450 if (unlikely(!_appActive))
453 /* the following creates a new OpenGL ES context with the API version we
454 * need if there is already an active context created by another OpenGL
455 * provider we cache it and restore analog to the lock/unlock pattern used
456 * through-out the class */
457 _previousEaglContext = [EAGLContext currentContext];
459 _eaglContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2];
461 if (unlikely(!_eaglContext))
463 if (unlikely(![EAGLContext setCurrentContext:_eaglContext]))
466 CAEAGLLayer *layer = (CAEAGLLayer *)self.layer;
467 layer.drawableProperties = [NSDictionary dictionaryWithObject:kEAGLColorFormatRGBA8 forKey: kEAGLDrawablePropertyColorFormat];
470 self.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
477 - (void)setVoutDisplay:(vout_display_t *)vd
481 [self createBuffers];
486 - (vout_display_t *)voutDisplay
491 - (void)fetchViewContainer
494 /* get the object we will draw into */
495 UIView *viewContainer = var_CreateGetAddress (_voutDisplay, "drawable-nsobject");
496 if (unlikely(viewContainer == nil)) {
497 msg_Err(_voutDisplay, "provided view container is nil");
501 if (unlikely(![viewContainer respondsToSelector:@selector(isKindOfClass:)])) {
502 msg_Err(_voutDisplay, "void pointer not an ObjC object");
506 [viewContainer retain];
508 @synchronized(viewContainer) {
509 if (![viewContainer isKindOfClass:[UIView class]]) {
510 msg_Err(_voutDisplay, "passed ObjC object not of class UIView");
514 vout_display_sys_t *sys = _voutDisplay->sys;
516 /* This will be released in Close(), on
517 * main thread, after we are done using it. */
518 sys->viewContainer = viewContainer;
520 self.frame = viewContainer.bounds;
523 [sys->viewContainer performSelectorOnMainThread:@selector(addSubview:)
527 /* add tap gesture recognizer for DVD menus and stuff */
528 sys->tapRecognizer = [[UITapGestureRecognizer alloc] initWithTarget:self
529 action:@selector(tapRecognized:)];
530 if (sys->viewContainer.window) {
531 if (sys->viewContainer.window.rootViewController) {
532 if (sys->viewContainer.window.rootViewController.view)
533 [sys->viewContainer.superview addGestureRecognizer:sys->tapRecognizer];
536 sys->tapRecognizer.cancelsTouchesInView = NO;
538 } @catch (NSException *exception) {
539 msg_Err(_voutDisplay, "Handling the view container failed due to an Obj-C exception (%s, %s", [exception.name UTF8String], [exception.reason UTF8String]);
540 vout_display_sys_t *sys = _voutDisplay->sys;
541 sys->viewContainer = nil;
547 [[NSNotificationCenter defaultCenter] removeObserver:self];
548 [_eaglContext release];
552 - (void)didMoveToWindow
554 self.contentScaleFactor = self.window.screen.scale;
555 _bufferNeedReset = YES;
558 - (void)createBuffers
560 if (![NSThread isMainThread])
562 [self performSelectorOnMainThread:@selector(createBuffers)
568 if (unlikely(!_appActive)) {
574 glDisable(GL_DEPTH_TEST);
576 glGenFramebuffers(1, &_frameBuffer);
577 glBindFramebuffer(GL_FRAMEBUFFER, _frameBuffer);
579 glGenRenderbuffers(1, &_renderBuffer);
580 glBindRenderbuffer(GL_RENDERBUFFER, _renderBuffer);
582 [_eaglContext renderbufferStorage:GL_RENDERBUFFER fromDrawable:(CAEAGLLayer *)self.layer];
584 glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, _renderBuffer);
585 if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
587 msg_Err(_voutDisplay, "Failed to make complete framebuffer object %x", glCheckFramebufferStatus(GL_FRAMEBUFFER));
593 - (void)destroyBuffers
595 if (![NSThread isMainThread])
597 [self performSelectorOnMainThread:@selector(destroyBuffers)
605 /* clear frame buffer */
606 glDeleteFramebuffers(1, &_frameBuffer);
609 /* clear render buffer */
610 glDeleteRenderbuffers(1, &_renderBuffer);
618 if (unlikely(_bufferNeedReset)) {
619 [self destroyBuffers];
620 [self createBuffers];
621 _bufferNeedReset = NO;
627 _previousEaglContext = [EAGLContext currentContext];
628 [EAGLContext setCurrentContext:_eaglContext];
633 [EAGLContext setCurrentContext:_previousEaglContext];
636 - (void)layoutSubviews
640 _bufferNeedReset = YES;
645 if (![NSThread isMainThread])
647 [self performSelectorOnMainThread:@selector(reshape)
655 CGSize viewSize = [self bounds].size;
656 CGFloat scaleFactor = self.contentScaleFactor;
657 vout_display_place_t place;
660 vout_display_cfg_t cfg_tmp = *(_voutDisplay->cfg);
662 cfg_tmp.display.width = viewSize.width * scaleFactor;
663 cfg_tmp.display.height = viewSize.height * scaleFactor;
665 vout_display_PlacePicture(&place, &_voutDisplay->source, &cfg_tmp, false);
666 _voutDisplay->sys->place = place;
667 vout_display_SendEventDisplaySize(_voutDisplay, viewSize.width * scaleFactor,
668 viewSize.height * scaleFactor);
671 // x / y are top left corner, but we need the lower left one
672 glViewport(place.x, place.y, place.width, place.height);
676 - (void)tapRecognized:(UITapGestureRecognizer *)tapRecognizer
678 UIGestureRecognizerState state = [tapRecognizer state];
679 CGPoint touchPoint = [tapRecognizer locationInView:self];
680 CGFloat scaleFactor = self.contentScaleFactor;
681 vout_display_SendMouseMovedDisplayCoordinates(_voutDisplay, ORIENT_NORMAL,
682 (int)touchPoint.x * scaleFactor, (int)touchPoint.y * scaleFactor,
683 &_voutDisplay->sys->place);
685 vout_display_SendEventMousePressed(_voutDisplay, MOUSE_BUTTON_LEFT);
686 vout_display_SendEventMouseReleased(_voutDisplay, MOUSE_BUTTON_LEFT);
689 - (void)applicationStateChanged:(NSNotification *)notification
691 @synchronized (self) {
692 if ([[notification name] isEqualToString:UIApplicationWillResignActiveNotification]
693 || [[notification name] isEqualToString:UIApplicationDidEnterBackgroundNotification]
694 || [[notification name] isEqualToString:UIApplicationWillTerminateNotification])
701 - (void)updateConstraints
703 [super updateConstraints];
712 - (BOOL)acceptsFirstResponder
717 - (void)propagateDimensionsToVoutCore
721 @synchronized(_voutDisplay->sys->viewContainer) {
722 scaleFactor = _voutDisplay->sys->viewContainer.contentScaleFactor;
723 viewSize = _voutDisplay->sys->viewContainer.bounds.size;
725 vout_display_SendEventDisplaySize(_voutDisplay, viewSize.width * scaleFactor, viewSize.height * scaleFactor);
728 - (void)mainThreadContentScaleFactor:(NSNumber *)scaleFactor
730 id *ret = [scaleFactor pointerValue];
731 *ret = [[NSNumber alloc] initWithFloat:[super contentScaleFactor]];
734 - (CGFloat)contentScaleFactor
736 if ([NSThread isMainThread]) {
737 return [super contentScaleFactor];
740 NSNumber *scaleFactor;
741 [self performSelectorOnMainThread:@selector(mainThreadContentScaleFactor:)
742 withObject:[NSValue valueWithPointer:&scaleFactor]
744 CGFloat ret = [scaleFactor floatValue];
745 [scaleFactor release];
749 - (void)mainThreadViewBounds:(NSValue *)viewBoundsString
751 id *ret = [viewBoundsString pointerValue];
752 *ret = [NSStringFromCGRect([super bounds]) retain];
757 if ([NSThread isMainThread]) {
758 return self.bounds.size;
761 NSString *viewBoundsString;
762 [self performSelectorOnMainThread:@selector(mainThreadViewBounds:)
763 withObject:[NSValue valueWithPointer:&viewBoundsString]
765 CGRect bounds = CGRectFromString(viewBoundsString);
766 [viewBoundsString release];