1 /*****************************************************************************
2 * ios.m: iOS OpenGL ES provider
3 *****************************************************************************
4 * Copyright (C) 2001-2017 VLC authors and VideoLAN
7 * Authors: Pierre d'Herbemont <pdherbemont at videolan dot org>
8 * Felix Paul Kühne <fkuehne at videolan dot org>
9 * David Fuhrmann <david dot fuhrmann at googlemail dot com>
11 * Laurent Aimar <fenrir _AT_ videolan _DOT_ org>
12 * Eric Petit <titer@m0k.org>
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU Lesser General Public License as published by
16 * the Free Software Foundation; either version 2.1 of the License, or
17 * (at your option) any later version.
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU Lesser General Public License for more details.
24 * You should have received a copy of the GNU Lesser General Public License
25 * along with this program; if not, write to the Free Software Foundation,
26 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
27 *****************************************************************************/
29 /*****************************************************************************
31 *****************************************************************************/
33 #import <UIKit/UIKit.h>
34 #import <OpenGLES/EAGL.h>
35 #import <OpenGLES/ES2/gl.h>
36 #import <OpenGLES/ES2/glext.h>
37 #import <QuartzCore/QuartzCore.h>
44 #import <vlc_common.h>
45 #import <vlc_plugin.h>
46 #import <vlc_vout_display.h>
47 #import <vlc_opengl.h>
48 #import <vlc_dialog.h>
49 #import "opengl/vout_helper.h"
52 * Forward declarations
55 struct picture_sys_t {
56 CVPixelBufferRef pixelBuffer;
59 static int Open(vlc_object_t *);
60 static void Close(vlc_object_t *);
62 static picture_pool_t* PicturePool(vout_display_t *, unsigned);
63 static void PictureRender(vout_display_t *, picture_t *, subpicture_t *);
64 static void PictureDisplay(vout_display_t *, picture_t *, subpicture_t *);
65 static int Control(vout_display_t*, int, va_list);
67 static void *OurGetProcAddress(vlc_gl_t *, const char *);
69 static int GLESMakeCurrent(vlc_gl_t *);
70 static void GLESSwap(vlc_gl_t *);
71 static void GLESReleaseCurrent(vlc_gl_t *);
77 set_shortname("iOS vout")
78 set_description("iOS OpenGL video output")
79 set_category(CAT_VIDEO)
80 set_subcategory(SUBCAT_VIDEO_VOUT)
81 set_capability("vout display", 300)
82 set_callbacks(Open, Close)
84 add_shortcut("vout_ios2", "vout_ios")
88 @interface VLCOpenGLES2VideoView : UIView {
89 vout_display_t *_voutDisplay;
90 EAGLContext *_eaglContext;
96 BOOL _bufferNeedReset;
99 /* Written from MT, read locked from vout */
100 vout_display_place_t _place;
102 @property (readonly) GLuint renderBuffer;
103 @property (readonly) GLuint frameBuffer;
104 @property (readwrite) vout_display_t* voutDisplay;
105 @property (readonly) EAGLContext* eaglContext;
106 @property GLuint shaderProgram;
108 - (void)createBuffers;
109 - (void)destroyBuffers;
110 - (void)resetBuffers;
111 - (BOOL)makeCurrent:(EAGLContext **)previousEaglContext;
112 - (void)releaseCurrent:(EAGLContext *)previousEaglContext;
114 - (void)setPlace:(const vout_display_place_t *)place;
116 - (void)propagateDimensionsToVoutCore;
120 struct vout_display_sys_t
122 VLCOpenGLES2VideoView *glESView;
123 UIView *viewContainer;
124 UITapGestureRecognizer *tapRecognizer;
127 vout_display_opengl_t *vgl;
129 picture_pool_t *picturePool;
134 VLCOpenGLES2VideoView *glESView;
135 EAGLContext *previousEaglContext;
138 static void *OurGetProcAddress(vlc_gl_t *gl, const char *name)
142 return dlsym(RTLD_DEFAULT, name);
145 static int Open(vlc_object_t *this)
147 vout_display_t *vd = (vout_display_t *)this;
149 if (vout_display_IsWindowed(vd))
152 vout_display_sys_t *sys = vlc_obj_calloc (this, 1, sizeof(*sys));
158 sys->picturePool = NULL;
161 var_Create(vd->obj.parent, "ios-eaglcontext", VLC_VAR_ADDRESS);
164 /* setup the actual OpenGL ES view */
165 [VLCOpenGLES2VideoView performSelectorOnMainThread:@selector(getNewView:)
166 withObject:[NSValue valueWithPointer:&sys->glESView]
168 if (!sys->glESView) {
169 msg_Err(vd, "Creating OpenGL ES 2 view failed");
173 [sys->glESView setVoutDisplay:vd];
175 [sys->glESView performSelectorOnMainThread:@selector(fetchViewContainer) withObject:nil waitUntilDone:YES];
176 if (!sys->viewContainer) {
177 msg_Err(vd, "Fetching view container failed");
181 const vlc_fourcc_t *subpicture_chromas;
182 video_format_t fmt = vd->fmt;
184 sys->gl = vlc_object_create(this, sizeof(*sys->gl));
188 struct gl_sys *glsys = sys->gl->sys =
189 vlc_obj_malloc(this, sizeof(struct gl_sys));
190 if (unlikely(!sys->gl->sys))
192 glsys->glESView = sys->glESView;
193 /* Initialize common OpenGL video display */
194 sys->gl->makeCurrent = GLESMakeCurrent;
195 sys->gl->releaseCurrent = GLESReleaseCurrent;
196 sys->gl->swap = GLESSwap;
197 sys->gl->getProcAddress = OurGetProcAddress;
199 if (vlc_gl_MakeCurrent(sys->gl) != VLC_SUCCESS)
202 var_SetAddress(vd->obj.parent, "ios-eaglcontext", [sys->glESView eaglContext]);
204 sys->vgl = vout_display_opengl_New(&vd->fmt, &subpicture_chromas,
205 sys->gl, &vd->cfg->viewpoint);
206 vlc_gl_ReleaseCurrent(sys->gl);
211 vout_display_info_t info = vd->info;
212 info.has_pictures_invalid = false;
213 info.subpicture_chromas = subpicture_chromas;
215 /* Setup vout_display_t once everything is fine */
218 vd->pool = PicturePool;
219 vd->prepare = PictureRender;
220 vd->display = PictureDisplay;
221 vd->control = Control;
223 /* forward our dimensions to the vout core */
224 [sys->glESView performSelectorOnMainThread:@selector(propagateDimensionsToVoutCore) withObject:nil waitUntilDone:YES];
227 [[NSNotificationCenter defaultCenter] addObserver:sys->glESView
228 selector:@selector(applicationStateChanged:)
229 name:UIApplicationWillResignActiveNotification
231 [[NSNotificationCenter defaultCenter] addObserver:sys->glESView
232 selector:@selector(applicationStateChanged:)
233 name:UIApplicationDidBecomeActiveNotification
235 [sys->glESView reshape];
244 static void Close (vlc_object_t *this)
246 vout_display_t *vd = (vout_display_t *)this;
247 vout_display_sys_t *sys = vd->sys;
250 if (sys->tapRecognizer) {
251 [sys->tapRecognizer.view performSelectorOnMainThread:@selector(removeGestureRecognizer:) withObject:sys->tapRecognizer waitUntilDone:YES];
252 [sys->tapRecognizer release];
255 [sys->glESView setVoutDisplay:nil];
257 var_Destroy (vd, "drawable-nsobject");
258 @synchronized(sys->viewContainer) {
259 [sys->glESView performSelectorOnMainThread:@selector(removeFromSuperview) withObject:nil waitUntilDone:NO];
260 [sys->viewContainer performSelectorOnMainThread:@selector(release) withObject:nil waitUntilDone:NO];
262 sys->viewContainer = nil;
264 if (sys->gl != NULL) {
265 msg_Dbg(this, "deleting display");
267 if (likely(sys->vgl))
269 vlc_gl_MakeCurrent(sys->gl);
270 vout_display_opengl_Delete(sys->vgl);
271 vlc_gl_ReleaseCurrent(sys->gl);
273 vlc_object_release(sys->gl);
276 [sys->glESView release];
278 var_Destroy(vd->obj.parent, "ios-eaglcontext");
281 /*****************************************************************************
282 * vout display callbacks
283 *****************************************************************************/
285 static int Control(vout_display_t *vd, int query, va_list ap)
287 vout_display_sys_t *sys = vd->sys;
290 case VOUT_DISPLAY_CHANGE_DISPLAY_FILLED:
291 case VOUT_DISPLAY_CHANGE_ZOOM:
292 case VOUT_DISPLAY_CHANGE_SOURCE_ASPECT:
293 case VOUT_DISPLAY_CHANGE_SOURCE_CROP:
294 case VOUT_DISPLAY_CHANGE_DISPLAY_SIZE:
300 const vout_display_cfg_t *cfg;
302 if (vlc_gl_MakeCurrent(sys->gl) != VLC_SUCCESS)
305 if (query == VOUT_DISPLAY_CHANGE_SOURCE_ASPECT ||
306 query == VOUT_DISPLAY_CHANGE_SOURCE_CROP) {
309 cfg = (const vout_display_cfg_t*)va_arg(ap, const vout_display_cfg_t *);
312 /* we don't adapt anything here regardless of what the vout core
313 * wants since we are not in a traditional desktop window */
317 vout_display_cfg_t cfg_tmp = *cfg;
319 viewSize = [sys->glESView viewSize];
321 /* on HiDPI displays, the point bounds don't equal the actual pixels */
322 CGFloat scaleFactor = sys->glESView.contentScaleFactor;
323 cfg_tmp.display.width = viewSize.width * scaleFactor;
324 cfg_tmp.display.height = viewSize.height * scaleFactor;
326 vout_display_place_t place;
327 vout_display_PlacePicture(&place, &vd->source, &cfg_tmp, false);
329 [sys->glESView setPlace:&place];
331 vout_display_opengl_SetWindowAspectRatio(sys->vgl, (float)place.width / place.height);
333 // x / y are top left corner, but we need the lower left one
334 if (query != VOUT_DISPLAY_CHANGE_DISPLAY_SIZE)
335 vout_display_opengl_Viewport(sys->vgl, place.x,
336 cfg_tmp.display.height - (place.y + place.height),
337 place.width, place.height);
338 vlc_gl_ReleaseCurrent(sys->gl);
343 case VOUT_DISPLAY_CHANGE_VIEWPOINT:
344 return vout_display_opengl_SetViewpoint(sys->vgl,
345 &va_arg (ap, const vout_display_cfg_t* )->viewpoint);
347 case VOUT_DISPLAY_RESET_PICTURES:
348 vlc_assert_unreachable ();
350 msg_Err(vd, "Unknown request %d", query);
355 static void PictureDisplay(vout_display_t *vd, picture_t *pic, subpicture_t *subpicture)
357 vout_display_sys_t *sys = vd->sys;
358 if (vlc_gl_MakeCurrent(sys->gl) == VLC_SUCCESS)
360 vout_display_opengl_Display(sys->vgl, &vd->source);
361 vlc_gl_ReleaseCurrent(sys->gl);
364 picture_Release(pic);
367 subpicture_Delete(subpicture);
370 static void PictureRender(vout_display_t *vd, picture_t *pic, subpicture_t *subpicture)
372 vout_display_sys_t *sys = vd->sys;
373 if (vlc_gl_MakeCurrent(sys->gl) == VLC_SUCCESS)
375 vout_display_opengl_Prepare(sys->vgl, pic, subpicture);
376 vlc_gl_ReleaseCurrent(sys->gl);
380 static picture_pool_t *PicturePool(vout_display_t *vd, unsigned requested_count)
382 vout_display_sys_t *sys = vd->sys;
384 if (!sys->picturePool && vlc_gl_MakeCurrent(sys->gl) == VLC_SUCCESS)
386 sys->picturePool = vout_display_opengl_GetPool(sys->vgl, requested_count);
387 vlc_gl_ReleaseCurrent(sys->gl);
389 return sys->picturePool;
392 /*****************************************************************************
393 * vout opengl callbacks
394 *****************************************************************************/
395 static int GLESMakeCurrent(vlc_gl_t *gl)
397 struct gl_sys *sys = gl->sys;
399 if (![sys->glESView makeCurrent:&sys->previousEaglContext])
402 [sys->glESView resetBuffers];
406 static void GLESReleaseCurrent(vlc_gl_t *gl)
408 struct gl_sys *sys = gl->sys;
410 [sys->glESView releaseCurrent:sys->previousEaglContext];
413 static void GLESSwap(vlc_gl_t *gl)
415 struct gl_sys *sys = gl->sys;
417 [[sys->glESView eaglContext] presentRenderbuffer:GL_RENDERBUFFER];
421 /*****************************************************************************
423 *****************************************************************************/
424 @implementation VLCOpenGLES2VideoView
425 @synthesize voutDisplay = _voutDisplay, eaglContext = _eaglContext;
429 return [CAEAGLLayer class];
432 + (void)getNewView:(NSValue *)value
434 id *ret = [value pointerValue];
435 *ret = [[self alloc] initWithFrame:CGRectMake(0.,0.,320.,240.)];
438 - (id)initWithFrame:(CGRect)frame
440 self = [super initWithFrame:frame];
445 _appActive = ([UIApplication sharedApplication].applicationState == UIApplicationStateActive);
446 if (unlikely(!_appActive))
449 vlc_mutex_init(&_mutex);
451 /* the following creates a new OpenGL ES context with the API version we
452 * need if there is already an active context created by another OpenGL
453 * provider we cache it and restore analog to the
454 * makeCurrent/releaseCurrent pattern used through-out the class */
455 EAGLContext *previousEaglContext = [EAGLContext currentContext];
457 _eaglContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2];
459 if (unlikely(!_eaglContext)
460 || unlikely(![EAGLContext setCurrentContext:_eaglContext]))
462 vlc_mutex_destroy(&_mutex);
466 CAEAGLLayer *layer = (CAEAGLLayer *)self.layer;
467 layer.drawableProperties = [NSDictionary dictionaryWithObject:kEAGLColorFormatRGBA8 forKey: kEAGLDrawablePropertyColorFormat];
470 self.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
472 [self releaseCurrent:previousEaglContext];
477 - (void)setVoutDisplay:(vout_display_t *)vd
481 [self createBuffers];
486 - (vout_display_t *)voutDisplay
491 - (void)fetchViewContainer
494 /* get the object we will draw into */
495 UIView *viewContainer = var_CreateGetAddress (_voutDisplay, "drawable-nsobject");
496 if (unlikely(viewContainer == nil)) {
497 msg_Err(_voutDisplay, "provided view container is nil");
501 if (unlikely(![viewContainer respondsToSelector:@selector(isKindOfClass:)])) {
502 msg_Err(_voutDisplay, "void pointer not an ObjC object");
506 [viewContainer retain];
508 @synchronized(viewContainer) {
509 if (![viewContainer isKindOfClass:[UIView class]]) {
510 msg_Err(_voutDisplay, "passed ObjC object not of class UIView");
514 vout_display_sys_t *sys = _voutDisplay->sys;
516 /* This will be released in Close(), on
517 * main thread, after we are done using it. */
518 sys->viewContainer = viewContainer;
520 self.frame = viewContainer.bounds;
523 [sys->viewContainer performSelectorOnMainThread:@selector(addSubview:)
527 /* add tap gesture recognizer for DVD menus and stuff */
528 sys->tapRecognizer = [[UITapGestureRecognizer alloc] initWithTarget:self
529 action:@selector(tapRecognized:)];
530 if (sys->viewContainer.window) {
531 if (sys->viewContainer.window.rootViewController) {
532 if (sys->viewContainer.window.rootViewController.view)
533 [sys->viewContainer.superview addGestureRecognizer:sys->tapRecognizer];
536 sys->tapRecognizer.cancelsTouchesInView = NO;
538 } @catch (NSException *exception) {
539 msg_Err(_voutDisplay, "Handling the view container failed due to an Obj-C exception (%s, %s", [exception.name UTF8String], [exception.reason UTF8String]);
540 vout_display_sys_t *sys = _voutDisplay->sys;
541 sys->viewContainer = nil;
547 [[NSNotificationCenter defaultCenter] removeObserver:self];
548 [_eaglContext release];
549 vlc_mutex_destroy(&_mutex);
553 - (void)didMoveToWindow
555 self.contentScaleFactor = self.window.screen.scale;
556 _bufferNeedReset = YES;
559 - (void)createBuffers
561 if (![NSThread isMainThread])
563 [self performSelectorOnMainThread:@selector(createBuffers)
569 EAGLContext *previousEaglContext;
570 if (![self makeCurrent:&previousEaglContext])
573 glDisable(GL_DEPTH_TEST);
575 glGenFramebuffers(1, &_frameBuffer);
576 glBindFramebuffer(GL_FRAMEBUFFER, _frameBuffer);
578 glGenRenderbuffers(1, &_renderBuffer);
579 glBindRenderbuffer(GL_RENDERBUFFER, _renderBuffer);
581 [_eaglContext renderbufferStorage:GL_RENDERBUFFER fromDrawable:(CAEAGLLayer *)self.layer];
583 glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, _renderBuffer);
584 if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
586 msg_Err(_voutDisplay, "Failed to make complete framebuffer object %x", glCheckFramebufferStatus(GL_FRAMEBUFFER));
589 [self releaseCurrent:previousEaglContext];
592 - (void)destroyBuffers
594 if (![NSThread isMainThread])
596 [self performSelectorOnMainThread:@selector(destroyBuffers)
602 EAGLContext *previousEaglContext;
603 if (![self makeCurrent:&previousEaglContext])
606 /* clear frame buffer */
607 glDeleteFramebuffers(1, &_frameBuffer);
610 /* clear render buffer */
611 glDeleteRenderbuffers(1, &_renderBuffer);
614 [self releaseCurrent:previousEaglContext];
619 if (unlikely(_bufferNeedReset)) {
620 [self destroyBuffers];
621 [self createBuffers];
622 _bufferNeedReset = NO;
626 - (BOOL)makeCurrent:(EAGLContext **)previousEaglContext
628 vlc_mutex_lock(&_mutex);
630 if (unlikely(!_appActive))
632 vlc_mutex_unlock(&_mutex);
636 *previousEaglContext = [EAGLContext currentContext];
638 BOOL success = [EAGLContext setCurrentContext:_eaglContext];
640 vlc_mutex_unlock(&_mutex);
644 - (void)releaseCurrent:(EAGLContext *)previousEaglContext
646 [EAGLContext setCurrentContext:previousEaglContext];
649 - (void)layoutSubviews
653 _bufferNeedReset = YES;
658 if (![NSThread isMainThread])
660 [self performSelectorOnMainThread:@selector(reshape)
666 EAGLContext *previousEaglContext;
667 if (![self makeCurrent:&previousEaglContext])
670 CGSize viewSize = [self bounds].size;
671 CGFloat scaleFactor = self.contentScaleFactor;
672 vout_display_place_t place;
675 vout_display_cfg_t cfg_tmp = *(_voutDisplay->cfg);
677 cfg_tmp.display.width = viewSize.width * scaleFactor;
678 cfg_tmp.display.height = viewSize.height * scaleFactor;
680 vout_display_PlacePicture(&place, &_voutDisplay->source, &cfg_tmp, false);
681 vout_display_SendEventDisplaySize(_voutDisplay, viewSize.width * scaleFactor,
682 viewSize.height * scaleFactor);
683 [self setPlace:&place];
686 // x / y are top left corner, but we need the lower left one
687 glViewport(place.x, place.y, place.width, place.height);
688 [self releaseCurrent:previousEaglContext];
691 - (void)tapRecognized:(UITapGestureRecognizer *)tapRecognizer
693 UIGestureRecognizerState state = [tapRecognizer state];
694 CGPoint touchPoint = [tapRecognizer locationInView:self];
695 CGFloat scaleFactor = self.contentScaleFactor;
696 vlc_mutex_lock(&_mutex);
697 vout_display_place_t place = _place;
698 vlc_mutex_unlock(&_mutex);
699 vout_display_SendMouseMovedDisplayCoordinates(_voutDisplay, ORIENT_NORMAL,
700 (int)touchPoint.x * scaleFactor, (int)touchPoint.y * scaleFactor,
703 vout_display_SendEventMousePressed(_voutDisplay, MOUSE_BUTTON_LEFT);
704 vout_display_SendEventMouseReleased(_voutDisplay, MOUSE_BUTTON_LEFT);
707 - (void)setPlace:(const vout_display_place_t *)place
709 vlc_mutex_lock(&_mutex);
711 vlc_mutex_unlock(&_mutex);
714 - (void)applicationStateChanged:(NSNotification *)notification
716 vlc_mutex_lock(&_mutex);
718 if ([[notification name] isEqualToString:UIApplicationWillResignActiveNotification]
719 || [[notification name] isEqualToString:UIApplicationDidEnterBackgroundNotification]
720 || [[notification name] isEqualToString:UIApplicationWillTerminateNotification])
725 vlc_mutex_unlock(&_mutex);
728 - (void)updateConstraints
730 [super updateConstraints];
739 - (BOOL)acceptsFirstResponder
744 - (void)propagateDimensionsToVoutCore
748 @synchronized(_voutDisplay->sys->viewContainer) {
749 scaleFactor = _voutDisplay->sys->viewContainer.contentScaleFactor;
750 viewSize = _voutDisplay->sys->viewContainer.bounds.size;
752 vout_display_SendEventDisplaySize(_voutDisplay, viewSize.width * scaleFactor, viewSize.height * scaleFactor);
755 - (void)mainThreadContentScaleFactor:(NSNumber *)scaleFactor
757 id *ret = [scaleFactor pointerValue];
758 *ret = [[NSNumber alloc] initWithFloat:[super contentScaleFactor]];
761 - (CGFloat)contentScaleFactor
763 if ([NSThread isMainThread]) {
764 return [super contentScaleFactor];
767 NSNumber *scaleFactor;
768 [self performSelectorOnMainThread:@selector(mainThreadContentScaleFactor:)
769 withObject:[NSValue valueWithPointer:&scaleFactor]
771 CGFloat ret = [scaleFactor floatValue];
772 [scaleFactor release];
776 - (void)mainThreadViewBounds:(NSValue *)viewBoundsString
778 id *ret = [viewBoundsString pointerValue];
779 *ret = [NSStringFromCGRect([super bounds]) retain];
784 if ([NSThread isMainThread]) {
785 return self.bounds.size;
788 NSString *viewBoundsString;
789 [self performSelectorOnMainThread:@selector(mainThreadViewBounds:)
790 withObject:[NSValue valueWithPointer:&viewBoundsString]
792 CGRect bounds = CGRectFromString(viewBoundsString);
793 [viewBoundsString release];