Flutter macOS Embedder
FlutterViewController.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
7 
8 #include <Carbon/Carbon.h>
9 #import <objc/message.h>
10 
20 #import "flutter/shell/platform/embedder/embedder.h"
21 
22 namespace {
25 
26 // Use different device ID for mouse and pan/zoom events, since we can't differentiate the actual
27 // device (mouse v.s. trackpad).
28 static constexpr int32_t kMousePointerDeviceId = 0;
29 static constexpr int32_t kPointerPanZoomDeviceId = 1;
30 
31 // A trackpad touch following inertial scrolling should cause an inertia cancel
32 // event to be issued. Use a window of 50 milliseconds after the scroll to account
33 // for delays in event propagation observed in macOS Ventura.
34 static constexpr double kTrackpadTouchInertiaCancelWindowMs = 0.050;
35 
36 /**
37  * State tracking for mouse events, to adapt between the events coming from the system and the
38  * events that the embedding API expects.
39  */
40 struct MouseState {
41  /**
42  * The currently pressed buttons, as represented in FlutterPointerEvent.
43  */
44  int64_t buttons = 0;
45 
46  /**
47  * The accumulated gesture pan.
48  */
49  CGFloat delta_x = 0;
50  CGFloat delta_y = 0;
51 
52  /**
53  * The accumulated gesture zoom scale.
54  */
55  CGFloat scale = 0;
56 
57  /**
58  * The accumulated gesture rotation.
59  */
60  CGFloat rotation = 0;
61 
62  /**
63  * Whether or not a kAdd event has been sent (or sent again since the last kRemove if tracking is
64  * enabled). Used to determine whether to send a kAdd event before sending an incoming mouse
65  * event, since Flutter expects pointers to be added before events are sent for them.
66  */
67  bool flutter_state_is_added = false;
68 
69  /**
70  * Whether or not a kDown has been sent since the last kAdd/kUp.
71  */
72  bool flutter_state_is_down = false;
73 
74  /**
75  * Whether or not mouseExited: was received while a button was down. Cocoa's behavior when
76  * dragging out of a tracked area is to send an exit, then keep sending drag events until the last
77  * button is released. Flutter doesn't expect to receive events after a kRemove, so the kRemove
78  * for the exit needs to be delayed until after the last mouse button is released. If cursor
79  * returns back to the window while still dragging, the flag is cleared in mouseEntered:.
80  */
81  bool has_pending_exit = false;
82 
83  /*
84  * Whether or not a kPanZoomStart has been sent since the last kAdd/kPanZoomEnd.
85  */
86  bool flutter_state_is_pan_zoom_started = false;
87 
88  /**
89  * State of pan gesture.
90  */
91  NSEventPhase pan_gesture_phase = NSEventPhaseNone;
92 
93  /**
94  * State of scale gesture.
95  */
96  NSEventPhase scale_gesture_phase = NSEventPhaseNone;
97 
98  /**
99  * State of rotate gesture.
100  */
101  NSEventPhase rotate_gesture_phase = NSEventPhaseNone;
102 
103  /**
104  * Time of last scroll momentum event.
105  */
106  NSTimeInterval last_scroll_momentum_changed_time = 0;
107 
108  /**
109  * Resets all gesture state to default values.
110  */
111  void GestureReset() {
112  delta_x = 0;
113  delta_y = 0;
114  scale = 0;
115  rotation = 0;
116  flutter_state_is_pan_zoom_started = false;
117  pan_gesture_phase = NSEventPhaseNone;
118  scale_gesture_phase = NSEventPhaseNone;
119  rotate_gesture_phase = NSEventPhaseNone;
120  }
121 
122  /**
123  * Resets all state to default values.
124  */
125  void Reset() {
126  flutter_state_is_added = false;
127  flutter_state_is_down = false;
128  has_pending_exit = false;
129  buttons = 0;
130  GestureReset();
131  }
132 };
133 
134 /**
135  * Returns the current Unicode layout data (kTISPropertyUnicodeKeyLayoutData).
136  *
137  * To use the returned data, convert it to CFDataRef first, finds its bytes
138  * with CFDataGetBytePtr, then reinterpret it into const UCKeyboardLayout*.
139  * It's returned in NSData* to enable auto reference count.
140  */
141 NSData* currentKeyboardLayoutData() {
142  TISInputSourceRef source = TISCopyCurrentKeyboardInputSource();
143  CFTypeRef layout_data = TISGetInputSourceProperty(source, kTISPropertyUnicodeKeyLayoutData);
144  if (layout_data == nil) {
145  CFRelease(source);
146  // TISGetInputSourceProperty returns null with Japanese keyboard layout.
147  // Using TISCopyCurrentKeyboardLayoutInputSource to fix NULL return.
148  // https://github.com/microsoft/node-native-keymap/blob/5f0699ded00179410a14c0e1b0e089fe4df8e130/src/keyboard_mac.mm#L91
149  source = TISCopyCurrentKeyboardLayoutInputSource();
150  layout_data = TISGetInputSourceProperty(source, kTISPropertyUnicodeKeyLayoutData);
151  }
152  return (__bridge_transfer NSData*)CFRetain(layout_data);
153 }
154 
155 } // namespace
156 
157 #pragma mark - Private interface declaration.
158 
159 /**
160  * FlutterViewWrapper is a convenience class that wraps a FlutterView and provides
161  * a mechanism to attach AppKit views such as FlutterTextField without affecting
162  * the accessibility subtree of the wrapped FlutterView itself.
163  *
164  * The FlutterViewController uses this class to create its content view. When
165  * any of the accessibility services (e.g. VoiceOver) is turned on, the accessibility
166  * bridge creates FlutterTextFields that interact with the service. The bridge has to
167  * attach the FlutterTextField somewhere in the view hierarchy in order for the
168  * FlutterTextField to interact correctly with VoiceOver. Those FlutterTextFields
169  * will be attached to this view so that they won't affect the accessibility subtree
170  * of FlutterView.
171  */
172 @interface FlutterViewWrapper : NSView
173 
174 - (void)setBackgroundColor:(NSColor*)color;
175 
176 @end
177 
178 /**
179  * Private interface declaration for FlutterViewController.
180  */
182 
183 /**
184  * The tracking area used to generate hover events, if enabled.
185  */
186 @property(nonatomic) NSTrackingArea* trackingArea;
187 
188 /**
189  * The current state of the mouse and the sent mouse events.
190  */
191 @property(nonatomic) MouseState mouseState;
192 
193 /**
194  * Event monitor for keyUp events.
195  */
196 @property(nonatomic) id keyUpMonitor;
197 
198 /**
199  * Pointer to a keyboard manager, a hub that manages how key events are
200  * dispatched to various Flutter key responders, and whether the event is
201  * propagated to the next NSResponder.
202  */
203 @property(nonatomic, readonly, nonnull) FlutterKeyboardManager* keyboardManager;
204 
205 @property(nonatomic) KeyboardLayoutNotifier keyboardLayoutNotifier;
206 
207 @property(nonatomic) NSData* keyboardLayoutData;
208 
209 /**
210  * Starts running |engine|, including any initial setup.
211  */
212 - (BOOL)launchEngine;
213 
214 /**
215  * Updates |trackingArea| for the current tracking settings, creating it with
216  * the correct mode if tracking is enabled, or removing it if not.
217  */
218 - (void)configureTrackingArea;
219 
220 /**
221  * Creates and registers keyboard related components.
222  */
223 - (void)initializeKeyboard;
224 
225 /**
226  * Calls dispatchMouseEvent:phase: with a phase determined by self.mouseState.
227  *
228  * mouseState.buttons should be updated before calling this method.
229  */
230 - (void)dispatchMouseEvent:(nonnull NSEvent*)event;
231 
232 /**
233  * Calls dispatchMouseEvent:phase: with a phase determined by event.phase.
234  */
235 - (void)dispatchGestureEvent:(nonnull NSEvent*)event;
236 
237 /**
238  * Converts |event| to a FlutterPointerEvent with the given phase, and sends it to the engine.
239  */
240 - (void)dispatchMouseEvent:(nonnull NSEvent*)event phase:(FlutterPointerPhase)phase;
241 
242 /**
243  * Called when the active keyboard input source changes.
244  *
245  * Input sources may be simple keyboard layouts, or more complex input methods involving an IME,
246  * such as Chinese, Japanese, and Korean.
247  */
248 - (void)onKeyboardLayoutChanged;
249 
250 @end
251 
252 #pragma mark - Private dependant functions
253 
254 namespace {
255 void OnKeyboardLayoutChanged(CFNotificationCenterRef center,
256  void* observer,
257  CFStringRef name,
258  const void* object,
259  CFDictionaryRef userInfo) {
260  FlutterViewController* controller = (__bridge FlutterViewController*)observer;
261  if (controller != nil) {
262  [controller onKeyboardLayoutChanged];
263  }
264 }
265 } // namespace
266 
267 #pragma mark - FlutterViewWrapper implementation.
268 
269 @implementation FlutterViewWrapper {
270  FlutterView* _flutterView;
272 }
273 
274 - (instancetype)initWithFlutterView:(FlutterView*)view
275  controller:(FlutterViewController*)controller {
276  self = [super initWithFrame:NSZeroRect];
277  if (self) {
278  _flutterView = view;
279  _controller = controller;
280  view.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable;
281  [self addSubview:view];
282  }
283  return self;
284 }
285 
286 - (void)setBackgroundColor:(NSColor*)color {
287  [_flutterView setBackgroundColor:color];
288 }
289 
290 - (BOOL)performKeyEquivalent:(NSEvent*)event {
291  // Do not intercept the event if flutterView is not first responder, otherwise this would
292  // interfere with TextInputPlugin, which also handles key equivalents.
293  //
294  // Also do not intercept the event if key equivalent is a product of an event being
295  // redispatched by the TextInputPlugin, in which case it needs to bubble up so that menus
296  // can handle key equivalents.
297  if (self.window.firstResponder != _flutterView || [_controller isDispatchingKeyEvent:event]) {
298  return [super performKeyEquivalent:event];
299  }
300  [_flutterView keyDown:event];
301  return YES;
302 }
303 
304 - (NSArray*)accessibilityChildren {
305  return @[ _flutterView ];
306 }
307 
308 - (void)mouseDown:(NSEvent*)event {
309  // Work around an AppKit bug where mouseDown/mouseUp are not called on the view controller if the
310  // view is the content view of an NSPopover AND macOS's Reduced Transparency accessibility setting
311  // is enabled.
312  //
313  // This simply calls mouseDown on the next responder in the responder chain as the default
314  // implementation on NSResponder is documented to do.
315  //
316  // See: https://github.com/flutter/flutter/issues/115015
317  // See: http://www.openradar.me/FB12050037
318  // See: https://developer.apple.com/documentation/appkit/nsresponder/1524634-mousedown
319  [self.nextResponder mouseDown:event];
320 }
321 
322 - (void)mouseUp:(NSEvent*)event {
323  // Work around an AppKit bug where mouseDown/mouseUp are not called on the view controller if the
324  // view is the content view of an NSPopover AND macOS's Reduced Transparency accessibility setting
325  // is enabled.
326  //
327  // This simply calls mouseUp on the next responder in the responder chain as the default
328  // implementation on NSResponder is documented to do.
329  //
330  // See: https://github.com/flutter/flutter/issues/115015
331  // See: http://www.openradar.me/FB12050037
332  // See: https://developer.apple.com/documentation/appkit/nsresponder/1535349-mouseup
333  [self.nextResponder mouseUp:event];
334 }
335 
336 @end
337 
338 #pragma mark - FlutterViewController implementation.
339 
340 @implementation FlutterViewController {
341  // The project to run in this controller's engine.
343 
344  std::shared_ptr<flutter::AccessibilityBridgeMac> _bridge;
345 
347 
348  // FlutterViewController does not actually uses the synchronizer, but only
349  // passes it to FlutterView.
351 }
352 
353 @synthesize viewId = _viewId;
354 @dynamic accessibilityBridge;
355 
356 /**
357  * Performs initialization that's common between the different init paths.
358  */
359 static void CommonInit(FlutterViewController* controller, FlutterEngine* engine) {
360  if (!engine) {
361  engine = [[FlutterEngine alloc] initWithName:@"io.flutter"
362  project:controller->_project
363  allowHeadlessExecution:NO];
364  }
365  NSCAssert(controller.engine == nil,
366  @"The FlutterViewController is unexpectedly attached to "
367  @"engine %@ before initialization.",
368  controller.engine);
369  [engine addViewController:controller];
370  NSCAssert(controller.engine != nil,
371  @"The FlutterViewController unexpectedly stays unattached after initialization. "
372  @"In unit tests, this is likely because either the FlutterViewController or "
373  @"the FlutterEngine is mocked. Please subclass these classes instead.",
374  controller.engine, controller.viewId);
375  controller->_mouseTrackingMode = kFlutterMouseTrackingModeInKeyWindow;
376  controller->_textInputPlugin = [[FlutterTextInputPlugin alloc] initWithViewController:controller];
377  [controller initializeKeyboard];
378  [controller notifySemanticsEnabledChanged];
379  // macOS fires this message when changing IMEs.
380  CFNotificationCenterRef cfCenter = CFNotificationCenterGetDistributedCenter();
381  __weak FlutterViewController* weakSelf = controller;
382  CFNotificationCenterAddObserver(cfCenter, (__bridge void*)weakSelf, OnKeyboardLayoutChanged,
383  kTISNotifySelectedKeyboardInputSourceChanged, NULL,
384  CFNotificationSuspensionBehaviorDeliverImmediately);
385 }
386 
387 - (instancetype)initWithCoder:(NSCoder*)coder {
388  self = [super initWithCoder:coder];
389  NSAssert(self, @"Super init cannot be nil");
390 
391  CommonInit(self, nil);
392  return self;
393 }
394 
395 - (instancetype)initWithNibName:(NSString*)nibNameOrNil bundle:(NSBundle*)nibBundleOrNil {
396  self = [super initWithNibName:nibNameOrNil bundle:nibBundleOrNil];
397  NSAssert(self, @"Super init cannot be nil");
398 
399  CommonInit(self, nil);
400  return self;
401 }
402 
403 - (instancetype)initWithProject:(nullable FlutterDartProject*)project {
404  self = [super initWithNibName:nil bundle:nil];
405  NSAssert(self, @"Super init cannot be nil");
406 
407  _project = project;
408  CommonInit(self, nil);
409  return self;
410 }
411 
412 - (instancetype)initWithEngine:(nonnull FlutterEngine*)engine
413  nibName:(nullable NSString*)nibName
414  bundle:(nullable NSBundle*)nibBundle {
415  NSAssert(engine != nil, @"Engine is required");
416 
417  self = [super initWithNibName:nibName bundle:nibBundle];
418  if (self) {
419  CommonInit(self, engine);
420  }
421 
422  return self;
423 }
424 
425 - (BOOL)isDispatchingKeyEvent:(NSEvent*)event {
426  return [_keyboardManager isDispatchingKeyEvent:event];
427 }
428 
429 - (void)loadView {
430  FlutterView* flutterView;
431  id<MTLDevice> device = _engine.renderer.device;
432  id<MTLCommandQueue> commandQueue = _engine.renderer.commandQueue;
433  if (!device || !commandQueue) {
434  NSLog(@"Unable to create FlutterView; no MTLDevice or MTLCommandQueue available.");
435  return;
436  }
437  flutterView = [self createFlutterViewWithMTLDevice:device commandQueue:commandQueue];
438  if (_backgroundColor != nil) {
439  [flutterView setBackgroundColor:_backgroundColor];
440  }
441  FlutterViewWrapper* wrapperView = [[FlutterViewWrapper alloc] initWithFlutterView:flutterView
442  controller:self];
443  self.view = wrapperView;
444  _flutterView = flutterView;
445 }
446 
447 - (void)viewDidLoad {
448  [self configureTrackingArea];
449  [self.view setAllowedTouchTypes:NSTouchTypeMaskIndirect];
450  [self.view setWantsRestingTouches:YES];
451  [_engine viewControllerViewDidLoad:self];
452 }
453 
454 - (void)viewWillAppear {
455  [super viewWillAppear];
456  if (!_engine.running) {
457  [self launchEngine];
458  }
459  [self listenForMetaModifiedKeyUpEvents];
460 }
461 
462 - (void)viewWillDisappear {
463  // Per Apple's documentation, it is discouraged to call removeMonitor: in dealloc, and it's
464  // recommended to be called earlier in the lifecycle.
465  [NSEvent removeMonitor:_keyUpMonitor];
466  _keyUpMonitor = nil;
467 }
468 
469 - (void)dealloc {
470  if ([self attached]) {
471  [_engine removeViewController:self];
472  }
473  CFNotificationCenterRef cfCenter = CFNotificationCenterGetDistributedCenter();
474  CFNotificationCenterRemoveEveryObserver(cfCenter, (__bridge void*)self);
475 }
476 
477 #pragma mark - Public methods
478 
479 - (void)setMouseTrackingMode:(FlutterMouseTrackingMode)mode {
480  if (_mouseTrackingMode == mode) {
481  return;
482  }
483  _mouseTrackingMode = mode;
484  [self configureTrackingArea];
485 }
486 
487 - (void)setBackgroundColor:(NSColor*)color {
488  _backgroundColor = color;
489  [_flutterView setBackgroundColor:_backgroundColor];
490 }
491 
492 - (FlutterViewId)viewId {
493  NSAssert([self attached], @"This view controller is not attached.");
494  return _viewId;
495 }
496 
497 - (void)onPreEngineRestart {
498  [self initializeKeyboard];
499 }
500 
501 - (void)notifySemanticsEnabledChanged {
502  BOOL mySemanticsEnabled = !!_bridge;
503  BOOL newSemanticsEnabled = _engine.semanticsEnabled;
504  if (newSemanticsEnabled == mySemanticsEnabled) {
505  return;
506  }
507  if (newSemanticsEnabled) {
508  _bridge = [self createAccessibilityBridgeWithEngine:_engine];
509  } else {
510  // Remove the accessibility children from flutter view before resetting the bridge.
511  _flutterView.accessibilityChildren = nil;
512  _bridge.reset();
513  }
514  NSAssert(newSemanticsEnabled == !!_bridge, @"Failed to update semantics for the view.");
515 }
516 
517 - (std::weak_ptr<flutter::AccessibilityBridgeMac>)accessibilityBridge {
518  return _bridge;
519 }
520 
521 - (void)setUpWithEngine:(FlutterEngine*)engine
522  viewId:(FlutterViewId)viewId
523  threadSynchronizer:(FlutterThreadSynchronizer*)threadSynchronizer {
524  NSAssert(_engine == nil, @"Already attached to an engine %@.", _engine);
525  _engine = engine;
526  _viewId = viewId;
527  _threadSynchronizer = threadSynchronizer;
528  [_threadSynchronizer registerView:_viewId];
529 }
530 
531 - (void)detachFromEngine {
532  NSAssert(_engine != nil, @"Not attached to any engine.");
533  [_threadSynchronizer deregisterView:_viewId];
534  _threadSynchronizer = nil;
535  _engine = nil;
536 }
537 
538 - (BOOL)attached {
539  return _engine != nil;
540 }
541 
542 - (void)updateSemantics:(const FlutterSemanticsUpdate2*)update {
543  NSAssert(_engine.semanticsEnabled, @"Semantics must be enabled.");
544  if (!_engine.semanticsEnabled) {
545  return;
546  }
547  for (size_t i = 0; i < update->node_count; i++) {
548  const FlutterSemanticsNode2* node = update->nodes[i];
549  _bridge->AddFlutterSemanticsNodeUpdate(*node);
550  }
551 
552  for (size_t i = 0; i < update->custom_action_count; i++) {
553  const FlutterSemanticsCustomAction2* action = update->custom_actions[i];
554  _bridge->AddFlutterSemanticsCustomActionUpdate(*action);
555  }
556 
557  _bridge->CommitUpdates();
558 
559  // Accessibility tree can only be used when the view is loaded.
560  if (!self.viewLoaded) {
561  return;
562  }
563  // Attaches the accessibility root to the flutter view.
564  auto root = _bridge->GetFlutterPlatformNodeDelegateFromID(0).lock();
565  if (root) {
566  if ([self.flutterView.accessibilityChildren count] == 0) {
567  NSAccessibilityElement* native_root = root->GetNativeViewAccessible();
568  self.flutterView.accessibilityChildren = @[ native_root ];
569  }
570  } else {
571  self.flutterView.accessibilityChildren = nil;
572  }
573 }
574 
575 #pragma mark - Private methods
576 
577 - (BOOL)launchEngine {
578  if (![_engine runWithEntrypoint:nil]) {
579  return NO;
580  }
581  return YES;
582 }
583 
584 // macOS does not call keyUp: on a key while the command key is pressed. This results in a loss
585 // of a key event once the modified key is released. This method registers the
586 // ViewController as a listener for a keyUp event before it's handled by NSApplication, and should
587 // NOT modify the event to avoid any unexpected behavior.
588 - (void)listenForMetaModifiedKeyUpEvents {
589  if (_keyUpMonitor != nil) {
590  // It is possible for [NSViewController viewWillAppear] to be invoked multiple times
591  // in a row. https://github.com/flutter/flutter/issues/105963
592  return;
593  }
594  FlutterViewController* __weak weakSelf = self;
595  _keyUpMonitor = [NSEvent
596  addLocalMonitorForEventsMatchingMask:NSEventMaskKeyUp
597  handler:^NSEvent*(NSEvent* event) {
598  // Intercept keyUp only for events triggered on the current
599  // view or textInputPlugin.
600  NSResponder* firstResponder = [[event window] firstResponder];
601  if (weakSelf.viewLoaded && weakSelf.flutterView &&
602  (firstResponder == weakSelf.flutterView ||
603  firstResponder == weakSelf.textInputPlugin) &&
604  ([event modifierFlags] & NSEventModifierFlagCommand) &&
605  ([event type] == NSEventTypeKeyUp)) {
606  [weakSelf keyUp:event];
607  }
608  return event;
609  }];
610 }
611 
612 - (void)configureTrackingArea {
613  if (!self.viewLoaded) {
614  // The viewDidLoad will call configureTrackingArea again when
615  // the view is actually loaded.
616  return;
617  }
618  if (_mouseTrackingMode != kFlutterMouseTrackingModeNone && self.flutterView) {
619  NSTrackingAreaOptions options = NSTrackingMouseEnteredAndExited | NSTrackingMouseMoved |
620  NSTrackingInVisibleRect | NSTrackingEnabledDuringMouseDrag;
621  switch (_mouseTrackingMode) {
622  case kFlutterMouseTrackingModeInKeyWindow:
623  options |= NSTrackingActiveInKeyWindow;
624  break;
625  case kFlutterMouseTrackingModeInActiveApp:
626  options |= NSTrackingActiveInActiveApp;
627  break;
628  case kFlutterMouseTrackingModeAlways:
629  options |= NSTrackingActiveAlways;
630  break;
631  default:
632  NSLog(@"Error: Unrecognized mouse tracking mode: %ld", _mouseTrackingMode);
633  return;
634  }
635  _trackingArea = [[NSTrackingArea alloc] initWithRect:NSZeroRect
636  options:options
637  owner:self
638  userInfo:nil];
639  [self.flutterView addTrackingArea:_trackingArea];
640  } else if (_trackingArea) {
641  [self.flutterView removeTrackingArea:_trackingArea];
642  _trackingArea = nil;
643  }
644 }
645 
646 - (void)initializeKeyboard {
647  // TODO(goderbauer): Seperate keyboard/textinput stuff into ViewController specific and Engine
648  // global parts. Move the global parts to FlutterEngine.
649  _keyboardManager = [[FlutterKeyboardManager alloc] initWithViewDelegate:self];
650 }
651 
652 - (void)dispatchMouseEvent:(nonnull NSEvent*)event {
653  FlutterPointerPhase phase = _mouseState.buttons == 0
654  ? (_mouseState.flutter_state_is_down ? kUp : kHover)
655  : (_mouseState.flutter_state_is_down ? kMove : kDown);
656  [self dispatchMouseEvent:event phase:phase];
657 }
658 
659 - (void)dispatchGestureEvent:(nonnull NSEvent*)event {
660  if (event.phase == NSEventPhaseBegan || event.phase == NSEventPhaseMayBegin) {
661  [self dispatchMouseEvent:event phase:kPanZoomStart];
662  } else if (event.phase == NSEventPhaseChanged) {
663  [self dispatchMouseEvent:event phase:kPanZoomUpdate];
664  } else if (event.phase == NSEventPhaseEnded || event.phase == NSEventPhaseCancelled) {
665  [self dispatchMouseEvent:event phase:kPanZoomEnd];
666  } else if (event.phase == NSEventPhaseNone && event.momentumPhase == NSEventPhaseNone) {
667  [self dispatchMouseEvent:event phase:kHover];
668  } else {
669  // Waiting until the first momentum change event is a workaround for an issue where
670  // touchesBegan: is called unexpectedly while in low power mode within the interval between
671  // momentum start and the first momentum change.
672  if (event.momentumPhase == NSEventPhaseChanged) {
673  _mouseState.last_scroll_momentum_changed_time = event.timestamp;
674  }
675  // Skip momentum update events, the framework will generate scroll momentum.
676  NSAssert(event.momentumPhase != NSEventPhaseNone,
677  @"Received gesture event with unexpected phase");
678  }
679 }
680 
681 - (void)dispatchMouseEvent:(NSEvent*)event phase:(FlutterPointerPhase)phase {
682  NSAssert(self.viewLoaded, @"View must be loaded before it handles the mouse event");
683  // There are edge cases where the system will deliver enter out of order relative to other
684  // events (e.g., drag out and back in, release, then click; mouseDown: will be called before
685  // mouseEntered:). Discard those events, since the add will already have been synthesized.
686  if (_mouseState.flutter_state_is_added && phase == kAdd) {
687  return;
688  }
689 
690  // Multiple gesture recognizers could be active at once, we can't send multiple kPanZoomStart.
691  // For example: rotation and magnification.
692  if (phase == kPanZoomStart || phase == kPanZoomEnd) {
693  if (event.type == NSEventTypeScrollWheel) {
694  _mouseState.pan_gesture_phase = event.phase;
695  } else if (event.type == NSEventTypeMagnify) {
696  _mouseState.scale_gesture_phase = event.phase;
697  } else if (event.type == NSEventTypeRotate) {
698  _mouseState.rotate_gesture_phase = event.phase;
699  }
700  }
701  if (phase == kPanZoomStart) {
702  if (event.type == NSEventTypeScrollWheel) {
703  // Ensure scroll inertia cancel event is not sent afterwards.
704  _mouseState.last_scroll_momentum_changed_time = 0;
705  }
706  if (_mouseState.flutter_state_is_pan_zoom_started) {
707  // Already started on a previous gesture type
708  return;
709  }
710  _mouseState.flutter_state_is_pan_zoom_started = true;
711  }
712  if (phase == kPanZoomEnd) {
713  if (!_mouseState.flutter_state_is_pan_zoom_started) {
714  // NSEventPhaseCancelled is sometimes received at incorrect times in the state
715  // machine, just ignore it here if it doesn't make sense
716  // (we have no active gesture to cancel).
717  NSAssert(event.phase == NSEventPhaseCancelled,
718  @"Received gesture event with unexpected phase");
719  return;
720  }
721  // NSEventPhase values are powers of two, we can use this to inspect merged phases.
722  NSEventPhase all_gestures_fields = _mouseState.pan_gesture_phase |
723  _mouseState.scale_gesture_phase |
724  _mouseState.rotate_gesture_phase;
725  NSEventPhase active_mask = NSEventPhaseBegan | NSEventPhaseChanged;
726  if ((all_gestures_fields & active_mask) != 0) {
727  // Even though this gesture type ended, a different type is still active.
728  return;
729  }
730  }
731 
732  // If a pointer added event hasn't been sent, synthesize one using this event for the basic
733  // information.
734  if (!_mouseState.flutter_state_is_added && phase != kAdd) {
735  // Only the values extracted for use in flutterEvent below matter, the rest are dummy values.
736  NSEvent* addEvent = [NSEvent enterExitEventWithType:NSEventTypeMouseEntered
737  location:event.locationInWindow
738  modifierFlags:0
739  timestamp:event.timestamp
740  windowNumber:event.windowNumber
741  context:nil
742  eventNumber:0
743  trackingNumber:0
744  userData:NULL];
745  [self dispatchMouseEvent:addEvent phase:kAdd];
746  }
747 
748  NSPoint locationInView = [self.flutterView convertPoint:event.locationInWindow fromView:nil];
749  NSPoint locationInBackingCoordinates = [self.flutterView convertPointToBacking:locationInView];
750  int32_t device = kMousePointerDeviceId;
751  FlutterPointerDeviceKind deviceKind = kFlutterPointerDeviceKindMouse;
752  if (phase == kPanZoomStart || phase == kPanZoomUpdate || phase == kPanZoomEnd) {
753  device = kPointerPanZoomDeviceId;
754  deviceKind = kFlutterPointerDeviceKindTrackpad;
755  }
756  FlutterPointerEvent flutterEvent = {
757  .struct_size = sizeof(flutterEvent),
758  .phase = phase,
759  .timestamp = static_cast<size_t>(event.timestamp * USEC_PER_SEC),
760  .x = locationInBackingCoordinates.x,
761  .y = -locationInBackingCoordinates.y, // convertPointToBacking makes this negative.
762  .device = device,
763  .device_kind = deviceKind,
764  // If a click triggered a synthesized kAdd, don't pass the buttons in that event.
765  .buttons = phase == kAdd ? 0 : _mouseState.buttons,
766  .view_id = static_cast<FlutterViewId>(_viewId),
767  };
768 
769  if (phase == kPanZoomUpdate) {
770  if (event.type == NSEventTypeScrollWheel) {
771  _mouseState.delta_x += event.scrollingDeltaX * self.flutterView.layer.contentsScale;
772  _mouseState.delta_y += event.scrollingDeltaY * self.flutterView.layer.contentsScale;
773  } else if (event.type == NSEventTypeMagnify) {
774  _mouseState.scale += event.magnification;
775  } else if (event.type == NSEventTypeRotate) {
776  _mouseState.rotation += event.rotation * (-M_PI / 180.0);
777  }
778  flutterEvent.pan_x = _mouseState.delta_x;
779  flutterEvent.pan_y = _mouseState.delta_y;
780  // Scale value needs to be normalized to range 0->infinity.
781  flutterEvent.scale = pow(2.0, _mouseState.scale);
782  flutterEvent.rotation = _mouseState.rotation;
783  } else if (phase == kPanZoomEnd) {
784  _mouseState.GestureReset();
785  } else if (phase != kPanZoomStart && event.type == NSEventTypeScrollWheel) {
786  flutterEvent.signal_kind = kFlutterPointerSignalKindScroll;
787 
788  double pixelsPerLine = 1.0;
789  if (!event.hasPreciseScrollingDeltas) {
790  // The scrollingDelta needs to be multiplied by the line height.
791  // CGEventSourceGetPixelsPerLine() will return 10, which will result in
792  // scrolling that is noticeably slower than in other applications.
793  // Using 40.0 as the multiplier to match Chromium.
794  // See https://source.chromium.org/chromium/chromium/src/+/main:ui/events/cocoa/events_mac.mm
795  pixelsPerLine = 40.0;
796  }
797  double scaleFactor = self.flutterView.layer.contentsScale;
798  // When mouse input is received while shift is pressed (regardless of
799  // any other pressed keys), Mac automatically flips the axis. Other
800  // platforms do not do this, so we flip it back to normalize the input
801  // received by the framework. The keyboard+mouse-scroll mechanism is exposed
802  // in the ScrollBehavior of the framework so developers can customize the
803  // behavior.
804  // At time of change, Apple does not expose any other type of API or signal
805  // that the X/Y axes have been flipped.
806  double scaledDeltaX = -event.scrollingDeltaX * pixelsPerLine * scaleFactor;
807  double scaledDeltaY = -event.scrollingDeltaY * pixelsPerLine * scaleFactor;
808  if (event.modifierFlags & NSShiftKeyMask) {
809  flutterEvent.scroll_delta_x = scaledDeltaY;
810  flutterEvent.scroll_delta_y = scaledDeltaX;
811  } else {
812  flutterEvent.scroll_delta_x = scaledDeltaX;
813  flutterEvent.scroll_delta_y = scaledDeltaY;
814  }
815  }
816 
817  [_keyboardManager syncModifiersIfNeeded:event.modifierFlags timestamp:event.timestamp];
818  [_engine sendPointerEvent:flutterEvent];
819 
820  // Update tracking of state as reported to Flutter.
821  if (phase == kDown) {
822  _mouseState.flutter_state_is_down = true;
823  } else if (phase == kUp) {
824  _mouseState.flutter_state_is_down = false;
825  if (_mouseState.has_pending_exit) {
826  [self dispatchMouseEvent:event phase:kRemove];
827  _mouseState.has_pending_exit = false;
828  }
829  } else if (phase == kAdd) {
830  _mouseState.flutter_state_is_added = true;
831  } else if (phase == kRemove) {
832  _mouseState.Reset();
833  }
834 }
835 
836 - (void)onAccessibilityStatusChanged:(BOOL)enabled {
837  if (!enabled && self.viewLoaded && [_textInputPlugin isFirstResponder]) {
838  // Normally TextInputPlugin, when editing, is child of FlutterViewWrapper.
839  // When accessiblity is enabled the TextInputPlugin gets added as an indirect
840  // child to FlutterTextField. When disabling the plugin needs to be reparented
841  // back.
842  [self.view addSubview:_textInputPlugin];
843  }
844 }
845 
846 - (std::shared_ptr<flutter::AccessibilityBridgeMac>)createAccessibilityBridgeWithEngine:
847  (nonnull FlutterEngine*)engine {
848  return std::make_shared<flutter::AccessibilityBridgeMac>(engine, self);
849 }
850 
851 - (nonnull FlutterView*)createFlutterViewWithMTLDevice:(id<MTLDevice>)device
852  commandQueue:(id<MTLCommandQueue>)commandQueue {
853  return [[FlutterView alloc] initWithMTLDevice:device
854  commandQueue:commandQueue
855  delegate:self
856  threadSynchronizer:_threadSynchronizer
857  viewId:_viewId];
858 }
859 
860 - (void)onKeyboardLayoutChanged {
861  _keyboardLayoutData = nil;
862  if (_keyboardLayoutNotifier != nil) {
864  }
865 }
866 
867 - (NSString*)lookupKeyForAsset:(NSString*)asset {
868  return [FlutterDartProject lookupKeyForAsset:asset];
869 }
870 
871 - (NSString*)lookupKeyForAsset:(NSString*)asset fromPackage:(NSString*)package {
872  return [FlutterDartProject lookupKeyForAsset:asset fromPackage:package];
873 }
874 
875 #pragma mark - FlutterViewDelegate
876 
877 /**
878  * Responds to view reshape by notifying the engine of the change in dimensions.
879  */
880 - (void)viewDidReshape:(NSView*)view {
881  FML_DCHECK(view == _flutterView);
882  [_engine updateWindowMetricsForViewController:self];
883 }
884 
885 - (BOOL)viewShouldAcceptFirstResponder:(NSView*)view {
886  FML_DCHECK(view == _flutterView);
887  // Only allow FlutterView to become first responder if TextInputPlugin is
888  // not active. Otherwise a mouse event inside FlutterView would cause the
889  // TextInputPlugin to lose first responder status.
890  return !_textInputPlugin.isFirstResponder;
891 }
892 
893 #pragma mark - FlutterPluginRegistry
894 
895 - (id<FlutterPluginRegistrar>)registrarForPlugin:(NSString*)pluginName {
896  return [_engine registrarForPlugin:pluginName];
897 }
898 
899 - (NSObject*)valuePublishedByPlugin:(NSString*)pluginKey {
900  return [_engine valuePublishedByPlugin:pluginKey];
901 }
902 
903 #pragma mark - FlutterKeyboardViewDelegate
904 
905 - (void)sendKeyEvent:(const FlutterKeyEvent&)event
906  callback:(nullable FlutterKeyEventCallback)callback
907  userData:(nullable void*)userData {
908  [_engine sendKeyEvent:event callback:callback userData:userData];
909 }
910 
911 - (id<FlutterBinaryMessenger>)getBinaryMessenger {
912  return _engine.binaryMessenger;
913 }
914 
915 - (BOOL)onTextInputKeyEvent:(nonnull NSEvent*)event {
916  return [_textInputPlugin handleKeyEvent:event];
917 }
918 
919 - (void)subscribeToKeyboardLayoutChange:(nullable KeyboardLayoutNotifier)callback {
920  _keyboardLayoutNotifier = callback;
921 }
922 
923 - (LayoutClue)lookUpLayoutForKeyCode:(uint16_t)keyCode shift:(BOOL)shift {
924  if (_keyboardLayoutData == nil) {
925  _keyboardLayoutData = currentKeyboardLayoutData();
926  }
927  const UCKeyboardLayout* layout = reinterpret_cast<const UCKeyboardLayout*>(
928  CFDataGetBytePtr((__bridge CFDataRef)_keyboardLayoutData));
929 
930  UInt32 deadKeyState = 0;
931  UniCharCount stringLength = 0;
932  UniChar resultChar;
933 
934  UInt32 modifierState = ((shift ? shiftKey : 0) >> 8) & 0xFF;
935  UInt32 keyboardType = LMGetKbdLast();
936 
937  bool isDeadKey = false;
938  OSStatus status =
939  UCKeyTranslate(layout, keyCode, kUCKeyActionDown, modifierState, keyboardType,
940  kUCKeyTranslateNoDeadKeysBit, &deadKeyState, 1, &stringLength, &resultChar);
941  // For dead keys, press the same key again to get the printable representation of the key.
942  if (status == noErr && stringLength == 0 && deadKeyState != 0) {
943  isDeadKey = true;
944  status =
945  UCKeyTranslate(layout, keyCode, kUCKeyActionDown, modifierState, keyboardType,
946  kUCKeyTranslateNoDeadKeysBit, &deadKeyState, 1, &stringLength, &resultChar);
947  }
948 
949  if (status == noErr && stringLength == 1 && !std::iscntrl(resultChar)) {
950  return LayoutClue{resultChar, isDeadKey};
951  }
952  return LayoutClue{0, false};
953 }
954 
955 - (nonnull NSDictionary*)getPressedState {
956  return [_keyboardManager getPressedState];
957 }
958 
959 #pragma mark - NSResponder
960 
961 - (BOOL)acceptsFirstResponder {
962  return YES;
963 }
964 
965 - (void)keyDown:(NSEvent*)event {
966  [_keyboardManager handleEvent:event];
967 }
968 
969 - (void)keyUp:(NSEvent*)event {
970  [_keyboardManager handleEvent:event];
971 }
972 
973 - (void)flagsChanged:(NSEvent*)event {
974  [_keyboardManager handleEvent:event];
975 }
976 
977 - (void)mouseEntered:(NSEvent*)event {
978  if (_mouseState.has_pending_exit) {
979  _mouseState.has_pending_exit = false;
980  } else {
981  [self dispatchMouseEvent:event phase:kAdd];
982  }
983 }
984 
985 - (void)mouseExited:(NSEvent*)event {
986  if (_mouseState.buttons != 0) {
987  _mouseState.has_pending_exit = true;
988  return;
989  }
990  [self dispatchMouseEvent:event phase:kRemove];
991 }
992 
993 - (void)mouseDown:(NSEvent*)event {
994  _mouseState.buttons |= kFlutterPointerButtonMousePrimary;
995  [self dispatchMouseEvent:event];
996 }
997 
998 - (void)mouseUp:(NSEvent*)event {
999  _mouseState.buttons &= ~static_cast<uint64_t>(kFlutterPointerButtonMousePrimary);
1000  [self dispatchMouseEvent:event];
1001 }
1002 
1003 - (void)mouseDragged:(NSEvent*)event {
1004  [self dispatchMouseEvent:event];
1005 }
1006 
1007 - (void)rightMouseDown:(NSEvent*)event {
1008  _mouseState.buttons |= kFlutterPointerButtonMouseSecondary;
1009  [self dispatchMouseEvent:event];
1010 }
1011 
1012 - (void)rightMouseUp:(NSEvent*)event {
1013  _mouseState.buttons &= ~static_cast<uint64_t>(kFlutterPointerButtonMouseSecondary);
1014  [self dispatchMouseEvent:event];
1015 }
1016 
1017 - (void)rightMouseDragged:(NSEvent*)event {
1018  [self dispatchMouseEvent:event];
1019 }
1020 
1021 - (void)otherMouseDown:(NSEvent*)event {
1022  _mouseState.buttons |= (1 << event.buttonNumber);
1023  [self dispatchMouseEvent:event];
1024 }
1025 
1026 - (void)otherMouseUp:(NSEvent*)event {
1027  _mouseState.buttons &= ~static_cast<uint64_t>(1 << event.buttonNumber);
1028  [self dispatchMouseEvent:event];
1029 }
1030 
1031 - (void)otherMouseDragged:(NSEvent*)event {
1032  [self dispatchMouseEvent:event];
1033 }
1034 
1035 - (void)mouseMoved:(NSEvent*)event {
1036  [self dispatchMouseEvent:event];
1037 }
1038 
1039 - (void)scrollWheel:(NSEvent*)event {
1040  [self dispatchGestureEvent:event];
1041 }
1042 
1043 - (void)magnifyWithEvent:(NSEvent*)event {
1044  [self dispatchGestureEvent:event];
1045 }
1046 
1047 - (void)rotateWithEvent:(NSEvent*)event {
1048  [self dispatchGestureEvent:event];
1049 }
1050 
1051 - (void)swipeWithEvent:(NSEvent*)event {
1052  // Not needed, it's handled by scrollWheel.
1053 }
1054 
1055 - (void)touchesBeganWithEvent:(NSEvent*)event {
1056  NSTouch* touch = event.allTouches.anyObject;
1057  if (touch != nil) {
1058  if ((event.timestamp - _mouseState.last_scroll_momentum_changed_time) <
1059  kTrackpadTouchInertiaCancelWindowMs) {
1060  // The trackpad has been touched following a scroll momentum event.
1061  // A scroll inertia cancel message should be sent to the framework.
1062  NSPoint locationInView = [self.flutterView convertPoint:event.locationInWindow fromView:nil];
1063  NSPoint locationInBackingCoordinates =
1064  [self.flutterView convertPointToBacking:locationInView];
1065  FlutterPointerEvent flutterEvent = {
1066  .struct_size = sizeof(flutterEvent),
1067  .timestamp = static_cast<size_t>(event.timestamp * USEC_PER_SEC),
1068  .x = locationInBackingCoordinates.x,
1069  .y = -locationInBackingCoordinates.y, // convertPointToBacking makes this negative.
1070  .device = kPointerPanZoomDeviceId,
1071  .signal_kind = kFlutterPointerSignalKindScrollInertiaCancel,
1072  .device_kind = kFlutterPointerDeviceKindTrackpad,
1073  .view_id = static_cast<FlutterViewId>(_viewId),
1074  };
1075 
1076  [_engine sendPointerEvent:flutterEvent];
1077  // Ensure no further scroll inertia cancel event will be sent.
1078  _mouseState.last_scroll_momentum_changed_time = 0;
1079  }
1080  }
1081 }
1082 
1083 @end
flutter::LayoutClue
Definition: FlutterKeyboardViewDelegate.h:20
FlutterEngine
Definition: FlutterEngine.h:30
FlutterViewController
Definition: FlutterViewController.h:65
FlutterEngine.h
FlutterViewWrapper
Definition: FlutterViewController.mm:172
FlutterEngine_Internal.h
+[FlutterDartProject lookupKeyForAsset:]
NSString * lookupKeyForAsset:(NSString *asset)
Definition: FlutterDartProject.mm:116
_keyboardLayoutNotifier
flutter::KeyboardLayoutNotifier _keyboardLayoutNotifier
Definition: FlutterKeyboardManagerTest.mm:243
_bridge
std::shared_ptr< flutter::AccessibilityBridgeMac > _bridge
Definition: FlutterViewController.mm:340
FlutterChannels.h
FlutterRenderer.h
_project
FlutterDartProject * _project
Definition: FlutterEngine.mm:403
FlutterViewController::engine
FlutterEngine * engine
Definition: FlutterViewController.h:70
FlutterPluginRegistrar-p
Definition: FlutterPluginRegistrarMacOS.h:28
FlutterKeyPrimaryResponder.h
-[FlutterView setBackgroundColor:]
void setBackgroundColor:(nonnull NSColor *color)
_controller
__weak FlutterViewController * _controller
Definition: FlutterViewController.mm:269
_id
FlutterViewId _id
Definition: FlutterViewController.mm:346
FlutterThreadSynchronizer
Definition: FlutterThreadSynchronizer.h:16
flutter::KeyboardLayoutNotifier
void(^ KeyboardLayoutNotifier)()
Definition: FlutterKeyboardViewDelegate.h:16
FlutterTextInputPlugin
Definition: FlutterTextInputPlugin.h:27
FlutterCodecs.h
FlutterKeyboardManager.h
FlutterViewController_Internal.h
_threadSynchronizer
FlutterThreadSynchronizer * _threadSynchronizer
Definition: FlutterViewController.mm:350
FlutterView
Definition: FlutterView.h:48
FlutterTextInputSemanticsObject.h
+[FlutterDartProject lookupKeyForAsset:fromPackage:]
NSString * lookupKeyForAsset:fromPackage:(NSString *asset,[fromPackage] NSString *package)
Definition: FlutterDartProject.mm:125
FlutterDartProject
Definition: FlutterDartProject.mm:24
FlutterKeyboardManager
Definition: FlutterKeyboardManager.h:27
FlutterBinaryMessenger-p
Definition: FlutterBinaryMessenger.h:49
FlutterView.h
FlutterViewDelegate-p
Definition: FlutterView.h:31
FlutterViewController.h
FlutterViewId
int64_t FlutterViewId
Definition: FlutterView.h:15