1 /*
   2  * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 #import "JNIUtilities.h"
  27 
  28 #import <ApplicationServices/ApplicationServices.h>
  29 
  30 #import "CRobotKeyCode.h"
  31 #import "LWCToolkit.h"
  32 #import "sun_lwawt_macosx_CRobot.h"
  33 #import "java_awt_event_InputEvent.h"
  34 #import "java_awt_event_KeyEvent.h"
  35 #import "sizecalc.h"
  36 #import "ThreadUtilities.h"
  37 
  38 // Starting number for event numbers generated by Robot.
  39 // Apple docs don't mention at all what are the requirements
  40 // for these numbers. It seems that they must be higher
  41 // than event numbers from real events, which start at some
  42 // value close to zero. There is no API for obtaining current
  43 // event number, so we have to start from some random number.
  44 // 32000 as starting value works for me, let's hope that it will
  45 // work for others as well.
  46 #define ROBOT_EVENT_NUMBER_START 32000
  47 
  48 #define k_JAVA_ROBOT_WHEEL_COUNT 1
  49 
  50 #if !defined(kCGBitmapByteOrder32Host)
  51 #define kCGBitmapByteOrder32Host 0
  52 #endif
  53 
  54 // In OS X, left and right mouse button share the same click count.
  55 // That is, if one starts clicking the left button rapidly and then
  56 // switches to the right button, then the click count will continue
  57 // increasing, without dropping to 1 in between. The middle button,
  58 // however, has its own click count.
  59 // For robot, we aren't going to emulate all that complexity. All our
  60 // synhtetic clicks share the same click count.
  61 static int gsClickCount;
  62 static NSTimeInterval gsLastClickTime;
  63 
  64 // Apparently, for mouse up/down events we have to set an event number
  65 // that is incremented on each button press. Otherwise, strange things
  66 // happen with z-order.
  67 static int gsEventNumber;
  68 static int* gsButtonEventNumber;
  69 static NSTimeInterval gNextKeyEventTime;
  70 
  71 static inline CGKeyCode GetCGKeyCode(jint javaKeyCode);
  72 
  73 static void PostMouseEvent(const CGPoint point, CGMouseButton button,
  74                            CGEventType type, int clickCount, int eventNumber);
  75 
  76 static int GetClickCount(BOOL isDown);
  77 
  78 static void
  79 CreateJavaException(JNIEnv* env, CGError err)
  80 {
  81     // Throw a java exception indicating what is wrong.
  82     NSString* s = [NSString stringWithFormat:@"Robot: CGError: %d", err];
  83     (*env)->ThrowNew(env, (*env)->FindClass(env, "java/awt/AWTException"),
  84                      [s UTF8String]);
  85 }
  86 
  87 /**
  88  * Saves the "safe moment" when the NEXT event can be posted by the robot safely
  89  * and sleeps for some time if the "safe moment" for the CURRENT event is not
  90  * reached.
  91  *
  92  * We need to sleep to give time for the macOS to update the state.
  93  *
  94  * The "mouse move" events are skipped, because it is not a big issue if we lost
  95  * some of them, the latest coordinates are saved in the peer and will be used
  96  * for clicks.
  97  */
  98 static inline void autoDelay(BOOL isMove) {
  99     if (!isMove){
 100         NSTimeInterval now = [[NSDate date] timeIntervalSinceReferenceDate];
 101         NSTimeInterval delay = gNextKeyEventTime - now;
 102         if (delay > 0) {
 103             [NSThread sleepForTimeInterval:delay];
 104         }
 105     }
 106     gNextKeyEventTime = [[NSDate date] timeIntervalSinceReferenceDate] + 0.050;
 107 }
 108 
 109 /*
 110  * Class:     sun_lwawt_macosx_CRobot
 111  * Method:    initRobot
 112  * Signature: (V)V
 113  */
 114 JNIEXPORT void JNICALL
 115 Java_sun_lwawt_macosx_CRobot_initRobot
 116 (JNIEnv *env, jobject peer)
 117 {
 118     // Set things up to let our app act like a synthetic keyboard and mouse.
 119     // Always set all states, in case Apple ever changes default behaviors.
 120     static int setupDone = 0;
 121     if (!setupDone) {
 122         [ThreadUtilities performOnMainThreadWaiting:NO block:^(){
 123             int i;
 124             jint* tmp;
 125             jboolean copy = JNI_FALSE;
 126 
 127             setupDone = 1;
 128             // Don't block local events after posting ours
 129             CGSetLocalEventsSuppressionInterval(0.0);
 130 
 131             // Let our event's modifier key state blend with local hardware events
 132             CGEnableEventStateCombining(TRUE);
 133 
 134             // Don't let our events block local hardware events
 135             CGSetLocalEventsFilterDuringSupressionState(
 136                                         kCGEventFilterMaskPermitAllEvents,
 137                                         kCGEventSupressionStateSupressionInterval);
 138             CGSetLocalEventsFilterDuringSupressionState(
 139                                         kCGEventFilterMaskPermitAllEvents,
 140                                         kCGEventSupressionStateRemoteMouseDrag);
 141 
 142             gsClickCount = 0;
 143             gsLastClickTime = 0;
 144             gNextKeyEventTime = 0;
 145             gsEventNumber = ROBOT_EVENT_NUMBER_START;
 146 
 147             gsButtonEventNumber = (int*)SAFE_SIZE_ARRAY_ALLOC(malloc, sizeof(int), gNumberOfButtons);
 148             if (gsButtonEventNumber == NULL) {
 149                 JNU_ThrowOutOfMemoryError(env, NULL);
 150                 return;
 151             }
 152 
 153             for (i = 0; i < gNumberOfButtons; ++i) {
 154                 gsButtonEventNumber[i] = ROBOT_EVENT_NUMBER_START;
 155             }
 156         }];
 157     }
 158 }
 159 
 160 /*
 161  * Class:     sun_lwawt_macosx_CRobot
 162  * Method:    mouseEvent
 163  * Signature: (IIIIZZ)V
 164  */
 165 JNIEXPORT void JNICALL
 166 Java_sun_lwawt_macosx_CRobot_mouseEvent
 167 (JNIEnv *env, jobject peer, jint mouseLastX, jint mouseLastY, jint buttonsState,
 168  jboolean isButtonsDownState, jboolean isMouseMove)
 169 {
 170     JNI_COCOA_ENTER(env);
 171     autoDelay(isMouseMove);
 172 
 173     // This is the native method called when Robot mouse events occur.
 174     // The CRobot tracks the mouse position, and which button was
 175     // pressed. The peer also tracks the mouse button desired state,
 176     // the appropriate key modifier state, and whether the mouse action
 177     // is simply a mouse move with no mouse button state changes.
 178 
 179     // volatile, otherwise it warns that it might be clobbered by 'longjmp'
 180     volatile CGPoint point;
 181 
 182     point.x = mouseLastX;
 183     point.y = mouseLastY;
 184 
 185     __block CGMouseButton button = kCGMouseButtonLeft;
 186     __block CGEventType type = kCGEventMouseMoved;
 187 
 188     void (^HandleRobotButton)(CGMouseButton, CGEventType, CGEventType, CGEventType) =
 189         ^(CGMouseButton cgButton, CGEventType cgButtonUp, CGEventType cgButtonDown,
 190           CGEventType cgButtonDragged) {
 191 
 192             button = cgButton;
 193             type = cgButtonUp;
 194 
 195             if (isButtonsDownState) {
 196                 if (isMouseMove) {
 197                     type = cgButtonDragged;
 198                 } else {
 199                     type = cgButtonDown;
 200                 }
 201             }
 202         };
 203 
 204     // Left
 205     if (buttonsState & java_awt_event_InputEvent_BUTTON1_MASK ||
 206         buttonsState & java_awt_event_InputEvent_BUTTON1_DOWN_MASK ) {
 207 
 208         HandleRobotButton(kCGMouseButtonLeft, kCGEventLeftMouseUp,
 209                           kCGEventLeftMouseDown, kCGEventLeftMouseDragged);
 210     }
 211 
 212     // Other
 213     if (buttonsState & java_awt_event_InputEvent_BUTTON2_MASK ||
 214         buttonsState & java_awt_event_InputEvent_BUTTON2_DOWN_MASK ) {
 215 
 216         HandleRobotButton(kCGMouseButtonCenter, kCGEventOtherMouseUp,
 217                           kCGEventOtherMouseDown, kCGEventOtherMouseDragged);
 218     }
 219 
 220     // Right
 221     if (buttonsState & java_awt_event_InputEvent_BUTTON3_MASK ||
 222         buttonsState & java_awt_event_InputEvent_BUTTON3_DOWN_MASK ) {
 223 
 224         HandleRobotButton(kCGMouseButtonRight, kCGEventRightMouseUp,
 225                           kCGEventRightMouseDown, kCGEventRightMouseDragged);
 226     }
 227 
 228     // Extra
 229     if (gNumberOfButtons > 3) {
 230         int extraButton;
 231         for (extraButton = 3; extraButton < gNumberOfButtons; ++extraButton) {
 232             if ((buttonsState & gButtonDownMasks[extraButton])) {
 233                 HandleRobotButton(extraButton, kCGEventOtherMouseUp,
 234                             kCGEventOtherMouseDown, kCGEventOtherMouseDragged);
 235             }
 236         }
 237     }
 238 
 239     int clickCount = 0;
 240     int eventNumber = gsEventNumber;
 241 
 242     if (isMouseMove) {
 243         // any mouse movement resets click count
 244         gsLastClickTime = 0;
 245     } else {
 246         clickCount = GetClickCount(isButtonsDownState);
 247 
 248         if (isButtonsDownState) {
 249             gsButtonEventNumber[button] = gsEventNumber++;
 250         }
 251         eventNumber = gsButtonEventNumber[button];
 252     }
 253 
 254     PostMouseEvent(point, button, type, clickCount, eventNumber);
 255 
 256     JNI_COCOA_EXIT(env);
 257 }
 258 
 259 /*
 260  * Class:     sun_lwawt_macosx_CRobot
 261  * Method:    mouseWheel
 262  * Signature: (I)V
 263  */
 264 JNIEXPORT void JNICALL
 265 Java_sun_lwawt_macosx_CRobot_mouseWheel
 266 (JNIEnv *env, jobject peer, jint wheelAmt)
 267 {
 268     autoDelay(NO);
 269     [ThreadUtilities performOnMainThreadWaiting:YES block:^(){
 270         CGEventSourceRef source = CGEventSourceCreate(kCGEventSourceStateHIDSystemState);
 271         CGEventRef event = CGEventCreateScrollWheelEvent(source,
 272                                                 kCGScrollEventUnitLine,
 273                                                 k_JAVA_ROBOT_WHEEL_COUNT, wheelAmt);
 274         if (event != NULL) {
 275             CGEventPost(kCGHIDEventTap, event);
 276             CFRelease(event);
 277         }
 278         if (source != NULL) {
 279             CFRelease(source);
 280         }
 281     }];
 282 }
 283 
 284 /*
 285  * Class:     sun_lwawt_macosx_CRobot
 286  * Method:    keyEvent
 287  * Signature: (IZ)V
 288  */
 289 JNIEXPORT void JNICALL
 290 Java_sun_lwawt_macosx_CRobot_keyEvent
 291 (JNIEnv *env, jobject peer, jint javaKeyCode, jboolean keyPressed)
 292 {
 293     autoDelay(NO);
 294     [ThreadUtilities performOnMainThreadWaiting:YES block:^(){
 295         CGEventSourceRef source = CGEventSourceCreate(kCGEventSourceStateHIDSystemState);
 296         CGKeyCode keyCode = GetCGKeyCode(javaKeyCode);
 297         CGEventRef event = CGEventCreateKeyboardEvent(source, keyCode, keyPressed);
 298         if (event != NULL) {
 299             CGEventPost(kCGHIDEventTap, event);
 300             CFRelease(event);
 301         }
 302         if (source != NULL) {
 303             CFRelease(source);
 304         }
 305     }];
 306 }
 307 
 308 /*
 309  * Class:     sun_lwawt_macosx_CRobot
 310  * Method:    nativeGetScreenPixels
 311  * Signature: (IIIII[I)V
 312  */
 313 JNIEXPORT void JNICALL
 314 Java_sun_lwawt_macosx_CRobot_nativeGetScreenPixels
 315 (JNIEnv *env, jobject peer,
 316  jint x, jint y, jint width, jint height, jdouble scale, jintArray pixels)
 317 {
 318     JNI_COCOA_ENTER(env);
 319 
 320     jint picX = x;
 321     jint picY = y;
 322     jint picWidth = width;
 323     jint picHeight = height;
 324 
 325     CGRect screenRect = CGRectMake(picX / scale, picY / scale,
 326                                 picWidth / scale, picHeight / scale);
 327     CGImageRef screenPixelsImage = CGWindowListCreateImage(screenRect,
 328                                         kCGWindowListOptionOnScreenOnly,
 329                                         kCGNullWindowID, kCGWindowImageBestResolution);
 330 
 331     if (screenPixelsImage == NULL) {
 332         return;
 333     }
 334 
 335     // get a pointer to the Java int array
 336     void *jPixelData = (*env)->GetPrimitiveArrayCritical(env, pixels, 0);
 337     CHECK_NULL(jPixelData);
 338 
 339     // create a graphics context around the Java int array
 340     CGColorSpaceRef picColorSpace = CGColorSpaceCreateWithName(
 341                                             kCGColorSpaceSRGB);
 342     CGContextRef jPicContextRef = CGBitmapContextCreate(
 343                                             jPixelData,
 344                                             picWidth, picHeight,
 345                                             8, picWidth * sizeof(jint),
 346                                             picColorSpace,
 347                                             kCGBitmapByteOrder32Host |
 348                                             kCGImageAlphaPremultipliedFirst);
 349 
 350     CGColorSpaceRelease(picColorSpace);
 351 
 352     // flip, scale, and color correct the screen image into the Java pixels
 353     CGRect bounds = { { 0, 0 }, { picWidth, picHeight } };
 354     CGContextDrawImage(jPicContextRef, bounds, screenPixelsImage);
 355     CGContextFlush(jPicContextRef);
 356 
 357     // cleanup
 358     CGContextRelease(jPicContextRef);
 359     CGImageRelease(screenPixelsImage);
 360 
 361     // release the Java int array back up to the JVM
 362     (*env)->ReleasePrimitiveArrayCritical(env, pixels, jPixelData, 0);
 363 
 364     JNI_COCOA_EXIT(env);
 365 }
 366 
 367 /****************************************************
 368  * Helper methods
 369  ****************************************************/
 370 
 371 static void PostMouseEvent(const CGPoint point, CGMouseButton button,
 372                            CGEventType type, int clickCount, int eventNumber)
 373 {
 374     [ThreadUtilities performOnMainThreadWaiting:YES block:^(){
 375         CGEventSourceRef source = CGEventSourceCreate(kCGEventSourceStateHIDSystemState);
 376         CGEventRef mouseEvent = CGEventCreateMouseEvent(source, type, point, button);
 377         if (mouseEvent != NULL) {
 378             CGEventSetIntegerValueField(mouseEvent, kCGMouseEventClickState, clickCount);
 379             CGEventSetIntegerValueField(mouseEvent, kCGMouseEventNumber, eventNumber);
 380             CGEventPost(kCGHIDEventTap, mouseEvent);
 381             CFRelease(mouseEvent);
 382         }
 383         if (source != NULL) {
 384             CFRelease(source);
 385         }
 386     }];
 387 }
 388 
 389 static inline CGKeyCode GetCGKeyCode(jint javaKeyCode)
 390 {
 391     CRobotKeyCodeMapping *keyCodeMapping = [CRobotKeyCodeMapping sharedInstance];
 392     return [keyCodeMapping getOSXKeyCodeForJavaKey:javaKeyCode];
 393 }
 394 
 395 static int GetClickCount(BOOL isDown) {
 396     NSTimeInterval now = [[NSDate date] timeIntervalSinceReferenceDate];
 397     NSTimeInterval clickInterval = now - gsLastClickTime;
 398     BOOL isWithinTreshold = clickInterval < [NSEvent doubleClickInterval];
 399 
 400     if (isDown) {
 401         if (isWithinTreshold) {
 402             gsClickCount++;
 403         } else {
 404             gsClickCount = 1;
 405         }
 406 
 407         gsLastClickTime = now;
 408     } else {
 409         // In OS X, a mouse up has the click count of the last mouse down
 410         // if an interval between up and down is within the double click
 411         // threshold, and 0 otherwise.
 412         if (!isWithinTreshold) {
 413             gsClickCount = 0;
 414         }
 415     }
 416 
 417     return gsClickCount;
 418 }