1 /*
   2  * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 #import "jni_util.h"
  27 
  28 #import <JavaNativeFoundation/JavaNativeFoundation.h>
  29 #import <ApplicationServices/ApplicationServices.h>
  30 
  31 #import "CRobotKeyCode.h"
  32 #import "LWCToolkit.h"
  33 #import "sun_lwawt_macosx_CRobot.h"
  34 #import "java_awt_event_InputEvent.h"
  35 #import "java_awt_event_KeyEvent.h"
  36 #import "sizecalc.h"
  37 #import "ThreadUtilities.h"
  38 
  39 // Starting number for event numbers generated by Robot.
  40 // Apple docs don't mention at all what are the requirements
  41 // for these numbers. It seems that they must be higher
  42 // than event numbers from real events, which start at some
  43 // value close to zero. There is no API for obtaining current
  44 // event number, so we have to start from some random number.
  45 // 32000 as starting value works for me, let's hope that it will
  46 // work for others as well.
  47 #define ROBOT_EVENT_NUMBER_START 32000
  48 
  49 #define k_JAVA_ROBOT_WHEEL_COUNT 1
  50 
  51 #if !defined(kCGBitmapByteOrder32Host)
  52 #define kCGBitmapByteOrder32Host 0
  53 #endif
  54 
  55 // In OS X, left and right mouse button share the same click count.
  56 // That is, if one starts clicking the left button rapidly and then
  57 // switches to the right button, then the click count will continue
  58 // increasing, without dropping to 1 in between. The middle button,
  59 // however, has its own click count.
  60 // For robot, we aren't going to emulate all that complexity. All our
  61 // synhtetic clicks share the same click count.
  62 static int gsClickCount;
  63 static NSTimeInterval gsLastClickTime;
  64 
  65 // Apparently, for mouse up/down events we have to set an event number
  66 // that is incremented on each button press. Otherwise, strange things
  67 // happen with z-order.
  68 static int gsEventNumber;
  69 static int* gsButtonEventNumber;
  70 static NSTimeInterval gNextKeyEventTime;
  71 
  72 static inline CGKeyCode GetCGKeyCode(jint javaKeyCode);
  73 
  74 static void PostMouseEvent(const CGPoint point, CGMouseButton button,
  75                            CGEventType type, int clickCount, int eventNumber);
  76 
  77 static int GetClickCount(BOOL isDown);
  78 
  79 static void
  80 CreateJavaException(JNIEnv* env, CGError err)
  81 {
  82     // Throw a java exception indicating what is wrong.
  83     NSString* s = [NSString stringWithFormat:@"Robot: CGError: %d", err];
  84     (*env)->ThrowNew(env, (*env)->FindClass(env, "java/awt/AWTException"),
  85                      [s UTF8String]);
  86 }
  87 
  88 /**
  89  * Saves the "safe moment" when the NEXT event can be posted by the robot safely
  90  * and sleeps for some time if the "safe moment" for the CURRENT event is not
  91  * reached.
  92  *
  93  * We need to sleep to give time for the macOS to update the state.
  94  *
  95  * The "mouse move" events are skipped, because it is not a big issue if we lost
  96  * some of them, the latest coordinates are saved in the peer and will be used
  97  * for clicks.
  98  */
  99 static inline void autoDelay(BOOL isMove) {
 100     if (!isMove){
 101         NSTimeInterval now = [[NSDate date] timeIntervalSinceReferenceDate];
 102         NSTimeInterval delay = gNextKeyEventTime - now;
 103         if (delay > 0) {
 104             [NSThread sleepForTimeInterval:delay];
 105         }
 106     }
 107     gNextKeyEventTime = [[NSDate date] timeIntervalSinceReferenceDate] + 0.050;
 108 }
 109 
 110 /*
 111  * Class:     sun_lwawt_macosx_CRobot
 112  * Method:    initRobot
 113  * Signature: (V)V
 114  */
 115 JNIEXPORT void JNICALL
 116 Java_sun_lwawt_macosx_CRobot_initRobot
 117 (JNIEnv *env, jobject peer)
 118 {
 119     // Set things up to let our app act like a synthetic keyboard and mouse.
 120     // Always set all states, in case Apple ever changes default behaviors.
 121     static int setupDone = 0;
 122     if (!setupDone) {
 123         [ThreadUtilities performOnMainThreadWaiting:NO block:^(){
 124             int i;
 125             jint* tmp;
 126             jboolean copy = JNI_FALSE;
 127 
 128             setupDone = 1;
 129             // Don't block local events after posting ours
 130             CGSetLocalEventsSuppressionInterval(0.0);
 131 
 132             // Let our event's modifier key state blend with local hardware events
 133             CGEnableEventStateCombining(TRUE);
 134 
 135             // Don't let our events block local hardware events
 136             CGSetLocalEventsFilterDuringSupressionState(
 137                                         kCGEventFilterMaskPermitAllEvents,
 138                                         kCGEventSupressionStateSupressionInterval);
 139             CGSetLocalEventsFilterDuringSupressionState(
 140                                         kCGEventFilterMaskPermitAllEvents,
 141                                         kCGEventSupressionStateRemoteMouseDrag);
 142 
 143             gsClickCount = 0;
 144             gsLastClickTime = 0;
 145             gNextKeyEventTime = 0;
 146             gsEventNumber = ROBOT_EVENT_NUMBER_START;
 147 
 148             gsButtonEventNumber = (int*)SAFE_SIZE_ARRAY_ALLOC(malloc, sizeof(int), gNumberOfButtons);
 149             if (gsButtonEventNumber == NULL) {
 150                 JNU_ThrowOutOfMemoryError(env, NULL);
 151                 return;
 152             }
 153 
 154             for (i = 0; i < gNumberOfButtons; ++i) {
 155                 gsButtonEventNumber[i] = ROBOT_EVENT_NUMBER_START;
 156             }
 157         }];
 158     }
 159 }
 160 
 161 /*
 162  * Class:     sun_lwawt_macosx_CRobot
 163  * Method:    mouseEvent
 164  * Signature: (IIIIZZ)V
 165  */
 166 JNIEXPORT void JNICALL
 167 Java_sun_lwawt_macosx_CRobot_mouseEvent
 168 (JNIEnv *env, jobject peer, jint mouseLastX, jint mouseLastY, jint buttonsState,
 169  jboolean isButtonsDownState, jboolean isMouseMove)
 170 {
 171     JNF_COCOA_ENTER(env);
 172     autoDelay(isMouseMove);
 173 
 174     // This is the native method called when Robot mouse events occur.
 175     // The CRobot tracks the mouse position, and which button was
 176     // pressed. The peer also tracks the mouse button desired state,
 177     // the appropriate key modifier state, and whether the mouse action
 178     // is simply a mouse move with no mouse button state changes.
 179 
 180     // volatile, otherwise it warns that it might be clobbered by 'longjmp'
 181     volatile CGPoint point;
 182 
 183     point.x = mouseLastX;
 184     point.y = mouseLastY;
 185 
 186     __block CGMouseButton button = kCGMouseButtonLeft;
 187     __block CGEventType type = kCGEventMouseMoved;
 188 
 189     void (^HandleRobotButton)(CGMouseButton, CGEventType, CGEventType, CGEventType) =
 190         ^(CGMouseButton cgButton, CGEventType cgButtonUp, CGEventType cgButtonDown,
 191           CGEventType cgButtonDragged) {
 192 
 193             button = cgButton;
 194             type = cgButtonUp;
 195 
 196             if (isButtonsDownState) {
 197                 if (isMouseMove) {
 198                     type = cgButtonDragged;
 199                 } else {
 200                     type = cgButtonDown;
 201                 }
 202             }
 203         };
 204 
 205     // Left
 206     if (buttonsState & java_awt_event_InputEvent_BUTTON1_MASK ||
 207         buttonsState & java_awt_event_InputEvent_BUTTON1_DOWN_MASK ) {
 208 
 209         HandleRobotButton(kCGMouseButtonLeft, kCGEventLeftMouseUp,
 210                           kCGEventLeftMouseDown, kCGEventLeftMouseDragged);
 211     }
 212 
 213     // Other
 214     if (buttonsState & java_awt_event_InputEvent_BUTTON2_MASK ||
 215         buttonsState & java_awt_event_InputEvent_BUTTON2_DOWN_MASK ) {
 216 
 217         HandleRobotButton(kCGMouseButtonCenter, kCGEventOtherMouseUp,
 218                           kCGEventOtherMouseDown, kCGEventOtherMouseDragged);
 219     }
 220 
 221     // Right
 222     if (buttonsState & java_awt_event_InputEvent_BUTTON3_MASK ||
 223         buttonsState & java_awt_event_InputEvent_BUTTON3_DOWN_MASK ) {
 224 
 225         HandleRobotButton(kCGMouseButtonRight, kCGEventRightMouseUp,
 226                           kCGEventRightMouseDown, kCGEventRightMouseDragged);
 227     }
 228 
 229     // Extra
 230     if (gNumberOfButtons > 3) {
 231         int extraButton;
 232         for (extraButton = 3; extraButton < gNumberOfButtons; ++extraButton) {
 233             if ((buttonsState & gButtonDownMasks[extraButton])) {
 234                 HandleRobotButton(extraButton, kCGEventOtherMouseUp,
 235                             kCGEventOtherMouseDown, kCGEventOtherMouseDragged);
 236             }
 237         }
 238     }
 239 
 240     int clickCount = 0;
 241     int eventNumber = gsEventNumber;
 242 
 243     if (isMouseMove) {
 244         // any mouse movement resets click count
 245         gsLastClickTime = 0;
 246     } else {
 247         clickCount = GetClickCount(isButtonsDownState);
 248 
 249         if (isButtonsDownState) {
 250             gsButtonEventNumber[button] = gsEventNumber++;
 251         }
 252         eventNumber = gsButtonEventNumber[button];
 253     }
 254 
 255     PostMouseEvent(point, button, type, clickCount, eventNumber);
 256 
 257     JNF_COCOA_EXIT(env);
 258 }
 259 
 260 /*
 261  * Class:     sun_lwawt_macosx_CRobot
 262  * Method:    mouseWheel
 263  * Signature: (I)V
 264  */
 265 JNIEXPORT void JNICALL
 266 Java_sun_lwawt_macosx_CRobot_mouseWheel
 267 (JNIEnv *env, jobject peer, jint wheelAmt)
 268 {
 269     autoDelay(NO);
 270     [ThreadUtilities performOnMainThreadWaiting:YES block:^(){
 271         CGEventSourceRef source = CGEventSourceCreate(kCGEventSourceStateHIDSystemState);
 272         CGEventRef event = CGEventCreateScrollWheelEvent(source,
 273                                                 kCGScrollEventUnitLine,
 274                                                 k_JAVA_ROBOT_WHEEL_COUNT, wheelAmt);
 275         if (event != NULL) {
 276             CGEventPost(kCGHIDEventTap, event);
 277             CFRelease(event);
 278         }
 279         if (source != NULL) {
 280             CFRelease(source);
 281         }
 282     }];
 283 }
 284 
 285 /*
 286  * Class:     sun_lwawt_macosx_CRobot
 287  * Method:    keyEvent
 288  * Signature: (IZ)V
 289  */
 290 JNIEXPORT void JNICALL
 291 Java_sun_lwawt_macosx_CRobot_keyEvent
 292 (JNIEnv *env, jobject peer, jint javaKeyCode, jboolean keyPressed)
 293 {
 294     autoDelay(NO);
 295     [ThreadUtilities performOnMainThreadWaiting:YES block:^(){
 296         CGEventSourceRef source = CGEventSourceCreate(kCGEventSourceStateHIDSystemState);
 297         CGKeyCode keyCode = GetCGKeyCode(javaKeyCode);
 298         CGEventRef event = CGEventCreateKeyboardEvent(source, keyCode, keyPressed);
 299         if (event != NULL) {
 300             CGEventPost(kCGHIDEventTap, event);
 301             CFRelease(event);
 302         }
 303         if (source != NULL) {
 304             CFRelease(source);
 305         }
 306     }];
 307 }
 308 
 309 /*
 310  * Class:     sun_lwawt_macosx_CRobot
 311  * Method:    nativeGetScreenPixels
 312  * Signature: (IIIII[I)V
 313  */
 314 JNIEXPORT void JNICALL
 315 Java_sun_lwawt_macosx_CRobot_nativeGetScreenPixels
 316 (JNIEnv *env, jobject peer,
 317  jint x, jint y, jint width, jint height, jdouble scale, jintArray pixels)
 318 {
 319     JNF_COCOA_ENTER(env);
 320 
 321     jint picX = x;
 322     jint picY = y;
 323     jint picWidth = width;
 324     jint picHeight = height;
 325 
 326     CGRect screenRect = CGRectMake(picX / scale, picY / scale,
 327                                 picWidth / scale, picHeight / scale);
 328     CGImageRef screenPixelsImage = CGWindowListCreateImage(screenRect,
 329                                         kCGWindowListOptionOnScreenOnly,
 330                                         kCGNullWindowID, kCGWindowImageBestResolution);
 331 
 332     if (screenPixelsImage == NULL) {
 333         return;
 334     }
 335 
 336     // get a pointer to the Java int array
 337     void *jPixelData = (*env)->GetPrimitiveArrayCritical(env, pixels, 0);
 338     CHECK_NULL(jPixelData);
 339 
 340     // create a graphics context around the Java int array
 341     CGColorSpaceRef picColorSpace = CGColorSpaceCreateWithName(
 342                                             kCGColorSpaceSRGB);
 343     CGContextRef jPicContextRef = CGBitmapContextCreate(
 344                                             jPixelData,
 345                                             picWidth, picHeight,
 346                                             8, picWidth * sizeof(jint),
 347                                             picColorSpace,
 348                                             kCGBitmapByteOrder32Host |
 349                                             kCGImageAlphaPremultipliedFirst);
 350 
 351     CGColorSpaceRelease(picColorSpace);
 352 
 353     // flip, scale, and color correct the screen image into the Java pixels
 354     CGRect bounds = { { 0, 0 }, { picWidth, picHeight } };
 355     CGContextDrawImage(jPicContextRef, bounds, screenPixelsImage);
 356     CGContextFlush(jPicContextRef);
 357 
 358     // cleanup
 359     CGContextRelease(jPicContextRef);
 360     CGImageRelease(screenPixelsImage);
 361 
 362     // release the Java int array back up to the JVM
 363     (*env)->ReleasePrimitiveArrayCritical(env, pixels, jPixelData, 0);
 364 
 365     JNF_COCOA_EXIT(env);
 366 }
 367 
 368 /****************************************************
 369  * Helper methods
 370  ****************************************************/
 371 
 372 static void PostMouseEvent(const CGPoint point, CGMouseButton button,
 373                            CGEventType type, int clickCount, int eventNumber)
 374 {
 375     [ThreadUtilities performOnMainThreadWaiting:YES block:^(){
 376         CGEventSourceRef source = CGEventSourceCreate(kCGEventSourceStateHIDSystemState);
 377         CGEventRef mouseEvent = CGEventCreateMouseEvent(source, type, point, button);
 378         if (mouseEvent != NULL) {
 379             CGEventSetIntegerValueField(mouseEvent, kCGMouseEventClickState, clickCount);
 380             CGEventSetIntegerValueField(mouseEvent, kCGMouseEventNumber, eventNumber);
 381             CGEventPost(kCGHIDEventTap, mouseEvent);
 382             CFRelease(mouseEvent);
 383         }
 384         if (source != NULL) {
 385             CFRelease(source);
 386         }
 387     }];
 388 }
 389 
 390 static inline CGKeyCode GetCGKeyCode(jint javaKeyCode)
 391 {
 392     CRobotKeyCodeMapping *keyCodeMapping = [CRobotKeyCodeMapping sharedInstance];
 393     return [keyCodeMapping getOSXKeyCodeForJavaKey:javaKeyCode];
 394 }
 395 
 396 static int GetClickCount(BOOL isDown) {
 397     NSTimeInterval now = [[NSDate date] timeIntervalSinceReferenceDate];
 398     NSTimeInterval clickInterval = now - gsLastClickTime;
 399     BOOL isWithinTreshold = clickInterval < [NSEvent doubleClickInterval];
 400 
 401     if (isDown) {
 402         if (isWithinTreshold) {
 403             gsClickCount++;
 404         } else {
 405             gsClickCount = 1;
 406         }
 407 
 408         gsLastClickTime = now;
 409     } else {
 410         // In OS X, a mouse up has the click count of the last mouse down
 411         // if an interval between up and down is within the double click
 412         // threshold, and 0 otherwise.
 413         if (!isWithinTreshold) {
 414             gsClickCount = 0;
 415         }
 416     }
 417 
 418     return gsClickCount;
 419 }