1 /* 2 * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26 #import "jni_util.h" 27 28 #import <JavaNativeFoundation/JavaNativeFoundation.h> 29 #import <ApplicationServices/ApplicationServices.h> 30 31 #import "CRobotKeyCode.h" 32 #import "LWCToolkit.h" 33 #import "sun_lwawt_macosx_CRobot.h" 34 #import "java_awt_event_InputEvent.h" 35 #import "java_awt_event_KeyEvent.h" 36 #import "sizecalc.h" 37 #import "ThreadUtilities.h" 38 39 // Starting number for event numbers generated by Robot. 40 // Apple docs don't mention at all what are the requirements 41 // for these numbers. It seems that they must be higher 42 // than event numbers from real events, which start at some 43 // value close to zero. There is no API for obtaining current 44 // event number, so we have to start from some random number. 45 // 32000 as starting value works for me, let's hope that it will 46 // work for others as well. 47 #define ROBOT_EVENT_NUMBER_START 32000 48 49 #define k_JAVA_ROBOT_WHEEL_COUNT 1 50 51 #if !defined(kCGBitmapByteOrder32Host) 52 #define kCGBitmapByteOrder32Host 0 53 #endif 54 55 // In OS X, left and right mouse button share the same click count. 56 // That is, if one starts clicking the left button rapidly and then 57 // switches to the right button, then the click count will continue 58 // increasing, without dropping to 1 in between. The middle button, 59 // however, has its own click count. 60 // For robot, we aren't going to emulate all that complexity. All our 61 // synhtetic clicks share the same click count. 62 static int gsClickCount; 63 static NSTimeInterval gsLastClickTime; 64 65 // Apparently, for mouse up/down events we have to set an event number 66 // that is incremented on each button press. Otherwise, strange things 67 // happen with z-order. 68 static int gsEventNumber; 69 static int* gsButtonEventNumber; 70 static NSTimeInterval gNextKeyEventTime; 71 72 static inline CGKeyCode GetCGKeyCode(jint javaKeyCode); 73 74 static void PostMouseEvent(const CGPoint point, CGMouseButton button, 75 CGEventType type, int clickCount, int eventNumber); 76 77 static int GetClickCount(BOOL isDown); 78 79 static void 80 CreateJavaException(JNIEnv* env, CGError err) 81 { 82 // Throw a java exception indicating what is wrong. 83 NSString* s = [NSString stringWithFormat:@"Robot: CGError: %d", err]; 84 (*env)->ThrowNew(env, (*env)->FindClass(env, "java/awt/AWTException"), 85 [s UTF8String]); 86 } 87 88 /** 89 * Saves the "safe moment" when the NEXT event can be posted by the robot safely 90 * and sleeps for some time if the "safe moment" for the CURRENT event is not 91 * reached. 92 * 93 * We need to sleep to give time for the macOS to update the state. 94 * 95 * The "mouse move" events are skipped, because it is not a big issue if we lost 96 * some of them, the latest coordinates are saved in the peer and will be used 97 * for clicks. 98 */ 99 static inline void autoDelay(BOOL isMove) { 100 if (!isMove){ 101 NSTimeInterval now = [[NSDate date] timeIntervalSinceReferenceDate]; 102 NSTimeInterval delay = gNextKeyEventTime - now; 103 if (delay > 0) { 104 [NSThread sleepForTimeInterval:delay]; 105 } 106 } 107 gNextKeyEventTime = [[NSDate date] timeIntervalSinceReferenceDate] + 0.050; 108 } 109 110 /* 111 * Class: sun_lwawt_macosx_CRobot 112 * Method: initRobot 113 * Signature: (V)V 114 */ 115 JNIEXPORT void JNICALL 116 Java_sun_lwawt_macosx_CRobot_initRobot 117 (JNIEnv *env, jobject peer) 118 { 119 // Set things up to let our app act like a synthetic keyboard and mouse. 120 // Always set all states, in case Apple ever changes default behaviors. 121 static int setupDone = 0; 122 if (!setupDone) { 123 [ThreadUtilities performOnMainThreadWaiting:NO block:^(){ 124 int i; 125 jint* tmp; 126 jboolean copy = JNI_FALSE; 127 128 setupDone = 1; 129 // Don't block local events after posting ours 130 CGSetLocalEventsSuppressionInterval(0.0); 131 132 // Let our event's modifier key state blend with local hardware events 133 CGEnableEventStateCombining(TRUE); 134 135 // Don't let our events block local hardware events 136 CGSetLocalEventsFilterDuringSupressionState( 137 kCGEventFilterMaskPermitAllEvents, 138 kCGEventSupressionStateSupressionInterval); 139 CGSetLocalEventsFilterDuringSupressionState( 140 kCGEventFilterMaskPermitAllEvents, 141 kCGEventSupressionStateRemoteMouseDrag); 142 143 gsClickCount = 0; 144 gsLastClickTime = 0; 145 gNextKeyEventTime = 0; 146 gsEventNumber = ROBOT_EVENT_NUMBER_START; 147 148 gsButtonEventNumber = (int*)SAFE_SIZE_ARRAY_ALLOC(malloc, sizeof(int), gNumberOfButtons); 149 if (gsButtonEventNumber == NULL) { 150 JNU_ThrowOutOfMemoryError(env, NULL); 151 return; 152 } 153 154 for (i = 0; i < gNumberOfButtons; ++i) { 155 gsButtonEventNumber[i] = ROBOT_EVENT_NUMBER_START; 156 } 157 }]; 158 } 159 } 160 161 /* 162 * Class: sun_lwawt_macosx_CRobot 163 * Method: mouseEvent 164 * Signature: (IIIIZZ)V 165 */ 166 JNIEXPORT void JNICALL 167 Java_sun_lwawt_macosx_CRobot_mouseEvent 168 (JNIEnv *env, jobject peer, jint mouseLastX, jint mouseLastY, jint buttonsState, 169 jboolean isButtonsDownState, jboolean isMouseMove) 170 { 171 JNF_COCOA_ENTER(env); 172 autoDelay(isMouseMove); 173 174 // This is the native method called when Robot mouse events occur. 175 // The CRobot tracks the mouse position, and which button was 176 // pressed. The peer also tracks the mouse button desired state, 177 // the appropriate key modifier state, and whether the mouse action 178 // is simply a mouse move with no mouse button state changes. 179 180 // volatile, otherwise it warns that it might be clobbered by 'longjmp' 181 volatile CGPoint point; 182 183 point.x = mouseLastX; 184 point.y = mouseLastY; 185 186 __block CGMouseButton button = kCGMouseButtonLeft; 187 __block CGEventType type = kCGEventMouseMoved; 188 189 void (^HandleRobotButton)(CGMouseButton, CGEventType, CGEventType, CGEventType) = 190 ^(CGMouseButton cgButton, CGEventType cgButtonUp, CGEventType cgButtonDown, 191 CGEventType cgButtonDragged) { 192 193 button = cgButton; 194 type = cgButtonUp; 195 196 if (isButtonsDownState) { 197 if (isMouseMove) { 198 type = cgButtonDragged; 199 } else { 200 type = cgButtonDown; 201 } 202 } 203 }; 204 205 // Left 206 if (buttonsState & java_awt_event_InputEvent_BUTTON1_MASK || 207 buttonsState & java_awt_event_InputEvent_BUTTON1_DOWN_MASK ) { 208 209 HandleRobotButton(kCGMouseButtonLeft, kCGEventLeftMouseUp, 210 kCGEventLeftMouseDown, kCGEventLeftMouseDragged); 211 } 212 213 // Other 214 if (buttonsState & java_awt_event_InputEvent_BUTTON2_MASK || 215 buttonsState & java_awt_event_InputEvent_BUTTON2_DOWN_MASK ) { 216 217 HandleRobotButton(kCGMouseButtonCenter, kCGEventOtherMouseUp, 218 kCGEventOtherMouseDown, kCGEventOtherMouseDragged); 219 } 220 221 // Right 222 if (buttonsState & java_awt_event_InputEvent_BUTTON3_MASK || 223 buttonsState & java_awt_event_InputEvent_BUTTON3_DOWN_MASK ) { 224 225 HandleRobotButton(kCGMouseButtonRight, kCGEventRightMouseUp, 226 kCGEventRightMouseDown, kCGEventRightMouseDragged); 227 } 228 229 // Extra 230 if (gNumberOfButtons > 3) { 231 int extraButton; 232 for (extraButton = 3; extraButton < gNumberOfButtons; ++extraButton) { 233 if ((buttonsState & gButtonDownMasks[extraButton])) { 234 HandleRobotButton(extraButton, kCGEventOtherMouseUp, 235 kCGEventOtherMouseDown, kCGEventOtherMouseDragged); 236 } 237 } 238 } 239 240 int clickCount = 0; 241 int eventNumber = gsEventNumber; 242 243 if (isMouseMove) { 244 // any mouse movement resets click count 245 gsLastClickTime = 0; 246 } else { 247 clickCount = GetClickCount(isButtonsDownState); 248 249 if (isButtonsDownState) { 250 gsButtonEventNumber[button] = gsEventNumber++; 251 } 252 eventNumber = gsButtonEventNumber[button]; 253 } 254 255 PostMouseEvent(point, button, type, clickCount, eventNumber); 256 257 JNF_COCOA_EXIT(env); 258 } 259 260 /* 261 * Class: sun_lwawt_macosx_CRobot 262 * Method: mouseWheel 263 * Signature: (I)V 264 */ 265 JNIEXPORT void JNICALL 266 Java_sun_lwawt_macosx_CRobot_mouseWheel 267 (JNIEnv *env, jobject peer, jint wheelAmt) 268 { 269 autoDelay(NO); 270 [ThreadUtilities performOnMainThreadWaiting:YES block:^(){ 271 CGEventSourceRef source = CGEventSourceCreate(kCGEventSourceStateHIDSystemState); 272 CGEventRef event = CGEventCreateScrollWheelEvent(source, 273 kCGScrollEventUnitLine, 274 k_JAVA_ROBOT_WHEEL_COUNT, wheelAmt); 275 if (event != NULL) { 276 CGEventPost(kCGHIDEventTap, event); 277 CFRelease(event); 278 } 279 if (source != NULL) { 280 CFRelease(source); 281 } 282 }]; 283 } 284 285 /* 286 * Class: sun_lwawt_macosx_CRobot 287 * Method: keyEvent 288 * Signature: (IZ)V 289 */ 290 JNIEXPORT void JNICALL 291 Java_sun_lwawt_macosx_CRobot_keyEvent 292 (JNIEnv *env, jobject peer, jint javaKeyCode, jboolean keyPressed) 293 { 294 autoDelay(NO); 295 [ThreadUtilities performOnMainThreadWaiting:YES block:^(){ 296 CGEventSourceRef source = CGEventSourceCreate(kCGEventSourceStateHIDSystemState); 297 CGKeyCode keyCode = GetCGKeyCode(javaKeyCode); 298 CGEventRef event = CGEventCreateKeyboardEvent(source, keyCode, keyPressed); 299 if (event != NULL) { 300 CGEventPost(kCGHIDEventTap, event); 301 CFRelease(event); 302 } 303 if (source != NULL) { 304 CFRelease(source); 305 } 306 }]; 307 } 308 309 /* 310 * Class: sun_lwawt_macosx_CRobot 311 * Method: nativeGetScreenPixels 312 * Signature: (IIIII[I)V 313 */ 314 JNIEXPORT void JNICALL 315 Java_sun_lwawt_macosx_CRobot_nativeGetScreenPixels 316 (JNIEnv *env, jobject peer, 317 jint x, jint y, jint width, jint height, jdouble scale, jintArray pixels) 318 { 319 JNF_COCOA_ENTER(env); 320 321 jint picX = x; 322 jint picY = y; 323 jint picWidth = width; 324 jint picHeight = height; 325 326 CGRect screenRect = CGRectMake(picX / scale, picY / scale, 327 picWidth / scale, picHeight / scale); 328 CGImageRef screenPixelsImage = CGWindowListCreateImage(screenRect, 329 kCGWindowListOptionOnScreenOnly, 330 kCGNullWindowID, kCGWindowImageBestResolution); 331 332 if (screenPixelsImage == NULL) { 333 return; 334 } 335 336 // get a pointer to the Java int array 337 void *jPixelData = (*env)->GetPrimitiveArrayCritical(env, pixels, 0); 338 CHECK_NULL(jPixelData); 339 340 // create a graphics context around the Java int array 341 CGColorSpaceRef picColorSpace = CGColorSpaceCreateWithName( 342 kCGColorSpaceSRGB); 343 CGContextRef jPicContextRef = CGBitmapContextCreate( 344 jPixelData, 345 picWidth, picHeight, 346 8, picWidth * sizeof(jint), 347 picColorSpace, 348 kCGBitmapByteOrder32Host | 349 kCGImageAlphaPremultipliedFirst); 350 351 CGColorSpaceRelease(picColorSpace); 352 353 // flip, scale, and color correct the screen image into the Java pixels 354 CGRect bounds = { { 0, 0 }, { picWidth, picHeight } }; 355 CGContextDrawImage(jPicContextRef, bounds, screenPixelsImage); 356 CGContextFlush(jPicContextRef); 357 358 // cleanup 359 CGContextRelease(jPicContextRef); 360 CGImageRelease(screenPixelsImage); 361 362 // release the Java int array back up to the JVM 363 (*env)->ReleasePrimitiveArrayCritical(env, pixels, jPixelData, 0); 364 365 JNF_COCOA_EXIT(env); 366 } 367 368 /**************************************************** 369 * Helper methods 370 ****************************************************/ 371 372 static void PostMouseEvent(const CGPoint point, CGMouseButton button, 373 CGEventType type, int clickCount, int eventNumber) 374 { 375 [ThreadUtilities performOnMainThreadWaiting:YES block:^(){ 376 CGEventSourceRef source = CGEventSourceCreate(kCGEventSourceStateHIDSystemState); 377 CGEventRef mouseEvent = CGEventCreateMouseEvent(source, type, point, button); 378 if (mouseEvent != NULL) { 379 CGEventSetIntegerValueField(mouseEvent, kCGMouseEventClickState, clickCount); 380 CGEventSetIntegerValueField(mouseEvent, kCGMouseEventNumber, eventNumber); 381 CGEventPost(kCGHIDEventTap, mouseEvent); 382 CFRelease(mouseEvent); 383 } 384 if (source != NULL) { 385 CFRelease(source); 386 } 387 }]; 388 } 389 390 static inline CGKeyCode GetCGKeyCode(jint javaKeyCode) 391 { 392 CRobotKeyCodeMapping *keyCodeMapping = [CRobotKeyCodeMapping sharedInstance]; 393 return [keyCodeMapping getOSXKeyCodeForJavaKey:javaKeyCode]; 394 } 395 396 static int GetClickCount(BOOL isDown) { 397 NSTimeInterval now = [[NSDate date] timeIntervalSinceReferenceDate]; 398 NSTimeInterval clickInterval = now - gsLastClickTime; 399 BOOL isWithinTreshold = clickInterval < [NSEvent doubleClickInterval]; 400 401 if (isDown) { 402 if (isWithinTreshold) { 403 gsClickCount++; 404 } else { 405 gsClickCount = 1; 406 } 407 408 gsLastClickTime = now; 409 } else { 410 // In OS X, a mouse up has the click count of the last mouse down 411 // if an interval between up and down is within the double click 412 // threshold, and 0 otherwise. 413 if (!isWithinTreshold) { 414 gsClickCount = 0; 415 } 416 } 417 418 return gsClickCount; 419 }