1 /*
2 * Copyright (c) 2007, 2013, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 // SunJSSE does not support dynamic system properties, no way to re-use
25 // system properties in samevm/agentvm mode.
26
27 /*
28 * @test
29 * @bug 6492872
30 * @summary Deadlock in SSLEngine
31 * @run main/othervm SSLEngineDeadlock
32 * @author Brad R. Wetmore
33 */
34
35 /**
36 * A SSLEngine usage example which simplifies the presentation
37 * by removing the I/O and multi-threading concerns.
38 *
39 * The test creates two SSLEngines, simulating a client and server.
40 * The "transport" layer consists two byte buffers: think of them
41 * as directly connected pipes.
42 *
43 * Note, this is a *very* simple example: real code will be much more
44 * involved. For example, different threading and I/O models could be
45 * used, transport mechanisms could close unexpectedly, and so on.
46 *
47 * When this application runs, notice that several messages
48 * (wrap/unwrap) pass before any application data is consumed or
49 * produced. (For more information, please see the SSL/TLS
50 * specifications.) There may several steps for a successful handshake,
51 * so it's typical to see the following series of operations:
52 *
53 * client server message
54 * ====== ====== =======
55 * wrap() ... ClientHello
56 * ... unwrap() ClientHello
57 * ... wrap() ServerHello/Certificate
58 * unwrap() ... ServerHello/Certificate
59 * wrap() ... ClientKeyExchange
60 * wrap() ... ChangeCipherSpec
61 * wrap() ... Finished
62 * ... unwrap() ClientKeyExchange
63 * ... unwrap() ChangeCipherSpec
64 * ... unwrap() Finished
65 * ... wrap() ChangeCipherSpec
66 * ... wrap() Finished
67 * unwrap() ... ChangeCipherSpec
68 * unwrap() ... Finished
69 */
70
71 import javax.net.ssl.*;
72 import javax.net.ssl.SSLEngineResult.*;
73 import java.io.*;
74 import java.security.*;
75 import java.nio.*;
76 import java.lang.management.*;
77
78 public class SSLEngineDeadlock {
79
80 /*
81 * Enables logging of the SSLEngine operations.
82 */
83 private static boolean logging = false;
84
85 /*
86 * Enables the JSSE system debugging system property:
87 *
88 * -Djavax.net.debug=all
89 *
90 * This gives a lot of low-level information about operations underway,
91 * including specific handshake messages, and might be best examined
92 * after gaining some familiarity with this application.
93 */
94 private static boolean debug = false;
95
96 private SSLContext sslc;
97
98 private SSLEngine clientEngine; // client Engine
99 private ByteBuffer clientOut; // write side of clientEngine
100 private ByteBuffer clientIn; // read side of clientEngine
101
102 private SSLEngine serverEngine; // server Engine
103 private ByteBuffer serverOut; // write side of serverEngine
104 private ByteBuffer serverIn; // read side of serverEngine
105
106 private volatile boolean testDone = false;
107
108 /*
109 * For data transport, this example uses local ByteBuffers. This
110 * isn't really useful, but the purpose of this example is to show
111 * SSLEngine concepts, not how to do network transport.
112 */
113 private ByteBuffer cTOs; // "reliable" transport client->server
114 private ByteBuffer sTOc; // "reliable" transport server->client
115
116 /*
117 * The following is to set up the keystores.
118 */
119 private static String pathToStores = "../../../../../../../etc";
120 private static String keyStoreFile = "keystore";
121 private static String trustStoreFile = "truststore";
122 private static String passwd = "passphrase";
123
124 private static String keyFilename =
125 System.getProperty("test.src", ".") + "/" + pathToStores +
126 "/" + keyStoreFile;
127 private static String trustFilename =
128 System.getProperty("test.src", ".") + "/" + pathToStores +
129 "/" + trustStoreFile;
130
131 /*
132 * Main entry point for this test.
133 */
134 public static void main(String args[]) throws Exception {
135 if (debug) {
136 System.setProperty("javax.net.debug", "all");
137 }
138
139 // Turn off logging, and only output the test iteration to keep
140 // the noise down.
141 for (int i = 1; i <= 200; i++) {
142 if ((i % 5) == 0) {
143 System.out.println("Test #: " + i);
144 }
145 SSLEngineDeadlock test = new SSLEngineDeadlock();
146 test.runTest();
147
148 detectDeadLock();
149 }
150 System.out.println("Test Passed.");
151 }
152
153 /*
154 * Create an initialized SSLContext to use for these tests.
155 */
156 public SSLEngineDeadlock() throws Exception {
157
158 KeyStore ks = KeyStore.getInstance("JKS");
159 KeyStore ts = KeyStore.getInstance("JKS");
160
161 char[] passphrase = "passphrase".toCharArray();
162
163 ks.load(new FileInputStream(keyFilename), passphrase);
164 ts.load(new FileInputStream(trustFilename), passphrase);
165
166 KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
167 kmf.init(ks, passphrase);
168
169 TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
170 tmf.init(ts);
171
172 SSLContext sslCtx = SSLContext.getInstance("TLS");
173
174 sslCtx.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
175
176 sslc = sslCtx;
177 }
178
179 /*
180 * Create a thread which simply spins on tasks. This will hopefully
181 * trigger a deadlock between the wrap/unwrap and the tasks. On our
182 * slow, single-CPU build machine (sol8), it was very repeatable.
183 */
184 private void doTask() {
185 Runnable task;
186
187 while (!testDone) {
188 if ((task = clientEngine.getDelegatedTask()) != null) {
189 task.run();
190 }
191 if ((task = serverEngine.getDelegatedTask()) != null) {
192 task.run();
193 }
194 }
195 }
196
197 /*
198 * Run the test.
199 *
200 * Sit in a tight loop, both engines calling wrap/unwrap regardless
201 * of whether data is available or not. We do this until both engines
202 * report back they are closed.
203 *
204 * The main loop handles all of the I/O phases of the SSLEngine's
205 * lifetime:
206 *
207 * initial handshaking
208 * application data transfer
209 * engine closing
210 *
211 * One could easily separate these phases into separate
212 * sections of code.
213 */
214 private void runTest() throws Exception {
215 boolean dataDone = false;
216
217 createSSLEngines();
218 createBuffers();
219
220 SSLEngineResult clientResult; // results from client's last operation
221 SSLEngineResult serverResult; // results from server's last operation
222
223 new Thread("SSLEngine Task Dispatcher") {
224 public void run() {
225 try {
226 doTask();
227 } catch (Exception e) {
228 System.err.println("Task thread died...test will hang");
229 }
230 }
231 }.start();
232
233 /*
234 * Examining the SSLEngineResults could be much more involved,
235 * and may alter the overall flow of the application.
236 *
237 * For example, if we received a BUFFER_OVERFLOW when trying
238 * to write to the output pipe, we could reallocate a larger
239 * pipe, but instead we wait for the peer to drain it.
240 */
241 while (!isEngineClosed(clientEngine) ||
242 !isEngineClosed(serverEngine)) {
243
244 log("================");
245
246 clientResult = clientEngine.wrap(clientOut, cTOs);
247 log("client wrap: ", clientResult);
248
249 serverResult = serverEngine.wrap(serverOut, sTOc);
250 log("server wrap: ", serverResult);
251
252 cTOs.flip();
253 sTOc.flip();
254
255 log("----");
256
257 clientResult = clientEngine.unwrap(sTOc, clientIn);
258 log("client unwrap: ", clientResult);
259
260 serverResult = serverEngine.unwrap(cTOs, serverIn);
261 log("server unwrap: ", serverResult);
262
263 cTOs.compact();
264 sTOc.compact();
265
266 /*
267 * After we've transfered all application data between the client
268 * and server, we close the clientEngine's outbound stream.
269 * This generates a close_notify handshake message, which the
270 * server engine receives and responds by closing itself.
271 */
272 if (!dataDone && (clientOut.limit() == serverIn.position()) &&
273 (serverOut.limit() == clientIn.position())) {
274
275 /*
276 * A sanity check to ensure we got what was sent.
277 */
278 checkTransfer(serverOut, clientIn);
279 checkTransfer(clientOut, serverIn);
280
281 log("\tClosing clientEngine's *OUTBOUND*...");
282 clientEngine.closeOutbound();
283 dataDone = true;
284 }
285 }
286 testDone = true;
287 }
288
289 /*
290 * Using the SSLContext created during object creation,
291 * create/configure the SSLEngines we'll use for this test.
292 */
293 private void createSSLEngines() throws Exception {
294 /*
295 * Configure the serverEngine to act as a server in the SSL/TLS
296 * handshake. Also, require SSL client authentication.
297 */
298 serverEngine = sslc.createSSLEngine();
299 serverEngine.setUseClientMode(false);
300 serverEngine.setNeedClientAuth(true);
301
302 /*
303 * Similar to above, but using client mode instead.
304 */
305 clientEngine = sslc.createSSLEngine("client", 80);
306 clientEngine.setUseClientMode(true);
307 }
308
309 /*
310 * Create and size the buffers appropriately.
311 */
312 private void createBuffers() {
313
314 /*
315 * We'll assume the buffer sizes are the same
316 * between client and server.
317 */
318 SSLSession session = clientEngine.getSession();
319 int appBufferMax = session.getApplicationBufferSize();
320 int netBufferMax = session.getPacketBufferSize();
321
322 /*
323 * We'll make the input buffers a bit bigger than the max needed
324 * size, so that unwrap()s following a successful data transfer
325 * won't generate BUFFER_OVERFLOWS.
326 *
327 * We'll use a mix of direct and indirect ByteBuffers for
328 * tutorial purposes only. In reality, only use direct
329 * ByteBuffers when they give a clear performance enhancement.
330 */
331 clientIn = ByteBuffer.allocate(appBufferMax + 50);
332 serverIn = ByteBuffer.allocate(appBufferMax + 50);
333
334 cTOs = ByteBuffer.allocateDirect(netBufferMax);
335 sTOc = ByteBuffer.allocateDirect(netBufferMax);
336
337 clientOut = ByteBuffer.wrap("Hi Server, I'm Client".getBytes());
338 serverOut = ByteBuffer.wrap("Hello Client, I'm Server".getBytes());
339 }
340
341 private static boolean isEngineClosed(SSLEngine engine) {
342 return (engine.isOutboundDone() && engine.isInboundDone());
343 }
344
345 /*
346 * Simple check to make sure everything came across as expected.
347 */
348 private static void checkTransfer(ByteBuffer a, ByteBuffer b)
349 throws Exception {
350 a.flip();
351 b.flip();
352
353 if (!a.equals(b)) {
354 throw new Exception("Data didn't transfer cleanly");
355 } else {
356 log("\tData transferred cleanly");
357 }
358
359 a.position(a.limit());
360 b.position(b.limit());
361 a.limit(a.capacity());
362 b.limit(b.capacity());
363 }
364
365 /*
366 * Detect dead lock
367 */
368 private static void detectDeadLock() throws Exception {
369 ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
370 long[] threadIds = threadBean.findDeadlockedThreads();
371 if (threadIds != null && threadIds.length != 0) {
372 for (long id : threadIds) {
373 ThreadInfo info =
374 threadBean.getThreadInfo(id, Integer.MAX_VALUE);
375 System.out.println("Deadlocked ThreadInfo: " + info);
376 }
377 throw new Exception("Found Deadlock!");
378 }
379 }
380
381 /*
382 * Logging code
383 */
384 private static boolean resultOnce = true;
385
386 private static void log(String str, SSLEngineResult result) {
387 if (!logging) {
388 return;
389 }
390 if (resultOnce) {
391 resultOnce = false;
392 System.out.println("The format of the SSLEngineResult is: \n" +
393 "\t\"getStatus() / getHandshakeStatus()\" +\n" +
394 "\t\"bytesConsumed() / bytesProduced()\"\n");
395 }
396 HandshakeStatus hsStatus = result.getHandshakeStatus();
397 log(str +
398 result.getStatus() + "/" + hsStatus + ", " +
399 result.bytesConsumed() + "/" + result.bytesProduced() +
400 " bytes");
401 if (hsStatus == HandshakeStatus.FINISHED) {
402 log("\t...ready for application data");
403 }
404 }
405
406 private static void log(String str) {
407 if (logging) {
408 System.out.println(str);
409 }
410 }
411 }