1 /*
2 * Copyright (c) 2007, 2011, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 /*
25 * @test
26 * @bug 6492872
27 * @summary Deadlock in SSLEngine
28 * @run main/othervm SSLEngineDeadlock
29 *
30 * SunJSSE does not support dynamic system properties, no way to re-use
31 * system properties in samevm/agentvm mode.
32 *
33 * @author Brad R. Wetmore
34 */
35
36 /**
37 * A SSLEngine usage example which simplifies the presentation
38 * by removing the I/O and multi-threading concerns.
39 *
40 * The test creates two SSLEngines, simulating a client and server.
41 * The "transport" layer consists two byte buffers: think of them
42 * as directly connected pipes.
43 *
44 * Note, this is a *very* simple example: real code will be much more
45 * involved. For example, different threading and I/O models could be
46 * used, transport mechanisms could close unexpectedly, and so on.
47 *
48 * When this application runs, notice that several messages
49 * (wrap/unwrap) pass before any application data is consumed or
50 * produced. (For more information, please see the SSL/TLS
51 * specifications.) There may several steps for a successful handshake,
52 * so it's typical to see the following series of operations:
53 *
54 * client server message
55 * ====== ====== =======
56 * wrap() ... ClientHello
57 * ... unwrap() ClientHello
58 * ... wrap() ServerHello/Certificate
59 * unwrap() ... ServerHello/Certificate
60 * wrap() ... ClientKeyExchange
61 * wrap() ... ChangeCipherSpec
62 * wrap() ... Finished
63 * ... unwrap() ClientKeyExchange
64 * ... unwrap() ChangeCipherSpec
65 * ... unwrap() Finished
66 * ... wrap() ChangeCipherSpec
67 * ... wrap() Finished
68 * unwrap() ... ChangeCipherSpec
69 * unwrap() ... Finished
70 */
71
72 import javax.net.ssl.*;
73 import javax.net.ssl.SSLEngineResult.*;
74 import java.io.*;
75 import java.security.*;
76 import java.nio.*;
77
78 public class SSLEngineDeadlock {
79
80 /*
81 * Enables logging of the SSLEngine operations.
82 */
83 private static boolean logging = false;
84
85 /*
86 * Enables the JSSE system debugging system property:
87 *
88 * -Djavax.net.debug=all
89 *
90 * This gives a lot of low-level information about operations underway,
91 * including specific handshake messages, and might be best examined
92 * after gaining some familiarity with this application.
93 */
94 private static boolean debug = false;
95
96 private SSLContext sslc;
97
98 private SSLEngine clientEngine; // client Engine
99 private ByteBuffer clientOut; // write side of clientEngine
100 private ByteBuffer clientIn; // read side of clientEngine
101
102 private SSLEngine serverEngine; // server Engine
103 private ByteBuffer serverOut; // write side of serverEngine
104 private ByteBuffer serverIn; // read side of serverEngine
105
106 private volatile boolean testDone = false;
107
108 /*
109 * For data transport, this example uses local ByteBuffers. This
110 * isn't really useful, but the purpose of this example is to show
111 * SSLEngine concepts, not how to do network transport.
112 */
113 private ByteBuffer cTOs; // "reliable" transport client->server
114 private ByteBuffer sTOc; // "reliable" transport server->client
115
116 /*
117 * The following is to set up the keystores.
118 */
119 private static String pathToStores = "../../../../../../../etc";
120 private static String keyStoreFile = "keystore";
121 private static String trustStoreFile = "truststore";
122 private static String passwd = "passphrase";
123
124 private static String keyFilename =
125 System.getProperty("test.src", ".") + "/" + pathToStores +
126 "/" + keyStoreFile;
127 private static String trustFilename =
128 System.getProperty("test.src", ".") + "/" + pathToStores +
129 "/" + trustStoreFile;
130
131 /*
132 * Main entry point for this test.
133 */
134 public static void main(String args[]) throws Exception {
135 if (debug) {
136 System.setProperty("javax.net.debug", "all");
137 }
138
139 // Turn off logging, and only output the test iteration to keep
140 // the noise down.
141 for (int i = 1; i <= 200; i++) {
142 if ((i % 5) == 0) {
143 System.out.println("Test #: " + i);
144 }
145 SSLEngineDeadlock test = new SSLEngineDeadlock();
146 test.runTest();
147 }
148 System.out.println("Test Passed.");
149 }
150
151 /*
152 * Create an initialized SSLContext to use for these tests.
153 */
154 public SSLEngineDeadlock() throws Exception {
155
156 KeyStore ks = KeyStore.getInstance("JKS");
157 KeyStore ts = KeyStore.getInstance("JKS");
158
159 char[] passphrase = "passphrase".toCharArray();
160
161 ks.load(new FileInputStream(keyFilename), passphrase);
162 ts.load(new FileInputStream(trustFilename), passphrase);
163
164 KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
165 kmf.init(ks, passphrase);
166
167 TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
168 tmf.init(ts);
169
170 SSLContext sslCtx = SSLContext.getInstance("TLS");
171
172 sslCtx.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
173
174 sslc = sslCtx;
175 }
176
177 /*
178 * Create a thread which simply spins on tasks. This will hopefully
179 * trigger a deadlock between the wrap/unwrap and the tasks. On our
180 * slow, single-CPU build machine (sol8), it was very repeatable.
181 */
182 private void doTask() {
183 Runnable task;
184
185 while (!testDone) {
186 if ((task = clientEngine.getDelegatedTask()) != null) {
187 task.run();
188 }
189 if ((task = serverEngine.getDelegatedTask()) != null) {
190 task.run();
191 }
192 }
193 }
194
195 /*
196 * Run the test.
197 *
198 * Sit in a tight loop, both engines calling wrap/unwrap regardless
199 * of whether data is available or not. We do this until both engines
200 * report back they are closed.
201 *
202 * The main loop handles all of the I/O phases of the SSLEngine's
203 * lifetime:
204 *
205 * initial handshaking
206 * application data transfer
207 * engine closing
208 *
209 * One could easily separate these phases into separate
210 * sections of code.
211 */
212 private void runTest() throws Exception {
213 boolean dataDone = false;
214
215 createSSLEngines();
216 createBuffers();
217
218 SSLEngineResult clientResult; // results from client's last operation
219 SSLEngineResult serverResult; // results from server's last operation
220
221 new Thread("SSLEngine Task Dispatcher") {
222 public void run() {
223 try {
224 doTask();
225 } catch (Exception e) {
226 System.err.println("Task thread died...test will hang");
227 }
228 }
229 }.start();
230
231 /*
232 * Examining the SSLEngineResults could be much more involved,
233 * and may alter the overall flow of the application.
234 *
235 * For example, if we received a BUFFER_OVERFLOW when trying
236 * to write to the output pipe, we could reallocate a larger
237 * pipe, but instead we wait for the peer to drain it.
238 */
239 while (!isEngineClosed(clientEngine) ||
240 !isEngineClosed(serverEngine)) {
241
242 log("================");
243
244 clientResult = clientEngine.wrap(clientOut, cTOs);
245 log("client wrap: ", clientResult);
246
247 serverResult = serverEngine.wrap(serverOut, sTOc);
248 log("server wrap: ", serverResult);
249
250 cTOs.flip();
251 sTOc.flip();
252
253 log("----");
254
255 clientResult = clientEngine.unwrap(sTOc, clientIn);
256 log("client unwrap: ", clientResult);
257
258 serverResult = serverEngine.unwrap(cTOs, serverIn);
259 log("server unwrap: ", serverResult);
260
261 cTOs.compact();
262 sTOc.compact();
263
264 /*
265 * After we've transfered all application data between the client
266 * and server, we close the clientEngine's outbound stream.
267 * This generates a close_notify handshake message, which the
268 * server engine receives and responds by closing itself.
269 */
270 if (!dataDone && (clientOut.limit() == serverIn.position()) &&
271 (serverOut.limit() == clientIn.position())) {
272
273 /*
274 * A sanity check to ensure we got what was sent.
275 */
276 checkTransfer(serverOut, clientIn);
277 checkTransfer(clientOut, serverIn);
278
279 log("\tClosing clientEngine's *OUTBOUND*...");
280 clientEngine.closeOutbound();
281 dataDone = true;
282 }
283 }
284 testDone = true;
285 }
286
287 /*
288 * Using the SSLContext created during object creation,
289 * create/configure the SSLEngines we'll use for this test.
290 */
291 private void createSSLEngines() throws Exception {
292 /*
293 * Configure the serverEngine to act as a server in the SSL/TLS
294 * handshake. Also, require SSL client authentication.
295 */
296 serverEngine = sslc.createSSLEngine();
297 serverEngine.setUseClientMode(false);
298 serverEngine.setNeedClientAuth(true);
299
300 /*
301 * Similar to above, but using client mode instead.
302 */
303 clientEngine = sslc.createSSLEngine("client", 80);
304 clientEngine.setUseClientMode(true);
305 }
306
307 /*
308 * Create and size the buffers appropriately.
309 */
310 private void createBuffers() {
311
312 /*
313 * We'll assume the buffer sizes are the same
314 * between client and server.
315 */
316 SSLSession session = clientEngine.getSession();
317 int appBufferMax = session.getApplicationBufferSize();
318 int netBufferMax = session.getPacketBufferSize();
319
320 /*
321 * We'll make the input buffers a bit bigger than the max needed
322 * size, so that unwrap()s following a successful data transfer
323 * won't generate BUFFER_OVERFLOWS.
324 *
325 * We'll use a mix of direct and indirect ByteBuffers for
326 * tutorial purposes only. In reality, only use direct
327 * ByteBuffers when they give a clear performance enhancement.
328 */
329 clientIn = ByteBuffer.allocate(appBufferMax + 50);
330 serverIn = ByteBuffer.allocate(appBufferMax + 50);
331
332 cTOs = ByteBuffer.allocateDirect(netBufferMax);
333 sTOc = ByteBuffer.allocateDirect(netBufferMax);
334
335 clientOut = ByteBuffer.wrap("Hi Server, I'm Client".getBytes());
336 serverOut = ByteBuffer.wrap("Hello Client, I'm Server".getBytes());
337 }
338
339 private static boolean isEngineClosed(SSLEngine engine) {
340 return (engine.isOutboundDone() && engine.isInboundDone());
341 }
342
343 /*
344 * Simple check to make sure everything came across as expected.
345 */
346 private static void checkTransfer(ByteBuffer a, ByteBuffer b)
347 throws Exception {
348 a.flip();
349 b.flip();
350
351 if (!a.equals(b)) {
352 throw new Exception("Data didn't transfer cleanly");
353 } else {
354 log("\tData transferred cleanly");
355 }
356
357 a.position(a.limit());
358 b.position(b.limit());
359 a.limit(a.capacity());
360 b.limit(b.capacity());
361 }
362
363 /*
364 * Logging code
365 */
366 private static boolean resultOnce = true;
367
368 private static void log(String str, SSLEngineResult result) {
369 if (!logging) {
370 return;
371 }
372 if (resultOnce) {
373 resultOnce = false;
374 System.out.println("The format of the SSLEngineResult is: \n" +
375 "\t\"getStatus() / getHandshakeStatus()\" +\n" +
376 "\t\"bytesConsumed() / bytesProduced()\"\n");
377 }
378 HandshakeStatus hsStatus = result.getHandshakeStatus();
379 log(str +
380 result.getStatus() + "/" + hsStatus + ", " +
381 result.bytesConsumed() + "/" + result.bytesProduced() +
382 " bytes");
383 if (hsStatus == HandshakeStatus.FINISHED) {
384 log("\t...ready for application data");
385 }
386 }
387
388 private static void log(String str) {
389 if (logging) {
390 System.out.println(str);
391 }
392 }
393 }