1 /*
   2  * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package com.sun.webkit.network;
  27 
  28 import com.sun.webkit.Invoker;
  29 import com.sun.webkit.LoadListenerClient;
  30 import com.sun.webkit.WebPage;
  31 import static com.sun.webkit.network.URLs.newURL;
  32 import java.io.EOFException;
  33 import java.io.File;
  34 import java.io.FileNotFoundException;
  35 import java.io.IOException;
  36 import java.io.InputStream;
  37 import java.io.OutputStream;
  38 import java.io.UnsupportedEncodingException;
  39 import java.net.ConnectException;
  40 import java.net.HttpRetryException;
  41 import java.net.HttpURLConnection;
  42 import java.net.MalformedURLException;
  43 import java.net.NoRouteToHostException;
  44 import java.net.SocketException;
  45 import java.net.SocketTimeoutException;
  46 import java.net.URL;
  47 import java.net.URLConnection;
  48 import java.net.URLDecoder;
  49 import java.net.UnknownHostException;
  50 import java.nio.ByteBuffer;
  51 import java.security.AccessControlException;
  52 import java.security.AccessController;
  53 import java.security.PrivilegedAction;
  54 import java.util.List;
  55 import java.util.Locale;
  56 import java.util.Map;
  57 import java.util.concurrent.CountDownLatch;
  58 import java.util.logging.Level;
  59 import java.util.logging.Logger;
  60 import java.util.zip.GZIPInputStream;
  61 import java.util.zip.InflaterInputStream;
  62 import javax.net.ssl.SSLHandshakeException;
  63 
  64 /**
  65  * A runnable that loads a resource specified by a URL.
  66  */
  67 final class URLLoader implements Runnable {
  68 
  69     private static final Logger logger =
  70             Logger.getLogger(URLLoader.class.getName());
  71     private static final int MAX_REDIRECTS = 10;
  72     private static final int MAX_BUF_COUNT = 3;
  73     private static final String GET = "GET";
  74     private static final String HEAD = "HEAD";
  75     private static final String DELETE = "DELETE";
  76 
  77 
  78     private final WebPage webPage;
  79     private final ByteBufferPool byteBufferPool;
  80     private final boolean asynchronous;
  81     private String url;
  82     private String method;
  83     private final String headers;
  84     private FormDataElement[] formDataElements;
  85     private final long data;
  86     private volatile boolean canceled = false;
  87 
  88 
  89     /**
  90      * Creates a new {@code URLLoader}.
  91      */
  92     URLLoader(WebPage webPage,
  93               ByteBufferPool byteBufferPool,
  94               boolean asynchronous,
  95               String url,
  96               String method,
  97               String headers,
  98               FormDataElement[] formDataElements,
  99               long data)
 100     {
 101         this.webPage = webPage;
 102         this.byteBufferPool = byteBufferPool;
 103         this.asynchronous = asynchronous;
 104         this.url = url;
 105         this.method = method;
 106         this.headers = headers;
 107         this.formDataElements = formDataElements;
 108         this.data = data;
 109     }
 110 
 111 
 112     /**
 113      * Cancels this loader.
 114      */
 115     private void fwkCancel() {
 116         if (logger.isLoggable(Level.FINEST)) {
 117             logger.log(Level.FINEST, String.format("data: [0x%016X]", data));
 118         }
 119         canceled = true;
 120     }
 121 
 122     /**
 123      * {@inheritDoc}
 124      */
 125     @Override
 126     public void run() {
 127         // Run the loader in the page's access control context
 128         AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
 129             doRun();
 130             return null;
 131         }, webPage.getAccessControlContext());
 132     }
 133 
 134     /**
 135      * Executes this loader.
 136      */
 137     private void doRun() {
 138         Throwable error = null;
 139         int errorCode = 0;
 140         try {
 141             int redirectCount = 0;
 142             boolean streaming = true;
 143             while (true) {
 144                 // RT-14438
 145                 String actualUrl = url;
 146                 if (url.startsWith("file:")) {
 147                     int questionMarkPosition = url.indexOf('?');
 148                     if (questionMarkPosition != -1) {
 149                         actualUrl = url.substring(0, questionMarkPosition);
 150                     }
 151                 }
 152 
 153                 URL urlObject = newURL(actualUrl);
 154 
 155                 // RT-22458
 156                 workaround7177996(urlObject);
 157 
 158                 URLConnection c = urlObject.openConnection();
 159                 prepareConnection(c);
 160 
 161                 Redirect redirect = null;
 162                 try {
 163                     sendRequest(c, streaming);
 164                     redirect = receiveResponse(c);
 165                 } catch (HttpRetryException ex) {
 166                     // RT-19914
 167                     if (streaming) {
 168                         streaming = false;
 169                         continue; // retry without streaming
 170                     } else {
 171                         throw ex;
 172                     }
 173                 } finally {
 174                     close(c);
 175                 }
 176 
 177                 if (redirect != null) {
 178                     if (redirectCount++ >= MAX_REDIRECTS) {
 179                         throw new TooManyRedirectsException();
 180                     }
 181                     boolean resetRequest = !redirect.preserveRequest
 182                             && !method.equals(GET) && !method.equals(HEAD);
 183                     String newMethod = resetRequest ? GET : method;
 184                     willSendRequest(redirect.url, newMethod, c);
 185                     // willSendRequest() may cancel this loader
 186                     if (canceled) {
 187                         break;
 188                     }
 189                     url = redirect.url;
 190                     method = newMethod;
 191                     formDataElements = resetRequest ? null : formDataElements;
 192                 } else {
 193                     break;
 194                 }
 195             }
 196         } catch (MalformedURLException ex) {
 197             error = ex;
 198             errorCode = LoadListenerClient.MALFORMED_URL;
 199         } catch (AccessControlException ex) {
 200             error = ex;
 201             errorCode = LoadListenerClient.PERMISSION_DENIED;
 202         } catch (UnknownHostException ex) {
 203             error = ex;
 204             errorCode = LoadListenerClient.UNKNOWN_HOST;
 205         } catch (NoRouteToHostException ex) {
 206             error = ex;
 207             errorCode = LoadListenerClient.NO_ROUTE_TO_HOST;
 208         } catch (ConnectException ex) {
 209             error = ex;
 210             errorCode = LoadListenerClient.CONNECTION_REFUSED;
 211         } catch (SocketException ex) {
 212             error = ex;
 213             errorCode = LoadListenerClient.CONNECTION_RESET;
 214         } catch (SSLHandshakeException ex) {
 215             error = ex;
 216             errorCode = LoadListenerClient.SSL_HANDSHAKE;
 217         } catch (SocketTimeoutException ex) {
 218             error = ex;
 219             errorCode = LoadListenerClient.CONNECTION_TIMED_OUT;
 220         } catch (InvalidResponseException ex) {
 221             error = ex;
 222             errorCode = LoadListenerClient.INVALID_RESPONSE;
 223         } catch (TooManyRedirectsException ex) {
 224             error = ex;
 225             errorCode = LoadListenerClient.TOO_MANY_REDIRECTS;
 226         } catch (FileNotFoundException ex) {
 227             error = ex;
 228             errorCode = LoadListenerClient.FILE_NOT_FOUND;
 229         } catch (Throwable th) {
 230             error = th;
 231             errorCode = LoadListenerClient.UNKNOWN_ERROR;
 232         }
 233 
 234         if (error != null) {
 235             if (errorCode == LoadListenerClient.UNKNOWN_ERROR) {
 236                 logger.log(Level.WARNING, "Unexpected error", error);
 237             } else {
 238                 logger.log(Level.FINEST, "Load error", error);
 239             }
 240             didFail(errorCode, error.getMessage());
 241         }
 242     }
 243 
 244     private static void workaround7177996(URL url)
 245         throws FileNotFoundException
 246     {
 247         if (!url.getProtocol().equals("file")) {
 248             return;
 249         }
 250 
 251         String host = url.getHost();
 252         if (host == null || host.equals("") || host.equals("~")
 253                 || host.equalsIgnoreCase("localhost") )
 254         {
 255            return;
 256         }
 257 
 258         if (System.getProperty("os.name").startsWith("Windows")) {
 259             String path = null;
 260             try {
 261                 path = URLDecoder.decode(url.getPath(), "UTF-8");
 262             } catch (UnsupportedEncodingException e) {
 263                 // The system should always have the platform default
 264             }
 265             path = path.replace('/', '\\');
 266             path = path.replace('|', ':');
 267             File file = new File("\\\\" + host + path);
 268             if (!file.exists()) {
 269                 throw new FileNotFoundException("File not found: " + url);
 270             }
 271         } else {
 272             throw new FileNotFoundException("File not found: " + url);
 273         }
 274     }
 275 
 276     /**
 277      * Prepares a connection.
 278      */
 279     private void prepareConnection(URLConnection c) throws IOException {
 280         // The following two timeouts are quite arbitrary and should
 281         // probably be configurable via an API
 282         c.setConnectTimeout(30000);   // 30 seconds
 283         c.setReadTimeout(60000 * 60); // 60 minutes
 284 
 285         // Given that WebKit has its own cache, do not use
 286         // any URLConnection caches, even if someone installs them.
 287         // As a side effect, this fixes the problem of WebPane not
 288         // working well with the plug-in cache, which was one of
 289         // the causes for RT-11880.
 290         c.setUseCaches(false);
 291 
 292         Locale loc = Locale.getDefault();
 293         String lang = "";
 294         if (!loc.equals(Locale.US) && !loc.equals(Locale.ENGLISH)) {
 295             lang = loc.getCountry().isEmpty() ?
 296                 loc.getLanguage() + ",":
 297                 loc.getLanguage() + "-" + loc.getCountry() + ",";
 298         }
 299         c.setRequestProperty("Accept-Language", lang.toLowerCase() + "en-us;q=0.8,en;q=0.7");
 300         c.setRequestProperty("Accept-Encoding", "gzip");
 301         c.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
 302 
 303         if (headers != null && headers.length() > 0) {
 304             for (String h : headers.split("\n")) {
 305                 int i = h.indexOf(':');
 306                 if (i > 0) {
 307                     c.addRequestProperty(h.substring(0, i), h.substring(i + 2));
 308                 }
 309             }
 310         }
 311 
 312         if (c instanceof HttpURLConnection) {
 313             HttpURLConnection httpConnection = (HttpURLConnection) c;
 314             httpConnection.setRequestMethod(method);
 315             // There are too many bugs in the way HttpURLConnection handles
 316             // redirects, so we will deal with them ourselves
 317             httpConnection.setInstanceFollowRedirects(false);
 318         }
 319     }
 320 
 321     /**
 322      * Sends request to the server.
 323      */
 324     private void sendRequest(URLConnection c, boolean streaming)
 325         throws IOException
 326     {
 327         OutputStream out = null;
 328         try {
 329             long bytesToBeSent = 0;
 330             boolean sendFormData = formDataElements != null
 331                     && c instanceof HttpURLConnection
 332                     && !method.equals(DELETE);
 333             boolean isGetOrHead = method.equals(GET) || method.equals(HEAD);
 334             if (sendFormData) {
 335                 c.setDoOutput(true);
 336 
 337                 for (FormDataElement formDataElement : formDataElements) {
 338                     formDataElement.open();
 339                     bytesToBeSent += formDataElement.getSize();
 340                 }
 341 
 342                 if (streaming) {
 343                     HttpURLConnection http = (HttpURLConnection) c;
 344                     if (bytesToBeSent <= Integer.MAX_VALUE) {
 345                         http.setFixedLengthStreamingMode((int) bytesToBeSent);
 346                     } else {
 347                         http.setChunkedStreamingMode(0);
 348                     }
 349                 }
 350             } else if (!isGetOrHead && (c instanceof HttpURLConnection)) {
 351                 c.setRequestProperty("Content-Length", "0");
 352             }
 353 
 354             int maxTryCount = isGetOrHead ? 3 : 1;
 355             c.setConnectTimeout(c.getConnectTimeout() / maxTryCount);
 356             int tryCount = 0;
 357             while (!canceled) {
 358                 try {
 359                     c.connect();
 360                     break;
 361                 } catch (SocketTimeoutException ex) {
 362                     if (++tryCount >= maxTryCount) {
 363                         throw ex;
 364                     }
 365                 } catch (IllegalArgumentException ex) {
 366                     // Happens with some malformed URLs
 367                     throw new MalformedURLException(url);
 368                 }
 369             }
 370 
 371             if (sendFormData) {
 372                 out = c.getOutputStream();
 373                 byte[] buffer = new byte[4096];
 374                 long bytesSent = 0;
 375                 for (FormDataElement formDataElement : formDataElements) {
 376                     InputStream in = formDataElement.getInputStream();
 377                     int count;
 378                     while ((count = in.read(buffer)) > 0) {
 379                         out.write(buffer, 0, count);
 380                         bytesSent += count;
 381                         didSendData(bytesSent, bytesToBeSent);
 382                     }
 383                     formDataElement.close();
 384                 }
 385                 out.flush();
 386                 out.close();
 387                 out = null;
 388             }
 389         } finally {
 390             if (out != null) {
 391                 try {
 392                     out.close();
 393                 } catch (IOException ignore) {}
 394             }
 395             if (formDataElements != null && c instanceof HttpURLConnection) {
 396                 for (FormDataElement formDataElement : formDataElements) {
 397                     try {
 398                         formDataElement.close();
 399                     } catch (IOException ignore) {}
 400                 }
 401             }
 402         }
 403     }
 404 
 405     /**
 406      * Receives response from the server.
 407      */
 408     private Redirect receiveResponse(URLConnection c)
 409         throws IOException, InterruptedException
 410     {
 411         if (canceled) {
 412             return null;
 413         }
 414 
 415         InputStream errorStream = null;
 416 
 417         if (c instanceof HttpURLConnection) {
 418             HttpURLConnection http = (HttpURLConnection) c;
 419 
 420             int code = http.getResponseCode();
 421             if (code == -1) {
 422                 throw new InvalidResponseException();
 423             }
 424 
 425             if (canceled) {
 426                 return null;
 427             }
 428 
 429             // See RT-17435
 430             switch (code) {
 431                 case 301: // Moved Permanently
 432                 case 302: // Found
 433                 case 303: // See Other
 434                 case 307: // Temporary Redirect
 435                     String newLoc = http.getHeaderField("Location");
 436                     if (newLoc != null) {
 437                         URL newUrl;
 438                         try {
 439                             newUrl = newURL(newLoc);
 440                         } catch (MalformedURLException mue) {
 441                             // Try to treat newLoc as a relative URI to conform
 442                             // to popular browsers
 443                             newUrl = newURL(c.getURL(), newLoc);
 444                         }
 445                         return new Redirect(newUrl.toExternalForm(),
 446                                             code == 307);
 447                     }
 448                     break;
 449 
 450                 case 304: // Not Modified
 451                     didReceiveResponse(c);
 452                     didFinishLoading();
 453                     return null;
 454             }
 455 
 456             if (code >= 400 && !method.equals(HEAD)) {
 457                 errorStream = http.getErrorStream();
 458             }
 459         }
 460 
 461         // Let's see if it's an ftp (or ftps) URL and we need to transform
 462         // a directory listing into HTML
 463         if (url.startsWith("ftp:") || url.startsWith("ftps:")) {
 464             boolean dir = false;
 465             boolean notsure = false;
 466             // Unfortunately, there is no clear way to determine if we are
 467             // accessing a directory, so a bit of guessing is in order
 468             String path = c.getURL().getPath();
 469             if (path == null || path.isEmpty() || path.endsWith("/")
 470                     || path.contains(";type=d"))
 471             {
 472                 dir = true;
 473             } else {
 474                 String type = c.getContentType();
 475                 if ("text/plain".equalsIgnoreCase(type)
 476                         || "text/html".equalsIgnoreCase(type))
 477                 {
 478                     dir = true;
 479                     notsure = true;
 480                 }
 481             }
 482             if (dir) {
 483                 c = new DirectoryURLConnection(c, notsure);
 484             }
 485         }
 486 
 487         // Same is true for FileURLConnection
 488         if (url.startsWith("file:")) {
 489             if("text/plain".equals(c.getContentType())
 490                     && c.getHeaderField("content-length") == null)
 491             {
 492                 // It is a directory
 493                 c = new DirectoryURLConnection(c);
 494             }
 495         }
 496 
 497         didReceiveResponse(c);
 498 
 499         if (method.equals(HEAD)) {
 500             didFinishLoading();
 501             return null;
 502         }
 503 
 504         InputStream inputStream = null;
 505         try {
 506             inputStream = errorStream == null
 507                 ? c.getInputStream() : errorStream;
 508         } catch (HttpRetryException ex) {
 509             // HttpRetryException is handled from doRun() method.
 510             // Hence rethrowing the exception to caller(doRun() method)
 511             throw ex;
 512         } catch (IOException e) {
 513             if (logger.isLoggable(Level.FINE)) {
 514                 logger.log(Level.FINE, String.format("Exception caught: [%s], %s",
 515                     e.getClass().getSimpleName(),
 516                     e.getMessage()));
 517             }
 518         }
 519 
 520         String encoding = c.getContentEncoding();
 521         if ("gzip".equalsIgnoreCase(encoding)) {
 522             inputStream = new GZIPInputStream(inputStream);
 523         } else if ("deflate".equalsIgnoreCase(encoding)) {
 524             inputStream = new InflaterInputStream(inputStream);
 525         }
 526 
 527         ByteBufferAllocator allocator =
 528                 byteBufferPool.newAllocator(MAX_BUF_COUNT);
 529         ByteBuffer byteBuffer = null;
 530         try {
 531             if (inputStream != null) {
 532                 // 8192 is the default size of a BufferedInputStream used in
 533                 // most URLConnections, by using the same size, we avoid quite
 534                 // a few System.arrayCopy() calls
 535                 byte[] buffer = new byte[8192];
 536                 while (!canceled) {
 537                     int count;
 538                     try {
 539                         count = inputStream.read(buffer);
 540                     } catch (EOFException ex) {
 541                         // can be thrown by GZIPInputStream signaling
 542                         // the end of the stream
 543                         count = -1;
 544                     }
 545 
 546                     if (count == -1) {
 547                         break;
 548                     }
 549 
 550                     if (byteBuffer == null) {
 551                         byteBuffer = allocator.allocate();
 552                     }
 553 
 554                     int remaining = byteBuffer.remaining();
 555                     if (count < remaining) {
 556                         byteBuffer.put(buffer, 0, count);
 557                     } else {
 558                         byteBuffer.put(buffer, 0, remaining);
 559 
 560                         byteBuffer.flip();
 561                         didReceiveData(byteBuffer, allocator);
 562                         byteBuffer = null;
 563 
 564                         int outstanding = count - remaining;
 565                         if (outstanding > 0) {
 566                             byteBuffer = allocator.allocate();
 567                             byteBuffer.put(buffer, remaining, outstanding);
 568                         }
 569                     }
 570                 }
 571             }
 572             if (!canceled) {
 573                 if (byteBuffer != null && byteBuffer.position() > 0) {
 574                     byteBuffer.flip();
 575                     didReceiveData(byteBuffer, allocator);
 576                     byteBuffer = null;
 577                 }
 578                 didFinishLoading();
 579             }
 580         } finally {
 581             if (byteBuffer != null) {
 582                 byteBuffer.clear();
 583                 allocator.release(byteBuffer);
 584             }
 585         }
 586         return null;
 587     }
 588 
 589     /**
 590      * Releases the resources that may be associated with a connection.
 591      */
 592     private static void close(URLConnection c) {
 593         if (c instanceof HttpURLConnection) {
 594             InputStream errorStream = ((HttpURLConnection) c).getErrorStream();
 595             if (errorStream != null) {
 596                 try {
 597                     errorStream.close();
 598                 } catch (IOException ignore) {}
 599             }
 600         }
 601         try {
 602             c.getInputStream().close();
 603         } catch (IOException ignore) {}
 604     }
 605 
 606 
 607     /**
 608      * A holder for redirect information.
 609      */
 610     private static final class Redirect {
 611         private final String url;
 612         private final boolean preserveRequest;
 613 
 614         private Redirect(String url, boolean preserveRequest) {
 615             this.url = url;
 616             this.preserveRequest = preserveRequest;
 617         }
 618     }
 619 
 620     /**
 621      * Signals an invalid response from the server.
 622      */
 623     private static final class InvalidResponseException extends IOException {
 624         private InvalidResponseException() {
 625             super("Invalid server response");
 626         }
 627     }
 628 
 629     /**
 630      * Signals that too many redirects have been encountered
 631      * while processing the request.
 632      */
 633     private static final class TooManyRedirectsException extends IOException {
 634         private TooManyRedirectsException() {
 635             super("Too many redirects");
 636         }
 637     }
 638 
 639     private void didSendData(final long totalBytesSent,
 640                              final long totalBytesToBeSent)
 641     {
 642         callBack(() -> {
 643             if (!canceled) {
 644                 notifyDidSendData(totalBytesSent, totalBytesToBeSent);
 645             }
 646         });
 647     }
 648 
 649     private void notifyDidSendData(long totalBytesSent,
 650                                    long totalBytesToBeSent)
 651     {
 652         if (logger.isLoggable(Level.FINEST)) {
 653             logger.log(Level.FINEST, String.format(
 654                     "totalBytesSent: [%d], "
 655                     + "totalBytesToBeSent: [%d], "
 656                     + "data: [0x%016X]",
 657                     totalBytesSent,
 658                     totalBytesToBeSent,
 659                     data));
 660         }
 661         twkDidSendData(totalBytesSent, totalBytesToBeSent, data);
 662     }
 663 
 664     private void willSendRequest(String newUrl,
 665                                  final String newMethod,
 666                                  URLConnection c) throws InterruptedException
 667     {
 668         final String adjustedNewUrl = adjustUrlForWebKit(newUrl);
 669         final int status = extractStatus(c);
 670         final String contentType = c.getContentType();
 671         final String contentEncoding = extractContentEncoding(c);
 672         final long contentLength = extractContentLength(c);
 673         final String responseHeaders = extractHeaders(c);
 674         final String adjustedUrl = adjustUrlForWebKit(url);
 675         final CountDownLatch latch =
 676                 asynchronous ? new CountDownLatch(1) : null;
 677         callBack(() -> {
 678             try {
 679                 if (!canceled) {
 680                     boolean keepGoing = notifyWillSendRequest(
 681                             adjustedNewUrl,
 682                             newMethod,
 683                             status,
 684                             contentType,
 685                             contentEncoding,
 686                             contentLength,
 687                             responseHeaders,
 688                             adjustedUrl);
 689                     if (!keepGoing) {
 690                         fwkCancel();
 691                     }
 692                 }
 693             } finally {
 694                 if (latch != null) {
 695                     latch.countDown();
 696                 }
 697             }
 698         });
 699         if (latch != null) {
 700             latch.await();
 701         }
 702     }
 703 
 704     private boolean notifyWillSendRequest(String newUrl,
 705                                           String newMethod,
 706                                           int status,
 707                                           String contentType,
 708                                           String contentEncoding,
 709                                           long contentLength,
 710                                           String headers,
 711                                           String url)
 712     {
 713         if (logger.isLoggable(Level.FINEST)) {
 714             logger.log(Level.FINEST, String.format(
 715                     "newUrl: [%s], "
 716                     + "newMethod: [%s], "
 717                     + "status: [%d], "
 718                     + "contentType: [%s], "
 719                     + "contentEncoding: [%s], "
 720                     + "contentLength: [%d], "
 721                     + "url: [%s], "
 722                     + "data: [0x%016X], "
 723                     + "headers:%n%s",
 724                     newUrl,
 725                     newMethod,
 726                     status,
 727                     contentType,
 728                     contentEncoding,
 729                     contentLength,
 730                     url,
 731                     data,
 732                     Util.formatHeaders(headers)));
 733         }
 734         boolean result = twkWillSendRequest(
 735                 newUrl,
 736                 newMethod,
 737                 status,
 738                 contentType,
 739                 contentEncoding,
 740                 contentLength,
 741                 headers,
 742                 url,
 743                 data);
 744         if (logger.isLoggable(Level.FINEST)) {
 745             logger.log(Level.FINEST, String.format("result: [%s]", result));
 746         }
 747         return result;
 748     }
 749 
 750     private void didReceiveResponse(URLConnection c) {
 751         final int status = extractStatus(c);
 752         final String contentType = c.getContentType();
 753         final String contentEncoding = extractContentEncoding(c);
 754         final long contentLength = extractContentLength(c);
 755         final String responseHeaders = extractHeaders(c);
 756         final String adjustedUrl = adjustUrlForWebKit(url);
 757         callBack(() -> {
 758             if (!canceled) {
 759                 notifyDidReceiveResponse(
 760                         status,
 761                         contentType,
 762                         contentEncoding,
 763                         contentLength,
 764                         responseHeaders,
 765                         adjustedUrl);
 766             }
 767         });
 768     }
 769 
 770     private void notifyDidReceiveResponse(int status,
 771                                           String contentType,
 772                                           String contentEncoding,
 773                                           long contentLength,
 774                                           String headers,
 775                                           String url)
 776     {
 777         if (logger.isLoggable(Level.FINEST)) {
 778             logger.log(Level.FINEST, String.format(
 779                     "status: [%d], "
 780                     + "contentType: [%s], "
 781                     + "contentEncoding: [%s], "
 782                     + "contentLength: [%d], "
 783                     + "url: [%s], "
 784                     + "data: [0x%016X], "
 785                     + "headers:%n%s",
 786                     status,
 787                     contentType,
 788                     contentEncoding,
 789                     contentLength,
 790                     url,
 791                     data,
 792                     Util.formatHeaders(headers)));
 793         }
 794         twkDidReceiveResponse(
 795                 status,
 796                 contentType,
 797                 contentEncoding,
 798                 contentLength,
 799                 headers,
 800                 url,
 801                 data);
 802     }
 803 
 804     private void didReceiveData(final ByteBuffer byteBuffer,
 805                                 final ByteBufferAllocator allocator)
 806     {
 807         callBack(() -> {
 808             if (!canceled) {
 809                 notifyDidReceiveData(
 810                         byteBuffer,
 811                         byteBuffer.position(),
 812                         byteBuffer.remaining());
 813             }
 814             byteBuffer.clear();
 815             allocator.release(byteBuffer);
 816         });
 817     }
 818 
 819     private void notifyDidReceiveData(ByteBuffer byteBuffer,
 820                                       int position,
 821                                       int remaining)
 822     {
 823         if (logger.isLoggable(Level.FINEST)) {
 824             logger.log(Level.FINEST, String.format(
 825                     "byteBuffer: [%s], "
 826                     + "position: [%s], "
 827                     + "remaining: [%s], "
 828                     + "data: [0x%016X]",
 829                     byteBuffer,
 830                     position,
 831                     remaining,
 832                     data));
 833         }
 834         twkDidReceiveData(byteBuffer, position, remaining, data);
 835     }
 836 
 837     private void didFinishLoading() {
 838         callBack(() -> {
 839             if (!canceled) {
 840                 notifyDidFinishLoading();
 841             }
 842         });
 843     }
 844 
 845     private void notifyDidFinishLoading() {
 846         if (logger.isLoggable(Level.FINEST)) {
 847             logger.log(Level.FINEST, String.format("data: [0x%016X]", data));
 848         }
 849         twkDidFinishLoading(data);
 850     }
 851 
 852     private void didFail(final int errorCode, final String message) {
 853         final String adjustedUrl = adjustUrlForWebKit(url);
 854         callBack(() -> {
 855             if (!canceled) {
 856                 notifyDidFail(errorCode, adjustedUrl, message);
 857             }
 858         });
 859     }
 860 
 861     private void notifyDidFail(int errorCode, String url, String message) {
 862         if (logger.isLoggable(Level.FINEST)) {
 863             logger.log(Level.FINEST, String.format(
 864                     "errorCode: [%d], "
 865                     + "url: [%s], "
 866                     + "message: [%s], "
 867                     + "data: [0x%016X]",
 868                     errorCode,
 869                     url,
 870                     message,
 871                     data));
 872         }
 873         twkDidFail(errorCode, url, message, data);
 874     }
 875 
 876     private void callBack(Runnable runnable) {
 877         if (asynchronous) {
 878             Invoker.getInvoker().invokeOnEventThread(runnable);
 879         } else {
 880             runnable.run();
 881         }
 882     }
 883 
 884     private static native void twkDidSendData(long totalBytesSent,
 885                                               long totalBytesToBeSent,
 886                                               long data);
 887 
 888     private static native boolean twkWillSendRequest(String newUrl,
 889                                                      String newMethod,
 890                                                      int status,
 891                                                      String contentType,
 892                                                      String contentEncoding,
 893                                                      long contentLength,
 894                                                      String headers,
 895                                                      String url,
 896                                                      long data);
 897 
 898     private static native void twkDidReceiveResponse(int status,
 899                                                      String contentType,
 900                                                      String contentEncoding,
 901                                                      long contentLength,
 902                                                      String headers,
 903                                                      String url,
 904                                                      long data);
 905 
 906     private static native void twkDidReceiveData(ByteBuffer byteBuffer,
 907                                                  int position,
 908                                                  int remaining,
 909                                                  long data);
 910 
 911     private static native void twkDidFinishLoading(long data);
 912 
 913     private static native void twkDidFail(int errorCode,
 914                                           String url,
 915                                           String message,
 916                                           long data);
 917 
 918     /**
 919      * Given a {@link URLConnection}, returns the connection status
 920      * for passing into native callbacks.
 921      */
 922     private static int extractStatus(URLConnection c) {
 923         int status = 0;
 924         if (c instanceof HttpURLConnection) {
 925             try {
 926                 status = ((HttpURLConnection) c).getResponseCode();
 927             } catch (java.io.IOException ignore) {}
 928         }
 929         return status;
 930     }
 931 
 932     /**
 933      * Given a {@link URLConnection}, returns the content encoding
 934      * for passing into native callbacks.
 935      */
 936     private static String extractContentEncoding(URLConnection c) {
 937         String contentEncoding = c.getContentEncoding();
 938         // For compressed streams, the encoding is in Content-Type
 939         if ("gzip".equalsIgnoreCase(contentEncoding) ||
 940             "deflate".equalsIgnoreCase(contentEncoding))
 941         {
 942             contentEncoding = null;
 943             String contentType  = c.getContentType();
 944             if (contentType != null) {
 945                 int i = contentType.indexOf("charset=");
 946                 if (i >= 0) {
 947                     contentEncoding = contentType.substring(i + 8);
 948                     i = contentEncoding.indexOf(";");
 949                     if (i > 0) {
 950                         contentEncoding = contentEncoding.substring(0, i);
 951                     }
 952                 }
 953             }
 954         }
 955         return contentEncoding;
 956     }
 957 
 958     /**
 959      * Given a {@link URLConnection}, returns the content length
 960      * for passing into native callbacks.
 961      */
 962     private static long extractContentLength(URLConnection c) {
 963         // Cannot use URLConnection.getContentLength()
 964         // as it only returns an int
 965         try {
 966             return Long.parseLong(c.getHeaderField("content-length"));
 967         } catch (Exception ex) {
 968             return -1;
 969         }
 970     }
 971 
 972     /**
 973      * Given a {@link URLConnection}, returns the headers string
 974      * for passing into native callbacks.
 975      */
 976     private static String extractHeaders(URLConnection c) {
 977         StringBuilder sb = new StringBuilder();
 978         Map<String, List<String>> headers = c.getHeaderFields();
 979         for (Map.Entry<String, List<String>> entry: headers.entrySet()) {
 980             String key = entry.getKey();
 981             List<String> values = entry.getValue();
 982             for (String value : values) {
 983                 sb.append(key != null ? key : "");
 984                 sb.append(':').append(value).append('\n');
 985             }
 986         }
 987         return sb.toString();
 988     }
 989 
 990     /**
 991      * Adjust a URL string for passing into WebKit.
 992      */
 993     private static String adjustUrlForWebKit(String url) {
 994         try {
 995             url = Util.adjustUrlForWebKit(url);
 996         } catch (Exception ignore) {
 997         }
 998         return url;
 999     }
1000 }