# HG changeset patch
# User stuefe
# Date 1504161918 -7200
# Node ID fc2017f2ac02bfaf300cb40696fb3f541ae08724
# Parent  162c0a6e1fe33e1fcfef4a37560745f401b5c046
8186665: [aix] buffer overflow in Java_java_nio_MappedByteBuffer_isLoaded0
Reviewed-by:

diff --git a/src/java.base/unix/native/libnio/MappedByteBuffer.c b/src/java.base/unix/native/libnio/MappedByteBuffer.c
--- a/src/java.base/unix/native/libnio/MappedByteBuffer.c
+++ b/src/java.base/unix/native/libnio/MappedByteBuffer.c
@@ -28,10 +28,33 @@
 #include "jvm.h"
 #include "jlong.h"
 #include "java_nio_MappedByteBuffer.h"
+#include <assert.h>
 #include <sys/mman.h>
 #include <stddef.h>
 #include <stdlib.h>
 
+#ifdef _AIX
+#include <unistd.h>
+#endif
+
+
+#ifdef _AIX
+static long calculate_number_of_pages_in_range(void* address, size_t len, size_t pagesize) {
+    uintptr_t address_unaligned = (uintptr_t) address;
+    uintptr_t address_aligned = address_unaligned & (~(pagesize - 1));
+    size_t len2 = len + (address_unaligned - address_aligned);
+    long numPages = (len2 + pagesize - 1) / pagesize;
+    return len2 + pagesize - 1 / pagesize;
+}
+#endif
+
+/* Output type for mincore(2) */
+#ifdef __linux__
+typedef unsigned char mincore_vec_t;
+#else
+typedef char mincore_vec_t;
+#endif
+
 JNIEXPORT jboolean JNICALL
 Java_java_nio_MappedByteBuffer_isLoaded0(JNIEnv *env, jobject obj, jlong address,
                                          jlong len, jint numPages)
@@ -40,18 +63,28 @@
     int result = 0;
     int i = 0;
     void *a = (void *) jlong_to_ptr(address);
-#ifdef __linux__
-    unsigned char *vec = (unsigned char *)malloc(numPages * sizeof(char));
-#else
-    char *vec = (char *)malloc(numPages * sizeof(char));
+    mincore_vec_t* vec = NULL;
+
+#ifdef _AIX
+    /* See JDK-8186665 */
+    size_t pagesize = (size_t)sysconf(_SC_PAGESIZE);
+    if ((long)pagesize == -1) {
+      return JNI_FALSE;
+    }
+    numPages = (jint) calculate_number_of_pages_in_range(a, len, pagesize);
 #endif
 
+    vec = (mincore_vec_t*) malloc(numPages + 1);
+
     if (vec == NULL) {
         JNU_ThrowOutOfMemoryError(env, NULL);
         return JNI_FALSE;
     }
 
+    vec[numPages] = '\x7f'; /* Sentinel */
     result = mincore(a, (size_t)len, vec);
+    assert(vec[numPages] == '\x7f');
+
     if (result == -1) {
         JNU_ThrowIOExceptionWithLastError(env, "mincore failed");
         free(vec);