--- old/src/java.desktop/share/native/libmlib_image/safe_alloc.h 2016-12-19 12:28:28.546670794 -0800 +++ new/src/java.desktop/share/native/libmlib_image/safe_alloc.h 2016-12-19 12:28:27.682670797 -0800 @@ -35,10 +35,10 @@ */ #define SAFE_TO_ALLOC_2(c, sz) \ (((c) > 0) && ((sz) > 0) && \ - ((0xffffffffu / ((juint)(c))) > ((juint)(sz)))) + ((0x7fffffff / (c)) > (sz))) #define SAFE_TO_ALLOC_3(w, h, sz) \ (((w) > 0) && ((h) > 0) && ((sz) > 0) && \ - (((0xffffffffu / ((juint)(w))) / ((juint)(h))) > ((juint)(sz)))) + (((0x7fffffff / (w)) / (h)) > (sz))) #endif // __SAFE_ALLOC_H__ --- old/src/java.desktop/share/native/libawt/awt/medialib/awt_ImagingLib.c 2016-12-19 12:28:30.586670787 -0800 +++ new/src/java.desktop/share/native/libawt/awt/medialib/awt_ImagingLib.c 2016-12-19 12:28:29.902670789 -0800 @@ -2419,7 +2419,7 @@ case sun_awt_image_IntegerComponentRaster_TYPE_INT_8BIT_SAMPLES: if (!((rasterP->chanOffsets[0] == 0 || SAFE_TO_ALLOC_2(rasterP->chanOffsets[0], 4)) && SAFE_TO_ALLOC_2(width, 4) && - SAFE_TO_ALLOC_3(height, rasterP->scanlineStride, 4))) + SAFE_TO_ALLOC_3(rasterP->scanlineStride, height, 4))) { return -1; } @@ -2428,7 +2428,7 @@ if (offset < 0 || offset >= dataSize || width > rasterP->scanlineStride || - height * rasterP->scanlineStride * 4 > dataSize - offset) + ((width + (height - 1) * rasterP->scanlineStride) * 4) > dataSize - offset) { // raster data buffer is too short return -1; @@ -2446,7 +2446,7 @@ return 0; case sun_awt_image_IntegerComponentRaster_TYPE_BYTE_SAMPLES: if (!(SAFE_TO_ALLOC_2(width, rasterP->numBands) && - SAFE_TO_ALLOC_2(height, rasterP->scanlineStride))) + SAFE_TO_ALLOC_2(rasterP->scanlineStride, height))) { return -1; } @@ -2455,7 +2455,8 @@ if (offset < 0 || offset >= dataSize || width * rasterP->numBands > rasterP->scanlineStride || - height * rasterP->scanlineStride > dataSize - offset) + ((width * rasterP->numBands) + + (height - 1) * rasterP->scanlineStride) > dataSize - offset) { // raster data buffer is too short return -1; @@ -2474,7 +2475,7 @@ case sun_awt_image_IntegerComponentRaster_TYPE_USHORT_SAMPLES: if (!((rasterP->chanOffsets[0] == 0 || SAFE_TO_ALLOC_2(rasterP->chanOffsets[0], 2)) && SAFE_TO_ALLOC_3(width, rasterP->numBands, 2) && - SAFE_TO_ALLOC_3(height, rasterP->scanlineStride, 2))) + SAFE_TO_ALLOC_3(rasterP->scanlineStride, height, 2))) { return -1; } @@ -2483,7 +2484,8 @@ if (offset < 0 || offset >= dataSize || width * rasterP->numBands > rasterP->scanlineStride || - height * rasterP->scanlineStride * 2 > dataSize - offset) + (((width * rasterP->numBands) + + (height - 1) * rasterP->scanlineStride)) * 2 > dataSize - offset) { // raster data buffer is too short return -1; --- /dev/null 2016-11-01 09:38:44.965684815 -0700 +++ new/test/java/awt/image/Raster/TestChildRasterOp.java 2016-12-19 12:28:30.798670786 -0800 @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @bug 8130737 + * @test + * @summary test no exception rasterop for child raster with non-zero offset + */ + +import java.awt.geom.AffineTransform; +import java.awt.image.AffineTransformOp; +import java.awt.image.DataBuffer; +import java.awt.image.DataBufferByte; +import java.awt.image.DataBufferInt; +import java.awt.image.DataBufferUShort; +import java.awt.image.Raster; +import java.awt.image.WritableRaster; + +public class TestChildRasterOp { + + private static AffineTransform at = new AffineTransform(); + private static final AffineTransformOp rop = + new AffineTransformOp(at, AffineTransformOp.TYPE_NEAREST_NEIGHBOR); + private static int[] offsets = {0}; + + public static void main(String[] args) { + testByteRaster(); + testShortRaster(); + testIntRaster(); + } + + private static void testByteRaster() { + WritableRaster srcRaster, dstRaster; + + byte[] pixels = + { 11, 12, 13, 14, + 21, 22, 23, 24, + 31, 32, 33, 34, + 41, 42, 43, 44 }; + + DataBuffer db = new DataBufferByte(pixels, pixels.length); + srcRaster = + Raster.createInterleavedRaster(db, 4, 4, 4, 1, offsets, null); + srcRaster = srcRaster.createWritableChild(1, 1, 3, 3, 0, 0, null); + dstRaster = rop.filter(srcRaster, null); + } + + private static void testShortRaster() { + WritableRaster srcRaster, dstRaster; + + short[] pixels = + { 11, 12, 13, 14, + 21, 22, 23, 24, + 31, 32, 33, 34, + 41, 42, 43, 44 }; + + DataBuffer db = new DataBufferUShort(pixels, pixels.length); + srcRaster = + Raster.createInterleavedRaster(db, 4, 4, 4, 1, offsets, null); + srcRaster = srcRaster.createWritableChild(1, 1, 3, 3, 0, 0, null); + dstRaster = rop.filter(srcRaster, null); + } + + private static void testIntRaster() { + WritableRaster srcRaster, dstRaster; + + int[] pixels = + { 11, 12, 13, 14, + 21, 22, 23, 24, + 31, 32, 33, 34, + 41, 42, 43, 44 }; + + DataBuffer db = new DataBufferInt(pixels, pixels.length); + srcRaster = + Raster.createPackedRaster(db, 4, 4, 4, offsets, null); + srcRaster = srcRaster.createWritableChild(1, 1, 3, 3, 0, 0, null); + dstRaster = rop.filter(srcRaster, null); + } +}