Overflow in resample.c, with tests

Tests/test_image_resample.py

@@ -0,0 +1,36 @@
+from helper import unittest, PillowTestCase, hopper
+from PIL import Image
+
+class TestImagingCoreResize(PillowTestCase):
+    #see https://github.com/python-pillow/Pillow/issues/1710
+    def test_overflow(self):
+        im = hopper('L')
+        xsize = 0x100000008 // 4
+        ysize = 1000 # unimportant
+        try:
+            # any resampling filter will do here
+            im.im.resize((xsize, ysize), Image.LINEAR) 
+            self.fail("Resize should raise MemoryError on invalid xsize")
+        except MemoryError:
+            self.assertTrue(True, "Should raise MemoryError")
+
+    def test_invalid_size(self):
+        im = hopper()
+
+        im.resize((100,100))
+        self.assertTrue(True, "Should not Crash")
+
+        try:
+            im.resize((-100,100))
+            self.fail("Resize should raise a value error on x negative size")
+        except ValueError:
+            self.assertTrue(True, "Should raise ValueError")
+
+        try:
+            im.resize((100,-100))
+            self.fail("Resize should raise a value error on y negative size")
+        except ValueError:
+            self.assertTrue(True, "Should raise ValueError")
+
+if __name__ == '__main__':
+    unittest.main()

_imaging.c

@@ -1532,6 +1532,10 @@ _resize(ImagingObject* self, PyObject* args)
 
     imIn = self->image;
 
+    if (xsize < 1 || ysize < 1) {
+        return ImagingError_ValueError("height and width must be > 0");
+    }
+
     if (imIn->xsize == xsize && imIn->ysize == ysize) {
         imOut = ImagingCopy(imIn);
     }

libImaging/Resample.c

@@ -138,11 +138,23 @@ ImagingResampleHorizontal(Imaging imIn, int xsize, int filter)
     /* maximum number of coofs */
     kmax = (int) ceil(support) * 2 + 1;
 
+    // check for overflow
+    if (kmax > 0 && xsize > SIZE_MAX / kmax)
+        return (Imaging) ImagingError_MemoryError();
+
+    // sizeof(float) should be greater than 0
+    if (xsize * kmax > SIZE_MAX / sizeof(float))
+        return (Imaging) ImagingError_MemoryError();
+
     /* coefficient buffer */
     kk = malloc(xsize * kmax * sizeof(float));
     if ( ! kk)
         return (Imaging) ImagingError_MemoryError();
 
+    // sizeof(int) should be greater than 0 as well
+    if (xsize > SIZE_MAX / (2 * sizeof(int)))
+        return (Imaging) ImagingError_MemoryError();
+
     xbounds = malloc(xsize * 2 * sizeof(int));
     if ( ! xbounds) {
         free(kk);