Tiff: Include two upstream CVE fixes in bundled libtiff

For issues CVE-2019-17546 and CVE-2019-14973, the following commits
were merged into the bundled libtiff:

4bb584a35f87af42d6cf09d15e9ce8909a839145 RGBA interface: fix integer
overflow potentially causing write heap buffer overflow, especially on
32 bit builds. Fixes
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=16443. Credit to
OSS Fuzz

1b5e3b6a23827c33acf19ad50ce5ce78f12b3773 Fix integer overflow in
_TIFFCheckMalloc() and other implementation-defined behaviour
(CVE-2019-14973)

Fixes: QTBUG-79397
Change-Id: I29257e6dbfbd816224d3dbaefdbe8afecd25f288
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

dist/changes-5.13.2

@@ -17,4 +17,9 @@ https://bugreports.qt.io/
 Each of these identifiers can be entered in the bug tracker to obtain more
 information about a particular change.
 
- - This release contains only minor code improvements.
+****************************************************************************
+*                                   TIFF                                   *
+****************************************************************************
+
+ - Two security-related upstream patches has been applied to the
+   bundled libtiff

src/3rdparty/libtiff/libtiff/tif_aux.c

@@ -57,18 +57,57 @@ _TIFFMultiply64(TIFF* tif, uint64 first, uint64 second, const char* where)
 	return bytes;
 }
 
+tmsize_t
+_TIFFMultiplySSize(TIFF* tif, tmsize_t first, tmsize_t second, const char* where)
+{
+    if( first <= 0 || second <= 0 )
+    {
+        if( tif != NULL && where != NULL )
+        {
+            TIFFErrorExt(tif->tif_clientdata, where,
+                        "Invalid argument to _TIFFMultiplySSize() in %s", where);
+        }
+        return 0;
+    }
+
+    if( first > TIFF_TMSIZE_T_MAX / second )
+    {
+        if( tif != NULL && where != NULL )
+        {
+            TIFFErrorExt(tif->tif_clientdata, where,
+                        "Integer overflow in %s", where);
+        }
+        return 0;
+    }
+    return first * second;
+}
+
+tmsize_t _TIFFCastUInt64ToSSize(TIFF* tif, uint64 val, const char* module)
+{
+    if( val > (uint64)TIFF_TMSIZE_T_MAX )
+    {
+        if( tif != NULL && module != NULL )
+        {
+            TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
+        }
+        return 0;
+    }
+    return (tmsize_t)val;
+}
+
 void*
 _TIFFCheckRealloc(TIFF* tif, void* buffer,
 		  tmsize_t nmemb, tmsize_t elem_size, const char* what)
 {
 	void* cp = NULL;
-	tmsize_t bytes = nmemb * elem_size;
+        tmsize_t count = _TIFFMultiplySSize(tif, nmemb, elem_size, NULL);
-
 	/*
-	 * XXX: Check for integer overflow.
+	 * Check for integer overflow.
 	 */
-	if (nmemb && elem_size && bytes / elem_size == nmemb)
+	if (count != 0)
-		cp = _TIFFrealloc(buffer, bytes);
+	{
+		cp = _TIFFrealloc(buffer, count);
+	}
 
 	if (cp == NULL) {
 		TIFFErrorExt(tif->tif_clientdata, tif->tif_name,

src/3rdparty/libtiff/libtiff/tif_getimage.c

@@ -755,9 +755,8 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 	uint32 leftmost_tw;
 
 	tilesize = TIFFTileSize(tif);  
-	bufsize = TIFFSafeMultiply(tmsize_t,alpha?4:3,tilesize);
+	bufsize = _TIFFMultiplySSize(tif, alpha?4:3,tilesize, "gtTileSeparate");
 	if (bufsize == 0) {
-		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtTileSeparate");
 		return (0);
 	}
 
@@ -950,16 +949,23 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 	fromskew = (w < imagewidth ? imagewidth - w : 0);
 	for (row = 0; row < h; row += nrow)
 	{
+		uint32 temp;
 		rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
 		nrow = (row + rowstoread > h ? h - row : rowstoread);
 		nrowsub = nrow;
 		if ((nrowsub%subsamplingver)!=0)
 			nrowsub+=subsamplingver-nrowsub%subsamplingver;
+		temp = (row + img->row_offset)%rowsperstrip + nrowsub;
+		if( scanline > 0 && temp > (size_t)(TIFF_TMSIZE_T_MAX / scanline) )
+		{
+			TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in gtStripContig");
+			return 0;
+		}
 		if (_TIFFReadEncodedStripAndAllocBuffer(tif,
 		    TIFFComputeStrip(tif,row+img->row_offset, 0),
 		    (void**)(&buf),
                     maxstripsize,
-		    ((row + img->row_offset)%rowsperstrip + nrowsub) * scanline)==(tmsize_t)(-1)
+		    temp * scanline)==(tmsize_t)(-1)
 		    && (buf == NULL || img->stoponerr))
 		{
 			ret = 0;
@@ -1019,9 +1025,8 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
         uint16 colorchannels;
 
 	stripsize = TIFFStripSize(tif);  
-	bufsize = TIFFSafeMultiply(tmsize_t,alpha?4:3,stripsize);
+	bufsize = _TIFFMultiplySSize(tif,alpha?4:3,stripsize, "gtStripSeparate");
 	if (bufsize == 0) {
-		TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
 		return (0);
 	}
 
@@ -1053,15 +1058,22 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 	fromskew = (w < imagewidth ? imagewidth - w : 0);
 	for (row = 0; row < h; row += nrow)
 	{
+                uint32 temp;
 		rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
 		nrow = (row + rowstoread > h ? h - row : rowstoread);
 		offset_row = row + img->row_offset;
+                temp = (row + img->row_offset)%rowsperstrip + nrow;
+                if( scanline > 0 && temp > (size_t)(TIFF_TMSIZE_T_MAX / scanline) )
+                {
+                        TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in gtStripSeparate");
+                        return 0;
+                }
                 if( buf == NULL )
                 {
                     if (_TIFFReadEncodedStripAndAllocBuffer(
                             tif, TIFFComputeStrip(tif, offset_row, 0),
                             (void**) &buf, bufsize,
-                            ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+                            temp * scanline)==(tmsize_t)(-1)
                         && (buf == NULL || img->stoponerr))
                     {
                             ret = 0;
@@ -1081,7 +1093,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
                     }
                 }
 		else if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
-		    p0, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+		    p0, temp * scanline)==(tmsize_t)(-1)
 		    && img->stoponerr)
 		{
 			ret = 0;
@@ -1089,7 +1101,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 		}
 		if (colorchannels > 1 
                     && TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 1),
-                                            p1, ((row + img->row_offset)%rowsperstrip + nrow) * scanline) == (tmsize_t)(-1)
+                                            p1, temp * scanline) == (tmsize_t)(-1)
 		    && img->stoponerr)
 		{
 			ret = 0;
@@ -1097,7 +1109,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 		}
 		if (colorchannels > 1 
                     && TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 2),
-                                            p2, ((row + img->row_offset)%rowsperstrip + nrow) * scanline) == (tmsize_t)(-1)
+                                            p2, temp * scanline) == (tmsize_t)(-1)
 		    && img->stoponerr)
 		{
 			ret = 0;
@@ -1106,7 +1118,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
 		if (alpha)
 		{
 			if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, colorchannels),
-			    pa, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+			    pa, temp * scanline)==(tmsize_t)(-1)
 			    && img->stoponerr)
 			{
 				ret = 0;

src/3rdparty/libtiff/libtiff/tif_luv.c

@@ -1264,16 +1264,10 @@ LogL16GuessDataFmt(TIFFDirectory *td)
 	return (SGILOGDATAFMT_UNKNOWN);
 }
 
-
-#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
-#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
-
 static tmsize_t
 multiply_ms(tmsize_t m1, tmsize_t m2)
 {
-        if( m1 == 0 || m2 > TIFF_TMSIZE_T_MAX / m1 )
+        return _TIFFMultiplySSize(NULL, m1, m2, NULL);
-            return 0;
-        return m1 * m2;
 }
 
 static int

src/3rdparty/libtiff/libtiff/tif_pixarlog.c

@@ -634,15 +634,10 @@ PixarLogGuessDataFmt(TIFFDirectory *td)
 	return guess;
 }
 
-#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
-#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
-
 static tmsize_t
 multiply_ms(tmsize_t m1, tmsize_t m2)
 {
-        if( m1 == 0 || m2 > TIFF_TMSIZE_T_MAX / m1 )
+	return _TIFFMultiplySSize(NULL, m1, m2, NULL);
-            return 0;
-        return m1 * m2;
 }
 
 static tmsize_t

src/3rdparty/libtiff/libtiff/tif_read.c

@@ -29,9 +29,6 @@
 #include "tiffiop.h"
 #include <stdio.h>
 
-#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
-#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
-
 int TIFFFillStrip(TIFF* tif, uint32 strip);
 int TIFFFillTile(TIFF* tif, uint32 tile);
 static int TIFFStartStrip(TIFF* tif, uint32 strip);
@@ -49,6 +46,8 @@ TIFFReadRawTile1(TIFF* tif, uint32 tile, void* buf, tmsize_t size, const char* m
 #define THRESHOLD_MULTIPLIER 10
 #define MAX_THRESHOLD (THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * INITIAL_THRESHOLD)
 
+#define TIFF_INT64_MAX ((((int64)0x7FFFFFFF) << 32) | 0xFFFFFFFF)
+
 /* Read 'size' bytes in tif_rawdata buffer starting at offset 'rawdata_offset'
  * Returns 1 in case of success, 0 otherwise. */
 static int TIFFReadAndRealloc( TIFF* tif, tmsize_t size,
@@ -734,23 +733,8 @@ TIFFReadRawStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
 		return ((tmsize_t)(-1));
 	}
 	bytecount = td->td_stripbytecount[strip];
-	if ((int64)bytecount <= 0) {
+        bytecountm = _TIFFCastUInt64ToSSize(tif, bytecount, module);
-#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+        if (bytecountm == 0) {
-		TIFFErrorExt(tif->tif_clientdata, module,
-			     "%I64u: Invalid strip byte count, strip %lu",
-			     (unsigned __int64) bytecount,
-			     (unsigned long) strip);
-#else
-		TIFFErrorExt(tif->tif_clientdata, module,
-			     "%llu: Invalid strip byte count, strip %lu",
-			     (unsigned long long) bytecount,
-			     (unsigned long) strip);
-#endif
-		return ((tmsize_t)(-1));
-	}
-	bytecountm = (tmsize_t)bytecount;
-	if ((uint64)bytecountm!=bytecount) {
-		TIFFErrorExt(tif->tif_clientdata, module, "Integer overflow");
 		return ((tmsize_t)(-1));
 	}
 	if (size != (tmsize_t)(-1) && size < bytecountm)
@@ -774,7 +758,7 @@ TIFFFillStrip(TIFF* tif, uint32 strip)
 	if ((tif->tif_flags&TIFF_NOREADRAW)==0)
 	{
 		uint64 bytecount = td->td_stripbytecount[strip];
-		if ((int64)bytecount <= 0) {
+		if( bytecount == 0 || bytecount > (uint64)TIFF_INT64_MAX ) {
 #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
 			TIFFErrorExt(tif->tif_clientdata, module,
 				"Invalid strip byte count %I64u, strip %lu",
@@ -801,7 +785,7 @@ TIFFFillStrip(TIFF* tif, uint32 strip)
 			    (bytecount - 4096) / 10 > (uint64)stripsize  )
 			{
 				uint64 newbytecount = (uint64)stripsize * 10 + 4096;
-				if( (int64)newbytecount >= 0 )
+				if( newbytecount == 0 || newbytecount > (uint64)TIFF_INT64_MAX )
 				{
 #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
 					TIFFWarningExt(tif->tif_clientdata, module,
@@ -1196,10 +1180,8 @@ TIFFReadRawTile(TIFF* tif, uint32 tile, void* buf, tmsize_t size)
 	bytecount64 = td->td_stripbytecount[tile];
 	if (size != (tmsize_t)(-1) && (uint64)size < bytecount64)
 		bytecount64 = (uint64)size;
-	bytecountm = (tmsize_t)bytecount64;
+	bytecountm = _TIFFCastUInt64ToSSize(tif, bytecount64, module);
-	if ((uint64)bytecountm!=bytecount64)
+        if( bytecountm == 0 ) {
-	{
-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
 		return ((tmsize_t)(-1));
 	}
 	return (TIFFReadRawTile1(tif, tile, buf, bytecountm, module));
@@ -1221,7 +1203,7 @@ TIFFFillTile(TIFF* tif, uint32 tile)
 	if ((tif->tif_flags&TIFF_NOREADRAW)==0)
 	{
 		uint64 bytecount = td->td_stripbytecount[tile];
-		if ((int64)bytecount <= 0) {
+		if( bytecount == 0 || bytecount > (uint64)TIFF_INT64_MAX ) {
 #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
 			TIFFErrorExt(tif->tif_clientdata, module,
 				"%I64u: Invalid tile byte count, tile %lu",
@@ -1248,7 +1230,7 @@ TIFFFillTile(TIFF* tif, uint32 tile)
 			    (bytecount - 4096) / 10 > (uint64)stripsize  )
 			{
 				uint64 newbytecount = (uint64)stripsize * 10 + 4096;
-				if( (int64)newbytecount >= 0 )
+				if( newbytecount == 0 || newbytecount > (uint64)TIFF_INT64_MAX )
 				{
 #if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
 					TIFFWarningExt(tif->tif_clientdata, module,

src/3rdparty/libtiff/libtiff/tif_strip.c

@@ -129,15 +129,8 @@ TIFFVStripSize(TIFF* tif, uint32 nrows)
 {
 	static const char module[] = "TIFFVStripSize";
 	uint64 m;
-	tmsize_t n;
 	m=TIFFVStripSize64(tif,nrows);
-	n=(tmsize_t)m;
+        return _TIFFCastUInt64ToSSize(tif, m, module);
-	if ((uint64)n!=m)
-	{
-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
-		n=0;
-	}
-	return(n);
 }
 
 /*
@@ -211,15 +204,8 @@ TIFFStripSize(TIFF* tif)
 {
 	static const char module[] = "TIFFStripSize";
 	uint64 m;
-	tmsize_t n;
 	m=TIFFStripSize64(tif);
-	n=(tmsize_t)m;
+	return _TIFFCastUInt64ToSSize(tif, m, module);
-	if ((uint64)n!=m)
-	{
-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
-		n=0;
-	}
-	return(n);
 }
 
 /*
@@ -330,14 +316,8 @@ TIFFScanlineSize(TIFF* tif)
 {
 	static const char module[] = "TIFFScanlineSize";
 	uint64 m;
-	tmsize_t n;
 	m=TIFFScanlineSize64(tif);
-	n=(tmsize_t)m;
+	return _TIFFCastUInt64ToSSize(tif, m, module);
-	if ((uint64)n!=m) {
-		TIFFErrorExt(tif->tif_clientdata,module,"Integer arithmetic overflow");
-		n=0;
-	}
-	return(n);
 }
 
 /*
@@ -366,15 +346,8 @@ TIFFRasterScanlineSize(TIFF* tif)
 {
 	static const char module[] = "TIFFRasterScanlineSize";
 	uint64 m;
-	tmsize_t n;
 	m=TIFFRasterScanlineSize64(tif);
-	n=(tmsize_t)m;
+	return _TIFFCastUInt64ToSSize(tif, m, module);
-	if ((uint64)n!=m)
-	{
-		TIFFErrorExt(tif->tif_clientdata,module,"Integer arithmetic overflow");
-		n=0;
-	}
-	return(n);
 }
 
 /* vim: set ts=8 sts=8 sw=8 noet: */

src/3rdparty/libtiff/libtiff/tif_tile.c

@@ -181,15 +181,8 @@ TIFFTileRowSize(TIFF* tif)
 {
 	static const char module[] = "TIFFTileRowSize";
 	uint64 m;
-	tmsize_t n;
 	m=TIFFTileRowSize64(tif);
-	n=(tmsize_t)m;
+	return _TIFFCastUInt64ToSSize(tif, m, module);
-	if ((uint64)n!=m)
-	{
-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
-		n=0;
-	}
-	return(n);
 }
 
 /*
@@ -248,15 +241,8 @@ TIFFVTileSize(TIFF* tif, uint32 nrows)
 {
 	static const char module[] = "TIFFVTileSize";
 	uint64 m;
-	tmsize_t n;
 	m=TIFFVTileSize64(tif,nrows);
-	n=(tmsize_t)m;
+	return _TIFFCastUInt64ToSSize(tif, m, module);
-	if ((uint64)n!=m)
-	{
-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
-		n=0;
-	}
-	return(n);
 }
 
 /*
@@ -272,15 +258,8 @@ TIFFTileSize(TIFF* tif)
 {
 	static const char module[] = "TIFFTileSize";
 	uint64 m;
-	tmsize_t n;
 	m=TIFFTileSize64(tif);
-	n=(tmsize_t)m;
+	return _TIFFCastUInt64ToSSize(tif, m, module);
-	if ((uint64)n!=m)
-	{
-		TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
-		n=0;
-	}
-	return(n);
 }
 
 /*

src/3rdparty/libtiff/libtiff/tiffiop.h

@@ -77,6 +77,9 @@ extern int snprintf(char* str, size_t size, const char* format, ...);
 #define	FALSE	0
 #endif
 
+#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
+#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
+
 typedef struct client_info {
     struct client_info *next;
     void *data;
@@ -258,7 +261,7 @@ struct tiff {
 #define TIFFhowmany8_64(x) (((x)&0x07)?((uint64)(x)>>3)+1:(uint64)(x)>>3)
 #define TIFFroundup_64(x, y) (TIFFhowmany_64(x,y)*(y))
 
-/* Safe multiply which returns zero if there is an integer overflow */
+/* Safe multiply which returns zero if there is an *unsigned* integer overflow. This macro is not safe for *signed* integer types */
 #define TIFFSafeMultiply(t,v,m) ((((t)(m) != (t)0) && (((t)(((v)*(m))/(m))) == (t)(v))) ? (t)((v)*(m)) : (t)0)
 
 #define TIFFmax(A,B) ((A)>(B)?(A):(B))
@@ -368,6 +371,8 @@ extern TIFFErrorHandlerExt _TIFFerrorHandlerExt;
 
 extern uint32 _TIFFMultiply32(TIFF*, uint32, uint32, const char*);
 extern uint64 _TIFFMultiply64(TIFF*, uint64, uint64, const char*);
+extern tmsize_t _TIFFMultiplySSize(TIFF*, tmsize_t, tmsize_t, const char*);
+extern tmsize_t _TIFFCastUInt64ToSSize(TIFF*, uint64, const char*);
 extern void* _TIFFCheckMalloc(TIFF*, tmsize_t, tmsize_t, const char*);
 extern void* _TIFFCheckRealloc(TIFF*, void*, tmsize_t, tmsize_t, const char*);