From 31c6d826a7f7a4ee7d83c8e99f25d82a4a248076 Mon Sep 17 00:00:00 2001
From: Richard Lowe <richlowe@richlowe.net>
Date: Thu, 12 Jun 2014 00:36:19 +0100
Subject: [PATCH] 4922 all calloc() implementations should check for overflow
 Reviewed by: Hans Rosenfeld <hans.rosenfeld@nexenta.com> Reviewed by: Marcel
 Telka <marcel.telka@nexenta.com> Approved by: Dan McDonald
 <danmcd@omniti.com>

---
 usr/src/cmd/sgs/rtld/common/malloc.c      | 18 +++++++++++++++---
 usr/src/lib/libmalloc/common/malloc.c     | 21 ++++++++++++++++-----
 usr/src/lib/libmapmalloc/common/calloc.c  | 22 +++++++++++++++++-----
 usr/src/lib/libmtmalloc/common/mtmalloc.c | 14 +++++++++++++-
 4 files changed, 61 insertions(+), 14 deletions(-)

diff --git a/usr/src/cmd/sgs/rtld/common/malloc.c b/usr/src/cmd/sgs/rtld/common/malloc.c
index 1069ec76e7..4e287d4d2f 100644
--- a/usr/src/cmd/sgs/rtld/common/malloc.c
+++ b/usr/src/cmd/sgs/rtld/common/malloc.c
@@ -207,11 +207,23 @@ void *
 calloc(size_t num, size_t size)
 {
 	void *	mp;
+	size_t	total;
 
-	num *= size;
-	if ((mp = malloc(num)) == NULL)
+	if (num == 0 || size == 0) {
+		total = 0;
+	} else {
+		total = num * size;
+
+		/* check for overflow */
+		if ((total / num) != size) {
+			errno = ENOMEM;
+			return (NULL);
+		}
+	}
+
+	if ((mp = malloc(total)) == NULL)
 		return (NULL);
-	(void) memset(mp, 0, num);
+	(void) memset(mp, 0, total);
 	return (mp);
 }
 
diff --git a/usr/src/lib/libmalloc/common/malloc.c b/usr/src/lib/libmalloc/common/malloc.c
index 1d2bbef56b..25343ea62c 100644
--- a/usr/src/lib/libmalloc/common/malloc.c
+++ b/usr/src/lib/libmalloc/common/malloc.c
@@ -27,8 +27,6 @@
 /*	Copyright (c) 1988 AT&T	*/
 /*	  All Rights Reserved  	*/
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/types.h>
 
 #ifndef debug
@@ -37,6 +35,7 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <errno.h>
 #include "assert.h"
 #include "malloc.h"
 #include "mallint.h"
@@ -845,12 +844,24 @@ void *
 calloc(size_t num, size_t size)
 {
 	char *mp;
+	size_t total;
+
+	if (num == 0 || size == 0) {
+		total = 0;
+	} else {
+		total = num * size;
+
+		/* check for overflow */
+		if ((total / num) != size) {
+			errno = ENOMEM;
+			return (NULL);
+		}
+	}
 
-	num *= size;
-	mp = malloc(num);
+	mp = malloc(total);
 	if (mp == NULL)
 		return (NULL);
-	(void) memset(mp, 0, num);
+	(void) memset(mp, 0, total);
 	return (mp);
 }
 
diff --git a/usr/src/lib/libmapmalloc/common/calloc.c b/usr/src/lib/libmapmalloc/common/calloc.c
index d654abd77d..f6bc623a8c 100644
--- a/usr/src/lib/libmapmalloc/common/calloc.c
+++ b/usr/src/lib/libmapmalloc/common/calloc.c
@@ -24,10 +24,10 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <stdlib.h>
 #include <string.h>
+#include <errno.h>
+
 /*
  * calloc - allocate and clear memory block
  */
@@ -36,12 +36,24 @@ void *
 calloc(size_t num, size_t size)
 {
 	void *mp;
+	size_t total;
+
+	if (num == 0 || size == 0) {
+		total = 0;
+	} else {
+		total = num * size;
+
+		/* check for overflow */
+		if ((total / num) != size) {
+			errno = ENOMEM;
+			return (NULL);
+		}
+	}
 
-	num *= size;
-	mp = malloc(num);
+	mp = malloc(total);
 	if (mp == NULL)
 		return (NULL);
-	(void) memset(mp, 0, num);
+	(void) memset(mp, 0, total);
 	return (mp);
 }
 
diff --git a/usr/src/lib/libmtmalloc/common/mtmalloc.c b/usr/src/lib/libmtmalloc/common/mtmalloc.c
index 91d9ae2371..0cf998c952 100644
--- a/usr/src/lib/libmtmalloc/common/mtmalloc.c
+++ b/usr/src/lib/libmtmalloc/common/mtmalloc.c
@@ -331,7 +331,19 @@ void *
 calloc(size_t nelem, size_t bytes)
 {
 	void * ptr;
-	size_t size = nelem * bytes;
+	size_t size;
+
+	if (nelem == 0 || bytes == 0) {
+		size = 0;
+	} else {
+		size = nelem * bytes;
+
+		/* check for overflow */
+		if ((size / nelem) != bytes) {
+			errno = ENOMEM;
+			return (NULL);
+		}
+	}
 
 	ptr = malloc(size);
 	if (ptr == NULL)
-- 
2.11.4.GIT