From 2994659f1e6c1ef260241491bceca91c9d2553b3 Mon Sep 17 00:00:00 2001
From: Venkatesh Srinivas <me@endeavour.zapto.org>
Date: Sun, 6 Mar 2011 05:18:24 -0800
Subject: [PATCH] kernel -- file desc malloc zone overflow handling.

Allow null return from mallocs for file descriptor arrays. Fork and exec
will ENOMEM when we cannot allocate FD arrays.

This is only a partial solution to bug 2019.
---
 sys/kern/kern_descrip.c | 16 +++++++++++-----
 sys/kern/kern_exec.c    |  4 +++-
 sys/kern/kern_fork.c    | 12 ++++++++++--
 sys/sys/filedesc.h      |  2 +-
 4 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index f38aeaf532..b0c43bac53 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -1813,8 +1813,8 @@ fdshare(struct proc *p)
  *
  * MPSAFE
  */
-struct filedesc *
-fdcopy(struct proc *p)
+int
+fdcopy(struct proc *p, struct filedesc **fpp)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct filedesc *newfdp;
@@ -1826,14 +1826,19 @@ fdcopy(struct proc *p)
 	 * Certain daemons might not have file descriptors. 
 	 */
 	if (fdp == NULL)
-		return (NULL);
+		return (0);
 
 	/*
 	 * Allocate the new filedesc and fd_files[] array.  This can race
 	 * with operations by other threads on the fdp so we have to be
 	 * careful.
 	 */
-	newfdp = kmalloc(sizeof(struct filedesc), M_FILEDESC, M_WAITOK | M_ZERO);
+	newfdp = kmalloc(sizeof(struct filedesc), 
+			 M_FILEDESC, M_WAITOK | M_ZERO | M_NULLOK);
+	if (newfdp == NULL) {
+		*fpp = NULL;
+		return (-1);
+	}
 again:
 	spin_lock(&fdp->fd_spin);
 	if (fdp->fd_lastfile < NDFILE) {
@@ -1925,7 +1930,8 @@ again:
 		}
 	}
 	spin_unlock(&fdp->fd_spin);
-	return (newfdp);
+	*fpp = newfdp;
+	return (0);
 }
 
 /*
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index 6b5816f034..e3d234ede1 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -338,7 +338,9 @@ interpret:
 	if (p->p_fd->fd_refcnt > 1) {
 		struct filedesc *tmp;
 
-		tmp = fdcopy(p);
+		error = fdcopy(p, &tmp);
+		if (error != 0)
+			goto exec_fail;
 		fdfree(p, tmp);
 	}
 
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index e926e66c0d..086c2cdd1b 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -283,7 +283,11 @@ fork1(struct lwp *lp1, int flags, struct proc **procp)
 		if (flags & RFFDG) {
 			if (p1->p_fd->fd_refcnt > 1) {
 				struct filedesc *newfd;
-				newfd = fdcopy(p1);
+				error = fdcopy(p1, &newfd);
+				if (error != 0) {
+					error = ENOMEM;
+					goto done;
+				}
 				fdfree(p1, newfd);
 			}
 		}
@@ -439,7 +443,11 @@ fork1(struct lwp *lp1, int flags, struct proc **procp)
 		p2->p_fd = fdinit(p1);
 		fdtol = NULL;
 	} else if (flags & RFFDG) {
-		p2->p_fd = fdcopy(p1);
+		error = fdcopy(p1, &p2->p_fd);
+		if (error != 0) {
+			error = ENOMEM;
+			goto done;
+		}
 		fdtol = NULL;
 	} else {
 		p2->p_fd = fdshare(p1);
diff --git a/sys/sys/filedesc.h b/sys/sys/filedesc.h
index 4e77991fa9..2be21d545f 100644
--- a/sys/sys/filedesc.h
+++ b/sys/sys/filedesc.h
@@ -164,7 +164,7 @@ void	fsetcred (struct file *fp, struct ucred *cr);
 void	fdinit_bootstrap(struct proc *p0, struct filedesc *fdp0, int cmask);
 struct	filedesc *fdinit (struct proc *p);
 struct	filedesc *fdshare (struct proc *p);
-struct	filedesc *fdcopy (struct proc *p);
+int	fdcopy (struct proc *p, struct filedesc **fpp);
 void	fdfree (struct proc *p, struct filedesc *repl);
 int	fdrevoke(void *f_data, short f_type, struct ucred *cred);
 int	closef (struct file *fp, struct proc *p);
-- 
2.11.4.GIT