af_unix: Return errno instead of NULL in unix_create1().

unix_create1() returns NULL on error, and the callers assume that it never
fails for reasons other than out of memory.  So, the callers always return
-ENOMEM when unix_create1() fails.

However, it also returns NULL when the number of af_unix sockets exceeds
twice the limit controlled by sysctl: fs.file-max.  In this case, the
callers should return -ENFILE like alloc_empty_file().

This patch changes unix_create1() to return the correct error value instead
of NULL on error.

Out of curiosity, the assumption has been wrong since 1999 due to this
change introduced in 2.2.4 [0].

  diff -u --recursive --new-file v2.2.3/linux/net/unix/af_unix.c linux/net/unix/af_unix.c
  --- v2.2.3/linux/net/unix/af_unix.c	Tue Jan 19 11:32:53 1999
  +++ linux/net/unix/af_unix.c	Sun Mar 21 07:22:00 1999
  @@ -388,6 +413,9 @@
   {
   	struct sock *sk;

  +	if (atomic_read(&unix_nr_socks) >= 2*max_files)
  +		return NULL;
  +
   	MOD_INC_USE_COUNT;
   	sk = sk_alloc(PF_UNIX, GFP_KERNEL, 1);
   	if (!sk) {

[0]: https://cdn.kernel.org/pub/linux/kernel/v2.2/patch-2.2.4.gz

Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2")
Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.co.jp>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
index 92345c9bb60cc3b469e7cf50effe122b81c7bb89..f505b89bda6adefe973806f842001d428fc6c5ca 100644 (file)
--- a/net/unix/af_unix.c
+++ b/net/unix/af_unix.c
@@ -828,20 +828,25 @@ struct proto unix_stream_proto = {
 
 static struct sock *unix_create1(struct net *net, struct socket *sock, int kern, int type)
 {
-       struct sock *sk = NULL;
        struct unix_sock *u;
+       struct sock *sk;
+       int err;
 
        atomic_long_inc(&unix_nr_socks);
-       if (atomic_long_read(&unix_nr_socks) > 2 * get_max_files())
-               goto out;
+       if (atomic_long_read(&unix_nr_socks) > 2 * get_max_files()) {
+               err = -ENFILE;
+               goto err;
+       }
 
        if (type == SOCK_STREAM)
                sk = sk_alloc(net, PF_UNIX, GFP_KERNEL, &unix_stream_proto, kern);
        else /*dgram and  seqpacket */
                sk = sk_alloc(net, PF_UNIX, GFP_KERNEL, &unix_dgram_proto, kern);
 
-       if (!sk)
-               goto out;
+       if (!sk) {
+               err = -ENOMEM;
+               goto err;
+       }
 
        sock_init_data(sock, sk);
 
@@ -861,20 +866,23 @@ static struct sock *unix_create1(struct net *net, struct socket *sock, int kern,
        init_waitqueue_func_entry(&u->peer_wake, unix_dgram_peer_wake_relay);
        memset(&u->scm_stat, 0, sizeof(struct scm_stat));
        unix_insert_socket(unix_sockets_unbound(sk), sk);
-out:
-       if (sk == NULL)
-               atomic_long_dec(&unix_nr_socks);
-       else {
-               local_bh_disable();
-               sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
-               local_bh_enable();
-       }
+
+       local_bh_disable();
+       sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
+       local_bh_enable();
+
        return sk;
+
+err:
+       atomic_long_dec(&unix_nr_socks);
+       return ERR_PTR(err);
 }
 
 static int unix_create(struct net *net, struct socket *sock, int protocol,
                       int kern)
 {
+       struct sock *sk;
+
        if (protocol && protocol != PF_UNIX)
                return -EPROTONOSUPPORT;
 
@@ -901,7 +909,11 @@ static int unix_create(struct net *net, struct socket *sock, int protocol,
                return -ESOCKTNOSUPPORT;
        }
 
-       return unix_create1(net, sock, kern, sock->type) ? 0 : -ENOMEM;
+       sk = unix_create1(net, sock, kern, sock->type);
+       if (IS_ERR(sk))
+               return PTR_ERR(sk);
+
+       return 0;
 }
 
 static int unix_release(struct socket *sock)
@@ -1314,12 +1326,15 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr,
           we will have to recheck all again in any case.
         */
 
-       err = -ENOMEM;
-
        /* create new sock for complete connection */
        newsk = unix_create1(sock_net(sk), NULL, 0, sock->type);
-       if (newsk == NULL)
+       if (IS_ERR(newsk)) {
+               err = PTR_ERR(newsk);
+               newsk = NULL;
                goto out;
+       }
+
+       err = -ENOMEM;
 
        /* Allocate skb for sending to listening sock */
        skb = sock_wmalloc(newsk, 1, 0, GFP_KERNEL);