SUNRPC: Fix xdr_expand_hole()

We do want to try to grow the buffer if possible, but if that attempt
fails, we still want to move the data and truncate the XDR message.

Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>

diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h
index 2b4e44bb06542c49e86d60866439fe9b6d200af0..178f499e2283d29b5c7b712e1e861b851d8914ef 100644 (file)
--- a/include/linux/sunrpc/xdr.h
+++ b/include/linux/sunrpc/xdr.h
@@ -253,7 +253,7 @@ extern unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len);
 extern void xdr_enter_page(struct xdr_stream *xdr, unsigned int len);
 extern int xdr_process_buf(struct xdr_buf *buf, unsigned int offset, unsigned int len, int (*actor)(struct scatterlist *, void *), void *data);
 extern unsigned int xdr_align_data(struct xdr_stream *, unsigned int offset, unsigned int length);
-extern uint64_t xdr_expand_hole(struct xdr_stream *, uint64_t, uint64_t);
+extern unsigned int xdr_expand_hole(struct xdr_stream *, unsigned int offset, unsigned int length);
 
 /**
  * xdr_stream_remaining - Return the number of bytes remaining in the stream

diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c
index c474339ba9acf55d833b2543dc9662f3f49d467b..e0906ed243744f2dccdb1ebaeba7d9a0bc0ffdd1 100644 (file)
--- a/net/sunrpc/xdr.c
+++ b/net/sunrpc/xdr.c
@@ -334,46 +334,6 @@ _shift_data_right_pages(struct page **pages, size_t pgto_base,
        } while ((len -= copy) != 0);
 }
 
-static unsigned int
-_shift_data_right_tail(struct xdr_buf *buf, unsigned int pgfrom, size_t len)
-{
-       struct kvec *tail = buf->tail;
-       unsigned int tailbuf_len;
-       unsigned int result = 0;
-       size_t copy;
-
-       tailbuf_len = buf->buflen - buf->head->iov_len - buf->page_len;
-
-       /* Shift the tail first */
-       if (tailbuf_len != 0) {
-               unsigned int free_space = tailbuf_len - tail->iov_len;
-
-               if (len < free_space)
-                       free_space = len;
-               if (len > free_space)
-                       len = free_space;
-
-               tail->iov_len += free_space;
-               copy = len;
-
-               if (tail->iov_len > len) {
-                       char *p = (char *)tail->iov_base + len;
-                       memmove(p, tail->iov_base, tail->iov_len - free_space);
-                       result += tail->iov_len - free_space;
-               } else
-                       copy = tail->iov_len;
-
-               /* Copy from the inlined pages into the tail */
-               _copy_from_pages((char *)tail->iov_base,
-                                        buf->pages,
-                                        buf->page_base + pgfrom,
-                                        copy);
-               result += copy;
-       }
-
-       return result;
-}
-
 /**
  * _copy_to_pages
  * @pages: array of pages
@@ -464,18 +424,42 @@ _copy_from_pages(char *p, struct page **pages, size_t pgbase, size_t len)
 }
 EXPORT_SYMBOL_GPL(_copy_from_pages);
 
+static void xdr_buf_iov_zero(const struct kvec *iov, unsigned int base,
+                            unsigned int len)
+{
+       if (base >= iov->iov_len)
+               return;
+       if (len > iov->iov_len - base)
+               len = iov->iov_len - base;
+       memset(iov->iov_base + base, 0, len);
+}
+
 /**
- * _zero_pages
- * @pages: array of pages
- * @pgbase: beginning page vector address
+ * xdr_buf_pages_zero
+ * @buf: xdr_buf
+ * @pgbase: beginning offset
  * @len: length
  */
-static void
-_zero_pages(struct page **pages, size_t pgbase, size_t len)
+static void xdr_buf_pages_zero(const struct xdr_buf *buf, unsigned int pgbase,
+                              unsigned int len)
 {
+       struct page **pages = buf->pages;
        struct page **page;
        char *vpage;
-       size_t zero;
+       unsigned int zero;
+
+       if (!len)
+               return;
+       if (pgbase >= buf->page_len) {
+               xdr_buf_iov_zero(buf->tail, pgbase - buf->page_len, len);
+               return;
+       }
+       if (pgbase + len > buf->page_len) {
+               xdr_buf_iov_zero(buf->tail, 0, pgbase + len - buf->page_len);
+               len = buf->page_len - pgbase;
+       }
+
+       pgbase += buf->page_base;
 
        page = pages + (pgbase >> PAGE_SHIFT);
        pgbase &= ~PAGE_MASK;
@@ -496,6 +480,103 @@ _zero_pages(struct page **pages, size_t pgbase, size_t len)
        } while ((len -= zero) != 0);
 }
 
+static void xdr_buf_try_expand(struct xdr_buf *buf, unsigned int len)
+{
+       struct kvec *head = buf->head;
+       struct kvec *tail = buf->tail;
+       unsigned int sum = head->iov_len + buf->page_len + tail->iov_len;
+       unsigned int free_space;
+
+       if (sum > buf->len) {
+               free_space = min_t(unsigned int, sum - buf->len, len);
+               buf->len += free_space;
+               len -= free_space;
+               if (!len)
+                       return;
+       }
+
+       if (buf->buflen > sum) {
+               /* Expand the tail buffer */
+               free_space = min_t(unsigned int, buf->buflen - sum, len);
+               tail->iov_len += free_space;
+               buf->len += free_space;
+       }
+}
+
+static void xdr_buf_tail_copy_right(const struct xdr_buf *buf,
+                                   unsigned int base, unsigned int len,
+                                   unsigned int shift)
+{
+       const struct kvec *tail = buf->tail;
+       unsigned int to = base + shift;
+
+       if (to >= tail->iov_len)
+               return;
+       if (len + to > tail->iov_len)
+               len = tail->iov_len - to;
+       memmove(tail->iov_base + to, tail->iov_base + base, len);
+}
+
+static void xdr_buf_pages_copy_right(const struct xdr_buf *buf,
+                                    unsigned int base, unsigned int len,
+                                    unsigned int shift)
+{
+       const struct kvec *tail = buf->tail;
+       unsigned int to = base + shift;
+       unsigned int pglen = 0;
+       unsigned int talen = 0, tato = 0;
+
+       if (base >= buf->page_len)
+               return;
+       if (len > buf->page_len - base)
+               len = buf->page_len - base;
+       if (to >= buf->page_len) {
+               tato = to - buf->page_len;
+               if (tail->iov_len >= len + tato)
+                       talen = len;
+               else if (tail->iov_len > tato)
+                       talen = tail->iov_len - tato;
+       } else if (len + to >= buf->page_len) {
+               pglen = buf->page_len - to;
+               talen = len - pglen;
+               if (talen > tail->iov_len)
+                       talen = tail->iov_len;
+       } else
+               pglen = len;
+
+       _copy_from_pages(tail->iov_base + tato, buf->pages,
+                        buf->page_base + base + pglen, talen);
+       _shift_data_right_pages(buf->pages, buf->page_base + to,
+                               buf->page_base + base, pglen);
+}
+
+static void xdr_buf_tail_shift_right(const struct xdr_buf *buf,
+                                    unsigned int base, unsigned int len,
+                                    unsigned int shift)
+{
+       const struct kvec *tail = buf->tail;
+
+       if (base >= tail->iov_len || !shift || !len)
+               return;
+       xdr_buf_tail_copy_right(buf, base, len, shift);
+}
+
+static void xdr_buf_pages_shift_right(const struct xdr_buf *buf,
+                                     unsigned int base, unsigned int len,
+                                     unsigned int shift)
+{
+       if (!shift || !len)
+               return;
+       if (base >= buf->page_len) {
+               xdr_buf_tail_shift_right(buf, base - buf->page_len, len, shift);
+               return;
+       }
+       if (base + len > buf->page_len)
+               xdr_buf_tail_shift_right(buf, 0, base + len - buf->page_len,
+                                        shift);
+       xdr_buf_pages_copy_right(buf, base, len, shift);
+}
+
 static void xdr_buf_tail_copy_left(const struct xdr_buf *buf, unsigned int base,
                                   unsigned int len, unsigned int shift)
 {
@@ -685,30 +766,33 @@ xdr_shrink_bufhead(struct xdr_buf *buf, size_t len)
 }
 
 /**
- * xdr_shrink_pagelen - shrinks buf->pages by up to @len bytes
+ * xdr_shrink_pagelen - shrinks buf->pages to @len bytes
  * @buf: xdr_buf
- * @len: bytes to remove from buf->pages
+ * @len: new page buffer length
  *
  * The extra data is not lost, but is instead moved into buf->tail.
  * Returns the actual number of bytes moved.
  */
-static unsigned int
-xdr_shrink_pagelen(struct xdr_buf *buf, size_t len)
+static unsigned int xdr_shrink_pagelen(struct xdr_buf *buf, unsigned int len)
 {
-       unsigned int pglen = buf->page_len;
-       unsigned int result;
-
-       if (len > buf->page_len)
-               len = buf-> page_len;
-
-       result = _shift_data_right_tail(buf, pglen - len, len);
-       buf->page_len -= len;
-       buf->buflen -= len;
-       /* Have we truncated the message? */
-       if (buf->len > buf->buflen)
-               buf->len = buf->buflen;
+       unsigned int shift, buflen = buf->len - buf->head->iov_len;
 
-       return result;
+       WARN_ON_ONCE(len > buf->page_len);
+       if (buf->head->iov_len >= buf->len || len > buflen)
+               buflen = len;
+       if (buf->page_len > buflen) {
+               buf->buflen -= buf->page_len - buflen;
+               buf->page_len = buflen;
+       }
+       if (len >= buf->page_len)
+               return 0;
+       shift = buf->page_len - len;
+       xdr_buf_try_expand(buf, shift);
+       xdr_buf_pages_shift_right(buf, len, buflen - len, shift);
+       buf->page_len = len;
+       buf->len -= shift;
+       buf->buflen -= shift;
+       return shift;
 }
 
 void
@@ -728,6 +812,18 @@ unsigned int xdr_stream_pos(const struct xdr_stream *xdr)
 }
 EXPORT_SYMBOL_GPL(xdr_stream_pos);
 
+static void xdr_stream_set_pos(struct xdr_stream *xdr, unsigned int pos)
+{
+       unsigned int blen = xdr->buf->len;
+
+       xdr->nwords = blen > pos ? XDR_QUADLEN(blen) - XDR_QUADLEN(pos) : 0;
+}
+
+static void xdr_stream_page_set_pos(struct xdr_stream *xdr, unsigned int pos)
+{
+       xdr_stream_set_pos(xdr, pos + xdr->buf->head[0].iov_len);
+}
+
 /**
  * xdr_page_pos - Return the current offset from the start of the xdr pages
  * @xdr: pointer to struct xdr_stream
@@ -1291,7 +1387,7 @@ static unsigned int xdr_align_pages(struct xdr_stream *xdr, unsigned int len)
        struct xdr_buf *buf = xdr->buf;
        unsigned int nwords = XDR_QUADLEN(len);
        unsigned int cur = xdr_stream_pos(xdr);
-       unsigned int copied, offset;
+       unsigned int copied;
 
        if (xdr->nwords == 0)
                return 0;
@@ -1305,9 +1401,8 @@ static unsigned int xdr_align_pages(struct xdr_stream *xdr, unsigned int len)
                len = buf->page_len;
        else if (nwords < xdr->nwords) {
                /* Truncate page data and move it into the tail */
-               offset = buf->page_len - len;
-               copied = xdr_shrink_pagelen(buf, offset);
-               trace_rpc_xdr_alignment(xdr, offset, copied);
+               copied = xdr_shrink_pagelen(buf, len);
+               trace_rpc_xdr_alignment(xdr, len, copied);
                xdr->nwords = XDR_QUADLEN(buf->len - cur);
        }
        return len;
@@ -1387,39 +1482,28 @@ unsigned int xdr_align_data(struct xdr_stream *xdr, unsigned int offset,
 }
 EXPORT_SYMBOL_GPL(xdr_align_data);
 
-uint64_t xdr_expand_hole(struct xdr_stream *xdr, uint64_t offset, uint64_t length)
+unsigned int xdr_expand_hole(struct xdr_stream *xdr, unsigned int offset,
+                            unsigned int length)
 {
        struct xdr_buf *buf = xdr->buf;
-       unsigned int bytes;
-       unsigned int from;
-       unsigned int truncated = 0;
-
-       if ((offset + length) < offset ||
-           (offset + length) > buf->page_len)
-               length = buf->page_len - offset;
+       unsigned int from, to, shift;
 
        xdr_realign_pages(xdr);
        from = xdr_page_pos(xdr);
-       bytes = xdr_stream_remaining(xdr);
-
-       if (offset + length + bytes > buf->page_len) {
-               unsigned int shift = (offset + length + bytes) - buf->page_len;
-               unsigned int res = _shift_data_right_tail(buf, from + bytes - shift, shift);
-               truncated = shift - res;
-               xdr->nwords -= XDR_QUADLEN(truncated);
-               bytes -= shift;
-       }
-
-       /* Now move the page data over and zero pages */
-       if (bytes > 0)
-               _shift_data_right_pages(buf->pages,
-                                       buf->page_base + offset + length,
-                                       buf->page_base + from,
-                                       bytes);
-       _zero_pages(buf->pages, buf->page_base + offset, length);
-
-       buf->len += length - (from - offset) - truncated;
-       xdr_set_page(xdr, offset + length, xdr_stream_remaining(xdr));
+       to = xdr_align_size(offset + length);
+
+       /* Could the hole be behind us? */
+       if (to > from) {
+               unsigned int buflen = buf->len - buf->head->iov_len;
+               shift = to - from;
+               xdr_buf_try_expand(buf, shift);
+               xdr_buf_pages_shift_right(buf, from, buflen, shift);
+               xdr_stream_page_set_pos(xdr, to);
+       } else if (to != from)
+               xdr_align_data(xdr, to, 0);
+       xdr_buf_pages_zero(buf, offset, length);
+
+       xdr_set_page(xdr, to, xdr_stream_remaining(xdr));
        return length;
 }
 EXPORT_SYMBOL_GPL(xdr_expand_hole);