}
EXPORT_SYMBOL(copy_page_from_iter_atomic);
-static inline void pipe_truncate(struct iov_iter *i)
-{
- struct pipe_inode_info *pipe = i->pipe;
- unsigned int p_tail = pipe->tail;
- unsigned int p_head = pipe->head;
- unsigned int p_mask = pipe->ring_size - 1;
-
- if (!pipe_empty(p_head, p_tail)) {
- struct pipe_buffer *buf;
- unsigned int i_head = i->head;
- size_t off = i->iov_offset;
-
- if (off) {
- buf = &pipe->bufs[i_head & p_mask];
- buf->len = off - buf->offset;
- i_head++;
- }
- while (p_head != i_head) {
- p_head--;
- pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]);
- }
-
- pipe->head = p_head;
- }
-}
-
static void pipe_advance(struct iov_iter *i, size_t size)
{
struct pipe_inode_info *pipe = i->pipe;
i->count += unroll;
if (unlikely(iov_iter_is_pipe(i))) {
struct pipe_inode_info *pipe = i->pipe;
- unsigned int p_mask = pipe->ring_size - 1;
- unsigned int i_head = i->head;
- size_t off = i->iov_offset;
- while (1) {
- struct pipe_buffer *b = &pipe->bufs[i_head & p_mask];
- size_t n = off - b->offset;
- if (unroll < n) {
- off -= unroll;
- break;
- }
- unroll -= n;
- if (!unroll && i_head == i->start_head) {
- off = 0;
- break;
+ unsigned int head = pipe->head;
+
+ while (head > i->start_head) {
+ struct pipe_buffer *b = pipe_buf(pipe, --head);
+ if (unroll < b->len) {
+ b->len -= unroll;
+ i->iov_offset = b->offset + b->len;
+ i->head = head;
+ return;
}
- i_head--;
- b = &pipe->bufs[i_head & p_mask];
- off = b->offset + b->len;
+ unroll -= b->len;
+ pipe_buf_release(pipe, b);
+ pipe->head--;
}
- i->iov_offset = off;
- i->head = i_head;
- pipe_truncate(i);
+ i->iov_offset = 0;
+ i->head = head;
return;
}
if (unlikely(iov_iter_is_discard(i)))