1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
/* SPDX-License-Identifier: GPL-2.0 */
/*
* bvec iterator
*
* Copyright (C) 2001 Ming Lei <ming.lei@canonical.com>
*/
#ifndef __LINUX_BVEC_ITER_H
#define __LINUX_BVEC_ITER_H
#include <linux/kernel.h>
#include <linux/bug.h>
#include <linux/errno.h>
#include <linux/mm.h>
/*
* was unsigned short, but we might as well be ready for > 64kB I/O pages
*/
struct bio_vec {
struct page *bv_page;
unsigned int bv_len;
unsigned int bv_offset;
};
struct bvec_iter {
sector_t bi_sector; /* device address in 512 byte
sectors */
unsigned int bi_size; /* residual I/O count */
unsigned int bi_idx; /* current index into bvl_vec */
unsigned int bi_bvec_done; /* number of bytes completed in
current bvec */
};
struct bvec_iter_all {
struct bio_vec bv;
int idx;
unsigned done;
};
/*
* various member access, note that bio_data should of course not be used
* on highmem page vectors
*/
#define __bvec_iter_bvec(bvec, iter) (&(bvec)[(iter).bi_idx])
/* multi-page (mp_bvec) helpers */
#define mp_bvec_iter_page(bvec, iter) \
(__bvec_iter_bvec((bvec), (iter))->bv_page)
#define mp_bvec_iter_len(bvec, iter) \
min((iter).bi_size, \
__bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done)
#define mp_bvec_iter_offset(bvec, iter) \
(__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done)
#define mp_bvec_iter_page_idx(bvec, iter) \
(mp_bvec_iter_offset((bvec), (iter)) / PAGE_SIZE)
#define mp_bvec_iter_bvec(bvec, iter) \
((struct bio_vec) { \
.bv_page = mp_bvec_iter_page((bvec), (iter)), \
.bv_len = mp_bvec_iter_len((bvec), (iter)), \
.bv_offset = mp_bvec_iter_offset((bvec), (iter)), \
})
/* For building single-page bvec in flight */
#define bvec_iter_offset(bvec, iter) \
(mp_bvec_iter_offset((bvec), (iter)) % PAGE_SIZE)
#define bvec_iter_len(bvec, iter) \
min_t(unsigned, mp_bvec_iter_len((bvec), (iter)), \
PAGE_SIZE - bvec_iter_offset((bvec), (iter)))
#define bvec_iter_page(bvec, iter) \
(mp_bvec_iter_page((bvec), (iter)) + \
mp_bvec_iter_page_idx((bvec), (iter)))
#define bvec_iter_bvec(bvec, iter) \
((struct bio_vec) { \
.bv_page = bvec_iter_page((bvec), (iter)), \
.bv_len = bvec_iter_len((bvec), (iter)), \
.bv_offset = bvec_iter_offset((bvec), (iter)), \
})
static inline bool bvec_iter_advance(const struct bio_vec *bv,
struct bvec_iter *iter, unsigned bytes)
{
if (WARN_ONCE(bytes > iter->bi_size,
"Attempted to advance past end of bvec iter\n")) {
iter->bi_size = 0;
return false;
}
while (bytes) {
const struct bio_vec *cur = bv + iter->bi_idx;
unsigned len = min3(bytes, iter->bi_size,
cur->bv_len - iter->bi_bvec_done);
bytes -= len;
iter->bi_size -= len;
iter->bi_bvec_done += len;
if (iter->bi_bvec_done == cur->bv_len) {
iter->bi_bvec_done = 0;
iter->bi_idx++;
}
}
return true;
}
#define for_each_bvec(bvl, bio_vec, iter, start) \
for (iter = (start); \
(iter).bi_size && \
((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \
bvec_iter_advance((bio_vec), &(iter), (bvl).bv_len))
/* for iterating one bio from start to end */
#define BVEC_ITER_ALL_INIT (struct bvec_iter) \
{ \
.bi_sector = 0, \
.bi_size = UINT_MAX, \
.bi_idx = 0, \
.bi_bvec_done = 0, \
}
static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all)
{
iter_all->done = 0;
iter_all->idx = 0;
return &iter_all->bv;
}
static inline struct page *bvec_nth_page(struct page *page, int idx)
{
return idx == 0 ? page : nth_page(page, idx);
}
static inline void bvec_advance(const struct bio_vec *bvec,
struct bvec_iter_all *iter_all)
{
struct bio_vec *bv = &iter_all->bv;
if (iter_all->done) {
bv->bv_page++;
bv->bv_offset = 0;
} else {
bv->bv_page = bvec_nth_page(bvec->bv_page, bvec->bv_offset /
PAGE_SIZE);
bv->bv_offset = bvec->bv_offset & ~PAGE_MASK;
}
bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset,
bvec->bv_len - iter_all->done);
iter_all->done += bv->bv_len;
if (iter_all->done == bvec->bv_len) {
iter_all->idx++;
iter_all->done = 0;
}
}
/*
* Get the last single-page segment from the multi-page bvec and store it
* in @seg
*/
static inline void mp_bvec_last_segment(const struct bio_vec *bvec,
struct bio_vec *seg)
{
unsigned total = bvec->bv_offset + bvec->bv_len;
unsigned last_page = (total - 1) / PAGE_SIZE;
seg->bv_page = bvec->bv_page + last_page;
/* the whole segment is inside the last page */
if (bvec->bv_offset >= last_page * PAGE_SIZE) {
seg->bv_offset = bvec->bv_offset % PAGE_SIZE;
seg->bv_len = bvec->bv_len;
} else {
seg->bv_offset = 0;
seg->bv_len = total - last_page * PAGE_SIZE;
}
}
#endif /* __LINUX_BVEC_ITER_H */
|
