Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0
2 : : /*
3 : : * Copyright (C) 1991, 1992 Linus Torvalds
4 : : * Copyright (C) 1994, Karl Keyte: Added support for disk statistics
5 : : * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
6 : : * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
7 : : * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
8 : : * - July2000
9 : : * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
10 : : */
11 : :
12 : : /*
13 : : * This handles all read/write requests to block devices
14 : : */
15 : : #include <linux/kernel.h>
16 : : #include <linux/module.h>
17 : : #include <linux/backing-dev.h>
18 : : #include <linux/bio.h>
19 : : #include <linux/blkdev.h>
20 : : #include <linux/blk-mq.h>
21 : : #include <linux/highmem.h>
22 : : #include <linux/mm.h>
23 : : #include <linux/kernel_stat.h>
24 : : #include <linux/string.h>
25 : : #include <linux/init.h>
26 : : #include <linux/completion.h>
27 : : #include <linux/slab.h>
28 : : #include <linux/swap.h>
29 : : #include <linux/writeback.h>
30 : : #include <linux/task_io_accounting_ops.h>
31 : : #include <linux/fault-inject.h>
32 : : #include <linux/list_sort.h>
33 : : #include <linux/delay.h>
34 : : #include <linux/ratelimit.h>
35 : : #include <linux/pm_runtime.h>
36 : : #include <linux/blk-cgroup.h>
37 : : #include <linux/t10-pi.h>
38 : : #include <linux/debugfs.h>
39 : : #include <linux/bpf.h>
40 : : #include <linux/psi.h>
41 : :
42 : : #define CREATE_TRACE_POINTS
43 : : #include <trace/events/block.h>
44 : :
45 : : #include "blk.h"
46 : : #include "blk-mq.h"
47 : : #include "blk-mq-sched.h"
48 : : #include "blk-pm.h"
49 : : #include "blk-rq-qos.h"
50 : :
51 : : #ifdef CONFIG_DEBUG_FS
52 : : struct dentry *blk_debugfs_root;
53 : : #endif
54 : :
55 : : EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
56 : : EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
57 : : EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
58 : : EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
59 : : EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
60 : :
61 : : DEFINE_IDA(blk_queue_ida);
62 : :
63 : : /*
64 : : * For queue allocation
65 : : */
66 : : struct kmem_cache *blk_requestq_cachep;
67 : :
68 : : /*
69 : : * Controlling structure to kblockd
70 : : */
71 : : static struct workqueue_struct *kblockd_workqueue;
72 : :
73 : : /**
74 : : * blk_queue_flag_set - atomically set a queue flag
75 : : * @flag: flag to be set
76 : : * @q: request queue
77 : : */
78 : 17595 : void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
79 : : {
80 : 17595 : set_bit(flag, &q->queue_flags);
81 : 17595 : }
82 : : EXPORT_SYMBOL(blk_queue_flag_set);
83 : :
84 : : /**
85 : : * blk_queue_flag_clear - atomically clear a queue flag
86 : : * @flag: flag to be cleared
87 : : * @q: request queue
88 : : */
89 : 5382 : void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
90 : : {
91 : 5382 : clear_bit(flag, &q->queue_flags);
92 : 5382 : }
93 : : EXPORT_SYMBOL(blk_queue_flag_clear);
94 : :
95 : : /**
96 : : * blk_queue_flag_test_and_set - atomically test and set a queue flag
97 : : * @flag: flag to be set
98 : : * @q: request queue
99 : : *
100 : : * Returns the previous value of @flag - 0 if the flag was not set and 1 if
101 : : * the flag was already set.
102 : : */
103 : 0 : bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q)
104 : : {
105 : 0 : return test_and_set_bit(flag, &q->queue_flags);
106 : : }
107 : : EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set);
108 : :
109 : 0 : void blk_rq_init(struct request_queue *q, struct request *rq)
110 : : {
111 : 0 : memset(rq, 0, sizeof(*rq));
112 : :
113 : 0 : INIT_LIST_HEAD(&rq->queuelist);
114 : 0 : rq->q = q;
115 : 0 : rq->__sector = (sector_t) -1;
116 : : INIT_HLIST_NODE(&rq->hash);
117 : 0 : RB_CLEAR_NODE(&rq->rb_node);
118 : 0 : rq->tag = -1;
119 : 0 : rq->internal_tag = -1;
120 : 0 : rq->start_time_ns = ktime_get_ns();
121 : 0 : rq->part = NULL;
122 : : refcount_set(&rq->ref, 1);
123 : 0 : }
124 : : EXPORT_SYMBOL(blk_rq_init);
125 : :
126 : : #define REQ_OP_NAME(name) [REQ_OP_##name] = #name
127 : : static const char *const blk_op_name[] = {
128 : : REQ_OP_NAME(READ),
129 : : REQ_OP_NAME(WRITE),
130 : : REQ_OP_NAME(FLUSH),
131 : : REQ_OP_NAME(DISCARD),
132 : : REQ_OP_NAME(SECURE_ERASE),
133 : : REQ_OP_NAME(ZONE_RESET),
134 : : REQ_OP_NAME(ZONE_RESET_ALL),
135 : : REQ_OP_NAME(WRITE_SAME),
136 : : REQ_OP_NAME(WRITE_ZEROES),
137 : : REQ_OP_NAME(SCSI_IN),
138 : : REQ_OP_NAME(SCSI_OUT),
139 : : REQ_OP_NAME(DRV_IN),
140 : : REQ_OP_NAME(DRV_OUT),
141 : : };
142 : : #undef REQ_OP_NAME
143 : :
144 : : /**
145 : : * blk_op_str - Return string XXX in the REQ_OP_XXX.
146 : : * @op: REQ_OP_XXX.
147 : : *
148 : : * Description: Centralize block layer function to convert REQ_OP_XXX into
149 : : * string format. Useful in the debugging and tracing bio or request. For
150 : : * invalid REQ_OP_XXX it returns string "UNKNOWN".
151 : : */
152 : 0 : inline const char *blk_op_str(unsigned int op)
153 : : {
154 : : const char *op_str = "UNKNOWN";
155 : :
156 [ # # # # : 0 : if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op])
# # # # ]
157 : : op_str = blk_op_name[op];
158 : :
159 : 0 : return op_str;
160 : : }
161 : : EXPORT_SYMBOL_GPL(blk_op_str);
162 : :
163 : : static const struct {
164 : : int errno;
165 : : const char *name;
166 : : } blk_errors[] = {
167 : : [BLK_STS_OK] = { 0, "" },
168 : : [BLK_STS_NOTSUPP] = { -EOPNOTSUPP, "operation not supported" },
169 : : [BLK_STS_TIMEOUT] = { -ETIMEDOUT, "timeout" },
170 : : [BLK_STS_NOSPC] = { -ENOSPC, "critical space allocation" },
171 : : [BLK_STS_TRANSPORT] = { -ENOLINK, "recoverable transport" },
172 : : [BLK_STS_TARGET] = { -EREMOTEIO, "critical target" },
173 : : [BLK_STS_NEXUS] = { -EBADE, "critical nexus" },
174 : : [BLK_STS_MEDIUM] = { -ENODATA, "critical medium" },
175 : : [BLK_STS_PROTECTION] = { -EILSEQ, "protection" },
176 : : [BLK_STS_RESOURCE] = { -ENOMEM, "kernel resource" },
177 : : [BLK_STS_DEV_RESOURCE] = { -EBUSY, "device resource" },
178 : : [BLK_STS_AGAIN] = { -EAGAIN, "nonblocking retry" },
179 : :
180 : : /* device mapper special case, should not leak out: */
181 : : [BLK_STS_DM_REQUEUE] = { -EREMCHG, "dm internal retry" },
182 : :
183 : : /* everything else not covered above: */
184 : : [BLK_STS_IOERR] = { -EIO, "I/O" },
185 : : };
186 : :
187 : 0 : blk_status_t errno_to_blk_status(int errno)
188 : : {
189 : : int i;
190 : :
191 [ # # ]: 0 : for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
192 [ # # ]: 0 : if (blk_errors[i].errno == errno)
193 : 0 : return (__force blk_status_t)i;
194 : : }
195 : :
196 : : return BLK_STS_IOERR;
197 : : }
198 : : EXPORT_SYMBOL_GPL(errno_to_blk_status);
199 : :
200 : 1743591 : int blk_status_to_errno(blk_status_t status)
201 : : {
202 : 1743591 : int idx = (__force int)status;
203 : :
204 [ - + # # : 1743591 : if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
+ - ]
205 : : return -EIO;
206 : 1743588 : return blk_errors[idx].errno;
207 : : }
208 : : EXPORT_SYMBOL_GPL(blk_status_to_errno);
209 : :
210 : 0 : static void print_req_error(struct request *req, blk_status_t status,
211 : : const char *caller)
212 : : {
213 : 0 : int idx = (__force int)status;
214 : :
215 [ # # # # : 0 : if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
# # ]
216 : 0 : return;
217 : :
218 [ # # # # ]: 0 : printk_ratelimited(KERN_ERR
219 : : "%s: %s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
220 : : "phys_seg %u prio class %u\n",
221 : : caller, blk_errors[idx].name,
222 : : req->rq_disk ? req->rq_disk->disk_name : "?",
223 : : blk_rq_pos(req), req_op(req), blk_op_str(req_op(req)),
224 : : req->cmd_flags & ~REQ_OP_MASK,
225 : : req->nr_phys_segments,
226 : : IOPRIO_PRIO_CLASS(req->ioprio));
227 : : }
228 : :
229 : 2733451 : static void req_bio_endio(struct request *rq, struct bio *bio,
230 : : unsigned int nbytes, blk_status_t error)
231 : : {
232 [ - + ]: 2733451 : if (error)
233 : 0 : bio->bi_status = error;
234 : :
235 [ - + ]: 2733451 : if (unlikely(rq->rq_flags & RQF_QUIET))
236 : : bio_set_flag(bio, BIO_QUIET);
237 : :
238 : 2733451 : bio_advance(bio, nbytes);
239 : :
240 : : /* don't actually finish bio if it's part of flush sequence */
241 [ + - + + ]: 2733449 : if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
242 : 2733451 : bio_endio(bio);
243 : 2733449 : }
244 : :
245 : 0 : void blk_dump_rq_flags(struct request *rq, char *msg)
246 : : {
247 [ # # ]: 0 : printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
248 : 0 : rq->rq_disk ? rq->rq_disk->disk_name : "?",
249 : 0 : (unsigned long long) rq->cmd_flags);
250 : :
251 : 0 : printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
252 : : (unsigned long long)blk_rq_pos(rq),
253 : : blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
254 : 0 : printk(KERN_INFO " bio %p, biotail %p, len %u\n",
255 : : rq->bio, rq->biotail, blk_rq_bytes(rq));
256 : 0 : }
257 : : EXPORT_SYMBOL(blk_dump_rq_flags);
258 : :
259 : : /**
260 : : * blk_sync_queue - cancel any pending callbacks on a queue
261 : : * @q: the queue
262 : : *
263 : : * Description:
264 : : * The block layer may perform asynchronous callback activity
265 : : * on a queue, such as calling the unplug function after a timeout.
266 : : * A block device may call blk_sync_queue to ensure that any
267 : : * such activity is cancelled, thus allowing it to release resources
268 : : * that the callbacks might use. The caller must already have made sure
269 : : * that its ->make_request_fn will not re-add plugging prior to calling
270 : : * this function.
271 : : *
272 : : * This function does not cancel any asynchronous activity arising
273 : : * out of elevator or throttling code. That would require elevator_exit()
274 : : * and blkcg_exit_queue() to be called with queue lock initialized.
275 : : *
276 : : */
277 : 0 : void blk_sync_queue(struct request_queue *q)
278 : : {
279 : 0 : del_timer_sync(&q->timeout);
280 : 0 : cancel_work_sync(&q->timeout_work);
281 : 0 : }
282 : : EXPORT_SYMBOL(blk_sync_queue);
283 : :
284 : : /**
285 : : * blk_set_pm_only - increment pm_only counter
286 : : * @q: request queue pointer
287 : : */
288 : 0 : void blk_set_pm_only(struct request_queue *q)
289 : : {
290 : 0 : atomic_inc(&q->pm_only);
291 : 0 : }
292 : : EXPORT_SYMBOL_GPL(blk_set_pm_only);
293 : :
294 : 0 : void blk_clear_pm_only(struct request_queue *q)
295 : : {
296 : : int pm_only;
297 : :
298 : 0 : pm_only = atomic_dec_return(&q->pm_only);
299 [ # # # # ]: 0 : WARN_ON_ONCE(pm_only < 0);
300 [ # # ]: 0 : if (pm_only == 0)
301 : 0 : wake_up_all(&q->mq_freeze_wq);
302 : 0 : }
303 : : EXPORT_SYMBOL_GPL(blk_clear_pm_only);
304 : :
305 : 0 : void blk_put_queue(struct request_queue *q)
306 : : {
307 : 0 : kobject_put(&q->kobj);
308 : 0 : }
309 : : EXPORT_SYMBOL(blk_put_queue);
310 : :
311 : 0 : void blk_set_queue_dying(struct request_queue *q)
312 : : {
313 : : blk_queue_flag_set(QUEUE_FLAG_DYING, q);
314 : :
315 : : /*
316 : : * When queue DYING flag is set, we need to block new req
317 : : * entering queue, so we call blk_freeze_queue_start() to
318 : : * prevent I/O from crossing blk_queue_enter().
319 : : */
320 : 0 : blk_freeze_queue_start(q);
321 : :
322 [ # # ]: 0 : if (queue_is_mq(q))
323 : 0 : blk_mq_wake_waiters(q);
324 : :
325 : : /* Make blk_queue_enter() reexamine the DYING flag. */
326 : 0 : wake_up_all(&q->mq_freeze_wq);
327 : 0 : }
328 : : EXPORT_SYMBOL_GPL(blk_set_queue_dying);
329 : :
330 : : /**
331 : : * blk_cleanup_queue - shutdown a request queue
332 : : * @q: request queue to shutdown
333 : : *
334 : : * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
335 : : * put it. All future requests will be failed immediately with -ENODEV.
336 : : */
337 : 0 : void blk_cleanup_queue(struct request_queue *q)
338 : : {
339 : : /* mark @q DYING, no new request or merges will be allowed afterwards */
340 : 0 : mutex_lock(&q->sysfs_lock);
341 : 0 : blk_set_queue_dying(q);
342 : :
343 : : blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
344 : : blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
345 : : blk_queue_flag_set(QUEUE_FLAG_DYING, q);
346 : 0 : mutex_unlock(&q->sysfs_lock);
347 : :
348 : : /*
349 : : * Drain all requests queued before DYING marking. Set DEAD flag to
350 : : * prevent that blk_mq_run_hw_queues() accesses the hardware queues
351 : : * after draining finished.
352 : : */
353 : 0 : blk_freeze_queue(q);
354 : :
355 : 0 : rq_qos_exit(q);
356 : :
357 : : blk_queue_flag_set(QUEUE_FLAG_DEAD, q);
358 : :
359 : : /* for synchronous bio-based driver finish in-flight integrity i/o */
360 : : blk_flush_integrity();
361 : :
362 : : /* @q won't process any more request, flush async actions */
363 : 0 : del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer);
364 : : blk_sync_queue(q);
365 : :
366 [ # # ]: 0 : if (queue_is_mq(q))
367 : 0 : blk_mq_exit_queue(q);
368 : :
369 : : /*
370 : : * In theory, request pool of sched_tags belongs to request queue.
371 : : * However, the current implementation requires tag_set for freeing
372 : : * requests, so free the pool now.
373 : : *
374 : : * Queue has become frozen, there can't be any in-queue requests, so
375 : : * it is safe to free requests now.
376 : : */
377 : 0 : mutex_lock(&q->sysfs_lock);
378 [ # # ]: 0 : if (q->elevator)
379 : 0 : blk_mq_sched_free_requests(q);
380 : 0 : mutex_unlock(&q->sysfs_lock);
381 : :
382 : 0 : percpu_ref_exit(&q->q_usage_counter);
383 : :
384 : : /* @q is and will stay empty, shutdown and put */
385 : : blk_put_queue(q);
386 : 0 : }
387 : : EXPORT_SYMBOL(blk_cleanup_queue);
388 : :
389 : 3312 : struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
390 : : {
391 : 3312 : return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE);
392 : : }
393 : : EXPORT_SYMBOL(blk_alloc_queue);
394 : :
395 : : /**
396 : : * blk_queue_enter() - try to increase q->q_usage_counter
397 : : * @q: request queue pointer
398 : : * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PREEMPT
399 : : */
400 : 2832341 : int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
401 : : {
402 : 2832341 : const bool pm = flags & BLK_MQ_REQ_PREEMPT;
403 : :
404 : : while (true) {
405 : : bool success = false;
406 : :
407 : : rcu_read_lock();
408 [ + - ]: 2830267 : if (percpu_ref_tryget_live(&q->q_usage_counter)) {
409 : : /*
410 : : * The code that increments the pm_only counter is
411 : : * responsible for ensuring that that counter is
412 : : * globally visible before the queue is unfrozen.
413 : : */
414 [ + + - + ]: 5662152 : if (pm || !blk_queue_pm_only(q)) {
415 : : success = true;
416 : : } else {
417 : : percpu_ref_put(&q->q_usage_counter);
418 : : }
419 : : }
420 : : rcu_read_unlock();
421 : :
422 [ - + ]: 2832798 : if (success)
423 : : return 0;
424 : :
425 [ # # ]: 0 : if (flags & BLK_MQ_REQ_NOWAIT)
426 : : return -EBUSY;
427 : :
428 : : /*
429 : : * read pair of barrier in blk_freeze_queue_start(),
430 : : * we need to order reading __PERCPU_REF_DEAD flag of
431 : : * .q_usage_counter and reading .mq_freeze_depth or
432 : : * queue dying flag, otherwise the following wait may
433 : : * never return if the two reads are reordered.
434 : : */
435 : 0 : smp_rmb();
436 : :
437 [ # # # # : 0 : wait_event(q->mq_freeze_wq,
# # # # #
# # # # #
# # ]
438 : : (!q->mq_freeze_depth &&
439 : : (pm || (blk_pm_request_resume(q),
440 : : !blk_queue_pm_only(q)))) ||
441 : : blk_queue_dying(q));
442 [ # # ]: 0 : if (blk_queue_dying(q))
443 : : return -ENODEV;
444 : : }
445 : : }
446 : :
447 : 1262104 : void blk_queue_exit(struct request_queue *q)
448 : : {
449 : 3994220 : percpu_ref_put(&q->q_usage_counter);
450 : 1262099 : }
451 : :
452 : 1863 : static void blk_queue_usage_counter_release(struct percpu_ref *ref)
453 : : {
454 : : struct request_queue *q =
455 : : container_of(ref, struct request_queue, q_usage_counter);
456 : :
457 : 1863 : wake_up_all(&q->mq_freeze_wq);
458 : 1863 : }
459 : :
460 : 2571 : static void blk_rq_timed_out_timer(struct timer_list *t)
461 : : {
462 : : struct request_queue *q = from_timer(q, t, timeout);
463 : :
464 : 2571 : kblockd_schedule_work(&q->timeout_work);
465 : 2571 : }
466 : :
467 : 0 : static void blk_timeout_work(struct work_struct *work)
468 : : {
469 : 0 : }
470 : :
471 : : /**
472 : : * blk_alloc_queue_node - allocate a request queue
473 : : * @gfp_mask: memory allocation flags
474 : : * @node_id: NUMA node to allocate memory from
475 : : */
476 : 5175 : struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
477 : : {
478 : : struct request_queue *q;
479 : : int ret;
480 : :
481 : 5175 : q = kmem_cache_alloc_node(blk_requestq_cachep,
482 : : gfp_mask | __GFP_ZERO, node_id);
483 [ + - ]: 5175 : if (!q)
484 : : return NULL;
485 : :
486 : 5175 : q->last_merge = NULL;
487 : :
488 : 5175 : q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
489 [ + - ]: 5175 : if (q->id < 0)
490 : : goto fail_q;
491 : :
492 : 5175 : ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
493 [ + - ]: 5175 : if (ret)
494 : : goto fail_id;
495 : :
496 : 5175 : q->backing_dev_info = bdi_alloc_node(gfp_mask, node_id);
497 [ + - ]: 5175 : if (!q->backing_dev_info)
498 : : goto fail_split;
499 : :
500 : 5175 : q->stats = blk_alloc_queue_stats();
501 [ + - ]: 5175 : if (!q->stats)
502 : : goto fail_stats;
503 : :
504 : 5175 : q->backing_dev_info->ra_pages = VM_READAHEAD_PAGES;
505 : 5175 : q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK;
506 : 5175 : q->backing_dev_info->name = "block";
507 : 5175 : q->node = node_id;
508 : :
509 : 5175 : timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,
510 : : laptop_mode_timer_fn, 0);
511 : 5175 : timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
512 : 10350 : INIT_WORK(&q->timeout_work, blk_timeout_work);
513 : 5175 : INIT_LIST_HEAD(&q->icq_list);
514 : : #ifdef CONFIG_BLK_CGROUP
515 : 5175 : INIT_LIST_HEAD(&q->blkg_list);
516 : : #endif
517 : :
518 : 5175 : kobject_init(&q->kobj, &blk_queue_ktype);
519 : :
520 : : #ifdef CONFIG_BLK_DEV_IO_TRACE
521 : 5175 : mutex_init(&q->blk_trace_mutex);
522 : : #endif
523 : 5175 : mutex_init(&q->sysfs_lock);
524 : 5175 : mutex_init(&q->sysfs_dir_lock);
525 : 5175 : spin_lock_init(&q->queue_lock);
526 : :
527 : 5175 : init_waitqueue_head(&q->mq_freeze_wq);
528 : 5175 : mutex_init(&q->mq_freeze_lock);
529 : :
530 : : /*
531 : : * Init percpu_ref in atomic mode so that it's faster to shutdown.
532 : : * See blk_register_queue() for details.
533 : : */
534 [ + - ]: 5175 : if (percpu_ref_init(&q->q_usage_counter,
535 : : blk_queue_usage_counter_release,
536 : : PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
537 : : goto fail_bdi;
538 : :
539 [ - + ]: 5175 : if (blkcg_init_queue(q))
540 : : goto fail_ref;
541 : :
542 : : return q;
543 : :
544 : : fail_ref:
545 : 0 : percpu_ref_exit(&q->q_usage_counter);
546 : : fail_bdi:
547 : 0 : blk_free_queue_stats(q->stats);
548 : : fail_stats:
549 : 0 : bdi_put(q->backing_dev_info);
550 : : fail_split:
551 : 0 : bioset_exit(&q->bio_split);
552 : : fail_id:
553 : 0 : ida_simple_remove(&blk_queue_ida, q->id);
554 : : fail_q:
555 : 0 : kmem_cache_free(blk_requestq_cachep, q);
556 : 0 : return NULL;
557 : : }
558 : : EXPORT_SYMBOL(blk_alloc_queue_node);
559 : :
560 : 5382 : bool blk_get_queue(struct request_queue *q)
561 : : {
562 [ + - ]: 5382 : if (likely(!blk_queue_dying(q))) {
563 : : __blk_get_queue(q);
564 : 5382 : return true;
565 : : }
566 : :
567 : : return false;
568 : : }
569 : : EXPORT_SYMBOL(blk_get_queue);
570 : :
571 : : /**
572 : : * blk_get_request - allocate a request
573 : : * @q: request queue to allocate a request for
574 : : * @op: operation (REQ_OP_*) and REQ_* flags, e.g. REQ_SYNC.
575 : : * @flags: BLK_MQ_REQ_* flags, e.g. BLK_MQ_REQ_NOWAIT.
576 : : */
577 : 0 : struct request *blk_get_request(struct request_queue *q, unsigned int op,
578 : : blk_mq_req_flags_t flags)
579 : : {
580 : : struct request *req;
581 : :
582 [ # # # # ]: 0 : WARN_ON_ONCE(op & REQ_NOWAIT);
583 [ # # # # ]: 0 : WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PREEMPT));
584 : :
585 : 0 : req = blk_mq_alloc_request(q, op, flags);
586 [ # # # # ]: 0 : if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)
587 : 0 : q->mq_ops->initialize_rq_fn(req);
588 : :
589 : 0 : return req;
590 : : }
591 : : EXPORT_SYMBOL(blk_get_request);
592 : :
593 : 24140 : void blk_put_request(struct request *req)
594 : : {
595 : 24140 : blk_mq_free_request(req);
596 : 24140 : }
597 : : EXPORT_SYMBOL(blk_put_request);
598 : :
599 : 1548955 : bool bio_attempt_back_merge(struct request *req, struct bio *bio,
600 : : unsigned int nr_segs)
601 : : {
602 : 1548955 : const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
603 : :
604 [ + + ]: 1548955 : if (!ll_back_merge_fn(req, bio, nr_segs))
605 : : return false;
606 : :
607 : 1548401 : trace_block_bio_backmerge(req->q, req, bio);
608 : 1548436 : rq_qos_merge(req->q, req, bio);
609 : :
610 [ + + ]: 1548394 : if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
611 : 1383317 : blk_rq_set_mixed_merge(req);
612 : :
613 : 1548437 : req->biotail->bi_next = bio;
614 : 1548437 : req->biotail = bio;
615 : 1548437 : req->__data_len += bio->bi_iter.bi_size;
616 : :
617 : 1548437 : blk_account_io_start(req, false);
618 : 1548438 : return true;
619 : : }
620 : :
621 : 24148 : bool bio_attempt_front_merge(struct request *req, struct bio *bio,
622 : : unsigned int nr_segs)
623 : : {
624 : 24148 : const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
625 : :
626 [ + + ]: 24148 : if (!ll_front_merge_fn(req, bio, nr_segs))
627 : : return false;
628 : :
629 : 24147 : trace_block_bio_frontmerge(req->q, req, bio);
630 : 24147 : rq_qos_merge(req->q, req, bio);
631 : :
632 [ + + ]: 24147 : if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
633 : 23396 : blk_rq_set_mixed_merge(req);
634 : :
635 : 24148 : bio->bi_next = req->bio;
636 : 24148 : req->bio = bio;
637 : :
638 : 24148 : req->__sector = bio->bi_iter.bi_sector;
639 : 24148 : req->__data_len += bio->bi_iter.bi_size;
640 : :
641 : 24148 : blk_account_io_start(req, false);
642 : 24148 : return true;
643 : : }
644 : :
645 : 0 : bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
646 : : struct bio *bio)
647 : : {
648 : : unsigned short segments = blk_rq_nr_discard_segments(req);
649 : :
650 [ # # ]: 0 : if (segments >= queue_max_discard_segments(q))
651 : : goto no_merge;
652 [ # # ]: 0 : if (blk_rq_sectors(req) + bio_sectors(bio) >
653 : 0 : blk_rq_get_max_sectors(req, blk_rq_pos(req)))
654 : : goto no_merge;
655 : :
656 : : rq_qos_merge(q, req, bio);
657 : :
658 : 0 : req->biotail->bi_next = bio;
659 : 0 : req->biotail = bio;
660 : 0 : req->__data_len += bio->bi_iter.bi_size;
661 : 0 : req->nr_phys_segments = segments + 1;
662 : :
663 : 0 : blk_account_io_start(req, false);
664 : 0 : return true;
665 : : no_merge:
666 : : req_set_nomerge(q, req);
667 : : return false;
668 : : }
669 : :
670 : : /**
671 : : * blk_attempt_plug_merge - try to merge with %current's plugged list
672 : : * @q: request_queue new bio is being queued at
673 : : * @bio: new bio being queued
674 : : * @nr_segs: number of segments in @bio
675 : : * @same_queue_rq: pointer to &struct request that gets filled in when
676 : : * another request associated with @q is found on the plug list
677 : : * (optional, may be %NULL)
678 : : *
679 : : * Determine whether @bio being queued on @q can be merged with a request
680 : : * on %current's plugged list. Returns %true if merge was successful,
681 : : * otherwise %false.
682 : : *
683 : : * Plugging coalesces IOs from the same issuer for the same purpose without
684 : : * going through @q->queue_lock. As such it's more of an issuing mechanism
685 : : * than scheduling, and the request, while may have elvpriv data, is not
686 : : * added on the elevator at this point. In addition, we don't have
687 : : * reliable access to the elevator outside queue lock. Only check basic
688 : : * merging parameters without querying the elevator.
689 : : *
690 : : * Caller must ensure !blk_queue_nomerges(q) beforehand.
691 : : */
692 : 2733266 : bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
693 : : unsigned int nr_segs, struct request **same_queue_rq)
694 : : {
695 : : struct blk_plug *plug;
696 : : struct request *rq;
697 : : struct list_head *plug_list;
698 : :
699 : : plug = blk_mq_plug(q, bio);
700 [ + + ]: 2733266 : if (!plug)
701 : : return false;
702 : :
703 : 1643415 : plug_list = &plug->mq_list;
704 : :
705 [ + + ]: 1980959 : list_for_each_entry_reverse(rq, plug_list, queuelist) {
706 : : bool merged = false;
707 : :
708 [ + + + + ]: 1221182 : if (rq->q == q && same_queue_rq) {
709 : : /*
710 : : * Only blk-mq multiple hardware queues case checks the
711 : : * rq in the same queue, there should be only one such
712 : : * rq in a queue
713 : : **/
714 : 1221217 : *same_queue_rq = rq;
715 : : }
716 : :
717 [ + + + + ]: 1221182 : if (rq->q != q || !blk_rq_merge_ok(rq, bio))
718 : 1804 : continue;
719 : :
720 [ + + - + ]: 1219419 : switch (blk_try_merge(rq, bio)) {
721 : : case ELEVATOR_BACK_MERGE:
722 : 859924 : merged = bio_attempt_back_merge(rq, bio, nr_segs);
723 : 859990 : break;
724 : : case ELEVATOR_FRONT_MERGE:
725 : 23932 : merged = bio_attempt_front_merge(rq, bio, nr_segs);
726 : 23932 : break;
727 : : case ELEVATOR_DISCARD_MERGE:
728 : 0 : merged = bio_attempt_discard_merge(q, rq, bio);
729 : 0 : break;
730 : : default:
731 : : break;
732 : : }
733 : :
734 [ + + ]: 1219478 : if (merged)
735 : : return true;
736 : : }
737 : :
738 : : return false;
739 : : }
740 : :
741 : 0 : static void handle_bad_sector(struct bio *bio, sector_t maxsector)
742 : : {
743 : : char b[BDEVNAME_SIZE];
744 : :
745 : 0 : printk(KERN_INFO "attempt to access beyond end of device\n");
746 : 0 : printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n",
747 : : bio_devname(bio, b), bio->bi_opf,
748 : 0 : (unsigned long long)bio_end_sector(bio),
749 : : (long long)maxsector);
750 : 0 : }
751 : :
752 : : #ifdef CONFIG_FAIL_MAKE_REQUEST
753 : :
754 : : static DECLARE_FAULT_ATTR(fail_make_request);
755 : :
756 : : static int __init setup_fail_make_request(char *str)
757 : : {
758 : : return setup_fault_attr(&fail_make_request, str);
759 : : }
760 : : __setup("fail_make_request=", setup_fail_make_request);
761 : :
762 : : static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
763 : : {
764 : : return part->make_it_fail && should_fail(&fail_make_request, bytes);
765 : : }
766 : :
767 : : static int __init fail_make_request_debugfs(void)
768 : : {
769 : : struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
770 : : NULL, &fail_make_request);
771 : :
772 : : return PTR_ERR_OR_ZERO(dir);
773 : : }
774 : :
775 : : late_initcall(fail_make_request_debugfs);
776 : :
777 : : #else /* CONFIG_FAIL_MAKE_REQUEST */
778 : :
779 : : static inline bool should_fail_request(struct hd_struct *part,
780 : : unsigned int bytes)
781 : : {
782 : : return false;
783 : : }
784 : :
785 : : #endif /* CONFIG_FAIL_MAKE_REQUEST */
786 : :
787 : 2736869 : static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part)
788 : : {
789 : 2736869 : const int op = bio_op(bio);
790 : :
791 [ - + # # ]: 2736869 : if (part->policy && op_is_write(op)) {
792 : : char b[BDEVNAME_SIZE];
793 : :
794 [ # # # # ]: 0 : if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
795 : : return false;
796 : :
797 [ # # ]: 0 : WARN_ONCE(1,
798 : : "generic_make_request: Trying to write "
799 : : "to read-only block-device %s (partno %d)\n",
800 : : bio_devname(bio, b), part->partno);
801 : : /* Older lvm-tools actually trigger this */
802 : : return false;
803 : : }
804 : :
805 : : return false;
806 : : }
807 : :
808 : 2737899 : static noinline int should_fail_bio(struct bio *bio)
809 : : {
810 : : if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size))
811 : : return -EIO;
812 : : return 0;
813 : : }
814 : : ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);
815 : :
816 : : /*
817 : : * Check whether this bio extends beyond the end of the device or partition.
818 : : * This may well happen - the kernel calls bread() without checking the size of
819 : : * the device, e.g., when mounting a file system.
820 : : */
821 : 2736688 : static inline int bio_check_eod(struct bio *bio, sector_t maxsector)
822 : : {
823 : 2736688 : unsigned int nr_sectors = bio_sectors(bio);
824 : :
825 [ + + + + ]: 5472556 : if (nr_sectors && maxsector &&
826 [ + + ]: 5471751 : (nr_sectors > maxsector ||
827 : 2735883 : bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
828 : 75 : handle_bad_sector(bio, maxsector);
829 : 0 : return -EIO;
830 : : }
831 : : return 0;
832 : : }
833 : :
834 : : /*
835 : : * Remap block n of partition p to block n+start(p) of the disk.
836 : : */
837 : 2724522 : static inline int blk_partition_remap(struct bio *bio)
838 : : {
839 : : struct hd_struct *p;
840 : : int ret = -EIO;
841 : :
842 : : rcu_read_lock();
843 : 2724002 : p = __disk_get_part(bio->bi_disk, bio->bi_partno);
844 [ + + ]: 2724457 : if (unlikely(!p))
845 : : goto out;
846 : : if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
847 : : goto out;
848 [ + - ]: 2723292 : if (unlikely(bio_check_ro(bio, p)))
849 : : goto out;
850 : :
851 : : /*
852 : : * Zone reset does not include bi_size so bio_sectors() is always 0.
853 : : * Include a test for the reset op code and perform the remap if needed.
854 : : */
855 [ + + + + ]: 2724459 : if (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET) {
856 [ + - ]: 2722053 : if (bio_check_eod(bio, part_nr_sects_read(p)))
857 : : goto out;
858 : 2721595 : bio->bi_iter.bi_sector += p->start_sect;
859 : 5443190 : trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p),
860 : 2721595 : bio->bi_iter.bi_sector - p->start_sect);
861 : : }
862 : 2723655 : bio->bi_partno = 0;
863 : : ret = 0;
864 : : out:
865 : : rcu_read_unlock();
866 : 2723970 : return ret;
867 : : }
868 : :
869 : : static noinline_for_stack bool
870 : 2737821 : generic_make_request_checks(struct bio *bio)
871 : : {
872 : : struct request_queue *q;
873 : 2737821 : int nr_sectors = bio_sectors(bio);
874 : : blk_status_t status = BLK_STS_IOERR;
875 : : char b[BDEVNAME_SIZE];
876 : :
877 : 2737821 : might_sleep();
878 : :
879 : 2738011 : q = bio->bi_disk->queue;
880 [ - + ]: 2738011 : if (unlikely(!q)) {
881 : 0 : printk(KERN_ERR
882 : : "generic_make_request: Trying to access "
883 : : "nonexistent block-device %s (%Lu)\n",
884 : 0 : bio_devname(bio, b), (long long)bio->bi_iter.bi_sector);
885 : 0 : goto end_io;
886 : : }
887 : :
888 : : /*
889 : : * For a REQ_NOWAIT based request, return -EOPNOTSUPP
890 : : * if queue is not a request based queue.
891 : : */
892 [ - + # # ]: 2738011 : if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_mq(q))
893 : : goto not_supported;
894 : :
895 [ + + ]: 2738011 : if (should_fail_bio(bio))
896 : : goto end_io;
897 : :
898 [ + + ]: 2736726 : if (bio->bi_partno) {
899 [ + + ]: 2723269 : if (unlikely(blk_partition_remap(bio)))
900 : : goto end_io;
901 : : } else {
902 [ + - ]: 13457 : if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0)))
903 : : goto end_io;
904 [ + - ]: 26914 : if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))
905 : : goto end_io;
906 : : }
907 : :
908 : : /*
909 : : * Filter flush bio's early so that make_request based
910 : : * drivers without flush support don't have to worry
911 : : * about them.
912 : : */
913 [ + + + - ]: 5479969 : if (op_is_flush(bio->bi_opf) &&
914 : : !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
915 : 4109 : bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
916 [ + + ]: 4109 : if (!nr_sectors) {
917 : : status = BLK_STS_OK;
918 : : goto end_io;
919 : : }
920 : : }
921 : :
922 [ + + ]: 2736767 : if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
923 : 2736537 : bio->bi_opf &= ~REQ_HIPRI;
924 : :
925 [ - - - - : 2736767 : switch (bio_op(bio)) {
- - + ]
926 : : case REQ_OP_DISCARD:
927 [ # # ]: 0 : if (!blk_queue_discard(q))
928 : : goto not_supported;
929 : : break;
930 : : case REQ_OP_SECURE_ERASE:
931 [ # # ]: 0 : if (!blk_queue_secure_erase(q))
932 : : goto not_supported;
933 : : break;
934 : : case REQ_OP_WRITE_SAME:
935 [ # # ]: 0 : if (!q->limits.max_write_same_sectors)
936 : : goto not_supported;
937 : : break;
938 : : case REQ_OP_ZONE_RESET:
939 [ # # ]: 0 : if (!blk_queue_is_zoned(q))
940 : : goto not_supported;
941 : : break;
942 : : case REQ_OP_ZONE_RESET_ALL:
943 [ # # # # ]: 0 : if (!blk_queue_is_zoned(q) || !blk_queue_zone_resetall(q))
944 : : goto not_supported;
945 : : break;
946 : : case REQ_OP_WRITE_ZEROES:
947 [ # # ]: 0 : if (!q->limits.max_write_zeroes_sectors)
948 : : goto not_supported;
949 : : break;
950 : : default:
951 : : break;
952 : : }
953 : :
954 : : /*
955 : : * Various block parts want %current->io_context and lazy ioc
956 : : * allocation ends up trading a lot of pain for a small amount of
957 : : * memory. Just allocate it upfront. This may fail and block
958 : : * layer knows how to live with it.
959 : : */
960 : 2736767 : create_io_context(GFP_ATOMIC, q->node);
961 : :
962 [ + + ]: 2736295 : if (!blkcg_bio_issue_check(q, bio))
963 : : return false;
964 : :
965 [ + + ]: 2734540 : if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
966 : 2735120 : trace_block_bio_queue(q, bio);
967 : : /* Now that enqueuing has been traced, we need to trace
968 : : * completion as well.
969 : : */
970 : : bio_set_flag(bio, BIO_TRACE_COMPLETION);
971 : : }
972 : : return true;
973 : :
974 : : not_supported:
975 : : status = BLK_STS_NOTSUPP;
976 : : end_io:
977 : 1658 : bio->bi_status = status;
978 : 1658 : bio_endio(bio);
979 : 1163 : return false;
980 : : }
981 : :
982 : : /**
983 : : * generic_make_request - hand a buffer to its device driver for I/O
984 : : * @bio: The bio describing the location in memory and on the device.
985 : : *
986 : : * generic_make_request() is used to make I/O requests of block
987 : : * devices. It is passed a &struct bio, which describes the I/O that needs
988 : : * to be done.
989 : : *
990 : : * generic_make_request() does not return any status. The
991 : : * success/failure status of the request, along with notification of
992 : : * completion, is delivered asynchronously through the bio->bi_end_io
993 : : * function described (one day) else where.
994 : : *
995 : : * The caller of generic_make_request must make sure that bi_io_vec
996 : : * are set to describe the memory buffer, and that bi_dev and bi_sector are
997 : : * set to describe the device address, and the
998 : : * bi_end_io and optionally bi_private are set to describe how
999 : : * completion notification should be signaled.
1000 : : *
1001 : : * generic_make_request and the drivers it calls may use bi_next if this
1002 : : * bio happens to be merged with someone else, and may resubmit the bio to
1003 : : * a lower device by calling into generic_make_request recursively, which
1004 : : * means the bio should NOT be touched after the call to ->make_request_fn.
1005 : : */
1006 : 2737816 : blk_qc_t generic_make_request(struct bio *bio)
1007 : : {
1008 : : /*
1009 : : * bio_list_on_stack[0] contains bios submitted by the current
1010 : : * make_request_fn.
1011 : : * bio_list_on_stack[1] contains bios that were submitted before
1012 : : * the current make_request_fn, but that haven't been processed
1013 : : * yet.
1014 : : */
1015 : : struct bio_list bio_list_on_stack[2];
1016 : : blk_qc_t ret = BLK_QC_T_NONE;
1017 : :
1018 [ + + ]: 2737816 : if (!generic_make_request_checks(bio))
1019 : : goto out;
1020 : :
1021 : : /*
1022 : : * We only want one ->make_request_fn to be active at a time, else
1023 : : * stack usage with stacked devices could be a problem. So use
1024 : : * current->bio_list to keep a list of requests submited by a
1025 : : * make_request_fn function. current->bio_list is also used as a
1026 : : * flag to say if generic_make_request is currently active in this
1027 : : * task or not. If it is NULL, then no make_request is active. If
1028 : : * it is non-NULL, then a make_request is active, and new requests
1029 : : * should be added at the tail
1030 : : */
1031 [ + + ]: 2734227 : if (current->bio_list) {
1032 : : bio_list_add(¤t->bio_list[0], bio);
1033 : : goto out;
1034 : : }
1035 : :
1036 : : /* following loop may be a bit non-obvious, and so deserves some
1037 : : * explanation.
1038 : : * Before entering the loop, bio->bi_next is NULL (as all callers
1039 : : * ensure that) so we have a list with a single bio.
1040 : : * We pretend that we have just taken it off a longer list, so
1041 : : * we assign bio_list to a pointer to the bio_list_on_stack,
1042 : : * thus initialising the bio_list of new bios to be
1043 : : * added. ->make_request() may indeed add some more bios
1044 : : * through a recursive call to generic_make_request. If it
1045 : : * did, we find a non-NULL value in bio_list and re-enter the loop
1046 : : * from the top. In this case we really did just take the bio
1047 : : * of the top of the list (no pretending) and so remove it from
1048 : : * bio_list, and call into ->make_request() again.
1049 : : */
1050 [ - + ]: 2734225 : BUG_ON(bio->bi_next);
1051 : : bio_list_init(&bio_list_on_stack[0]);
1052 : 2734225 : current->bio_list = bio_list_on_stack;
1053 : : do {
1054 : 2734227 : struct request_queue *q = bio->bi_disk->queue;
1055 : 5468454 : blk_mq_req_flags_t flags = bio->bi_opf & REQ_NOWAIT ?
1056 : 2734227 : BLK_MQ_REQ_NOWAIT : 0;
1057 : :
1058 [ + - ]: 2734227 : if (likely(blk_queue_enter(q, flags) == 0)) {
1059 : : struct bio_list lower, same;
1060 : :
1061 : : /* Create a fresh bio_list for all subordinate requests */
1062 : 2734610 : bio_list_on_stack[1] = bio_list_on_stack[0];
1063 : : bio_list_init(&bio_list_on_stack[0]);
1064 : 2734610 : ret = q->make_request_fn(q, bio);
1065 : :
1066 : : blk_queue_exit(q);
1067 : :
1068 : : /* sort new bios into those for a lower level
1069 : : * and those for the same level
1070 : : */
1071 : : bio_list_init(&lower);
1072 : : bio_list_init(&same);
1073 [ + + ]: 2736557 : while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
1074 [ + - ]: 2 : if (q == bio->bi_disk->queue)
1075 : : bio_list_add(&same, bio);
1076 : : else
1077 : : bio_list_add(&lower, bio);
1078 : : /* now assemble so we handle the lowest level first */
1079 : : bio_list_merge(&bio_list_on_stack[0], &lower);
1080 : : bio_list_merge(&bio_list_on_stack[0], &same);
1081 : : bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
1082 : : } else {
1083 [ # # # # ]: 0 : if (unlikely(!blk_queue_dying(q) &&
1084 : : (bio->bi_opf & REQ_NOWAIT)))
1085 : : bio_wouldblock_error(bio);
1086 : : else
1087 : : bio_io_error(bio);
1088 : : }
1089 : : bio = bio_list_pop(&bio_list_on_stack[0]);
1090 [ + + ]: 2732665 : } while (bio);
1091 : 2732663 : current->bio_list = NULL; /* deactivate */
1092 : :
1093 : : out:
1094 : 2736156 : return ret;
1095 : : }
1096 : : EXPORT_SYMBOL(generic_make_request);
1097 : :
1098 : : /**
1099 : : * direct_make_request - hand a buffer directly to its device driver for I/O
1100 : : * @bio: The bio describing the location in memory and on the device.
1101 : : *
1102 : : * This function behaves like generic_make_request(), but does not protect
1103 : : * against recursion. Must only be used if the called driver is known
1104 : : * to not call generic_make_request (or direct_make_request) again from
1105 : : * its make_request function. (Calling direct_make_request again from
1106 : : * a workqueue is perfectly fine as that doesn't recurse).
1107 : : */
1108 : 0 : blk_qc_t direct_make_request(struct bio *bio)
1109 : : {
1110 : 0 : struct request_queue *q = bio->bi_disk->queue;
1111 : 0 : bool nowait = bio->bi_opf & REQ_NOWAIT;
1112 : : blk_qc_t ret;
1113 : :
1114 [ # # ]: 0 : if (!generic_make_request_checks(bio))
1115 : : return BLK_QC_T_NONE;
1116 : :
1117 [ # # ]: 0 : if (unlikely(blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0))) {
1118 [ # # # # ]: 0 : if (nowait && !blk_queue_dying(q))
1119 : 0 : bio->bi_status = BLK_STS_AGAIN;
1120 : : else
1121 : 0 : bio->bi_status = BLK_STS_IOERR;
1122 : 0 : bio_endio(bio);
1123 : 0 : return BLK_QC_T_NONE;
1124 : : }
1125 : :
1126 : 0 : ret = q->make_request_fn(q, bio);
1127 : : blk_queue_exit(q);
1128 : 0 : return ret;
1129 : : }
1130 : : EXPORT_SYMBOL_GPL(direct_make_request);
1131 : :
1132 : : /**
1133 : : * submit_bio - submit a bio to the block device layer for I/O
1134 : : * @bio: The &struct bio which describes the I/O
1135 : : *
1136 : : * submit_bio() is very similar in purpose to generic_make_request(), and
1137 : : * uses that function to do most of the work. Both are fairly rough
1138 : : * interfaces; @bio must be presetup and ready for I/O.
1139 : : *
1140 : : */
1141 : 2737921 : blk_qc_t submit_bio(struct bio *bio)
1142 : : {
1143 : : bool workingset_read = false;
1144 : : unsigned long pflags;
1145 : : blk_qc_t ret;
1146 : :
1147 [ + - ]: 2737789 : if (blkcg_punt_bio_submit(bio))
1148 : : return BLK_QC_T_NONE;
1149 : :
1150 : : /*
1151 : : * If it's a regular read/write or a barrier with data attached,
1152 : : * go through the normal accounting stuff before submission.
1153 : : */
1154 [ + + ]: 2737789 : if (bio_has_data(bio)) {
1155 : : unsigned int count;
1156 : :
1157 [ + + ]: 2736858 : if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
1158 : 496 : count = queue_logical_block_size(bio->bi_disk->queue) >> 9;
1159 : : else
1160 : 2736611 : count = bio_sectors(bio);
1161 : :
1162 [ + + ]: 2736858 : if (op_is_write(bio_op(bio))) {
1163 : : count_vm_events(PGPGOUT, count);
1164 : : } else {
1165 : : if (bio_flagged(bio, BIO_WORKINGSET))
1166 : : workingset_read = true;
1167 : 2523191 : task_io_account_read(bio->bi_iter.bi_size);
1168 : : count_vm_events(PGPGIN, count);
1169 : : }
1170 : :
1171 [ - + ]: 2736693 : if (unlikely(block_dump)) {
1172 : : char b[BDEVNAME_SIZE];
1173 [ # # ]: 0 : printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
1174 : 0 : current->comm, task_pid_nr(current),
1175 : 0 : op_is_write(bio_op(bio)) ? "WRITE" : "READ",
1176 : : (unsigned long long)bio->bi_iter.bi_sector,
1177 : : bio_devname(bio, b), count);
1178 : : }
1179 : : }
1180 : :
1181 : : /*
1182 : : * If we're reading data that is part of the userspace
1183 : : * workingset, count submission time as memory stall. When the
1184 : : * device is congested, or the submitting cgroup IO-throttled,
1185 : : * submission can be a significant part of overall IO time.
1186 : : */
1187 : : if (workingset_read)
1188 : : psi_memstall_enter(&pflags);
1189 : :
1190 : 2737624 : ret = generic_make_request(bio);
1191 : :
1192 : : if (workingset_read)
1193 : : psi_memstall_leave(&pflags);
1194 : :
1195 : 2735010 : return ret;
1196 : : }
1197 : : EXPORT_SYMBOL(submit_bio);
1198 : :
1199 : : /**
1200 : : * blk_cloned_rq_check_limits - Helper function to check a cloned request
1201 : : * for new the queue limits
1202 : : * @q: the queue
1203 : : * @rq: the request being checked
1204 : : *
1205 : : * Description:
1206 : : * @rq may have been made based on weaker limitations of upper-level queues
1207 : : * in request stacking drivers, and it may violate the limitation of @q.
1208 : : * Since the block layer and the underlying device driver trust @rq
1209 : : * after it is inserted to @q, it should be checked against @q before
1210 : : * the insertion using this generic function.
1211 : : *
1212 : : * Request stacking drivers like request-based dm may change the queue
1213 : : * limits when retrying requests on other queues. Those requests need
1214 : : * to be checked against the new queue limits again during dispatch.
1215 : : */
1216 : 0 : static int blk_cloned_rq_check_limits(struct request_queue *q,
1217 : : struct request *rq)
1218 : : {
1219 [ # # ]: 0 : if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) {
1220 : 0 : printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
1221 : : __func__, blk_rq_sectors(rq),
1222 : : blk_queue_get_max_sectors(q, req_op(rq)));
1223 : 0 : return -EIO;
1224 : : }
1225 : :
1226 : : /*
1227 : : * queue's settings related to segment counting like q->bounce_pfn
1228 : : * may differ from that of other stacking queues.
1229 : : * Recalculate it to check the request correctly on this queue's
1230 : : * limitation.
1231 : : */
1232 : 0 : rq->nr_phys_segments = blk_recalc_rq_segments(rq);
1233 [ # # ]: 0 : if (rq->nr_phys_segments > queue_max_segments(q)) {
1234 : 0 : printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n",
1235 : : __func__, rq->nr_phys_segments, queue_max_segments(q));
1236 : 0 : return -EIO;
1237 : : }
1238 : :
1239 : : return 0;
1240 : : }
1241 : :
1242 : : /**
1243 : : * blk_insert_cloned_request - Helper for stacking drivers to submit a request
1244 : : * @q: the queue to submit the request
1245 : : * @rq: the request being queued
1246 : : */
1247 : 0 : blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
1248 : : {
1249 [ # # ]: 0 : if (blk_cloned_rq_check_limits(q, rq))
1250 : : return BLK_STS_IOERR;
1251 : :
1252 : : if (rq->rq_disk &&
1253 : : should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
1254 : : return BLK_STS_IOERR;
1255 : :
1256 [ # # ]: 0 : if (blk_queue_io_stat(q))
1257 : 0 : blk_account_io_start(rq, true);
1258 : :
1259 : : /*
1260 : : * Since we have a scheduler attached on the top device,
1261 : : * bypass a potential scheduler on the bottom device for
1262 : : * insert.
1263 : : */
1264 : 0 : return blk_mq_request_issue_directly(rq, true);
1265 : : }
1266 : : EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
1267 : :
1268 : : /**
1269 : : * blk_rq_err_bytes - determine number of bytes till the next failure boundary
1270 : : * @rq: request to examine
1271 : : *
1272 : : * Description:
1273 : : * A request could be merge of IOs which require different failure
1274 : : * handling. This function determines the number of bytes which
1275 : : * can be failed from the beginning of the request without
1276 : : * crossing into area which need to be retried further.
1277 : : *
1278 : : * Return:
1279 : : * The number of bytes to fail.
1280 : : */
1281 : 0 : unsigned int blk_rq_err_bytes(const struct request *rq)
1282 : : {
1283 : 0 : unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
1284 : : unsigned int bytes = 0;
1285 : : struct bio *bio;
1286 : :
1287 [ # # ]: 0 : if (!(rq->rq_flags & RQF_MIXED_MERGE))
1288 : 0 : return blk_rq_bytes(rq);
1289 : :
1290 : : /*
1291 : : * Currently the only 'mixing' which can happen is between
1292 : : * different fastfail types. We can safely fail portions
1293 : : * which have all the failfast bits that the first one has -
1294 : : * the ones which are at least as eager to fail as the first
1295 : : * one.
1296 : : */
1297 [ # # ]: 0 : for (bio = rq->bio; bio; bio = bio->bi_next) {
1298 [ # # ]: 0 : if ((bio->bi_opf & ff) != ff)
1299 : : break;
1300 : 0 : bytes += bio->bi_iter.bi_size;
1301 : : }
1302 : :
1303 : : /* this could lead to infinite loop */
1304 [ # # # # ]: 0 : BUG_ON(blk_rq_bytes(rq) && !bytes);
1305 : 0 : return bytes;
1306 : : }
1307 : : EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
1308 : :
1309 : 1136377 : void blk_account_io_completion(struct request *req, unsigned int bytes)
1310 : : {
1311 [ + - ]: 1136377 : if (blk_do_io_stat(req)) {
1312 : 1136377 : const int sgrp = op_stat_group(req_op(req));
1313 : : struct hd_struct *part;
1314 : :
1315 : 1136376 : part_stat_lock();
1316 : 1136375 : part = req->part;
1317 [ + + ]: 3404987 : part_stat_add(part, sectors[sgrp], bytes >> 9);
1318 : 1136375 : part_stat_unlock();
1319 : : }
1320 : 1136377 : }
1321 : :
1322 : 1136363 : void blk_account_io_done(struct request *req, u64 now)
1323 : : {
1324 : : /*
1325 : : * Account IO completion. flush_rq isn't accounted as a
1326 : : * normal IO on queueing nor completion. Accounting the
1327 : : * containing request is enough.
1328 : : */
1329 [ + - + + ]: 1136363 : if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {
1330 : 1136366 : const int sgrp = op_stat_group(req_op(req));
1331 : : struct hd_struct *part;
1332 : :
1333 : 1136368 : part_stat_lock();
1334 : 1136364 : part = req->part;
1335 : :
1336 : 1136364 : update_io_ticks(part, jiffies);
1337 [ + + ]: 3404975 : part_stat_inc(part, ios[sgrp]);
1338 [ + + ]: 3400814 : part_stat_add(part, nsecs[sgrp], now - req->start_time_ns);
1339 [ + + ]: 4537190 : part_stat_add(part, time_in_queue, nsecs_to_jiffies64(now - req->start_time_ns));
1340 : 2272748 : part_dec_in_flight(req->q, part, rq_data_dir(req));
1341 : :
1342 : : hd_struct_put(part);
1343 : 1136355 : part_stat_unlock();
1344 : : }
1345 : 1136373 : }
1346 : :
1347 : 2732901 : void blk_account_io_start(struct request *rq, bool new_io)
1348 : : {
1349 : : struct hd_struct *part;
1350 : 5465802 : int rw = rq_data_dir(rq);
1351 : :
1352 [ + + ]: 2732901 : if (!blk_do_io_stat(rq))
1353 : 2733266 : return;
1354 : :
1355 : 2731820 : part_stat_lock();
1356 : :
1357 [ + + ]: 2728925 : if (!new_io) {
1358 : 1572558 : part = rq->part;
1359 [ + + ]: 4717696 : part_stat_inc(part, merges[rw]);
1360 : : } else {
1361 : 2312734 : part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
1362 [ - + ]: 1159990 : if (!hd_struct_try_get(part)) {
1363 : : /*
1364 : : * The partition is already being removed,
1365 : : * the request will be accounted on the disk only
1366 : : *
1367 : : * We take a reference on disk->part0 although that
1368 : : * partition will never be deleted, so we can treat
1369 : : * it as any other partition.
1370 : : */
1371 : 0 : part = &rq->rq_disk->part0;
1372 : : hd_struct_get(part);
1373 : : }
1374 : 1159990 : part_inc_in_flight(rq->q, part, rw);
1375 : 1159661 : rq->part = part;
1376 : : }
1377 : :
1378 : 2732219 : update_io_ticks(part, jiffies);
1379 : :
1380 : 2733149 : part_stat_unlock();
1381 : : }
1382 : :
1383 : : /*
1384 : : * Steal bios from a request and add them to a bio list.
1385 : : * The request must not have been partially completed before.
1386 : : */
1387 : 0 : void blk_steal_bios(struct bio_list *list, struct request *rq)
1388 : : {
1389 [ # # ]: 0 : if (rq->bio) {
1390 [ # # ]: 0 : if (list->tail)
1391 : 0 : list->tail->bi_next = rq->bio;
1392 : : else
1393 : 0 : list->head = rq->bio;
1394 : 0 : list->tail = rq->biotail;
1395 : :
1396 : 0 : rq->bio = NULL;
1397 : 0 : rq->biotail = NULL;
1398 : : }
1399 : :
1400 : 0 : rq->__data_len = 0;
1401 : 0 : }
1402 : : EXPORT_SYMBOL_GPL(blk_steal_bios);
1403 : :
1404 : : /**
1405 : : * blk_update_request - Special helper function for request stacking drivers
1406 : : * @req: the request being processed
1407 : : * @error: block status code
1408 : : * @nr_bytes: number of bytes to complete @req
1409 : : *
1410 : : * Description:
1411 : : * Ends I/O on a number of bytes attached to @req, but doesn't complete
1412 : : * the request structure even if @req doesn't have leftover.
1413 : : * If @req has leftover, sets it up for the next range of segments.
1414 : : *
1415 : : * This special helper function is only for request stacking drivers
1416 : : * (e.g. request-based dm) so that they can handle partial completion.
1417 : : * Actual device drivers should use blk_mq_end_request instead.
1418 : : *
1419 : : * Passing the result of blk_rq_bytes() as @nr_bytes guarantees
1420 : : * %false return from this function.
1421 : : *
1422 : : * Note:
1423 : : * The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in both
1424 : : * blk_rq_bytes() and in blk_update_request().
1425 : : *
1426 : : * Return:
1427 : : * %false - this request doesn't have any more data
1428 : : * %true - this request has more data
1429 : : **/
1430 : 1136379 : bool blk_update_request(struct request *req, blk_status_t error,
1431 : : unsigned int nr_bytes)
1432 : : {
1433 : : int total_bytes;
1434 : :
1435 : 1136379 : trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
1436 : :
1437 [ + - ]: 1136378 : if (!req->bio)
1438 : : return false;
1439 : :
1440 : : #ifdef CONFIG_BLK_DEV_INTEGRITY
1441 : : if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
1442 : : error == BLK_STS_OK)
1443 : : req->q->integrity.profile->complete_fn(req, nr_bytes);
1444 : : #endif
1445 : :
1446 [ - + # # : 1136379 : if (unlikely(error && !blk_rq_is_passthrough(req) &&
- + # # ]
1447 : : !(req->rq_flags & RQF_QUIET)))
1448 : 0 : print_req_error(req, error, __func__);
1449 : :
1450 : 1136379 : blk_account_io_completion(req, nr_bytes);
1451 : :
1452 : : total_bytes = 0;
1453 [ + - ]: 3869805 : while (req->bio) {
1454 : : struct bio *bio = req->bio;
1455 : 2733439 : unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
1456 : :
1457 [ + + ]: 2733439 : if (bio_bytes == bio->bi_iter.bi_size)
1458 : 2733447 : req->bio = bio->bi_next;
1459 : :
1460 : : /* Completion has already been traced */
1461 : : bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1462 : 2733439 : req_bio_endio(req, bio, bio_bytes, error);
1463 : :
1464 : 2733449 : total_bytes += bio_bytes;
1465 : 2733449 : nr_bytes -= bio_bytes;
1466 : :
1467 [ + + ]: 2733449 : if (!nr_bytes)
1468 : : break;
1469 : : }
1470 : :
1471 : : /*
1472 : : * completely done
1473 : : */
1474 [ + - ]: 1136376 : if (!req->bio) {
1475 : : /*
1476 : : * Reset counters so that the request stacking driver
1477 : : * can find how many bytes remain in the request
1478 : : * later.
1479 : : */
1480 : 1136376 : req->__data_len = 0;
1481 : 1136376 : return false;
1482 : : }
1483 : :
1484 : 0 : req->__data_len -= total_bytes;
1485 : :
1486 : : /* update sector only for requests with clear definition of sector */
1487 [ # # ]: 0 : if (!blk_rq_is_passthrough(req))
1488 : 0 : req->__sector += total_bytes >> 9;
1489 : :
1490 : : /* mixed attributes always follow the first bio */
1491 [ # # ]: 0 : if (req->rq_flags & RQF_MIXED_MERGE) {
1492 : 0 : req->cmd_flags &= ~REQ_FAILFAST_MASK;
1493 : 0 : req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
1494 : : }
1495 : :
1496 [ # # ]: 0 : if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
1497 : : /*
1498 : : * If total number of sectors is less than the first segment
1499 : : * size, something has gone terribly wrong.
1500 : : */
1501 [ # # ]: 0 : if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
1502 : 0 : blk_dump_rq_flags(req, "request botched");
1503 : 0 : req->__data_len = blk_rq_cur_bytes(req);
1504 : : }
1505 : :
1506 : : /* recalculate the number of segments */
1507 : 0 : req->nr_phys_segments = blk_recalc_rq_segments(req);
1508 : : }
1509 : :
1510 : : return true;
1511 : : }
1512 : : EXPORT_SYMBOL_GPL(blk_update_request);
1513 : :
1514 : : #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
1515 : : /**
1516 : : * rq_flush_dcache_pages - Helper function to flush all pages in a request
1517 : : * @rq: the request to be flushed
1518 : : *
1519 : : * Description:
1520 : : * Flush all pages in @rq.
1521 : : */
1522 : 0 : void rq_flush_dcache_pages(struct request *rq)
1523 : : {
1524 : : struct req_iterator iter;
1525 : : struct bio_vec bvec;
1526 : :
1527 [ # # # # : 0 : rq_for_each_segment(bvec, rq, iter)
# # ]
1528 : 0 : flush_dcache_page(bvec.bv_page);
1529 : 0 : }
1530 : : EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
1531 : : #endif
1532 : :
1533 : : /**
1534 : : * blk_lld_busy - Check if underlying low-level drivers of a device are busy
1535 : : * @q : the queue of the device being checked
1536 : : *
1537 : : * Description:
1538 : : * Check if underlying low-level drivers of a device are busy.
1539 : : * If the drivers want to export their busy state, they must set own
1540 : : * exporting function using blk_queue_lld_busy() first.
1541 : : *
1542 : : * Basically, this function is used only by request stacking drivers
1543 : : * to stop dispatching requests to underlying devices when underlying
1544 : : * devices are busy. This behavior helps more I/O merging on the queue
1545 : : * of the request stacking driver and prevents I/O throughput regression
1546 : : * on burst I/O load.
1547 : : *
1548 : : * Return:
1549 : : * 0 - Not busy (The request stacking driver should dispatch request)
1550 : : * 1 - Busy (The request stacking driver should stop dispatching request)
1551 : : */
1552 : 0 : int blk_lld_busy(struct request_queue *q)
1553 : : {
1554 [ # # # # ]: 0 : if (queue_is_mq(q) && q->mq_ops->busy)
1555 : 0 : return q->mq_ops->busy(q);
1556 : :
1557 : : return 0;
1558 : : }
1559 : : EXPORT_SYMBOL_GPL(blk_lld_busy);
1560 : :
1561 : : /**
1562 : : * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
1563 : : * @rq: the clone request to be cleaned up
1564 : : *
1565 : : * Description:
1566 : : * Free all bios in @rq for a cloned request.
1567 : : */
1568 : 0 : void blk_rq_unprep_clone(struct request *rq)
1569 : : {
1570 : : struct bio *bio;
1571 : :
1572 [ # # # # ]: 0 : while ((bio = rq->bio) != NULL) {
1573 : 0 : rq->bio = bio->bi_next;
1574 : :
1575 : 0 : bio_put(bio);
1576 : : }
1577 : 0 : }
1578 : : EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
1579 : :
1580 : : /*
1581 : : * Copy attributes of the original request to the clone request.
1582 : : * The actual data parts (e.g. ->cmd, ->sense) are not copied.
1583 : : */
1584 : 0 : static void __blk_rq_prep_clone(struct request *dst, struct request *src)
1585 : : {
1586 : 0 : dst->__sector = blk_rq_pos(src);
1587 : 0 : dst->__data_len = blk_rq_bytes(src);
1588 [ # # ]: 0 : if (src->rq_flags & RQF_SPECIAL_PAYLOAD) {
1589 : 0 : dst->rq_flags |= RQF_SPECIAL_PAYLOAD;
1590 : 0 : dst->special_vec = src->special_vec;
1591 : : }
1592 : 0 : dst->nr_phys_segments = src->nr_phys_segments;
1593 : 0 : dst->ioprio = src->ioprio;
1594 : 0 : dst->extra_len = src->extra_len;
1595 : 0 : }
1596 : :
1597 : : /**
1598 : : * blk_rq_prep_clone - Helper function to setup clone request
1599 : : * @rq: the request to be setup
1600 : : * @rq_src: original request to be cloned
1601 : : * @bs: bio_set that bios for clone are allocated from
1602 : : * @gfp_mask: memory allocation mask for bio
1603 : : * @bio_ctr: setup function to be called for each clone bio.
1604 : : * Returns %0 for success, non %0 for failure.
1605 : : * @data: private data to be passed to @bio_ctr
1606 : : *
1607 : : * Description:
1608 : : * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
1609 : : * The actual data parts of @rq_src (e.g. ->cmd, ->sense)
1610 : : * are not copied, and copying such parts is the caller's responsibility.
1611 : : * Also, pages which the original bios are pointing to are not copied
1612 : : * and the cloned bios just point same pages.
1613 : : * So cloned bios must be completed before original bios, which means
1614 : : * the caller must complete @rq before @rq_src.
1615 : : */
1616 : 0 : int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
1617 : : struct bio_set *bs, gfp_t gfp_mask,
1618 : : int (*bio_ctr)(struct bio *, struct bio *, void *),
1619 : : void *data)
1620 : : {
1621 : : struct bio *bio, *bio_src;
1622 : :
1623 [ # # ]: 0 : if (!bs)
1624 : : bs = &fs_bio_set;
1625 : :
1626 [ # # # # ]: 0 : __rq_for_each_bio(bio_src, rq_src) {
1627 : 0 : bio = bio_clone_fast(bio_src, gfp_mask, bs);
1628 [ # # ]: 0 : if (!bio)
1629 : : goto free_and_out;
1630 : :
1631 [ # # # # ]: 0 : if (bio_ctr && bio_ctr(bio, bio_src, data))
1632 : : goto free_and_out;
1633 : :
1634 [ # # ]: 0 : if (rq->bio) {
1635 : 0 : rq->biotail->bi_next = bio;
1636 : 0 : rq->biotail = bio;
1637 : : } else
1638 : 0 : rq->bio = rq->biotail = bio;
1639 : : }
1640 : :
1641 : 0 : __blk_rq_prep_clone(rq, rq_src);
1642 : :
1643 : 0 : return 0;
1644 : :
1645 : : free_and_out:
1646 [ # # ]: 0 : if (bio)
1647 : 0 : bio_put(bio);
1648 : : blk_rq_unprep_clone(rq);
1649 : :
1650 : : return -ENOMEM;
1651 : : }
1652 : : EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
1653 : :
1654 : 0 : int kblockd_schedule_work(struct work_struct *work)
1655 : : {
1656 : 2571 : return queue_work(kblockd_workqueue, work);
1657 : : }
1658 : : EXPORT_SYMBOL(kblockd_schedule_work);
1659 : :
1660 : 0 : int kblockd_schedule_work_on(int cpu, struct work_struct *work)
1661 : : {
1662 : 0 : return queue_work_on(cpu, kblockd_workqueue, work);
1663 : : }
1664 : : EXPORT_SYMBOL(kblockd_schedule_work_on);
1665 : :
1666 : 1097526 : int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
1667 : : unsigned long delay)
1668 : : {
1669 : 1097526 : return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
1670 : : }
1671 : : EXPORT_SYMBOL(kblockd_mod_delayed_work_on);
1672 : :
1673 : : /**
1674 : : * blk_start_plug - initialize blk_plug and track it inside the task_struct
1675 : : * @plug: The &struct blk_plug that needs to be initialized
1676 : : *
1677 : : * Description:
1678 : : * blk_start_plug() indicates to the block layer an intent by the caller
1679 : : * to submit multiple I/O requests in a batch. The block layer may use
1680 : : * this hint to defer submitting I/Os from the caller until blk_finish_plug()
1681 : : * is called. However, the block layer may choose to submit requests
1682 : : * before a call to blk_finish_plug() if the number of queued I/Os
1683 : : * exceeds %BLK_MAX_REQUEST_COUNT, or if the size of the I/O is larger than
1684 : : * %BLK_PLUG_FLUSH_SIZE. The queued I/Os may also be submitted early if
1685 : : * the task schedules (see below).
1686 : : *
1687 : : * Tracking blk_plug inside the task_struct will help with auto-flushing the
1688 : : * pending I/O should the task end up blocking between blk_start_plug() and
1689 : : * blk_finish_plug(). This is important from a performance perspective, but
1690 : : * also ensures that we don't deadlock. For instance, if the task is blocking
1691 : : * for a memory allocation, memory reclaim could end up wanting to free a
1692 : : * page belonging to that request that is currently residing in our private
1693 : : * plug. By flushing the pending I/O when the process goes to sleep, we avoid
1694 : : * this kind of deadlock.
1695 : : */
1696 : 826594 : void blk_start_plug(struct blk_plug *plug)
1697 : : {
1698 : 826594 : struct task_struct *tsk = current;
1699 : :
1700 : : /*
1701 : : * If this is a nested plug, don't actually assign it.
1702 : : */
1703 [ + + ]: 826594 : if (tsk->plug)
1704 : 826594 : return;
1705 : :
1706 : 796439 : INIT_LIST_HEAD(&plug->mq_list);
1707 : 796439 : INIT_LIST_HEAD(&plug->cb_list);
1708 : 796439 : plug->rq_count = 0;
1709 : 796439 : plug->multiple_queues = false;
1710 : :
1711 : : /*
1712 : : * Store ordering should not be needed here, since a potential
1713 : : * preempt will imply a full memory barrier
1714 : : */
1715 : 796439 : tsk->plug = plug;
1716 : : }
1717 : : EXPORT_SYMBOL(blk_start_plug);
1718 : :
1719 : 800492 : static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
1720 : : {
1721 : 800492 : LIST_HEAD(callbacks);
1722 : :
1723 [ - + ]: 2401476 : while (!list_empty(&plug->cb_list)) {
1724 : : list_splice_init(&plug->cb_list, &callbacks);
1725 : :
1726 [ # # ]: 0 : while (!list_empty(&callbacks)) {
1727 : 0 : struct blk_plug_cb *cb = list_first_entry(&callbacks,
1728 : : struct blk_plug_cb,
1729 : : list);
1730 : : list_del(&cb->list);
1731 : 0 : cb->callback(cb, from_schedule);
1732 : : }
1733 : : }
1734 : 800492 : }
1735 : :
1736 : 0 : struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
1737 : : int size)
1738 : : {
1739 : 0 : struct blk_plug *plug = current->plug;
1740 : : struct blk_plug_cb *cb;
1741 : :
1742 [ # # ]: 0 : if (!plug)
1743 : : return NULL;
1744 : :
1745 [ # # ]: 0 : list_for_each_entry(cb, &plug->cb_list, list)
1746 [ # # # # ]: 0 : if (cb->callback == unplug && cb->data == data)
1747 : 0 : return cb;
1748 : :
1749 : : /* Not currently on the callback list */
1750 [ # # ]: 0 : BUG_ON(size < sizeof(*cb));
1751 : 0 : cb = kzalloc(size, GFP_ATOMIC);
1752 [ # # ]: 0 : if (cb) {
1753 : 0 : cb->data = data;
1754 : 0 : cb->callback = unplug;
1755 : 0 : list_add(&cb->list, &plug->cb_list);
1756 : : }
1757 : 0 : return cb;
1758 : : }
1759 : : EXPORT_SYMBOL(blk_check_plugged);
1760 : :
1761 : 800469 : void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1762 : : {
1763 : 800469 : flush_plug_callbacks(plug, from_schedule);
1764 : :
1765 [ + + ]: 1600972 : if (!list_empty(&plug->mq_list))
1766 : 689320 : blk_mq_flush_plug_list(plug, from_schedule);
1767 : 804241 : }
1768 : :
1769 : : /**
1770 : : * blk_finish_plug - mark the end of a batch of submitted I/O
1771 : : * @plug: The &struct blk_plug passed to blk_start_plug()
1772 : : *
1773 : : * Description:
1774 : : * Indicate that a batch of I/O submissions is complete. This function
1775 : : * must be paired with an initial call to blk_start_plug(). The intent
1776 : : * is to allow the block layer to optimize I/O submission. See the
1777 : : * documentation for blk_start_plug() for more information.
1778 : : */
1779 : 826314 : void blk_finish_plug(struct blk_plug *plug)
1780 : : {
1781 [ + + ]: 826314 : if (plug != current->plug)
1782 : 826574 : return;
1783 : 796440 : blk_flush_plug_list(plug, false);
1784 : :
1785 : 796700 : current->plug = NULL;
1786 : : }
1787 : : EXPORT_SYMBOL(blk_finish_plug);
1788 : :
1789 : 207 : int __init blk_dev_init(void)
1790 : : {
1791 : : BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
1792 : : BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
1793 : : FIELD_SIZEOF(struct request, cmd_flags));
1794 : : BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
1795 : : FIELD_SIZEOF(struct bio, bi_opf));
1796 : :
1797 : : /* used for unplugging and affects IO latency/throughput - HIGHPRI */
1798 : 207 : kblockd_workqueue = alloc_workqueue("kblockd",
1799 : : WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
1800 [ - + ]: 207 : if (!kblockd_workqueue)
1801 : 0 : panic("Failed to create kblockd\n");
1802 : :
1803 : 207 : blk_requestq_cachep = kmem_cache_create("request_queue",
1804 : : sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
1805 : :
1806 : : #ifdef CONFIG_DEBUG_FS
1807 : 207 : blk_debugfs_root = debugfs_create_dir("block", NULL);
1808 : : #endif
1809 : :
1810 : 207 : return 0;
1811 : : }
|
