md: fix a potential deadlock of raid5/raid10 reshape
[sfrench/cifs-2.6.git] / drivers / md / raid10.c
1 /*
2  * raid10.c : Multiple Devices driver for Linux
3  *
4  * Copyright (C) 2000-2004 Neil Brown
5  *
6  * RAID-10 support for md.
7  *
8  * Base on code in raid1.c.  See raid1.c for further copyright information.
9  *
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2, or (at your option)
14  * any later version.
15  *
16  * You should have received a copy of the GNU General Public License
17  * (for example /usr/src/linux/COPYING); if not, write to the Free
18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19  */
20
21 #include <linux/slab.h>
22 #include <linux/delay.h>
23 #include <linux/blkdev.h>
24 #include <linux/module.h>
25 #include <linux/seq_file.h>
26 #include <linux/ratelimit.h>
27 #include <linux/kthread.h>
28 #include <trace/events/block.h>
29 #include "md.h"
30 #include "raid10.h"
31 #include "raid0.h"
32 #include "md-bitmap.h"
33
34 /*
35  * RAID10 provides a combination of RAID0 and RAID1 functionality.
36  * The layout of data is defined by
37  *    chunk_size
38  *    raid_disks
39  *    near_copies (stored in low byte of layout)
40  *    far_copies (stored in second byte of layout)
41  *    far_offset (stored in bit 16 of layout )
42  *    use_far_sets (stored in bit 17 of layout )
43  *    use_far_sets_bugfixed (stored in bit 18 of layout )
44  *
45  * The data to be stored is divided into chunks using chunksize.  Each device
46  * is divided into far_copies sections.   In each section, chunks are laid out
47  * in a style similar to raid0, but near_copies copies of each chunk is stored
48  * (each on a different drive).  The starting device for each section is offset
49  * near_copies from the starting device of the previous section.  Thus there
50  * are (near_copies * far_copies) of each chunk, and each is on a different
51  * drive.  near_copies and far_copies must be at least one, and their product
52  * is at most raid_disks.
53  *
54  * If far_offset is true, then the far_copies are handled a bit differently.
55  * The copies are still in different stripes, but instead of being very far
56  * apart on disk, there are adjacent stripes.
57  *
58  * The far and offset algorithms are handled slightly differently if
59  * 'use_far_sets' is true.  In this case, the array's devices are grouped into
60  * sets that are (near_copies * far_copies) in size.  The far copied stripes
61  * are still shifted by 'near_copies' devices, but this shifting stays confined
62  * to the set rather than the entire array.  This is done to improve the number
63  * of device combinations that can fail without causing the array to fail.
64  * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk
65  * on a device):
66  *    A B C D    A B C D E
67  *      ...         ...
68  *    D A B C    E A B C D
69  * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s):
70  *    [A B] [C D]    [A B] [C D E]
71  *    |...| |...|    |...| | ... |
72  *    [B A] [D C]    [B A] [E C D]
73  */
74
75 /*
76  * Number of guaranteed r10bios in case of extreme VM load:
77  */
78 #define NR_RAID10_BIOS 256
79
80 /* when we get a read error on a read-only array, we redirect to another
81  * device without failing the first device, or trying to over-write to
82  * correct the read error.  To keep track of bad blocks on a per-bio
83  * level, we store IO_BLOCKED in the appropriate 'bios' pointer
84  */
85 #define IO_BLOCKED ((struct bio *)1)
86 /* When we successfully write to a known bad-block, we need to remove the
87  * bad-block marking which must be done from process context.  So we record
88  * the success by setting devs[n].bio to IO_MADE_GOOD
89  */
90 #define IO_MADE_GOOD ((struct bio *)2)
91
92 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)
93
94 /* When there are this many requests queued to be written by
95  * the raid10 thread, we become 'congested' to provide back-pressure
96  * for writeback.
97  */
98 static int max_queued_requests = 1024;
99
100 static void allow_barrier(struct r10conf *conf);
101 static void lower_barrier(struct r10conf *conf);
102 static int _enough(struct r10conf *conf, int previous, int ignore);
103 static int enough(struct r10conf *conf, int ignore);
104 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
105                                 int *skipped);
106 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
107 static void end_reshape_write(struct bio *bio);
108 static void end_reshape(struct r10conf *conf);
109
110 #define raid10_log(md, fmt, args...)                            \
111         do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid10 " fmt, ##args); } while (0)
112
113 #include "raid1-10.c"
114
115 /*
116  * for resync bio, r10bio pointer can be retrieved from the per-bio
117  * 'struct resync_pages'.
118  */
119 static inline struct r10bio *get_resync_r10bio(struct bio *bio)
120 {
121         return get_resync_pages(bio)->raid_bio;
122 }
123
124 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
125 {
126         struct r10conf *conf = data;
127         int size = offsetof(struct r10bio, devs[conf->copies]);
128
129         /* allocate a r10bio with room for raid_disks entries in the
130          * bios array */
131         return kzalloc(size, gfp_flags);
132 }
133
134 static void r10bio_pool_free(void *r10_bio, void *data)
135 {
136         kfree(r10_bio);
137 }
138
139 #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
140 /* amount of memory to reserve for resync requests */
141 #define RESYNC_WINDOW (1024*1024)
142 /* maximum number of concurrent requests, memory permitting */
143 #define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
144 #define CLUSTER_RESYNC_WINDOW (32 * RESYNC_WINDOW)
145 #define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9)
146
147 /*
148  * When performing a resync, we need to read and compare, so
149  * we need as many pages are there are copies.
150  * When performing a recovery, we need 2 bios, one for read,
151  * one for write (we recover only one drive per r10buf)
152  *
153  */
154 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
155 {
156         struct r10conf *conf = data;
157         struct r10bio *r10_bio;
158         struct bio *bio;
159         int j;
160         int nalloc, nalloc_rp;
161         struct resync_pages *rps;
162
163         r10_bio = r10bio_pool_alloc(gfp_flags, conf);
164         if (!r10_bio)
165                 return NULL;
166
167         if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
168             test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
169                 nalloc = conf->copies; /* resync */
170         else
171                 nalloc = 2; /* recovery */
172
173         /* allocate once for all bios */
174         if (!conf->have_replacement)
175                 nalloc_rp = nalloc;
176         else
177                 nalloc_rp = nalloc * 2;
178         rps = kmalloc(sizeof(struct resync_pages) * nalloc_rp, gfp_flags);
179         if (!rps)
180                 goto out_free_r10bio;
181
182         /*
183          * Allocate bios.
184          */
185         for (j = nalloc ; j-- ; ) {
186                 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
187                 if (!bio)
188                         goto out_free_bio;
189                 r10_bio->devs[j].bio = bio;
190                 if (!conf->have_replacement)
191                         continue;
192                 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
193                 if (!bio)
194                         goto out_free_bio;
195                 r10_bio->devs[j].repl_bio = bio;
196         }
197         /*
198          * Allocate RESYNC_PAGES data pages and attach them
199          * where needed.
200          */
201         for (j = 0; j < nalloc; j++) {
202                 struct bio *rbio = r10_bio->devs[j].repl_bio;
203                 struct resync_pages *rp, *rp_repl;
204
205                 rp = &rps[j];
206                 if (rbio)
207                         rp_repl = &rps[nalloc + j];
208
209                 bio = r10_bio->devs[j].bio;
210
211                 if (!j || test_bit(MD_RECOVERY_SYNC,
212                                    &conf->mddev->recovery)) {
213                         if (resync_alloc_pages(rp, gfp_flags))
214                                 goto out_free_pages;
215                 } else {
216                         memcpy(rp, &rps[0], sizeof(*rp));
217                         resync_get_all_pages(rp);
218                 }
219
220                 rp->raid_bio = r10_bio;
221                 bio->bi_private = rp;
222                 if (rbio) {
223                         memcpy(rp_repl, rp, sizeof(*rp));
224                         rbio->bi_private = rp_repl;
225                 }
226         }
227
228         return r10_bio;
229
230 out_free_pages:
231         while (--j >= 0)
232                 resync_free_pages(&rps[j * 2]);
233
234         j = 0;
235 out_free_bio:
236         for ( ; j < nalloc; j++) {
237                 if (r10_bio->devs[j].bio)
238                         bio_put(r10_bio->devs[j].bio);
239                 if (r10_bio->devs[j].repl_bio)
240                         bio_put(r10_bio->devs[j].repl_bio);
241         }
242         kfree(rps);
243 out_free_r10bio:
244         r10bio_pool_free(r10_bio, conf);
245         return NULL;
246 }
247
248 static void r10buf_pool_free(void *__r10_bio, void *data)
249 {
250         struct r10conf *conf = data;
251         struct r10bio *r10bio = __r10_bio;
252         int j;
253         struct resync_pages *rp = NULL;
254
255         for (j = conf->copies; j--; ) {
256                 struct bio *bio = r10bio->devs[j].bio;
257
258                 rp = get_resync_pages(bio);
259                 resync_free_pages(rp);
260                 bio_put(bio);
261
262                 bio = r10bio->devs[j].repl_bio;
263                 if (bio)
264                         bio_put(bio);
265         }
266
267         /* resync pages array stored in the 1st bio's .bi_private */
268         kfree(rp);
269
270         r10bio_pool_free(r10bio, conf);
271 }
272
273 static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
274 {
275         int i;
276
277         for (i = 0; i < conf->copies; i++) {
278                 struct bio **bio = & r10_bio->devs[i].bio;
279                 if (!BIO_SPECIAL(*bio))
280                         bio_put(*bio);
281                 *bio = NULL;
282                 bio = &r10_bio->devs[i].repl_bio;
283                 if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
284                         bio_put(*bio);
285                 *bio = NULL;
286         }
287 }
288
289 static void free_r10bio(struct r10bio *r10_bio)
290 {
291         struct r10conf *conf = r10_bio->mddev->private;
292
293         put_all_bios(conf, r10_bio);
294         mempool_free(r10_bio, conf->r10bio_pool);
295 }
296
297 static void put_buf(struct r10bio *r10_bio)
298 {
299         struct r10conf *conf = r10_bio->mddev->private;
300
301         mempool_free(r10_bio, conf->r10buf_pool);
302
303         lower_barrier(conf);
304 }
305
306 static void reschedule_retry(struct r10bio *r10_bio)
307 {
308         unsigned long flags;
309         struct mddev *mddev = r10_bio->mddev;
310         struct r10conf *conf = mddev->private;
311
312         spin_lock_irqsave(&conf->device_lock, flags);
313         list_add(&r10_bio->retry_list, &conf->retry_list);
314         conf->nr_queued ++;
315         spin_unlock_irqrestore(&conf->device_lock, flags);
316
317         /* wake up frozen array... */
318         wake_up(&conf->wait_barrier);
319
320         md_wakeup_thread(mddev->thread);
321 }
322
323 /*
324  * raid_end_bio_io() is called when we have finished servicing a mirrored
325  * operation and are ready to return a success/failure code to the buffer
326  * cache layer.
327  */
328 static void raid_end_bio_io(struct r10bio *r10_bio)
329 {
330         struct bio *bio = r10_bio->master_bio;
331         struct r10conf *conf = r10_bio->mddev->private;
332
333         if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
334                 bio->bi_status = BLK_STS_IOERR;
335
336         bio_endio(bio);
337         /*
338          * Wake up any possible resync thread that waits for the device
339          * to go idle.
340          */
341         allow_barrier(conf);
342
343         free_r10bio(r10_bio);
344 }
345
346 /*
347  * Update disk head position estimator based on IRQ completion info.
348  */
349 static inline void update_head_pos(int slot, struct r10bio *r10_bio)
350 {
351         struct r10conf *conf = r10_bio->mddev->private;
352
353         conf->mirrors[r10_bio->devs[slot].devnum].head_position =
354                 r10_bio->devs[slot].addr + (r10_bio->sectors);
355 }
356
357 /*
358  * Find the disk number which triggered given bio
359  */
360 static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
361                          struct bio *bio, int *slotp, int *replp)
362 {
363         int slot;
364         int repl = 0;
365
366         for (slot = 0; slot < conf->copies; slot++) {
367                 if (r10_bio->devs[slot].bio == bio)
368                         break;
369                 if (r10_bio->devs[slot].repl_bio == bio) {
370                         repl = 1;
371                         break;
372                 }
373         }
374
375         BUG_ON(slot == conf->copies);
376         update_head_pos(slot, r10_bio);
377
378         if (slotp)
379                 *slotp = slot;
380         if (replp)
381                 *replp = repl;
382         return r10_bio->devs[slot].devnum;
383 }
384
385 static void raid10_end_read_request(struct bio *bio)
386 {
387         int uptodate = !bio->bi_status;
388         struct r10bio *r10_bio = bio->bi_private;
389         int slot;
390         struct md_rdev *rdev;
391         struct r10conf *conf = r10_bio->mddev->private;
392
393         slot = r10_bio->read_slot;
394         rdev = r10_bio->devs[slot].rdev;
395         /*
396          * this branch is our 'one mirror IO has finished' event handler:
397          */
398         update_head_pos(slot, r10_bio);
399
400         if (uptodate) {
401                 /*
402                  * Set R10BIO_Uptodate in our master bio, so that
403                  * we will return a good error code to the higher
404                  * levels even if IO on some other mirrored buffer fails.
405                  *
406                  * The 'master' represents the composite IO operation to
407                  * user-side. So if something waits for IO, then it will
408                  * wait for the 'master' bio.
409                  */
410                 set_bit(R10BIO_Uptodate, &r10_bio->state);
411         } else {
412                 /* If all other devices that store this block have
413                  * failed, we want to return the error upwards rather
414                  * than fail the last device.  Here we redefine
415                  * "uptodate" to mean "Don't want to retry"
416                  */
417                 if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state),
418                              rdev->raid_disk))
419                         uptodate = 1;
420         }
421         if (uptodate) {
422                 raid_end_bio_io(r10_bio);
423                 rdev_dec_pending(rdev, conf->mddev);
424         } else {
425                 /*
426                  * oops, read error - keep the refcount on the rdev
427                  */
428                 char b[BDEVNAME_SIZE];
429                 pr_err_ratelimited("md/raid10:%s: %s: rescheduling sector %llu\n",
430                                    mdname(conf->mddev),
431                                    bdevname(rdev->bdev, b),
432                                    (unsigned long long)r10_bio->sector);
433                 set_bit(R10BIO_ReadError, &r10_bio->state);
434                 reschedule_retry(r10_bio);
435         }
436 }
437
438 static void close_write(struct r10bio *r10_bio)
439 {
440         /* clear the bitmap if all writes complete successfully */
441         bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
442                         r10_bio->sectors,
443                         !test_bit(R10BIO_Degraded, &r10_bio->state),
444                         0);
445         md_write_end(r10_bio->mddev);
446 }
447
448 static void one_write_done(struct r10bio *r10_bio)
449 {
450         if (atomic_dec_and_test(&r10_bio->remaining)) {
451                 if (test_bit(R10BIO_WriteError, &r10_bio->state))
452                         reschedule_retry(r10_bio);
453                 else {
454                         close_write(r10_bio);
455                         if (test_bit(R10BIO_MadeGood, &r10_bio->state))
456                                 reschedule_retry(r10_bio);
457                         else
458                                 raid_end_bio_io(r10_bio);
459                 }
460         }
461 }
462
463 static void raid10_end_write_request(struct bio *bio)
464 {
465         struct r10bio *r10_bio = bio->bi_private;
466         int dev;
467         int dec_rdev = 1;
468         struct r10conf *conf = r10_bio->mddev->private;
469         int slot, repl;
470         struct md_rdev *rdev = NULL;
471         struct bio *to_put = NULL;
472         bool discard_error;
473
474         discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD;
475
476         dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
477
478         if (repl)
479                 rdev = conf->mirrors[dev].replacement;
480         if (!rdev) {
481                 smp_rmb();
482                 repl = 0;
483                 rdev = conf->mirrors[dev].rdev;
484         }
485         /*
486          * this branch is our 'one mirror IO has finished' event handler:
487          */
488         if (bio->bi_status && !discard_error) {
489                 if (repl)
490                         /* Never record new bad blocks to replacement,
491                          * just fail it.
492                          */
493                         md_error(rdev->mddev, rdev);
494                 else {
495                         set_bit(WriteErrorSeen, &rdev->flags);
496                         if (!test_and_set_bit(WantReplacement, &rdev->flags))
497                                 set_bit(MD_RECOVERY_NEEDED,
498                                         &rdev->mddev->recovery);
499
500                         dec_rdev = 0;
501                         if (test_bit(FailFast, &rdev->flags) &&
502                             (bio->bi_opf & MD_FAILFAST)) {
503                                 md_error(rdev->mddev, rdev);
504                                 if (!test_bit(Faulty, &rdev->flags))
505                                         /* This is the only remaining device,
506                                          * We need to retry the write without
507                                          * FailFast
508                                          */
509                                         set_bit(R10BIO_WriteError, &r10_bio->state);
510                                 else {
511                                         r10_bio->devs[slot].bio = NULL;
512                                         to_put = bio;
513                                         dec_rdev = 1;
514                                 }
515                         } else
516                                 set_bit(R10BIO_WriteError, &r10_bio->state);
517                 }
518         } else {
519                 /*
520                  * Set R10BIO_Uptodate in our master bio, so that
521                  * we will return a good error code for to the higher
522                  * levels even if IO on some other mirrored buffer fails.
523                  *
524                  * The 'master' represents the composite IO operation to
525                  * user-side. So if something waits for IO, then it will
526                  * wait for the 'master' bio.
527                  */
528                 sector_t first_bad;
529                 int bad_sectors;
530
531                 /*
532                  * Do not set R10BIO_Uptodate if the current device is
533                  * rebuilding or Faulty. This is because we cannot use
534                  * such device for properly reading the data back (we could
535                  * potentially use it, if the current write would have felt
536                  * before rdev->recovery_offset, but for simplicity we don't
537                  * check this here.
538                  */
539                 if (test_bit(In_sync, &rdev->flags) &&
540                     !test_bit(Faulty, &rdev->flags))
541                         set_bit(R10BIO_Uptodate, &r10_bio->state);
542
543                 /* Maybe we can clear some bad blocks. */
544                 if (is_badblock(rdev,
545                                 r10_bio->devs[slot].addr,
546                                 r10_bio->sectors,
547                                 &first_bad, &bad_sectors) && !discard_error) {
548                         bio_put(bio);
549                         if (repl)
550                                 r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
551                         else
552                                 r10_bio->devs[slot].bio = IO_MADE_GOOD;
553                         dec_rdev = 0;
554                         set_bit(R10BIO_MadeGood, &r10_bio->state);
555                 }
556         }
557
558         /*
559          *
560          * Let's see if all mirrored write operations have finished
561          * already.
562          */
563         one_write_done(r10_bio);
564         if (dec_rdev)
565                 rdev_dec_pending(rdev, conf->mddev);
566         if (to_put)
567                 bio_put(to_put);
568 }
569
570 /*
571  * RAID10 layout manager
572  * As well as the chunksize and raid_disks count, there are two
573  * parameters: near_copies and far_copies.
574  * near_copies * far_copies must be <= raid_disks.
575  * Normally one of these will be 1.
576  * If both are 1, we get raid0.
577  * If near_copies == raid_disks, we get raid1.
578  *
579  * Chunks are laid out in raid0 style with near_copies copies of the
580  * first chunk, followed by near_copies copies of the next chunk and
581  * so on.
582  * If far_copies > 1, then after 1/far_copies of the array has been assigned
583  * as described above, we start again with a device offset of near_copies.
584  * So we effectively have another copy of the whole array further down all
585  * the drives, but with blocks on different drives.
586  * With this layout, and block is never stored twice on the one device.
587  *
588  * raid10_find_phys finds the sector offset of a given virtual sector
589  * on each device that it is on.
590  *
591  * raid10_find_virt does the reverse mapping, from a device and a
592  * sector offset to a virtual address
593  */
594
595 static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
596 {
597         int n,f;
598         sector_t sector;
599         sector_t chunk;
600         sector_t stripe;
601         int dev;
602         int slot = 0;
603         int last_far_set_start, last_far_set_size;
604
605         last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
606         last_far_set_start *= geo->far_set_size;
607
608         last_far_set_size = geo->far_set_size;
609         last_far_set_size += (geo->raid_disks % geo->far_set_size);
610
611         /* now calculate first sector/dev */
612         chunk = r10bio->sector >> geo->chunk_shift;
613         sector = r10bio->sector & geo->chunk_mask;
614
615         chunk *= geo->near_copies;
616         stripe = chunk;
617         dev = sector_div(stripe, geo->raid_disks);
618         if (geo->far_offset)
619                 stripe *= geo->far_copies;
620
621         sector += stripe << geo->chunk_shift;
622
623         /* and calculate all the others */
624         for (n = 0; n < geo->near_copies; n++) {
625                 int d = dev;
626                 int set;
627                 sector_t s = sector;
628                 r10bio->devs[slot].devnum = d;
629                 r10bio->devs[slot].addr = s;
630                 slot++;
631
632                 for (f = 1; f < geo->far_copies; f++) {
633                         set = d / geo->far_set_size;
634                         d += geo->near_copies;
635
636                         if ((geo->raid_disks % geo->far_set_size) &&
637                             (d > last_far_set_start)) {
638                                 d -= last_far_set_start;
639                                 d %= last_far_set_size;
640                                 d += last_far_set_start;
641                         } else {
642                                 d %= geo->far_set_size;
643                                 d += geo->far_set_size * set;
644                         }
645                         s += geo->stride;
646                         r10bio->devs[slot].devnum = d;
647                         r10bio->devs[slot].addr = s;
648                         slot++;
649                 }
650                 dev++;
651                 if (dev >= geo->raid_disks) {
652                         dev = 0;
653                         sector += (geo->chunk_mask + 1);
654                 }
655         }
656 }
657
658 static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
659 {
660         struct geom *geo = &conf->geo;
661
662         if (conf->reshape_progress != MaxSector &&
663             ((r10bio->sector >= conf->reshape_progress) !=
664              conf->mddev->reshape_backwards)) {
665                 set_bit(R10BIO_Previous, &r10bio->state);
666                 geo = &conf->prev;
667         } else
668                 clear_bit(R10BIO_Previous, &r10bio->state);
669
670         __raid10_find_phys(geo, r10bio);
671 }
672
673 static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
674 {
675         sector_t offset, chunk, vchunk;
676         /* Never use conf->prev as this is only called during resync
677          * or recovery, so reshape isn't happening
678          */
679         struct geom *geo = &conf->geo;
680         int far_set_start = (dev / geo->far_set_size) * geo->far_set_size;
681         int far_set_size = geo->far_set_size;
682         int last_far_set_start;
683
684         if (geo->raid_disks % geo->far_set_size) {
685                 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
686                 last_far_set_start *= geo->far_set_size;
687
688                 if (dev >= last_far_set_start) {
689                         far_set_size = geo->far_set_size;
690                         far_set_size += (geo->raid_disks % geo->far_set_size);
691                         far_set_start = last_far_set_start;
692                 }
693         }
694
695         offset = sector & geo->chunk_mask;
696         if (geo->far_offset) {
697                 int fc;
698                 chunk = sector >> geo->chunk_shift;
699                 fc = sector_div(chunk, geo->far_copies);
700                 dev -= fc * geo->near_copies;
701                 if (dev < far_set_start)
702                         dev += far_set_size;
703         } else {
704                 while (sector >= geo->stride) {
705                         sector -= geo->stride;
706                         if (dev < (geo->near_copies + far_set_start))
707                                 dev += far_set_size - geo->near_copies;
708                         else
709                                 dev -= geo->near_copies;
710                 }
711                 chunk = sector >> geo->chunk_shift;
712         }
713         vchunk = chunk * geo->raid_disks + dev;
714         sector_div(vchunk, geo->near_copies);
715         return (vchunk << geo->chunk_shift) + offset;
716 }
717
718 /*
719  * This routine returns the disk from which the requested read should
720  * be done. There is a per-array 'next expected sequential IO' sector
721  * number - if this matches on the next IO then we use the last disk.
722  * There is also a per-disk 'last know head position' sector that is
723  * maintained from IRQ contexts, both the normal and the resync IO
724  * completion handlers update this position correctly. If there is no
725  * perfect sequential match then we pick the disk whose head is closest.
726  *
727  * If there are 2 mirrors in the same 2 devices, performance degrades
728  * because position is mirror, not device based.
729  *
730  * The rdev for the device selected will have nr_pending incremented.
731  */
732
733 /*
734  * FIXME: possibly should rethink readbalancing and do it differently
735  * depending on near_copies / far_copies geometry.
736  */
737 static struct md_rdev *read_balance(struct r10conf *conf,
738                                     struct r10bio *r10_bio,
739                                     int *max_sectors)
740 {
741         const sector_t this_sector = r10_bio->sector;
742         int disk, slot;
743         int sectors = r10_bio->sectors;
744         int best_good_sectors;
745         sector_t new_distance, best_dist;
746         struct md_rdev *best_rdev, *rdev = NULL;
747         int do_balance;
748         int best_slot;
749         struct geom *geo = &conf->geo;
750
751         raid10_find_phys(conf, r10_bio);
752         rcu_read_lock();
753         best_slot = -1;
754         best_rdev = NULL;
755         best_dist = MaxSector;
756         best_good_sectors = 0;
757         do_balance = 1;
758         clear_bit(R10BIO_FailFast, &r10_bio->state);
759         /*
760          * Check if we can balance. We can balance on the whole
761          * device if no resync is going on (recovery is ok), or below
762          * the resync window. We take the first readable disk when
763          * above the resync window.
764          */
765         if ((conf->mddev->recovery_cp < MaxSector
766              && (this_sector + sectors >= conf->next_resync)) ||
767             (mddev_is_clustered(conf->mddev) &&
768              md_cluster_ops->area_resyncing(conf->mddev, READ, this_sector,
769                                             this_sector + sectors)))
770                 do_balance = 0;
771
772         for (slot = 0; slot < conf->copies ; slot++) {
773                 sector_t first_bad;
774                 int bad_sectors;
775                 sector_t dev_sector;
776
777                 if (r10_bio->devs[slot].bio == IO_BLOCKED)
778                         continue;
779                 disk = r10_bio->devs[slot].devnum;
780                 rdev = rcu_dereference(conf->mirrors[disk].replacement);
781                 if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
782                     r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
783                         rdev = rcu_dereference(conf->mirrors[disk].rdev);
784                 if (rdev == NULL ||
785                     test_bit(Faulty, &rdev->flags))
786                         continue;
787                 if (!test_bit(In_sync, &rdev->flags) &&
788                     r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
789                         continue;
790
791                 dev_sector = r10_bio->devs[slot].addr;
792                 if (is_badblock(rdev, dev_sector, sectors,
793                                 &first_bad, &bad_sectors)) {
794                         if (best_dist < MaxSector)
795                                 /* Already have a better slot */
796                                 continue;
797                         if (first_bad <= dev_sector) {
798                                 /* Cannot read here.  If this is the
799                                  * 'primary' device, then we must not read
800                                  * beyond 'bad_sectors' from another device.
801                                  */
802                                 bad_sectors -= (dev_sector - first_bad);
803                                 if (!do_balance && sectors > bad_sectors)
804                                         sectors = bad_sectors;
805                                 if (best_good_sectors > sectors)
806                                         best_good_sectors = sectors;
807                         } else {
808                                 sector_t good_sectors =
809                                         first_bad - dev_sector;
810                                 if (good_sectors > best_good_sectors) {
811                                         best_good_sectors = good_sectors;
812                                         best_slot = slot;
813                                         best_rdev = rdev;
814                                 }
815                                 if (!do_balance)
816                                         /* Must read from here */
817                                         break;
818                         }
819                         continue;
820                 } else
821                         best_good_sectors = sectors;
822
823                 if (!do_balance)
824                         break;
825
826                 if (best_slot >= 0)
827                         /* At least 2 disks to choose from so failfast is OK */
828                         set_bit(R10BIO_FailFast, &r10_bio->state);
829                 /* This optimisation is debatable, and completely destroys
830                  * sequential read speed for 'far copies' arrays.  So only
831                  * keep it for 'near' arrays, and review those later.
832                  */
833                 if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
834                         new_distance = 0;
835
836                 /* for far > 1 always use the lowest address */
837                 else if (geo->far_copies > 1)
838                         new_distance = r10_bio->devs[slot].addr;
839                 else
840                         new_distance = abs(r10_bio->devs[slot].addr -
841                                            conf->mirrors[disk].head_position);
842                 if (new_distance < best_dist) {
843                         best_dist = new_distance;
844                         best_slot = slot;
845                         best_rdev = rdev;
846                 }
847         }
848         if (slot >= conf->copies) {
849                 slot = best_slot;
850                 rdev = best_rdev;
851         }
852
853         if (slot >= 0) {
854                 atomic_inc(&rdev->nr_pending);
855                 r10_bio->read_slot = slot;
856         } else
857                 rdev = NULL;
858         rcu_read_unlock();
859         *max_sectors = best_good_sectors;
860
861         return rdev;
862 }
863
864 static int raid10_congested(struct mddev *mddev, int bits)
865 {
866         struct r10conf *conf = mddev->private;
867         int i, ret = 0;
868
869         if ((bits & (1 << WB_async_congested)) &&
870             conf->pending_count >= max_queued_requests)
871                 return 1;
872
873         rcu_read_lock();
874         for (i = 0;
875              (i < conf->geo.raid_disks || i < conf->prev.raid_disks)
876                      && ret == 0;
877              i++) {
878                 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
879                 if (rdev && !test_bit(Faulty, &rdev->flags)) {
880                         struct request_queue *q = bdev_get_queue(rdev->bdev);
881
882                         ret |= bdi_congested(q->backing_dev_info, bits);
883                 }
884         }
885         rcu_read_unlock();
886         return ret;
887 }
888
889 static void flush_pending_writes(struct r10conf *conf)
890 {
891         /* Any writes that have been queued but are awaiting
892          * bitmap updates get flushed here.
893          */
894         spin_lock_irq(&conf->device_lock);
895
896         if (conf->pending_bio_list.head) {
897                 struct blk_plug plug;
898                 struct bio *bio;
899
900                 bio = bio_list_get(&conf->pending_bio_list);
901                 conf->pending_count = 0;
902                 spin_unlock_irq(&conf->device_lock);
903
904                 /*
905                  * As this is called in a wait_event() loop (see freeze_array),
906                  * current->state might be TASK_UNINTERRUPTIBLE which will
907                  * cause a warning when we prepare to wait again.  As it is
908                  * rare that this path is taken, it is perfectly safe to force
909                  * us to go around the wait_event() loop again, so the warning
910                  * is a false-positive. Silence the warning by resetting
911                  * thread state
912                  */
913                 __set_current_state(TASK_RUNNING);
914
915                 blk_start_plug(&plug);
916                 /* flush any pending bitmap writes to disk
917                  * before proceeding w/ I/O */
918                 bitmap_unplug(conf->mddev->bitmap);
919                 wake_up(&conf->wait_barrier);
920
921                 while (bio) { /* submit pending writes */
922                         struct bio *next = bio->bi_next;
923                         struct md_rdev *rdev = (void*)bio->bi_disk;
924                         bio->bi_next = NULL;
925                         bio_set_dev(bio, rdev->bdev);
926                         if (test_bit(Faulty, &rdev->flags)) {
927                                 bio_io_error(bio);
928                         } else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
929                                             !blk_queue_discard(bio->bi_disk->queue)))
930                                 /* Just ignore it */
931                                 bio_endio(bio);
932                         else
933                                 generic_make_request(bio);
934                         bio = next;
935                 }
936                 blk_finish_plug(&plug);
937         } else
938                 spin_unlock_irq(&conf->device_lock);
939 }
940
941 /* Barriers....
942  * Sometimes we need to suspend IO while we do something else,
943  * either some resync/recovery, or reconfigure the array.
944  * To do this we raise a 'barrier'.
945  * The 'barrier' is a counter that can be raised multiple times
946  * to count how many activities are happening which preclude
947  * normal IO.
948  * We can only raise the barrier if there is no pending IO.
949  * i.e. if nr_pending == 0.
950  * We choose only to raise the barrier if no-one is waiting for the
951  * barrier to go down.  This means that as soon as an IO request
952  * is ready, no other operations which require a barrier will start
953  * until the IO request has had a chance.
954  *
955  * So: regular IO calls 'wait_barrier'.  When that returns there
956  *    is no backgroup IO happening,  It must arrange to call
957  *    allow_barrier when it has finished its IO.
958  * backgroup IO calls must call raise_barrier.  Once that returns
959  *    there is no normal IO happeing.  It must arrange to call
960  *    lower_barrier when the particular background IO completes.
961  */
962
963 static void raise_barrier(struct r10conf *conf, int force)
964 {
965         BUG_ON(force && !conf->barrier);
966         spin_lock_irq(&conf->resync_lock);
967
968         /* Wait until no block IO is waiting (unless 'force') */
969         wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
970                             conf->resync_lock);
971
972         /* block any new IO from starting */
973         conf->barrier++;
974
975         /* Now wait for all pending IO to complete */
976         wait_event_lock_irq(conf->wait_barrier,
977                             !atomic_read(&conf->nr_pending) && conf->barrier < RESYNC_DEPTH,
978                             conf->resync_lock);
979
980         spin_unlock_irq(&conf->resync_lock);
981 }
982
983 static void lower_barrier(struct r10conf *conf)
984 {
985         unsigned long flags;
986         spin_lock_irqsave(&conf->resync_lock, flags);
987         conf->barrier--;
988         spin_unlock_irqrestore(&conf->resync_lock, flags);
989         wake_up(&conf->wait_barrier);
990 }
991
992 static void wait_barrier(struct r10conf *conf)
993 {
994         spin_lock_irq(&conf->resync_lock);
995         if (conf->barrier) {
996                 conf->nr_waiting++;
997                 /* Wait for the barrier to drop.
998                  * However if there are already pending
999                  * requests (preventing the barrier from
1000                  * rising completely), and the
1001                  * pre-process bio queue isn't empty,
1002                  * then don't wait, as we need to empty
1003                  * that queue to get the nr_pending
1004                  * count down.
1005                  */
1006                 raid10_log(conf->mddev, "wait barrier");
1007                 wait_event_lock_irq(conf->wait_barrier,
1008                                     !conf->barrier ||
1009                                     (atomic_read(&conf->nr_pending) &&
1010                                      current->bio_list &&
1011                                      (!bio_list_empty(&current->bio_list[0]) ||
1012                                       !bio_list_empty(&current->bio_list[1]))),
1013                                     conf->resync_lock);
1014                 conf->nr_waiting--;
1015                 if (!conf->nr_waiting)
1016                         wake_up(&conf->wait_barrier);
1017         }
1018         atomic_inc(&conf->nr_pending);
1019         spin_unlock_irq(&conf->resync_lock);
1020 }
1021
1022 static void allow_barrier(struct r10conf *conf)
1023 {
1024         if ((atomic_dec_and_test(&conf->nr_pending)) ||
1025                         (conf->array_freeze_pending))
1026                 wake_up(&conf->wait_barrier);
1027 }
1028
1029 static void freeze_array(struct r10conf *conf, int extra)
1030 {
1031         /* stop syncio and normal IO and wait for everything to
1032          * go quiet.
1033          * We increment barrier and nr_waiting, and then
1034          * wait until nr_pending match nr_queued+extra
1035          * This is called in the context of one normal IO request
1036          * that has failed. Thus any sync request that might be pending
1037          * will be blocked by nr_pending, and we need to wait for
1038          * pending IO requests to complete or be queued for re-try.
1039          * Thus the number queued (nr_queued) plus this request (extra)
1040          * must match the number of pending IOs (nr_pending) before
1041          * we continue.
1042          */
1043         spin_lock_irq(&conf->resync_lock);
1044         conf->array_freeze_pending++;
1045         conf->barrier++;
1046         conf->nr_waiting++;
1047         wait_event_lock_irq_cmd(conf->wait_barrier,
1048                                 atomic_read(&conf->nr_pending) == conf->nr_queued+extra,
1049                                 conf->resync_lock,
1050                                 flush_pending_writes(conf));
1051
1052         conf->array_freeze_pending--;
1053         spin_unlock_irq(&conf->resync_lock);
1054 }
1055
1056 static void unfreeze_array(struct r10conf *conf)
1057 {
1058         /* reverse the effect of the freeze */
1059         spin_lock_irq(&conf->resync_lock);
1060         conf->barrier--;
1061         conf->nr_waiting--;
1062         wake_up(&conf->wait_barrier);
1063         spin_unlock_irq(&conf->resync_lock);
1064 }
1065
1066 static sector_t choose_data_offset(struct r10bio *r10_bio,
1067                                    struct md_rdev *rdev)
1068 {
1069         if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
1070             test_bit(R10BIO_Previous, &r10_bio->state))
1071                 return rdev->data_offset;
1072         else
1073                 return rdev->new_data_offset;
1074 }
1075
1076 struct raid10_plug_cb {
1077         struct blk_plug_cb      cb;
1078         struct bio_list         pending;
1079         int                     pending_cnt;
1080 };
1081
1082 static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
1083 {
1084         struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb,
1085                                                    cb);
1086         struct mddev *mddev = plug->cb.data;
1087         struct r10conf *conf = mddev->private;
1088         struct bio *bio;
1089
1090         if (from_schedule || current->bio_list) {
1091                 spin_lock_irq(&conf->device_lock);
1092                 bio_list_merge(&conf->pending_bio_list, &plug->pending);
1093                 conf->pending_count += plug->pending_cnt;
1094                 spin_unlock_irq(&conf->device_lock);
1095                 wake_up(&conf->wait_barrier);
1096                 md_wakeup_thread(mddev->thread);
1097                 kfree(plug);
1098                 return;
1099         }
1100
1101         /* we aren't scheduling, so we can do the write-out directly. */
1102         bio = bio_list_get(&plug->pending);
1103         bitmap_unplug(mddev->bitmap);
1104         wake_up(&conf->wait_barrier);
1105
1106         while (bio) { /* submit pending writes */
1107                 struct bio *next = bio->bi_next;
1108                 struct md_rdev *rdev = (void*)bio->bi_disk;
1109                 bio->bi_next = NULL;
1110                 bio_set_dev(bio, rdev->bdev);
1111                 if (test_bit(Faulty, &rdev->flags)) {
1112                         bio_io_error(bio);
1113                 } else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
1114                                     !blk_queue_discard(bio->bi_disk->queue)))
1115                         /* Just ignore it */
1116                         bio_endio(bio);
1117                 else
1118                         generic_make_request(bio);
1119                 bio = next;
1120         }
1121         kfree(plug);
1122 }
1123
1124 static void raid10_read_request(struct mddev *mddev, struct bio *bio,
1125                                 struct r10bio *r10_bio)
1126 {
1127         struct r10conf *conf = mddev->private;
1128         struct bio *read_bio;
1129         const int op = bio_op(bio);
1130         const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
1131         int max_sectors;
1132         sector_t sectors;
1133         struct md_rdev *rdev;
1134         char b[BDEVNAME_SIZE];
1135         int slot = r10_bio->read_slot;
1136         struct md_rdev *err_rdev = NULL;
1137         gfp_t gfp = GFP_NOIO;
1138
1139         if (r10_bio->devs[slot].rdev) {
1140                 /*
1141                  * This is an error retry, but we cannot
1142                  * safely dereference the rdev in the r10_bio,
1143                  * we must use the one in conf.
1144                  * If it has already been disconnected (unlikely)
1145                  * we lose the device name in error messages.
1146                  */
1147                 int disk;
1148                 /*
1149                  * As we are blocking raid10, it is a little safer to
1150                  * use __GFP_HIGH.
1151                  */
1152                 gfp = GFP_NOIO | __GFP_HIGH;
1153
1154                 rcu_read_lock();
1155                 disk = r10_bio->devs[slot].devnum;
1156                 err_rdev = rcu_dereference(conf->mirrors[disk].rdev);
1157                 if (err_rdev)
1158                         bdevname(err_rdev->bdev, b);
1159                 else {
1160                         strcpy(b, "???");
1161                         /* This never gets dereferenced */
1162                         err_rdev = r10_bio->devs[slot].rdev;
1163                 }
1164                 rcu_read_unlock();
1165         }
1166         /*
1167          * Register the new request and wait if the reconstruction
1168          * thread has put up a bar for new requests.
1169          * Continue immediately if no resync is active currently.
1170          */
1171         wait_barrier(conf);
1172
1173         sectors = r10_bio->sectors;
1174         while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1175             bio->bi_iter.bi_sector < conf->reshape_progress &&
1176             bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
1177                 /*
1178                  * IO spans the reshape position.  Need to wait for reshape to
1179                  * pass
1180                  */
1181                 raid10_log(conf->mddev, "wait reshape");
1182                 allow_barrier(conf);
1183                 wait_event(conf->wait_barrier,
1184                            conf->reshape_progress <= bio->bi_iter.bi_sector ||
1185                            conf->reshape_progress >= bio->bi_iter.bi_sector +
1186                            sectors);
1187                 wait_barrier(conf);
1188         }
1189
1190         rdev = read_balance(conf, r10_bio, &max_sectors);
1191         if (!rdev) {
1192                 if (err_rdev) {
1193                         pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n",
1194                                             mdname(mddev), b,
1195                                             (unsigned long long)r10_bio->sector);
1196                 }
1197                 raid_end_bio_io(r10_bio);
1198                 return;
1199         }
1200         if (err_rdev)
1201                 pr_err_ratelimited("md/raid10:%s: %s: redirecting sector %llu to another mirror\n",
1202                                    mdname(mddev),
1203                                    bdevname(rdev->bdev, b),
1204                                    (unsigned long long)r10_bio->sector);
1205         if (max_sectors < bio_sectors(bio)) {
1206                 struct bio *split = bio_split(bio, max_sectors,
1207                                               gfp, conf->bio_split);
1208                 bio_chain(split, bio);
1209                 generic_make_request(bio);
1210                 bio = split;
1211                 r10_bio->master_bio = bio;
1212                 r10_bio->sectors = max_sectors;
1213         }
1214         slot = r10_bio->read_slot;
1215
1216         read_bio = bio_clone_fast(bio, gfp, mddev->bio_set);
1217
1218         r10_bio->devs[slot].bio = read_bio;
1219         r10_bio->devs[slot].rdev = rdev;
1220
1221         read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr +
1222                 choose_data_offset(r10_bio, rdev);
1223         bio_set_dev(read_bio, rdev->bdev);
1224         read_bio->bi_end_io = raid10_end_read_request;
1225         bio_set_op_attrs(read_bio, op, do_sync);
1226         if (test_bit(FailFast, &rdev->flags) &&
1227             test_bit(R10BIO_FailFast, &r10_bio->state))
1228                 read_bio->bi_opf |= MD_FAILFAST;
1229         read_bio->bi_private = r10_bio;
1230
1231         if (mddev->gendisk)
1232                 trace_block_bio_remap(read_bio->bi_disk->queue,
1233                                       read_bio, disk_devt(mddev->gendisk),
1234                                       r10_bio->sector);
1235         generic_make_request(read_bio);
1236         return;
1237 }
1238
1239 static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,
1240                                   struct bio *bio, bool replacement,
1241                                   int n_copy)
1242 {
1243         const int op = bio_op(bio);
1244         const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
1245         const unsigned long do_fua = (bio->bi_opf & REQ_FUA);
1246         unsigned long flags;
1247         struct blk_plug_cb *cb;
1248         struct raid10_plug_cb *plug = NULL;
1249         struct r10conf *conf = mddev->private;
1250         struct md_rdev *rdev;
1251         int devnum = r10_bio->devs[n_copy].devnum;
1252         struct bio *mbio;
1253
1254         if (replacement) {
1255                 rdev = conf->mirrors[devnum].replacement;
1256                 if (rdev == NULL) {
1257                         /* Replacement just got moved to main 'rdev' */
1258                         smp_mb();
1259                         rdev = conf->mirrors[devnum].rdev;
1260                 }
1261         } else
1262                 rdev = conf->mirrors[devnum].rdev;
1263
1264         mbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
1265         if (replacement)
1266                 r10_bio->devs[n_copy].repl_bio = mbio;
1267         else
1268                 r10_bio->devs[n_copy].bio = mbio;
1269
1270         mbio->bi_iter.bi_sector = (r10_bio->devs[n_copy].addr +
1271                                    choose_data_offset(r10_bio, rdev));
1272         bio_set_dev(mbio, rdev->bdev);
1273         mbio->bi_end_io = raid10_end_write_request;
1274         bio_set_op_attrs(mbio, op, do_sync | do_fua);
1275         if (!replacement && test_bit(FailFast,
1276                                      &conf->mirrors[devnum].rdev->flags)
1277                          && enough(conf, devnum))
1278                 mbio->bi_opf |= MD_FAILFAST;
1279         mbio->bi_private = r10_bio;
1280
1281         if (conf->mddev->gendisk)
1282                 trace_block_bio_remap(mbio->bi_disk->queue,
1283                                       mbio, disk_devt(conf->mddev->gendisk),
1284                                       r10_bio->sector);
1285         /* flush_pending_writes() needs access to the rdev so...*/
1286         mbio->bi_disk = (void *)rdev;
1287
1288         atomic_inc(&r10_bio->remaining);
1289
1290         cb = blk_check_plugged(raid10_unplug, mddev, sizeof(*plug));
1291         if (cb)
1292                 plug = container_of(cb, struct raid10_plug_cb, cb);
1293         else
1294                 plug = NULL;
1295         if (plug) {
1296                 bio_list_add(&plug->pending, mbio);
1297                 plug->pending_cnt++;
1298         } else {
1299                 spin_lock_irqsave(&conf->device_lock, flags);
1300                 bio_list_add(&conf->pending_bio_list, mbio);
1301                 conf->pending_count++;
1302                 spin_unlock_irqrestore(&conf->device_lock, flags);
1303                 md_wakeup_thread(mddev->thread);
1304         }
1305 }
1306
1307 static void raid10_write_request(struct mddev *mddev, struct bio *bio,
1308                                  struct r10bio *r10_bio)
1309 {
1310         struct r10conf *conf = mddev->private;
1311         int i;
1312         struct md_rdev *blocked_rdev;
1313         sector_t sectors;
1314         int max_sectors;
1315
1316         if ((mddev_is_clustered(mddev) &&
1317              md_cluster_ops->area_resyncing(mddev, WRITE,
1318                                             bio->bi_iter.bi_sector,
1319                                             bio_end_sector(bio)))) {
1320                 DEFINE_WAIT(w);
1321                 for (;;) {
1322                         prepare_to_wait(&conf->wait_barrier,
1323                                         &w, TASK_IDLE);
1324                         if (!md_cluster_ops->area_resyncing(mddev, WRITE,
1325                                  bio->bi_iter.bi_sector, bio_end_sector(bio)))
1326                                 break;
1327                         schedule();
1328                 }
1329                 finish_wait(&conf->wait_barrier, &w);
1330         }
1331
1332         /*
1333          * Register the new request and wait if the reconstruction
1334          * thread has put up a bar for new requests.
1335          * Continue immediately if no resync is active currently.
1336          */
1337         wait_barrier(conf);
1338
1339         sectors = r10_bio->sectors;
1340         while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1341             bio->bi_iter.bi_sector < conf->reshape_progress &&
1342             bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
1343                 /*
1344                  * IO spans the reshape position.  Need to wait for reshape to
1345                  * pass
1346                  */
1347                 raid10_log(conf->mddev, "wait reshape");
1348                 allow_barrier(conf);
1349                 wait_event(conf->wait_barrier,
1350                            conf->reshape_progress <= bio->bi_iter.bi_sector ||
1351                            conf->reshape_progress >= bio->bi_iter.bi_sector +
1352                            sectors);
1353                 wait_barrier(conf);
1354         }
1355
1356         if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1357             (mddev->reshape_backwards
1358              ? (bio->bi_iter.bi_sector < conf->reshape_safe &&
1359                 bio->bi_iter.bi_sector + sectors > conf->reshape_progress)
1360              : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe &&
1361                 bio->bi_iter.bi_sector < conf->reshape_progress))) {
1362                 /* Need to update reshape_position in metadata */
1363                 mddev->reshape_position = conf->reshape_progress;
1364                 set_mask_bits(&mddev->sb_flags, 0,
1365                               BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1366                 md_wakeup_thread(mddev->thread);
1367                 raid10_log(conf->mddev, "wait reshape metadata");
1368                 wait_event(mddev->sb_wait,
1369                            !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
1370
1371                 conf->reshape_safe = mddev->reshape_position;
1372         }
1373
1374         if (conf->pending_count >= max_queued_requests) {
1375                 md_wakeup_thread(mddev->thread);
1376                 raid10_log(mddev, "wait queued");
1377                 wait_event(conf->wait_barrier,
1378                            conf->pending_count < max_queued_requests);
1379         }
1380         /* first select target devices under rcu_lock and
1381          * inc refcount on their rdev.  Record them by setting
1382          * bios[x] to bio
1383          * If there are known/acknowledged bad blocks on any device
1384          * on which we have seen a write error, we want to avoid
1385          * writing to those blocks.  This potentially requires several
1386          * writes to write around the bad blocks.  Each set of writes
1387          * gets its own r10_bio with a set of bios attached.
1388          */
1389
1390         r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
1391         raid10_find_phys(conf, r10_bio);
1392 retry_write:
1393         blocked_rdev = NULL;
1394         rcu_read_lock();
1395         max_sectors = r10_bio->sectors;
1396
1397         for (i = 0;  i < conf->copies; i++) {
1398                 int d = r10_bio->devs[i].devnum;
1399                 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
1400                 struct md_rdev *rrdev = rcu_dereference(
1401                         conf->mirrors[d].replacement);
1402                 if (rdev == rrdev)
1403                         rrdev = NULL;
1404                 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
1405                         atomic_inc(&rdev->nr_pending);
1406                         blocked_rdev = rdev;
1407                         break;
1408                 }
1409                 if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
1410                         atomic_inc(&rrdev->nr_pending);
1411                         blocked_rdev = rrdev;
1412                         break;
1413                 }
1414                 if (rdev && (test_bit(Faulty, &rdev->flags)))
1415                         rdev = NULL;
1416                 if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1417                         rrdev = NULL;
1418
1419                 r10_bio->devs[i].bio = NULL;
1420                 r10_bio->devs[i].repl_bio = NULL;
1421
1422                 if (!rdev && !rrdev) {
1423                         set_bit(R10BIO_Degraded, &r10_bio->state);
1424                         continue;
1425                 }
1426                 if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1427                         sector_t first_bad;
1428                         sector_t dev_sector = r10_bio->devs[i].addr;
1429                         int bad_sectors;
1430                         int is_bad;
1431
1432                         is_bad = is_badblock(rdev, dev_sector, max_sectors,
1433                                              &first_bad, &bad_sectors);
1434                         if (is_bad < 0) {
1435                                 /* Mustn't write here until the bad block
1436                                  * is acknowledged
1437                                  */
1438                                 atomic_inc(&rdev->nr_pending);
1439                                 set_bit(BlockedBadBlocks, &rdev->flags);
1440                                 blocked_rdev = rdev;
1441                                 break;
1442                         }
1443                         if (is_bad && first_bad <= dev_sector) {
1444                                 /* Cannot write here at all */
1445                                 bad_sectors -= (dev_sector - first_bad);
1446                                 if (bad_sectors < max_sectors)
1447                                         /* Mustn't write more than bad_sectors
1448                                          * to other devices yet
1449                                          */
1450                                         max_sectors = bad_sectors;
1451                                 /* We don't set R10BIO_Degraded as that
1452                                  * only applies if the disk is missing,
1453                                  * so it might be re-added, and we want to
1454                                  * know to recover this chunk.
1455                                  * In this case the device is here, and the
1456                                  * fact that this chunk is not in-sync is
1457                                  * recorded in the bad block log.
1458                                  */
1459                                 continue;
1460                         }
1461                         if (is_bad) {
1462                                 int good_sectors = first_bad - dev_sector;
1463                                 if (good_sectors < max_sectors)
1464                                         max_sectors = good_sectors;
1465                         }
1466                 }
1467                 if (rdev) {
1468                         r10_bio->devs[i].bio = bio;
1469                         atomic_inc(&rdev->nr_pending);
1470                 }
1471                 if (rrdev) {
1472                         r10_bio->devs[i].repl_bio = bio;
1473                         atomic_inc(&rrdev->nr_pending);
1474                 }
1475         }
1476         rcu_read_unlock();
1477
1478         if (unlikely(blocked_rdev)) {
1479                 /* Have to wait for this device to get unblocked, then retry */
1480                 int j;
1481                 int d;
1482
1483                 for (j = 0; j < i; j++) {
1484                         if (r10_bio->devs[j].bio) {
1485                                 d = r10_bio->devs[j].devnum;
1486                                 rdev_dec_pending(conf->mirrors[d].rdev, mddev);
1487                         }
1488                         if (r10_bio->devs[j].repl_bio) {
1489                                 struct md_rdev *rdev;
1490                                 d = r10_bio->devs[j].devnum;
1491                                 rdev = conf->mirrors[d].replacement;
1492                                 if (!rdev) {
1493                                         /* Race with remove_disk */
1494                                         smp_mb();
1495                                         rdev = conf->mirrors[d].rdev;
1496                                 }
1497                                 rdev_dec_pending(rdev, mddev);
1498                         }
1499                 }
1500                 allow_barrier(conf);
1501                 raid10_log(conf->mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);
1502                 md_wait_for_blocked_rdev(blocked_rdev, mddev);
1503                 wait_barrier(conf);
1504                 goto retry_write;
1505         }
1506
1507         if (max_sectors < r10_bio->sectors)
1508                 r10_bio->sectors = max_sectors;
1509
1510         if (r10_bio->sectors < bio_sectors(bio)) {
1511                 struct bio *split = bio_split(bio, r10_bio->sectors,
1512                                               GFP_NOIO, conf->bio_split);
1513                 bio_chain(split, bio);
1514                 generic_make_request(bio);
1515                 bio = split;
1516                 r10_bio->master_bio = bio;
1517         }
1518
1519         atomic_set(&r10_bio->remaining, 1);
1520         bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1521
1522         for (i = 0; i < conf->copies; i++) {
1523                 if (r10_bio->devs[i].bio)
1524                         raid10_write_one_disk(mddev, r10_bio, bio, false, i);
1525                 if (r10_bio->devs[i].repl_bio)
1526                         raid10_write_one_disk(mddev, r10_bio, bio, true, i);
1527         }
1528         one_write_done(r10_bio);
1529 }
1530
1531 static void __make_request(struct mddev *mddev, struct bio *bio, int sectors)
1532 {
1533         struct r10conf *conf = mddev->private;
1534         struct r10bio *r10_bio;
1535
1536         r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1537
1538         r10_bio->master_bio = bio;
1539         r10_bio->sectors = sectors;
1540
1541         r10_bio->mddev = mddev;
1542         r10_bio->sector = bio->bi_iter.bi_sector;
1543         r10_bio->state = 0;
1544         memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * conf->copies);
1545
1546         if (bio_data_dir(bio) == READ)
1547                 raid10_read_request(mddev, bio, r10_bio);
1548         else
1549                 raid10_write_request(mddev, bio, r10_bio);
1550 }
1551
1552 static bool raid10_make_request(struct mddev *mddev, struct bio *bio)
1553 {
1554         struct r10conf *conf = mddev->private;
1555         sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
1556         int chunk_sects = chunk_mask + 1;
1557         int sectors = bio_sectors(bio);
1558
1559         if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1560                 md_flush_request(mddev, bio);
1561                 return true;
1562         }
1563
1564         if (!md_write_start(mddev, bio))
1565                 return false;
1566
1567         /*
1568          * If this request crosses a chunk boundary, we need to split
1569          * it.
1570          */
1571         if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
1572                      sectors > chunk_sects
1573                      && (conf->geo.near_copies < conf->geo.raid_disks
1574                          || conf->prev.near_copies <
1575                          conf->prev.raid_disks)))
1576                 sectors = chunk_sects -
1577                         (bio->bi_iter.bi_sector &
1578                          (chunk_sects - 1));
1579         __make_request(mddev, bio, sectors);
1580
1581         /* In case raid10d snuck in to freeze_array */
1582         wake_up(&conf->wait_barrier);
1583         return true;
1584 }
1585
1586 static void raid10_status(struct seq_file *seq, struct mddev *mddev)
1587 {
1588         struct r10conf *conf = mddev->private;
1589         int i;
1590
1591         if (conf->geo.near_copies < conf->geo.raid_disks)
1592                 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
1593         if (conf->geo.near_copies > 1)
1594                 seq_printf(seq, " %d near-copies", conf->geo.near_copies);
1595         if (conf->geo.far_copies > 1) {
1596                 if (conf->geo.far_offset)
1597                         seq_printf(seq, " %d offset-copies", conf->geo.far_copies);
1598                 else
1599                         seq_printf(seq, " %d far-copies", conf->geo.far_copies);
1600                 if (conf->geo.far_set_size != conf->geo.raid_disks)
1601                         seq_printf(seq, " %d devices per set", conf->geo.far_set_size);
1602         }
1603         seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,
1604                                         conf->geo.raid_disks - mddev->degraded);
1605         rcu_read_lock();
1606         for (i = 0; i < conf->geo.raid_disks; i++) {
1607                 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1608                 seq_printf(seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
1609         }
1610         rcu_read_unlock();
1611         seq_printf(seq, "]");
1612 }
1613
1614 /* check if there are enough drives for
1615  * every block to appear on atleast one.
1616  * Don't consider the device numbered 'ignore'
1617  * as we might be about to remove it.
1618  */
1619 static int _enough(struct r10conf *conf, int previous, int ignore)
1620 {
1621         int first = 0;
1622         int has_enough = 0;
1623         int disks, ncopies;
1624         if (previous) {
1625                 disks = conf->prev.raid_disks;
1626                 ncopies = conf->prev.near_copies;
1627         } else {
1628                 disks = conf->geo.raid_disks;
1629                 ncopies = conf->geo.near_copies;
1630         }
1631
1632         rcu_read_lock();
1633         do {
1634                 int n = conf->copies;
1635                 int cnt = 0;
1636                 int this = first;
1637                 while (n--) {
1638                         struct md_rdev *rdev;
1639                         if (this != ignore &&
1640                             (rdev = rcu_dereference(conf->mirrors[this].rdev)) &&
1641                             test_bit(In_sync, &rdev->flags))
1642                                 cnt++;
1643                         this = (this+1) % disks;
1644                 }
1645                 if (cnt == 0)
1646                         goto out;
1647                 first = (first + ncopies) % disks;
1648         } while (first != 0);
1649         has_enough = 1;
1650 out:
1651         rcu_read_unlock();
1652         return has_enough;
1653 }
1654
1655 static int enough(struct r10conf *conf, int ignore)
1656 {
1657         /* when calling 'enough', both 'prev' and 'geo' must
1658          * be stable.
1659          * This is ensured if ->reconfig_mutex or ->device_lock
1660          * is held.
1661          */
1662         return _enough(conf, 0, ignore) &&
1663                 _enough(conf, 1, ignore);
1664 }
1665
1666 static void raid10_error(struct mddev *mddev, struct md_rdev *rdev)
1667 {
1668         char b[BDEVNAME_SIZE];
1669         struct r10conf *conf = mddev->private;
1670         unsigned long flags;
1671
1672         /*
1673          * If it is not operational, then we have already marked it as dead
1674          * else if it is the last working disks, ignore the error, let the
1675          * next level up know.
1676          * else mark the drive as failed
1677          */
1678         spin_lock_irqsave(&conf->device_lock, flags);
1679         if (test_bit(In_sync, &rdev->flags)
1680             && !enough(conf, rdev->raid_disk)) {
1681                 /*
1682                  * Don't fail the drive, just return an IO error.
1683                  */
1684                 spin_unlock_irqrestore(&conf->device_lock, flags);
1685                 return;
1686         }
1687         if (test_and_clear_bit(In_sync, &rdev->flags))
1688                 mddev->degraded++;
1689         /*
1690          * If recovery is running, make sure it aborts.
1691          */
1692         set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1693         set_bit(Blocked, &rdev->flags);
1694         set_bit(Faulty, &rdev->flags);
1695         set_mask_bits(&mddev->sb_flags, 0,
1696                       BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1697         spin_unlock_irqrestore(&conf->device_lock, flags);
1698         pr_crit("md/raid10:%s: Disk failure on %s, disabling device.\n"
1699                 "md/raid10:%s: Operation continuing on %d devices.\n",
1700                 mdname(mddev), bdevname(rdev->bdev, b),
1701                 mdname(mddev), conf->geo.raid_disks - mddev->degraded);
1702 }
1703
1704 static void print_conf(struct r10conf *conf)
1705 {
1706         int i;
1707         struct md_rdev *rdev;
1708
1709         pr_debug("RAID10 conf printout:\n");
1710         if (!conf) {
1711                 pr_debug("(!conf)\n");
1712                 return;
1713         }
1714         pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
1715                  conf->geo.raid_disks);
1716
1717         /* This is only called with ->reconfix_mutex held, so
1718          * rcu protection of rdev is not needed */
1719         for (i = 0; i < conf->geo.raid_disks; i++) {
1720                 char b[BDEVNAME_SIZE];
1721                 rdev = conf->mirrors[i].rdev;
1722                 if (rdev)
1723                         pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n",
1724                                  i, !test_bit(In_sync, &rdev->flags),
1725                                  !test_bit(Faulty, &rdev->flags),
1726                                  bdevname(rdev->bdev,b));
1727         }
1728 }
1729
1730 static void close_sync(struct r10conf *conf)
1731 {
1732         wait_barrier(conf);
1733         allow_barrier(conf);
1734
1735         mempool_destroy(conf->r10buf_pool);
1736         conf->r10buf_pool = NULL;
1737 }
1738
1739 static int raid10_spare_active(struct mddev *mddev)
1740 {
1741         int i;
1742         struct r10conf *conf = mddev->private;
1743         struct raid10_info *tmp;
1744         int count = 0;
1745         unsigned long flags;
1746
1747         /*
1748          * Find all non-in_sync disks within the RAID10 configuration
1749          * and mark them in_sync
1750          */
1751         for (i = 0; i < conf->geo.raid_disks; i++) {
1752                 tmp = conf->mirrors + i;
1753                 if (tmp->replacement
1754                     && tmp->replacement->recovery_offset == MaxSector
1755                     && !test_bit(Faulty, &tmp->replacement->flags)
1756                     && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
1757                         /* Replacement has just become active */
1758                         if (!tmp->rdev
1759                             || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
1760                                 count++;
1761                         if (tmp->rdev) {
1762                                 /* Replaced device not technically faulty,
1763                                  * but we need to be sure it gets removed
1764                                  * and never re-added.
1765                                  */
1766                                 set_bit(Faulty, &tmp->rdev->flags);
1767                                 sysfs_notify_dirent_safe(
1768                                         tmp->rdev->sysfs_state);
1769                         }
1770                         sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
1771                 } else if (tmp->rdev
1772                            && tmp->rdev->recovery_offset == MaxSector
1773                            && !test_bit(Faulty, &tmp->rdev->flags)
1774                            && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
1775                         count++;
1776                         sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
1777                 }
1778         }
1779         spin_lock_irqsave(&conf->device_lock, flags);
1780         mddev->degraded -= count;
1781         spin_unlock_irqrestore(&conf->device_lock, flags);
1782
1783         print_conf(conf);
1784         return count;
1785 }
1786
1787 static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1788 {
1789         struct r10conf *conf = mddev->private;
1790         int err = -EEXIST;
1791         int mirror;
1792         int first = 0;
1793         int last = conf->geo.raid_disks - 1;
1794
1795         if (mddev->recovery_cp < MaxSector)
1796                 /* only hot-add to in-sync arrays, as recovery is
1797                  * very different from resync
1798                  */
1799                 return -EBUSY;
1800         if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1))
1801                 return -EINVAL;
1802
1803         if (md_integrity_add_rdev(rdev, mddev))
1804                 return -ENXIO;
1805
1806         if (rdev->raid_disk >= 0)
1807                 first = last = rdev->raid_disk;
1808
1809         if (rdev->saved_raid_disk >= first &&
1810             conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1811                 mirror = rdev->saved_raid_disk;
1812         else
1813                 mirror = first;
1814         for ( ; mirror <= last ; mirror++) {
1815                 struct raid10_info *p = &conf->mirrors[mirror];
1816                 if (p->recovery_disabled == mddev->recovery_disabled)
1817                         continue;
1818                 if (p->rdev) {
1819                         if (!test_bit(WantReplacement, &p->rdev->flags) ||
1820                             p->replacement != NULL)
1821                                 continue;
1822                         clear_bit(In_sync, &rdev->flags);
1823                         set_bit(Replacement, &rdev->flags);
1824                         rdev->raid_disk = mirror;
1825                         err = 0;
1826                         if (mddev->gendisk)
1827                                 disk_stack_limits(mddev->gendisk, rdev->bdev,
1828                                                   rdev->data_offset << 9);
1829                         conf->fullsync = 1;
1830                         rcu_assign_pointer(p->replacement, rdev);
1831                         break;
1832                 }
1833
1834                 if (mddev->gendisk)
1835                         disk_stack_limits(mddev->gendisk, rdev->bdev,
1836                                           rdev->data_offset << 9);
1837
1838                 p->head_position = 0;
1839                 p->recovery_disabled = mddev->recovery_disabled - 1;
1840                 rdev->raid_disk = mirror;
1841                 err = 0;
1842                 if (rdev->saved_raid_disk != mirror)
1843                         conf->fullsync = 1;
1844                 rcu_assign_pointer(p->rdev, rdev);
1845                 break;
1846         }
1847         if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
1848                 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
1849
1850         print_conf(conf);
1851         return err;
1852 }
1853
1854 static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
1855 {
1856         struct r10conf *conf = mddev->private;
1857         int err = 0;
1858         int number = rdev->raid_disk;
1859         struct md_rdev **rdevp;
1860         struct raid10_info *p = conf->mirrors + number;
1861
1862         print_conf(conf);
1863         if (rdev == p->rdev)
1864                 rdevp = &p->rdev;
1865         else if (rdev == p->replacement)
1866                 rdevp = &p->replacement;
1867         else
1868                 return 0;
1869
1870         if (test_bit(In_sync, &rdev->flags) ||
1871             atomic_read(&rdev->nr_pending)) {
1872                 err = -EBUSY;
1873                 goto abort;
1874         }
1875         /* Only remove non-faulty devices if recovery
1876          * is not possible.
1877          */
1878         if (!test_bit(Faulty, &rdev->flags) &&
1879             mddev->recovery_disabled != p->recovery_disabled &&
1880             (!p->replacement || p->replacement == rdev) &&
1881             number < conf->geo.raid_disks &&
1882             enough(conf, -1)) {
1883                 err = -EBUSY;
1884                 goto abort;
1885         }
1886         *rdevp = NULL;
1887         if (!test_bit(RemoveSynchronized, &rdev->flags)) {
1888                 synchronize_rcu();
1889                 if (atomic_read(&rdev->nr_pending)) {
1890                         /* lost the race, try later */
1891                         err = -EBUSY;
1892                         *rdevp = rdev;
1893                         goto abort;
1894                 }
1895         }
1896         if (p->replacement) {
1897                 /* We must have just cleared 'rdev' */
1898                 p->rdev = p->replacement;
1899                 clear_bit(Replacement, &p->replacement->flags);
1900                 smp_mb(); /* Make sure other CPUs may see both as identical
1901                            * but will never see neither -- if they are careful.
1902                            */
1903                 p->replacement = NULL;
1904         }
1905
1906         clear_bit(WantReplacement, &rdev->flags);
1907         err = md_integrity_register(mddev);
1908
1909 abort:
1910
1911         print_conf(conf);
1912         return err;
1913 }
1914
1915 static void __end_sync_read(struct r10bio *r10_bio, struct bio *bio, int d)
1916 {
1917         struct r10conf *conf = r10_bio->mddev->private;
1918
1919         if (!bio->bi_status)
1920                 set_bit(R10BIO_Uptodate, &r10_bio->state);
1921         else
1922                 /* The write handler will notice the lack of
1923                  * R10BIO_Uptodate and record any errors etc
1924                  */
1925                 atomic_add(r10_bio->sectors,
1926                            &conf->mirrors[d].rdev->corrected_errors);
1927
1928         /* for reconstruct, we always reschedule after a read.
1929          * for resync, only after all reads
1930          */
1931         rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
1932         if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
1933             atomic_dec_and_test(&r10_bio->remaining)) {
1934                 /* we have read all the blocks,
1935                  * do the comparison in process context in raid10d
1936                  */
1937                 reschedule_retry(r10_bio);
1938         }
1939 }
1940
1941 static void end_sync_read(struct bio *bio)
1942 {
1943         struct r10bio *r10_bio = get_resync_r10bio(bio);
1944         struct r10conf *conf = r10_bio->mddev->private;
1945         int d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
1946
1947         __end_sync_read(r10_bio, bio, d);
1948 }
1949
1950 static void end_reshape_read(struct bio *bio)
1951 {
1952         /* reshape read bio isn't allocated from r10buf_pool */
1953         struct r10bio *r10_bio = bio->bi_private;
1954
1955         __end_sync_read(r10_bio, bio, r10_bio->read_slot);
1956 }
1957
1958 static void end_sync_request(struct r10bio *r10_bio)
1959 {
1960         struct mddev *mddev = r10_bio->mddev;
1961
1962         while (atomic_dec_and_test(&r10_bio->remaining)) {
1963                 if (r10_bio->master_bio == NULL) {
1964                         /* the primary of several recovery bios */
1965                         sector_t s = r10_bio->sectors;
1966                         if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1967                             test_bit(R10BIO_WriteError, &r10_bio->state))
1968                                 reschedule_retry(r10_bio);
1969                         else
1970                                 put_buf(r10_bio);
1971                         md_done_sync(mddev, s, 1);
1972                         break;
1973                 } else {
1974                         struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
1975                         if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1976                             test_bit(R10BIO_WriteError, &r10_bio->state))
1977                                 reschedule_retry(r10_bio);
1978                         else
1979                                 put_buf(r10_bio);
1980                         r10_bio = r10_bio2;
1981                 }
1982         }
1983 }
1984
1985 static void end_sync_write(struct bio *bio)
1986 {
1987         struct r10bio *r10_bio = get_resync_r10bio(bio);
1988         struct mddev *mddev = r10_bio->mddev;
1989         struct r10conf *conf = mddev->private;
1990         int d;
1991         sector_t first_bad;
1992         int bad_sectors;
1993         int slot;
1994         int repl;
1995         struct md_rdev *rdev = NULL;
1996
1997         d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
1998         if (repl)
1999                 rdev = conf->mirrors[d].replacement;
2000         else
2001                 rdev = conf->mirrors[d].rdev;
2002
2003         if (bio->bi_status) {
2004                 if (repl)
2005                         md_error(mddev, rdev);
2006                 else {
2007                         set_bit(WriteErrorSeen, &rdev->flags);
2008                         if (!test_and_set_bit(WantReplacement, &rdev->flags))
2009                                 set_bit(MD_RECOVERY_NEEDED,
2010                                         &rdev->mddev->recovery);
2011                         set_bit(R10BIO_WriteError, &r10_bio->state);
2012                 }
2013         } else if (is_badblock(rdev,
2014                              r10_bio->devs[slot].addr,
2015                              r10_bio->sectors,
2016                              &first_bad, &bad_sectors))
2017                 set_bit(R10BIO_MadeGood, &r10_bio->state);
2018
2019         rdev_dec_pending(rdev, mddev);
2020
2021         end_sync_request(r10_bio);
2022 }
2023
2024 /*
2025  * Note: sync and recover and handled very differently for raid10
2026  * This code is for resync.
2027  * For resync, we read through virtual addresses and read all blocks.
2028  * If there is any error, we schedule a write.  The lowest numbered
2029  * drive is authoritative.
2030  * However requests come for physical address, so we need to map.
2031  * For every physical address there are raid_disks/copies virtual addresses,
2032  * which is always are least one, but is not necessarly an integer.
2033  * This means that a physical address can span multiple chunks, so we may
2034  * have to submit multiple io requests for a single sync request.
2035  */
2036 /*
2037  * We check if all blocks are in-sync and only write to blocks that
2038  * aren't in sync
2039  */
2040 static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2041 {
2042         struct r10conf *conf = mddev->private;
2043         int i, first;
2044         struct bio *tbio, *fbio;
2045         int vcnt;
2046         struct page **tpages, **fpages;
2047
2048         atomic_set(&r10_bio->remaining, 1);
2049
2050         /* find the first device with a block */
2051         for (i=0; i<conf->copies; i++)
2052                 if (!r10_bio->devs[i].bio->bi_status)
2053                         break;
2054
2055         if (i == conf->copies)
2056                 goto done;
2057
2058         first = i;
2059         fbio = r10_bio->devs[i].bio;
2060         fbio->bi_iter.bi_size = r10_bio->sectors << 9;
2061         fbio->bi_iter.bi_idx = 0;
2062         fpages = get_resync_pages(fbio)->pages;
2063
2064         vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
2065         /* now find blocks with errors */
2066         for (i=0 ; i < conf->copies ; i++) {
2067                 int  j, d;
2068                 struct md_rdev *rdev;
2069                 struct resync_pages *rp;
2070
2071                 tbio = r10_bio->devs[i].bio;
2072
2073                 if (tbio->bi_end_io != end_sync_read)
2074                         continue;
2075                 if (i == first)
2076                         continue;
2077
2078                 tpages = get_resync_pages(tbio)->pages;
2079                 d = r10_bio->devs[i].devnum;
2080                 rdev = conf->mirrors[d].rdev;
2081                 if (!r10_bio->devs[i].bio->bi_status) {
2082                         /* We know that the bi_io_vec layout is the same for
2083                          * both 'first' and 'i', so we just compare them.
2084                          * All vec entries are PAGE_SIZE;
2085                          */
2086                         int sectors = r10_bio->sectors;
2087                         for (j = 0; j < vcnt; j++) {
2088                                 int len = PAGE_SIZE;
2089                                 if (sectors < (len / 512))
2090                                         len = sectors * 512;
2091                                 if (memcmp(page_address(fpages[j]),
2092                                            page_address(tpages[j]),
2093                                            len))
2094                                         break;
2095                                 sectors -= len/512;
2096                         }
2097                         if (j == vcnt)
2098                                 continue;
2099                         atomic64_add(r10_bio->sectors, &mddev->resync_mismatches);
2100                         if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
2101                                 /* Don't fix anything. */
2102                                 continue;
2103                 } else if (test_bit(FailFast, &rdev->flags)) {
2104                         /* Just give up on this device */
2105                         md_error(rdev->mddev, rdev);
2106                         continue;
2107                 }
2108                 /* Ok, we need to write this bio, either to correct an
2109                  * inconsistency or to correct an unreadable block.
2110                  * First we need to fixup bv_offset, bv_len and
2111                  * bi_vecs, as the read request might have corrupted these
2112                  */
2113                 rp = get_resync_pages(tbio);
2114                 bio_reset(tbio);
2115
2116                 md_bio_reset_resync_pages(tbio, rp, fbio->bi_iter.bi_size);
2117
2118                 rp->raid_bio = r10_bio;
2119                 tbio->bi_private = rp;
2120                 tbio->bi_iter.bi_sector = r10_bio->devs[i].addr;
2121                 tbio->bi_end_io = end_sync_write;
2122                 bio_set_op_attrs(tbio, REQ_OP_WRITE, 0);
2123
2124                 bio_copy_data(tbio, fbio);
2125
2126                 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2127                 atomic_inc(&r10_bio->remaining);
2128                 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio));
2129
2130                 if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
2131                         tbio->bi_opf |= MD_FAILFAST;
2132                 tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
2133                 bio_set_dev(tbio, conf->mirrors[d].rdev->bdev);
2134                 generic_make_request(tbio);
2135         }
2136
2137         /* Now write out to any replacement devices
2138          * that are active
2139          */
2140         for (i = 0; i < conf->copies; i++) {
2141                 int d;
2142
2143                 tbio = r10_bio->devs[i].repl_bio;
2144                 if (!tbio || !tbio->bi_end_io)
2145                         continue;
2146                 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
2147                     && r10_bio->devs[i].bio != fbio)
2148                         bio_copy_data(tbio, fbio);
2149                 d = r10_bio->devs[i].devnum;
2150                 atomic_inc(&r10_bio->remaining);
2151                 md_sync_acct(conf->mirrors[d].replacement->bdev,
2152                              bio_sectors(tbio));
2153                 generic_make_request(tbio);
2154         }
2155
2156 done:
2157         if (atomic_dec_and_test(&r10_bio->remaining)) {
2158                 md_done_sync(mddev, r10_bio->sectors, 1);
2159                 put_buf(r10_bio);
2160         }
2161 }
2162
2163 /*
2164  * Now for the recovery code.
2165  * Recovery happens across physical sectors.
2166  * We recover all non-is_sync drives by finding the virtual address of
2167  * each, and then choose a working drive that also has that virt address.
2168  * There is a separate r10_bio for each non-in_sync drive.
2169  * Only the first two slots are in use. The first for reading,
2170  * The second for writing.
2171  *
2172  */
2173 static void fix_recovery_read_error(struct r10bio *r10_bio)
2174 {
2175         /* We got a read error during recovery.
2176          * We repeat the read in smaller page-sized sections.
2177          * If a read succeeds, write it to the new device or record
2178          * a bad block if we cannot.
2179          * If a read fails, record a bad block on both old and
2180          * new devices.
2181          */
2182         struct mddev *mddev = r10_bio->mddev;
2183         struct r10conf *conf = mddev->private;
2184         struct bio *bio = r10_bio->devs[0].bio;
2185         sector_t sect = 0;
2186         int sectors = r10_bio->sectors;
2187         int idx = 0;
2188         int dr = r10_bio->devs[0].devnum;
2189         int dw = r10_bio->devs[1].devnum;
2190         struct page **pages = get_resync_pages(bio)->pages;
2191
2192         while (sectors) {
2193                 int s = sectors;
2194                 struct md_rdev *rdev;
2195                 sector_t addr;
2196                 int ok;
2197
2198                 if (s > (PAGE_SIZE>>9))
2199                         s = PAGE_SIZE >> 9;
2200
2201                 rdev = conf->mirrors[dr].rdev;
2202                 addr = r10_bio->devs[0].addr + sect,
2203                 ok = sync_page_io(rdev,
2204                                   addr,
2205                                   s << 9,
2206                                   pages[idx],
2207                                   REQ_OP_READ, 0, false);
2208                 if (ok) {
2209                         rdev = conf->mirrors[dw].rdev;
2210                         addr = r10_bio->devs[1].addr + sect;
2211                         ok = sync_page_io(rdev,
2212                                           addr,
2213                                           s << 9,
2214                                           pages[idx],
2215                                           REQ_OP_WRITE, 0, false);
2216                         if (!ok) {
2217                                 set_bit(WriteErrorSeen, &rdev->flags);
2218                                 if (!test_and_set_bit(WantReplacement,
2219                                                       &rdev->flags))
2220                                         set_bit(MD_RECOVERY_NEEDED,
2221                                                 &rdev->mddev->recovery);
2222                         }
2223                 }
2224                 if (!ok) {
2225                         /* We don't worry if we cannot set a bad block -
2226                          * it really is bad so there is no loss in not
2227                          * recording it yet
2228                          */
2229                         rdev_set_badblocks(rdev, addr, s, 0);
2230
2231                         if (rdev != conf->mirrors[dw].rdev) {
2232                                 /* need bad block on destination too */
2233                                 struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2234                                 addr = r10_bio->devs[1].addr + sect;
2235                                 ok = rdev_set_badblocks(rdev2, addr, s, 0);
2236                                 if (!ok) {
2237                                         /* just abort the recovery */
2238                                         pr_notice("md/raid10:%s: recovery aborted due to read error\n",
2239                                                   mdname(mddev));
2240
2241                                         conf->mirrors[dw].recovery_disabled
2242                                                 = mddev->recovery_disabled;
2243                                         set_bit(MD_RECOVERY_INTR,
2244                                                 &mddev->recovery);
2245                                         break;
2246                                 }
2247                         }
2248                 }
2249
2250                 sectors -= s;
2251                 sect += s;
2252                 idx++;
2253         }
2254 }
2255
2256 static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2257 {
2258         struct r10conf *conf = mddev->private;
2259         int d;
2260         struct bio *wbio, *wbio2;
2261
2262         if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
2263                 fix_recovery_read_error(r10_bio);
2264                 end_sync_request(r10_bio);
2265                 return;
2266         }
2267
2268         /*
2269          * share the pages with the first bio
2270          * and submit the write request
2271          */
2272         d = r10_bio->devs[1].devnum;
2273         wbio = r10_bio->devs[1].bio;
2274         wbio2 = r10_bio->devs[1].repl_bio;
2275         /* Need to test wbio2->bi_end_io before we call
2276          * generic_make_request as if the former is NULL,
2277          * the latter is free to free wbio2.
2278          */
2279         if (wbio2 && !wbio2->bi_end_io)
2280                 wbio2 = NULL;
2281         if (wbio->bi_end_io) {
2282                 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2283                 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio));
2284                 generic_make_request(wbio);
2285         }
2286         if (wbio2) {
2287                 atomic_inc(&conf->mirrors[d].replacement->nr_pending);
2288                 md_sync_acct(conf->mirrors[d].replacement->bdev,
2289                              bio_sectors(wbio2));
2290                 generic_make_request(wbio2);
2291         }
2292 }
2293
2294 /*
2295  * Used by fix_read_error() to decay the per rdev read_errors.
2296  * We halve the read error count for every hour that has elapsed
2297  * since the last recorded read error.
2298  *
2299  */
2300 static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
2301 {
2302         long cur_time_mon;
2303         unsigned long hours_since_last;
2304         unsigned int read_errors = atomic_read(&rdev->read_errors);
2305
2306         cur_time_mon = ktime_get_seconds();
2307
2308         if (rdev->last_read_error == 0) {
2309                 /* first time we've seen a read error */
2310                 rdev->last_read_error = cur_time_mon;
2311                 return;
2312         }
2313
2314         hours_since_last = (long)(cur_time_mon -
2315                             rdev->last_read_error) / 3600;
2316
2317         rdev->last_read_error = cur_time_mon;
2318
2319         /*
2320          * if hours_since_last is > the number of bits in read_errors
2321          * just set read errors to 0. We do this to avoid
2322          * overflowing the shift of read_errors by hours_since_last.
2323          */
2324         if (hours_since_last >= 8 * sizeof(read_errors))
2325                 atomic_set(&rdev->read_errors, 0);
2326         else
2327                 atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
2328 }
2329
2330 static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2331                             int sectors, struct page *page, int rw)
2332 {
2333         sector_t first_bad;
2334         int bad_sectors;
2335
2336         if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
2337             && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
2338                 return -1;
2339         if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false))
2340                 /* success */
2341                 return 1;
2342         if (rw == WRITE) {
2343                 set_bit(WriteErrorSeen, &rdev->flags);
2344                 if (!test_and_set_bit(WantReplacement, &rdev->flags))
2345                         set_bit(MD_RECOVERY_NEEDED,
2346                                 &rdev->mddev->recovery);
2347         }
2348         /* need to record an error - either for the block or the device */
2349         if (!rdev_set_badblocks(rdev, sector, sectors, 0))
2350                 md_error(rdev->mddev, rdev);
2351         return 0;
2352 }
2353
2354 /*
2355  * This is a kernel thread which:
2356  *
2357  *      1.      Retries failed read operations on working mirrors.
2358  *      2.      Updates the raid superblock when problems encounter.
2359  *      3.      Performs writes following reads for array synchronising.
2360  */
2361
2362 static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2363 {
2364         int sect = 0; /* Offset from r10_bio->sector */
2365         int sectors = r10_bio->sectors;
2366         struct md_rdev*rdev;
2367         int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
2368         int d = r10_bio->devs[r10_bio->read_slot].devnum;
2369
2370         /* still own a reference to this rdev, so it cannot
2371          * have been cleared recently.
2372          */
2373         rdev = conf->mirrors[d].rdev;
2374
2375         if (test_bit(Faulty, &rdev->flags))
2376                 /* drive has already been failed, just ignore any
2377                    more fix_read_error() attempts */
2378                 return;
2379
2380         check_decay_read_errors(mddev, rdev);
2381         atomic_inc(&rdev->read_errors);
2382         if (atomic_read(&rdev->read_errors) > max_read_errors) {
2383                 char b[BDEVNAME_SIZE];
2384                 bdevname(rdev->bdev, b);
2385
2386                 pr_notice("md/raid10:%s: %s: Raid device exceeded read_error threshold [cur %d:max %d]\n",
2387                           mdname(mddev), b,
2388                           atomic_read(&rdev->read_errors), max_read_errors);
2389                 pr_notice("md/raid10:%s: %s: Failing raid device\n",
2390                           mdname(mddev), b);
2391                 md_error(mddev, rdev);
2392                 r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
2393                 return;
2394         }
2395
2396         while(sectors) {
2397                 int s = sectors;
2398                 int sl = r10_bio->read_slot;
2399                 int success = 0;
2400                 int start;
2401
2402                 if (s > (PAGE_SIZE>>9))
2403                         s = PAGE_SIZE >> 9;
2404
2405                 rcu_read_lock();
2406                 do {
2407                         sector_t first_bad;
2408                         int bad_sectors;
2409
2410                         d = r10_bio->devs[sl].devnum;
2411                         rdev = rcu_dereference(conf->mirrors[d].rdev);
2412                         if (rdev &&
2413                             test_bit(In_sync, &rdev->flags) &&
2414                             !test_bit(Faulty, &rdev->flags) &&
2415                             is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
2416                                         &first_bad, &bad_sectors) == 0) {
2417                                 atomic_inc(&rdev->nr_pending);
2418                                 rcu_read_unlock();
2419                                 success = sync_page_io(rdev,
2420                                                        r10_bio->devs[sl].addr +
2421                                                        sect,
2422                                                        s<<9,
2423                                                        conf->tmppage,
2424                                                        REQ_OP_READ, 0, false);
2425                                 rdev_dec_pending(rdev, mddev);
2426                                 rcu_read_lock();
2427                                 if (success)
2428                                         break;
2429                         }
2430                         sl++;
2431                         if (sl == conf->copies)
2432                                 sl = 0;
2433                 } while (!success && sl != r10_bio->read_slot);
2434                 rcu_read_unlock();
2435
2436                 if (!success) {
2437                         /* Cannot read from anywhere, just mark the block
2438                          * as bad on the first device to discourage future
2439                          * reads.
2440                          */
2441                         int dn = r10_bio->devs[r10_bio->read_slot].devnum;
2442                         rdev = conf->mirrors[dn].rdev;
2443
2444                         if (!rdev_set_badblocks(
2445                                     rdev,
2446                                     r10_bio->devs[r10_bio->read_slot].addr
2447                                     + sect,
2448                                     s, 0)) {
2449                                 md_error(mddev, rdev);
2450                                 r10_bio->devs[r10_bio->read_slot].bio
2451                                         = IO_BLOCKED;
2452                         }
2453                         break;
2454                 }
2455
2456                 start = sl;
2457                 /* write it back and re-read */
2458                 rcu_read_lock();
2459                 while (sl != r10_bio->read_slot) {
2460                         char b[BDEVNAME_SIZE];
2461
2462                         if (sl==0)
2463                                 sl = conf->copies;
2464                         sl--;
2465                         d = r10_bio->devs[sl].devnum;
2466                         rdev = rcu_dereference(conf->mirrors[d].rdev);
2467                         if (!rdev ||
2468                             test_bit(Faulty, &rdev->flags) ||
2469                             !test_bit(In_sync, &rdev->flags))
2470                                 continue;
2471
2472                         atomic_inc(&rdev->nr_pending);
2473                         rcu_read_unlock();
2474                         if (r10_sync_page_io(rdev,
2475                                              r10_bio->devs[sl].addr +
2476                                              sect,
2477                                              s, conf->tmppage, WRITE)
2478                             == 0) {
2479                                 /* Well, this device is dead */
2480                                 pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %s)\n",
2481                                           mdname(mddev), s,
2482                                           (unsigned long long)(
2483                                                   sect +
2484                                                   choose_data_offset(r10_bio,
2485                                                                      rdev)),
2486                                           bdevname(rdev->bdev, b));
2487                                 pr_notice("md/raid10:%s: %s: failing drive\n",
2488                                           mdname(mddev),
2489                                           bdevname(rdev->bdev, b));
2490                         }
2491                         rdev_dec_pending(rdev, mddev);
2492                         rcu_read_lock();
2493                 }
2494                 sl = start;
2495                 while (sl != r10_bio->read_slot) {
2496                         char b[BDEVNAME_SIZE];
2497
2498                         if (sl==0)
2499                                 sl = conf->copies;
2500                         sl--;
2501                         d = r10_bio->devs[sl].devnum;
2502                         rdev = rcu_dereference(conf->mirrors[d].rdev);
2503                         if (!rdev ||
2504                             test_bit(Faulty, &rdev->flags) ||
2505                             !test_bit(In_sync, &rdev->flags))
2506                                 continue;
2507
2508                         atomic_inc(&rdev->nr_pending);
2509                         rcu_read_unlock();
2510                         switch (r10_sync_page_io(rdev,
2511                                              r10_bio->devs[sl].addr +
2512                                              sect,
2513                                              s, conf->tmppage,
2514                                                  READ)) {
2515                         case 0:
2516                                 /* Well, this device is dead */
2517                                 pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %s)\n",
2518                                        mdname(mddev), s,
2519                                        (unsigned long long)(
2520                                                sect +
2521                                                choose_data_offset(r10_bio, rdev)),
2522                                        bdevname(rdev->bdev, b));
2523                                 pr_notice("md/raid10:%s: %s: failing drive\n",
2524                                        mdname(mddev),
2525                                        bdevname(rdev->bdev, b));
2526                                 break;
2527                         case 1:
2528                                 pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %s)\n",
2529                                        mdname(mddev), s,
2530                                        (unsigned long long)(
2531                                                sect +
2532                                                choose_data_offset(r10_bio, rdev)),
2533                                        bdevname(rdev->bdev, b));
2534                                 atomic_add(s, &rdev->corrected_errors);
2535                         }
2536
2537                         rdev_dec_pending(rdev, mddev);
2538                         rcu_read_lock();
2539                 }
2540                 rcu_read_unlock();
2541
2542                 sectors -= s;
2543                 sect += s;
2544         }
2545 }
2546
2547 static int narrow_write_error(struct r10bio *r10_bio, int i)
2548 {
2549         struct bio *bio = r10_bio->master_bio;
2550         struct mddev *mddev = r10_bio->mddev;
2551         struct r10conf *conf = mddev->private;
2552         struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2553         /* bio has the data to be written to slot 'i' where
2554          * we just recently had a write error.
2555          * We repeatedly clone the bio and trim down to one block,
2556          * then try the write.  Where the write fails we record
2557          * a bad block.
2558          * It is conceivable that the bio doesn't exactly align with
2559          * blocks.  We must handle this.
2560          *
2561          * We currently own a reference to the rdev.
2562          */
2563
2564         int block_sectors;
2565         sector_t sector;
2566         int sectors;
2567         int sect_to_write = r10_bio->sectors;
2568         int ok = 1;
2569
2570         if (rdev->badblocks.shift < 0)
2571                 return 0;
2572
2573         block_sectors = roundup(1 << rdev->badblocks.shift,
2574                                 bdev_logical_block_size(rdev->bdev) >> 9);
2575         sector = r10_bio->sector;
2576         sectors = ((r10_bio->sector + block_sectors)
2577                    & ~(sector_t)(block_sectors - 1))
2578                 - sector;
2579
2580         while (sect_to_write) {
2581                 struct bio *wbio;
2582                 sector_t wsector;
2583                 if (sectors > sect_to_write)
2584                         sectors = sect_to_write;
2585                 /* Write at 'sector' for 'sectors' */
2586                 wbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
2587                 bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors);
2588                 wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector);
2589                 wbio->bi_iter.bi_sector = wsector +
2590                                    choose_data_offset(r10_bio, rdev);
2591                 bio_set_dev(wbio, rdev->bdev);
2592                 bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
2593
2594                 if (submit_bio_wait(wbio) < 0)
2595                         /* Failure! */
2596                         ok = rdev_set_badblocks(rdev, wsector,
2597                                                 sectors, 0)
2598                                 && ok;
2599
2600                 bio_put(wbio);
2601                 sect_to_write -= sectors;
2602                 sector += sectors;
2603                 sectors = block_sectors;
2604         }
2605         return ok;
2606 }
2607
2608 static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2609 {
2610         int slot = r10_bio->read_slot;
2611         struct bio *bio;
2612         struct r10conf *conf = mddev->private;
2613         struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2614
2615         /* we got a read error. Maybe the drive is bad.  Maybe just
2616          * the block and we can fix it.
2617          * We freeze all other IO, and try reading the block from
2618          * other devices.  When we find one, we re-write
2619          * and check it that fixes the read error.
2620          * This is all done synchronously while the array is
2621          * frozen.
2622          */
2623         bio = r10_bio->devs[slot].bio;
2624         bio_put(bio);
2625         r10_bio->devs[slot].bio = NULL;
2626
2627         if (mddev->ro)
2628                 r10_bio->devs[slot].bio = IO_BLOCKED;
2629         else if (!test_bit(FailFast, &rdev->flags)) {
2630                 freeze_array(conf, 1);
2631                 fix_read_error(conf, mddev, r10_bio);
2632                 unfreeze_array(conf);
2633         } else
2634                 md_error(mddev, rdev);
2635
2636         rdev_dec_pending(rdev, mddev);
2637         allow_barrier(conf);
2638         r10_bio->state = 0;
2639         raid10_read_request(mddev, r10_bio->master_bio, r10_bio);
2640 }
2641
2642 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2643 {
2644         /* Some sort of write request has finished and it
2645          * succeeded in writing where we thought there was a
2646          * bad block.  So forget the bad block.
2647          * Or possibly if failed and we need to record
2648          * a bad block.
2649          */
2650         int m;
2651         struct md_rdev *rdev;
2652
2653         if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
2654             test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2655                 for (m = 0; m < conf->copies; m++) {
2656                         int dev = r10_bio->devs[m].devnum;
2657                         rdev = conf->mirrors[dev].rdev;
2658                         if (r10_bio->devs[m].bio == NULL ||
2659                                 r10_bio->devs[m].bio->bi_end_io == NULL)
2660                                 continue;
2661                         if (!r10_bio->devs[m].bio->bi_status) {
2662                                 rdev_clear_badblocks(
2663                                         rdev,
2664                                         r10_bio->devs[m].addr,
2665                                         r10_bio->sectors, 0);
2666                         } else {
2667                                 if (!rdev_set_badblocks(
2668                                             rdev,
2669                                             r10_bio->devs[m].addr,
2670                                             r10_bio->sectors, 0))
2671                                         md_error(conf->mddev, rdev);
2672                         }
2673                         rdev = conf->mirrors[dev].replacement;
2674                         if (r10_bio->devs[m].repl_bio == NULL ||
2675                                 r10_bio->devs[m].repl_bio->bi_end_io == NULL)
2676                                 continue;
2677
2678                         if (!r10_bio->devs[m].repl_bio->bi_status) {
2679                                 rdev_clear_badblocks(
2680                                         rdev,
2681                                         r10_bio->devs[m].addr,
2682                                         r10_bio->sectors, 0);
2683                         } else {
2684                                 if (!rdev_set_badblocks(
2685                                             rdev,
2686                                             r10_bio->devs[m].addr,
2687                                             r10_bio->sectors, 0))
2688                                         md_error(conf->mddev, rdev);
2689                         }
2690                 }
2691                 put_buf(r10_bio);
2692         } else {
2693                 bool fail = false;
2694                 for (m = 0; m < conf->copies; m++) {
2695                         int dev = r10_bio->devs[m].devnum;
2696                         struct bio *bio = r10_bio->devs[m].bio;
2697                         rdev = conf->mirrors[dev].rdev;
2698                         if (bio == IO_MADE_GOOD) {
2699                                 rdev_clear_badblocks(
2700                                         rdev,
2701                                         r10_bio->devs[m].addr,
2702                                         r10_bio->sectors, 0);
2703                                 rdev_dec_pending(rdev, conf->mddev);
2704                         } else if (bio != NULL && bio->bi_status) {
2705                                 fail = true;
2706                                 if (!narrow_write_error(r10_bio, m)) {
2707                                         md_error(conf->mddev, rdev);
2708                                         set_bit(R10BIO_Degraded,
2709                                                 &r10_bio->state);
2710                                 }
2711                                 rdev_dec_pending(rdev, conf->mddev);
2712                         }
2713                         bio = r10_bio->devs[m].repl_bio;
2714                         rdev = conf->mirrors[dev].replacement;
2715                         if (rdev && bio == IO_MADE_GOOD) {
2716                                 rdev_clear_badblocks(
2717                                         rdev,
2718                                         r10_bio->devs[m].addr,
2719                                         r10_bio->sectors, 0);
2720                                 rdev_dec_pending(rdev, conf->mddev);
2721                         }
2722                 }
2723                 if (fail) {
2724                         spin_lock_irq(&conf->device_lock);
2725                         list_add(&r10_bio->retry_list, &conf->bio_end_io_list);
2726                         conf->nr_queued++;
2727                         spin_unlock_irq(&conf->device_lock);
2728                         /*
2729                          * In case freeze_array() is waiting for condition
2730                          * nr_pending == nr_queued + extra to be true.
2731                          */
2732                         wake_up(&conf->wait_barrier);
2733                         md_wakeup_thread(conf->mddev->thread);
2734                 } else {
2735                         if (test_bit(R10BIO_WriteError,
2736                                      &r10_bio->state))
2737                                 close_write(r10_bio);
2738                         raid_end_bio_io(r10_bio);
2739                 }
2740         }
2741 }
2742
2743 static void raid10d(struct md_thread *thread)
2744 {
2745         struct mddev *mddev = thread->mddev;
2746         struct r10bio *r10_bio;
2747         unsigned long flags;
2748         struct r10conf *conf = mddev->private;
2749         struct list_head *head = &conf->retry_list;
2750         struct blk_plug plug;
2751
2752         md_check_recovery(mddev);
2753
2754         if (!list_empty_careful(&conf->bio_end_io_list) &&
2755             !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2756                 LIST_HEAD(tmp);
2757                 spin_lock_irqsave(&conf->device_lock, flags);
2758                 if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2759                         while (!list_empty(&conf->bio_end_io_list)) {
2760                                 list_move(conf->bio_end_io_list.prev, &tmp);
2761                                 conf->nr_queued--;
2762                         }
2763                 }
2764                 spin_unlock_irqrestore(&conf->device_lock, flags);
2765                 while (!list_empty(&tmp)) {
2766                         r10_bio = list_first_entry(&tmp, struct r10bio,
2767                                                    retry_list);
2768                         list_del(&r10_bio->retry_list);
2769                         if (mddev->degraded)
2770                                 set_bit(R10BIO_Degraded, &r10_bio->state);
2771
2772                         if (test_bit(R10BIO_WriteError,
2773                                      &r10_bio->state))
2774                                 close_write(r10_bio);
2775                         raid_end_bio_io(r10_bio);
2776                 }
2777         }
2778
2779         blk_start_plug(&plug);
2780         for (;;) {
2781
2782                 flush_pending_writes(conf);
2783
2784                 spin_lock_irqsave(&conf->device_lock, flags);
2785                 if (list_empty(head)) {
2786                         spin_unlock_irqrestore(&conf->device_lock, flags);
2787                         break;
2788                 }
2789                 r10_bio = list_entry(head->prev, struct r10bio, retry_list);
2790                 list_del(head->prev);
2791                 conf->nr_queued--;
2792                 spin_unlock_irqrestore(&conf->device_lock, flags);
2793
2794                 mddev = r10_bio->mddev;
2795                 conf = mddev->private;
2796                 if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2797                     test_bit(R10BIO_WriteError, &r10_bio->state))
2798                         handle_write_completed(conf, r10_bio);
2799                 else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
2800                         reshape_request_write(mddev, r10_bio);
2801                 else if (test_bit(R10BIO_IsSync, &r10_bio->state))
2802                         sync_request_write(mddev, r10_bio);
2803                 else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
2804                         recovery_request_write(mddev, r10_bio);
2805                 else if (test_bit(R10BIO_ReadError, &r10_bio->state))
2806                         handle_read_error(mddev, r10_bio);
2807                 else
2808                         WARN_ON_ONCE(1);
2809
2810                 cond_resched();
2811                 if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
2812                         md_check_recovery(mddev);
2813         }
2814         blk_finish_plug(&plug);
2815 }
2816
2817 static int init_resync(struct r10conf *conf)
2818 {
2819         int buffs;
2820         int i;
2821
2822         buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2823         BUG_ON(conf->r10buf_pool);
2824         conf->have_replacement = 0;
2825         for (i = 0; i < conf->geo.raid_disks; i++)
2826                 if (conf->mirrors[i].replacement)
2827                         conf->have_replacement = 1;
2828         conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
2829         if (!conf->r10buf_pool)
2830                 return -ENOMEM;
2831         conf->next_resync = 0;
2832         return 0;
2833 }
2834
2835 static struct r10bio *raid10_alloc_init_r10buf(struct r10conf *conf)
2836 {
2837         struct r10bio *r10bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
2838         struct rsync_pages *rp;
2839         struct bio *bio;
2840         int nalloc;
2841         int i;
2842
2843         if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
2844             test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
2845                 nalloc = conf->copies; /* resync */
2846         else
2847                 nalloc = 2; /* recovery */
2848
2849         for (i = 0; i < nalloc; i++) {
2850                 bio = r10bio->devs[i].bio;
2851                 rp = bio->bi_private;
2852                 bio_reset(bio);
2853                 bio->bi_private = rp;
2854                 bio = r10bio->devs[i].repl_bio;
2855                 if (bio) {
2856                         rp = bio->bi_private;
2857                         bio_reset(bio);
2858                         bio->bi_private = rp;
2859                 }
2860         }
2861         return r10bio;
2862 }
2863
2864 /*
2865  * Set cluster_sync_high since we need other nodes to add the
2866  * range [cluster_sync_low, cluster_sync_high] to suspend list.
2867  */
2868 static void raid10_set_cluster_sync_high(struct r10conf *conf)
2869 {
2870         sector_t window_size;
2871         int extra_chunk, chunks;
2872
2873         /*
2874          * First, here we define "stripe" as a unit which across
2875          * all member devices one time, so we get chunks by use
2876          * raid_disks / near_copies. Otherwise, if near_copies is
2877          * close to raid_disks, then resync window could increases
2878          * linearly with the increase of raid_disks, which means
2879          * we will suspend a really large IO window while it is not
2880          * necessary. If raid_disks is not divisible by near_copies,
2881          * an extra chunk is needed to ensure the whole "stripe" is
2882          * covered.
2883          */
2884
2885         chunks = conf->geo.raid_disks / conf->geo.near_copies;
2886         if (conf->geo.raid_disks % conf->geo.near_copies == 0)
2887                 extra_chunk = 0;
2888         else
2889                 extra_chunk = 1;
2890         window_size = (chunks + extra_chunk) * conf->mddev->chunk_sectors;
2891
2892         /*
2893          * At least use a 32M window to align with raid1's resync window
2894          */
2895         window_size = (CLUSTER_RESYNC_WINDOW_SECTORS > window_size) ?
2896                         CLUSTER_RESYNC_WINDOW_SECTORS : window_size;
2897
2898         conf->cluster_sync_high = conf->cluster_sync_low + window_size;
2899 }
2900
2901 /*
2902  * perform a "sync" on one "block"
2903  *
2904  * We need to make sure that no normal I/O request - particularly write
2905  * requests - conflict with active sync requests.
2906  *
2907  * This is achieved by tracking pending requests and a 'barrier' concept
2908  * that can be installed to exclude normal IO requests.
2909  *
2910  * Resync and recovery are handled very differently.
2911  * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
2912  *
2913  * For resync, we iterate over virtual addresses, read all copies,
2914  * and update if there are differences.  If only one copy is live,
2915  * skip it.
2916  * For recovery, we iterate over physical addresses, read a good
2917  * value for each non-in_sync drive, and over-write.
2918  *
2919  * So, for recovery we may have several outstanding complex requests for a
2920  * given address, one for each out-of-sync device.  We model this by allocating
2921  * a number of r10_bio structures, one for each out-of-sync device.
2922  * As we setup these structures, we collect all bio's together into a list
2923  * which we then process collectively to add pages, and then process again
2924  * to pass to generic_make_request.
2925  *
2926  * The r10_bio structures are linked using a borrowed master_bio pointer.
2927  * This link is counted in ->remaining.  When the r10_bio that points to NULL
2928  * has its remaining count decremented to 0, the whole complex operation
2929  * is complete.
2930  *
2931  */
2932
2933 static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
2934                              int *skipped)
2935 {
2936         struct r10conf *conf = mddev->private;
2937         struct r10bio *r10_bio;
2938         struct bio *biolist = NULL, *bio;
2939         sector_t max_sector, nr_sectors;
2940         int i;
2941         int max_sync;
2942         sector_t sync_blocks;
2943         sector_t sectors_skipped = 0;
2944         int chunks_skipped = 0;
2945         sector_t chunk_mask = conf->geo.chunk_mask;
2946         int page_idx = 0;
2947
2948         if (!conf->r10buf_pool)
2949                 if (init_resync(conf))
2950                         return 0;
2951
2952         /*
2953          * Allow skipping a full rebuild for incremental assembly
2954          * of a clean array, like RAID1 does.
2955          */
2956         if (mddev->bitmap == NULL &&
2957             mddev->recovery_cp == MaxSector &&
2958             mddev->reshape_position == MaxSector &&
2959             !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
2960             !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
2961             !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
2962             conf->fullsync == 0) {
2963                 *skipped = 1;
2964                 return mddev->dev_sectors - sector_nr;
2965         }
2966
2967  skipped:
2968         max_sector = mddev->dev_sectors;
2969         if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
2970             test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
2971                 max_sector = mddev->resync_max_sectors;
2972         if (sector_nr >= max_sector) {
2973                 conf->cluster_sync_low = 0;
2974                 conf->cluster_sync_high = 0;
2975
2976                 /* If we aborted, we need to abort the
2977                  * sync on the 'current' bitmap chucks (there can
2978                  * be several when recovering multiple devices).
2979                  * as we may have started syncing it but not finished.
2980                  * We can find the current address in
2981                  * mddev->curr_resync, but for recovery,
2982                  * we need to convert that to several
2983                  * virtual addresses.
2984                  */
2985                 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
2986                         end_reshape(conf);
2987                         close_sync(conf);
2988                         return 0;
2989                 }
2990
2991                 if (mddev->curr_resync < max_sector) { /* aborted */
2992                         if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
2993                                 bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
2994                                                 &sync_blocks, 1);
2995                         else for (i = 0; i < conf->geo.raid_disks; i++) {
2996                                 sector_t sect =
2997                                         raid10_find_virt(conf, mddev->curr_resync, i);
2998                                 bitmap_end_sync(mddev->bitmap, sect,
2999                                                 &sync_blocks, 1);
3000                         }
3001                 } else {
3002                         /* completed sync */
3003                         if ((!mddev->bitmap || conf->fullsync)
3004                             && conf->have_replacement
3005                             && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3006                                 /* Completed a full sync so the replacements
3007                                  * are now fully recovered.
3008                                  */
3009                                 rcu_read_lock();
3010                                 for (i = 0; i < conf->geo.raid_disks; i++) {
3011                                         struct md_rdev *rdev =
3012                                                 rcu_dereference(conf->mirrors[i].replacement);
3013                                         if (rdev)
3014                                                 rdev->recovery_offset = MaxSector;
3015                                 }
3016                                 rcu_read_unlock();
3017                         }
3018                         conf->fullsync = 0;
3019                 }
3020                 bitmap_close_sync(mddev->bitmap);
3021                 close_sync(conf);
3022                 *skipped = 1;
3023                 return sectors_skipped;
3024         }
3025
3026         if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
3027                 return reshape_request(mddev, sector_nr, skipped);
3028
3029         if (chunks_skipped >= conf->geo.raid_disks) {
3030                 /* if there has been nothing to do on any drive,
3031                  * then there is nothing to do at all..
3032                  */
3033                 *skipped = 1;