2 * Internal header file _only_ for device mapper core
4 * Copyright (C) 2016 Red Hat, Inc. All rights reserved.
6 * This file is released under the LGPL.
9 #ifndef DM_CORE_INTERNAL_H
10 #define DM_CORE_INTERNAL_H
12 #include <linux/kthread.h>
13 #include <linux/ktime.h>
14 #include <linux/blk-mq.h>
15 #include <linux/blk-crypto-profile.h>
17 #include <trace/events/block.h>
22 #define DM_RESERVED_MAX_IOS 1024
24 struct dm_kobject_holder {
26 struct completion completion;
30 * DM core internal structures used directly by dm.c, dm-rq.c and dm-table.c.
31 * DM targets must _not_ deference a mapped_device or dm_table to directly
32 * access their members!
35 struct mapped_device {
36 struct mutex suspend_lock;
38 struct mutex table_devices_lock;
39 struct list_head table_devices;
42 * The current mapping (struct dm_table *).
43 * Use dm_get_live_table{_fast} or take suspend_lock for
50 /* Protect queue and type against concurrent access. */
51 struct mutex type_lock;
52 enum dm_queue_mode type;
55 struct request_queue *queue;
60 struct dm_target *immutable_target;
61 struct target_type *immutable_target_type;
65 struct dax_device *dax_dev;
67 wait_queue_head_t wait;
68 unsigned long __percpu *pending_io;
70 /* forced geometry settings */
71 struct hd_geometry geometry;
74 * Processing queue (flush)
76 struct workqueue_struct *wq;
79 * A list of ios that arrived while we were suspended.
81 struct work_struct work;
82 spinlock_t deferred_lock;
83 struct bio_list deferred;
90 wait_queue_head_t eventq;
93 struct list_head uevent_list;
94 spinlock_t uevent_lock; /* Protect access to uevent_list */
96 /* for blk-mq request-based DM support */
98 struct blk_mq_tag_set *tag_set;
100 struct dm_stats stats;
102 /* the number of internal suspends */
103 unsigned internal_suspend_count;
106 struct semaphore swap_bios_semaphore;
107 struct mutex swap_bios_lock;
110 * io objects are allocated from here.
112 struct bio_set io_bs;
115 /* kobject and completion */
116 struct dm_kobject_holder kobj_holder;
118 struct srcu_struct io_barrier;
120 #ifdef CONFIG_BLK_DEV_ZONED
121 unsigned int nr_zones;
122 unsigned int *zwp_offset;
126 struct dm_ima_measurements ima;
131 * Bits for the flags field of struct mapped_device.
133 #define DMF_BLOCK_IO_FOR_SUSPEND 0
134 #define DMF_SUSPENDED 1
136 #define DMF_FREEING 3
137 #define DMF_DELETING 4
138 #define DMF_NOFLUSH_SUSPENDING 5
139 #define DMF_DEFERRED_REMOVE 6
140 #define DMF_SUSPENDED_INTERNALLY 7
141 #define DMF_POST_SUSPENDING 8
142 #define DMF_EMULATE_ZONE_APPEND 9
144 void disable_discard(struct mapped_device *md);
145 void disable_write_zeroes(struct mapped_device *md);
147 static inline sector_t dm_get_size(struct mapped_device *md)
149 return get_capacity(md->disk);
152 static inline struct dm_stats *dm_get_stats(struct mapped_device *md)
157 static inline bool dm_emulate_zone_append(struct mapped_device *md)
159 if (blk_queue_is_zoned(md->queue))
160 return test_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
164 #define DM_TABLE_MAX_DEPTH 16
167 struct mapped_device *md;
168 enum dm_queue_mode type;
172 unsigned int counts[DM_TABLE_MAX_DEPTH]; /* in nodes */
173 sector_t *index[DM_TABLE_MAX_DEPTH];
175 unsigned int num_targets;
176 unsigned int num_allocated;
178 struct dm_target *targets;
180 struct target_type *immutable_target_type;
182 bool integrity_supported:1;
184 unsigned integrity_added:1;
187 * Indicates the rw permissions for the new logical
188 * device. This should be a combination of FMODE_READ
193 /* a list of devices used by this table */
194 struct list_head devices;
196 /* events get handed up using this callback */
197 void (*event_fn)(void *);
200 struct dm_md_mempools *mempools;
202 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
203 struct blk_crypto_profile *crypto_profile;
208 * One of these is allocated per clone bio.
210 #define DM_TIO_MAGIC 28714
211 struct dm_target_io {
212 unsigned short magic;
213 unsigned short flags;
214 unsigned int target_bio_nr;
216 struct dm_target *ti;
217 unsigned int *len_ptr;
227 DM_TIO_IS_DUPLICATE_BIO
230 static inline bool dm_tio_flagged(struct dm_target_io *tio, unsigned int bit)
232 return (tio->flags & (1U << bit)) != 0;
235 static inline void dm_tio_set_flag(struct dm_target_io *tio, unsigned int bit)
237 tio->flags |= (1U << bit);
241 * One of these is allocated per original bio.
242 * It contains the first clone used for that original.
244 #define DM_IO_MAGIC 19577
246 unsigned short magic;
247 unsigned short flags;
249 struct mapped_device *md;
250 struct bio *orig_bio;
253 unsigned long start_time;
255 struct hlist_node node;
256 struct task_struct *map_task;
257 struct dm_stats_aux stats_aux;
258 /* last member of dm_target_io is 'struct bio' */
259 struct dm_target_io tio;
270 static inline bool dm_io_flagged(struct dm_io *io, unsigned int bit)
272 return (io->flags & (1U << bit)) != 0;
275 static inline void dm_io_set_flag(struct dm_io *io, unsigned int bit)
277 io->flags |= (1U << bit);
280 static inline void dm_io_inc_pending(struct dm_io *io)
282 atomic_inc(&io->io_count);
285 void dm_io_dec_pending(struct dm_io *io, blk_status_t error);
287 static inline struct completion *dm_get_completion_from_kobject(struct kobject *kobj)
289 return &container_of(kobj, struct dm_kobject_holder, kobj)->completion;
292 unsigned __dm_get_module_param(unsigned *module_param, unsigned def, unsigned max);
294 static inline bool dm_message_test_buffer_overflow(char *result, unsigned maxlen)
296 return !maxlen || strlen(result) + 1 >= maxlen;
299 extern atomic_t dm_global_event_nr;
300 extern wait_queue_head_t dm_global_eventq;
301 void dm_issue_global_event(void);