1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#define pr_fmt(fmt) "blk-crypto: " fmt
30
31#include <linux/keyslot-manager.h>
32#include <linux/atomic.h>
33#include <linux/mutex.h>
34#include <linux/pm_runtime.h>
35#include <linux/wait.h>
36#include <linux/blkdev.h>
37
38struct blk_ksm_keyslot {
39 atomic_t slot_refs;
40 struct list_head idle_slot_node;
41 struct hlist_node hash_node;
42 const struct blk_crypto_key *key;
43 struct blk_keyslot_manager *ksm;
44};
45
46static inline void blk_ksm_hw_enter(struct blk_keyslot_manager *ksm)
47{
48
49
50
51
52
53 if (ksm->dev)
54 pm_runtime_get_sync(ksm->dev);
55 down_write(&ksm->lock);
56}
57
58static inline void blk_ksm_hw_exit(struct blk_keyslot_manager *ksm)
59{
60 up_write(&ksm->lock);
61 if (ksm->dev)
62 pm_runtime_put_sync(ksm->dev);
63}
64
65
66
67
68
69
70
71
72
73
74
75int blk_ksm_init(struct blk_keyslot_manager *ksm, unsigned int num_slots)
76{
77 unsigned int slot;
78 unsigned int i;
79 unsigned int slot_hashtable_size;
80
81 memset(ksm, 0, sizeof(*ksm));
82
83 if (num_slots == 0)
84 return -EINVAL;
85
86 ksm->slots = kvcalloc(num_slots, sizeof(ksm->slots[0]), GFP_KERNEL);
87 if (!ksm->slots)
88 return -ENOMEM;
89
90 ksm->num_slots = num_slots;
91
92 init_rwsem(&ksm->lock);
93
94 init_waitqueue_head(&ksm->idle_slots_wait_queue);
95 INIT_LIST_HEAD(&ksm->idle_slots);
96
97 for (slot = 0; slot < num_slots; slot++) {
98 ksm->slots[slot].ksm = ksm;
99 list_add_tail(&ksm->slots[slot].idle_slot_node,
100 &ksm->idle_slots);
101 }
102
103 spin_lock_init(&ksm->idle_slots_lock);
104
105 slot_hashtable_size = roundup_pow_of_two(num_slots);
106
107
108
109
110 if (slot_hashtable_size < 2)
111 slot_hashtable_size = 2;
112
113 ksm->log_slot_ht_size = ilog2(slot_hashtable_size);
114 ksm->slot_hashtable = kvmalloc_array(slot_hashtable_size,
115 sizeof(ksm->slot_hashtable[0]),
116 GFP_KERNEL);
117 if (!ksm->slot_hashtable)
118 goto err_destroy_ksm;
119 for (i = 0; i < slot_hashtable_size; i++)
120 INIT_HLIST_HEAD(&ksm->slot_hashtable[i]);
121
122 return 0;
123
124err_destroy_ksm:
125 blk_ksm_destroy(ksm);
126 return -ENOMEM;
127}
128EXPORT_SYMBOL_GPL(blk_ksm_init);
129
130static inline struct hlist_head *
131blk_ksm_hash_bucket_for_key(struct blk_keyslot_manager *ksm,
132 const struct blk_crypto_key *key)
133{
134 return &ksm->slot_hashtable[hash_ptr(key, ksm->log_slot_ht_size)];
135}
136
137static void blk_ksm_remove_slot_from_lru_list(struct blk_ksm_keyslot *slot)
138{
139 struct blk_keyslot_manager *ksm = slot->ksm;
140 unsigned long flags;
141
142 spin_lock_irqsave(&ksm->idle_slots_lock, flags);
143 list_del(&slot->idle_slot_node);
144 spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);
145}
146
147static struct blk_ksm_keyslot *blk_ksm_find_keyslot(
148 struct blk_keyslot_manager *ksm,
149 const struct blk_crypto_key *key)
150{
151 const struct hlist_head *head = blk_ksm_hash_bucket_for_key(ksm, key);
152 struct blk_ksm_keyslot *slotp;
153
154 hlist_for_each_entry(slotp, head, hash_node) {
155 if (slotp->key == key)
156 return slotp;
157 }
158 return NULL;
159}
160
161static struct blk_ksm_keyslot *blk_ksm_find_and_grab_keyslot(
162 struct blk_keyslot_manager *ksm,
163 const struct blk_crypto_key *key)
164{
165 struct blk_ksm_keyslot *slot;
166
167 slot = blk_ksm_find_keyslot(ksm, key);
168 if (!slot)
169 return NULL;
170 if (atomic_inc_return(&slot->slot_refs) == 1) {
171
172 blk_ksm_remove_slot_from_lru_list(slot);
173 }
174 return slot;
175}
176
177unsigned int blk_ksm_get_slot_idx(struct blk_ksm_keyslot *slot)
178{
179 return slot - slot->ksm->slots;
180}
181EXPORT_SYMBOL_GPL(blk_ksm_get_slot_idx);
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199blk_status_t blk_ksm_get_slot_for_key(struct blk_keyslot_manager *ksm,
200 const struct blk_crypto_key *key,
201 struct blk_ksm_keyslot **slot_ptr)
202{
203 struct blk_ksm_keyslot *slot;
204 int slot_idx;
205 int err;
206
207 *slot_ptr = NULL;
208 down_read(&ksm->lock);
209 slot = blk_ksm_find_and_grab_keyslot(ksm, key);
210 up_read(&ksm->lock);
211 if (slot)
212 goto success;
213
214 for (;;) {
215 blk_ksm_hw_enter(ksm);
216 slot = blk_ksm_find_and_grab_keyslot(ksm, key);
217 if (slot) {
218 blk_ksm_hw_exit(ksm);
219 goto success;
220 }
221
222
223
224
225
226 if (!list_empty(&ksm->idle_slots))
227 break;
228
229 blk_ksm_hw_exit(ksm);
230 wait_event(ksm->idle_slots_wait_queue,
231 !list_empty(&ksm->idle_slots));
232 }
233
234 slot = list_first_entry(&ksm->idle_slots, struct blk_ksm_keyslot,
235 idle_slot_node);
236 slot_idx = blk_ksm_get_slot_idx(slot);
237
238 err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot_idx);
239 if (err) {
240 wake_up(&ksm->idle_slots_wait_queue);
241 blk_ksm_hw_exit(ksm);
242 return errno_to_blk_status(err);
243 }
244
245
246 if (slot->key)
247 hlist_del(&slot->hash_node);
248 slot->key = key;
249 hlist_add_head(&slot->hash_node, blk_ksm_hash_bucket_for_key(ksm, key));
250
251 atomic_set(&slot->slot_refs, 1);
252
253 blk_ksm_remove_slot_from_lru_list(slot);
254
255 blk_ksm_hw_exit(ksm);
256success:
257 *slot_ptr = slot;
258 return BLK_STS_OK;
259}
260
261
262
263
264
265
266
267void blk_ksm_put_slot(struct blk_ksm_keyslot *slot)
268{
269 struct blk_keyslot_manager *ksm;
270 unsigned long flags;
271
272 if (!slot)
273 return;
274
275 ksm = slot->ksm;
276
277 if (atomic_dec_and_lock_irqsave(&slot->slot_refs,
278 &ksm->idle_slots_lock, flags)) {
279 list_add_tail(&slot->idle_slot_node, &ksm->idle_slots);
280 spin_unlock_irqrestore(&ksm->idle_slots_lock, flags);
281 wake_up(&ksm->idle_slots_wait_queue);
282 }
283}
284
285
286
287
288
289
290
291
292
293
294
295bool blk_ksm_crypto_cfg_supported(struct blk_keyslot_manager *ksm,
296 const struct blk_crypto_config *cfg)
297{
298 if (!ksm)
299 return false;
300 if (!(ksm->crypto_modes_supported[cfg->crypto_mode] &
301 cfg->data_unit_size))
302 return false;
303 if (ksm->max_dun_bytes_supported < cfg->dun_bytes)
304 return false;
305 return true;
306}
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322int blk_ksm_evict_key(struct blk_keyslot_manager *ksm,
323 const struct blk_crypto_key *key)
324{
325 struct blk_ksm_keyslot *slot;
326 int err = 0;
327
328 blk_ksm_hw_enter(ksm);
329 slot = blk_ksm_find_keyslot(ksm, key);
330 if (!slot)
331 goto out_unlock;
332
333 if (WARN_ON_ONCE(atomic_read(&slot->slot_refs) != 0)) {
334 err = -EBUSY;
335 goto out_unlock;
336 }
337 err = ksm->ksm_ll_ops.keyslot_evict(ksm, key,
338 blk_ksm_get_slot_idx(slot));
339 if (err)
340 goto out_unlock;
341
342 hlist_del(&slot->hash_node);
343 slot->key = NULL;
344 err = 0;
345out_unlock:
346 blk_ksm_hw_exit(ksm);
347 return err;
348}
349
350
351
352
353
354
355
356
357
358
359void blk_ksm_reprogram_all_keys(struct blk_keyslot_manager *ksm)
360{
361 unsigned int slot;
362
363
364 down_write(&ksm->lock);
365 for (slot = 0; slot < ksm->num_slots; slot++) {
366 const struct blk_crypto_key *key = ksm->slots[slot].key;
367 int err;
368
369 if (!key)
370 continue;
371
372 err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot);
373 WARN_ON(err);
374 }
375 up_write(&ksm->lock);
376}
377EXPORT_SYMBOL_GPL(blk_ksm_reprogram_all_keys);
378
379void blk_ksm_destroy(struct blk_keyslot_manager *ksm)
380{
381 if (!ksm)
382 return;
383 kvfree(ksm->slot_hashtable);
384 kvfree_sensitive(ksm->slots, sizeof(ksm->slots[0]) * ksm->num_slots);
385 memzero_explicit(ksm, sizeof(*ksm));
386}
387EXPORT_SYMBOL_GPL(blk_ksm_destroy);
388
389bool blk_ksm_register(struct blk_keyslot_manager *ksm, struct request_queue *q)
390{
391 if (blk_integrity_queue_supports_integrity(q)) {
392 pr_warn("Integrity and hardware inline encryption are not supported together. Disabling hardware inline encryption.\n");
393 return false;
394 }
395 q->ksm = ksm;
396 return true;
397}
398EXPORT_SYMBOL_GPL(blk_ksm_register);
399
400void blk_ksm_unregister(struct request_queue *q)
401{
402 q->ksm = NULL;
403}
404