1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#ifndef _RDS_IB_MR_H
33#define _RDS_IB_MR_H
34
35#include <linux/kernel.h>
36
37#include "rds.h"
38#include "ib.h"
39
40#define RDS_MR_1M_POOL_SIZE (8192 / 2)
41#define RDS_MR_1M_MSG_SIZE 256
42#define RDS_MR_8K_MSG_SIZE 2
43#define RDS_MR_8K_SCALE (256 / (RDS_MR_8K_MSG_SIZE + 1))
44#define RDS_MR_8K_POOL_SIZE (RDS_MR_8K_SCALE * (8192 / 2))
45
46enum rds_ib_fr_state {
47 FRMR_IS_FREE,
48 FRMR_IS_INUSE,
49 FRMR_IS_STALE,
50};
51
52struct rds_ib_frmr {
53 struct ib_mr *mr;
54 enum rds_ib_fr_state fr_state;
55 bool fr_inv;
56 wait_queue_head_t fr_inv_done;
57 bool fr_reg;
58 wait_queue_head_t fr_reg_done;
59 struct ib_send_wr fr_wr;
60 unsigned int dma_npages;
61 unsigned int sg_byte_len;
62};
63
64
65struct rds_ib_mr {
66 struct delayed_work work;
67 struct rds_ib_device *device;
68 struct rds_ib_mr_pool *pool;
69 struct rds_ib_connection *ic;
70
71 struct llist_node llnode;
72
73
74 struct list_head unmap_list;
75 unsigned int remap_count;
76
77 struct scatterlist *sg;
78 unsigned int sg_len;
79 int sg_dma_len;
80
81 u8 odp:1;
82 union {
83 struct rds_ib_frmr frmr;
84 struct ib_mr *mr;
85 } u;
86};
87
88
89struct rds_ib_mr_pool {
90 unsigned int pool_type;
91 struct mutex flush_lock;
92 struct delayed_work flush_worker;
93
94 atomic_t item_count;
95 atomic_t dirty_count;
96
97 struct llist_head drop_list;
98 struct llist_head free_list;
99 struct llist_head clean_list;
100 wait_queue_head_t flush_wait;
101 spinlock_t clean_lock;
102
103 atomic_t free_pinned;
104 unsigned long max_items;
105 unsigned long max_items_soft;
106 unsigned long max_free_pinned;
107 unsigned int max_pages;
108};
109
110extern struct workqueue_struct *rds_ib_mr_wq;
111extern bool prefer_frmr;
112
113struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_dev,
114 int npages);
115void rds_ib_get_mr_info(struct rds_ib_device *rds_ibdev,
116 struct rds_info_rdma_connection *iinfo);
117void rds6_ib_get_mr_info(struct rds_ib_device *rds_ibdev,
118 struct rds6_info_rdma_connection *iinfo6);
119void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *);
120void *rds_ib_get_mr(struct scatterlist *sg, unsigned long nents,
121 struct rds_sock *rs, u32 *key_ret,
122 struct rds_connection *conn, u64 start, u64 length,
123 int need_odp);
124void rds_ib_sync_mr(void *trans_private, int dir);
125void rds_ib_free_mr(void *trans_private, int invalidate);
126void rds_ib_flush_mrs(void);
127int rds_ib_mr_init(void);
128void rds_ib_mr_exit(void);
129u32 rds_ib_get_lkey(void *trans_private);
130
131void __rds_ib_teardown_mr(struct rds_ib_mr *);
132void rds_ib_teardown_mr(struct rds_ib_mr *);
133struct rds_ib_mr *rds_ib_reuse_mr(struct rds_ib_mr_pool *);
134int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *, int, struct rds_ib_mr **);
135struct rds_ib_mr *rds_ib_try_reuse_ibmr(struct rds_ib_mr_pool *);
136struct rds_ib_mr *rds_ib_reg_frmr(struct rds_ib_device *rds_ibdev,
137 struct rds_ib_connection *ic,
138 struct scatterlist *sg,
139 unsigned long nents, u32 *key);
140void rds_ib_unreg_frmr(struct list_head *list, unsigned int *nfreed,
141 unsigned long *unpinned, unsigned int goal);
142void rds_ib_free_frmr_list(struct rds_ib_mr *);
143#endif
144