1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#ifndef __LNET_LIB_TYPES_H__
36#define __LNET_LIB_TYPES_H__
37
38#include <linux/kthread.h>
39#include <linux/uio.h>
40#include <linux/types.h>
41
42#include "types.h"
43#include "lnetctl.h"
44
45
46#define LNET_MAX_PAYLOAD CONFIG_LNET_MAX_PAYLOAD
47#if (LNET_MAX_PAYLOAD < LNET_MTU)
48# error "LNET_MAX_PAYLOAD too small - error in configure --with-max-payload-mb"
49#elif (LNET_MAX_PAYLOAD > (PAGE_SIZE * LNET_MAX_IOV))
50# error "LNET_MAX_PAYLOAD too large - error in configure --with-max-payload-mb"
51#endif
52
53
54struct lnet_libmd;
55
56typedef struct lnet_msg {
57 struct list_head msg_activelist;
58 struct list_head msg_list;
59
60 lnet_process_id_t msg_target;
61
62 lnet_nid_t msg_from;
63 __u32 msg_type;
64
65
66 unsigned int msg_tx_committed:1;
67
68 unsigned int msg_tx_cpt:15;
69
70 unsigned int msg_rx_committed:1;
71
72 unsigned int msg_rx_cpt:15;
73
74 unsigned int msg_tx_delayed:1;
75
76 unsigned int msg_rx_delayed:1;
77
78 unsigned int msg_rx_ready_delay:1;
79
80 unsigned int msg_vmflush:1;
81 unsigned int msg_target_is_router:1;
82 unsigned int msg_routing:1;
83 unsigned int msg_ack:1;
84 unsigned int msg_sending:1;
85 unsigned int msg_receiving:1;
86 unsigned int msg_txcredit:1;
87 unsigned int msg_peertxcredit:1;
88 unsigned int msg_rtrcredit:1;
89 unsigned int msg_peerrtrcredit:1;
90 unsigned int msg_onactivelist:1;
91 unsigned int msg_rdma_get:1;
92
93 struct lnet_peer *msg_txpeer;
94 struct lnet_peer *msg_rxpeer;
95
96 void *msg_private;
97 struct lnet_libmd *msg_md;
98
99 unsigned int msg_len;
100 unsigned int msg_wanted;
101 unsigned int msg_offset;
102 unsigned int msg_niov;
103 struct kvec *msg_iov;
104 lnet_kiov_t *msg_kiov;
105
106 lnet_event_t msg_ev;
107 lnet_hdr_t msg_hdr;
108} lnet_msg_t;
109
110typedef struct lnet_libhandle {
111 struct list_head lh_hash_chain;
112 __u64 lh_cookie;
113} lnet_libhandle_t;
114
115#define lh_entry(ptr, type, member) \
116 ((type *)((char *)(ptr) - (char *)(&((type *)0)->member)))
117
118typedef struct lnet_eq {
119 struct list_head eq_list;
120 lnet_libhandle_t eq_lh;
121 lnet_seq_t eq_enq_seq;
122 lnet_seq_t eq_deq_seq;
123 unsigned int eq_size;
124 lnet_eq_handler_t eq_callback;
125 lnet_event_t *eq_events;
126 int **eq_refs;
127} lnet_eq_t;
128
129typedef struct lnet_me {
130 struct list_head me_list;
131 lnet_libhandle_t me_lh;
132 lnet_process_id_t me_match_id;
133 unsigned int me_portal;
134 unsigned int me_pos;
135 __u64 me_match_bits;
136 __u64 me_ignore_bits;
137 lnet_unlink_t me_unlink;
138 struct lnet_libmd *me_md;
139} lnet_me_t;
140
141typedef struct lnet_libmd {
142 struct list_head md_list;
143 lnet_libhandle_t md_lh;
144 lnet_me_t *md_me;
145 char *md_start;
146 unsigned int md_offset;
147 unsigned int md_length;
148 unsigned int md_max_size;
149 int md_threshold;
150 int md_refcount;
151 unsigned int md_options;
152 unsigned int md_flags;
153 void *md_user_ptr;
154 lnet_eq_t *md_eq;
155 unsigned int md_niov;
156 union {
157 struct kvec iov[LNET_MAX_IOV];
158 lnet_kiov_t kiov[LNET_MAX_IOV];
159 } md_iov;
160} lnet_libmd_t;
161
162#define LNET_MD_FLAG_ZOMBIE (1 << 0)
163#define LNET_MD_FLAG_AUTO_UNLINK (1 << 1)
164#define LNET_MD_FLAG_ABORTED (1 << 2)
165
166typedef struct {
167
168 struct list_head tp_list;
169 lnet_nid_t tp_nid;
170 unsigned int tp_threshold;
171} lnet_test_peer_t;
172
173#define LNET_COOKIE_TYPE_MD 1
174#define LNET_COOKIE_TYPE_ME 2
175#define LNET_COOKIE_TYPE_EQ 3
176#define LNET_COOKIE_TYPE_BITS 2
177#define LNET_COOKIE_MASK ((1ULL << LNET_COOKIE_TYPE_BITS) - 1ULL)
178
179struct lnet_ni;
180
181typedef struct lnet_lnd {
182
183 struct list_head lnd_list;
184 int lnd_refcount;
185
186
187 __u32 lnd_type;
188
189 int (*lnd_startup)(struct lnet_ni *ni);
190 void (*lnd_shutdown)(struct lnet_ni *ni);
191 int (*lnd_ctl)(struct lnet_ni *ni, unsigned int cmd, void *arg);
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212 int (*lnd_send)(struct lnet_ni *ni, void *private, lnet_msg_t *msg);
213
214
215
216
217
218
219
220
221 int (*lnd_recv)(struct lnet_ni *ni, void *private, lnet_msg_t *msg,
222 int delayed, unsigned int niov,
223 struct kvec *iov, lnet_kiov_t *kiov,
224 unsigned int offset, unsigned int mlen,
225 unsigned int rlen);
226
227
228
229
230
231
232
233
234
235 int (*lnd_eager_recv)(struct lnet_ni *ni, void *private,
236 lnet_msg_t *msg, void **new_privatep);
237
238
239 void (*lnd_notify)(struct lnet_ni *ni, lnet_nid_t peer, int alive);
240
241
242 void (*lnd_query)(struct lnet_ni *ni, lnet_nid_t peer,
243 unsigned long *when);
244
245
246 int (*lnd_accept)(struct lnet_ni *ni, struct socket *sock);
247} lnd_t;
248
249struct lnet_tx_queue {
250 int tq_credits;
251 int tq_credits_min;
252 int tq_credits_max;
253 struct list_head tq_delayed;
254};
255
256typedef struct lnet_ni {
257 spinlock_t ni_lock;
258 struct list_head ni_list;
259 struct list_head ni_cptlist;
260 int ni_maxtxcredits;
261
262 int ni_peertxcredits;
263
264 int ni_peerrtrcredits;
265
266 int ni_peertimeout;
267 int ni_ncpts;
268 __u32 *ni_cpts;
269 lnet_nid_t ni_nid;
270 void *ni_data;
271 lnd_t *ni_lnd;
272 struct lnet_tx_queue **ni_tx_queues;
273 int **ni_refs;
274 time64_t ni_last_alive;
275 lnet_ni_status_t *ni_status;
276
277 char *ni_interfaces[LNET_MAX_INTERFACES];
278} lnet_ni_t;
279
280#define LNET_PROTO_PING_MATCHBITS 0x8000000000000000LL
281
282
283
284
285
286#define LNET_PING_FEAT_INVAL (0)
287#define LNET_PING_FEAT_BASE (1 << 0)
288#define LNET_PING_FEAT_NI_STATUS (1 << 1)
289#define LNET_PING_FEAT_RTE_DISABLED (1 << 2)
290
291#define LNET_PING_FEAT_MASK (LNET_PING_FEAT_BASE | \
292 LNET_PING_FEAT_NI_STATUS)
293
294
295#define LNET_MAX_RTR_NIS 16
296#define LNET_PINGINFO_SIZE offsetof(lnet_ping_info_t, pi_ni[LNET_MAX_RTR_NIS])
297typedef struct {
298
299 struct list_head rcd_list;
300 lnet_handle_md_t rcd_mdh;
301 struct lnet_peer *rcd_gateway;
302 lnet_ping_info_t *rcd_pinginfo;
303} lnet_rc_data_t;
304
305typedef struct lnet_peer {
306 struct list_head lp_hashlist;
307 struct list_head lp_txq;
308
309 struct list_head lp_rtrq;
310
311 struct list_head lp_rtr_list;
312 int lp_txcredits;
313 int lp_mintxcredits;
314 int lp_rtrcredits;
315 int lp_minrtrcredits;
316 unsigned int lp_alive:1;
317 unsigned int lp_notify:1;
318 unsigned int lp_notifylnd:1;
319
320 unsigned int lp_notifying:1;
321
322 unsigned int lp_ping_notsent;
323
324 int lp_alive_count;
325
326 long lp_txqnob;
327 unsigned long lp_timestamp;
328
329 unsigned long lp_ping_timestamp;
330
331 unsigned long lp_ping_deadline;
332
333 unsigned long lp_last_alive;
334 unsigned long lp_last_query;
335
336 lnet_ni_t *lp_ni;
337 lnet_nid_t lp_nid;
338 int lp_refcount;
339 int lp_cpt;
340
341 int lp_rtr_refcount;
342
343 unsigned int lp_ping_feats;
344 struct list_head lp_routes;
345 lnet_rc_data_t *lp_rcd;
346} lnet_peer_t;
347
348
349#define LNET_PEER_HASH_BITS 9
350#define LNET_PEER_HASH_SIZE (1 << LNET_PEER_HASH_BITS)
351
352
353struct lnet_peer_table {
354 int pt_version;
355 int pt_number;
356
357 int pt_zombies;
358 struct list_head pt_deathrow;
359 struct list_head *pt_hash;
360};
361
362
363
364
365
366#define lnet_peer_aliveness_enabled(lp) (the_lnet.ln_routing && \
367 (lp)->lp_ni->ni_peertimeout > 0)
368
369typedef struct {
370 struct list_head lr_list;
371 struct list_head lr_gwlist;
372 lnet_peer_t *lr_gateway;
373 __u32 lr_net;
374 int lr_seq;
375 unsigned int lr_downis;
376 __u32 lr_hops;
377 unsigned int lr_priority;
378} lnet_route_t;
379
380#define LNET_REMOTE_NETS_HASH_DEFAULT (1U << 7)
381#define LNET_REMOTE_NETS_HASH_MAX (1U << 16)
382#define LNET_REMOTE_NETS_HASH_SIZE (1 << the_lnet.ln_remote_nets_hbits)
383
384typedef struct {
385 struct list_head lrn_list;
386
387 struct list_head lrn_routes;
388 __u32 lrn_net;
389} lnet_remotenet_t;
390
391
392#define LNET_CREDIT_OK 0
393
394#define LNET_CREDIT_WAIT 1
395
396typedef struct {
397 struct list_head rbp_bufs;
398 struct list_head rbp_msgs;
399
400 int rbp_npages;
401
402 int rbp_req_nbuffers;
403
404 int rbp_nbuffers;
405 int rbp_credits;
406
407 int rbp_mincredits;
408} lnet_rtrbufpool_t;
409
410typedef struct {
411 struct list_head rb_list;
412 lnet_rtrbufpool_t *rb_pool;
413 lnet_kiov_t rb_kiov[0];
414} lnet_rtrbuf_t;
415
416#define LNET_PEER_HASHSIZE 503
417
418#define LNET_TINY_BUF_IDX 0
419#define LNET_SMALL_BUF_IDX 1
420#define LNET_LARGE_BUF_IDX 2
421
422
423#define LNET_NRBPOOLS (LNET_LARGE_BUF_IDX + 1)
424
425enum {
426
427 LNET_MATCHMD_NONE = (1 << 0),
428
429 LNET_MATCHMD_OK = (1 << 1),
430
431 LNET_MATCHMD_DROP = (1 << 2),
432
433 LNET_MATCHMD_EXHAUSTED = (1 << 3),
434
435 LNET_MATCHMD_FINISH = (LNET_MATCHMD_OK | LNET_MATCHMD_DROP),
436};
437
438
439#define LNET_PTL_LAZY (1 << 0)
440#define LNET_PTL_MATCH_UNIQUE (1 << 1)
441#define LNET_PTL_MATCH_WILDCARD (1 << 2)
442
443
444
445struct lnet_match_info {
446 __u64 mi_mbits;
447 lnet_process_id_t mi_id;
448 unsigned int mi_opc;
449 unsigned int mi_portal;
450 unsigned int mi_rlength;
451 unsigned int mi_roffset;
452};
453
454
455#define LNET_MT_HASH_BITS 8
456#define LNET_MT_HASH_SIZE (1 << LNET_MT_HASH_BITS)
457#define LNET_MT_HASH_MASK (LNET_MT_HASH_SIZE - 1)
458
459
460
461
462#define LNET_MT_HASH_IGNORE LNET_MT_HASH_SIZE
463
464
465
466
467
468#define LNET_MT_BITS_U64 6
469#define LNET_MT_EXHAUSTED_BITS (LNET_MT_HASH_BITS - LNET_MT_BITS_U64)
470#define LNET_MT_EXHAUSTED_BMAP ((1 << LNET_MT_EXHAUSTED_BITS) + 1)
471
472
473struct lnet_match_table {
474
475 unsigned int mt_cpt;
476 unsigned int mt_portal;
477
478
479
480
481 unsigned int mt_enabled;
482
483 __u64 mt_exhausted[LNET_MT_EXHAUSTED_BMAP];
484 struct list_head *mt_mhash;
485};
486
487
488
489#define LNET_PTL_ROTOR_OFF 0
490
491#define LNET_PTL_ROTOR_ON 1
492
493#define LNET_PTL_ROTOR_RR_RT 2
494
495#define LNET_PTL_ROTOR_HASH_RT 3
496
497typedef struct lnet_portal {
498 spinlock_t ptl_lock;
499 unsigned int ptl_index;
500
501 unsigned int ptl_options;
502
503 struct list_head ptl_msg_stealing;
504
505 struct list_head ptl_msg_delayed;
506
507 struct lnet_match_table **ptl_mtables;
508
509 unsigned int ptl_rotor;
510
511 int ptl_mt_nmaps;
512
513 int ptl_mt_maps[0];
514} lnet_portal_t;
515
516#define LNET_LH_HASH_BITS 12
517#define LNET_LH_HASH_SIZE (1ULL << LNET_LH_HASH_BITS)
518#define LNET_LH_HASH_MASK (LNET_LH_HASH_SIZE - 1)
519
520
521struct lnet_res_container {
522 unsigned int rec_type;
523 __u64 rec_lh_cookie;
524 struct list_head rec_active;
525 struct list_head *rec_lh_hash;
526};
527
528
529struct lnet_msg_container {
530 int msc_init;
531
532 int msc_nfinalizers;
533
534 struct list_head msc_finalizing;
535 struct list_head msc_active;
536
537 void **msc_finalizers;
538};
539
540
541#define LNET_RC_STATE_SHUTDOWN 0
542#define LNET_RC_STATE_RUNNING 1
543#define LNET_RC_STATE_STOPPING 2
544
545typedef struct {
546
547 struct cfs_cpt_table *ln_cpt_table;
548
549 unsigned int ln_cpt_number;
550 unsigned int ln_cpt_bits;
551
552
553 struct cfs_percpt_lock *ln_res_lock;
554
555 int ln_nportals;
556
557 lnet_portal_t **ln_portals;
558
559 struct lnet_res_container **ln_me_containers;
560
561 struct lnet_res_container **ln_md_containers;
562
563
564 struct lnet_res_container ln_eq_container;
565 wait_queue_head_t ln_eq_waitq;
566 spinlock_t ln_eq_wait_lock;
567 unsigned int ln_remote_nets_hbits;
568
569
570 struct cfs_percpt_lock *ln_net_lock;
571
572 struct lnet_msg_container **ln_msg_containers;
573 lnet_counters_t **ln_counters;
574 struct lnet_peer_table **ln_peer_tables;
575
576 struct list_head ln_test_peers;
577 struct list_head ln_drop_rules;
578 struct list_head ln_delay_rules;
579
580 struct list_head ln_nis;
581
582 struct list_head ln_nis_cpt;
583
584 struct list_head ln_nis_zombie;
585 lnet_ni_t *ln_loni;
586
587
588 struct list_head *ln_remote_nets_hash;
589
590 __u64 ln_remote_nets_version;
591
592 struct list_head ln_routers;
593
594 __u64 ln_routers_version;
595
596 lnet_rtrbufpool_t **ln_rtrpools;
597
598 lnet_handle_md_t ln_ping_target_md;
599 lnet_handle_eq_t ln_ping_target_eq;
600 lnet_ping_info_t *ln_ping_info;
601
602
603 int ln_rc_state;
604
605 lnet_handle_eq_t ln_rc_eqh;
606
607 struct list_head ln_rcd_deathrow;
608
609 struct list_head ln_rcd_zombie;
610
611 struct semaphore ln_rc_signal;
612
613 struct mutex ln_api_mutex;
614 struct mutex ln_lnd_mutex;
615 struct mutex ln_delay_mutex;
616
617 int ln_niinit_self;
618
619 int ln_refcount;
620
621 int ln_shutdown;
622
623 int ln_routing;
624 lnet_pid_t ln_pid;
625
626 __u64 ln_interface_cookie;
627
628 struct list_head ln_lnds;
629
630
631 int ln_testprotocompat;
632
633
634
635
636
637
638
639 bool ln_nis_from_mod_params;
640
641
642
643
644
645
646 wait_queue_head_t ln_rc_waitq;
647
648} lnet_t;
649
650#endif
651