1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#ifndef __LNET_LIB_LNET_H__
36#define __LNET_LIB_LNET_H__
37
38#include "../libcfs/libcfs.h"
39#include "api.h"
40#include "lnet.h"
41#include "lib-types.h"
42
43extern lnet_t the_lnet;
44
45#if (BITS_PER_LONG == 32)
46
47#define LNET_CPT_MAX_BITS 1
48
49#else
50
51
52
53
54#define LNET_CPT_MAX_BITS 8
55#endif
56
57
58#define LNET_CPT_MAX (1 << LNET_CPT_MAX_BITS)
59
60#define LNET_CPT_NUMBER (the_lnet.ln_cpt_number)
61#define LNET_CPT_BITS (the_lnet.ln_cpt_bits)
62#define LNET_CPT_MASK ((1ULL << LNET_CPT_BITS) - 1)
63
64
65#define LNET_LOCK_EX CFS_PERCPT_LOCK_EX
66
67static inline int lnet_is_wire_handle_none(lnet_handle_wire_t *wh)
68{
69 return (wh->wh_interface_cookie == LNET_WIRE_HANDLE_COOKIE_NONE &&
70 wh->wh_object_cookie == LNET_WIRE_HANDLE_COOKIE_NONE);
71}
72
73static inline int lnet_md_exhausted(lnet_libmd_t *md)
74{
75 return (md->md_threshold == 0 ||
76 ((md->md_options & LNET_MD_MAX_SIZE) != 0 &&
77 md->md_offset + md->md_max_size > md->md_length));
78}
79
80static inline int lnet_md_unlinkable(lnet_libmd_t *md)
81{
82
83
84
85
86
87 if (md->md_refcount != 0)
88 return 0;
89
90 if ((md->md_flags & LNET_MD_FLAG_ZOMBIE) != 0)
91 return 1;
92
93 return ((md->md_flags & LNET_MD_FLAG_AUTO_UNLINK) != 0 &&
94 lnet_md_exhausted(md));
95}
96
97#define lnet_cpt_table() (the_lnet.ln_cpt_table)
98#define lnet_cpt_current() cfs_cpt_current(the_lnet.ln_cpt_table, 1)
99
100static inline int
101lnet_cpt_of_cookie(__u64 cookie)
102{
103 unsigned int cpt = (cookie >> LNET_COOKIE_TYPE_BITS) & LNET_CPT_MASK;
104
105
106
107 return cpt < LNET_CPT_NUMBER ? cpt : cpt % LNET_CPT_NUMBER;
108}
109
110static inline void
111lnet_res_lock(int cpt)
112{
113 cfs_percpt_lock(the_lnet.ln_res_lock, cpt);
114}
115
116static inline void
117lnet_res_unlock(int cpt)
118{
119 cfs_percpt_unlock(the_lnet.ln_res_lock, cpt);
120}
121
122static inline int
123lnet_res_lock_current(void)
124{
125 int cpt = lnet_cpt_current();
126
127 lnet_res_lock(cpt);
128 return cpt;
129}
130
131static inline void
132lnet_net_lock(int cpt)
133{
134 cfs_percpt_lock(the_lnet.ln_net_lock, cpt);
135}
136
137static inline void
138lnet_net_unlock(int cpt)
139{
140 cfs_percpt_unlock(the_lnet.ln_net_lock, cpt);
141}
142
143static inline int
144lnet_net_lock_current(void)
145{
146 int cpt = lnet_cpt_current();
147
148 lnet_net_lock(cpt);
149 return cpt;
150}
151
152#define LNET_LOCK() lnet_net_lock(LNET_LOCK_EX)
153#define LNET_UNLOCK() lnet_net_unlock(LNET_LOCK_EX)
154
155#define lnet_ptl_lock(ptl) spin_lock(&(ptl)->ptl_lock)
156#define lnet_ptl_unlock(ptl) spin_unlock(&(ptl)->ptl_lock)
157#define lnet_eq_wait_lock() spin_lock(&the_lnet.ln_eq_wait_lock)
158#define lnet_eq_wait_unlock() spin_unlock(&the_lnet.ln_eq_wait_lock)
159#define lnet_ni_lock(ni) spin_lock(&(ni)->ni_lock)
160#define lnet_ni_unlock(ni) spin_unlock(&(ni)->ni_lock)
161
162#define MAX_PORTALS 64
163
164static inline lnet_eq_t *
165lnet_eq_alloc(void)
166{
167 lnet_eq_t *eq;
168
169 LIBCFS_ALLOC(eq, sizeof(*eq));
170 return eq;
171}
172
173static inline void
174lnet_eq_free(lnet_eq_t *eq)
175{
176 LIBCFS_FREE(eq, sizeof(*eq));
177}
178
179static inline lnet_libmd_t *
180lnet_md_alloc(lnet_md_t *umd)
181{
182 lnet_libmd_t *md;
183 unsigned int size;
184 unsigned int niov;
185
186 if ((umd->options & LNET_MD_KIOV) != 0) {
187 niov = umd->length;
188 size = offsetof(lnet_libmd_t, md_iov.kiov[niov]);
189 } else {
190 niov = ((umd->options & LNET_MD_IOVEC) != 0) ?
191 umd->length : 1;
192 size = offsetof(lnet_libmd_t, md_iov.iov[niov]);
193 }
194
195 LIBCFS_ALLOC(md, size);
196
197 if (md != NULL) {
198
199 md->md_options = umd->options;
200 md->md_niov = niov;
201 INIT_LIST_HEAD(&md->md_list);
202 }
203
204 return md;
205}
206
207static inline void
208lnet_md_free(lnet_libmd_t *md)
209{
210 unsigned int size;
211
212 if ((md->md_options & LNET_MD_KIOV) != 0)
213 size = offsetof(lnet_libmd_t, md_iov.kiov[md->md_niov]);
214 else
215 size = offsetof(lnet_libmd_t, md_iov.iov[md->md_niov]);
216
217 LIBCFS_FREE(md, size);
218}
219
220static inline lnet_me_t *
221lnet_me_alloc(void)
222{
223 lnet_me_t *me;
224
225 LIBCFS_ALLOC(me, sizeof(*me));
226 return me;
227}
228
229static inline void
230lnet_me_free(lnet_me_t *me)
231{
232 LIBCFS_FREE(me, sizeof(*me));
233}
234
235static inline lnet_msg_t *
236lnet_msg_alloc(void)
237{
238 lnet_msg_t *msg;
239
240 LIBCFS_ALLOC(msg, sizeof(*msg));
241
242
243 return msg;
244}
245
246static inline void
247lnet_msg_free(lnet_msg_t *msg)
248{
249 LASSERT(!msg->msg_onactivelist);
250 LIBCFS_FREE(msg, sizeof(*msg));
251}
252
253lnet_libhandle_t *lnet_res_lh_lookup(struct lnet_res_container *rec,
254 __u64 cookie);
255void lnet_res_lh_initialize(struct lnet_res_container *rec,
256 lnet_libhandle_t *lh);
257static inline void
258lnet_res_lh_invalidate(lnet_libhandle_t *lh)
259{
260
261 list_del(&lh->lh_hash_chain);
262}
263
264static inline void
265lnet_eq2handle(lnet_handle_eq_t *handle, lnet_eq_t *eq)
266{
267 if (eq == NULL) {
268 LNetInvalidateHandle(handle);
269 return;
270 }
271
272 handle->cookie = eq->eq_lh.lh_cookie;
273}
274
275static inline lnet_eq_t *
276lnet_handle2eq(lnet_handle_eq_t *handle)
277{
278 lnet_libhandle_t *lh;
279
280 lh = lnet_res_lh_lookup(&the_lnet.ln_eq_container, handle->cookie);
281 if (lh == NULL)
282 return NULL;
283
284 return lh_entry(lh, lnet_eq_t, eq_lh);
285}
286
287static inline void
288lnet_md2handle(lnet_handle_md_t *handle, lnet_libmd_t *md)
289{
290 handle->cookie = md->md_lh.lh_cookie;
291}
292
293static inline lnet_libmd_t *
294lnet_handle2md(lnet_handle_md_t *handle)
295{
296
297 lnet_libhandle_t *lh;
298 int cpt;
299
300 cpt = lnet_cpt_of_cookie(handle->cookie);
301 lh = lnet_res_lh_lookup(the_lnet.ln_md_containers[cpt],
302 handle->cookie);
303 if (lh == NULL)
304 return NULL;
305
306 return lh_entry(lh, lnet_libmd_t, md_lh);
307}
308
309static inline lnet_libmd_t *
310lnet_wire_handle2md(lnet_handle_wire_t *wh)
311{
312
313 lnet_libhandle_t *lh;
314 int cpt;
315
316 if (wh->wh_interface_cookie != the_lnet.ln_interface_cookie)
317 return NULL;
318
319 cpt = lnet_cpt_of_cookie(wh->wh_object_cookie);
320 lh = lnet_res_lh_lookup(the_lnet.ln_md_containers[cpt],
321 wh->wh_object_cookie);
322 if (lh == NULL)
323 return NULL;
324
325 return lh_entry(lh, lnet_libmd_t, md_lh);
326}
327
328static inline void
329lnet_me2handle(lnet_handle_me_t *handle, lnet_me_t *me)
330{
331 handle->cookie = me->me_lh.lh_cookie;
332}
333
334static inline lnet_me_t *
335lnet_handle2me(lnet_handle_me_t *handle)
336{
337
338 lnet_libhandle_t *lh;
339 int cpt;
340
341 cpt = lnet_cpt_of_cookie(handle->cookie);
342 lh = lnet_res_lh_lookup(the_lnet.ln_me_containers[cpt],
343 handle->cookie);
344 if (lh == NULL)
345 return NULL;
346
347 return lh_entry(lh, lnet_me_t, me_lh);
348}
349
350static inline void
351lnet_peer_addref_locked(lnet_peer_t *lp)
352{
353 LASSERT(lp->lp_refcount > 0);
354 lp->lp_refcount++;
355}
356
357void lnet_destroy_peer_locked(lnet_peer_t *lp);
358
359static inline void
360lnet_peer_decref_locked(lnet_peer_t *lp)
361{
362 LASSERT(lp->lp_refcount > 0);
363 lp->lp_refcount--;
364 if (lp->lp_refcount == 0)
365 lnet_destroy_peer_locked(lp);
366}
367
368static inline int
369lnet_isrouter(lnet_peer_t *lp)
370{
371 return lp->lp_rtr_refcount != 0;
372}
373
374static inline void
375lnet_ni_addref_locked(lnet_ni_t *ni, int cpt)
376{
377 LASSERT(cpt >= 0 && cpt < LNET_CPT_NUMBER);
378 LASSERT(*ni->ni_refs[cpt] >= 0);
379
380 (*ni->ni_refs[cpt])++;
381}
382
383static inline void
384lnet_ni_addref(lnet_ni_t *ni)
385{
386 lnet_net_lock(0);
387 lnet_ni_addref_locked(ni, 0);
388 lnet_net_unlock(0);
389}
390
391static inline void
392lnet_ni_decref_locked(lnet_ni_t *ni, int cpt)
393{
394 LASSERT(cpt >= 0 && cpt < LNET_CPT_NUMBER);
395 LASSERT(*ni->ni_refs[cpt] > 0);
396
397 (*ni->ni_refs[cpt])--;
398}
399
400static inline void
401lnet_ni_decref(lnet_ni_t *ni)
402{
403 lnet_net_lock(0);
404 lnet_ni_decref_locked(ni, 0);
405 lnet_net_unlock(0);
406}
407
408void lnet_ni_free(lnet_ni_t *ni);
409
410static inline int
411lnet_nid2peerhash(lnet_nid_t nid)
412{
413 return hash_long(nid, LNET_PEER_HASH_BITS);
414}
415
416static inline struct list_head *
417lnet_net2rnethash(__u32 net)
418{
419 return &the_lnet.ln_remote_nets_hash[(LNET_NETNUM(net) +
420 LNET_NETTYP(net)) &
421 ((1U << the_lnet.ln_remote_nets_hbits) - 1)];
422}
423
424extern lnd_t the_lolnd;
425extern int avoid_asym_router_failure;
426
427int lnet_cpt_of_nid_locked(lnet_nid_t nid);
428int lnet_cpt_of_nid(lnet_nid_t nid);
429lnet_ni_t *lnet_nid2ni_locked(lnet_nid_t nid, int cpt);
430lnet_ni_t *lnet_net2ni_locked(__u32 net, int cpt);
431lnet_ni_t *lnet_net2ni(__u32 net);
432
433int lnet_init(void);
434void lnet_fini(void);
435
436int lnet_notify(lnet_ni_t *ni, lnet_nid_t peer, int alive, unsigned long when);
437void lnet_notify_locked(lnet_peer_t *lp, int notifylnd, int alive,
438 unsigned long when);
439int lnet_add_route(__u32 net, unsigned int hops, lnet_nid_t gateway_nid,
440 unsigned int priority);
441int lnet_check_routes(void);
442int lnet_del_route(__u32 net, lnet_nid_t gw_nid);
443void lnet_destroy_routes(void);
444int lnet_get_route(int idx, __u32 *net, __u32 *hops,
445 lnet_nid_t *gateway, __u32 *alive, __u32 *priority);
446void lnet_router_debugfs_init(void);
447void lnet_router_debugfs_fini(void);
448int lnet_rtrpools_alloc(int im_a_router);
449void lnet_rtrpools_free(void);
450lnet_remotenet_t *lnet_find_net_locked(__u32 net);
451
452int lnet_islocalnid(lnet_nid_t nid);
453int lnet_islocalnet(__u32 net);
454
455void lnet_msg_attach_md(lnet_msg_t *msg, lnet_libmd_t *md,
456 unsigned int offset, unsigned int mlen);
457void lnet_msg_detach_md(lnet_msg_t *msg, int status);
458void lnet_build_unlink_event(lnet_libmd_t *md, lnet_event_t *ev);
459void lnet_build_msg_event(lnet_msg_t *msg, lnet_event_kind_t ev_type);
460void lnet_msg_commit(lnet_msg_t *msg, int cpt);
461void lnet_msg_decommit(lnet_msg_t *msg, int cpt, int status);
462
463void lnet_eq_enqueue_event(lnet_eq_t *eq, lnet_event_t *ev);
464void lnet_prep_send(lnet_msg_t *msg, int type, lnet_process_id_t target,
465 unsigned int offset, unsigned int len);
466int lnet_send(lnet_nid_t nid, lnet_msg_t *msg, lnet_nid_t rtr_nid);
467void lnet_return_tx_credits_locked(lnet_msg_t *msg);
468void lnet_return_rx_credits_locked(lnet_msg_t *msg);
469
470
471
472static inline int
473lnet_ptl_is_lazy(lnet_portal_t *ptl)
474{
475 return !!(ptl->ptl_options & LNET_PTL_LAZY);
476}
477
478static inline int
479lnet_ptl_is_unique(lnet_portal_t *ptl)
480{
481 return !!(ptl->ptl_options & LNET_PTL_MATCH_UNIQUE);
482}
483
484static inline int
485lnet_ptl_is_wildcard(lnet_portal_t *ptl)
486{
487 return !!(ptl->ptl_options & LNET_PTL_MATCH_WILDCARD);
488}
489
490static inline void
491lnet_ptl_setopt(lnet_portal_t *ptl, int opt)
492{
493 ptl->ptl_options |= opt;
494}
495
496static inline void
497lnet_ptl_unsetopt(lnet_portal_t *ptl, int opt)
498{
499 ptl->ptl_options &= ~opt;
500}
501
502
503struct list_head *lnet_mt_match_head(struct lnet_match_table *mtable,
504 lnet_process_id_t id, __u64 mbits);
505struct lnet_match_table *lnet_mt_of_attach(unsigned int index,
506 lnet_process_id_t id, __u64 mbits,
507 __u64 ignore_bits,
508 lnet_ins_pos_t pos);
509int lnet_mt_match_md(struct lnet_match_table *mtable,
510 struct lnet_match_info *info, struct lnet_msg *msg);
511
512
513void lnet_ptl_attach_md(lnet_me_t *me, lnet_libmd_t *md,
514 struct list_head *matches, struct list_head *drops);
515void lnet_ptl_detach_md(lnet_me_t *me, lnet_libmd_t *md);
516int lnet_ptl_match_md(struct lnet_match_info *info, struct lnet_msg *msg);
517
518
519int lnet_portals_create(void);
520void lnet_portals_destroy(void);
521
522
523int lnet_parse(lnet_ni_t *ni, lnet_hdr_t *hdr,
524 lnet_nid_t fromnid, void *private, int rdma_req);
525void lnet_recv(lnet_ni_t *ni, void *private, lnet_msg_t *msg, int delayed,
526 unsigned int offset, unsigned int mlen, unsigned int rlen);
527lnet_msg_t *lnet_create_reply_msg(lnet_ni_t *ni, lnet_msg_t *get_msg);
528void lnet_set_reply_msg_len(lnet_ni_t *ni, lnet_msg_t *msg, unsigned int len);
529
530void lnet_finalize(lnet_ni_t *ni, lnet_msg_t *msg, int rc);
531
532void lnet_drop_delayed_msg_list(struct list_head *head, char *reason);
533void lnet_recv_delayed_msg_list(struct list_head *head);
534
535int lnet_msg_container_setup(struct lnet_msg_container *container, int cpt);
536void lnet_msg_container_cleanup(struct lnet_msg_container *container);
537void lnet_msg_containers_destroy(void);
538int lnet_msg_containers_create(void);
539
540char *lnet_msgtyp2str(int type);
541void lnet_print_hdr(lnet_hdr_t *hdr);
542int lnet_fail_nid(lnet_nid_t nid, unsigned int threshold);
543
544void lnet_counters_get(lnet_counters_t *counters);
545void lnet_counters_reset(void);
546
547unsigned int lnet_iov_nob(unsigned int niov, struct kvec *iov);
548int lnet_extract_iov(int dst_niov, struct kvec *dst,
549 int src_niov, struct kvec *src,
550 unsigned int offset, unsigned int len);
551
552unsigned int lnet_kiov_nob(unsigned int niov, lnet_kiov_t *iov);
553int lnet_extract_kiov(int dst_niov, lnet_kiov_t *dst,
554 int src_niov, lnet_kiov_t *src,
555 unsigned int offset, unsigned int len);
556
557void lnet_copy_iov2iov(unsigned int ndiov, struct kvec *diov,
558 unsigned int doffset,
559 unsigned int nsiov, struct kvec *siov,
560 unsigned int soffset, unsigned int nob);
561void lnet_copy_kiov2iov(unsigned int niov, struct kvec *iov,
562 unsigned int iovoffset,
563 unsigned int nkiov, lnet_kiov_t *kiov,
564 unsigned int kiovoffset, unsigned int nob);
565void lnet_copy_iov2kiov(unsigned int nkiov, lnet_kiov_t *kiov,
566 unsigned int kiovoffset,
567 unsigned int niov, struct kvec *iov,
568 unsigned int iovoffset, unsigned int nob);
569void lnet_copy_kiov2kiov(unsigned int ndkiov, lnet_kiov_t *dkiov,
570 unsigned int doffset,
571 unsigned int nskiov, lnet_kiov_t *skiov,
572 unsigned int soffset, unsigned int nob);
573
574static inline void
575lnet_copy_iov2flat(int dlen, void *dest, unsigned int doffset,
576 unsigned int nsiov, struct kvec *siov, unsigned int soffset,
577 unsigned int nob)
578{
579 struct kvec diov = { dest, dlen};
580
581 lnet_copy_iov2iov(1, &diov, doffset,
582 nsiov, siov, soffset, nob);
583}
584
585static inline void
586lnet_copy_kiov2flat(int dlen, void *dest, unsigned int doffset,
587 unsigned int nsiov, lnet_kiov_t *skiov,
588 unsigned int soffset, unsigned int nob)
589{
590 struct kvec diov = { dest, dlen};
591
592 lnet_copy_kiov2iov(1, &diov, doffset,
593 nsiov, skiov, soffset, nob);
594}
595
596static inline void
597lnet_copy_flat2iov(unsigned int ndiov, struct kvec *diov, unsigned int doffset,
598 int slen, void *src, unsigned int soffset, unsigned int nob)
599{
600 struct kvec siov = { src, slen};
601
602 lnet_copy_iov2iov(ndiov, diov, doffset,
603 1, &siov, soffset, nob);
604}
605
606static inline void
607lnet_copy_flat2kiov(unsigned int ndiov, lnet_kiov_t *dkiov,
608 unsigned int doffset, int slen, void *src,
609 unsigned int soffset, unsigned int nob)
610{
611 struct kvec siov = { src, slen};
612
613 lnet_copy_iov2kiov(ndiov, dkiov, doffset,
614 1, &siov, soffset, nob);
615}
616
617void lnet_me_unlink(lnet_me_t *me);
618
619void lnet_md_unlink(lnet_libmd_t *md);
620void lnet_md_deconstruct(lnet_libmd_t *lmd, lnet_md_t *umd);
621
622void lnet_register_lnd(lnd_t *lnd);
623void lnet_unregister_lnd(lnd_t *lnd);
624
625int lnet_connect(struct socket **sockp, lnet_nid_t peer_nid,
626 __u32 local_ip, __u32 peer_ip, int peer_port);
627void lnet_connect_console_error(int rc, lnet_nid_t peer_nid,
628 __u32 peer_ip, int port);
629int lnet_count_acceptor_nis(void);
630int lnet_acceptor_timeout(void);
631int lnet_acceptor_port(void);
632
633int lnet_count_acceptor_nis(void);
634int lnet_acceptor_port(void);
635
636int lnet_acceptor_start(void);
637void lnet_acceptor_stop(void);
638
639int lnet_ipif_query(char *name, int *up, __u32 *ip, __u32 *mask);
640int lnet_ipif_enumerate(char ***names);
641void lnet_ipif_free_enumeration(char **names, int n);
642int lnet_sock_setbuf(struct socket *socket, int txbufsize, int rxbufsize);
643int lnet_sock_getbuf(struct socket *socket, int *txbufsize, int *rxbufsize);
644int lnet_sock_getaddr(struct socket *socket, bool remote, __u32 *ip, int *port);
645int lnet_sock_write(struct socket *sock, void *buffer, int nob, int timeout);
646int lnet_sock_read(struct socket *sock, void *buffer, int nob, int timeout);
647
648int lnet_sock_listen(struct socket **sockp, __u32 ip, int port, int backlog);
649int lnet_sock_accept(struct socket **newsockp, struct socket *sock);
650int lnet_sock_connect(struct socket **sockp, int *fatal,
651 __u32 local_ip, int local_port,
652 __u32 peer_ip, int peer_port);
653void libcfs_sock_release(struct socket *sock);
654
655int lnet_peers_start_down(void);
656int lnet_peer_buffer_credits(lnet_ni_t *ni);
657
658int lnet_router_checker_start(void);
659void lnet_router_checker_stop(void);
660void lnet_router_ni_update_locked(lnet_peer_t *gw, __u32 net);
661void lnet_swap_pinginfo(lnet_ping_info_t *info);
662
663int lnet_ping_target_init(void);
664void lnet_ping_target_fini(void);
665int lnet_ping(lnet_process_id_t id, int timeout_ms,
666 lnet_process_id_t *ids, int n_ids);
667
668int lnet_parse_ip2nets(char **networksp, char *ip2nets);
669int lnet_parse_routes(char *route_str, int *im_a_router);
670int lnet_parse_networks(struct list_head *nilist, char *networks);
671
672int lnet_nid2peer_locked(lnet_peer_t **lpp, lnet_nid_t nid, int cpt);
673lnet_peer_t *lnet_find_peer_locked(struct lnet_peer_table *ptable,
674 lnet_nid_t nid);
675void lnet_peer_tables_cleanup(void);
676void lnet_peer_tables_destroy(void);
677int lnet_peer_tables_create(void);
678void lnet_debug_peer(lnet_nid_t nid);
679
680static inline void
681lnet_peer_set_alive(lnet_peer_t *lp)
682{
683 lp->lp_last_alive = lp->lp_last_query = jiffies;
684 if (!lp->lp_alive)
685 lnet_notify_locked(lp, 0, 1, lp->lp_last_alive);
686}
687
688#endif
689