1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37#define DEBUG_SUBSYSTEM S_LNET
38#include <linux/lnet/lib-lnet.h>
39
40
41static int accept_port = 988;
42static int accept_backlog = 127;
43static int accept_timeout = 5;
44
45struct {
46 int pta_shutdown;
47 socket_t *pta_sock;
48 struct completion pta_signal;
49} lnet_acceptor_state;
50
51int
52lnet_acceptor_port(void)
53{
54 return accept_port;
55}
56
57static inline int
58lnet_accept_magic(__u32 magic, __u32 constant)
59{
60 return (magic == constant ||
61 magic == __swab32(constant));
62}
63
64
65EXPORT_SYMBOL(lnet_acceptor_port);
66
67static char *accept = "secure";
68
69CFS_MODULE_PARM(accept, "s", charp, 0444,
70 "Accept connections (secure|all|none)");
71CFS_MODULE_PARM(accept_port, "i", int, 0444,
72 "Acceptor's port (same on all nodes)");
73CFS_MODULE_PARM(accept_backlog, "i", int, 0444,
74 "Acceptor's listen backlog");
75CFS_MODULE_PARM(accept_timeout, "i", int, 0644,
76 "Acceptor's timeout (seconds)");
77
78static char *accept_type = NULL;
79
80int
81lnet_acceptor_get_tunables(void)
82{
83
84
85
86 accept_type = accept;
87 return 0;
88}
89
90int
91lnet_acceptor_timeout(void)
92{
93 return accept_timeout;
94}
95EXPORT_SYMBOL(lnet_acceptor_timeout);
96
97void
98lnet_connect_console_error (int rc, lnet_nid_t peer_nid,
99 __u32 peer_ip, int peer_port)
100{
101 switch (rc) {
102
103 case -ECONNREFUSED:
104 CNETERR("Connection to %s at host %u.%u.%u.%u on port %d was "
105 "refused: check that Lustre is running on that node.\n",
106 libcfs_nid2str(peer_nid),
107 HIPQUAD(peer_ip), peer_port);
108 break;
109 case -EHOSTUNREACH:
110 case -ENETUNREACH:
111 CNETERR("Connection to %s at host %u.%u.%u.%u "
112 "was unreachable: the network or that node may "
113 "be down, or Lustre may be misconfigured.\n",
114 libcfs_nid2str(peer_nid), HIPQUAD(peer_ip));
115 break;
116 case -ETIMEDOUT:
117 CNETERR("Connection to %s at host %u.%u.%u.%u on "
118 "port %d took too long: that node may be hung "
119 "or experiencing high load.\n",
120 libcfs_nid2str(peer_nid),
121 HIPQUAD(peer_ip), peer_port);
122 break;
123 case -ECONNRESET:
124 LCONSOLE_ERROR_MSG(0x11b, "Connection to %s at host %u.%u.%u.%u"
125 " on port %d was reset: "
126 "is it running a compatible version of "
127 "Lustre and is %s one of its NIDs?\n",
128 libcfs_nid2str(peer_nid),
129 HIPQUAD(peer_ip), peer_port,
130 libcfs_nid2str(peer_nid));
131 break;
132 case -EPROTO:
133 LCONSOLE_ERROR_MSG(0x11c, "Protocol error connecting to %s at "
134 "host %u.%u.%u.%u on port %d: is it running "
135 "a compatible version of Lustre?\n",
136 libcfs_nid2str(peer_nid),
137 HIPQUAD(peer_ip), peer_port);
138 break;
139 case -EADDRINUSE:
140 LCONSOLE_ERROR_MSG(0x11d, "No privileged ports available to "
141 "connect to %s at host %u.%u.%u.%u on port "
142 "%d\n", libcfs_nid2str(peer_nid),
143 HIPQUAD(peer_ip), peer_port);
144 break;
145 default:
146 LCONSOLE_ERROR_MSG(0x11e, "Unexpected error %d connecting to %s"
147 " at host %u.%u.%u.%u on port %d\n", rc,
148 libcfs_nid2str(peer_nid),
149 HIPQUAD(peer_ip), peer_port);
150 break;
151 }
152}
153EXPORT_SYMBOL(lnet_connect_console_error);
154
155int
156lnet_connect(socket_t **sockp, lnet_nid_t peer_nid,
157 __u32 local_ip, __u32 peer_ip, int peer_port)
158{
159 lnet_acceptor_connreq_t cr;
160 socket_t *sock;
161 int rc;
162 int port;
163 int fatal;
164
165 CLASSERT (sizeof(cr) <= 16);
166
167 for (port = LNET_ACCEPTOR_MAX_RESERVED_PORT;
168 port >= LNET_ACCEPTOR_MIN_RESERVED_PORT;
169 --port) {
170
171
172 rc = libcfs_sock_connect(&sock, &fatal,
173 local_ip, port,
174 peer_ip, peer_port);
175 if (rc != 0) {
176 if (fatal)
177 goto failed;
178 continue;
179 }
180
181 CLASSERT (LNET_PROTO_ACCEPTOR_VERSION == 1);
182
183 cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
184 cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
185 cr.acr_nid = peer_nid;
186
187 if (the_lnet.ln_testprotocompat != 0) {
188
189 lnet_net_lock(LNET_LOCK_EX);
190 if ((the_lnet.ln_testprotocompat & 4) != 0) {
191 cr.acr_version++;
192 the_lnet.ln_testprotocompat &= ~4;
193 }
194 if ((the_lnet.ln_testprotocompat & 8) != 0) {
195 cr.acr_magic = LNET_PROTO_MAGIC;
196 the_lnet.ln_testprotocompat &= ~8;
197 }
198 lnet_net_unlock(LNET_LOCK_EX);
199 }
200
201 rc = libcfs_sock_write(sock, &cr, sizeof(cr),
202 accept_timeout);
203 if (rc != 0)
204 goto failed_sock;
205
206 *sockp = sock;
207 return 0;
208 }
209
210 rc = -EADDRINUSE;
211 goto failed;
212
213 failed_sock:
214 libcfs_sock_release(sock);
215 failed:
216 lnet_connect_console_error(rc, peer_nid, peer_ip, peer_port);
217 return rc;
218}
219EXPORT_SYMBOL(lnet_connect);
220
221
222
223
224int
225lnet_accept(socket_t *sock, __u32 magic)
226{
227 lnet_acceptor_connreq_t cr;
228 __u32 peer_ip;
229 int peer_port;
230 int rc;
231 int flip;
232 lnet_ni_t *ni;
233 char *str;
234
235 LASSERT (sizeof(cr) <= 16);
236
237 rc = libcfs_sock_getaddr(sock, 1, &peer_ip, &peer_port);
238 LASSERT (rc == 0);
239
240 if (!lnet_accept_magic(magic, LNET_PROTO_ACCEPTOR_MAGIC)) {
241
242 if (lnet_accept_magic(magic, LNET_PROTO_MAGIC)) {
243
244
245
246
247
248 memset (&cr, 0, sizeof(cr));
249 cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
250 cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
251 rc = libcfs_sock_write(sock, &cr, sizeof(cr),
252 accept_timeout);
253
254 if (rc != 0)
255 CERROR("Error sending magic+version in response"
256 "to LNET magic from %u.%u.%u.%u: %d\n",
257 HIPQUAD(peer_ip), rc);
258 return -EPROTO;
259 }
260
261 if (magic == le32_to_cpu(LNET_PROTO_TCP_MAGIC))
262 str = "'old' socknal/tcpnal";
263 else if (lnet_accept_magic(magic, LNET_PROTO_RA_MAGIC))
264 str = "'old' ranal";
265 else
266 str = "unrecognised";
267
268 LCONSOLE_ERROR_MSG(0x11f, "Refusing connection from %u.%u.%u.%u"
269 " magic %08x: %s acceptor protocol\n",
270 HIPQUAD(peer_ip), magic, str);
271 return -EPROTO;
272 }
273
274 flip = (magic != LNET_PROTO_ACCEPTOR_MAGIC);
275
276 rc = libcfs_sock_read(sock, &cr.acr_version,
277 sizeof(cr.acr_version),
278 accept_timeout);
279 if (rc != 0) {
280 CERROR("Error %d reading connection request version from "
281 "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
282 return -EIO;
283 }
284
285 if (flip)
286 __swab32s(&cr.acr_version);
287
288 if (cr.acr_version != LNET_PROTO_ACCEPTOR_VERSION) {
289
290
291
292
293 int peer_version = cr.acr_version;
294
295 memset (&cr, 0, sizeof(cr));
296 cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
297 cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
298
299 rc = libcfs_sock_write(sock, &cr, sizeof(cr),
300 accept_timeout);
301
302 if (rc != 0)
303 CERROR("Error sending magic+version in response"
304 "to version %d from %u.%u.%u.%u: %d\n",
305 peer_version, HIPQUAD(peer_ip), rc);
306 return -EPROTO;
307 }
308
309 rc = libcfs_sock_read(sock, &cr.acr_nid,
310 sizeof(cr) -
311 offsetof(lnet_acceptor_connreq_t, acr_nid),
312 accept_timeout);
313 if (rc != 0) {
314 CERROR("Error %d reading connection request from "
315 "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
316 return -EIO;
317 }
318
319 if (flip)
320 __swab64s(&cr.acr_nid);
321
322 ni = lnet_net2ni(LNET_NIDNET(cr.acr_nid));
323 if (ni == NULL ||
324 ni->ni_nid != cr.acr_nid) {
325 if (ni != NULL)
326 lnet_ni_decref(ni);
327 LCONSOLE_ERROR_MSG(0x120, "Refusing connection from %u.%u.%u.%u"
328 " for %s: No matching NI\n",
329 HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid));
330 return -EPERM;
331 }
332
333 if (ni->ni_lnd->lnd_accept == NULL) {
334
335 lnet_ni_decref(ni);
336 LCONSOLE_ERROR_MSG(0x121, "Refusing connection from %u.%u.%u.%u"
337 " for %s: NI doesn not accept IP connections\n",
338 HIPQUAD(peer_ip), libcfs_nid2str(cr.acr_nid));
339 return -EPERM;
340 }
341
342 CDEBUG(D_NET, "Accept %s from %u.%u.%u.%u\n",
343 libcfs_nid2str(cr.acr_nid), HIPQUAD(peer_ip));
344
345 rc = ni->ni_lnd->lnd_accept(ni, sock);
346
347 lnet_ni_decref(ni);
348 return rc;
349}
350
351int
352lnet_acceptor(void *arg)
353{
354 socket_t *newsock;
355 int rc;
356 __u32 magic;
357 __u32 peer_ip;
358 int peer_port;
359 int secure = (int)((long_ptr_t)arg);
360
361 LASSERT (lnet_acceptor_state.pta_sock == NULL);
362
363 cfs_block_allsigs();
364
365 rc = libcfs_sock_listen(&lnet_acceptor_state.pta_sock,
366 0, accept_port, accept_backlog);
367 if (rc != 0) {
368 if (rc == -EADDRINUSE)
369 LCONSOLE_ERROR_MSG(0x122, "Can't start acceptor on port"
370 " %d: port already in use\n",
371 accept_port);
372 else
373 LCONSOLE_ERROR_MSG(0x123, "Can't start acceptor on port "
374 "%d: unexpected error %d\n",
375 accept_port, rc);
376
377 lnet_acceptor_state.pta_sock = NULL;
378 } else {
379 LCONSOLE(0, "Accept %s, port %d\n", accept_type, accept_port);
380 }
381
382
383 lnet_acceptor_state.pta_shutdown = rc;
384 complete(&lnet_acceptor_state.pta_signal);
385
386 if (rc != 0)
387 return rc;
388
389 while (!lnet_acceptor_state.pta_shutdown) {
390
391 rc = libcfs_sock_accept(&newsock, lnet_acceptor_state.pta_sock);
392 if (rc != 0) {
393 if (rc != -EAGAIN) {
394 CWARN("Accept error %d: pausing...\n", rc);
395 cfs_pause(cfs_time_seconds(1));
396 }
397 continue;
398 }
399
400
401 if (lnet_acceptor_state.pta_shutdown) {
402 libcfs_sock_release(newsock);
403 break;
404 }
405
406 rc = libcfs_sock_getaddr(newsock, 1, &peer_ip, &peer_port);
407 if (rc != 0) {
408 CERROR("Can't determine new connection's address\n");
409 goto failed;
410 }
411
412 if (secure && peer_port > LNET_ACCEPTOR_MAX_RESERVED_PORT) {
413 CERROR("Refusing connection from %u.%u.%u.%u: "
414 "insecure port %d\n",
415 HIPQUAD(peer_ip), peer_port);
416 goto failed;
417 }
418
419 rc = libcfs_sock_read(newsock, &magic, sizeof(magic),
420 accept_timeout);
421 if (rc != 0) {
422 CERROR("Error %d reading connection request from "
423 "%u.%u.%u.%u\n", rc, HIPQUAD(peer_ip));
424 goto failed;
425 }
426
427 rc = lnet_accept(newsock, magic);
428 if (rc != 0)
429 goto failed;
430
431 continue;
432
433 failed:
434 libcfs_sock_release(newsock);
435 }
436
437 libcfs_sock_release(lnet_acceptor_state.pta_sock);
438 lnet_acceptor_state.pta_sock = NULL;
439
440 CDEBUG(D_NET, "Acceptor stopping\n");
441
442
443 complete(&lnet_acceptor_state.pta_signal);
444 return 0;
445}
446
447static inline int
448accept2secure(const char *acc, long *sec)
449{
450 if (!strcmp(acc, "secure")) {
451 *sec = 1;
452 return 1;
453 } else if (!strcmp(acc, "all")) {
454 *sec = 0;
455 return 1;
456 } else if (!strcmp(acc, "none")) {
457 return 0;
458 } else {
459 LCONSOLE_ERROR_MSG(0x124, "Can't parse 'accept=\"%s\"'\n",
460 acc);
461 return -EINVAL;
462 }
463}
464
465int
466lnet_acceptor_start(void)
467{
468 int rc;
469 long rc2;
470 long secure;
471
472 LASSERT (lnet_acceptor_state.pta_sock == NULL);
473
474 rc = lnet_acceptor_get_tunables();
475 if (rc != 0)
476 return rc;
477
478
479 init_completion(&lnet_acceptor_state.pta_signal);
480 rc = accept2secure(accept_type, &secure);
481 if (rc <= 0) {
482 fini_completion(&lnet_acceptor_state.pta_signal);
483 return rc;
484 }
485
486 if (lnet_count_acceptor_nis() == 0)
487 return 0;
488
489 rc2 = PTR_ERR(kthread_run(lnet_acceptor,
490 (void *)(ulong_ptr_t)secure,
491 "acceptor_%03ld", secure));
492 if (IS_ERR_VALUE(rc2)) {
493 CERROR("Can't start acceptor thread: %ld\n", rc2);
494 fini_completion(&lnet_acceptor_state.pta_signal);
495
496 return -ESRCH;
497 }
498
499
500 wait_for_completion(&lnet_acceptor_state.pta_signal);
501
502 if (!lnet_acceptor_state.pta_shutdown) {
503
504 LASSERT(lnet_acceptor_state.pta_sock != NULL);
505 return 0;
506 }
507
508 LASSERT(lnet_acceptor_state.pta_sock == NULL);
509 fini_completion(&lnet_acceptor_state.pta_signal);
510
511 return -ENETDOWN;
512}
513
514void
515lnet_acceptor_stop(void)
516{
517 if (lnet_acceptor_state.pta_sock == NULL)
518 return;
519
520 lnet_acceptor_state.pta_shutdown = 1;
521 libcfs_sock_abort_accept(lnet_acceptor_state.pta_sock);
522
523
524 wait_for_completion(&lnet_acceptor_state.pta_signal);
525
526 fini_completion(&lnet_acceptor_state.pta_signal);
527}
528
