1
2
3
4
5#include <stdlib.h>
6#include <fcntl.h>
7#include <unistd.h>
8#include <sys/types.h>
9#include <sys/socket.h>
10#include <sys/ioctl.h>
11#include <errno.h>
12
13#include <rte_version.h>
14#include <rte_mbuf.h>
15#include <rte_ether.h>
16#include <ethdev_driver.h>
17#include <ethdev_vdev.h>
18#include <rte_malloc.h>
19#include <rte_kvargs.h>
20#include <rte_bus_vdev.h>
21#include <rte_hash.h>
22#include <rte_jhash.h>
23#include <rte_string_fns.h>
24
25#include "rte_eth_memif.h"
26#include "memif_socket.h"
27
28static void memif_intr_handler(void *arg);
29
30static ssize_t
31memif_msg_send(int fd, memif_msg_t *msg, int afd)
32{
33 struct msghdr mh = { 0 };
34 struct iovec iov[1];
35 struct cmsghdr *cmsg;
36 char ctl[CMSG_SPACE(sizeof(int))];
37
38 iov[0].iov_base = msg;
39 iov[0].iov_len = sizeof(memif_msg_t);
40 mh.msg_iov = iov;
41 mh.msg_iovlen = 1;
42
43 if (afd > 0) {
44 memset(&ctl, 0, sizeof(ctl));
45 mh.msg_control = ctl;
46 mh.msg_controllen = sizeof(ctl);
47 cmsg = CMSG_FIRSTHDR(&mh);
48 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
49 cmsg->cmsg_level = SOL_SOCKET;
50 cmsg->cmsg_type = SCM_RIGHTS;
51 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
52 }
53
54 return sendmsg(fd, &mh, 0);
55}
56
57static int
58memif_msg_send_from_queue(struct memif_control_channel *cc)
59{
60 ssize_t size;
61 int ret = 0;
62 struct memif_msg_queue_elt *e;
63
64 e = TAILQ_FIRST(&cc->msg_queue);
65 if (e == NULL)
66 return 0;
67
68 size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd);
69 if (size != sizeof(memif_msg_t)) {
70 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
71 ret = -1;
72 } else {
73 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
74 }
75 TAILQ_REMOVE(&cc->msg_queue, e, next);
76 rte_free(e);
77
78 return ret;
79}
80
81static struct memif_msg_queue_elt *
82memif_msg_enq(struct memif_control_channel *cc)
83{
84 struct memif_msg_queue_elt *e;
85
86 e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
87 if (e == NULL) {
88 MIF_LOG(ERR, "Failed to allocate control message.");
89 return NULL;
90 }
91
92 e->fd = -1;
93 TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
94
95 return e;
96}
97
98void
99memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
100 int err_code)
101{
102 struct memif_msg_queue_elt *e;
103 struct pmd_internals *pmd;
104 memif_msg_disconnect_t *d;
105
106 if (cc == NULL) {
107 MIF_LOG(DEBUG, "Missing control channel.");
108 return;
109 }
110
111 e = memif_msg_enq(cc);
112 if (e == NULL) {
113 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
114 return;
115 }
116
117 d = &e->msg.disconnect;
118
119 e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
120 d->code = err_code;
121
122 if (reason != NULL) {
123 strlcpy((char *)d->string, reason, sizeof(d->string));
124 if (cc->dev != NULL) {
125 pmd = cc->dev->data->dev_private;
126 strlcpy(pmd->local_disc_string, reason,
127 sizeof(pmd->local_disc_string));
128 }
129 }
130}
131
132static int
133memif_msg_enq_hello(struct memif_control_channel *cc)
134{
135 struct memif_msg_queue_elt *e = memif_msg_enq(cc);
136 memif_msg_hello_t *h;
137
138 if (e == NULL)
139 return -1;
140
141 h = &e->msg.hello;
142
143 e->msg.type = MEMIF_MSG_TYPE_HELLO;
144 h->min_version = MEMIF_VERSION;
145 h->max_version = MEMIF_VERSION;
146 h->max_c2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
147 h->max_s2c_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
148 h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
149 h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
150
151 strlcpy((char *)h->name, rte_version(), sizeof(h->name));
152
153 return 0;
154}
155
156static int
157memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
158{
159 struct pmd_internals *pmd = dev->data->dev_private;
160 memif_msg_hello_t *h = &msg->hello;
161
162 if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
163 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
164 return -1;
165 }
166
167
168 pmd->run.num_c2s_rings = RTE_MIN(h->max_c2s_ring + 1,
169 pmd->cfg.num_c2s_rings);
170 pmd->run.num_s2c_rings = RTE_MIN(h->max_s2c_ring + 1,
171 pmd->cfg.num_s2c_rings);
172 pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
173 pmd->cfg.log2_ring_size);
174 pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
175
176 strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
177
178 MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name);
179
180 return 0;
181}
182
183static int
184memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
185{
186 memif_msg_init_t *i = &msg->init;
187 struct memif_socket_dev_list_elt *elt;
188 struct pmd_internals *pmd;
189 struct rte_eth_dev *dev;
190
191 if (i->version != MEMIF_VERSION) {
192 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
193 return -1;
194 }
195
196 if (cc->socket == NULL) {
197 memif_msg_enq_disconnect(cc, "Device error", 0);
198 return -1;
199 }
200
201
202 TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
203 dev = elt->dev;
204 pmd = dev->data->dev_private;
205 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
206 (pmd->id == i->id) && (pmd->role == MEMIF_ROLE_SERVER)) {
207 if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
208 ETH_MEMIF_FLAG_CONNECTED)) {
209 memif_msg_enq_disconnect(cc,
210 "Already connected", 0);
211 return -1;
212 }
213
214
215 cc->dev = dev;
216 pmd->cc = cc;
217
218 if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
219 memif_msg_enq_disconnect(pmd->cc,
220 "Only ethernet mode supported",
221 0);
222 return -1;
223 }
224
225 strlcpy(pmd->remote_name, (char *)i->name,
226 sizeof(pmd->remote_name));
227
228 if (*pmd->secret != '\0') {
229 if (*i->secret == '\0') {
230 memif_msg_enq_disconnect(pmd->cc,
231 "Secret required", 0);
232 return -1;
233 }
234 if (strncmp(pmd->secret, (char *)i->secret,
235 ETH_MEMIF_SECRET_SIZE) != 0) {
236 memif_msg_enq_disconnect(pmd->cc,
237 "Incorrect secret", 0);
238 return -1;
239 }
240 }
241
242 pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
243 return 0;
244 }
245 }
246
247
248 MIF_LOG(DEBUG, "ID %u not found.", i->id);
249 memif_msg_enq_disconnect(cc, "ID not found", 0);
250 return -1;
251}
252
253static int
254memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
255 int fd)
256{
257 struct pmd_internals *pmd = dev->data->dev_private;
258 struct pmd_process_private *proc_private = dev->process_private;
259 memif_msg_add_region_t *ar = &msg->add_region;
260 struct memif_region *r;
261
262 if (fd < 0) {
263 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
264 return -1;
265 }
266
267 if (ar->index >= ETH_MEMIF_MAX_REGION_NUM ||
268 ar->index != proc_private->regions_num ||
269 proc_private->regions[ar->index] != NULL) {
270 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
271 return -1;
272 }
273
274 r = rte_zmalloc("region", sizeof(struct memif_region), 0);
275 if (r == NULL) {
276 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0);
277 return -ENOMEM;
278 }
279
280 r->fd = fd;
281 r->region_size = ar->size;
282 r->addr = NULL;
283
284 proc_private->regions[ar->index] = r;
285 proc_private->regions_num++;
286
287 return 0;
288}
289
290static int
291memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
292{
293 struct pmd_internals *pmd = dev->data->dev_private;
294 memif_msg_add_ring_t *ar = &msg->add_ring;
295 struct memif_queue *mq;
296
297 if (fd < 0) {
298 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
299 return -1;
300 }
301
302
303 if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) {
304 if (ar->index >= pmd->cfg.num_c2s_rings) {
305 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
306 return -1;
307 }
308 pmd->run.num_c2s_rings++;
309 } else {
310 if (ar->index >= pmd->cfg.num_s2c_rings) {
311 memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
312 return -1;
313 }
314 pmd->run.num_s2c_rings++;
315 }
316
317 mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) ?
318 dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
319
320 mq->intr_handle.fd = fd;
321 mq->log2_ring_size = ar->log2_ring_size;
322 mq->region = ar->region;
323 mq->ring_offset = ar->offset;
324
325 return 0;
326}
327
328static int
329memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
330{
331 struct pmd_internals *pmd = dev->data->dev_private;
332 memif_msg_connect_t *c = &msg->connect;
333 int ret;
334
335 ret = memif_connect(dev);
336 if (ret < 0)
337 return ret;
338
339 strlcpy(pmd->remote_if_name, (char *)c->if_name,
340 sizeof(pmd->remote_if_name));
341 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
342
343 return 0;
344}
345
346static int
347memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
348{
349 struct pmd_internals *pmd = dev->data->dev_private;
350 memif_msg_connected_t *c = &msg->connected;
351 int ret;
352
353 ret = memif_connect(dev);
354 if (ret < 0)
355 return ret;
356
357 strlcpy(pmd->remote_if_name, (char *)c->if_name,
358 sizeof(pmd->remote_if_name));
359 MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
360
361 return 0;
362}
363
364static int
365memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
366{
367 struct pmd_internals *pmd = dev->data->dev_private;
368 memif_msg_disconnect_t *d = &msg->disconnect;
369
370 memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string));
371 strlcpy(pmd->remote_disc_string, (char *)d->string,
372 sizeof(pmd->remote_disc_string));
373
374 MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string);
375
376 memset(pmd->local_disc_string, 0, 96);
377 memif_disconnect(dev);
378 return 0;
379}
380
381static int
382memif_msg_enq_ack(struct rte_eth_dev *dev)
383{
384 struct pmd_internals *pmd = dev->data->dev_private;
385 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
386 if (e == NULL)
387 return -1;
388
389 e->msg.type = MEMIF_MSG_TYPE_ACK;
390
391 return 0;
392}
393
394static int
395memif_msg_enq_init(struct rte_eth_dev *dev)
396{
397 struct pmd_internals *pmd = dev->data->dev_private;
398 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
399 memif_msg_init_t *i = &e->msg.init;
400
401 if (e == NULL)
402 return -1;
403
404 i = &e->msg.init;
405 e->msg.type = MEMIF_MSG_TYPE_INIT;
406 i->version = MEMIF_VERSION;
407 i->id = pmd->id;
408 i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
409
410 strlcpy((char *)i->name, rte_version(), sizeof(i->name));
411
412 if (*pmd->secret != '\0')
413 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
414
415 return 0;
416}
417
418static int
419memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
420{
421 struct pmd_internals *pmd = dev->data->dev_private;
422 struct pmd_process_private *proc_private = dev->process_private;
423 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
424 memif_msg_add_region_t *ar;
425 struct memif_region *mr = proc_private->regions[idx];
426
427 if (e == NULL)
428 return -1;
429
430 ar = &e->msg.add_region;
431 e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
432 e->fd = mr->fd;
433 ar->index = idx;
434 ar->size = mr->region_size;
435
436 return 0;
437}
438
439static int
440memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
441 memif_ring_type_t type)
442{
443 struct pmd_internals *pmd = dev->data->dev_private;
444 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
445 struct memif_queue *mq;
446 memif_msg_add_ring_t *ar;
447
448 if (e == NULL)
449 return -1;
450
451 ar = &e->msg.add_ring;
452 mq = (type == MEMIF_RING_C2S) ? dev->data->tx_queues[idx] :
453 dev->data->rx_queues[idx];
454
455 e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
456 e->fd = mq->intr_handle.fd;
457 ar->index = idx;
458 ar->offset = mq->ring_offset;
459 ar->region = mq->region;
460 ar->log2_ring_size = mq->log2_ring_size;
461 ar->flags = (type == MEMIF_RING_C2S) ? MEMIF_MSG_ADD_RING_FLAG_C2S : 0;
462 ar->private_hdr_size = 0;
463
464 return 0;
465}
466
467static int
468memif_msg_enq_connect(struct rte_eth_dev *dev)
469{
470 struct pmd_internals *pmd = dev->data->dev_private;
471 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
472 memif_msg_connect_t *c;
473
474 if (e == NULL)
475 return -1;
476
477 c = &e->msg.connect;
478 e->msg.type = MEMIF_MSG_TYPE_CONNECT;
479 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
480
481 return 0;
482}
483
484static int
485memif_msg_enq_connected(struct rte_eth_dev *dev)
486{
487 struct pmd_internals *pmd = dev->data->dev_private;
488 struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
489 memif_msg_connected_t *c;
490
491 if (e == NULL)
492 return -1;
493
494 c = &e->msg.connected;
495 e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
496 strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
497
498 return 0;
499}
500
501static void
502memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
503{
504 struct memif_msg_queue_elt *elt;
505 struct memif_control_channel *cc = arg;
506
507
508 close(intr_handle->fd);
509
510 while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
511 TAILQ_REMOVE(&cc->msg_queue, elt, next);
512 rte_free(elt);
513 }
514
515 rte_free(cc);
516}
517
518void
519memif_disconnect(struct rte_eth_dev *dev)
520{
521 struct pmd_internals *pmd = dev->data->dev_private;
522 struct memif_msg_queue_elt *elt, *next;
523 struct memif_queue *mq;
524 struct rte_intr_handle *ih;
525 int i;
526 int ret;
527
528 dev->data->dev_link.link_status = ETH_LINK_DOWN;
529 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
530 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
531
532 rte_spinlock_lock(&pmd->cc_lock);
533 if (pmd->cc != NULL) {
534
535 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
536 next = TAILQ_NEXT(elt, next);
537 if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
538 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
539 rte_free(elt);
540 }
541 }
542
543 memif_msg_send_from_queue(pmd->cc);
544
545
546 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
547 MIF_LOG(WARNING,
548 "Unexpected message(s) in message queue.");
549 }
550
551 ih = &pmd->cc->intr_handle;
552 if (ih->fd > 0) {
553 ret = rte_intr_callback_unregister(ih,
554 memif_intr_handler,
555 pmd->cc);
556
557
558
559
560 if (ret == -EAGAIN) {
561 ret = rte_intr_callback_unregister_pending(ih,
562 memif_intr_handler,
563 pmd->cc,
564 memif_intr_unregister_handler);
565 } else if (ret > 0) {
566 close(ih->fd);
567 rte_free(pmd->cc);
568 }
569 pmd->cc = NULL;
570 if (ret <= 0)
571 MIF_LOG(WARNING,
572 "Failed to unregister control channel callback.");
573 }
574 }
575 rte_spinlock_unlock(&pmd->cc_lock);
576
577
578 for (i = 0; i < pmd->cfg.num_c2s_rings; i++) {
579 if (pmd->role == MEMIF_ROLE_CLIENT) {
580 if (dev->data->tx_queues != NULL)
581 mq = dev->data->tx_queues[i];
582 else
583 continue;
584 } else {
585 if (dev->data->rx_queues != NULL)
586 mq = dev->data->rx_queues[i];
587 else
588 continue;
589 }
590 if (mq->intr_handle.fd > 0) {
591 close(mq->intr_handle.fd);
592 mq->intr_handle.fd = -1;
593 }
594 }
595 for (i = 0; i < pmd->cfg.num_s2c_rings; i++) {
596 if (pmd->role == MEMIF_ROLE_SERVER) {
597 if (dev->data->tx_queues != NULL)
598 mq = dev->data->tx_queues[i];
599 else
600 continue;
601 } else {
602 if (dev->data->rx_queues != NULL)
603 mq = dev->data->rx_queues[i];
604 else
605 continue;
606 }
607 if (mq->intr_handle.fd > 0) {
608 close(mq->intr_handle.fd);
609 mq->intr_handle.fd = -1;
610 }
611 }
612
613 memif_free_regions(dev);
614
615
616 memset(&pmd->run, 0, sizeof(pmd->run));
617
618 MIF_LOG(DEBUG, "Disconnected, id: %d, role: %s.", pmd->id,
619 (pmd->role == MEMIF_ROLE_SERVER) ? "server" : "client");
620}
621
622static int
623memif_msg_receive(struct memif_control_channel *cc)
624{
625 char ctl[CMSG_SPACE(sizeof(int)) +
626 CMSG_SPACE(sizeof(struct ucred))] = { 0 };
627 struct msghdr mh = { 0 };
628 struct iovec iov[1];
629 memif_msg_t msg = { 0 };
630 ssize_t size;
631 int ret = 0;
632 struct ucred *cr __rte_unused;
633 cr = 0;
634 struct cmsghdr *cmsg;
635 int afd = -1;
636 int i;
637 struct pmd_internals *pmd;
638 struct pmd_process_private *proc_private;
639
640 iov[0].iov_base = (void *)&msg;
641 iov[0].iov_len = sizeof(memif_msg_t);
642 mh.msg_iov = iov;
643 mh.msg_iovlen = 1;
644 mh.msg_control = ctl;
645 mh.msg_controllen = sizeof(ctl);
646
647 size = recvmsg(cc->intr_handle.fd, &mh, 0);
648 if (size != sizeof(memif_msg_t)) {
649 MIF_LOG(DEBUG, "Invalid message size = %zd", size);
650 if (size > 0)
651
652
653
654 memif_msg_enq_disconnect(cc, "Invalid message size", 0);
655 return -1;
656 }
657 MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
658
659 cmsg = CMSG_FIRSTHDR(&mh);
660 while (cmsg) {
661 if (cmsg->cmsg_level == SOL_SOCKET) {
662 if (cmsg->cmsg_type == SCM_CREDENTIALS)
663 cr = (struct ucred *)CMSG_DATA(cmsg);
664 else if (cmsg->cmsg_type == SCM_RIGHTS)
665 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
666 }
667 cmsg = CMSG_NXTHDR(&mh, cmsg);
668 }
669
670 if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
671 MIF_LOG(DEBUG, "Unexpected message.");
672 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
673 return -1;
674 }
675
676
677 switch (msg.type) {
678 case MEMIF_MSG_TYPE_ACK:
679 break;
680 case MEMIF_MSG_TYPE_HELLO:
681 ret = memif_msg_receive_hello(cc->dev, &msg);
682 if (ret < 0)
683 goto exit;
684 ret = memif_init_regions_and_queues(cc->dev);
685 if (ret < 0)
686 goto exit;
687 ret = memif_msg_enq_init(cc->dev);
688 if (ret < 0)
689 goto exit;
690 pmd = cc->dev->data->dev_private;
691 proc_private = cc->dev->process_private;
692 for (i = 0; i < proc_private->regions_num; i++) {
693 ret = memif_msg_enq_add_region(cc->dev, i);
694 if (ret < 0)
695 goto exit;
696 }
697 for (i = 0; i < pmd->run.num_c2s_rings; i++) {
698 ret = memif_msg_enq_add_ring(cc->dev, i,
699 MEMIF_RING_C2S);
700 if (ret < 0)
701 goto exit;
702 }
703 for (i = 0; i < pmd->run.num_s2c_rings; i++) {
704 ret = memif_msg_enq_add_ring(cc->dev, i,
705 MEMIF_RING_S2C);
706 if (ret < 0)
707 goto exit;
708 }
709 ret = memif_msg_enq_connect(cc->dev);
710 if (ret < 0)
711 goto exit;
712 break;
713 case MEMIF_MSG_TYPE_INIT:
714
715
716
717
718 ret = memif_msg_receive_init(cc, &msg);
719 if (ret < 0)
720 goto exit;
721 ret = memif_msg_enq_ack(cc->dev);
722 if (ret < 0)
723 goto exit;
724 break;
725 case MEMIF_MSG_TYPE_ADD_REGION:
726 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
727 if (ret < 0)
728 goto exit;
729 ret = memif_msg_enq_ack(cc->dev);
730 if (ret < 0)
731 goto exit;
732 break;
733 case MEMIF_MSG_TYPE_ADD_RING:
734 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
735 if (ret < 0)
736 goto exit;
737 ret = memif_msg_enq_ack(cc->dev);
738 if (ret < 0)
739 goto exit;
740 break;
741 case MEMIF_MSG_TYPE_CONNECT:
742 ret = memif_msg_receive_connect(cc->dev, &msg);
743 if (ret < 0)
744 goto exit;
745 ret = memif_msg_enq_connected(cc->dev);
746 if (ret < 0)
747 goto exit;
748 break;
749 case MEMIF_MSG_TYPE_CONNECTED:
750 ret = memif_msg_receive_connected(cc->dev, &msg);
751 break;
752 case MEMIF_MSG_TYPE_DISCONNECT:
753 ret = memif_msg_receive_disconnect(cc->dev, &msg);
754 if (ret < 0)
755 goto exit;
756 break;
757 default:
758 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
759 ret = -1;
760 goto exit;
761 }
762
763 exit:
764 return ret;
765}
766
767static void
768memif_intr_handler(void *arg)
769{
770 struct memif_control_channel *cc = arg;
771 int ret;
772
773 ret = memif_msg_receive(cc);
774
775 if (cc->dev == NULL) {
776 memif_msg_send_from_queue(cc);
777 ret = rte_intr_callback_unregister_pending(&cc->intr_handle,
778 memif_intr_handler,
779 cc,
780 memif_intr_unregister_handler);
781 if (ret < 0)
782 MIF_LOG(WARNING,
783 "Failed to unregister control channel callback.");
784 return;
785 }
786
787 if (ret < 0)
788 goto disconnect;
789
790 ret = memif_msg_send_from_queue(cc);
791 if (ret < 0)
792 goto disconnect;
793
794 return;
795
796 disconnect:
797 if (cc->dev == NULL) {
798 MIF_LOG(WARNING, "eth dev not allocated");
799 return;
800 }
801 memif_disconnect(cc->dev);
802}
803
804static void
805memif_listener_handler(void *arg)
806{
807 struct memif_socket *socket = arg;
808 int sockfd;
809 int addr_len;
810 struct sockaddr_un client;
811 struct memif_control_channel *cc;
812 int ret;
813
814 addr_len = sizeof(client);
815 sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client,
816 (socklen_t *)&addr_len);
817 if (sockfd < 0) {
818 MIF_LOG(ERR,
819 "Failed to accept connection request on socket fd %d",
820 socket->intr_handle.fd);
821 return;
822 }
823
824 MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
825
826 cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
827 if (cc == NULL) {
828 MIF_LOG(ERR, "Failed to allocate control channel.");
829 goto error;
830 }
831
832 cc->intr_handle.fd = sockfd;
833 cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
834 cc->socket = socket;
835 cc->dev = NULL;
836 TAILQ_INIT(&cc->msg_queue);
837
838 ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc);
839 if (ret < 0) {
840 MIF_LOG(ERR, "Failed to register control channel callback.");
841 goto error;
842 }
843
844 ret = memif_msg_enq_hello(cc);
845 if (ret < 0) {
846 MIF_LOG(ERR, "Failed to enqueue hello message.");
847 goto error;
848 }
849 ret = memif_msg_send_from_queue(cc);
850 if (ret < 0)
851 goto error;
852
853 return;
854
855 error:
856 if (sockfd >= 0) {
857 close(sockfd);
858 sockfd = -1;
859 }
860 if (cc != NULL)
861 rte_free(cc);
862}
863
864static struct memif_socket *
865memif_socket_create(char *key, uint8_t listener, bool is_abstract)
866{
867 struct memif_socket *sock;
868 struct sockaddr_un un = { 0 };
869 uint32_t sunlen;
870 int sockfd;
871 int ret;
872 int on = 1;
873
874 sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
875 if (sock == NULL) {
876 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
877 return NULL;
878 }
879
880 sock->listener = listener;
881 strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE);
882 TAILQ_INIT(&sock->dev_queue);
883
884 if (listener != 0) {
885 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
886 if (sockfd < 0)
887 goto error;
888
889 un.sun_family = AF_UNIX;
890 if (is_abstract) {
891
892 un.sun_path[0] = '\0';
893 strlcpy(un.sun_path + 1, sock->filename, MEMIF_SOCKET_UN_SIZE - 1);
894 sunlen = RTE_MIN(1 + strlen(sock->filename),
895 MEMIF_SOCKET_UN_SIZE) +
896 sizeof(un) - sizeof(un.sun_path);
897 } else {
898 sunlen = sizeof(un);
899 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE);
900 }
901
902 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
903 sizeof(on));
904 if (ret < 0)
905 goto error;
906
907 ret = bind(sockfd, (struct sockaddr *)&un, sunlen);
908 if (ret < 0)
909 goto error;
910
911 ret = listen(sockfd, 1);
912 if (ret < 0)
913 goto error;
914
915 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename);
916
917 sock->intr_handle.fd = sockfd;
918 sock->intr_handle.type = RTE_INTR_HANDLE_EXT;
919 ret = rte_intr_callback_register(&sock->intr_handle,
920 memif_listener_handler, sock);
921 if (ret < 0) {
922 MIF_LOG(ERR, "Failed to register interrupt "
923 "callback for listener socket");
924 return NULL;
925 }
926 }
927
928 return sock;
929
930 error:
931 MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno));
932 if (sock != NULL)
933 rte_free(sock);
934 if (sockfd >= 0)
935 close(sockfd);
936 return NULL;
937}
938
939static struct rte_hash *
940memif_create_socket_hash(void)
941{
942 struct rte_hash_parameters params = { 0 };
943
944 params.name = MEMIF_SOCKET_HASH_NAME;
945 params.entries = 256;
946 params.key_len = MEMIF_SOCKET_UN_SIZE;
947 params.hash_func = rte_jhash;
948 params.hash_func_init_val = 0;
949 return rte_hash_create(¶ms);
950}
951
952int
953memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
954{
955 struct pmd_internals *pmd = dev->data->dev_private;
956 struct memif_socket *socket = NULL;
957 struct memif_socket_dev_list_elt *elt;
958 struct pmd_internals *tmp_pmd;
959 struct rte_hash *hash;
960 int ret;
961 char key[MEMIF_SOCKET_UN_SIZE];
962
963 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
964 if (hash == NULL) {
965 hash = memif_create_socket_hash();
966 if (hash == NULL) {
967 MIF_LOG(ERR, "Failed to create memif socket hash.");
968 return -1;
969 }
970 }
971
972 memset(key, 0, MEMIF_SOCKET_UN_SIZE);
973 strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE);
974 ret = rte_hash_lookup_data(hash, key, (void **)&socket);
975 if (ret < 0) {
976 socket = memif_socket_create(key,
977 (pmd->role == MEMIF_ROLE_CLIENT) ? 0 : 1,
978 pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT);
979 if (socket == NULL)
980 return -1;
981 ret = rte_hash_add_key_data(hash, key, socket);
982 if (ret < 0) {
983 MIF_LOG(ERR, "Failed to add socket to socket hash.");
984 return ret;
985 }
986 }
987 pmd->socket_filename = socket->filename;
988
989 TAILQ_FOREACH(elt, &socket->dev_queue, next) {
990 tmp_pmd = elt->dev->data->dev_private;
991 if (tmp_pmd->id == pmd->id && tmp_pmd->role == pmd->role) {
992 MIF_LOG(ERR, "Two interfaces with the same id (%d) can "
993 "not have the same role.", pmd->id);
994 return -1;
995 }
996 }
997
998 elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
999 if (elt == NULL) {
1000 MIF_LOG(ERR, "Failed to add device to socket device list.");
1001 return -1;
1002 }
1003 elt->dev = dev;
1004 TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
1005
1006 return 0;
1007}
1008
1009void
1010memif_socket_remove_device(struct rte_eth_dev *dev)
1011{
1012 struct pmd_internals *pmd = dev->data->dev_private;
1013 struct memif_socket *socket = NULL;
1014 struct memif_socket_dev_list_elt *elt, *next;
1015 struct rte_hash *hash;
1016 int ret;
1017
1018 hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1019 if (hash == NULL)
1020 return;
1021
1022 if (pmd->socket_filename == NULL)
1023 return;
1024
1025 if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1026 return;
1027
1028 for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1029 next = TAILQ_NEXT(elt, next);
1030 if (elt->dev == dev) {
1031 TAILQ_REMOVE(&socket->dev_queue, elt, next);
1032 rte_free(elt);
1033 pmd->socket_filename = NULL;
1034 }
1035 }
1036
1037
1038 if (TAILQ_EMPTY(&socket->dev_queue)) {
1039 rte_hash_del_key(hash, socket->filename);
1040 if (socket->listener && !(pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT)) {
1041
1042
1043
1044 ret = remove(socket->filename);
1045 if (ret < 0)
1046 MIF_LOG(ERR, "Failed to remove socket file: %s",
1047 socket->filename);
1048 }
1049 rte_free(socket);
1050 }
1051}
1052
1053int
1054memif_connect_server(struct rte_eth_dev *dev)
1055{
1056 struct pmd_internals *pmd = dev->data->dev_private;
1057
1058 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1059 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1060 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1061 return 0;
1062}
1063
1064int
1065memif_connect_client(struct rte_eth_dev *dev)
1066{
1067 int sockfd;
1068 int ret;
1069 uint32_t sunlen;
1070 struct sockaddr_un sun = { 0 };
1071 struct pmd_internals *pmd = dev->data->dev_private;
1072
1073 memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1074 memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1075 pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1076
1077 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1078 if (sockfd < 0) {
1079 MIF_LOG(ERR, "Failed to open socket.");
1080 return -1;
1081 }
1082
1083 sun.sun_family = AF_UNIX;
1084 sunlen = sizeof(struct sockaddr_un);
1085 if (pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT) {
1086
1087 sun.sun_path[0] = '\0';
1088 strlcpy(sun.sun_path + 1, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE - 1);
1089 sunlen = RTE_MIN(strlen(pmd->socket_filename) + 1,
1090 MEMIF_SOCKET_UN_SIZE) +
1091 sizeof(sun) - sizeof(sun.sun_path);
1092 } else {
1093 strlcpy(sun.sun_path, pmd->socket_filename, MEMIF_SOCKET_UN_SIZE);
1094 }
1095
1096 ret = connect(sockfd, (struct sockaddr *)&sun, sunlen);
1097 if (ret < 0) {
1098 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename);
1099 goto error;
1100 }
1101
1102 MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename);
1103
1104 pmd->cc = rte_zmalloc("memif-cc",
1105 sizeof(struct memif_control_channel), 0);
1106 if (pmd->cc == NULL) {
1107 MIF_LOG(ERR, "Failed to allocate control channel.");
1108 goto error;
1109 }
1110
1111 pmd->cc->intr_handle.fd = sockfd;
1112 pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
1113 pmd->cc->socket = NULL;
1114 pmd->cc->dev = dev;
1115 TAILQ_INIT(&pmd->cc->msg_queue);
1116
1117 ret = rte_intr_callback_register(&pmd->cc->intr_handle,
1118 memif_intr_handler, pmd->cc);
1119 if (ret < 0) {
1120 MIF_LOG(ERR, "Failed to register interrupt callback for control fd");
1121 goto error;
1122 }
1123
1124 return 0;
1125
1126 error:
1127 if (sockfd >= 0) {
1128 close(sockfd);
1129 sockfd = -1;
1130 }
1131 if (pmd->cc != NULL) {
1132 rte_free(pmd->cc);
1133 pmd->cc = NULL;
1134 }
1135 return -1;
1136}
1137
