1
2
3
4
5
6#include "queueing.h"
7#include "device.h"
8#include "peer.h"
9#include "timers.h"
10#include "messages.h"
11#include "cookie.h"
12#include "socket.h"
13
14#include <linux/ip.h>
15#include <linux/ipv6.h>
16#include <linux/udp.h>
17#include <net/ip_tunnels.h>
18
19
20static void update_rx_stats(struct wg_peer *peer, size_t len)
21{
22 struct pcpu_sw_netstats *tstats =
23 get_cpu_ptr(peer->device->dev->tstats);
24
25 u64_stats_update_begin(&tstats->syncp);
26 ++tstats->rx_packets;
27 tstats->rx_bytes += len;
28 peer->rx_bytes += len;
29 u64_stats_update_end(&tstats->syncp);
30 put_cpu_ptr(tstats);
31}
32
33#define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type)
34
35static size_t validate_header_len(struct sk_buff *skb)
36{
37 if (unlikely(skb->len < sizeof(struct message_header)))
38 return 0;
39 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) &&
40 skb->len >= MESSAGE_MINIMUM_LENGTH)
41 return sizeof(struct message_data);
42 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) &&
43 skb->len == sizeof(struct message_handshake_initiation))
44 return sizeof(struct message_handshake_initiation);
45 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) &&
46 skb->len == sizeof(struct message_handshake_response))
47 return sizeof(struct message_handshake_response);
48 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) &&
49 skb->len == sizeof(struct message_handshake_cookie))
50 return sizeof(struct message_handshake_cookie);
51 return 0;
52}
53
54static int prepare_skb_header(struct sk_buff *skb, struct wg_device *wg)
55{
56 size_t data_offset, data_len, header_len;
57 struct udphdr *udp;
58
59 if (unlikely(!wg_check_packet_protocol(skb) ||
60 skb_transport_header(skb) < skb->head ||
61 (skb_transport_header(skb) + sizeof(struct udphdr)) >
62 skb_tail_pointer(skb)))
63 return -EINVAL;
64 udp = udp_hdr(skb);
65 data_offset = (u8 *)udp - skb->data;
66 if (unlikely(data_offset > U16_MAX ||
67 data_offset + sizeof(struct udphdr) > skb->len))
68
69
70
71 return -EINVAL;
72 data_len = ntohs(udp->len);
73 if (unlikely(data_len < sizeof(struct udphdr) ||
74 data_len > skb->len - data_offset))
75
76
77
78 return -EINVAL;
79 data_len -= sizeof(struct udphdr);
80 data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data;
81 if (unlikely(!pskb_may_pull(skb,
82 data_offset + sizeof(struct message_header)) ||
83 pskb_trim(skb, data_len + data_offset) < 0))
84 return -EINVAL;
85 skb_pull(skb, data_offset);
86 if (unlikely(skb->len != data_len))
87
88 return -EINVAL;
89 header_len = validate_header_len(skb);
90 if (unlikely(!header_len))
91 return -EINVAL;
92 __skb_push(skb, data_offset);
93 if (unlikely(!pskb_may_pull(skb, data_offset + header_len)))
94 return -EINVAL;
95 __skb_pull(skb, data_offset);
96 return 0;
97}
98
99static void wg_receive_handshake_packet(struct wg_device *wg,
100 struct sk_buff *skb)
101{
102 enum cookie_mac_state mac_state;
103 struct wg_peer *peer = NULL;
104
105
106
107 static u64 last_under_load;
108 bool packet_needs_cookie;
109 bool under_load;
110
111 if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) {
112 net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n",
113 wg->dev->name, skb);
114 wg_cookie_message_consume(
115 (struct message_handshake_cookie *)skb->data, wg);
116 return;
117 }
118
119 under_load = atomic_read(&wg->handshake_queue_len) >=
120 MAX_QUEUED_INCOMING_HANDSHAKES / 8;
121 if (under_load) {
122 last_under_load = ktime_get_coarse_boottime_ns();
123 } else if (last_under_load) {
124 under_load = !wg_birthdate_has_expired(last_under_load, 1);
125 if (!under_load)
126 last_under_load = 0;
127 }
128 mac_state = wg_cookie_validate_packet(&wg->cookie_checker, skb,
129 under_load);
130 if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) ||
131 (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)) {
132 packet_needs_cookie = false;
133 } else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) {
134 packet_needs_cookie = true;
135 } else {
136 net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n",
137 wg->dev->name, skb);
138 return;
139 }
140
141 switch (SKB_TYPE_LE32(skb)) {
142 case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): {
143 struct message_handshake_initiation *message =
144 (struct message_handshake_initiation *)skb->data;
145
146 if (packet_needs_cookie) {
147 wg_packet_send_handshake_cookie(wg, skb,
148 message->sender_index);
149 return;
150 }
151 peer = wg_noise_handshake_consume_initiation(message, wg);
152 if (unlikely(!peer)) {
153 net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n",
154 wg->dev->name, skb);
155 return;
156 }
157 wg_socket_set_peer_endpoint_from_skb(peer, skb);
158 net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n",
159 wg->dev->name, peer->internal_id,
160 &peer->endpoint.addr);
161 wg_packet_send_handshake_response(peer);
162 break;
163 }
164 case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): {
165 struct message_handshake_response *message =
166 (struct message_handshake_response *)skb->data;
167
168 if (packet_needs_cookie) {
169 wg_packet_send_handshake_cookie(wg, skb,
170 message->sender_index);
171 return;
172 }
173 peer = wg_noise_handshake_consume_response(message, wg);
174 if (unlikely(!peer)) {
175 net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n",
176 wg->dev->name, skb);
177 return;
178 }
179 wg_socket_set_peer_endpoint_from_skb(peer, skb);
180 net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n",
181 wg->dev->name, peer->internal_id,
182 &peer->endpoint.addr);
183 if (wg_noise_handshake_begin_session(&peer->handshake,
184 &peer->keypairs)) {
185 wg_timers_session_derived(peer);
186 wg_timers_handshake_complete(peer);
187
188
189
190
191
192
193 wg_packet_send_keepalive(peer);
194 }
195 break;
196 }
197 }
198
199 if (unlikely(!peer)) {
200 WARN(1, "Somehow a wrong type of packet wound up in the handshake queue!\n");
201 return;
202 }
203
204 local_bh_disable();
205 update_rx_stats(peer, skb->len);
206 local_bh_enable();
207
208 wg_timers_any_authenticated_packet_received(peer);
209 wg_timers_any_authenticated_packet_traversal(peer);
210 wg_peer_put(peer);
211}
212
213void wg_packet_handshake_receive_worker(struct work_struct *work)
214{
215 struct crypt_queue *queue = container_of(work, struct multicore_worker, work)->ptr;
216 struct wg_device *wg = container_of(queue, struct wg_device, handshake_queue);
217 struct sk_buff *skb;
218
219 while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) {
220 wg_receive_handshake_packet(wg, skb);
221 dev_kfree_skb(skb);
222 atomic_dec(&wg->handshake_queue_len);
223 cond_resched();
224 }
225}
226
227static void keep_key_fresh(struct wg_peer *peer)
228{
229 struct noise_keypair *keypair;
230 bool send;
231
232 if (peer->sent_lastminute_handshake)
233 return;
234
235 rcu_read_lock_bh();
236 keypair = rcu_dereference_bh(peer->keypairs.current_keypair);
237 send = keypair && READ_ONCE(keypair->sending.is_valid) &&
238 keypair->i_am_the_initiator &&
239 wg_birthdate_has_expired(keypair->sending.birthdate,
240 REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT);
241 rcu_read_unlock_bh();
242
243 if (unlikely(send)) {
244 peer->sent_lastminute_handshake = true;
245 wg_packet_send_queued_handshake_initiation(peer, false);
246 }
247}
248
249static bool decrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair)
250{
251 struct scatterlist sg[MAX_SKB_FRAGS + 8];
252 struct sk_buff *trailer;
253 unsigned int offset;
254 int num_frags;
255
256 if (unlikely(!keypair))
257 return false;
258
259 if (unlikely(!READ_ONCE(keypair->receiving.is_valid) ||
260 wg_birthdate_has_expired(keypair->receiving.birthdate, REJECT_AFTER_TIME) ||
261 keypair->receiving_counter.counter >= REJECT_AFTER_MESSAGES)) {
262 WRITE_ONCE(keypair->receiving.is_valid, false);
263 return false;
264 }
265
266 PACKET_CB(skb)->nonce =
267 le64_to_cpu(((struct message_data *)skb->data)->counter);
268
269
270
271
272
273 offset = skb->data - skb_network_header(skb);
274 skb_push(skb, offset);
275 num_frags = skb_cow_data(skb, 0, &trailer);
276 offset += sizeof(struct message_data);
277 skb_pull(skb, offset);
278 if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg)))
279 return false;
280
281 sg_init_table(sg, num_frags);
282 if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0)
283 return false;
284
285 if (!chacha20poly1305_decrypt_sg_inplace(sg, skb->len, NULL, 0,
286 PACKET_CB(skb)->nonce,
287 keypair->receiving.key))
288 return false;
289
290
291
292
293 skb_push(skb, offset);
294 if (pskb_trim(skb, skb->len - noise_encrypted_len(0)))
295 return false;
296 skb_pull(skb, offset);
297
298 return true;
299}
300
301
302static bool counter_validate(struct noise_replay_counter *counter, u64 their_counter)
303{
304 unsigned long index, index_current, top, i;
305 bool ret = false;
306
307 spin_lock_bh(&counter->lock);
308
309 if (unlikely(counter->counter >= REJECT_AFTER_MESSAGES + 1 ||
310 their_counter >= REJECT_AFTER_MESSAGES))
311 goto out;
312
313 ++their_counter;
314
315 if (unlikely((COUNTER_WINDOW_SIZE + their_counter) <
316 counter->counter))
317 goto out;
318
319 index = their_counter >> ilog2(BITS_PER_LONG);
320
321 if (likely(their_counter > counter->counter)) {
322 index_current = counter->counter >> ilog2(BITS_PER_LONG);
323 top = min_t(unsigned long, index - index_current,
324 COUNTER_BITS_TOTAL / BITS_PER_LONG);
325 for (i = 1; i <= top; ++i)
326 counter->backtrack[(i + index_current) &
327 ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0;
328 counter->counter = their_counter;
329 }
330
331 index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1;
332 ret = !test_and_set_bit(their_counter & (BITS_PER_LONG - 1),
333 &counter->backtrack[index]);
334
335out:
336 spin_unlock_bh(&counter->lock);
337 return ret;
338}
339
340#include "selftest/counter.c"
341
342static void wg_packet_consume_data_done(struct wg_peer *peer,
343 struct sk_buff *skb,
344 struct endpoint *endpoint)
345{
346 struct net_device *dev = peer->device->dev;
347 unsigned int len, len_before_trim;
348 struct wg_peer *routed_peer;
349
350 wg_socket_set_peer_endpoint(peer, endpoint);
351
352 if (unlikely(wg_noise_received_with_keypair(&peer->keypairs,
353 PACKET_CB(skb)->keypair))) {
354 wg_timers_handshake_complete(peer);
355 wg_packet_send_staged_packets(peer);
356 }
357
358 keep_key_fresh(peer);
359
360 wg_timers_any_authenticated_packet_received(peer);
361 wg_timers_any_authenticated_packet_traversal(peer);
362
363
364 if (unlikely(!skb->len)) {
365 update_rx_stats(peer, message_data_len(0));
366 net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n",
367 dev->name, peer->internal_id,
368 &peer->endpoint.addr);
369 goto packet_processed;
370 }
371
372 wg_timers_data_received(peer);
373
374 if (unlikely(skb_network_header(skb) < skb->head))
375 goto dishonest_packet_size;
376 if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) &&
377 (ip_hdr(skb)->version == 4 ||
378 (ip_hdr(skb)->version == 6 &&
379 pskb_network_may_pull(skb, sizeof(struct ipv6hdr)))))))
380 goto dishonest_packet_type;
381
382 skb->dev = dev;
383
384
385
386
387
388
389 skb->ip_summed = CHECKSUM_UNNECESSARY;
390 skb->csum_level = ~0;
391 skb->protocol = ip_tunnel_parse_protocol(skb);
392 if (skb->protocol == htons(ETH_P_IP)) {
393 len = ntohs(ip_hdr(skb)->tot_len);
394 if (unlikely(len < sizeof(struct iphdr)))
395 goto dishonest_packet_size;
396 INET_ECN_decapsulate(skb, PACKET_CB(skb)->ds, ip_hdr(skb)->tos);
397 } else if (skb->protocol == htons(ETH_P_IPV6)) {
398 len = ntohs(ipv6_hdr(skb)->payload_len) +
399 sizeof(struct ipv6hdr);
400 INET_ECN_decapsulate(skb, PACKET_CB(skb)->ds, ipv6_get_dsfield(ipv6_hdr(skb)));
401 } else {
402 goto dishonest_packet_type;
403 }
404
405 if (unlikely(len > skb->len))
406 goto dishonest_packet_size;
407 len_before_trim = skb->len;
408 if (unlikely(pskb_trim(skb, len)))
409 goto packet_processed;
410
411 routed_peer = wg_allowedips_lookup_src(&peer->device->peer_allowedips,
412 skb);
413 wg_peer_put(routed_peer);
414
415 if (unlikely(routed_peer != peer))
416 goto dishonest_packet_peer;
417
418 napi_gro_receive(&peer->napi, skb);
419 update_rx_stats(peer, message_data_len(len_before_trim));
420 return;
421
422dishonest_packet_peer:
423 net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n",
424 dev->name, skb, peer->internal_id,
425 &peer->endpoint.addr);
426 ++dev->stats.rx_errors;
427 ++dev->stats.rx_frame_errors;
428 goto packet_processed;
429dishonest_packet_type:
430 net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n",
431 dev->name, peer->internal_id, &peer->endpoint.addr);
432 ++dev->stats.rx_errors;
433 ++dev->stats.rx_frame_errors;
434 goto packet_processed;
435dishonest_packet_size:
436 net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n",
437 dev->name, peer->internal_id, &peer->endpoint.addr);
438 ++dev->stats.rx_errors;
439 ++dev->stats.rx_length_errors;
440 goto packet_processed;
441packet_processed:
442 dev_kfree_skb(skb);
443}
444
445int wg_packet_rx_poll(struct napi_struct *napi, int budget)
446{
447 struct wg_peer *peer = container_of(napi, struct wg_peer, napi);
448 struct noise_keypair *keypair;
449 struct endpoint endpoint;
450 enum packet_state state;
451 struct sk_buff *skb;
452 int work_done = 0;
453 bool free;
454
455 if (unlikely(budget <= 0))
456 return 0;
457
458 while ((skb = wg_prev_queue_peek(&peer->rx_queue)) != NULL &&
459 (state = atomic_read_acquire(&PACKET_CB(skb)->state)) !=
460 PACKET_STATE_UNCRYPTED) {
461 wg_prev_queue_drop_peeked(&peer->rx_queue);
462 keypair = PACKET_CB(skb)->keypair;
463 free = true;
464
465 if (unlikely(state != PACKET_STATE_CRYPTED))
466 goto next;
467
468 if (unlikely(!counter_validate(&keypair->receiving_counter,
469 PACKET_CB(skb)->nonce))) {
470 net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n",
471 peer->device->dev->name,
472 PACKET_CB(skb)->nonce,
473 keypair->receiving_counter.counter);
474 goto next;
475 }
476
477 if (unlikely(wg_socket_endpoint_from_skb(&endpoint, skb)))
478 goto next;
479
480 wg_reset_packet(skb, false);
481 wg_packet_consume_data_done(peer, skb, &endpoint);
482 free = false;
483
484next:
485 wg_noise_keypair_put(keypair, false);
486 wg_peer_put(peer);
487 if (unlikely(free))
488 dev_kfree_skb(skb);
489
490 if (++work_done >= budget)
491 break;
492 }
493
494 if (work_done < budget)
495 napi_complete_done(napi, work_done);
496
497 return work_done;
498}
499
500void wg_packet_decrypt_worker(struct work_struct *work)
501{
502 struct crypt_queue *queue = container_of(work, struct multicore_worker,
503 work)->ptr;
504 struct sk_buff *skb;
505
506 while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) {
507 enum packet_state state =
508 likely(decrypt_packet(skb, PACKET_CB(skb)->keypair)) ?
509 PACKET_STATE_CRYPTED : PACKET_STATE_DEAD;
510 wg_queue_enqueue_per_peer_rx(skb, state);
511 if (need_resched())
512 cond_resched();
513 }
514}
515
516static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb)
517{
518 __le32 idx = ((struct message_data *)skb->data)->key_idx;
519 struct wg_peer *peer = NULL;
520 int ret;
521
522 rcu_read_lock_bh();
523 PACKET_CB(skb)->keypair =
524 (struct noise_keypair *)wg_index_hashtable_lookup(
525 wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx,
526 &peer);
527 if (unlikely(!wg_noise_keypair_get(PACKET_CB(skb)->keypair)))
528 goto err_keypair;
529
530 if (unlikely(READ_ONCE(peer->is_dead)))
531 goto err;
532
533 ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, &peer->rx_queue, skb,
534 wg->packet_crypt_wq, &wg->decrypt_queue.last_cpu);
535 if (unlikely(ret == -EPIPE))
536 wg_queue_enqueue_per_peer_rx(skb, PACKET_STATE_DEAD);
537 if (likely(!ret || ret == -EPIPE)) {
538 rcu_read_unlock_bh();
539 return;
540 }
541err:
542 wg_noise_keypair_put(PACKET_CB(skb)->keypair, false);
543err_keypair:
544 rcu_read_unlock_bh();
545 wg_peer_put(peer);
546 dev_kfree_skb(skb);
547}
548
549void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb)
550{
551 if (unlikely(prepare_skb_header(skb, wg) < 0))
552 goto err;
553 switch (SKB_TYPE_LE32(skb)) {
554 case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION):
555 case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE):
556 case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): {
557 int cpu, ret = -EBUSY;
558
559 if (unlikely(!rng_is_initialized()))
560 goto drop;
561 if (atomic_read(&wg->handshake_queue_len) > MAX_QUEUED_INCOMING_HANDSHAKES / 2) {
562 if (spin_trylock_bh(&wg->handshake_queue.ring.producer_lock)) {
563 ret = __ptr_ring_produce(&wg->handshake_queue.ring, skb);
564 spin_unlock_bh(&wg->handshake_queue.ring.producer_lock);
565 }
566 } else
567 ret = ptr_ring_produce_bh(&wg->handshake_queue.ring, skb);
568 if (ret) {
569 drop:
570 net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n",
571 wg->dev->name, skb);
572 goto err;
573 }
574 atomic_inc(&wg->handshake_queue_len);
575 cpu = wg_cpumask_next_online(&wg->handshake_queue.last_cpu);
576
577 queue_work_on(cpu, wg->handshake_receive_wq,
578 &per_cpu_ptr(wg->handshake_queue.worker, cpu)->work);
579 break;
580 }
581 case cpu_to_le32(MESSAGE_DATA):
582 PACKET_CB(skb)->ds = ip_tunnel_get_dsfield(ip_hdr(skb), skb);
583 wg_packet_consume_data(wg, skb);
584 break;
585 default:
586 WARN(1, "Non-exhaustive parsing of packet header lead to unknown packet type!\n");
587 goto err;
588 }
589 return;
590
591err:
592 dev_kfree_skb(skb);
593}
594
