linux/net/xfrm/xfrm_input.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * xfrm_input.c
   4 *
   5 * Changes:
   6 *      YOSHIFUJI Hideaki @USAGI
   7 *              Split up af-specific portion
   8 *
   9 */
  10
  11#include <linux/bottom_half.h>
  12#include <linux/cache.h>
  13#include <linux/interrupt.h>
  14#include <linux/slab.h>
  15#include <linux/module.h>
  16#include <linux/netdevice.h>
  17#include <linux/percpu.h>
  18#include <net/dst.h>
  19#include <net/ip.h>
  20#include <net/xfrm.h>
  21#include <net/ip_tunnels.h>
  22#include <net/ip6_tunnel.h>
  23
  24#include "xfrm_inout.h"
  25
  26struct xfrm_trans_tasklet {
  27        struct tasklet_struct tasklet;
  28        struct sk_buff_head queue;
  29};
  30
  31struct xfrm_trans_cb {
  32        union {
  33                struct inet_skb_parm    h4;
  34#if IS_ENABLED(CONFIG_IPV6)
  35                struct inet6_skb_parm   h6;
  36#endif
  37        } header;
  38        int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb);
  39        struct net *net;
  40};
  41
  42#define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))
  43
  44static DEFINE_SPINLOCK(xfrm_input_afinfo_lock);
  45static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[2][AF_INET6 + 1];
  46
  47static struct gro_cells gro_cells;
  48static struct net_device xfrm_napi_dev;
  49
  50static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet);
  51
  52int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo)
  53{
  54        int err = 0;
  55
  56        if (WARN_ON(afinfo->family > AF_INET6))
  57                return -EAFNOSUPPORT;
  58
  59        spin_lock_bh(&xfrm_input_afinfo_lock);
  60        if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family]))
  61                err = -EEXIST;
  62        else
  63                rcu_assign_pointer(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], afinfo);
  64        spin_unlock_bh(&xfrm_input_afinfo_lock);
  65        return err;
  66}
  67EXPORT_SYMBOL(xfrm_input_register_afinfo);
  68
  69int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo)
  70{
  71        int err = 0;
  72
  73        spin_lock_bh(&xfrm_input_afinfo_lock);
  74        if (likely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family])) {
  75                if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family] != afinfo))
  76                        err = -EINVAL;
  77                else
  78                        RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], NULL);
  79        }
  80        spin_unlock_bh(&xfrm_input_afinfo_lock);
  81        synchronize_rcu();
  82        return err;
  83}
  84EXPORT_SYMBOL(xfrm_input_unregister_afinfo);
  85
  86static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(u8 family, bool is_ipip)
  87{
  88        const struct xfrm_input_afinfo *afinfo;
  89
  90        if (WARN_ON_ONCE(family > AF_INET6))
  91                return NULL;
  92
  93        rcu_read_lock();
  94        afinfo = rcu_dereference(xfrm_input_afinfo[is_ipip][family]);
  95        if (unlikely(!afinfo))
  96                rcu_read_unlock();
  97        return afinfo;
  98}
  99
 100static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,
 101                       int err)
 102{
 103        bool is_ipip = (protocol == IPPROTO_IPIP || protocol == IPPROTO_IPV6);
 104        const struct xfrm_input_afinfo *afinfo;
 105        int ret;
 106
 107        afinfo = xfrm_input_get_afinfo(family, is_ipip);
 108        if (!afinfo)
 109                return -EAFNOSUPPORT;
 110
 111        ret = afinfo->callback(skb, protocol, err);
 112        rcu_read_unlock();
 113
 114        return ret;
 115}
 116
 117struct sec_path *secpath_set(struct sk_buff *skb)
 118{
 119        struct sec_path *sp, *tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH);
 120
 121        sp = skb_ext_add(skb, SKB_EXT_SEC_PATH);
 122        if (!sp)
 123                return NULL;
 124
 125        if (tmp) /* reused existing one (was COW'd if needed) */
 126                return sp;
 127
 128        /* allocated new secpath */
 129        memset(sp->ovec, 0, sizeof(sp->ovec));
 130        sp->olen = 0;
 131        sp->len = 0;
 132
 133        return sp;
 134}
 135EXPORT_SYMBOL(secpath_set);
 136
 137/* Fetch spi and seq from ipsec header */
 138
 139int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq)
 140{
 141        int offset, offset_seq;
 142        int hlen;
 143
 144        switch (nexthdr) {
 145        case IPPROTO_AH:
 146                hlen = sizeof(struct ip_auth_hdr);
 147                offset = offsetof(struct ip_auth_hdr, spi);
 148                offset_seq = offsetof(struct ip_auth_hdr, seq_no);
 149                break;
 150        case IPPROTO_ESP:
 151                hlen = sizeof(struct ip_esp_hdr);
 152                offset = offsetof(struct ip_esp_hdr, spi);
 153                offset_seq = offsetof(struct ip_esp_hdr, seq_no);
 154                break;
 155        case IPPROTO_COMP:
 156                if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr)))
 157                        return -EINVAL;
 158                *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2)));
 159                *seq = 0;
 160                return 0;
 161        default:
 162                return 1;
 163        }
 164
 165        if (!pskb_may_pull(skb, hlen))
 166                return -EINVAL;
 167
 168        *spi = *(__be32 *)(skb_transport_header(skb) + offset);
 169        *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq);
 170        return 0;
 171}
 172EXPORT_SYMBOL(xfrm_parse_spi);
 173
 174static int xfrm4_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
 175{
 176        struct iphdr *iph;
 177        int optlen = 0;
 178        int err = -EINVAL;
 179
 180        if (unlikely(XFRM_MODE_SKB_CB(skb)->protocol == IPPROTO_BEETPH)) {
 181                struct ip_beet_phdr *ph;
 182                int phlen;
 183
 184                if (!pskb_may_pull(skb, sizeof(*ph)))
 185                        goto out;
 186
 187                ph = (struct ip_beet_phdr *)skb->data;
 188
 189                phlen = sizeof(*ph) + ph->padlen;
 190                optlen = ph->hdrlen * 8 + (IPV4_BEET_PHMAXLEN - phlen);
 191                if (optlen < 0 || optlen & 3 || optlen > 250)
 192                        goto out;
 193
 194                XFRM_MODE_SKB_CB(skb)->protocol = ph->nexthdr;
 195
 196                if (!pskb_may_pull(skb, phlen))
 197                        goto out;
 198                __skb_pull(skb, phlen);
 199        }
 200
 201        skb_push(skb, sizeof(*iph));
 202        skb_reset_network_header(skb);
 203        skb_mac_header_rebuild(skb);
 204
 205        xfrm4_beet_make_header(skb);
 206
 207        iph = ip_hdr(skb);
 208
 209        iph->ihl += optlen / 4;
 210        iph->tot_len = htons(skb->len);
 211        iph->daddr = x->sel.daddr.a4;
 212        iph->saddr = x->sel.saddr.a4;
 213        iph->check = 0;
 214        iph->check = ip_fast_csum(skb_network_header(skb), iph->ihl);
 215        err = 0;
 216out:
 217        return err;
 218}
 219
 220static void ipip_ecn_decapsulate(struct sk_buff *skb)
 221{
 222        struct iphdr *inner_iph = ipip_hdr(skb);
 223
 224        if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
 225                IP_ECN_set_ce(inner_iph);
 226}
 227
 228static int xfrm4_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
 229{
 230        int err = -EINVAL;
 231
 232        if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPIP)
 233                goto out;
 234
 235        if (!pskb_may_pull(skb, sizeof(struct iphdr)))
 236                goto out;
 237
 238        err = skb_unclone(skb, GFP_ATOMIC);
 239        if (err)
 240                goto out;
 241
 242        if (x->props.flags & XFRM_STATE_DECAP_DSCP)
 243                ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipip_hdr(skb));
 244        if (!(x->props.flags & XFRM_STATE_NOECN))
 245                ipip_ecn_decapsulate(skb);
 246
 247        skb_reset_network_header(skb);
 248        skb_mac_header_rebuild(skb);
 249        if (skb->mac_len)
 250                eth_hdr(skb)->h_proto = skb->protocol;
 251
 252        err = 0;
 253
 254out:
 255        return err;
 256}
 257
 258static void ipip6_ecn_decapsulate(struct sk_buff *skb)
 259{
 260        struct ipv6hdr *inner_iph = ipipv6_hdr(skb);
 261
 262        if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
 263                IP6_ECN_set_ce(skb, inner_iph);
 264}
 265
 266static int xfrm6_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
 267{
 268        int err = -EINVAL;
 269
 270        if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPV6)
 271                goto out;
 272        if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
 273                goto out;
 274
 275        err = skb_unclone(skb, GFP_ATOMIC);
 276        if (err)
 277                goto out;
 278
 279        if (x->props.flags & XFRM_STATE_DECAP_DSCP)
 280                ipv6_copy_dscp(ipv6_get_dsfield(ipv6_hdr(skb)),
 281                               ipipv6_hdr(skb));
 282        if (!(x->props.flags & XFRM_STATE_NOECN))
 283                ipip6_ecn_decapsulate(skb);
 284
 285        skb_reset_network_header(skb);
 286        skb_mac_header_rebuild(skb);
 287        if (skb->mac_len)
 288                eth_hdr(skb)->h_proto = skb->protocol;
 289
 290        err = 0;
 291
 292out:
 293        return err;
 294}
 295
 296static int xfrm6_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
 297{
 298        struct ipv6hdr *ip6h;
 299        int size = sizeof(struct ipv6hdr);
 300        int err;
 301
 302        err = skb_cow_head(skb, size + skb->mac_len);
 303        if (err)
 304                goto out;
 305
 306        __skb_push(skb, size);
 307        skb_reset_network_header(skb);
 308        skb_mac_header_rebuild(skb);
 309
 310        xfrm6_beet_make_header(skb);
 311
 312        ip6h = ipv6_hdr(skb);
 313        ip6h->payload_len = htons(skb->len - size);
 314        ip6h->daddr = x->sel.daddr.in6;
 315        ip6h->saddr = x->sel.saddr.in6;
 316        err = 0;
 317out:
 318        return err;
 319}
 320
 321/* Remove encapsulation header.
 322 *
 323 * The IP header will be moved over the top of the encapsulation
 324 * header.
 325 *
 326 * On entry, the transport header shall point to where the IP header
 327 * should be and the network header shall be set to where the IP
 328 * header currently is.  skb->data shall point to the start of the
 329 * payload.
 330 */
 331static int
 332xfrm_inner_mode_encap_remove(struct xfrm_state *x,
 333                             const struct xfrm_mode *inner_mode,
 334                             struct sk_buff *skb)
 335{
 336        switch (inner_mode->encap) {
 337        case XFRM_MODE_BEET:
 338                if (inner_mode->family == AF_INET)
 339                        return xfrm4_remove_beet_encap(x, skb);
 340                if (inner_mode->family == AF_INET6)
 341                        return xfrm6_remove_beet_encap(x, skb);
 342                break;
 343        case XFRM_MODE_TUNNEL:
 344                if (inner_mode->family == AF_INET)
 345                        return xfrm4_remove_tunnel_encap(x, skb);
 346                if (inner_mode->family == AF_INET6)
 347                        return xfrm6_remove_tunnel_encap(x, skb);
 348                break;
 349        }
 350
 351        WARN_ON_ONCE(1);
 352        return -EOPNOTSUPP;
 353}
 354
 355static int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb)
 356{
 357        const struct xfrm_mode *inner_mode = &x->inner_mode;
 358
 359        switch (x->outer_mode.family) {
 360        case AF_INET:
 361                xfrm4_extract_header(skb);
 362                break;
 363        case AF_INET6:
 364                xfrm6_extract_header(skb);
 365                break;
 366        default:
 367                WARN_ON_ONCE(1);
 368                return -EAFNOSUPPORT;
 369        }
 370
 371        if (x->sel.family == AF_UNSPEC) {
 372                inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
 373                if (!inner_mode)
 374                        return -EAFNOSUPPORT;
 375        }
 376
 377        switch (inner_mode->family) {
 378        case AF_INET:
 379                skb->protocol = htons(ETH_P_IP);
 380                break;
 381        case AF_INET6:
 382                skb->protocol = htons(ETH_P_IPV6);
 383                break;
 384        default:
 385                WARN_ON_ONCE(1);
 386                break;
 387        }
 388
 389        return xfrm_inner_mode_encap_remove(x, inner_mode, skb);
 390}
 391
 392/* Remove encapsulation header.
 393 *
 394 * The IP header will be moved over the top of the encapsulation header.
 395 *
 396 * On entry, skb_transport_header() shall point to where the IP header
 397 * should be and skb_network_header() shall be set to where the IP header
 398 * currently is.  skb->data shall point to the start of the payload.
 399 */
 400static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb)
 401{
 402        int ihl = skb->data - skb_transport_header(skb);
 403
 404        if (skb->transport_header != skb->network_header) {
 405                memmove(skb_transport_header(skb),
 406                        skb_network_header(skb), ihl);
 407                skb->network_header = skb->transport_header;
 408        }
 409        ip_hdr(skb)->tot_len = htons(skb->len + ihl);
 410        skb_reset_transport_header(skb);
 411        return 0;
 412}
 413
 414static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb)
 415{
 416#if IS_ENABLED(CONFIG_IPV6)
 417        int ihl = skb->data - skb_transport_header(skb);
 418
 419        if (skb->transport_header != skb->network_header) {
 420                memmove(skb_transport_header(skb),
 421                        skb_network_header(skb), ihl);
 422                skb->network_header = skb->transport_header;
 423        }
 424        ipv6_hdr(skb)->payload_len = htons(skb->len + ihl -
 425                                           sizeof(struct ipv6hdr));
 426        skb_reset_transport_header(skb);
 427        return 0;
 428#else
 429        WARN_ON_ONCE(1);
 430        return -EAFNOSUPPORT;
 431#endif
 432}
 433
 434static int xfrm_inner_mode_input(struct xfrm_state *x,
 435                                 const struct xfrm_mode *inner_mode,
 436                                 struct sk_buff *skb)
 437{
 438        switch (inner_mode->encap) {
 439        case XFRM_MODE_BEET:
 440        case XFRM_MODE_TUNNEL:
 441                return xfrm_prepare_input(x, skb);
 442        case XFRM_MODE_TRANSPORT:
 443                if (inner_mode->family == AF_INET)
 444                        return xfrm4_transport_input(x, skb);
 445                if (inner_mode->family == AF_INET6)
 446                        return xfrm6_transport_input(x, skb);
 447                break;
 448        case XFRM_MODE_ROUTEOPTIMIZATION:
 449                WARN_ON_ONCE(1);
 450                break;
 451        default:
 452                WARN_ON_ONCE(1);
 453                break;
 454        }
 455
 456        return -EOPNOTSUPP;
 457}
 458
 459int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
 460{
 461        const struct xfrm_state_afinfo *afinfo;
 462        struct net *net = dev_net(skb->dev);
 463        const struct xfrm_mode *inner_mode;
 464        int err;
 465        __be32 seq;
 466        __be32 seq_hi;
 467        struct xfrm_state *x = NULL;
 468        xfrm_address_t *daddr;
 469        u32 mark = skb->mark;
 470        unsigned int family = AF_UNSPEC;
 471        int decaps = 0;
 472        int async = 0;
 473        bool xfrm_gro = false;
 474        bool crypto_done = false;
 475        struct xfrm_offload *xo = xfrm_offload(skb);
 476        struct sec_path *sp;
 477
 478        if (encap_type < 0) {
 479                x = xfrm_input_state(skb);
 480
 481                if (unlikely(x->km.state != XFRM_STATE_VALID)) {
 482                        if (x->km.state == XFRM_STATE_ACQ)
 483                                XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
 484                        else
 485                                XFRM_INC_STATS(net,
 486                                               LINUX_MIB_XFRMINSTATEINVALID);
 487
 488                        if (encap_type == -1)
 489                                dev_put(skb->dev);
 490                        goto drop;
 491                }
 492
 493                family = x->outer_mode.family;
 494
 495                /* An encap_type of -1 indicates async resumption. */
 496                if (encap_type == -1) {
 497                        async = 1;
 498                        seq = XFRM_SKB_CB(skb)->seq.input.low;
 499                        goto resume;
 500                }
 501
 502                /* encap_type < -1 indicates a GRO call. */
 503                encap_type = 0;
 504                seq = XFRM_SPI_SKB_CB(skb)->seq;
 505
 506                if (xo && (xo->flags & CRYPTO_DONE)) {
 507                        crypto_done = true;
 508                        family = XFRM_SPI_SKB_CB(skb)->family;
 509
 510                        if (!(xo->status & CRYPTO_SUCCESS)) {
 511                                if (xo->status &
 512                                    (CRYPTO_TRANSPORT_AH_AUTH_FAILED |
 513                                     CRYPTO_TRANSPORT_ESP_AUTH_FAILED |
 514                                     CRYPTO_TUNNEL_AH_AUTH_FAILED |
 515                                     CRYPTO_TUNNEL_ESP_AUTH_FAILED)) {
 516
 517                                        xfrm_audit_state_icvfail(x, skb,
 518                                                                 x->type->proto);
 519                                        x->stats.integrity_failed++;
 520                                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
 521                                        goto drop;
 522                                }
 523
 524                                if (xo->status & CRYPTO_INVALID_PROTOCOL) {
 525                                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
 526                                        goto drop;
 527                                }
 528
 529                                XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
 530                                goto drop;
 531                        }
 532
 533                        if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
 534                                XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
 535                                goto drop;
 536                        }
 537                }
 538
 539                goto lock;
 540        }
 541
 542        family = XFRM_SPI_SKB_CB(skb)->family;
 543
 544        /* if tunnel is present override skb->mark value with tunnel i_key */
 545        switch (family) {
 546        case AF_INET:
 547                if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)
 548                        mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key);
 549                break;
 550        case AF_INET6:
 551                if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)
 552                        mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key);
 553                break;
 554        }
 555
 556        sp = secpath_set(skb);
 557        if (!sp) {
 558                XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
 559                goto drop;
 560        }
 561
 562        seq = 0;
 563        if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
 564                secpath_reset(skb);
 565                XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
 566                goto drop;
 567        }
 568
 569        daddr = (xfrm_address_t *)(skb_network_header(skb) +
 570                                   XFRM_SPI_SKB_CB(skb)->daddroff);
 571        do {
 572                sp = skb_sec_path(skb);
 573
 574                if (sp->len == XFRM_MAX_DEPTH) {
 575                        secpath_reset(skb);
 576                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
 577                        goto drop;
 578                }
 579
 580                x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family);
 581                if (x == NULL) {
 582                        secpath_reset(skb);
 583                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES);
 584                        xfrm_audit_state_notfound(skb, family, spi, seq);
 585                        goto drop;
 586                }
 587
 588                skb->mark = xfrm_smark_get(skb->mark, x);
 589
 590                sp->xvec[sp->len++] = x;
 591
 592                skb_dst_force(skb);
 593                if (!skb_dst(skb)) {
 594                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
 595                        goto drop;
 596                }
 597
 598lock:
 599                spin_lock(&x->lock);
 600
 601                if (unlikely(x->km.state != XFRM_STATE_VALID)) {
 602                        if (x->km.state == XFRM_STATE_ACQ)
 603                                XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
 604                        else
 605                                XFRM_INC_STATS(net,
 606                                               LINUX_MIB_XFRMINSTATEINVALID);
 607                        goto drop_unlock;
 608                }
 609
 610                if ((x->encap ? x->encap->encap_type : 0) != encap_type) {
 611                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
 612                        goto drop_unlock;
 613                }
 614
 615                if (x->repl->check(x, skb, seq)) {
 616                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
 617                        goto drop_unlock;
 618                }
 619
 620                if (xfrm_state_check_expire(x)) {
 621                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED);
 622                        goto drop_unlock;
 623                }
 624
 625                spin_unlock(&x->lock);
 626
 627                if (xfrm_tunnel_check(skb, x, family)) {
 628                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
 629                        goto drop;
 630                }
 631
 632                seq_hi = htonl(xfrm_replay_seqhi(x, seq));
 633
 634                XFRM_SKB_CB(skb)->seq.input.low = seq;
 635                XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;
 636
 637                dev_hold(skb->dev);
 638
 639                if (crypto_done)
 640                        nexthdr = x->type_offload->input_tail(x, skb);
 641                else
 642                        nexthdr = x->type->input(x, skb);
 643
 644                if (nexthdr == -EINPROGRESS)
 645                        return 0;
 646resume:
 647                dev_put(skb->dev);
 648
 649                spin_lock(&x->lock);
 650                if (nexthdr < 0) {
 651                        if (nexthdr == -EBADMSG) {
 652                                xfrm_audit_state_icvfail(x, skb,
 653                                                         x->type->proto);
 654                                x->stats.integrity_failed++;
 655                        }
 656                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
 657                        goto drop_unlock;
 658                }
 659
 660                /* only the first xfrm gets the encap type */
 661                encap_type = 0;
 662
 663                if (x->repl->recheck(x, skb, seq)) {
 664                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
 665                        goto drop_unlock;
 666                }
 667
 668                x->repl->advance(x, seq);
 669
 670                x->curlft.bytes += skb->len;
 671                x->curlft.packets++;
 672
 673                spin_unlock(&x->lock);
 674
 675                XFRM_MODE_SKB_CB(skb)->protocol = nexthdr;
 676
 677                inner_mode = &x->inner_mode;
 678
 679                if (x->sel.family == AF_UNSPEC) {
 680                        inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
 681                        if (inner_mode == NULL) {
 682                                XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
 683                                goto drop;
 684                        }
 685                }
 686
 687                if (xfrm_inner_mode_input(x, inner_mode, skb)) {
 688                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
 689                        goto drop;
 690                }
 691
 692                if (x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL) {
 693                        decaps = 1;
 694                        break;
 695                }
 696
 697                /*
 698                 * We need the inner address.  However, we only get here for
 699                 * transport mode so the outer address is identical.
 700                 */
 701                daddr = &x->id.daddr;
 702                family = x->outer_mode.family;
 703
 704                err = xfrm_parse_spi(skb, nexthdr, &spi, &seq);
 705                if (err < 0) {
 706                        XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
 707                        goto drop;
 708                }
 709                crypto_done = false;
 710        } while (!err);
 711
 712        err = xfrm_rcv_cb(skb, family, x->type->proto, 0);
 713        if (err)
 714                goto drop;
 715
 716        nf_reset_ct(skb);
 717
 718        if (decaps) {
 719                sp = skb_sec_path(skb);
 720                if (sp)
 721                        sp->olen = 0;
 722                skb_dst_drop(skb);
 723                gro_cells_receive(&gro_cells, skb);
 724                return 0;
 725        } else {
 726                xo = xfrm_offload(skb);
 727                if (xo)
 728                        xfrm_gro = xo->flags & XFRM_GRO;
 729
 730                err = -EAFNOSUPPORT;
 731                rcu_read_lock();
 732                afinfo = xfrm_state_afinfo_get_rcu(x->inner_mode.family);
 733                if (likely(afinfo))
 734                        err = afinfo->transport_finish(skb, xfrm_gro || async);
 735                rcu_read_unlock();
 736                if (xfrm_gro) {
 737                        sp = skb_sec_path(skb);
 738                        if (sp)
 739                                sp->olen = 0;
 740                        skb_dst_drop(skb);
 741                        gro_cells_receive(&gro_cells, skb);
 742                        return err;
 743                }
 744
 745                return err;
 746        }
 747
 748drop_unlock:
 749        spin_unlock(&x->lock);
 750drop:
 751        xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1);
 752        kfree_skb(skb);
 753        return 0;
 754}
 755EXPORT_SYMBOL(xfrm_input);
 756
 757int xfrm_input_resume(struct sk_buff *skb, int nexthdr)
 758{
 759        return xfrm_input(skb, nexthdr, 0, -1);
 760}
 761EXPORT_SYMBOL(xfrm_input_resume);
 762
 763static void xfrm_trans_reinject(struct tasklet_struct *t)
 764{
 765        struct xfrm_trans_tasklet *trans = from_tasklet(trans, t, tasklet);
 766        struct sk_buff_head queue;
 767        struct sk_buff *skb;
 768
 769        __skb_queue_head_init(&queue);
 770        skb_queue_splice_init(&trans->queue, &queue);
 771
 772        while ((skb = __skb_dequeue(&queue)))
 773                XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net,
 774                                               NULL, skb);
 775}
 776
 777int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
 778                         int (*finish)(struct net *, struct sock *,
 779                                       struct sk_buff *))
 780{
 781        struct xfrm_trans_tasklet *trans;
 782
 783        trans = this_cpu_ptr(&xfrm_trans_tasklet);
 784
 785        if (skb_queue_len(&trans->queue) >= netdev_max_backlog)
 786                return -ENOBUFS;
 787
 788        BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));
 789
 790        XFRM_TRANS_SKB_CB(skb)->finish = finish;
 791        XFRM_TRANS_SKB_CB(skb)->net = net;
 792        __skb_queue_tail(&trans->queue, skb);
 793        tasklet_schedule(&trans->tasklet);
 794        return 0;
 795}
 796EXPORT_SYMBOL(xfrm_trans_queue_net);
 797
 798int xfrm_trans_queue(struct sk_buff *skb,
 799                     int (*finish)(struct net *, struct sock *,
 800                                   struct sk_buff *))
 801{
 802        return xfrm_trans_queue_net(dev_net(skb->dev), skb, finish);
 803}
 804EXPORT_SYMBOL(xfrm_trans_queue);
 805
 806void __init xfrm_input_init(void)
 807{
 808        int err;
 809        int i;
 810
 811        init_dummy_netdev(&xfrm_napi_dev);
 812        err = gro_cells_init(&gro_cells, &xfrm_napi_dev);
 813        if (err)
 814                gro_cells.cells = NULL;
 815
 816        for_each_possible_cpu(i) {
 817                struct xfrm_trans_tasklet *trans;
 818
 819                trans = &per_cpu(xfrm_trans_tasklet, i);
 820                __skb_queue_head_init(&trans->queue);
 821                tasklet_setup(&trans->tasklet, xfrm_trans_reinject);
 822        }
 823}
 824