2 * net/sched/cls_flower.c Flower classifier
4 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
12 #include <linux/kernel.h>
13 #include <linux/init.h>
14 #include <linux/module.h>
15 #include <linux/rhashtable.h>
16 #include <linux/workqueue.h>
18 #include <linux/if_ether.h>
19 #include <linux/in6.h>
22 #include <net/sch_generic.h>
23 #include <net/pkt_cls.h>
25 #include <net/flow_dissector.h>
28 #include <net/dst_metadata.h>
32 struct flow_dissector_key_control control;
33 struct flow_dissector_key_control enc_control;
34 struct flow_dissector_key_basic basic;
35 struct flow_dissector_key_eth_addrs eth;
36 struct flow_dissector_key_vlan vlan;
38 struct flow_dissector_key_ipv4_addrs ipv4;
39 struct flow_dissector_key_ipv6_addrs ipv6;
41 struct flow_dissector_key_ports tp;
42 struct flow_dissector_key_keyid enc_key_id;
44 struct flow_dissector_key_ipv4_addrs enc_ipv4;
45 struct flow_dissector_key_ipv6_addrs enc_ipv6;
47 struct flow_dissector_key_ports enc_tp;
48 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
50 struct fl_flow_mask_range {
51 unsigned short int start;
52 unsigned short int end;
56 struct fl_flow_key key;
57 struct fl_flow_mask_range range;
63 struct fl_flow_mask mask;
64 struct flow_dissector dissector;
67 struct list_head filters;
68 struct rhashtable_params ht_params;
70 struct work_struct work;
75 struct cls_fl_filter {
76 struct rhash_head ht_node;
77 struct fl_flow_key mkey;
79 struct tcf_result res;
80 struct fl_flow_key key;
81 struct list_head list;
85 struct tc_to_netdev tc;
86 struct net_device *hw_dev;
89 static unsigned short int fl_mask_range(const struct fl_flow_mask *mask)
91 return mask->range.end - mask->range.start;
94 static void fl_mask_update_range(struct fl_flow_mask *mask)
96 const u8 *bytes = (const u8 *) &mask->key;
97 size_t size = sizeof(mask->key);
98 size_t i, first = 0, last = size - 1;
100 for (i = 0; i < sizeof(mask->key); i++) {
107 mask->range.start = rounddown(first, sizeof(long));
108 mask->range.end = roundup(last + 1, sizeof(long));
111 static void *fl_key_get_start(struct fl_flow_key *key,
112 const struct fl_flow_mask *mask)
114 return (u8 *) key + mask->range.start;
117 static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key,
118 struct fl_flow_mask *mask)
120 const long *lkey = fl_key_get_start(key, mask);
121 const long *lmask = fl_key_get_start(&mask->key, mask);
122 long *lmkey = fl_key_get_start(mkey, mask);
125 for (i = 0; i < fl_mask_range(mask); i += sizeof(long))
126 *lmkey++ = *lkey++ & *lmask++;
129 static void fl_clear_masked_range(struct fl_flow_key *key,
130 struct fl_flow_mask *mask)
132 memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask));
135 static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
136 struct tcf_result *res)
138 struct cls_fl_head *head = rcu_dereference_bh(tp->root);
139 struct cls_fl_filter *f;
140 struct fl_flow_key skb_key;
141 struct fl_flow_key skb_mkey;
142 struct ip_tunnel_info *info;
144 if (!atomic_read(&head->ht.nelems))
147 fl_clear_masked_range(&skb_key, &head->mask);
149 info = skb_tunnel_info(skb);
151 struct ip_tunnel_key *key = &info->key;
153 switch (ip_tunnel_info_af(info)) {
155 skb_key.enc_ipv4.src = key->u.ipv4.src;
156 skb_key.enc_ipv4.dst = key->u.ipv4.dst;
159 skb_key.enc_ipv6.src = key->u.ipv6.src;
160 skb_key.enc_ipv6.dst = key->u.ipv6.dst;
164 skb_key.enc_key_id.keyid = tunnel_id_to_key32(key->tun_id);
165 skb_key.enc_tp.src = key->tp_src;
166 skb_key.enc_tp.dst = key->tp_dst;
169 skb_key.indev_ifindex = skb->skb_iif;
170 /* skb_flow_dissect() does not set n_proto in case an unknown protocol,
171 * so do it rather here.
173 skb_key.basic.n_proto = skb->protocol;
174 skb_flow_dissect(skb, &head->dissector, &skb_key, 0);
176 fl_set_masked_key(&skb_mkey, &skb_key, &head->mask);
178 f = rhashtable_lookup_fast(&head->ht,
179 fl_key_get_start(&skb_mkey, &head->mask),
181 if (f && !tc_skip_sw(f->flags)) {
183 return tcf_exts_exec(skb, &f->exts, res);
188 static int fl_init(struct tcf_proto *tp)
190 struct cls_fl_head *head;
192 head = kzalloc(sizeof(*head), GFP_KERNEL);
196 INIT_LIST_HEAD_RCU(&head->filters);
197 rcu_assign_pointer(tp->root, head);
202 static void fl_destroy_filter(struct rcu_head *head)
204 struct cls_fl_filter *f = container_of(head, struct cls_fl_filter, rcu);
206 tcf_exts_destroy(&f->exts);
210 static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f)
212 struct tc_cls_flower_offload offload = {0};
213 struct net_device *dev = f->hw_dev;
214 struct tc_to_netdev *tc = &f->tc;
216 if (!tc_can_offload(dev, tp))
219 offload.command = TC_CLSFLOWER_DESTROY;
220 offload.cookie = (unsigned long)f;
222 tc->type = TC_SETUP_CLSFLOWER;
223 tc->cls_flower = &offload;
225 dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, tc);
228 static int fl_hw_replace_filter(struct tcf_proto *tp,
229 struct flow_dissector *dissector,
230 struct fl_flow_key *mask,
231 struct cls_fl_filter *f)
233 struct net_device *dev = tp->q->dev_queue->dev;
234 struct tc_cls_flower_offload offload = {0};
235 struct tc_to_netdev *tc = &f->tc;
238 if (!tc_can_offload(dev, tp)) {
239 if (tcf_exts_get_dev(dev, &f->exts, &f->hw_dev) ||
240 (f->hw_dev && !tc_can_offload(f->hw_dev, tp))) {
242 return tc_skip_sw(f->flags) ? -EINVAL : 0;
245 tc->egress_dev = true;
250 offload.command = TC_CLSFLOWER_REPLACE;
251 offload.cookie = (unsigned long)f;
252 offload.dissector = dissector;
254 offload.key = &f->key;
255 offload.exts = &f->exts;
257 tc->type = TC_SETUP_CLSFLOWER;
258 tc->cls_flower = &offload;
260 err = dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol,
263 if (tc_skip_sw(f->flags))
268 static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f)
270 struct tc_cls_flower_offload offload = {0};
271 struct net_device *dev = f->hw_dev;
272 struct tc_to_netdev *tc = &f->tc;
274 if (!tc_can_offload(dev, tp))
277 offload.command = TC_CLSFLOWER_STATS;
278 offload.cookie = (unsigned long)f;
279 offload.exts = &f->exts;
281 tc->type = TC_SETUP_CLSFLOWER;
282 tc->cls_flower = &offload;
284 dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, tc);
287 static void __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f)
289 list_del_rcu(&f->list);
290 if (!tc_skip_hw(f->flags))
291 fl_hw_destroy_filter(tp, f);
292 tcf_unbind_filter(tp, &f->res);
293 call_rcu(&f->rcu, fl_destroy_filter);
296 static void fl_destroy_sleepable(struct work_struct *work)
298 struct cls_fl_head *head = container_of(work, struct cls_fl_head,
300 if (head->mask_assigned)
301 rhashtable_destroy(&head->ht);
303 module_put(THIS_MODULE);
306 static void fl_destroy_rcu(struct rcu_head *rcu)
308 struct cls_fl_head *head = container_of(rcu, struct cls_fl_head, rcu);
310 INIT_WORK(&head->work, fl_destroy_sleepable);
311 schedule_work(&head->work);
314 static bool fl_destroy(struct tcf_proto *tp, bool force)
316 struct cls_fl_head *head = rtnl_dereference(tp->root);
317 struct cls_fl_filter *f, *next;
319 if (!force && !list_empty(&head->filters))
322 list_for_each_entry_safe(f, next, &head->filters, list)
325 __module_get(THIS_MODULE);
326 call_rcu(&head->rcu, fl_destroy_rcu);
331 static unsigned long fl_get(struct tcf_proto *tp, u32 handle)
333 struct cls_fl_head *head = rtnl_dereference(tp->root);
334 struct cls_fl_filter *f;
336 list_for_each_entry(f, &head->filters, list)
337 if (f->handle == handle)
338 return (unsigned long) f;
342 static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = {
343 [TCA_FLOWER_UNSPEC] = { .type = NLA_UNSPEC },
344 [TCA_FLOWER_CLASSID] = { .type = NLA_U32 },
345 [TCA_FLOWER_INDEV] = { .type = NLA_STRING,
347 [TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN },
348 [TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN },
349 [TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN },
350 [TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN },
351 [TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 },
352 [TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 },
353 [TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 },
354 [TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 },
355 [TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 },
356 [TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 },
357 [TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) },
358 [TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
359 [TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) },
360 [TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
361 [TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 },
362 [TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 },
363 [TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 },
364 [TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 },
365 [TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 },
366 [TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 },
367 [TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 },
368 [TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 },
369 [TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 },
370 [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 },
371 [TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 },
372 [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 },
373 [TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) },
374 [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
375 [TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) },
376 [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
377 [TCA_FLOWER_KEY_TCP_SRC_MASK] = { .type = NLA_U16 },
378 [TCA_FLOWER_KEY_TCP_DST_MASK] = { .type = NLA_U16 },
379 [TCA_FLOWER_KEY_UDP_SRC_MASK] = { .type = NLA_U16 },
380 [TCA_FLOWER_KEY_UDP_DST_MASK] = { .type = NLA_U16 },
381 [TCA_FLOWER_KEY_SCTP_SRC_MASK] = { .type = NLA_U16 },
382 [TCA_FLOWER_KEY_SCTP_DST_MASK] = { .type = NLA_U16 },
383 [TCA_FLOWER_KEY_SCTP_SRC] = { .type = NLA_U16 },
384 [TCA_FLOWER_KEY_SCTP_DST] = { .type = NLA_U16 },
385 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT] = { .type = NLA_U16 },
386 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK] = { .type = NLA_U16 },
387 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT] = { .type = NLA_U16 },
388 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK] = { .type = NLA_U16 },
389 [TCA_FLOWER_KEY_FLAGS] = { .type = NLA_U32 },
390 [TCA_FLOWER_KEY_FLAGS_MASK] = { .type = NLA_U32 },
393 static void fl_set_key_val(struct nlattr **tb,
394 void *val, int val_type,
395 void *mask, int mask_type, int len)
399 memcpy(val, nla_data(tb[val_type]), len);
400 if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type])
401 memset(mask, 0xff, len);
403 memcpy(mask, nla_data(tb[mask_type]), len);
406 static void fl_set_key_vlan(struct nlattr **tb,
407 struct flow_dissector_key_vlan *key_val,
408 struct flow_dissector_key_vlan *key_mask)
410 #define VLAN_PRIORITY_MASK 0x7
412 if (tb[TCA_FLOWER_KEY_VLAN_ID]) {
414 nla_get_u16(tb[TCA_FLOWER_KEY_VLAN_ID]) & VLAN_VID_MASK;
415 key_mask->vlan_id = VLAN_VID_MASK;
417 if (tb[TCA_FLOWER_KEY_VLAN_PRIO]) {
418 key_val->vlan_priority =
419 nla_get_u8(tb[TCA_FLOWER_KEY_VLAN_PRIO]) &
421 key_mask->vlan_priority = VLAN_PRIORITY_MASK;
425 static void fl_set_key_flag(u32 flower_key, u32 flower_mask,
426 u32 *dissector_key, u32 *dissector_mask,
427 u32 flower_flag_bit, u32 dissector_flag_bit)
429 if (flower_mask & flower_flag_bit) {
430 *dissector_mask |= dissector_flag_bit;
431 if (flower_key & flower_flag_bit)
432 *dissector_key |= dissector_flag_bit;
436 static void fl_set_key_flags(struct nlattr **tb,
437 u32 *flags_key, u32 *flags_mask)
441 if (!tb[TCA_FLOWER_KEY_FLAGS])
444 key = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS]));
446 if (!tb[TCA_FLOWER_KEY_FLAGS_MASK])
449 mask = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS_MASK]));
454 fl_set_key_flag(key, mask, flags_key, flags_mask,
455 TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
458 static int fl_set_key(struct net *net, struct nlattr **tb,
459 struct fl_flow_key *key, struct fl_flow_key *mask)
462 #ifdef CONFIG_NET_CLS_IND
463 if (tb[TCA_FLOWER_INDEV]) {
464 int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV]);
467 key->indev_ifindex = err;
468 mask->indev_ifindex = 0xffffffff;
472 fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
473 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
474 sizeof(key->eth.dst));
475 fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
476 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
477 sizeof(key->eth.src));
479 if (tb[TCA_FLOWER_KEY_ETH_TYPE]) {
480 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]);
482 if (ethertype == htons(ETH_P_8021Q)) {
483 fl_set_key_vlan(tb, &key->vlan, &mask->vlan);
484 fl_set_key_val(tb, &key->basic.n_proto,
485 TCA_FLOWER_KEY_VLAN_ETH_TYPE,
486 &mask->basic.n_proto, TCA_FLOWER_UNSPEC,
487 sizeof(key->basic.n_proto));
489 key->basic.n_proto = ethertype;
490 mask->basic.n_proto = cpu_to_be16(~0);
494 if (key->basic.n_proto == htons(ETH_P_IP) ||
495 key->basic.n_proto == htons(ETH_P_IPV6)) {
496 fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
497 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
498 sizeof(key->basic.ip_proto));
501 if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) {
502 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
503 fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
504 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
505 sizeof(key->ipv4.src));
506 fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
507 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
508 sizeof(key->ipv4.dst));
509 } else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) {
510 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
511 fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
512 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
513 sizeof(key->ipv6.src));
514 fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
515 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
516 sizeof(key->ipv6.dst));
519 if (key->basic.ip_proto == IPPROTO_TCP) {
520 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
521 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
522 sizeof(key->tp.src));
523 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
524 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
525 sizeof(key->tp.dst));
526 } else if (key->basic.ip_proto == IPPROTO_UDP) {
527 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
528 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
529 sizeof(key->tp.src));
530 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
531 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
532 sizeof(key->tp.dst));
533 } else if (key->basic.ip_proto == IPPROTO_SCTP) {
534 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
535 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
536 sizeof(key->tp.src));
537 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
538 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
539 sizeof(key->tp.dst));
542 if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] ||
543 tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) {
544 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
545 fl_set_key_val(tb, &key->enc_ipv4.src,
546 TCA_FLOWER_KEY_ENC_IPV4_SRC,
548 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
549 sizeof(key->enc_ipv4.src));
550 fl_set_key_val(tb, &key->enc_ipv4.dst,
551 TCA_FLOWER_KEY_ENC_IPV4_DST,
553 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
554 sizeof(key->enc_ipv4.dst));
557 if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] ||
558 tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) {
559 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
560 fl_set_key_val(tb, &key->enc_ipv6.src,
561 TCA_FLOWER_KEY_ENC_IPV6_SRC,
563 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
564 sizeof(key->enc_ipv6.src));
565 fl_set_key_val(tb, &key->enc_ipv6.dst,
566 TCA_FLOWER_KEY_ENC_IPV6_DST,
568 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
569 sizeof(key->enc_ipv6.dst));
572 fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID,
573 &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC,
574 sizeof(key->enc_key_id.keyid));
576 fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
577 &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
578 sizeof(key->enc_tp.src));
580 fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
581 &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
582 sizeof(key->enc_tp.dst));
584 fl_set_key_flags(tb, &key->control.flags, &mask->control.flags);
589 static bool fl_mask_eq(struct fl_flow_mask *mask1,
590 struct fl_flow_mask *mask2)
592 const long *lmask1 = fl_key_get_start(&mask1->key, mask1);
593 const long *lmask2 = fl_key_get_start(&mask2->key, mask2);
595 return !memcmp(&mask1->range, &mask2->range, sizeof(mask1->range)) &&
596 !memcmp(lmask1, lmask2, fl_mask_range(mask1));
599 static const struct rhashtable_params fl_ht_params = {
600 .key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */
601 .head_offset = offsetof(struct cls_fl_filter, ht_node),
602 .automatic_shrinking = true,
605 static int fl_init_hashtable(struct cls_fl_head *head,
606 struct fl_flow_mask *mask)
608 head->ht_params = fl_ht_params;
609 head->ht_params.key_len = fl_mask_range(mask);
610 head->ht_params.key_offset += mask->range.start;
612 return rhashtable_init(&head->ht, &head->ht_params);
615 #define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member)
616 #define FL_KEY_MEMBER_SIZE(member) (sizeof(((struct fl_flow_key *) 0)->member))
618 #define FL_KEY_IS_MASKED(mask, member) \
619 memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member), \
620 0, FL_KEY_MEMBER_SIZE(member)) \
622 #define FL_KEY_SET(keys, cnt, id, member) \
624 keys[cnt].key_id = id; \
625 keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member); \
629 #define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member) \
631 if (FL_KEY_IS_MASKED(mask, member)) \
632 FL_KEY_SET(keys, cnt, id, member); \
635 static void fl_init_dissector(struct cls_fl_head *head,
636 struct fl_flow_mask *mask)
638 struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX];
641 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control);
642 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic);
643 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
644 FLOW_DISSECTOR_KEY_ETH_ADDRS, eth);
645 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
646 FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
647 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
648 FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
649 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
650 FLOW_DISSECTOR_KEY_PORTS, tp);
651 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
652 FLOW_DISSECTOR_KEY_VLAN, vlan);
653 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
654 FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
655 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
656 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4);
657 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
658 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6);
659 if (FL_KEY_IS_MASKED(&mask->key, enc_ipv4) ||
660 FL_KEY_IS_MASKED(&mask->key, enc_ipv6))
661 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL,
663 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
664 FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp);
666 skb_flow_dissector_init(&head->dissector, keys, cnt);
669 static int fl_check_assign_mask(struct cls_fl_head *head,
670 struct fl_flow_mask *mask)
674 if (head->mask_assigned) {
675 if (!fl_mask_eq(&head->mask, mask))
681 /* Mask is not assigned yet. So assign it and init hashtable
684 err = fl_init_hashtable(head, mask);
687 memcpy(&head->mask, mask, sizeof(head->mask));
688 head->mask_assigned = true;
690 fl_init_dissector(head, mask);
695 static int fl_set_parms(struct net *net, struct tcf_proto *tp,
696 struct cls_fl_filter *f, struct fl_flow_mask *mask,
697 unsigned long base, struct nlattr **tb,
698 struct nlattr *est, bool ovr)
703 err = tcf_exts_init(&e, TCA_FLOWER_ACT, 0);
706 err = tcf_exts_validate(net, tp, tb, est, &e, ovr);
710 if (tb[TCA_FLOWER_CLASSID]) {
711 f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]);
712 tcf_bind_filter(tp, &f->res, base);
715 err = fl_set_key(net, tb, &f->key, &mask->key);
719 fl_mask_update_range(mask);
720 fl_set_masked_key(&f->mkey, &f->key, mask);
722 tcf_exts_change(tp, &f->exts, &e);
726 tcf_exts_destroy(&e);
730 static u32 fl_grab_new_handle(struct tcf_proto *tp,
731 struct cls_fl_head *head)
733 unsigned int i = 0x80000000;
737 if (++head->hgen == 0x7FFFFFFF)
739 } while (--i > 0 && fl_get(tp, head->hgen));
741 if (unlikely(i == 0)) {
742 pr_err("Insufficient number of handles\n");
751 static int fl_change(struct net *net, struct sk_buff *in_skb,
752 struct tcf_proto *tp, unsigned long base,
753 u32 handle, struct nlattr **tca,
754 unsigned long *arg, bool ovr)
756 struct cls_fl_head *head = rtnl_dereference(tp->root);
757 struct cls_fl_filter *fold = (struct cls_fl_filter *) *arg;
758 struct cls_fl_filter *fnew;
759 struct nlattr *tb[TCA_FLOWER_MAX + 1];
760 struct fl_flow_mask mask = {};
763 if (!tca[TCA_OPTIONS])
766 err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS], fl_policy);
770 if (fold && handle && fold->handle != handle)
773 fnew = kzalloc(sizeof(*fnew), GFP_KERNEL);
777 err = tcf_exts_init(&fnew->exts, TCA_FLOWER_ACT, 0);
782 handle = fl_grab_new_handle(tp, head);
788 fnew->handle = handle;
790 if (tb[TCA_FLOWER_FLAGS]) {
791 fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]);
793 if (!tc_flags_valid(fnew->flags)) {
799 err = fl_set_parms(net, tp, fnew, &mask, base, tb, tca[TCA_RATE], ovr);
803 err = fl_check_assign_mask(head, &mask);
807 if (!tc_skip_sw(fnew->flags)) {
808 err = rhashtable_insert_fast(&head->ht, &fnew->ht_node,
814 if (!tc_skip_hw(fnew->flags)) {
815 err = fl_hw_replace_filter(tp,
824 if (!tc_skip_sw(fold->flags))
825 rhashtable_remove_fast(&head->ht, &fold->ht_node,
827 if (!tc_skip_hw(fold->flags))
828 fl_hw_destroy_filter(tp, fold);
831 *arg = (unsigned long) fnew;
834 list_replace_rcu(&fold->list, &fnew->list);
835 tcf_unbind_filter(tp, &fold->res);
836 call_rcu(&fold->rcu, fl_destroy_filter);
838 list_add_tail_rcu(&fnew->list, &head->filters);
844 tcf_exts_destroy(&fnew->exts);
849 static int fl_delete(struct tcf_proto *tp, unsigned long arg)
851 struct cls_fl_head *head = rtnl_dereference(tp->root);
852 struct cls_fl_filter *f = (struct cls_fl_filter *) arg;
854 if (!tc_skip_sw(f->flags))
855 rhashtable_remove_fast(&head->ht, &f->ht_node,
861 static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg)
863 struct cls_fl_head *head = rtnl_dereference(tp->root);
864 struct cls_fl_filter *f;
866 list_for_each_entry_rcu(f, &head->filters, list) {
867 if (arg->count < arg->skip)
869 if (arg->fn(tp, (unsigned long) f, arg) < 0) {
878 static int fl_dump_key_val(struct sk_buff *skb,
879 void *val, int val_type,
880 void *mask, int mask_type, int len)
884 if (!memchr_inv(mask, 0, len))
886 err = nla_put(skb, val_type, len, val);
889 if (mask_type != TCA_FLOWER_UNSPEC) {
890 err = nla_put(skb, mask_type, len, mask);
897 static int fl_dump_key_vlan(struct sk_buff *skb,
898 struct flow_dissector_key_vlan *vlan_key,
899 struct flow_dissector_key_vlan *vlan_mask)
903 if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask)))
905 if (vlan_mask->vlan_id) {
906 err = nla_put_u16(skb, TCA_FLOWER_KEY_VLAN_ID,
911 if (vlan_mask->vlan_priority) {
912 err = nla_put_u8(skb, TCA_FLOWER_KEY_VLAN_PRIO,
913 vlan_key->vlan_priority);
920 static void fl_get_key_flag(u32 dissector_key, u32 dissector_mask,
921 u32 *flower_key, u32 *flower_mask,
922 u32 flower_flag_bit, u32 dissector_flag_bit)
924 if (dissector_mask & dissector_flag_bit) {
925 *flower_mask |= flower_flag_bit;
926 if (dissector_key & dissector_flag_bit)
927 *flower_key |= flower_flag_bit;
931 static int fl_dump_key_flags(struct sk_buff *skb, u32 flags_key, u32 flags_mask)
937 if (!memchr_inv(&flags_mask, 0, sizeof(flags_mask)))
943 fl_get_key_flag(flags_key, flags_mask, &key, &mask,
944 TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
946 _key = cpu_to_be32(key);
947 _mask = cpu_to_be32(mask);
949 err = nla_put(skb, TCA_FLOWER_KEY_FLAGS, 4, &_key);
953 return nla_put(skb, TCA_FLOWER_KEY_FLAGS_MASK, 4, &_mask);
956 static int fl_dump(struct net *net, struct tcf_proto *tp, unsigned long fh,
957 struct sk_buff *skb, struct tcmsg *t)
959 struct cls_fl_head *head = rtnl_dereference(tp->root);
960 struct cls_fl_filter *f = (struct cls_fl_filter *) fh;
962 struct fl_flow_key *key, *mask;
967 t->tcm_handle = f->handle;
969 nest = nla_nest_start(skb, TCA_OPTIONS);
971 goto nla_put_failure;
973 if (f->res.classid &&
974 nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid))
975 goto nla_put_failure;
978 mask = &head->mask.key;
980 if (mask->indev_ifindex) {
981 struct net_device *dev;
983 dev = __dev_get_by_index(net, key->indev_ifindex);
984 if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name))
985 goto nla_put_failure;
988 if (!tc_skip_hw(f->flags))
989 fl_hw_update_stats(tp, f);
991 if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
992 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
993 sizeof(key->eth.dst)) ||
994 fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
995 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
996 sizeof(key->eth.src)) ||
997 fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE,
998 &mask->basic.n_proto, TCA_FLOWER_UNSPEC,
999 sizeof(key->basic.n_proto)))
1000 goto nla_put_failure;
1002 if (fl_dump_key_vlan(skb, &key->vlan, &mask->vlan))
1003 goto nla_put_failure;
1005 if ((key->basic.n_proto == htons(ETH_P_IP) ||
1006 key->basic.n_proto == htons(ETH_P_IPV6)) &&
1007 fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
1008 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
1009 sizeof(key->basic.ip_proto)))
1010 goto nla_put_failure;
1012 if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
1013 (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
1014 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
1015 sizeof(key->ipv4.src)) ||
1016 fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
1017 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
1018 sizeof(key->ipv4.dst))))
1019 goto nla_put_failure;
1020 else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
1021 (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
1022 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
1023 sizeof(key->ipv6.src)) ||
1024 fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
1025 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
1026 sizeof(key->ipv6.dst))))
1027 goto nla_put_failure;
1029 if (key->basic.ip_proto == IPPROTO_TCP &&
1030 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
1031 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
1032 sizeof(key->tp.src)) ||
1033 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
1034 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
1035 sizeof(key->tp.dst))))
1036 goto nla_put_failure;
1037 else if (key->basic.ip_proto == IPPROTO_UDP &&
1038 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
1039 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
1040 sizeof(key->tp.src)) ||
1041 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
1042 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
1043 sizeof(key->tp.dst))))
1044 goto nla_put_failure;
1045 else if (key->basic.ip_proto == IPPROTO_SCTP &&
1046 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
1047 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
1048 sizeof(key->tp.src)) ||
1049 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
1050 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
1051 sizeof(key->tp.dst))))
1052 goto nla_put_failure;
1054 if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
1055 (fl_dump_key_val(skb, &key->enc_ipv4.src,
1056 TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src,
1057 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
1058 sizeof(key->enc_ipv4.src)) ||
1059 fl_dump_key_val(skb, &key->enc_ipv4.dst,
1060 TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst,
1061 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
1062 sizeof(key->enc_ipv4.dst))))
1063 goto nla_put_failure;
1064 else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
1065 (fl_dump_key_val(skb, &key->enc_ipv6.src,
1066 TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src,
1067 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
1068 sizeof(key->enc_ipv6.src)) ||
1069 fl_dump_key_val(skb, &key->enc_ipv6.dst,
1070 TCA_FLOWER_KEY_ENC_IPV6_DST,
1071 &mask->enc_ipv6.dst,
1072 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
1073 sizeof(key->enc_ipv6.dst))))
1074 goto nla_put_failure;
1076 if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID,
1077 &mask->enc_key_id, TCA_FLOWER_UNSPEC,
1078 sizeof(key->enc_key_id)) ||
1079 fl_dump_key_val(skb, &key->enc_tp.src,
1080 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
1082 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
1083 sizeof(key->enc_tp.src)) ||
1084 fl_dump_key_val(skb, &key->enc_tp.dst,
1085 TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
1087 TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
1088 sizeof(key->enc_tp.dst)))
1089 goto nla_put_failure;
1091 if (fl_dump_key_flags(skb, key->control.flags, mask->control.flags))
1092 goto nla_put_failure;
1094 nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags);
1096 if (tcf_exts_dump(skb, &f->exts))
1097 goto nla_put_failure;
1099 nla_nest_end(skb, nest);
1101 if (tcf_exts_dump_stats(skb, &f->exts) < 0)
1102 goto nla_put_failure;
1107 nla_nest_cancel(skb, nest);
1111 static struct tcf_proto_ops cls_fl_ops __read_mostly = {
1113 .classify = fl_classify,
1115 .destroy = fl_destroy,
1117 .change = fl_change,
1118 .delete = fl_delete,
1121 .owner = THIS_MODULE,
1124 static int __init cls_fl_init(void)
1126 return register_tcf_proto_ops(&cls_fl_ops);
1129 static void __exit cls_fl_exit(void)
1131 unregister_tcf_proto_ops(&cls_fl_ops);
1134 module_init(cls_fl_init);
1135 module_exit(cls_fl_exit);
1137 MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
1138 MODULE_DESCRIPTION("Flower classifier");
1139 MODULE_LICENSE("GPL v2");