2 * Copyright (C) 2017 Netronome Systems, Inc.
4 * This software is dual licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree or the BSD 2-Clause License provided below. You have the
7 * option to license this software under the complete terms of either license.
9 * The BSD 2-Clause License:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * 1. Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * 2. Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/hash.h>
35 #include <linux/hashtable.h>
36 #include <linux/jhash.h>
37 #include <linux/vmalloc.h>
38 #include <net/pkt_cls.h>
42 #include "../nfp_app.h"
44 struct nfp_mask_id_table {
45 struct hlist_node link;
51 static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id)
53 struct nfp_flower_priv *priv = app->priv;
54 struct circ_buf *ring;
56 ring = &priv->stats_ids.free_list;
57 /* Check if buffer is full. */
58 if (!CIRC_SPACE(ring->head, ring->tail, NFP_FL_STATS_ENTRY_RS *
59 NFP_FL_STATS_ELEM_RS -
60 NFP_FL_STATS_ELEM_RS + 1))
63 memcpy(&ring->buf[ring->head], &stats_context_id, NFP_FL_STATS_ELEM_RS);
64 ring->head = (ring->head + NFP_FL_STATS_ELEM_RS) %
65 (NFP_FL_STATS_ENTRY_RS * NFP_FL_STATS_ELEM_RS);
70 static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id)
72 struct nfp_flower_priv *priv = app->priv;
73 u32 freed_stats_id, temp_stats_id;
74 struct circ_buf *ring;
76 ring = &priv->stats_ids.free_list;
77 freed_stats_id = NFP_FL_STATS_ENTRY_RS;
78 /* Check for unallocated entries first. */
79 if (priv->stats_ids.init_unalloc > 0) {
80 *stats_context_id = priv->stats_ids.init_unalloc - 1;
81 priv->stats_ids.init_unalloc--;
85 /* Check if buffer is empty. */
86 if (ring->head == ring->tail) {
87 *stats_context_id = freed_stats_id;
91 memcpy(&temp_stats_id, &ring->buf[ring->tail], NFP_FL_STATS_ELEM_RS);
92 *stats_context_id = temp_stats_id;
93 memcpy(&ring->buf[ring->tail], &freed_stats_id, NFP_FL_STATS_ELEM_RS);
94 ring->tail = (ring->tail + NFP_FL_STATS_ELEM_RS) %
95 (NFP_FL_STATS_ENTRY_RS * NFP_FL_STATS_ELEM_RS);
100 /* Must be called with either RTNL or rcu_read_lock */
101 struct nfp_fl_payload *
102 nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie,
103 struct net_device *netdev, __be32 host_ctx)
105 struct nfp_flower_priv *priv = app->priv;
106 struct nfp_fl_payload *flower_entry;
108 hash_for_each_possible_rcu(priv->flow_table, flower_entry, link,
110 if (flower_entry->tc_flower_cookie == tc_flower_cookie &&
111 (!netdev || flower_entry->ingress_dev == netdev) &&
112 (host_ctx == NFP_FL_STATS_CTX_DONT_CARE ||
113 flower_entry->meta.host_ctx_id == host_ctx))
120 nfp_flower_update_stats(struct nfp_app *app, struct nfp_fl_stats_frame *stats)
122 struct nfp_fl_payload *nfp_flow;
123 unsigned long flower_cookie;
125 flower_cookie = be64_to_cpu(stats->stats_cookie);
128 nfp_flow = nfp_flower_search_fl_table(app, flower_cookie, NULL,
129 stats->stats_con_id);
131 goto exit_rcu_unlock;
133 spin_lock(&nfp_flow->lock);
134 nfp_flow->stats.pkts += be32_to_cpu(stats->pkt_count);
135 nfp_flow->stats.bytes += be64_to_cpu(stats->byte_count);
136 nfp_flow->stats.used = jiffies;
137 spin_unlock(&nfp_flow->lock);
143 void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb)
145 unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb);
146 struct nfp_fl_stats_frame *stats_frame;
150 msg = nfp_flower_cmsg_get_data(skb);
152 stats_frame = (struct nfp_fl_stats_frame *)msg;
153 for (i = 0; i < msg_len / sizeof(*stats_frame); i++)
154 nfp_flower_update_stats(app, stats_frame + i);
157 static int nfp_release_mask_id(struct nfp_app *app, u8 mask_id)
159 struct nfp_flower_priv *priv = app->priv;
160 struct circ_buf *ring;
161 struct timespec64 now;
163 ring = &priv->mask_ids.mask_id_free_list;
164 /* Checking if buffer is full. */
165 if (CIRC_SPACE(ring->head, ring->tail, NFP_FLOWER_MASK_ENTRY_RS) == 0)
168 memcpy(&ring->buf[ring->head], &mask_id, NFP_FLOWER_MASK_ELEMENT_RS);
169 ring->head = (ring->head + NFP_FLOWER_MASK_ELEMENT_RS) %
170 (NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS);
172 getnstimeofday64(&now);
173 priv->mask_ids.last_used[mask_id] = now;
178 static int nfp_mask_alloc(struct nfp_app *app, u8 *mask_id)
180 struct nfp_flower_priv *priv = app->priv;
181 struct timespec64 delta, now;
182 struct circ_buf *ring;
183 u8 temp_id, freed_id;
185 ring = &priv->mask_ids.mask_id_free_list;
186 freed_id = NFP_FLOWER_MASK_ENTRY_RS - 1;
187 /* Checking for unallocated entries first. */
188 if (priv->mask_ids.init_unallocated > 0) {
189 *mask_id = priv->mask_ids.init_unallocated;
190 priv->mask_ids.init_unallocated--;
194 /* Checking if buffer is empty. */
195 if (ring->head == ring->tail)
198 memcpy(&temp_id, &ring->buf[ring->tail], NFP_FLOWER_MASK_ELEMENT_RS);
201 getnstimeofday64(&now);
202 delta = timespec64_sub(now, priv->mask_ids.last_used[*mask_id]);
204 if (timespec64_to_ns(&delta) < NFP_FL_MASK_REUSE_TIME_NS)
207 memcpy(&ring->buf[ring->tail], &freed_id, NFP_FLOWER_MASK_ELEMENT_RS);
208 ring->tail = (ring->tail + NFP_FLOWER_MASK_ELEMENT_RS) %
209 (NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS);
219 nfp_add_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
221 struct nfp_flower_priv *priv = app->priv;
222 struct nfp_mask_id_table *mask_entry;
223 unsigned long hash_key;
226 if (nfp_mask_alloc(app, &mask_id))
229 mask_entry = kmalloc(sizeof(*mask_entry), GFP_KERNEL);
231 nfp_release_mask_id(app, mask_id);
235 INIT_HLIST_NODE(&mask_entry->link);
236 mask_entry->mask_id = mask_id;
237 hash_key = jhash(mask_data, mask_len, priv->mask_id_seed);
238 mask_entry->hash_key = hash_key;
239 mask_entry->ref_cnt = 1;
240 hash_add(priv->mask_table, &mask_entry->link, hash_key);
245 static struct nfp_mask_id_table *
246 nfp_search_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
248 struct nfp_flower_priv *priv = app->priv;
249 struct nfp_mask_id_table *mask_entry;
250 unsigned long hash_key;
252 hash_key = jhash(mask_data, mask_len, priv->mask_id_seed);
254 hash_for_each_possible(priv->mask_table, mask_entry, link, hash_key)
255 if (mask_entry->hash_key == hash_key)
262 nfp_find_in_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len)
264 struct nfp_mask_id_table *mask_entry;
266 mask_entry = nfp_search_mask_table(app, mask_data, mask_len);
270 mask_entry->ref_cnt++;
272 /* Casting u8 to int for later use. */
273 return mask_entry->mask_id;
277 nfp_check_mask_add(struct nfp_app *app, char *mask_data, u32 mask_len,
278 u8 *meta_flags, u8 *mask_id)
282 id = nfp_find_in_mask_table(app, mask_data, mask_len);
284 id = nfp_add_mask_table(app, mask_data, mask_len);
287 *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK;
295 nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len,
296 u8 *meta_flags, u8 *mask_id)
298 struct nfp_mask_id_table *mask_entry;
300 mask_entry = nfp_search_mask_table(app, mask_data, mask_len);
305 *meta_flags &= ~NFP_FL_META_FLAG_MANAGE_MASK;
307 *mask_id = mask_entry->mask_id;
308 mask_entry->ref_cnt--;
309 if (!mask_entry->ref_cnt) {
310 hash_del(&mask_entry->link);
311 nfp_release_mask_id(app, *mask_id);
314 *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK;
320 int nfp_compile_flow_metadata(struct nfp_app *app,
321 struct tc_cls_flower_offload *flow,
322 struct nfp_fl_payload *nfp_flow,
323 struct net_device *netdev)
325 struct nfp_flower_priv *priv = app->priv;
326 struct nfp_fl_payload *check_entry;
330 if (nfp_get_stats_entry(app, &stats_cxt))
333 nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt);
334 nfp_flow->meta.host_cookie = cpu_to_be64(flow->cookie);
337 if (!nfp_check_mask_add(app, nfp_flow->mask_data,
338 nfp_flow->meta.mask_len,
339 &nfp_flow->meta.flags, &new_mask_id)) {
340 if (nfp_release_stats_entry(app, stats_cxt))
345 nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version);
346 priv->flower_version++;
348 /* Update flow payload with mask ids. */
349 nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id;
350 nfp_flow->stats.pkts = 0;
351 nfp_flow->stats.bytes = 0;
352 nfp_flow->stats.used = jiffies;
354 check_entry = nfp_flower_search_fl_table(app, flow->cookie, netdev,
355 NFP_FL_STATS_CTX_DONT_CARE);
357 if (nfp_release_stats_entry(app, stats_cxt))
360 if (!nfp_check_mask_remove(app, nfp_flow->mask_data,
361 nfp_flow->meta.mask_len,
371 int nfp_modify_flow_metadata(struct nfp_app *app,
372 struct nfp_fl_payload *nfp_flow)
374 struct nfp_flower_priv *priv = app->priv;
378 nfp_check_mask_remove(app, nfp_flow->mask_data,
379 nfp_flow->meta.mask_len, &nfp_flow->meta.flags,
382 nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version);
383 priv->flower_version++;
385 /* Update flow payload with mask ids. */
386 nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id;
388 /* Release the stats ctx id. */
389 temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id);
391 return nfp_release_stats_entry(app, temp_ctx_id);
394 int nfp_flower_metadata_init(struct nfp_app *app)
396 struct nfp_flower_priv *priv = app->priv;
398 hash_init(priv->mask_table);
399 hash_init(priv->flow_table);
400 get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed));
402 /* Init ring buffer and unallocated mask_ids. */
403 priv->mask_ids.mask_id_free_list.buf =
404 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
405 NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL);
406 if (!priv->mask_ids.mask_id_free_list.buf)
409 priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1;
411 /* Init timestamps for mask id*/
412 priv->mask_ids.last_used =
413 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
414 sizeof(*priv->mask_ids.last_used), GFP_KERNEL);
415 if (!priv->mask_ids.last_used)
416 goto err_free_mask_id;
418 /* Init ring buffer and unallocated stats_ids. */
419 priv->stats_ids.free_list.buf =
420 vmalloc(NFP_FL_STATS_ENTRY_RS * NFP_FL_STATS_ELEM_RS);
421 if (!priv->stats_ids.free_list.buf)
422 goto err_free_last_used;
424 priv->stats_ids.init_unalloc = NFP_FL_REPEATED_HASH_MAX;
429 kfree(priv->mask_ids.last_used);
431 kfree(priv->mask_ids.mask_id_free_list.buf);
435 void nfp_flower_metadata_cleanup(struct nfp_app *app)
437 struct nfp_flower_priv *priv = app->priv;
442 kfree(priv->mask_ids.mask_id_free_list.buf);
443 kfree(priv->mask_ids.last_used);
444 vfree(priv->stats_ids.free_list.buf);