Index: sys/amd64/vmm/net/art.c =================================================================== --- /dev/null +++ sys/amd64/vmm/net/art.c @@ -0,0 +1,1006 @@ +/* + * Copyright (c) 2012, Armon Dadgar + * Copyright (C) 2017 Joyent Inc. + * Copyright (C) 2017 Matthew Macy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the organization nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL ARMON DADGAR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include + +#if 0 +#ifdef __i386__ + #include +#else +#ifdef __amd64__ + #include +#endif +#endif +#endif + +MALLOC_DEFINE(M_ART, "ART", "adaptive rt"); +#define abort() panic("woops!") +/** + * Macros to manipulate pointer tags + */ +#define IS_LEAF(x) (((uintptr_t)x & 1)) +#define SET_LEAF(x) ((void*)((uintptr_t)x | 1)) +#define LEAF_RAW(x) ((art_leaf*)((void*)((uintptr_t)x & ~1))) + +/** + * Allocates a node of the given type, + * initializes to zero and sets the type. + */ + +uint16_t node_sizes[] = { + sizeof(art_node4), + sizeof(art_node16), + sizeof(art_node48), + sizeof(art_node256) +}; + +static art_node* +alloc_node(uint8_t type) +{ + art_node* n; + + MPASS(type <= NODE_MAX); + n = (art_node*)malloc(node_sizes[type], M_ART, M_ZERO|M_WAITOK); + n->type = type; + return (n); +} + +/** + * Initializes an ART tree + * @return 0 on success. + */ +int +art_tree_init(art_tree *t, uint32_t key_len) { + t->root = NULL; + t->size = 0; + t->key_len = key_len; + return 0; +} + +// Recursively destroys the tree +static void destroy_node(art_node *n) { + // Break if null + if (!n) return; + + // Special case leafs + if (IS_LEAF(n)) { + free(LEAF_RAW(n), M_ART); + return; + } + + // Handle each node type + int i; + union { + art_node4 *p1; + art_node16 *p2; + art_node48 *p3; + art_node256 *p4; + } p; + switch (n->type) { + case NODE4: + p.p1 = (art_node4*)n; + for (i=0;inum_children;i++) { + destroy_node(p.p1->children[i]); + } + break; + + case NODE16: + p.p2 = (art_node16*)n; + for (i=0;inum_children;i++) { + destroy_node(p.p2->children[i]); + } + break; + + case NODE48: + p.p3 = (art_node48*)n; + for (i=0;inum_children;i++) { + destroy_node(p.p3->children[i]); + } + break; + + case NODE256: + p.p4 = (art_node256*)n; + for (i=0;i<256;i++) { + if (p.p4->children[i]) + destroy_node(p.p4->children[i]); + } + break; + + default: + abort(); + } + + // Free ourself on the way up + free(n, M_ART); +} + +/** + * Destroys an ART tree + * @return 0 on success. + */ +int art_tree_destroy(art_tree *t) { + destroy_node(t->root); + return 0; +} + +/** + * Returns the size of the ART tree. + */ + +static art_node** find_child(art_node *n, unsigned char c) { + int i, mask, bitfield; + union { + art_node4 *p1; + art_node16 *p2; + art_node48 *p3; + art_node256 *p4; + } p; + switch (n->type) { + case NODE4: + p.p1 = (art_node4*)n; + for (i=0 ; i < n->num_children; i++) { + /* this cast works around a bug in gcc 5.1 when unrolling loops + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124 + */ + if (((unsigned char*)p.p1->keys)[i] == c) + return &p.p1->children[i]; + } + break; + + { + case NODE16: + p.p2 = (art_node16*)n; + + // support non-86 architectures + /* + #ifdef __i386__ + // Compare the key to all 16 stored keys + __m128i cmp; + cmp = _mm_cmpeq_epi8(_mm_set1_epi8(c), + _mm_loadu_si128((__m128i*)p.p2->keys)); + + // Use a mask to ignore children that don't exist + mask = (1 << n->num_children) - 1; + bitfield = _mm_movemask_epi8(cmp) & mask; + #else + #ifdef __amd64__ + // Compare the key to all 16 stored keys + __m128i cmp; + cmp = _mm_cmpeq_epi8(_mm_set1_epi8(c), + _mm_loadu_si128((__m128i*)p.p2->keys)); + + // Use a mask to ignore children that don't exist + mask = (1 << n->num_children) - 1; + bitfield = _mm_movemask_epi8(cmp) & mask; +#else + */ + // Compare the key to all 16 stored keys + bitfield = 0; + for (i = 0; i < 16; ++i) { + if (p.p2->keys[i] == c) + bitfield |= (1 << i); + } + + // Use a mask to ignore children that don't exist + mask = (1 << n->num_children) - 1; + bitfield &= mask; + /* + #endif + #endif + */ + /* + * If we have a match (any bit set) then we can + * return the pointer match using ctz to get + * the index. + */ + if (bitfield) + return &p.p2->children[__builtin_ctz(bitfield)]; + break; + } + + case NODE48: + p.p3 = (art_node48*)n; + i = p.p3->keys[c]; + if (i) + return &p.p3->children[i-1]; + break; + + case NODE256: + p.p4 = (art_node256*)n; + if (p.p4->children[c]) + return &p.p4->children[c]; + break; + + default: + panic("woops"); + } + return NULL; +} + +/** + * Returns the number of prefix characters shared between + * the key and node. + */ +static int check_prefix(const art_node *n, const unsigned char *key, int key_len, int depth) { + int max_cmp = min(min(n->partial_len, MAX_PREFIX_LEN), key_len - depth); + int idx; + for (idx=0; idx < max_cmp; idx++) { + if (n->partial[idx] != key[depth+idx]) + return idx; + } + return idx; +} + +#define XOR(type, a, b, n) (((const type *)(a))[n] ^ ((const type *)(b))[n]) +#define CMP6(a, b) XOR(uint16_t, a, b, 0) | XOR(uint16_t, a, b, 1) | XOR(uint16_t, a, b, 2) +#define CMP16(a, b) XOR(uint64_t, a, b, 0) | XOR(uint64_t, a, b, 1) +/** + * Checks if a leaf matches + * @return 0 on success. + */ +static inline int +leaf_matches(const art_leaf *n, const unsigned char *key, int key_len) +{ + switch (key_len) { + case 4: + return (uintptr_t)n->key == (uintptr_t)key; + case 6: + return CMP6(n->key, key); + case 16: + return CMP16(n->key, key); + default: + return memcmp(n->key, key, key_len); + } +} + +/** + * Searches for a value in the ART tree + * @arg t The tree + * @arg key The key + * @arg key_len The length of the key + * @return NULL if the item was not found, otherwise + * the value pointer is returned. + */ +void* art_search(const art_tree *t, const unsigned char *key) { + art_node **child; + art_node *n = t->root; + int prefix_len, depth = 0, key_len; + key_len = t->key_len; + while (n) { + // Might be a leaf + if (IS_LEAF(n)) { + + n = (art_node*)LEAF_RAW(n); + // Check if the expanded path matches + if (!leaf_matches((art_leaf*)n, key, key_len)) + return ((art_leaf*)n)->value; + return NULL; + } + + // Bail if the prefix does not match + if (n->partial_len) { + prefix_len = check_prefix(n, key, key_len, depth); + if (prefix_len != min(MAX_PREFIX_LEN, n->partial_len)) + return NULL; + depth = depth + n->partial_len; + } + + // Recursively search + child = find_child(n, key[depth]); + n = (child) ? *child : NULL; + depth++; + } + return NULL; +} + +// Find the minimum leaf under a node +static art_leaf* minimum(const art_node *n) { + // Handle base cases + if (!n) return NULL; + if (IS_LEAF(n)) return LEAF_RAW(n); + + int idx; + switch (n->type) { + case NODE4: + return minimum(((const art_node4*)n)->children[0]); + case NODE16: + return minimum(((const art_node16*)n)->children[0]); + case NODE48: + idx=0; + while (!((const art_node48*)n)->keys[idx]) idx++; + idx = ((const art_node48*)n)->keys[idx] - 1; + return minimum(((const art_node48*)n)->children[idx]); + case NODE256: + idx=0; + while (!((const art_node256*)n)->children[idx]) idx++; + return minimum(((const art_node256*)n)->children[idx]); + default: + abort(); + } +} + +// Find the maximum leaf under a node +static art_leaf* maximum(const art_node *n) { + // Handle base cases + if (!n) return NULL; + if (IS_LEAF(n)) return LEAF_RAW(n); + + int idx; + switch (n->type) { + case NODE4: + return maximum(((const art_node4*)n)->children[n->num_children-1]); + case NODE16: + return maximum(((const art_node16*)n)->children[n->num_children-1]); + case NODE48: + idx=255; + while (!((const art_node48*)n)->keys[idx]) idx--; + idx = ((const art_node48*)n)->keys[idx] - 1; + return maximum(((const art_node48*)n)->children[idx]); + case NODE256: + idx=255; + while (!((const art_node256*)n)->children[idx]) idx--; + return maximum(((const art_node256*)n)->children[idx]); + default: + abort(); + } +} + +/** + * Returns the minimum valued leaf + */ +art_leaf* art_minimum(art_tree *t) { + return minimum((art_node*)t->root); +} + +/** + * Returns the maximum valued leaf + */ +art_leaf* art_maximum(art_tree *t) { + return maximum((art_node*)t->root); +} + +static art_leaf* make_leaf(const unsigned char *key, int key_len, void *value) { + art_leaf *l = (art_leaf*)malloc(sizeof(art_leaf)+key_len, M_ART, M_ZERO|M_WAITOK); + l->value = value; + memcpy(l->key, key, key_len); + return l; +} + +static int +longest_common_prefix(art_leaf *l1, art_leaf *l2, int depth, int key_len) { + int max_cmp = key_len - depth; + int idx; + for (idx=0; idx < max_cmp; idx++) { + if (l1->key[depth+idx] != l2->key[depth+idx]) + return idx; + } + return idx; +} + +static void copy_header(art_node *dest, art_node *src) { + dest->num_children = src->num_children; + dest->partial_len = src->partial_len; + memcpy(dest->partial, src->partial, min(MAX_PREFIX_LEN, src->partial_len)); +} + +static void add_child256(art_node256 *n, art_node **ref, unsigned char c, void *child) { + (void)ref; + n->n.num_children++; + n->children[c] = (art_node*)child; +} + +static void add_child48(art_node48 *n, art_node **ref, unsigned char c, void *child) { + if (n->n.num_children < 48) { + int pos = 0; + while (n->children[pos]) pos++; + n->children[pos] = (art_node*)child; + n->keys[c] = pos + 1; + n->n.num_children++; + } else { + art_node256 *new_node = (art_node256*)alloc_node(NODE256); + for (int i=0;i<256;i++) { + if (n->keys[i]) { + new_node->children[i] = n->children[n->keys[i] - 1]; + } + } + copy_header((art_node*)new_node, (art_node*)n); + *ref = (art_node*)new_node; + free(n, M_ART); + add_child256(new_node, ref, c, child); + } +} + +static void add_child16(art_node16 *n, art_node **ref, unsigned char c, void *child) { + if (n->n.num_children < 16) { + unsigned mask = (1 << n->n.num_children) - 1; + + /* + // support non-x86 architectures + #ifdef __i386__ + __m128i cmp; + + // Compare the key to all 16 stored keys + cmp = _mm_cmplt_epi8(_mm_set1_epi8(c), + _mm_loadu_si128((__m128i*)n->keys)); + + // Use a mask to ignore children that don't exist + unsigned bitfield = _mm_movemask_epi8(cmp) & mask; + #else + #ifdef __amd64__ + __m128i cmp; + + // Compare the key to all 16 stored keys + cmp = _mm_cmplt_epi8(_mm_set1_epi8(c), + _mm_loadu_si128((__m128i*)n->keys)); + + // Use a mask to ignore children that don't exist + unsigned bitfield = _mm_movemask_epi8(cmp) & mask; +#else + */ + // Compare the key to all 16 stored keys + unsigned bitfield = 0; + for (short i = 0; i < 16; ++i) { + if (c < n->keys[i]) + bitfield |= (1 << i); + } + + // Use a mask to ignore children that don't exist + bitfield &= mask; + /* +#endif + #endif + */ + // Check if less than any + unsigned idx; + if (bitfield) { + idx = __builtin_ctz(bitfield); + memmove(n->keys+idx+1,n->keys+idx,n->n.num_children-idx); + memmove(n->children+idx+1,n->children+idx, + (n->n.num_children-idx)*sizeof(void*)); + } else + idx = n->n.num_children; + + // Set the child + n->keys[idx] = c; + n->children[idx] = (art_node*)child; + n->n.num_children++; + + } else { + art_node48 *new_node = (art_node48*)alloc_node(NODE48); + + // Copy the child pointers and populate the key map + memcpy(new_node->children, n->children, + sizeof(void*)*n->n.num_children); + for (int i=0;in.num_children;i++) { + new_node->keys[n->keys[i]] = i + 1; + } + copy_header((art_node*)new_node, (art_node*)n); + *ref = (art_node*)new_node; + free(n, M_ART); + add_child48(new_node, ref, c, child); + } +} + +static void add_child4(art_node4 *n, art_node **ref, unsigned char c, void *child) { + if (n->n.num_children < 4) { + int idx; + for (idx=0; idx < n->n.num_children; idx++) { + if (c < n->keys[idx]) break; + } + + // Shift to make room + memmove(n->keys+idx+1, n->keys+idx, n->n.num_children - idx); + memmove(n->children+idx+1, n->children+idx, + (n->n.num_children - idx)*sizeof(void*)); + + // Insert element + n->keys[idx] = c; + n->children[idx] = (art_node*)child; + n->n.num_children++; + + } else { + art_node16 *new_node = (art_node16*)alloc_node(NODE16); + + // Copy the child pointers and the key map + memcpy(new_node->children, n->children, + sizeof(void*)*n->n.num_children); + memcpy(new_node->keys, n->keys, + sizeof(unsigned char)*n->n.num_children); + copy_header((art_node*)new_node, (art_node*)n); + *ref = (art_node*)new_node; + free(n, M_ART); + add_child16(new_node, ref, c, child); + } +} + +static void add_child(art_node *n, art_node **ref, unsigned char c, void *child) { + switch (n->type) { + case NODE4: + return add_child4((art_node4*)n, ref, c, child); + case NODE16: + return add_child16((art_node16*)n, ref, c, child); + case NODE48: + return add_child48((art_node48*)n, ref, c, child); + case NODE256: + return add_child256((art_node256*)n, ref, c, child); + default: + abort(); + } +} + +/** + * Calculates the index at which the prefixes mismatch + */ +static int prefix_mismatch(const art_node *n, const unsigned char *key, int key_len, int depth) { + int max_cmp = min(min(MAX_PREFIX_LEN, n->partial_len), key_len - depth); + int idx; + for (idx=0; idx < max_cmp; idx++) { + if (n->partial[idx] != key[depth+idx]) + return idx; + } + + // If the prefix is short we can avoid finding a leaf + if (n->partial_len > MAX_PREFIX_LEN) { + // Prefix is longer than what we've checked, find a leaf + art_leaf *l = minimum(n); + max_cmp = key_len - depth; + for (; idx < max_cmp; idx++) { + if (l->key[idx+depth] != key[depth+idx]) + return idx; + } + } + return idx; +} + +static void* recursive_insert(art_node *n, art_node **ref, const unsigned char *key, int key_len, void *value, int depth, int *old) { + // If we are at a NULL node, inject a leaf + if (!n) { + *ref = (art_node*)SET_LEAF(make_leaf(key, key_len, value)); + return NULL; + } + + // If we are at a leaf, we need to replace it with a node + if (IS_LEAF(n)) { + art_leaf *l = LEAF_RAW(n); + + // Check if we are updating an existing value + if (!leaf_matches(l, key, key_len)) { + *old = 1; + void *old_val = l->value; + l->value = value; + return old_val; + } + + // New value, we must split the leaf into a node4 + art_node4 *new_node = (art_node4*)alloc_node(NODE4); + + // Create a new leaf + art_leaf *l2 = make_leaf(key, key_len, value); + + // Determine longest prefix + int longest_prefix = longest_common_prefix(l, l2, depth, key_len); + new_node->n.partial_len = longest_prefix; + memcpy(new_node->n.partial, key+depth, min(MAX_PREFIX_LEN, longest_prefix)); + // Add the leafs to the new node4 + *ref = (art_node*)new_node; + add_child4(new_node, ref, l->key[depth+longest_prefix], SET_LEAF(l)); + add_child4(new_node, ref, l2->key[depth+longest_prefix], SET_LEAF(l2)); + return NULL; + } + + // Check if given node has a prefix + if (n->partial_len) { + // Determine if the prefixes differ, since we need to split + int prefix_diff = prefix_mismatch(n, key, key_len, depth); + if ((uint32_t)prefix_diff >= n->partial_len) { + depth += n->partial_len; + goto RECURSE_SEARCH; + } + + // Create a new node + art_node4 *new_node = (art_node4*)alloc_node(NODE4); + *ref = (art_node*)new_node; + new_node->n.partial_len = prefix_diff; + memcpy(new_node->n.partial, n->partial, min(MAX_PREFIX_LEN, prefix_diff)); + + // Adjust the prefix of the old node + if (n->partial_len <= MAX_PREFIX_LEN) { + add_child4(new_node, ref, n->partial[prefix_diff], n); + n->partial_len -= (prefix_diff+1); + memmove(n->partial, n->partial+prefix_diff+1, + min(MAX_PREFIX_LEN, n->partial_len)); + } else { + n->partial_len -= (prefix_diff+1); + art_leaf *l = minimum(n); + add_child4(new_node, ref, l->key[depth+prefix_diff], n); + memcpy(n->partial, l->key+depth+prefix_diff+1, + min(MAX_PREFIX_LEN, n->partial_len)); + } + + // Insert the new leaf + art_leaf *l = make_leaf(key, key_len, value); + add_child4(new_node, ref, key[depth+prefix_diff], SET_LEAF(l)); + return NULL; + } + +RECURSE_SEARCH:; + + // Find a child to recurse to + art_node **child = find_child(n, key[depth]); + if (child) { + return recursive_insert(*child, child, key, key_len, value, depth+1, old); + } + + // No child, node goes within us + art_leaf *l = make_leaf(key, key_len, value); + add_child(n, ref, key[depth], SET_LEAF(l)); + return NULL; +} + +/** + * Inserts a new value into the ART tree + * @arg t The tree + * @arg key The key + * @arg key_len The length of the key + * @arg value Opaque value. + * @return NULL if the item was newly inserted, otherwise + * the old value pointer is returned. + */ +void* art_insert(art_tree *t, const unsigned char *key, void *value) { + int old_val = 0; + void *old = recursive_insert(t->root, &t->root, key, t->key_len, value, 0, &old_val); + if (!old_val) t->size++; + return old; +} + +static void remove_child256(art_node256 *n, art_node **ref, unsigned char c) { + n->children[c] = NULL; + n->n.num_children--; + + // Resize to a node48 on underflow, not immediately to prevent + // trashing if we sit on the 48/49 boundary + if (n->n.num_children == 37) { + art_node48 *new_node = (art_node48*)alloc_node(NODE48); + *ref = (art_node*)new_node; + copy_header((art_node*)new_node, (art_node*)n); + + int pos = 0; + for (int i=0;i<256;i++) { + if (n->children[i]) { + new_node->children[pos] = n->children[i]; + new_node->keys[i] = pos + 1; + pos++; + } + } + free(n, M_ART); + } +} + +static void remove_child48(art_node48 *n, art_node **ref, unsigned char c) { + int pos = n->keys[c]; + n->keys[c] = 0; + n->children[pos-1] = NULL; + n->n.num_children--; + + if (n->n.num_children == 12) { + art_node16 *new_node = (art_node16*)alloc_node(NODE16); + *ref = (art_node*)new_node; + copy_header((art_node*)new_node, (art_node*)n); + + int child = 0; + for (int i=0;i<256;i++) { + pos = n->keys[i]; + if (pos) { + new_node->keys[child] = i; + new_node->children[child] = n->children[pos - 1]; + child++; + } + } + free(n, M_ART); + } +} + +static void remove_child16(art_node16 *n, art_node **ref, art_node **l) { + int pos = l - n->children; + memmove(n->keys+pos, n->keys+pos+1, n->n.num_children - 1 - pos); + memmove(n->children+pos, n->children+pos+1, (n->n.num_children - 1 - pos)*sizeof(void*)); + n->n.num_children--; + + if (n->n.num_children == 3) { + art_node4 *new_node = (art_node4*)alloc_node(NODE4); + *ref = (art_node*)new_node; + copy_header((art_node*)new_node, (art_node*)n); + memcpy(new_node->keys, n->keys, 4); + memcpy(new_node->children, n->children, 4*sizeof(void*)); + free(n, M_ART); + } +} + +static void remove_child4(art_node4 *n, art_node **ref, art_node **l) { + int pos = l - n->children; + memmove(n->keys+pos, n->keys+pos+1, n->n.num_children - 1 - pos); + memmove(n->children+pos, n->children+pos+1, (n->n.num_children - 1 - pos)*sizeof(void*)); + n->n.num_children--; + + // Remove nodes with only a single child + if (n->n.num_children == 1) { + art_node *child = n->children[0]; + if (!IS_LEAF(child)) { + // Concatenate the prefixes + int prefix = n->n.partial_len; + if (prefix < MAX_PREFIX_LEN) { + n->n.partial[prefix] = n->keys[0]; + prefix++; + } + if (prefix < MAX_PREFIX_LEN) { + int sub_prefix = min(child->partial_len, MAX_PREFIX_LEN - prefix); + memcpy(n->n.partial+prefix, child->partial, sub_prefix); + prefix += sub_prefix; + } + + // Store the prefix in the child + memcpy(child->partial, n->n.partial, min(prefix, MAX_PREFIX_LEN)); + child->partial_len += n->n.partial_len + 1; + } + *ref = child; + free(n, M_ART); + } +} + +static void remove_child(art_node *n, art_node **ref, unsigned char c, art_node **l) { + switch (n->type) { + case NODE4: + return remove_child4((art_node4*)n, ref, l); + case NODE16: + return remove_child16((art_node16*)n, ref, l); + case NODE48: + return remove_child48((art_node48*)n, ref, c); + case NODE256: + return remove_child256((art_node256*)n, ref, c); + default: + abort(); + } +} + +static art_leaf* recursive_delete(art_node *n, art_node **ref, const unsigned char *key, int key_len, int depth) { + // Search terminated + if (!n) return NULL; + + // Handle hitting a leaf node + if (IS_LEAF(n)) { + art_leaf *l = LEAF_RAW(n); + if (!leaf_matches(l, key, key_len)) { + *ref = NULL; + return l; + } + return NULL; + } + + // Bail if the prefix does not match + if (n->partial_len) { + int prefix_len = check_prefix(n, key, key_len, depth); + if (prefix_len != min(MAX_PREFIX_LEN, n->partial_len)) { + return NULL; + } + depth = depth + n->partial_len; + } + + // Find child node + art_node **child = find_child(n, key[depth]); + if (!child) return NULL; + + // If the child is leaf, delete from this node + if (IS_LEAF(*child)) { + art_leaf *l = LEAF_RAW(*child); + if (!leaf_matches(l, key, key_len)) { + remove_child(n, ref, key[depth], child); + return l; + } + return NULL; + + // Recurse + } else { + return recursive_delete(*child, child, key, key_len, depth+1); + } +} + +/** + * Deletes a value from the ART tree + * @arg t The tree + * @arg key The key + * @arg key_len The length of the key + * @return NULL if the item was not found, otherwise + * the value pointer is returned. + */ +void* art_delete(art_tree *t, const unsigned char *key) { + art_leaf *l = recursive_delete(t->root, &t->root, key, t->key_len, 0); + if (l) { + t->size--; + void *old = l->value; + free(l, M_ART); + return old; + } + return NULL; +} + +// Recursively iterates over the tree +static int recursive_iter(art_node *n, art_callback cb, void *data, int key_len) { + // Handle base cases + if (!n) return 0; + if (IS_LEAF(n)) { + art_leaf *l = LEAF_RAW(n); + return cb(data, (const unsigned char*)l->key, key_len, l->value); + } + + int idx, res; + switch (n->type) { + case NODE4: + for (int i=0; i < n->num_children; i++) { + res = recursive_iter(((art_node4*)n)->children[i], cb, data, key_len); + if (res) return res; + } + break; + + case NODE16: + for (int i=0; i < n->num_children; i++) { + res = recursive_iter(((art_node16*)n)->children[i], cb, data, key_len); + if (res) return res; + } + break; + + case NODE48: + for (int i=0; i < 256; i++) { + idx = ((art_node48*)n)->keys[i]; + if (!idx) continue; + + res = recursive_iter(((art_node48*)n)->children[idx-1], cb, data, key_len); + if (res) return res; + } + break; + + case NODE256: + for (int i=0; i < 256; i++) { + if (!((art_node256*)n)->children[i]) continue; + res = recursive_iter(((art_node256*)n)->children[i], cb, data, key_len); + if (res) return res; + } + break; + + default: + abort(); + } + return 0; +} + +/** + * Iterates through the entries pairs in the map, + * invoking a callback for each. The call back gets a + * key, value for each and returns an integer stop value. + * If the callback returns non-zero, then the iteration stops. + * @arg t The tree to iterate over + * @arg cb The callback function to invoke + * @arg data Opaque handle passed to the callback + * @return 0 on success, or the return of the callback. + */ +int art_iter(art_tree *t, art_callback cb, void *data) { + return recursive_iter(t->root, cb, data, t->key_len); +} + +/** + * Checks if a leaf prefix matches + * @return 0 on success. + */ +static int +leaf_prefix_matches(const art_leaf *n, const unsigned char *prefix, int prefix_len, + int key_len) { + // Fail if the key length is too short + if (key_len < (uint32_t)prefix_len) return 1; + + // Compare the keys + return memcmp(n->key, prefix, prefix_len); +} + +/** + * Iterates through the entries pairs in the map, + * invoking a callback for each that matches a given prefix. + * The call back gets a key, value for each and returns an integer stop value. + * If the callback returns non-zero, then the iteration stops. + * @arg t The tree to iterate over + * @arg prefix The prefix of keys to read + * @arg prefix_len The length of the prefix + * @arg cb The callback function to invoke + * @arg data Opaque handle passed to the callback + * @return 0 on success, or the return of the callback. + */ +int +art_iter_prefix(art_tree *t, const unsigned char *key, int prefix_len, + art_callback cb, void *data) { + art_node **child; + art_node *n = t->root; + int depth = 0; + while (n) { + // Might be a leaf + if (IS_LEAF(n)) { + n = (art_node*)LEAF_RAW(n); + // Check if the expanded path matches + if (!leaf_prefix_matches((art_leaf*)n, key, prefix_len, t->key_len)) { + art_leaf *l = (art_leaf*)n; + return cb(data, (const unsigned char*)l->key, t->key_len, l->value); + } + return 0; + } + + // If the depth matches the prefix, we need to handle this node + if (depth == prefix_len) { + art_leaf *l = minimum(n); + if (!leaf_prefix_matches(l, key, prefix_len, t->key_len)) + return recursive_iter(n, cb, data, t->key_len); + return 0; + } + + // Bail if the prefix does not match + if (n->partial_len) { + prefix_len = prefix_mismatch(n, key, t->key_len, depth); + + // Guard if the mis-match is longer than the MAX_PREFIX_LEN + if ((uint32_t)prefix_len > n->partial_len) { + prefix_len = n->partial_len; + } + + // If there is no match, search is terminated + if (!prefix_len) { + return 0; + + // If we've matched the prefix, iterate on this node + } else if (depth + prefix_len == t->key_len) { + return recursive_iter(n, cb, data, t->key_len); + } + + // if there is a full match, go deeper + depth = depth + n->partial_len; + } + + // Recursively search + child = find_child(n, key[depth]); + n = (child) ? *child : NULL; + depth++; + } + return 0; +} Index: sys/amd64/vmm/net/if_ethlink.c =================================================================== --- /dev/null +++ sys/amd64/vmm/net/if_ethlink.c @@ -0,0 +1,438 @@ +/* + * Copyright (C) 2018 Matthew Macy + * Copyright (C) 2018 Joyent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ifdi_if.h" + +static MALLOC_DEFINE(M_ETHLINK, "ethlink", "virtual private cloud interface"); + + +#define ETHLINK_DEBUG + +#ifdef ETHLINK_DEBUG +#define DPRINTF printf +#else +#define DPRINTF(...) +#endif + +static int ethlink_transmit(if_t ifp, struct mbuf *m); + +struct ethlink_softc { + if_softc_ctx_t shared; + if_ctx_t es_ctx; + struct ifnet *es_ifp; + struct ifnet *es_underlay_ifp; + uint32_t es_mflags; + uint16_t es_vtag; +}; + +static int clone_count; + +static int +ethlink_mbuf_to_qid(if_t ifp __unused, struct mbuf *m __unused) +{ + return (0); +} + +static void +ethlink_disconnect(struct ethlink_softc *es) +{ + struct ifnet *oifp = es->es_underlay_ifp; + + if (oifp) { + oifp->if_bridge = NULL; + oifp->if_bridge_input = NULL; + oifp->if_bridge_output = NULL; + oifp->if_bridge_linkstate = NULL; + oifp->if_capabilities &= ~IFCAP_BRIDGE_BATCH; + if_rele(oifp); + es->es_underlay_ifp = NULL; + } +} + +#define ETHLINK_CAPS \ + IFCAP_TSO | IFCAP_HWCSUM | IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \ + IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU | IFCAP_TXCSUM_IPV6 | IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU | \ + IFCAP_LINKSTATE + +static int +ethlink_cloneattach(if_ctx_t ctx, struct if_clone *ifc, const char *name, caddr_t params) +{ + struct ethlink_softc *es = iflib_get_softc(ctx); + if_softc_ctx_t scctx; + + atomic_add_int(&clone_count, 1); + es->es_ctx = ctx; + es->es_ifp = iflib_get_ifp(ctx); + scctx = es->shared = iflib_get_softc_ctx(ctx); + scctx->isc_capenable = ETHLINK_CAPS; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_IP6_TCP \ + | CSUM_IP6_UDP | CSUM_IP6_TCP; + return (0); +} + +static int +ethlink_attach_post(if_ctx_t ctx) +{ + struct ifnet *ifp; + + ifp = iflib_get_ifp(ctx); + if_settransmitfn(ifp, ethlink_transmit); + if_settransmittxqfn(ifp, ethlink_transmit); + if_setmbuftoqidfn(ifp, ethlink_mbuf_to_qid); + return (0); +} + +static int +ethlink_detach(if_ctx_t ctx) +{ + struct ethlink_softc *es = iflib_get_softc(ctx); + + if (es->es_ifp->if_bridge) + vpcp_port_disconnect_ifp(es->es_ifp); + ethlink_disconnect(es); + atomic_add_int(&clone_count, -1); + + return (0); +} + +static void +ethlink_init(if_ctx_t ctx) +{ +} + +static void +ethlink_stop(if_ctx_t ctx) +{ +} + +struct ifnet * +ethlink_ifp_get(if_ctx_t ctx) +{ + struct ethlink_softc *es = iflib_get_softc(ctx); + + return (es->es_underlay_ifp); +} + +static int +ethlink_transmit(if_t ifp, struct mbuf *m) +{ + struct ifnet *oifp; + if_ctx_t ctx; + struct ethlink_softc *es; + struct mbuf *mp, *mnext; + bool can_batch; + int lasterr, rc, qid; + + ctx = ifp->if_softc; + es = iflib_get_softc(ctx); + oifp = es->es_underlay_ifp; + can_batch = true; + if (bpf_peers_present(ifp->if_bpf)) { + mp = m; + do { + ETHER_BPF_MTAP(ifp, mp); + mp = mp->m_nextpkt; + } while (mp); + } + if (__predict_false(oifp == NULL)) { + m_freechain(m); + return (ENOBUFS); + } + MPASS(ifp == es->es_ifp); + if (es->es_vtag) { + m->m_flags |= M_VLANTAG; + m->m_pkthdr.ether_vtag = es->es_vtag; + } + m->m_pkthdr.rcvif = NULL; + qid = oifp->if_mbuf_to_qid(oifp, m); + mp = m->m_nextpkt; + while (mp) { + if (es->es_vtag) { + mp->m_flags |= M_VLANTAG; + mp->m_pkthdr.ether_vtag = es->es_vtag; + } + if (can_batch && (qid != oifp->if_mbuf_to_qid(oifp, m))) + can_batch = false; + mp->m_pkthdr.rcvif = NULL; + mp = mp->m_nextpkt; + } + if (can_batch) + return (oifp->if_transmit_txq(oifp, m)); + + do { + mnext = mp->m_nextpkt; + mp->m_nextpkt = NULL; + rc = oifp->if_transmit_txq(oifp, mp); + if (rc) + lasterr = rc; + mp = mnext; + } while (mp != NULL); + return (lasterr); +} + +static struct mbuf * +ethlink_bridge_input(if_t ifp, struct mbuf *m) +{ + struct ethlink_softc *es; + struct mbuf *mp, *mh, *mt, *mnext; + + es = ifp->if_bridge; + if (es->es_ifp->if_bridge == NULL) + return (m); + mh = mt = NULL; + mp = m; + do { + mnext = mp->m_nextpkt; + mp->m_nextpkt = NULL; + if (mp->m_flags & M_VLANTAG) { + if (__predict_false(mp->m_pkthdr.ether_vtag != es->es_vtag)) { + m_freem(mp); + goto next; + } + mp->m_flags &= ~M_VLANTAG; + mp->m_pkthdr.ether_vtag = 0; + } + mp->m_flags |= M_TRUNK; + if (mh != NULL) { + mt->m_nextpkt = mp; + mt = mp; + } else + mh = mt = mp; + next: + mp = mnext; + } while (mp != NULL); + if (__predict_false(mh == NULL)) + return (NULL); + + return (*(es->es_ifp)->if_bridge_input)(es->es_ifp, mh); +} + +static int +ethlink_bridge_output(struct ifnet *ifp, struct mbuf *m, + struct sockaddr *s __unused, struct rtentry *r__unused) +{ + struct ethlink_softc *es; + + es = ifp->if_bridge; + return (ethlink_transmit(es->es_ifp, m)); +} + +static void +ethlink_bridge_linkstate(struct ifnet *ifp __unused) +{ +} + +int +ethlink_ctl(vpc_ctx_t ctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata) +{ + if_ctx_t ifctx = ctx->v_ifp->if_softc; + struct ethlink_softc *es; + struct sockaddr_dl *sdl; + char buf[IFNAMSIZ]; + int rc; + + rc = 0; + es = iflib_get_softc(ifctx); + switch (op) { + case VPC_ETHLINK_OP_CONNECT: { + struct ifnet *ifp; + + ethlink_disconnect(es); + bzero(buf, IFNAMSIZ); + strncpy(buf, in, min(inlen, IFNAMSIZ-1)); + if ((ifp = ifunit_ref(buf)) == NULL) + return (ENOENT); + if (ifp->if_addr == NULL) { + if_rele(ifp); + return (ENXIO); + } + sdl = (struct sockaddr_dl *)ifp->if_addr->ifa_addr; + if (sdl->sdl_type == IFT_ETHER) + iflib_set_mac(ifctx, LLADDR(sdl)); + es->es_ifp->if_capabilities = ifp->if_capabilities; + es->es_underlay_ifp = ifp; + ifp->if_capabilities |= IFCAP_BRIDGE_BATCH; + ifp->if_bridge_input = ethlink_bridge_input; + ifp->if_bridge_output = ethlink_bridge_output; + ifp->if_bridge_linkstate = ethlink_bridge_linkstate; + wmb(); + ifp->if_bridge = es; + break; + } + case VPC_ETHLINK_OP_DISCONNECT: + ethlink_disconnect(es); + break; + case VPC_ETHLINK_OP_CONNECTED_NAME_GET: { + char *ifname; + + if (*outlen < IFNAMSIZ) + return (EOVERFLOW); + if (es->es_underlay_ifp == NULL) + return (EAGAIN); + ifname = malloc(IFNAMSIZ, M_TEMP, M_WAITOK|M_ZERO); + strncpy(ifname, es->es_underlay_ifp->if_xname, IFNAMSIZ); + *outdata = ifname; + *outlen = IFNAMSIZ; + break; + } + case VPC_ETHLINK_OP_VTAG_GET: { + uint16_t *out; + + out = malloc(sizeof(uint16_t), M_TEMP, M_WAITOK); + *outlen = sizeof(uint16_t); + *out = es->es_vtag; + *outdata = out; + break; + } + case VPC_ETHLINK_OP_VTAG_SET: { + uint16_t vtag; + + if (inlen != sizeof(uint16_t)) + return (EBADRPC); + vtag = *(const uint16_t *)in; + es->es_vtag = vtag; + break; + } + default: + rc = EOPNOTSUPP; + break; + } + return (rc); +} + +static device_method_t ethlink_if_methods[] = { + DEVMETHOD(ifdi_cloneattach, ethlink_cloneattach), + DEVMETHOD(ifdi_attach_post, ethlink_attach_post), + DEVMETHOD(ifdi_detach, ethlink_detach), + DEVMETHOD(ifdi_init, ethlink_init), + DEVMETHOD(ifdi_stop, ethlink_stop), + DEVMETHOD_END +}; + +static driver_t ethlink_iflib_driver = { + "ethlink", ethlink_if_methods, sizeof(struct ethlink_softc) +}; + +char ethlink_driver_version[] = "0.0.1"; + +static struct if_shared_ctx ethlink_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_driver_version = ethlink_driver_version, + .isc_driver = ðlink_iflib_driver, + .isc_flags = IFLIB_PSEUDO, + .isc_name = "ethlink", +}; + +if_shared_ctx_t ethlink_sctx = ðlink_sctx_init; + + +static if_pseudo_t ethlink_pseudo; + +static int +ethlink_module_init(void) +{ + ethlink_pseudo = iflib_clone_register(ethlink_sctx); + + return ethlink_pseudo != NULL ? 0 : ENXIO; +} + +static void +ethlink_module_deinit(void) +{ + iflib_clone_deregister(ethlink_pseudo); +} + +static int +ethlink_module_event_handler(module_t mod, int what, void *arg) +{ + int err; + + switch (what) { + case MOD_LOAD: + if ((err = ethlink_module_init()) != 0) + return (err); + break; + case MOD_UNLOAD: + if (clone_count == 0) + ethlink_module_deinit(); + else + return (EBUSY); + break; + default: + return (EOPNOTSUPP); + } + + return (0); +} + +static moduledata_t ethlink_moduledata = { + "ethlink", + ethlink_module_event_handler, + NULL +}; + +DECLARE_MODULE(ethlink, ethlink_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY); +MODULE_VERSION(ethlink, 1); +MODULE_DEPEND(ethlink, iflib, 1, 1, 1); Index: sys/amd64/vmm/net/if_hostif.c =================================================================== --- /dev/null +++ sys/amd64/vmm/net/if_hostif.c @@ -0,0 +1,265 @@ +/* + * Copyright (C) 2017-2018 Matthew Macy + * Copyright (C) 2017 Joyent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ifdi_if.h" + +static MALLOC_DEFINE(M_HOSTIF, "hostif", "virtual private cloud interface"); + + +#define HOSTIF_DEBUG + +#ifdef HOSTIF_DEBUG +#define DPRINTF printf +#else +#define DPRINTF(...) +#endif + + +static int soft_tso; +static uint64_t soft_tso_calls; +static SYSCTL_NODE(_net, OID_AUTO, hostif, CTLFLAG_RD, 0, + "hostif parameters"); + +SYSCTL_INT(_net_hostif, OID_AUTO, soft_tso, CTLFLAG_RW, + &soft_tso, 0, "enable soft tso"); +SYSCTL_QUAD(_net_hostif, OID_AUTO, soft_tso_calls, CTLFLAG_RD, + &soft_tso_calls, 0, "soft tso conversions"); + + +struct hostif_softc { + if_softc_ctx_t shared; + if_ctx_t vs_ctx; +}; + +static int clone_count; + +static int +hostif_mbuf_to_qid(if_t ifp __unused, struct mbuf *m __unused) +{ + return (0); +} + +static int +hostif_transmit(if_t ifp, struct mbuf *m) +{ + struct mbuf *mp = m; + struct mbuf_ext *mext; + + do { + ETHER_BPF_MTAP(ifp, mp); + mp = mp->m_nextpkt; + } while (mp); + + if (ifp->if_bridge == NULL) { + m_freechain(m); + return (ENOBUFS); + } + if (soft_tso) { + mext = pktchain_to_mvec(m, ifp->if_mtu, M_NOWAIT); + if (__predict_false(mext == NULL)) + return (ENOMEM); + if (m->m_pkthdr.csum_flags & CSUM_TSO) { + soft_tso_calls++; + m = (void*)mvec_tso(mext, 0, true); + if (__predict_false(m == NULL)) + return (ENOMEM); + } else + m = (void *)mext; + } + return ((*(ifp)->if_bridge_output)(ifp, m, NULL, NULL)); +} + +#define HOSTIF_CAPS \ + IFCAP_TSO | IFCAP_HWCSUM | IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \ + IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU | IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU | \ + IFCAP_LINKSTATE + +static int +hostif_cloneattach(if_ctx_t ctx, struct if_clone *ifc, const char *name, caddr_t params) +{ + struct hostif_softc *vs = iflib_get_softc(ctx); + if_softc_ctx_t scctx; + + atomic_add_int(&clone_count, 1); + vs->vs_ctx = ctx; + + scctx = vs->shared = iflib_get_softc_ctx(ctx); + scctx->isc_capenable = HOSTIF_CAPS; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_IP6_TCP \ + | CSUM_IP6_UDP | CSUM_IP6_TCP; + return (0); +} + +static int +hostif_attach_post(if_ctx_t ctx) +{ + struct ifnet *ifp; + + ifp = iflib_get_ifp(ctx); + if_settransmitfn(ifp, hostif_transmit); + if_settransmittxqfn(ifp, hostif_transmit); + if_setmbuftoqidfn(ifp, hostif_mbuf_to_qid); + iflib_link_state_change(ctx, LINK_STATE_UP, IF_Gbps(50)); + return (0); +} + +static int +hostif_detach(if_ctx_t ctx) +{ + iflib_link_state_change(ctx, LINK_STATE_DOWN, IF_Gbps(50)); + atomic_add_int(&clone_count, -1); + return (0); +} + +static void +hostif_init(if_ctx_t ctx) +{ +} + +static void +hostif_stop(if_ctx_t ctx) +{ +} + +int +hostif_ctl(vpc_ctx_t vctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata) +{ + return (0); +} + +static device_method_t hostif_if_methods[] = { + DEVMETHOD(ifdi_cloneattach, hostif_cloneattach), + DEVMETHOD(ifdi_attach_post, hostif_attach_post), + DEVMETHOD(ifdi_detach, hostif_detach), + DEVMETHOD(ifdi_init, hostif_init), + DEVMETHOD(ifdi_stop, hostif_stop), + DEVMETHOD_END +}; + +static driver_t hostif_iflib_driver = { + "hostif", hostif_if_methods, sizeof(struct hostif_softc) +}; + +char hostif_driver_version[] = "0.0.1"; + +static struct if_shared_ctx hostif_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_driver_version = hostif_driver_version, + .isc_driver = &hostif_iflib_driver, + .isc_flags = IFLIB_PSEUDO, + .isc_name = "hostif", +}; + +if_shared_ctx_t hostif_sctx = &hostif_sctx_init; + + +static if_pseudo_t hostif_pseudo; + +static int +hostif_module_init(void) +{ + hostif_pseudo = iflib_clone_register(hostif_sctx); + + return hostif_pseudo != NULL ? 0 : ENXIO; +} + +static void +hostif_module_deinit(void) +{ + iflib_clone_deregister(hostif_pseudo); +} + +static int +hostif_module_event_handler(module_t mod, int what, void *arg) +{ + int err; + + switch (what) { + case MOD_LOAD: + if ((err = hostif_module_init()) != 0) + return (err); + break; + case MOD_UNLOAD: + if (clone_count == 0) + hostif_module_deinit(); + else + return (EBUSY); + break; + default: + return (EOPNOTSUPP); + } + + return (0); +} + +static moduledata_t hostif_moduledata = { + "hostif", + hostif_module_event_handler, + NULL +}; + +DECLARE_MODULE(hostif, hostif_moduledata, SI_SUB_INIT_IF, SI_ORDER_ANY); +MODULE_VERSION(hostif, 1); +MODULE_DEPEND(hostif, iflib, 1, 1, 1); Index: sys/amd64/vmm/net/if_vmnic.h =================================================================== --- /dev/null +++ sys/amd64/vmm/net/if_vmnic.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2017-2018 Joyent Inc. + * Copyright (C) 2017-2018 Matthew Macy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _VTNETBE_H_ +#define _VTNETBE_H_ +struct iflib_ctx; + +extern struct vtnet_be *(*vtnet_be_init)(struct vm *); +extern void (*vtnet_be_cleanup)(struct vtnet_be *); + +#endif /* _VTNETBE_H_ */ Index: sys/amd64/vmm/net/if_vmnic.c =================================================================== --- /dev/null +++ sys/amd64/vmm/net/if_vmnic.c @@ -0,0 +1,2115 @@ +/* + * Copyright (C) 2017-2018 Joyent Inc. + * Copyright (C) 2017-2018 Matthew Macy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + + +#include "vmm_ktr.h" +#include "vmm_ioport.h" +#include "vmm_lapic.h" +#include "if_vmnic.h" +#include "ifdi_if.h" + +const char * pci_cmds[] = { + "VIRTIO_PCI_HOST_FEATURES", + "BAD1", + "BAD2", + "BAD3", + "VIRTIO_PCI_GUEST_FEATURES", + "BAD5", + "BAD6", + "BAD7", + "VIRTIO_PCI_QUEUE_PFN", + "BAD9", + "BAD10", + "BAD11", + "VIRTIO_PCI_QUEUE_NUM", + "BAD13", + "VIRTIO_PCI_QUEUE_SEL", + "BAD15", + "VIRTIO_PCI_QUEUE_NOTIFY", + "BAD17", + "VIRTIO_PCI_STATUS", + "VIRTIO_PCI_ISR", + "VIRTIO_MSI_CONFIG_VECTOR", + "BAD21", + "VIRTIO_MSI_QUEUE_VECTOR", +}; +#define VIRTIO_CMD_MAX VIRTIO_MSI_QUEUE_VECTOR + + +#define VB_NQUEUES_DEFAULT 8 +#define VB_MAX_TX_SEGS 64 +#define VB_RXQ_IDX 0 +#define VB_TXQ_IDX 1 +#define VMNIC_RX_BYTES_MAX (256*1024) +#define VMNIC_RX_QDEPTH 256 +#define VB_TSO_SIZE (65535 + sizeof(struct ether_vlan_header)) +#define VB_TSO_SEG_SIZE USHRT_MAX +#define VB_MAX_SCATTER VB_MAX_TX_SEGS +#define VB_CAPS \ + (IFCAP_TSO | IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO | IFCAP_HWCSUM | \ + IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | IFCAP_VLAN_MTU | \ + IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU) + +static MALLOC_DEFINE(M_VMNIC, "vmnic", "virtio-net backend"); + +static volatile int32_t modrefcnt; + +#define DEVEL_BRINGUP + +#ifdef VB_STATUS_DEBUG +#define SDPRINTF printf +#else +#define SDPRINTF(...) +#endif + +#ifdef VB_TX_DEBUG + +#define DPRINTF printf + +static void +vb_print_vhdr(struct virtio_net_hdr_mrg_rxbuf *vh) +{ + struct virtio_net_hdr *h = &vh->hdr; + if (h->flags | h->gso_type | h->hdr_len | h->gso_size | + h->csum_start | h->csum_offset) + printf("vhdr:\n \tflags: %d gso_type: %d hdr_len: %d\n" + "\tgso_size: %d, csum_start: %d, csum_offset: %d\n" + "\tnum_buffers: %d\n", + h->flags, h->gso_type, h->hdr_len, h->gso_size, + h->csum_start, h->csum_offset, vh->num_buffers); +} + +#else +#define DPRINTF(...) +static void vb_print_vhdr(struct virtio_net_hdr_mrg_rxbuf *vh __unused) {} +#endif + + +#ifdef RXDEBUG +#define RXDPRINTF printf +#else +#define RXDPRINTF(...) +#endif + +static int +vq2txq(int vq) +{ + return (vq - 1) / 2; +} + +static int +txq2vq(int txq) +{ + return txq * 2 + 1; +} +static int +vq2rxq(int vq) +{ + return vq / 2; +} + +static int +rxq2vq(int rxq) +{ + return rxq * 2; +} + +/* + * This is an in-kernel backend for the guest virtio net driver. + * + * For the interface see: + * http://docs.oasis-open.org/virtio/virtio/v1.0/cs04/virtio-v1.0-cs04.html#x1-1570001 + * + * It uses iflib device cloning for node creation. The path + * through the driver is an idiosyncratic inverted ifnet. + * + * Assuming an iflib hardware driver, [hw] represents the hw ifp value + * and [vb] represents the vtnet_be ifp, the path follows what we see below. + * + * On receive: + * (wire) -> iflib_rxeof -> ifp[hw]->if_input() [vmbridge_input] -> + * vb_hw_if_input() -> + * ifp[vb]->if_transmit [iflib_if_transmit] -> + * vb_txd_encap -> (guest) + * + * On transmit: + * (guest) -> iflib_rxeof -> vb_rxd_pkt_get(); + * ifp[vb]->if_input() [vb_if_input] -> + * vmbridge_transmit() -> [vpc_transmit()] -> + * ifp[hw]->if_transmit() [iflib_if_transmit] -> (wire) + * + */ + +struct vb_softc; + +struct vtnet_be { + SLIST_ENTRY(vtnet_be) next; + SLIST_HEAD(, vb_softc) dev; + struct vm *vm; + const char *name; /* can point back to VM name */ +}; + +/* Packet parse info */ +struct pinfo { + uint16_t etype; + uint8_t ehdrlen; /* eth header len, includes VLAN tag */ + uint8_t l4type; /* layer 4 protocol type */ + uint8_t l3size:7; /* size of l3 header */ + uint8_t l4size; /* layer 4 protocol type */ + uint8_t l3valid:1; /* size of l3 header */ +}; + +static SLIST_HEAD(, vtnet_be) vb_head; + +struct vb_queue { + uint16_t vq_qsize; + uint32_t vq_lastpfn; + uint32_t vq_pfn; + uint16_t vq_id; + uint16_t vq_msix_idx; + uint16_t vq_avail_idx; + + /* queue mgmt */ + volatile struct vring_desc *vq_desc; + volatile struct vring_avail *vq_avail; + volatile struct vring_used *vq_used; + + vm_offset_t vq_addr; +#ifdef GUEST_OVERCOMMIT + uint16_t vq_pages; + vm_page_t vq_m[54]; /* 54 pages for 16k descriptors */ +#endif +}; + +#define VS_ENQUEUED 0x001 +#define VS_VERS_1 0x002 +#define VS_READY 0x004 + +#define VS_ATTACHED 0x010 +#define VS_IMMUTABLE 0x020 + +#define VS_NQS_CHANGED 0x100 + +#define VB_CIDX_VALID (1 << 18) + +#define SCRATCHCHECK(vs) \ + MPASS(((vs)->pad[0] | (vs)->pad[1] | (vs)->pad[2] | (vs)->pad[3]) == 0) + +struct vb_softc { + uint64_t pad[4]; + if_softc_ctx_t shared; + if_ctx_t vs_ctx; +#define tx_num_queues shared->isc_ntxqsets +#define rx_num_queues shared->isc_nrxqsets + struct vtnet_be *vs_vn; + struct vb_rxq *vs_rx_queues; + struct vb_txq *vs_tx_queues; + struct proc *vs_proc; /* proc that did the cloneattach */ + SLIST_ENTRY(vb_softc) vs_next; + uint32_t vs_flags; + uint16_t vs_gpa_hint; + + /* virtio state */ + uint64_t vs_generation; + uint16_t vs_io_start; + uint16_t vs_io_size; + uint16_t vs_nqs; + uint16_t vs_nvqs; + uint8_t vs_msix_enabled; + uint8_t vs_curq; + uint8_t vs_status; + uint8_t vs_origmac[6]; + uint32_t vs_hv_caps; + uint32_t vs_negotiated_caps; + + struct virtio_net_config vs_cfg; + struct grouptask vs_deferred_uptask; + struct vb_msix_vector *vs_msix; + struct vb_queue *vs_queues; + struct vb_queue *vs_cvq; +}; + +#define VB_HDR_MAX 18 + 40 + +struct vb_rxq { + uint16_t vr_cidx; + uint16_t vr_pidx; + struct vb_softc *vr_vs; + caddr_t *vr_sdcl; + uint16_t *vr_avail; + struct vring_desc *vr_base; + uint8_t *vr_completion; + uint16_t *vr_used; + uint8_t vr_shift; + uint8_t vr_idx; + char vr_pkttmp[VB_HDR_MAX] __aligned(CACHE_LINE_SIZE); +}; + +struct vb_txq { + struct vb_softc *vt_vs; + struct vring_desc *vt_base; + uint16_t vt_cidx; /* iflib current index */ + uint8_t vt_idx; + qidx_t vt_vpidxs[VB_MAX_TX_SEGS]; + bus_dma_segment_t vt_segs[VB_MAX_TX_SEGS]; +}; + +static if_pseudo_t vb_clone_register(void); +static void vb_intr_msix(struct vb_softc *vs, int q); +static void vb_rxq_init(struct vb_softc *vs, struct vb_rxq *rxq, int i); +static void vb_txq_init(struct vb_softc *vs, struct vb_txq *txq, int i); +static int vb_pparse(caddr_t data, struct pinfo *pinfo); + +static int +vb_txd_encap(void *arg, if_pkt_info_t pi) +{ + struct vb_softc *vs = arg; + struct vb_txq *txq = &vs->vs_tx_queues[pi->ipi_qsidx]; + struct vm *vm = vs->vs_vn->vm; + if_softc_ctx_t scctx = vs->shared; + volatile struct vring_used *vu; + volatile struct vring_avail *va; + volatile struct vring_used_elem *vue; + bus_dma_segment_t *segs, *tx_segs; + struct virtio_net_hdr_mrg_rxbuf *vhd; + struct vring_desc *txd; + int i, total, freespc, ndesc, pidx, mask; + int soff, doff, sidx, didx; + uint16_t *vpidxs; + uint16_t vidx, rxvq; + + pidx = pi->ipi_pidx; + segs = pi->ipi_segs; + tx_segs = txq->vt_segs; + +#ifdef INVARIANTS + for (total = i = 0; i < pi->ipi_nsegs; i++) { + total += segs[i].ds_len; + DPRINTF("\t data: %p len: %ld\n", + (void *)segs[i].ds_addr, segs[i].ds_len); + } + MPASS(pi->ipi_len == total); +#endif + ndesc = VB_MAX_TX_SEGS; + mask = scctx->isc_nrxd[0] - 1; + vpidxs = txq->vt_vpidxs; + rxvq = rxq2vq(pi->ipi_qsidx); + va = vs->vs_queues[rxvq].vq_avail; + + /* + * Determine how much space the ring has -- the TSO segs check + * should be sufficient to avoid missing out here + */ + DPRINTF("vpidxs: "); + for (freespc = i = 0; freespc < pi->ipi_len && i < ndesc; i++) { + vm_offset_t kva; + uint16_t vpidx; + + vpidx = va->ring[pidx]; + vpidxs[i] = vpidx; + /* Can we always assume 1:1 between avail and descriptors? */ + txd = &txq->vt_base[vpidx]; + kva = vm_gpa_to_kva(vm, txd->addr, txd->len, &vs->vs_gpa_hint); + if (__predict_false(kva == 0)) { + panic("XXX do vm_gpa_hold"); + } + tx_segs[i].ds_addr = kva; + tx_segs[i].ds_len = txd->len; + freespc += txd->len; + pidx = (pidx + 1) & mask; + } + + DPRINTF("\n"); + if (__predict_false(freespc < pi->ipi_len)) { + DPRINTF("freespc=%d < len=%d\n", freespc, pi->ipi_len); + return (ENOBUFS); + } + vhd = (void *)tx_segs[0].ds_addr; + bzero(vhd, sizeof(*vhd)); + vhd->num_buffers = i; + if (__predict_true(pi->ipi_csum_flags & CSUM_DATA_VALID)) { + vhd->hdr.flags = VIRTIO_NET_HDR_F_DATA_VALID; + vhd->hdr.csum_start = pi->ipi_ehdrlen + pi->ipi_ip_hlen; + switch (pi->ipi_ipproto) { + case IPPROTO_TCP: + vhd->hdr.csum_offset = + offsetof(struct tcphdr, th_sum); + if (pi->ipi_len > vs->vs_cfg.mtu) { + caddr_t hdr = (caddr_t)segs[0].ds_addr; + struct pinfo pinfo; + int hdrlen; + vb_pparse(hdr, &pinfo); + hdrlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen + + max(sizeof(struct tcphdr), pinfo.l4size); + vhd->hdr.gso_size = vs->vs_cfg.mtu - hdrlen; +#if 0 + printf("len: %d mtu: %d ehdrlen: %d iphlen: %d tcp_hlen: %d gso_size: %d\n", + pi->ipi_len, vs->vs_cfg.mtu, pi->ipi_ehdrlen, pi->ipi_ehdrlen, + pinfo.l4size, vhd->hdr.gso_size); +#endif + if (pi->ipi_etype == ETHERTYPE_IP) + vhd->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4; + else if (pi->ipi_etype == ETHERTYPE_IPV6) + vhd->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6; + } + break; + case IPPROTO_UDP: + vhd->hdr.csum_offset = + offsetof(struct udphdr, uh_sum); + break; + case IPPROTO_SCTP: + vhd->hdr.csum_offset = + offsetof(struct sctphdr, checksum); + break; + default: + vhd->hdr.flags = 0; + vhd->hdr.csum_start = 0; + break; + } + } + if (pi->ipi_nsegs > 1) + vb_print_vhdr(vhd); + doff = sizeof(*vhd); + ndesc = i; + total = pi->ipi_len; + soff = didx = sidx = 0; + vu = vs->vs_queues[rxvq].vq_used; + vidx = vu->idx; + + DPRINTF("vidx:%d vu->idx: %d vu->flags: %x\n", + vidx, vu->idx, vu->flags); + do { + /* copy length / source delta / destination delta */ + int len, srem, drem, dlen; + char *src, *dst; + bool update_used; + + update_used = false; + srem = segs[sidx].ds_len - soff; + dlen = tx_segs[didx].ds_len; + drem = dlen - doff; + src = (caddr_t)segs[sidx].ds_addr + soff; + dst = (caddr_t)tx_segs[didx].ds_addr + doff; + /* + * Mark size of last descriptor and end loop + */ + if (total == srem && (srem <= drem)) { + txd = &txq->vt_base[(pi->ipi_pidx + didx) & mask]; + + dlen = doff + srem; + DPRINTF("last descriptor: %d len: %d doff: %d, srem: %d, dsize: %lu\n", + (pi->ipi_pidx + didx) & mask, dlen, doff, srem, tx_segs[didx].ds_len); + MPASS(dlen <= tx_segs[didx].ds_len); + update_used = true; + } + if (srem < drem) { + len = srem; + doff += len; + soff = 0; + sidx++; + } else if (srem > drem) { + len = drem; + soff += len; + doff = 0; + update_used = true; + } else { + len = srem; + doff = soff = 0; + sidx++; + update_used = true; + } + total -= len; + bcopy(src, dst, len); + if (update_used) { + vue = &vu->ring[vidx++ & mask]; + vue->id = /* pi->ipi_pidx + didx */ vpidxs[didx]; + vue->len = dlen; + didx++; + if ((vidx & 63) == 0) + DPRINTF("%s called pidx: %d nsegs: %d %d ndesc avail " + "vidx: %d vue->id: %d vue->len: %d vpidxs[%d]: %d\n", + __func__, pidx, pi->ipi_nsegs, ndesc, + vidx, vue->id, vue->len, didx, vpidxs[didx]); + } + } while (didx < ndesc); + + pi->ipi_new_pidx = pidx; + /* Update used ring to reflect final state */ + wmb(); + vu->idx = vidx; + return (0); +} + +static void +vb_txd_flush(void *arg, uint16_t txqid, qidx_t pidx __unused) +{ + struct vb_softc *vs = arg; + + /* Interrupt the guest */ + vb_intr_msix(vs, rxq2vq(txqid)); +} + +static int +vb_txd_credits_update(void *arg, uint16_t txqid, bool clear) +{ + + struct vb_softc *vs = arg; + if_softc_ctx_t scctx = vs->shared; + struct vb_txq *txq = &vs->vs_tx_queues[txqid]; + int16_t vpidx, rxvq; + int32_t delta; + + rxvq = rxq2vq(txqid); + vpidx = vs->vs_queues[rxvq].vq_avail->idx & (scctx->isc_ntxd[0]-1); + + /* credits updated should reflect new + * descriptors available in the ring -- + * so long as we copy in to the guest the + * mbuf chain can be freed instantly + */ + delta = (int)vpidx - (int)txq->vt_cidx; + if (delta < 0) + delta += scctx->isc_ntxd[0]; + DPRINTF("%s vt_cidx: %d vpidx: %d delta: %d\n", + __func__, txq->vt_cidx, vpidx, delta); + + if (clear && delta) + txq->vt_cidx = vpidx; + MPASS(delta < scctx->isc_ntxd[0]); + return (delta); +} + +static void +vb_rxd_refill(void *arg __unused, if_rxd_update_t iru __unused) +{ + //panic("XXX %s shouldn't be called for vtnet_be", __func__); + // log refill +} + +#define idx2gen(rxq, idx) ((uint8_t)(!(((idx) >> (rxq)->vr_shift) & 1UL))) + +static void +vb_rxd_reclaim(struct vb_rxq *rxq) +{ + struct vb_softc *vs = rxq->vr_vs; + volatile struct vring_used *vu; + volatile struct vring_used_elem *vue; + uint16_t idx, vidx, pidx, nrxd; + int mask, mask_used; +#ifdef INVARIANTS + int count = 0; +#endif + + if (__predict_false((vs->vs_flags & VS_READY) == 0)) { + RXDPRINTF("%s not ready\n", __func__); + return; + } + RXDPRINTF("q: %128D\n", rxq->vr_completion, ""); + + nrxd = vs->shared->isc_nrxd[0]; + mask_used = (nrxd*2)-1; + mask = nrxd-1; + + pidx = rxq->vr_pidx; + /* Update the element in the used ring */ + vu = vs->vs_queues[txq2vq(rxq->vr_idx)].vq_used; + MPASS(vu); + vidx = vu->idx; + for (pidx = rxq->vr_pidx; rxq->vr_completion[pidx & mask] == idx2gen(rxq, pidx); pidx++) { + idx = rxq->vr_used[pidx & mask_used]; + vue = &vu->ring[vidx++ & mask]; + vue->id = idx; + MPASS(count++ < nrxd); + } + /* ensure that all prior vue updates are written first */ + wmb(); + RXDPRINTF("vu->idx: %d vidx: %d\n", vu->idx, vidx); + vu->idx = vidx; + rxq->vr_pidx = pidx; + vb_intr_msix(vs, txq2vq(rxq->vr_idx)); +} + +static void +vb_rxd_flush(void *arg, uint16_t rxqid, + uint8_t flid __unused, qidx_t pidx __unused) +{ +} + +static void * +vb_cl_map(struct vb_softc *vs, volatile struct vring_desc *desc) +{ + + struct vm *vm = vs->vs_vn->vm; + + return ((void *)vm_gpa_to_kva(vm, desc->addr, desc->len, &vs->vs_gpa_hint)); +} + +static int +vb_rxd_available(void *arg, qidx_t rxqid, qidx_t cidx, qidx_t budget) +{ + struct vb_softc *vs = arg; + struct vb_rxq *rxq = &vs->vs_rx_queues[rxqid]; + uint16_t idx, nrxd = vs->shared->isc_nrxd[0]; + int cnt; + + idx = vs->vs_queues[txq2vq(rxqid)].vq_avail->idx; + idx &= (nrxd-1); + cnt = (int32_t)idx - (int32_t)cidx; + if (cnt < 0) + cnt += nrxd; + RXDPRINTF("vr_pidx: %d vr_cidx: %d cnt: %d idx: %d\n", + rxq->vr_pidx, rxq->vr_cidx, cnt, idx); + if (__predict_false(abs(rxq->vr_pidx - rxq->vr_cidx) >= (nrxd >> 4))) + vb_rxd_reclaim(rxq); + + return (cnt); +} + +static int +vb_pparse(caddr_t data, struct pinfo *pinfo) +{ + struct ether_vlan_header *eh; + struct ip *ip; + struct ip6_hdr *ip6; + struct tcphdr *th; + int ehdrlen; + int l3valid, l3size, l4type; + uint16_t etype; + + eh = (struct ether_vlan_header *)data; + if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + etype = ntohs(eh->evl_proto); + ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + } else { + etype = ntohs(eh->evl_encap_proto); + ehdrlen = ETHER_HDR_LEN; + } + th = NULL; + switch (etype) { + case ETHERTYPE_IP: + ip = (struct ip *)(data + ehdrlen); + l3valid = 1; + l3size = ip->ip_hl << 2; + l4type = ip->ip_p; + break; + case ETHERTYPE_IPV6: + ip6 = (struct ip6_hdr *)(data + ehdrlen); + l3valid = 1; + l3size = sizeof(struct ip6_hdr); + /* XXX lasthdr */ + l4type = ip6->ip6_nxt; + break; + default: + l3valid = 0; + l3size = 0; + l4type = 0; + } + if (l4type == IPPROTO_TCP) { + th = (void*)(((caddr_t)ip) + l3size); + pinfo->l4size = th->th_off << 2; + } + pinfo->etype = etype; + pinfo->ehdrlen = ehdrlen; + pinfo->l3valid = l3valid; + pinfo->l3size = l3size; + pinfo->l4type = l4type; + return (1); +} +static void +vb_rx_vhdr_process(struct virtio_net_hdr_mrg_rxbuf *vh, + struct vring_desc *rxd, caddr_t buf, + if_rxd_info_t ri) +{ + struct pinfo pinfo; + int flags; + + switch (vh->hdr.gso_type) { + case VIRTIO_NET_HDR_GSO_TCPV4: + ri->iri_tso_segsz = vh->hdr.gso_size; + ri->iri_csum_data = vh->hdr.csum_offset; + ri->iri_csum_flags = CSUM_TSO | CSUM_TCP; + return; + break; + case VIRTIO_NET_HDR_GSO_TCPV6: + ri->iri_tso_segsz = vh->hdr.gso_size; + ri->iri_csum_data = vh->hdr.csum_offset; + ri->iri_csum_flags = CSUM_IP6_TSO | CSUM_TCP_IPV6; + return; + break; + default: + break; + } + if ((vh->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) == 0) + return; + + vb_pparse(buf, &pinfo); + flags = 0; + switch (pinfo.etype) { + case ETHERTYPE_IP: + switch (pinfo.l4type) { + case IPPROTO_TCP: + flags = CSUM_TCP; + break; + case IPPROTO_UDP: + flags = CSUM_UDP; + break; + case IPPROTO_SCTP: + flags = CSUM_SCTP; + } + break; + case ETHERTYPE_IPV6: + switch (pinfo.l4type) { + case IPPROTO_TCP: + flags = CSUM_TCP_IPV6; + break; + case IPPROTO_UDP: + flags = CSUM_UDP_IPV6; + break; + case IPPROTO_SCTP: + flags = CSUM_SCTP_IPV6; + break; + } + break; + } + if (__predict_false(flags == 0)) + printf("XXX no flags, l4type: %d\n", pinfo.l4type); + ri->iri_csum_flags = flags; + ri->iri_csum_data = vh->hdr.csum_offset; +} + +static int +vb_rxd_pkt_get(void *arg, if_rxd_info_t ri) +{ + struct virtio_net_hdr_mrg_rxbuf *vh; + struct vb_softc *vs = arg; + if_softc_ctx_t scctx = vs->shared; + struct vb_rxq *rxq = &vs->vs_rx_queues[ri->iri_qsidx]; + struct vring_desc *rxd; + int i, cidx, vcidx, count, mask, used_mask; + caddr_t data; + + MPASS(ri->iri_cidx < scctx->isc_nrxd[0]); + i = 0; + cidx = ri->iri_cidx; + vcidx = rxq->vr_avail[cidx]; + mask = scctx->isc_nrxd[0]-1; + used_mask = (2*scctx->isc_nrxd[0])-1; + if (cidx != (rxq->vr_cidx & mask)) + printf("mismatch cidx: %d vr_cidx_masked:%d vr_cidx: %d", + cidx, rxq->vr_cidx & mask, rxq->vr_cidx); + + rxq->vr_used[rxq->vr_cidx & used_mask] = vcidx; + /* later passed to ext_free to return used descriptors */ + ri->iri_cookie1 = (void *)rxq; + ri->iri_cookie2 = (rxq->vr_cidx++ | VB_CIDX_VALID); + + ri->iri_cidx = rxq->vr_cidx & (scctx->isc_nrxd[0]-1); + rxd = (struct vring_desc *)&rxq->vr_base[vcidx]; + + if (__predict_false(rxd->len < sizeof(*vh))) { + DPRINTF("%s rxd->len short: %d\n", __func__, rxd->len); + return (ENXIO); + } + rxq->vr_sdcl[vcidx] = vb_cl_map(vs, rxd); + vh = (void *)rxq->vr_sdcl[vcidx]; + if (vh == NULL) + return (ENXIO); + if (__predict_true(rxd->len == sizeof(*vh))) { + rxq->vr_sdcl[vcidx] = NULL; + if (__predict_true(rxd->flags & VRING_DESC_F_NEXT)) { + vcidx = rxd->next; + rxq->vr_sdcl[vcidx] = vb_cl_map(vs, rxd); + if (__predict_false(rxq->vr_sdcl[vcidx] == NULL)) + return (ENXIO); + } else { + return (ENXIO); + } + if (__predict_false(vcidx >= scctx->isc_nrxd[0])) + return (ENXIO); + } else { + RXDPRINTF("didn't get separate vhdr assuming it's inplace rxd->len: %d\n", + rxd->len); + ri->iri_pad = sizeof(*vh); + } + rxd = (struct vring_desc *)&rxq->vr_base[vcidx]; + data = rxq->vr_sdcl[vcidx] = vb_cl_map(vs, rxd); + vb_rx_vhdr_process(vh, rxd, data, ri); + + do { + if (__predict_false(vcidx >= scctx->isc_nrxd[0])) { + RXDPRINTF("descriptor out of range: %d\n", vcidx); + return (ENXIO); + } + if (__predict_false(i == IFLIB_MAX_RX_SEGS)) { + RXDPRINTF("descriptor chain too long: %d\n", vcidx); + return (ENXIO); + } + rxd = (struct vring_desc *)&rxq->vr_base[vcidx]; + if ((int)ri->iri_len + (int)rxd->len > VB_TSO_SIZE) { + RXDPRINTF("chain exceeds maximum size: %d\n", + (int)ri->iri_len + (int)rxd->len); + return (ENXIO); + } + if (__predict_false(rxd->flags & VRING_DESC_F_INDIRECT)) { + RXDPRINTF("indirect not supported!!!!\n"); + return (ENXIO); + } + rxq->vr_sdcl[vcidx] = vb_cl_map(vs, rxd); + if (__predict_false(rxq->vr_sdcl[vcidx] == NULL)) { + RXDPRINTF("failed cluster map\n"); + return (ENXIO); + } + ri->iri_frags[i].irf_idx = vcidx; + ri->iri_frags[i].irf_len = rxd->len; + ri->iri_len += rxd->len; + i++; + vcidx = rxd->next; + } while (rxd->flags & VRING_DESC_F_NEXT); + + count = i; + for (i = 0; i < count; i++) + CTR5(KTR_SPARE3, "cidx: %d [%d].irf_idx: %d [%d].irf_len: %d\n", + cidx, i, ri->iri_frags[i].irf_idx, i, + ri->iri_frags[i].irf_len); + ri->iri_nfrags = count; + return (0); +} + +static void +vb_rx_completion(struct mbuf *m) +{ + struct vb_softc *vs; + struct vb_rxq *rxq; + int cidx, mask; + + if ((m->m_flags & M_PKTHDR) == 0) + return; + rxq = (struct vb_rxq *)m->m_ext.ext_arg1; + MPASS(rxq != NULL); + vs = rxq->vr_vs; + cidx = (uintptr_t)m->m_ext.ext_arg2; + /* + * Is this just a buffer post-processing? + */ + if (m_ismvec(m)) + mvec_free((void*)m); + + MPASS(cidx & VB_CIDX_VALID); + cidx &= ~VB_CIDX_VALID; + mask = vs->shared->isc_nrxd[0]-1; + rxq->vr_completion[cidx & mask] = idx2gen(rxq, cidx); +} + +static int +vb_intr(void *arg) +{ + return (0); +} + +static struct if_txrx vb_txrx = { + vb_txd_encap, /* copy packet in to rx ring */ + vb_txd_flush, /* mark RX "dma complete" up to doorbell */ + vb_txd_credits_update, /* all descriptors sent to */ + vb_rxd_available, /* # of descriptors in the tx ring */ + vb_rxd_pkt_get, /* get packet from tx ring */ + vb_rxd_refill, /* nop -- never called */ + vb_rxd_flush, /* nop -- never called */ + vb_intr +}; + +static struct mtx vb_mtx; +#define VB_LOCK mtx_lock(&vb_mtx) +#define VB_UNLOCK mtx_unlock(&vb_mtx) +static void vb_dev_kick(struct vb_softc *vs, uint32_t q); +static void vb_dev_pfn(struct vb_softc *vs, uint32_t pfn); +static void vb_vring_munmap(struct vb_softc *vs, int q); + +/* + * Utility routines + */ +static struct vtnet_be * +vb_find_vmname(const char *vmname) +{ + struct vtnet_be *vb; + + VB_LOCK; + SLIST_FOREACH(vb, &vb_head, next) { + if (!strcmp(vmname, vb->name)) + break; + } + VB_UNLOCK; + + return (vb); +} + +static void +deferred_up(void *arg) +{ + if_ctx_t ctx = arg; + struct ifnet *ifp = iflib_get_ifp(ctx); + struct mtx *ctxlock; + + ctxlock = iflib_ctx_lock_get(ctx); + mtx_lock(ctxlock); + if_setflagbits(ifp, IFF_UP, 0); + iflib_link_state_change(ctx, LINK_STATE_UP, IF_Gbps(200)); + mtx_unlock(ctxlock); + ifp->if_init(ctx); +} + +static void +deferred_up_timer(void *arg) +{ + struct vb_softc *vs = arg; + + GROUPTASK_ENQUEUE(&vs->vs_deferred_uptask); +} + +static void +vb_status_change(struct vb_softc *vs, uint32_t val) +{ + struct ifnet *ifp = iflib_get_ifp(vs->vs_ctx); + + SCRATCHCHECK(vs); + /* + * For legacy guest drivers - e.g. FreeBSD + */ + if (val == VIRTIO_CONFIG_STATUS_RESET) { + SDPRINTF("VIRTIO_CONFIG_STATUS_RESET\n"); + if_setflagbits(ifp, 0, IFF_UP); + ifp->if_init(vs->vs_ctx); + } + if (val & VIRTIO_CONFIG_STATUS_DRIVER) + SDPRINTF("VIRTIO_CONFIG_STATUS_DRIVER "); + if (val & VIRTIO_CONFIG_STATUS_ACK) + SDPRINTF("VIRTIO_CONFIG_STATUS_ACK "); + if (val & VIRTIO_CONFIG_STATUS_NEEDS_RESET) { + SDPRINTF("VIRTIO_CONFIG_STATUS_NEEDS_RESET "); + if_setflagbits(ifp, 0, IFF_UP); + ifp->if_init(vs->vs_ctx); + } + if (val & VIRTIO_CONFIG_STATUS_DRIVER_OK) { + SDPRINTF("VIRTIO_CONFIG_STATUS_DRIVER_OK "); + /* Up interface */ + timeout(deferred_up_timer, vs, hz*5); + } + if (val & VIRTIO_CONFIG_STATUS_FEATURES_OK) { + SDPRINTF("VIRTIO_CONFIG_STATUS_FEATURES_OK "); + vs->vs_flags |= VS_VERS_1; + } + if (val & VIRTIO_CONFIG_STATUS_FAILED) { + SDPRINTF("VIRTIO_CONFIG_STATUS_FAILED"); + if_setflagbits(ifp, 0, IFF_UP); + ifp->if_init(vs->vs_ctx); + } + SDPRINTF("\n"); + vs->vs_status = val | VIRTIO_CONFIG_STATUS_FEATURES_OK; +} + + +/* + * Virtio register-handling code + */ +static int +vb_handle_config(struct vb_softc *vs, int offset, bool in, int bytes, + uint32_t *val) +{ + uint8_t *ptr; + + SDPRINTF("%s %s offset: %d len: %d val: %x\n", __func__, in ? "in" : "out", + offset, bytes, *val); + + ptr = (uint8_t *)&vs->vs_cfg + offset; + if (in) { + if (bytes + offset <= sizeof(struct virtio_net_config)) + memcpy(val, ptr, bytes); + else + memset(val, -1, bytes); + } else if (offset + bytes <= ETHER_ADDR_LEN) { + memcpy(ptr, val, bytes); + } + + return (0); +} + +static int +vb_handle(struct vm *vm, int vcpuid, bool in, int port, int bytes, + uint32_t *val, void *arg) +{ + struct vb_softc *vs = arg; + int cfgoffset; + uint16_t offset; +#ifdef VB_STATUS_DEBUG + const char *cmd; +#endif + offset = port - vs->vs_io_start; + + SCRATCHCHECK(vs); + cfgoffset = VIRTIO_PCI_CONFIG_OFF(vs->vs_msix_enabled); + + /* Handle the device-specific config region */ + if (offset >= cfgoffset) { + offset -= cfgoffset; + return (vb_handle_config(vs, offset, in, bytes, val)); + } +#ifdef VB_STATUS_DEBUG + cmd = offset > VIRTIO_CMD_MAX ? "invalid offset" : pci_cmds[offset]; +#endif + if (in) { + switch (offset) { + case VIRTIO_PCI_HOST_FEATURES: + *val = vs->vs_hv_caps; + break; + case VIRTIO_PCI_GUEST_FEATURES: + *val = vs->vs_negotiated_caps; + break; + case VIRTIO_PCI_QUEUE_PFN: + if (vs->vs_curq < vs->vs_nvqs) + *val = vs->vs_queues[vs->vs_curq].vq_pfn; + else + *val = 0; + break; + case VIRTIO_PCI_QUEUE_NUM: + if (vs->vs_curq < vs->vs_nvqs) + *val = vs->vs_queues[vs->vs_curq].vq_qsize; + else + *val = 0; + break; + case VIRTIO_PCI_QUEUE_SEL: + *val = vs->vs_curq; + break; + case VIRTIO_PCI_QUEUE_NOTIFY: + /* write-only reg */ + *val = 0; + break; + case VIRTIO_PCI_STATUS: + /* return what was written */ + *val = vs->vs_status | VIRTIO_CONFIG_STATUS_FEATURES_OK; + break; + case VIRTIO_PCI_ISR: + /* only used for legacy interrupts */ + *val = 0; + break; + case VIRTIO_MSI_CONFIG_VECTOR: + /* not supported */ + *val = 0; + break; + case VIRTIO_MSI_QUEUE_VECTOR: + if (vs->vs_curq < vs->vs_nvqs) + *val = vs->vs_queues[vs->vs_curq].vq_msix_idx; + else + *val = VIRTIO_MSI_NO_VECTOR; + break; + default: + printf("vtnet_be: illegal reg read %d (%d)\n", + port, offset); + ; + } + } else { + /* + * Register writes + */ + switch (offset) { + case VIRTIO_PCI_HOST_FEATURES: + /* r/o reg */ + break; + case VIRTIO_PCI_GUEST_FEATURES: + printf("hv_caps: %08x hv_capenable: %08x\n", + vs->vs_hv_caps, *val & vs->vs_hv_caps); + vs->vs_negotiated_caps = *val & vs->vs_hv_caps; + break; + case VIRTIO_PCI_QUEUE_PFN: + vb_dev_pfn(vs, *val); + break; + case VIRTIO_PCI_QUEUE_NUM: + /* r/o reg */ + break; + case VIRTIO_PCI_QUEUE_SEL: + vs->vs_curq = *val; + break; + case VIRTIO_PCI_QUEUE_NOTIFY: + vb_dev_kick(vs, *val); + break; + case VIRTIO_PCI_STATUS: + vb_status_change(vs, *val); + break; + case VIRTIO_PCI_ISR: + /* ignore */ + break; + case VIRTIO_MSI_CONFIG_VECTOR: + /* ignore */ + break; + case VIRTIO_MSI_QUEUE_VECTOR: + if (vs->vs_curq < vs->vs_nvqs) + vs->vs_queues[vs->vs_curq].vq_msix_idx = *val; + break; + default: + printf("vtnet_be: illegal reg write %d (%d)\n", + port, offset); + break; + } + } + SDPRINTF("%s %s cmd: %s val: %x\n", __func__, in ? "in" : "out", + cmd, *val); + return (0); +} + +static void +vb_intr_msix(struct vb_softc *vs, int q) +{ + uint64_t addr, data; + int midx; + + if (vs->vs_msix_enabled && + !(vs->vs_queues[q].vq_avail->flags & + VRING_AVAIL_F_NO_INTERRUPT)) { + midx = vs->vs_queues[q].vq_msix_idx; + if (midx > vs->vs_nvqs) + return; + addr = vs->vs_msix[midx].addr; + data = vs->vs_msix[midx].msg; + lapic_intr_msi(vs->vs_vn->vm, addr, data); + } +} + +/* + * Input from vtnet_be iflib_rxeof (output from guest) + */ +static void +vb_if_input(struct ifnet *ifp, struct mbuf *m) +{ + + if (ifp->if_bridge == NULL) { + m_freechain(m); + return; + } + (void)(*(ifp)->if_bridge_output)(ifp, m, NULL, NULL); +} + +static void +vb_dev_kick(struct vb_softc *vs, uint32_t q) +{ + if (__predict_false(q == vs->vs_nvqs-1)) { + iflib_admin_intr_deferred(vs->vs_ctx); + return; + } + switch (q & 1) { + case VB_RXQ_IDX: + iflib_tx_intr_deferred(vs->vs_ctx, vq2rxq(q)); + break; + case VB_TXQ_IDX: + iflib_rx_intr_deferred(vs->vs_ctx, vq2txq(q)); + break; + } +} + +static void +vb_vring_munmap(struct vb_softc *vs, int q) +{ + int qid; + +#ifdef GUEST_OVERCOMMIT + int i; + + if (vs->vs_queues[q].vq_addr == 0) + return; + + vs->vs_flags &= ~VS_READY; + pmap_qremove(vs->vs_queues[q].vq_addr, vs->vs_queues[q].vq_pages); + + kva_free(vs->vs_queues[q].vq_addr, + vs->vs_queues[q].vq_pages * PAGE_SIZE); + + for (i = 0; i < vs->vs_queues[q].vq_pages; i++) { + /* Free this up */ + vm_gpa_release(vs->vs_queues[q].vq_m[i]); + vs->vs_queues[q].vq_m[i] = NULL; + } +#else + vs->vs_flags &= ~VS_READY; +#endif + vs->vs_queues[q].vq_lastpfn = 0; + vs->vs_queues[q].vq_addr = 0; + vs->vs_queues[q].vq_avail = NULL; + vs->vs_queues[q].vq_used = NULL; + + if (q == vs->vs_nvqs-1) { + vs->vs_cvq = NULL; + } else if ((q & 1) == VB_RXQ_IDX) { + qid = vq2rxq(q); + vs->vs_tx_queues[qid].vt_base = NULL; + } else if ((q & 1) == VB_TXQ_IDX) { + qid = vq2txq(q); + vs->vs_rx_queues[qid].vr_base = NULL; + vs->vs_rx_queues[qid].vr_avail = NULL; + } +} + +static void +vb_vring_mmap(struct vb_softc *vs, uint32_t pfn, int q) +{ + vm_offset_t vaddr; + uint64_t gpa, pfn64; + int qsz, len, qid; + + pfn64 = pfn; + gpa = pfn64 << PAGE_SHIFT; + + if (q >= vs->vs_nvqs) + return; + if (q >= vs->vs_nvqs) + vs->vs_nvqs = q+1; + /* + * Guest memory is linear and direct-mapped: the translation + * to a host virtual address is a simple base+offset. + * The use of 'lastpfn' to try and cache mappings is not + * needed, but since it's used in common code it's probably + * worth keeping in sync. + */ +#ifdef GUEST_OVERCOMMIT + /* + * In bhyve, guest memory may not be contiguous, or even + * present, in host memory. To simplify the virtio implementation, + * the pages will be wired in host memory, and then remapped + * to contiguous host virtual memory. + */ + + /* First, release the current set of pages if any */ + vb_vring_munmap(vs, q); + + /* Alloc new mapping */ + vaddr = kva_alloc(vs->vs_queues[q].vq_pages * PAGE_SIZE); + KASSERT(vaddr != 0, ("vtnet_be: kva_alloc NULL")); + for (int i = 0; i < vs->vs_queues[q].vq_pages; i++) { + void *vret; + vm_paddr_t tgpa; + + tgpa = gpa + (i * PAGE_SIZE); + vret = vm_gpa_hold(vs->vs_vn->vm, 0 /* XXX */, tgpa, PAGE_SIZE, + VM_PROT_RW, + (void **) &vs->vs_queues[q].vq_m[i]); + KASSERT(vret != NULL, ("vtnet_be: NULL gpa_hold")); + } + pmap_qenter(vaddr, vs->vs_queues[q].vq_m, + vs->vs_queues[q].vq_pages); + +#endif + + /* + * Set up queue pointers. Same logic as bhyve's vi_vq_init() + */ + qsz = vs->vs_queues[q].vq_qsize; + vaddr = vm_gpa_to_kva(vs->vs_vn->vm, gpa, qsz*sizeof(struct vring_desc), + &vs->vs_gpa_hint); + MPASS(vaddr); + vs->vs_queues[q].vq_addr = vaddr; + vs->vs_queues[q].vq_desc = (struct vring_desc *)vaddr; + + gpa += qsz * sizeof(struct vring_desc); + len = (2 + qsz + 1) * sizeof(uint16_t); + vaddr = vm_gpa_to_kva(vs->vs_vn->vm, gpa, len, &vs->vs_gpa_hint); + MPASS(vaddr); + vs->vs_queues[q].vq_avail = (struct vring_avail *)vaddr; + if (q == vs->vs_nvqs-1) { + vs->vs_cvq = (void *)(uintptr_t)&vs->vs_queues[q]; + } else if ((q & 1) == VB_RXQ_IDX) { + qid = vq2rxq(q); + vs->vs_tx_queues[qid].vt_base = (void *)(uintptr_t)vs->vs_queues[q].vq_desc; + } else if ((q & 1) == VB_TXQ_IDX) { + qid = vq2txq(q); + vs->vs_rx_queues[qid].vr_base = (void *)(uintptr_t)vs->vs_queues[q].vq_desc; + vs->vs_rx_queues[qid].vr_avail = (void *)(uintptr_t)vs->vs_queues[q].vq_avail->ring; + } + gpa += (2 + qsz + 1) * sizeof(uint16_t); + gpa = roundup2(gpa, PAGE_SIZE); + len = (2 + qsz + 1) * sizeof(uint16_t); + vaddr = vm_gpa_to_kva(vs->vs_vn->vm, gpa, (2 + qsz + 1) * sizeof(uint16_t), + &vs->vs_gpa_hint); + /* XXX should fail out if it's outside the range of the guest */ + MPASS(vaddr); + vs->vs_queues[q].vq_used = (struct vring_used *)vaddr; + printf("[%d].vq_used = %p flags: %d qsize: %d len: %d\n", q, (void *)vaddr, vs->vs_queues[q].vq_used->flags, qsz, len); + + if (q == vs->vs_nvqs-1) + vs->vs_flags |= VS_READY; + SCRATCHCHECK(vs); +} + +static void +vb_dev_pfn(struct vb_softc *vs, uint32_t pfn) +{ + int q; + + q = vs->vs_curq; + if (q >= vs->vs_nvqs) + return; + + /* Only map if the pfn has changed */ + if (vs->vs_queues[q].vq_lastpfn != pfn && pfn == 0) { + vb_vring_munmap(vs, q); + return; + } + /* Only map if the pfn has changed */ + if (vs->vs_queues[q].vq_lastpfn != pfn) + vb_vring_mmap(vs, pfn, q); + + vs->vs_queues[q].vq_pfn = pfn; +} + +static void +vb_dev_reset(struct vb_softc *vs) +{ + int i; + + vs->vs_generation++; + + vs->vs_status = 0; + vs->vs_curq = 0; + vs->vs_negotiated_caps = 0; + vs->vs_cvq = NULL; + + for (i = 0; i < vs->vs_nvqs; i++) { + vs->vs_queues[i].vq_qsize = vs->shared->isc_nrxd[0]; + vs->vs_queues[i].vq_lastpfn = vs->vs_queues[i].vq_pfn; + vs->vs_queues[i].vq_pfn = 0; + vs->vs_queues[i].vq_msix_idx = 0; + vs->vs_queues[i].vq_id = i; + } + SCRATCHCHECK(vs); +} + +static int +vb_dev_msix(struct vb_softc *vs, const struct vb_msix *vx, int length) +{ + int i, size; + + if (vs->vs_proc != curproc) + return (EINVAL); + + size = sizeof(*vx) + vx->vm_count*sizeof(struct vb_msix_vector); + if (length != (sizeof(*vx) + vx->vm_count*sizeof(struct vb_msix_vector))) { + printf("__func__, bad length for vb_msix %d -- expected %d \n", length, size); + return (EINVAL); + } + if (vx->vm_count < 3 || vx->vm_count > (2*VB_QUEUES_MAX) + 1) { + printf("%s bad count %d\n", __func__, vx->vm_count); + return (EINVAL); + } + + vs->vs_msix_enabled = vx->vm_status; + if (vx->vm_status) { + for (i = 0; i < vx->vm_count; i++) { + vs->vs_msix[i].addr = vx->vm_q[i].addr; + vs->vs_msix[i].msg = vx->vm_q[i].msg; + } + } + SCRATCHCHECK(vs); + return (0); +} + +static struct vtnet_be * +vtnet_be_init_(struct vm *vm) +{ + struct vtnet_be *vb; + + vb = malloc(sizeof(struct vtnet_be), M_VMNIC, M_WAITOK | M_ZERO); + vb->vm = vm; + vb->name = vm_name(vm); + + VB_LOCK; + SLIST_INSERT_HEAD(&vb_head, vb, next); + VB_UNLOCK; + + return (vb); +} + +static void +vtnet_be_cleanup_(struct vtnet_be *vb) +{ + struct vb_softc *vs; + int i; + + VB_LOCK; + while (!SLIST_EMPTY(&vb->dev)) { + vs = SLIST_FIRST(&vb->dev); + SLIST_REMOVE(&vb->dev, vs, vb_softc, vs_next); + VB_UNLOCK; + iflib_link_state_change_unlocked(vs->vs_ctx, LINK_STATE_DOWN, IF_Gbps(200)); + vs->vs_flags &= ~(VS_ENQUEUED|VS_READY); + + /* + * Only destroy the interface if it was + * created by bhyve + */ + vs->vs_vn = NULL; + vs->vs_proc = NULL; + vs->vs_negotiated_caps = 0; + vs->vs_io_start = 0; + vs->vs_io_size = 0; + vs->vs_msix_enabled = 0; + for (i = 0; i < vs->shared->isc_nrxqsets; i++) { + vs->vs_rx_queues[i].vr_cidx = 0; + vs->vs_rx_queues[i].vr_pidx = 0; + } + for (i = 0; i < vs->shared->isc_ntxqsets; i++) { + vs->vs_tx_queues[i].vt_cidx = 0; + } + VB_LOCK; + } + SLIST_REMOVE(&vb_head, vb, vtnet_be, next); + VB_UNLOCK; + + free(vb, M_VMNIC); +} + +static if_pseudo_t vb_pseudo; + +static int +vmnic_module_init(void) +{ + vb_pseudo = vb_clone_register(); + mtx_init(&vb_mtx, "vtnet_be", NULL, MTX_DEF); + vtnet_be_cleanup = vtnet_be_cleanup_; + vtnet_be_init = vtnet_be_init_; + return (0); +} + +static void +vmnic_module_deinit(void) +{ + /* XXX refcount --- check */ + iflib_clone_deregister(vb_pseudo); + mtx_destroy(&vb_mtx); + vtnet_be_cleanup = NULL; + vtnet_be_init = NULL; +} + +static void +vb_rxq_init(struct vb_softc *vs, struct vb_rxq *rxq, int idx) +{ + rxq->vr_vs = vs; + rxq->vr_pidx = rxq->vr_cidx = 0; + rxq->vr_idx = idx; + /* one bit beyond indicates whether we've wrapped an + * even or odd number of times + */ + rxq->vr_shift = ffs(vs->shared->isc_nrxd[0])-1; + rxq->vr_used = malloc(sizeof(uint16_t)*vs->shared->isc_nrxd[0]*2, + M_VMNIC, M_WAITOK|M_ZERO); + rxq->vr_completion = malloc(sizeof(uint8_t)*vs->shared->isc_nrxd[0], + M_VMNIC, M_WAITOK|M_ZERO); +} + +static void +vb_rxq_deinit(struct vb_softc *vs, struct vb_rxq *rxq) +{ + free(rxq->vr_used, M_VMNIC); + free(rxq->vr_completion, M_VMNIC); +} + +static void +vb_txq_init(struct vb_softc *vs, struct vb_txq *txq, int idx) +{ + txq->vt_idx = idx; + txq->vt_vs = vs; + txq->vt_cidx = 0; +} + +static int +vb_get_queues(SYSCTL_HANDLER_ARGS) +{ + struct vb_softc *vs = oidp->oid_arg1; + struct sbuf *sb; + struct vb_rxq *rxq; + int i, j, rc; + uint8_t *completion; + + if ((rc = sysctl_wire_old_buffer(req, 0))) + return (rc); + sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); + if (sb == NULL) + return (ENOMEM); + sbuf_printf(sb, "Queues:\n"); + for (i = 0; i < vs->vs_nqs; i++) { + rxq = &vs->vs_rx_queues[i]; + sbuf_printf(sb, "\t rx(%d) state:\n", i); + sbuf_printf(sb, "\t\t cidx: %d pidx: %d\n", + rxq->vr_cidx, rxq->vr_pidx); + completion = rxq->vr_completion; + for (j = 0; j < 16; j++) { + sbuf_printf(sb, "\t\t%64D\n", completion, ""); + completion += 64; + } + } + return (0); +} + +static int +vb_cloneattach(if_ctx_t ctx, struct if_clone *ifc, const char *name, caddr_t params) +{ + struct vb_softc *vs = iflib_get_softc(ctx); + if_softc_ctx_t scctx; + device_t dev; + + vs->vs_nqs = VB_NQUEUES_DEFAULT; + vs->vs_nvqs = 2*VB_NQUEUES_DEFAULT + 1; + scctx = vs->shared = iflib_get_softc_ctx(ctx); + vs->vs_ctx = ctx; + vs->vs_gpa_hint = 0; + dev = iflib_get_dev(ctx); + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "queue_dump", CTLTYPE_STRING | CTLFLAG_RD, vs, 0, + vb_get_queues, "A", "Dump private queues"); + + scctx->isc_tx_nsegments = VB_MAX_SCATTER; + scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments; + scctx->isc_tx_tso_size_max = VB_TSO_SIZE; + scctx->isc_tx_tso_segsize_max = VB_TSO_SEG_SIZE; + scctx->isc_nrxqsets_max = scctx->isc_ntxqsets_max = + scctx->isc_nrxqsets = scctx->isc_ntxqsets = vs->vs_nqs; + scctx->isc_capenable = VB_CAPS; + scctx->isc_txrx = &vb_txrx; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_IP6_TCP \ + | CSUM_IP6_UDP | CSUM_IP6_TCP; +#ifdef notyet + scctx->isc_tx_csum_flags |= CSUM_SCTP | CSUM_IP6_SCTP; +#endif + + /* Set up host capabilities */ + vs->vs_hv_caps |= VIRTIO_NET_F_MAC | VIRTIO_NET_F_MRG_RXBUF | + VIRTIO_NET_F_STATUS | VIRTIO_F_NOTIFY_ON_EMPTY | VIRTIO_NET_F_CTRL_VQ | \ + VIRTIO_NET_F_MQ; + return (0); +} + +static int +vb_attach_post(if_ctx_t ctx) +{ + struct vb_softc *vs = iflib_get_softc(ctx); + if_softc_ctx_t scctx = vs->shared; + struct ifnet *ifp; + char buf[32]; + int i, nvq; + + MPASS(scctx->isc_nrxqsets); + MPASS(scctx->isc_ntxqsets); + MPASS(scctx->isc_ntxqsets == scctx->isc_nrxqsets); + vs->vs_nqs = scctx->isc_nrxqsets; + nvq = vs->vs_nvqs; + vs->vs_queues = malloc(nvq*sizeof(struct vb_queue), M_VMNIC, M_WAITOK|M_ZERO); + vs->vs_msix = malloc(nvq*sizeof(struct vb_msix), M_VMNIC, M_WAITOK|M_ZERO); + vs->vs_rx_queues = malloc(sizeof(struct vb_rxq)*scctx->isc_nrxqsets, + M_VMNIC, M_WAITOK|M_ZERO); + vs->vs_tx_queues = malloc(sizeof(struct vb_txq)*scctx->isc_ntxqsets, + M_VMNIC, M_WAITOK|M_ZERO); + MPASS(scctx->isc_nrxqsets >= 1); + MPASS(scctx->isc_ntxqsets >= 1); + for (i = 0; i < scctx->isc_nrxqsets; i++) { + vb_rxq_init(vs, &vs->vs_rx_queues[i], i); + snprintf(buf, sizeof(buf), "rxq%d", i); + iflib_softirq_alloc_generic(ctx, NULL, IFLIB_INTR_RX, i, buf); + } + for (i = 0; i < scctx->isc_ntxqsets; i++) { + vb_txq_init(vs, &vs->vs_tx_queues[i], i); + snprintf(buf, sizeof(buf), "txq%d", i); + iflib_softirq_alloc_generic(ctx, NULL, IFLIB_INTR_TX, i, buf); + } + vb_dev_reset(vs); + iflib_config_gtask_init(vs->vs_ctx, &vs->vs_deferred_uptask, deferred_up, "deferred up"); + scctx->isc_min_tx_latency = 1; + scctx->isc_rx_mvec_enable = 1; + scctx->isc_txrx_budget_bytes_max = VMNIC_RX_BYTES_MAX; + scctx->isc_tx_qdepth = VMNIC_RX_QDEPTH; + /* + * If interface was created by bhyve + * plug everything together here + */ + ifp = iflib_get_ifp(ctx); + ifp->if_input = vb_if_input; + SCRATCHCHECK(vs); + return (0); +} + +static int +vb_if_attach(struct vb_softc *vs) +{ + struct ifnet *ifp = iflib_get_ifp(vs->vs_ctx); + + if (ifp->if_capabilities & IFCAP_TXCSUM) + vs->vs_hv_caps |= VIRTIO_NET_F_CSUM; + if (ifp->if_capabilities & IFCAP_TSO4) + vs->vs_hv_caps |= VIRTIO_NET_F_HOST_TSO4; + if (ifp->if_capabilities & IFCAP_TSO6) + vs->vs_hv_caps |= VIRTIO_NET_F_HOST_TSO6; + if ((ifp->if_capabilities & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) == + (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) + vs->vs_hv_caps |= VIRTIO_NET_F_GUEST_CSUM; + if (ifp->if_capabilities & IFCAP_LRO) + vs->vs_hv_caps |= VIRTIO_NET_F_GUEST_TSO4 | VIRTIO_NET_F_GUEST_TSO6; + + vs->vs_hv_caps |= VIRTIO_NET_F_MTU; + if (bootverbose) { + printf("caps: %016lx encaps: %016lx hv_caps: %08x\n", + ifp->if_capabilities, ifp->if_capenable, vs->vs_hv_caps); + printf("mac: %6D status: %x max_vq_pairs: %d mtu: %d\n", + vs->vs_cfg.mac, ":", vs->vs_cfg.status, vs->vs_cfg.max_virtqueue_pairs, + vs->vs_cfg.mtu); + } + SCRATCHCHECK(vs); + return (0); +} + +static int +vb_vm_attach(struct vb_softc *vs, const struct vb_vm_attach *vva) +{ + struct vtnet_be *vb; + int rc; + + /* Locate VM */ + vb = vb_find_vmname(vva->vva_vmparent); + if (vb == NULL) { + printf("vtnet_be: vmname %s doesn't exist\n", vva->vva_vmparent); + return (ENOENT); + } + vs->vs_vn = vb; + vs->vs_proc = curproc; + vs->vs_io_start = vva->vva_io_start; + vs->vs_io_size = vva->vva_io_size; + vs->vs_cfg.status = VIRTIO_NET_S_LINK_UP; + vs->vs_cfg.max_virtqueue_pairs = vs->vs_nqs; + /* Register the memory region with the VM */ + rc = vm_register_ioport(vs->vs_vn->vm, vb_handle, vs, + vs->vs_io_start, vs->vs_io_size); + + if (rc) { + printf("vm_register_ioport failed: %d\n", rc); + vs->vs_proc = NULL; + return (rc); + } + VB_LOCK; + SLIST_INSERT_HEAD(&vb->dev, vs, vs_next); + VB_UNLOCK; + vs->vs_flags |= VS_ENQUEUED; + SCRATCHCHECK(vs); + return (0); +} + +/* reset queue sizes and redrive attach */ +static int +vb_vm_deferred_attach(struct vb_softc *vs, const struct vb_vm_attach *va) +{ + int rc; + + if ((rc = vb_vm_attach(vs, va))) { + printf("vb_vm_attach failed %d\n", rc); + return (rc); + } + if ((rc = vb_if_attach(vs))) + return (rc); + if (vs->vs_flags & VS_NQS_CHANGED) { + rc = iflib_device_reinit(vs->vs_ctx); + if (!rc) + vs->vs_flags &= ~VS_NQS_CHANGED; + } + SCRATCHCHECK(vs); + return (rc); +} + +static int +vb_detach(if_ctx_t ctx) +{ + struct vb_softc *vs = iflib_get_softc(ctx); + if_softc_ctx_t scctx = vs->shared; + struct vtnet_be *vb = vs->vs_vn; + struct ifnet *ifp = iflib_get_ifp(ctx);; + int i; + + if (vs->vs_flags & VS_ENQUEUED) { + VB_LOCK; + SLIST_REMOVE(&vb->dev, vs, vb_softc, vs_next); + VB_UNLOCK; + } + for (i = 0; i < vs->vs_nvqs; i++) { + vb_vring_munmap(vs, i); + } + if (ifp->if_bridge) + vpcp_port_disconnect_ifp(ifp); + MPASS(ifp->if_bridge == NULL); + iflib_link_state_change(vs->vs_ctx, LINK_STATE_DOWN, IF_Gbps(200)); + + for (i = 0; i < scctx->isc_nrxqsets; i++) + vb_rxq_deinit(vs, &vs->vs_rx_queues[i]); + iflib_config_gtask_deinit(&vs->vs_deferred_uptask); + free(vs->vs_rx_queues, M_VMNIC); + free(vs->vs_tx_queues, M_VMNIC); + free(vs->vs_msix, M_VMNIC); + free(vs->vs_queues, M_VMNIC); + + return (0); +} + +static int +vmnic_mtu_set(if_ctx_t ctx, uint32_t mtu) +{ + struct vb_softc *vs = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + + if (vs->vs_flags & VS_IMMUTABLE) + return (EBUSY); + ifp->if_mtu = mtu; + vs->vs_cfg.mtu = mtu; + SCRATCHCHECK(vs); + return (0); +} + +static int +vmnic_mac_set(if_ctx_t ctx, const uint8_t *mac) +{ + struct vb_softc *vs = iflib_get_softc(ctx); + struct ifnet *ifp; + struct sockaddr_dl *sdl; + + if (vs->vs_flags & VS_IMMUTABLE) + return (EBUSY); + ifp = iflib_get_ifp(ctx); + memcpy(vs->vs_origmac, mac, ETHER_ADDR_LEN); + memcpy(vs->vs_cfg.mac, mac, ETHER_ADDR_LEN); + sdl = (struct sockaddr_dl *)ifp->if_addr->ifa_addr; + MPASS(sdl->sdl_type == IFT_ETHER); + memcpy(LLADDR(sdl), vs->vs_origmac, ETHER_ADDR_LEN); + SCRATCHCHECK(vs); + return (0); +} + +static void +vb_init(if_ctx_t ctx) +{ +} + +static void +vb_stop(if_ctx_t ctx) +{ + struct vb_softc *vs = iflib_get_softc(ctx); + if_softc_ctx_t scctx = vs->shared; + int i, j; + + MPASS(scctx->isc_nrxqsets); + MPASS(scctx->isc_nrxd[0]); + + for (i = 0; i < scctx->isc_nrxqsets; i++) { + if (vs->vs_rx_queues[i].vr_sdcl) { + for (j = 0; j < scctx->isc_nrxd[0]; j++) + vs->vs_rx_queues[i].vr_sdcl[j] = NULL; + } + } + for (i = 0; i < scctx->isc_ntxqsets; i++) + vs->vs_tx_queues[i].vt_cidx = 0; + SCRATCHCHECK(vs); +} + +static void +vb_rx_clset(if_ctx_t ctx, uint16_t fl __unused, uint16_t qidx, + caddr_t *sdcl) +{ + struct vb_softc *vs = iflib_get_softc(ctx); + + vs->vs_rx_queues[qidx].vr_sdcl = sdcl; + SCRATCHCHECK(vs); +} + +int +vmnic_ctl(vpc_ctx_t vctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata) +{ + if_ctx_t ctx = vctx->v_ifp->if_softc; + struct vb_softc *vs = iflib_get_softc(ctx); + int rc; + + rc = 0; + switch (op) { + case VPC_VMNIC_OP_NQUEUES_GET: { + uint16_t *nqsp; + + if (*outlen < sizeof(uint16_t)) + return (EOVERFLOW); + *outlen = sizeof(uint16_t); + nqsp = malloc(sizeof(uint16_t), M_TEMP, M_WAITOK); + *nqsp = vs->vs_nqs; + *outdata = nqsp; + break; + } + case VPC_VMNIC_OP_NQUEUES_SET: { + const uint16_t *nqsp = in; + + if (vs->vs_flags & VS_IMMUTABLE) + return (EBUSY); + if (inlen != sizeof(uint16_t)) + return (EBADRPC); + if (*nqsp == 0) + return (ENXIO); + if (*nqsp > VB_QUEUES_MAX) + return (E2BIG); + if (vs->vs_nqs != *nqsp) { + vs->vs_nqs = *nqsp; + vs->vs_nvqs = (vs->vs_nqs * 2) + 1; + vs->vs_flags |= VS_NQS_CHANGED; + } + break; + } + case VPC_VMNIC_OP_CONNECT: { +#ifdef DEVEL_BRINGUP + iflib_set_mtu(ctx, 1500); +#else + if (!(vs->vs_flags & VS_IMMUTABLE)) + return (EINPROGRESS); +#endif + vs->vs_proc = curproc; + rc = vb_vm_deferred_attach(vs, in); + break; + } + case VPC_VMNIC_OP_MSIX: + rc = vb_dev_msix(vs, (const struct vb_msix *)in, inlen); + break; + case VPC_VMNIC_OP_FREEZE: + vs->vs_flags |= VS_IMMUTABLE; + break; + case VPC_VMNIC_OP_UNFREEZE: + if (vs->vs_flags & VS_ATTACHED) + return (EBUSY); + vs->vs_flags &= ~VS_IMMUTABLE; + break; + default: + rc = ENOIOCTL; + break; + } + SCRATCHCHECK(vs); + return (rc); +} + +static int +vb_if_txq_intr_enable(if_ctx_t ctx, uint16_t txqid) +{ + return (0); +} + +static int +vb_if_rxq_intr_enable(if_ctx_t ctx, uint16_t rxqid) +{ + return (0); +} + + +typedef int (*ctrl_cmd_handler_t)(struct vb_softc *vs, int cmd, void *arg, int len); +static int vb_ctrl_rx(struct vb_softc *vs, int cmd, void *arg, int len) { return (ENOTSUP); } +static int vb_ctrl_mac(struct vb_softc *vs, int cmd, void *arg, int len) { return (ENOTSUP); } +static int vb_ctrl_vlan(struct vb_softc *vs, int cmd, void *arg, int len) { return (ENOTSUP); } +static int vb_ctrl_announce(struct vb_softc *vs, int cmd, void *arg, int len) { return (ENOTSUP); } +static int vb_ctrl_mq(struct vb_softc *vs, int cmd, void *arg, int len); +static int vb_ctrl_guest_offloads(struct vb_softc *vs, int cmd, void *arg, int len) { return (ENOTSUP); } +static ctrl_cmd_handler_t handlers[] = { + vb_ctrl_rx, + vb_ctrl_mac, + vb_ctrl_vlan, + vb_ctrl_announce, + vb_ctrl_mq, + vb_ctrl_guest_offloads +}; + +static int +vb_ctrl_mq(struct vb_softc *vs, int cmd, void *arg, int len) +{ + struct virtio_net_ctrl_mq *mq = arg; + uint16_t requested; + + if (cmd != VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { + printf("unexpected command %u in %s\n", cmd, __func__); + return (EINVAL); + } + requested = mq->virtqueue_pairs; + if (requested < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || + requested > vs->vs_cfg.max_virtqueue_pairs) { + printf("attempted to set bad value virtqueue_pairs=%d -- limit is 1-%d\n", + requested, vs->vs_cfg.max_virtqueue_pairs); + return (EINVAL); + } + printf("%s cmd: %d requested: %d\n", + __func__, cmd, requested); +#ifdef notyet + if (requested != vs->vs_nqs) { + vs->vs_nqs = requested; + vs->vs_nvqs = 2*requested + 1; + vb_dev_reset(vs); + } +#endif + return (0); +} + +static int +vb_ctrl_cmd_dispatch(struct vb_softc *vs, struct virtio_net_ctrl_hdr *hdr, void *data, int len) +{ + + if (hdr->class > VIRTIO_NET_CTRL_GUEST_OFFLOADS) + return (EINVAL); + return (handlers[hdr->class])(vs, hdr->cmd, data, len); +} + +static void +vb_update_admin_status(if_ctx_t ctx) +{ + struct vb_softc *vs = iflib_get_softc(ctx); + struct vb_queue *cvq = vs->vs_cvq; + volatile struct vring_desc *desc; + volatile struct vring_used_elem *vue; + struct virtio_net_ctrl_hdr *hdr; + int aidx, didx, vidx, len, rc, mask; + uint8_t *ackp; + + /* We aren't set up yet */ + if (cvq == NULL) { + DPRINTF("%s called - cvq not setup\n", __func__); + return; + } + mask = cvq->vq_qsize-1; + vidx = cvq->vq_used->idx; + /* handle queue requests */ + for (aidx = cvq->vq_avail_idx; aidx != cvq->vq_avail->idx; aidx++) { + len = 0; + didx = cvq->vq_avail->ring[aidx]; + desc = &cvq->vq_desc[didx]; + if (desc->len != sizeof(*hdr)) { + printf("control fail: desc->len=%d\n", desc->len); + goto update_used; + } + len = desc->len; + hdr = vb_cl_map(vs, desc); + if (hdr == NULL || !(desc->flags & VRING_DESC_F_NEXT) || + desc->next >= cvq->vq_qsize) { + printf("control fail 1) flags=%x next=%d\n", desc->flags, desc->next); + goto update_used; + } + desc = &cvq->vq_desc[desc->next]; + len += desc->len; + rc = vb_ctrl_cmd_dispatch(vs, hdr, vb_cl_map(vs, desc), desc->len); + if (rc || !(desc->flags & VRING_DESC_F_NEXT) || + (desc->next >= cvq->vq_qsize)) { + printf("control fail 2) flags=%x next=%d\n", desc->flags, desc->next); + goto update_used; + } + desc = &cvq->vq_desc[desc->next]; + if ((!(desc->flags & VRING_DESC_F_WRITE)) || + desc->len != sizeof(uint8_t)) { + goto update_used; + } + len += desc->len; + ackp = vb_cl_map(vs, desc); + *ackp = VIRTIO_NET_OK; + update_used: + vue = &cvq->vq_used->ring[vidx++ & mask]; + vue->id = didx; + vue->len = len; + } +/* Update used ring to reflect final state */ + wmb(); + cvq->vq_used->idx = vidx; + cvq->vq_avail_idx = aidx; + vb_intr_msix(vs, cvq->vq_id); +} + +static void +vmnic_watchdog_reset_queue(if_ctx_t ctx, uint16_t qid) +{ + struct vb_softc *vs = iflib_get_softc(ctx); + + vb_intr_msix(vs, qid); +} + +static int +vmnic_promisc_set(if_ctx_t ctx __unused, int flags __unused) +{ + return (0); +} + +static device_method_t vb_if_methods[] = { + DEVMETHOD(ifdi_rx_queue_intr_enable, vb_if_rxq_intr_enable), + DEVMETHOD(ifdi_tx_queue_intr_enable, vb_if_txq_intr_enable), + DEVMETHOD(ifdi_cloneattach, vb_cloneattach), + DEVMETHOD(ifdi_attach_post, vb_attach_post), + DEVMETHOD(ifdi_watchdog_reset_queue, vmnic_watchdog_reset_queue), + DEVMETHOD(ifdi_detach, vb_detach), + DEVMETHOD(ifdi_mtu_set, vmnic_mtu_set), + DEVMETHOD(ifdi_mac_set, vmnic_mac_set), + DEVMETHOD(ifdi_init, vb_init), + DEVMETHOD(ifdi_stop, vb_stop), + DEVMETHOD(ifdi_rx_clset, vb_rx_clset), + DEVMETHOD(ifdi_promisc_set, vmnic_promisc_set), + DEVMETHOD(ifdi_update_admin_status, vb_update_admin_status), + DEVMETHOD_END +}; + +static driver_t vb_iflib_driver = { + "vmnic", vb_if_methods, sizeof(struct vb_softc) +}; + +char vb_driver_version[] = "0.0.1"; + +#define VB_MIN_TXD 128 +#define VB_MAX_TXD 4096 +#define VB_DEFAULT_TXD 1024 +#define VB_MIN_RXD 128 +#define VB_MAX_RXD 4096 +#define VB_DEFAULT_RXD 1024 + +static struct if_shared_ctx vb_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_tx_maxsize = VB_TSO_SIZE, + .isc_tx_maxsegsize = USHRT_MAX, + .isc_rx_maxsize = USHRT_MAX, + .isc_rx_nsegments = 1, + .isc_rx_maxsegsize = USHRT_MAX, + .isc_nfl = 1, + .isc_nrxqs = 2, + .isc_ntxqs = 1, + .isc_driver_version = vb_driver_version, + .isc_driver = &vb_iflib_driver, + .isc_flags = IFLIB_TXD_ENCAP_PIO | IFLIB_RX_COMPLETION | \ + IFLIB_SKIP_CLREFILL | IFLIB_HAS_RXCQ | IFLIB_NO_HANG_RESET | \ + IFLIB_VIRTUAL, + + .isc_nrxd_min = {VB_MIN_RXD, VB_MIN_RXD}, + .isc_ntxd_min = {VB_MIN_TXD}, + .isc_nrxd_max = {VB_MAX_RXD, VB_MAX_RXD}, + .isc_ntxd_max = {VB_MAX_TXD}, + .isc_nrxd_default = {VB_DEFAULT_RXD, VB_DEFAULT_RXD}, + .isc_ntxd_default = {VB_DEFAULT_TXD}, + .isc_name = "vmnic", + .isc_rx_completion = vb_rx_completion, +}; + +if_shared_ctx_t vb_sctx = &vb_sctx_init; + +static if_pseudo_t +vb_clone_register(void) +{ + return (iflib_clone_register(vb_sctx)); +} + +static int +vmnic_module_event_handler(module_t mod, int what, void *arg) +{ + int err; + + switch (what) { + case MOD_LOAD: + if ((err = vmnic_module_init()) != 0) + return (err); + break; + case MOD_UNLOAD: + if (modrefcnt == 0) + vmnic_module_deinit(); + else + return (EBUSY); + break; + default: + return (EOPNOTSUPP); + } + + return (0); +} + +static moduledata_t vmnic_moduledata = { + "vmnic", + vmnic_module_event_handler, + NULL +}; +DECLARE_MODULE(vmnic, vmnic_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY); +MODULE_VERSION(vmnic, 1); +MODULE_DEPEND(vmnic, iflib, 1, 1, 1); +MODULE_DEPEND(vmnic, vmm, 1, 1, 1); Index: sys/amd64/vmm/net/if_vpcmux.c =================================================================== --- /dev/null +++ sys/amd64/vmm/net/if_vpcmux.c @@ -0,0 +1,1233 @@ +/* + * Copyright (C) 2017-2018 Matthew Macy + * Copyright (C) 2017-2018 Joyent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "ifdi_if.h" + +#include + +static int do_hw_vxtso = 0; +static SYSCTL_NODE(_net, OID_AUTO, vpcmux, CTLFLAG_RD, 0, + "vpcmux parameters"); +SYSCTL_INT(_net_vpcmux, OID_AUTO, hw_vxtso, CTLFLAG_RW, + &do_hw_vxtso, 0, "enable hw vxlan tso"); + + +#define DPRINTF(...) do { \ + if (__predict_false(bootverbose)) \ + printf(__VA_ARGS__); \ +} while (0) + +/* + * IPv4 w/o VLAN + */ +struct vxlan_header_l3 { + /* outer ether header */ + struct ether_header vh_ehdr; + /* outer IP header */ + struct ip vh_iphdr; + /* outer UDP header */ + struct udphdr vh_udphdr; + /* outer vxlan id header */ + struct vxlan_header vh_vxlanhdr; +} __packed; + + +struct vpcmux_ftable { + uint32_t vf_vni; + struct vpcmux_softc *vf_vs; + art_tree vf_ftable; +}; + +struct egress_cache { + uint16_t ec_hdr[3]; + uint16_t ec_pad; + int ec_ticks; + struct vxlan_header_l3 ec_vh; +}; + +static struct sx vpcmux_lock; +SX_SYSINIT(vpcmux, &vpcmux_lock, "VPC global"); + +#define VPCMUX_LOCK() sx_xlock(&vpcmux_lock) +#define VPCMUX_UNLOCK() sx_xunlock(&vpcmux_lock) + + +static DPCPU_DEFINE(struct egress_cache *, hdr_cache); + +static MALLOC_DEFINE(M_VPCMUX, "vpcmux", "virtual private cloud mux (vxlan encap)"); + +struct vpcmux_softc { + if_softc_ctx_t shared; + if_ctx_t vs_ctx; + struct ifnet *vs_ifp; + struct sockaddr vs_addr; + uint16_t vs_vxlan_port; + uint16_t vs_fibnum; + uint16_t vs_min_port; + uint16_t vs_max_port; + art_tree vs_vxftable; /* vxlanid -> ftable */ + ck_epoch_record_t vs_record; + struct ifnet *vs_underlay_ifp; + struct ifnet *vs_ethlink_ifp; + vpc_ctx_t vs_underlay_vctx; +}; + +static void vpcmux_fte_print(struct vpcmux_softc *vs); +static int vpcmux_underlay_disconnect(struct vpcmux_softc *vs); +static int clone_count; + +static __inline int +hdrcmp(uint16_t *lhs, uint16_t *rhs) +{ + return ((lhs[0] ^ rhs[0]) | + (lhs[1] ^ rhs[1]) | + (lhs[2] ^ rhs[2])); +} + +static struct vpcmux_ftable * +vpcmux_vxlanid_lookup(struct vpcmux_softc *vs, uint32_t vxlanid) +{ + + return (art_search(&vs->vs_vxftable, (const unsigned char *)&vxlanid)); +} + +static int +vpcmux_ftable_lookup(struct vpcmux_ftable *vf, struct ether_vlan_header *evh, + struct vpcmux_fte **pvfte) +{ + struct vpcmux_fte *vfte; + + vfte = art_search(&vf->vf_ftable, evh->evl_dhost); + if (__predict_false(vfte == NULL)) + return (ENOENT); + *pvfte = vfte; + return (0); +} + +static void +vpcmux_ftable_insert(struct vpcmux_ftable *vf, const struct vpcmux_fte *vftenew) +{ + struct vpcmux_fte *vfte; + + vfte = malloc(sizeof(*vfte), M_VPCMUX, M_WAITOK); + + bcopy(vftenew, vfte, sizeof(*vfte)); + art_insert(&vf->vf_ftable, (const unsigned char *)vfte->vf_hwaddr, vfte); +} + +static uint16_t +vpcmux_sport_hash(struct vpcmux_softc *vs, caddr_t data, uint32_t crcbuf[4]) +{ + uint16_t range; + uint32_t hash; + + range = vs->vs_max_port - vs->vs_min_port; + bcopy(data, &crcbuf[1], 12); + hash = sse42_crc32c(0, (const unsigned char *)crcbuf, 16); + hash %= range; + return (vs->vs_min_port + hash); +} + +static void +vpcmux_ip_init(struct vpcmux_ftable *vf, struct sockaddr *dstip, struct mbuf *m, + struct vpc_pkt_info *vpi) +{ + struct ip *ip; + struct sockaddr_in *sin; + struct vxlan_header_l3 *vh; + int len; + + len = m->m_pkthdr.len; + vh = (void *)m->m_data; + ip = (struct ip *)(uintptr_t)&vh->vh_iphdr; + ip->ip_len = htons(len - sizeof(struct ether_header)); + ip->ip_sum = 0; + ip->ip_tos = vpi->vpi_tos; + if (vf) { + ip->ip_hl = sizeof(*ip) >> 2; + ip->ip_v = 4; /* v4 only now */ + ip->ip_tos = 0; + ip->ip_id = 0; + ip->ip_off = htons(IP_DF); + ip->ip_ttl = 255; + ip->ip_p = IPPROTO_UDP; + sin = (struct sockaddr_in *)&vf->vf_vs->vs_addr; + ip->ip_src.s_addr = sin->sin_addr.s_addr; + sin = (struct sockaddr_in *)dstip; + ip->ip_dst.s_addr = sin->sin_addr.s_addr; + } + ip->ip_sum = in_cksum_hdr(ip); +} + + +static uint16_t +vpc_cksum_skip(struct mbuf *m, int len, int skip) +{ + uint16_t csum; + + if (m_ismvec(m)) + csum = mvec_cksum_skip(m, len, skip); + else + csum = in_cksum_skip(m, len, skip); + + if (__predict_false(csum == 0)) + csum = 0xffff; + return (csum); +} +static void +vpcmux_udphdr_init(struct mbuf *m) +{ + struct vxlan_header_l3 *vh; + struct udphdr *uh; + struct ip *ip; + int len; + + vh = (void*)m->m_data; + uh = (struct udphdr*)(uintptr_t)&vh->vh_udphdr; + ip = (struct ip *)(uintptr_t)&vh->vh_iphdr; + len = m->m_pkthdr.len - sizeof(*ip) - sizeof(struct ether_header); + uh->uh_ulen = htons(len); + uh->uh_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, + htons(ip->ip_p + len)); +} + +static void +vpcmux_vxlanhdr_init(struct vpcmux_ftable *vf, + struct vpcmux_fte *vfte, struct mbuf *m, + caddr_t hdr, struct vpc_pkt_info *tpi) +{ + struct ether_header *eh; + struct ip *ip; + struct udphdr *uh; + struct vxlan_header_l3 *vh; + struct vxlan_header *vhdr; + struct sockaddr *dstip; + struct ifnet *ifp; + caddr_t smac; + int len; + uint32_t crcbuf[4]; + struct mvec_cursor mc; + + vh = (void *)m->m_data; + ifp = m->m_pkthdr.rcvif; + MPASS(!(m->m_flags & M_EXT) || m_ismvec(m)); + crcbuf[0] = 0; + len = m->m_pkthdr.len; + smac = ifp->if_hw_addr; + eh = &vh->vh_ehdr; + dstip = &vfte->vf_protoaddr; + if (tpi->vpi_l4_len) { + struct tcphdr *th; + + mc.mc_idx = mc.mc_off = 0; + th = mvec_advance(m, &mc, m->m_pkthdr.encaplen + tpi->vpi_l2_len + tpi->vpi_l3_len); + crcbuf[0] = *(uint32_t *)&th->th_sport; + } + + eh->ether_type = htons(ETHERTYPE_IP); /* v4 only to start */ + /* arp resolve fills in dest */ + bcopy(smac, eh->ether_shost, ETHER_ADDR_LEN); + + vpcmux_ip_init(vf, dstip, m, tpi); + + uh = (struct udphdr*)(uintptr_t)&vh->vh_udphdr; + uh->uh_sport = htons(vpcmux_sport_hash(vf->vf_vs, hdr, crcbuf)); + //m->m_pkthdr.rsstype = M_HASHTYPE_OPAQUE; + //m->m_pkthdr.flowid = uh->uh_sport; + uh->uh_dport = vf->vf_vs->vs_vxlan_port; + vpcmux_udphdr_init(m); + vhdr = (struct vxlan_header *)(uintptr_t)&vh->vh_vxlanhdr; + vhdr->vxlh_flags = htonl(VXLAN_HDR_FLAGS_VALID_VNI); + vhdr->vxlh_vni = vf->vf_vni; + if (!(ifp->if_capenable & IFCAP_TXCSUM)) { + ip = (struct ip *)(uintptr_t)&vh->vh_iphdr; + uh->uh_sum = vpc_cksum_skip(m, ntohs(ip->ip_len) + sizeof(*eh), sizeof(*ip) + sizeof(*eh)); + } +} + +static int +vpcmux_cache_lookup(struct vpcmux_softc *vs, struct mbuf *m, struct ether_vlan_header *evh, + struct vpc_pkt_info *vpi) +{ + struct egress_cache *ecp; + + _critical_enter(); + ecp = DPCPU_GET(hdr_cache); + if (__predict_false(ecp->ec_ticks == 0)) + goto skip; + /* + * Is still in caching window + */ + if (__predict_false(ticks - ecp->ec_ticks > hz/4)) { + ecp->ec_ticks = 0; + goto skip; + } + /* + * dmac & vxlanid match + */ + if (hdrcmp(ecp->ec_hdr, (uint16_t *)evh->evl_dhost) == 0 && + (m->m_pkthdr.vxlanid == ecp->ec_vh.vh_vxlanhdr.vxlh_vni)) { + /* re-use last header */ + bcopy(&ecp->ec_vh, m->m_data, sizeof(struct vxlan_header_l3)); + _critical_exit(); + if ((m->m_pkthdr.csum_flags & CSUM_VX_TSO) == 0) { + vpcmux_ip_init(NULL, NULL, m, vpi); + vpcmux_udphdr_init(m); + } + return (1); + } + skip: + _critical_exit(); + return (0); +} + +static void +vpcmux_cache_update(struct mbuf *m, struct ether_vlan_header *evh) +{ + struct egress_cache *ecp; + uint16_t *src; + + src = (uint16_t *)evh->evl_dhost; + _critical_enter(); + /* update pcpu cache */ + ecp = DPCPU_GET(hdr_cache); + ecp->ec_hdr[0] = src[0]; + ecp->ec_hdr[1] = src[1]; + ecp->ec_hdr[2] = src[2]; + bcopy(m->m_data, &ecp->ec_vh, sizeof(struct vxlan_header_l3)); + ecp->ec_ticks = ticks; + _critical_exit(); +} + +static int +vpcmux_nd_lookup(struct vpcmux_softc *vs, const struct vpcmux_fte *vfte, uint8_t *ether_addr) +{ + const struct sockaddr *dst; + struct rtentry *rt; + if_t ifp; + int rc; + + if (__predict_false(vs->vs_ethlink_ifp == NULL)) + return (EAGAIN); + ifp = vs->vs_ethlink_ifp; + dst = &vfte->vf_protoaddr; + + rt = rtalloc1_fib((struct sockaddr*)(uintptr_t)dst, 0, 0, vs->vs_fibnum); + if (__predict_false(rt == NULL)) + return (ENETUNREACH); + if (__predict_false(!(rt->rt_flags & RTF_UP) || + (rt->rt_ifp == NULL) || + !RT_LINK_IS_UP(rt->rt_ifp))) { + RTFREE_LOCKED(rt); + return (ENETUNREACH); + } + if (rt->rt_flags & RTF_GATEWAY) + dst = rt->rt_gateway; + else + RTFREE_LOCKED(rt); + + /* get dmac */ + switch(dst->sa_family) { + case AF_INET: + rc = arpresolve(ifp, 0, NULL, dst, + ether_addr, NULL, NULL); + break; + case AF_INET6: + rc = nd6_resolve(ifp, 0, NULL, dst, + ether_addr, NULL, NULL); + break; + default: + rc = EOPNOTSUPP; + } + if (dst != &vfte->vf_protoaddr) + RTFREE_LOCKED(rt); + return (rc); +} + +static struct mbuf * +vpcmux_header_pullup(struct mbuf *mp, struct vpc_pkt_info *tpi) +{ + int minhlen; + struct mbuf *m; + + minhlen = mp->m_pkthdr.encaplen + ETHER_HDR_LEN + sizeof(struct ip) + sizeof(struct tcphdr); + MPASS(mp->m_pkthdr.len >= minhlen); + m = m_pullup(mp, mp->m_pkthdr.encaplen + tpi->vpi_l2_len + + tpi->vpi_l3_len + tpi->vpi_l4_len); + if (__predict_false(m == NULL)) + m_freem(mp); + return (m); +} + +static int +vpcmux_vxlan_encap(struct vpcmux_softc *vs, struct mbuf **mp) +{ + struct ether_vlan_header *evho, *evhi; + struct vxlan_header_l3 *vh; + struct mbuf *m; + struct vpcmux_ftable *vf; + struct vpcmux_fte *vfte; + struct ifnet *ifp; + struct vpc_pkt_info tpi; + uint32_t oldflags; + int rc, hdrsize, pktlen; + + m = *mp; + *mp = NULL; + ETHER_BPF_MTAP(iflib_get_ifp(vs->vs_ctx), m); + MPASS(m->m_flags & M_VXLANTAG); + vpc_parse_pkt(m, &tpi); + ifp = vs->vs_ethlink_ifp; + + evhi = (struct ether_vlan_header *)m->m_data; + MPASS(m->m_pkthdr.vxlanid); + hdrsize = sizeof(struct vxlan_header_l3); + oldflags = m->m_pkthdr.csum_flags; + pktlen = m->m_pkthdr.len; + m->m_nextpkt = NULL; + /* temporary */ + if (m_ismvec(m)) { + m = mvec_prepend(m, hdrsize); + } else { + struct mbuf *mh; + mh = m_gethdr(M_NOWAIT, MT_NOINIT); + if (__predict_false(mh == NULL)) { + m_freem(m); + DPRINTF("%s failed to gethdr\n", __func__); + return (ENOMEM); + } + bcopy(&m->m_pkthdr, &mh->m_pkthdr, sizeof(struct pkthdr)); + mh->m_data = mh->m_pktdat; + mh->m_nextpkt = NULL; + mh->m_flags = M_PKTHDR; + /* XXX v4 only */ + mh->m_pkthdr.len += hdrsize; + mh->m_len = hdrsize; + mh->m_next = m; + m->m_flags &= ~(M_PKTHDR|M_VXLANTAG); + m = mh; + } + MPASS(m->m_pkthdr.len == pktlen + hdrsize); + pktlen = m->m_pkthdr.len; + m->m_pkthdr.encaplen = hdrsize; + if (oldflags & CSUM_TSO) { + m = vpcmux_header_pullup(m, &tpi); + if (__predict_false(m == NULL)) { + DPRINTF("vpcmux_header_pullup failed\n"); + return (ENOMEM); + } + evhi = (void *)(m->m_data + hdrsize); + } + MPASS(m->m_pkthdr.len == pktlen); + m->m_pkthdr.csum_flags = CSUM_UDP; + m->m_pkthdr.csum_flags |= ((oldflags & CSUM_TSO) << 2); + m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); + m->m_pkthdr.rcvif = vs->vs_underlay_ifp; + + vh = (struct vxlan_header_l3 *)m->m_data; + evho = (struct ether_vlan_header *)&vh->vh_ehdr; + if (__predict_true(vpcmux_cache_lookup(vs, m, evhi, &tpi))) + goto tso_check; + /* lookup MAC->IP forwarding table */ + vf = vpcmux_vxlanid_lookup(vs, m->m_pkthdr.vxlanid); + if (__predict_false(vf == NULL)) { + DPRINTF("vxlanid %d not found\n", m->m_pkthdr.vxlanid); + m_freem(m); + return (ENOENT); + } + /* lookup IP using encapsulated dmac */ + rc = vpcmux_ftable_lookup(vf, evhi, &vfte); + if (__predict_false(rc)) { + DPRINTF("no forwarding entry for dmac: %*D\n", + ETHER_ADDR_LEN, (caddr_t)evhi, ":"); + vpcmux_fte_print(vs); + m_freem(m); + return (rc); + } + if ((rc = vpcmux_nd_lookup(vs, vfte, evho->evl_dhost))) { + DPRINTF("%s failed in nd_lookup\n", __func__); + return (rc); + } + m->m_pkthdr.rcvif = ifp; + vpcmux_vxlanhdr_init(vf, vfte, m, (caddr_t)evhi, &tpi); + vpcmux_cache_update(m, evhi); + + MPASS(m->m_pkthdr.len == m_length(m, NULL)); + /* + * do soft TSO if hardware doesn't support VXLAN offload + */ + tso_check: + if ((m->m_pkthdr.csum_flags & CSUM_VX_TSO) && + (!(ifp->if_capabilities & IFCAP_VXTSO) || !do_hw_vxtso)) { + MPASS(m_ismvec(m)); + *mp = (void *)mvec_tso((struct mbuf_ext*)m, hdrsize, true); + if (__predict_false(*mp == NULL)) { + DPRINTF("%s mvec_tso failed\n", __func__); + m_freem(m); + return (ENOMEM); + } + (*mp)->m_pkthdr.csum_flags &= ~(CSUM_TSO|CSUM_VX_TSO); + } else if (!(m->m_pkthdr.csum_flags & CSUM_VX_TSO) && + (m->m_pkthdr.len - ETHER_HDR_LEN > ifp->if_mtu)) { + DPRINTF("error: pktlen=%d > mtu + ETHER_HDR_LEN =%d\n", + m->m_pkthdr.len, ifp->if_mtu + ETHER_HDR_LEN); + m_freem(m); + } else + *mp = m; + return (0); +} + +static int +vpcmux_vxlan_encap_chain(struct vpcmux_softc *vs, struct mbuf **mp) +{ + struct mbuf *mh, *mt, *mnext, *m; + int rc; + + vpc_epoch_begin(); + + m = *mp; + *mp = mh = mt = NULL; + rc = 0; + do { + mnext = m->m_nextpkt; + m->m_nextpkt = NULL; + rc = vpcmux_vxlan_encap(vs, &m); + if (__predict_false(rc)) { + DPRINTF("encap fail in %s with %d\n", __func__, rc); + break; + } + if (mh == NULL) { + mh = mt = m; + } else { + mt->m_nextpkt = m; + mt = m; + } + MPASS(m != mnext); + m = mnext; + } while (m != NULL); + if (__predict_false(mnext != NULL)) { + DPRINTF("%s freeing after failed encap\n", __func__); + m_freechain(mnext); + } + *mp = mh; + vpc_epoch_end(); + return (rc); +} + +static int +vpcmux_mbuf_to_qid(if_t ifp __unused, struct mbuf *m __unused) +{ + return (0); +} + +static int +vpcmux_transmit(if_t ifp, struct mbuf *m) +{ + struct ifnet *oifp; + if_ctx_t ctx; + struct vpcmux_softc *vs; + struct mbuf *mp, *mh, *mt, *mhv, *mtv, *mnext; + int rc, needsconvert; + + ctx = ifp->if_softc; + vs = iflib_get_softc(ctx); + oifp = vs->vs_underlay_ifp; + mh = mt = mhv = mtv = NULL; + needsconvert = false; + rc = 0; + if (__predict_false(vs->vs_ethlink_ifp == NULL)) { + if (oifp == NULL) + return (ENOBUFS); + vs->vs_ethlink_ifp = ethlink_ifp_get(oifp->if_softc); + if (vs->vs_ethlink_ifp == NULL) + return (ENOBUFS); + } + mp = m; + do { + mnext = mp->m_nextpkt; + mp->m_nextpkt = NULL; + if (bpf_peers_present(ifp->if_bpf)) + ETHER_BPF_MTAP(ifp, mp); + if (mp->m_flags & M_VXLANTAG) { + if (mhv != NULL) { + mtv->m_nextpkt = mp; + mtv = mp; + } else + mhv = mtv = mp; + if (!m_ismvec(mp)) + needsconvert = true; + } else { + if (mh != NULL) { + mt->m_nextpkt = mp; + mt = mp; + } else + mh = mt = mp; + } + mp = mnext; + } while (mp); + if (__predict_false(oifp == NULL)) { + m_freechain(mhv); + m_freechain(mh); + return (ENOBUFS); + } + if (mh) { + rc = oifp->if_transmit_txq(oifp, mh); + if (__predict_false(rc)) { + m_freechain(mhv); + return (rc); + } + } + if (__predict_false(mhv == NULL)) + return (rc); + if (needsconvert) { + mhv = (void*)pktchain_to_mvec(mhv, 0, M_NOWAIT); + if (__predict_false(mhv == NULL)) { + return (ENOBUFS); + } + } +#ifdef INVARIANTS + mp = mhv; + do { + MPASS(mp->m_flags & M_VXLANTAG); + MPASS(m_ismvec(mp)); + mp = mp->m_nextpkt; + } while (mp != NULL); +#endif + rc = vpcmux_vxlan_encap_chain(vs, &mhv); + if (__predict_false(rc || mhv == NULL)) { + return (rc); + } +#ifdef INVARIANTS + mp = mhv; + do { + mvec_sanity(mp); + mp = mp->m_nextpkt; + } while (mp != NULL); +#endif + + return (oifp->if_transmit_txq(oifp, mhv)); +} + +static struct mbuf * +vpcmux_bridge_input(if_t ifp, struct mbuf *m) +{ + struct vpcmux_softc *vs; + + vs = ifp->if_bridge; + ETHER_BPF_MTAP(vs->vs_ifp, m); + if (vs->vs_ifp->if_bridge == NULL) + return (m); + return (*(vs->vs_ifp)->if_bridge_input)(vs->vs_ifp, m); +} + +static int +vpcmux_bridge_output(struct ifnet *ifp, struct mbuf *m, + struct sockaddr *s __unused, struct rtentry *r__unused) +{ + panic("%s should not be called", __func__); + m_freechain(m); + return (0); +} + +static void +vpcmux_bridge_linkstate(struct ifnet *ifp __unused) +{ +} + +static int +vpcmux_ftable_free_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + free(value, M_VPCMUX); + return (0); +} + +static int +vpcmux_vxftable_free_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + struct vpcmux_ftable *ftable = value; + + art_iter(&ftable->vf_ftable, vpcmux_ftable_free_callback, NULL); + free(value, M_VPCMUX); + return (0); +} + +static int +vpcmux_set_listen(struct vpcmux_softc *vs, const struct sockaddr *addr) +{ + const struct sockaddr_in *sin; + + /* v4 only XXX */ + sin = (const struct sockaddr_in *)addr; + //vs->vs_vxlan_port = sin->sin_port; + vs->vs_vxlan_port = htons(4789); + bcopy(sin, &vs->vs_addr, sizeof(*sin)); + return (0); +} + +static void +vpcmux_get_listen(struct vpcmux_softc *vs, struct sockaddr *addr) +{ + struct sockaddr_in *sin; + + /* v4 only XXX */ + sin = (struct sockaddr_in *)addr; + bcopy(&vs->vs_addr, sin, sizeof(*sin)); +} + +static int +vpcmux_underlay_connect(struct vpcmux_softc *vs, const vpc_id_t *id) +{ + struct ifnet *ifp, *vifp; + struct ifreq ifr; + vpc_ctx_t vctx; + if_ctx_t ethctx; + device_t dev; + vpc_handle_type_t *htype; + uint8_t objtype; + int rc; + + dev = iflib_get_dev(vs->vs_ctx); + if (vs->vs_vxlan_port == 0) { + device_printf(dev, "%s vxlan port not set", __func__); + return (EAGAIN); + } + if ((vctx = vmmnet_lookup(id)) == NULL) + return (ENOENT); + htype = (vpc_handle_type_t*)&vctx->v_handle_type; + objtype = htype->vht_obj_type; + if (bootverbose) { + printf("vctx: %32D\n", vctx, ":"); + printf("htype: %8D\n", htype, ":"); + } + if (objtype != VPC_OBJ_ETHLINK) { + if (bootverbose) + device_printf(dev,"bad type passed to %s id: %16D type: %d expected type: %d\n", + __func__, id, ":", objtype, VPC_OBJ_ETHLINK); + return (EINVAL); + } + if (vctx->v_ifp == NULL) { + if (bootverbose) + device_printf(dev, "underlay ethlink not attached %s id: %16D\n", + __func__, id, ":"); + return (EAGAIN); + + } + ethctx = vctx->v_ifp->if_softc; + vifp = vctx->v_ifp; + ifp = ethlink_ifp_get(ethctx); + if (!(ifp->if_capabilities & IFCAP_VXLANDECAP)) { + if (bootverbose) + device_printf(dev, "%s underlay interface %s id: %16D doesn't support vxlan decap\n", + __func__, ifp->if_xname, id, ":"); + return (EOPNOTSUPP); + } + ifr.ifr_index = vs->vs_vxlan_port; + + rc = ifp->if_ioctl(ifp, SIOCSIFVXLANPORT, (caddr_t)&ifr); + if (vs->vs_underlay_vctx) + vpcmux_underlay_disconnect(vs); + if (rc == 0) { + vmmnet_ref(vctx); + vs->vs_underlay_vctx = vctx; + vs->vs_underlay_ifp = vifp; + vs->vs_ethlink_ifp = ifp; + vifp->if_capabilities |= IFCAP_BRIDGE_BATCH; + vifp->if_bridge_input = vpcmux_bridge_input; + vifp->if_bridge_output = vpcmux_bridge_output; + vifp->if_bridge_linkstate = vpcmux_bridge_linkstate; + wmb(); + vifp->if_bridge = vs; + } + return (rc); +} + +static int +vpcmux_underlay_disconnect(struct vpcmux_softc *vs) +{ + struct ifreq ifr; + struct ifnet *ifp; + + if (vs->vs_underlay_vctx) { + ifr.ifr_index = 0; + ifp = vs->vs_underlay_ifp; + (void)ifp->if_ioctl(ifp, SIOCSIFVXLANPORT, (caddr_t)&ifr); + ifp->if_capabilities &= ~IFCAP_BRIDGE_BATCH; + ifp->if_bridge = NULL; + ifp->if_bridge_input = NULL; + ifp->if_bridge_output = NULL; + ifp->if_bridge_linkstate = NULL; + vmmnet_rele(vs->vs_underlay_vctx); + } + return (0); +} + + +static int +vpcmux_fte_update(struct vpcmux_softc *vs, const struct vpcmux_fte *vfte, bool add) +{ + struct vpcmux_ftable *ftable; + uint32_t addr, *addrp; + char buf[ETHER_ADDR_LEN]; + + /* XXX v4 */ + if (vfte->vf_protoaddr.sa_family != AF_INET) + return (EAFNOSUPPORT); + addr = ((const struct sockaddr_in *)(&vfte->vf_protoaddr))->sin_addr.s_addr; /* XXX v4 */ + ftable = vpcmux_vxlanid_lookup(vs, vfte->vf_vni); + if (ftable == NULL) { + if (add == false) + return (0); + ftable = malloc(sizeof(*ftable), M_VPCMUX, M_WAITOK|M_ZERO); + art_tree_init(&ftable->vf_ftable, ETHER_ADDR_LEN); + ftable->vf_vni = vfte->vf_vni; + ftable->vf_vs = vs; + art_insert(&vs->vs_vxftable, (const char *)&vfte->vf_vni, ftable); + } + if (add == false) { + addrp = art_delete(&ftable->vf_ftable, (const char *)&addr); + free(addrp, M_VPCMUX); + if (art_size(&ftable->vf_ftable) == 0) { + art_delete(&vs->vs_vxftable, (const char *)&vfte->vf_vni); + free(ftable, M_VPCMUX); + } + } else { + /* do an arp resolve on proto addr so that it's in cache */ + (void)vpcmux_nd_lookup(vs, vfte, buf); + vpcmux_ftable_insert(ftable, vfte); + } + return (0); +} + +static int +vpcmux_ftable_print_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + char buf[5]; + + buf[4] = 0; + inet_ntoa_r(((struct sockaddr_in *)value)->sin_addr, buf); + DPRINTF("vni: 0x%x dmac: %*D ip: %s\n", + *(uint32_t *)data, ETHER_ADDR_LEN, key, ":", buf); + return (0); +} + +static int +vpcmux_vxftable_print_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + struct vpcmux_ftable *ftable = value; + + art_iter(&ftable->vf_ftable, vpcmux_ftable_print_callback, (void*)(uintptr_t)key); + return (0); +} + +static void +vpcmux_fte_print(struct vpcmux_softc *vs) +{ + + art_iter(&vs->vs_vxftable, vpcmux_vxftable_print_callback, NULL); +} + +static int +vpcmux_ftable_count_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + uint32_t *count = data; + (*count)++; + return (0); +} + +static int +vpcmux_vxftable_count_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + struct vpcmux_ftable *ftable = value; + + art_iter(&ftable->vf_ftable, vpcmux_ftable_count_callback, data); + return (0); +} + +static int +vpc_fte_count(struct vpcmux_softc *vs) +{ + uint32_t count = 0; + + art_iter(&vs->vs_vxftable, vpcmux_vxftable_count_callback, &count); + return (count); +} + + +static int +vpcmux_ftable_copy_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + caddr_t *dst = data; +#ifdef VPCMUX_DEBUG + struct vpcmux_fte *e = value; + struct sockaddr_in *sin = (void *)&e->vf_protoaddr; + char l3s[INET_ADDRSTRLEN]; + printf("hwaddr: %6D protoaddr: %s vtag: %d vni: %d\n", + e->vf_hwaddr, ":", inet_ntoa_r(sin->sin_addr, l3s), + ntohs(e->vf_vlanid), ntohl(e->vf_vni<<8)); +#endif + memcpy(*dst, value, sizeof(struct vpcmux_fte)); + + *dst += sizeof(struct vpcmux_fte); + return (0); +} + +static int +vpcmux_vxftable_copy_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + struct vpcmux_ftable *ftable = value; + + art_iter(&ftable->vf_ftable, vpcmux_ftable_copy_callback, data); + return (0); +} + +static int +vpc_fte_copy(struct vpcmux_softc *vs, caddr_t *dst) +{ + return (art_iter(&vs->vs_vxftable, vpcmux_vxftable_copy_callback, &dst)); +} + +static int +vpcmux_fte_list(struct vpcmux_softc *vs, struct vpcmux_fte_list **vflp, size_t *length) +{ + struct vpcmux_fte_list *vfl; + int count; + + count = vpc_fte_count(vs); + if (*length == sizeof(struct vpcmux_fte_list)) { + vfl = malloc(sizeof(*vfl), M_TEMP, M_WAITOK|M_ZERO); + vfl->vfl_count = count; + *vflp = vfl; + return (0); + } + + if (*length < (sizeof(struct vpcmux_fte_list) + + count*sizeof(struct vpcmux_fte))) + return (EOVERFLOW); + *length = (sizeof(struct vpcmux_fte_list) + + count*sizeof(struct vpcmux_fte)); + vfl = malloc(*length, M_TEMP, M_WAITOK|M_ZERO); + vfl->vfl_count = count; + vpc_fte_copy(vs, (caddr_t *)vfl->vfl_vftes); + *vflp = vfl; + return (0); +} + +int +vpcmux_ctl(vpc_ctx_t ctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata) +{ + struct ifnet *ifp = ctx->v_ifp; + if_ctx_t ifctx = ifp->if_softc; + struct vpcmux_softc *vs = iflib_get_softc(ifctx); + + switch(op) { + case VPC_VPCMUX_OP_LISTEN: { + const struct sockaddr *vl_addr = in; + + if (inlen != sizeof(*vl_addr)) + return (EBADRPC); + return (vpcmux_set_listen(vs, vl_addr)); + break; + } + case VPC_VPCMUX_OP_LISTEN_ADDR_GET: { + struct sockaddr *vl_addr; + + if (*outlen < sizeof(*vl_addr)) + return (EOVERFLOW); + *outlen = sizeof(*vl_addr); + vl_addr = malloc(*outlen, M_TEMP, M_WAITOK|M_ZERO); + *outdata = vl_addr; + vpcmux_get_listen(vs, vl_addr); + break; + } + case VPC_VPCMUX_OP_UNDERLAY_CONNECT: { + const vpc_id_t *id = in; + + if (inlen != sizeof(*id)) + return (EBADRPC); + return (vpcmux_underlay_connect(vs, id)); + } + case VPC_VPCMUX_OP_UNDERLAY_DISCONNECT: { + return (vpcmux_underlay_disconnect(vs)); + } + case VPC_VPCMUX_OP_CONNECTED_ID_GET: { + vpc_id_t *id; + + if (vs->vs_underlay_vctx == NULL) + return (EAGAIN); + if (*outlen < sizeof(vpc_id_t)) + return (EOVERFLOW); + *outlen = sizeof(vpc_id_t); + id = malloc(*outlen, M_TEMP, M_WAITOK); + memcpy(id, &vs->vs_underlay_vctx->v_id, *outlen); + break; + } + case VPC_VPCMUX_OP_FTE_DEL: + case VPC_VPCMUX_OP_FTE_SET: { + const struct vpcmux_fte *vfte = in; + + if (inlen != sizeof(*vfte)) + return (EBADRPC); + return (vpcmux_fte_update(vs, vfte, op == VPC_VPCMUX_OP_FTE_SET)); + break; + } + case VPC_VPCMUX_OP_FTE_LIST: { + struct vpcmux_fte_list *vfl; + int rc; + + if ((rc = vpcmux_fte_list(vs, &vfl, outlen))) + return (rc); + *outdata = vfl; + break; + } + default: + return (EOPNOTSUPP); + } + return (0); +} + +#define VPCMUX_CAPS \ + IFCAP_TSO |IFCAP_HWCSUM | IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \ + IFCAP_VLAN_MTU | IFCAP_TXCSUM_IPV6 | IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU | IFCAP_LINKSTATE + +static int +vpcmux_cloneattach(if_ctx_t ctx, struct if_clone *ifc, const char *name, caddr_t params) +{ + struct vpcmux_softc *vs = iflib_get_softc(ctx); + if_softc_ctx_t scctx; + + atomic_add_int(&clone_count, 1); + scctx = vs->shared = iflib_get_softc_ctx(ctx); + scctx->isc_capenable = VPCMUX_CAPS; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_IP6_TCP \ + | CSUM_IP6_UDP | CSUM_IP6_TCP; + /* register vs_record */ + ck_epoch_register(&vpc_epoch, &vs->vs_record, NULL); + vs->vs_ctx = ctx; + vs->vs_ifp = iflib_get_ifp(ctx); + vs->vs_min_port = IPPORT_HIFIRSTAUTO; /* 49152 */ + vs->vs_max_port = IPPORT_HILASTAUTO; /* 65535 */ + + /* init vs_vxftable */ + art_tree_init(&vs->vs_vxftable, 3 /* VXLANID is 3 bytes */); + return (0); +} + +static int +vpcmux_attach_post(if_ctx_t ctx) +{ + if_t ifp; + + ifp = iflib_get_ifp(ctx); + if_settransmitfn(ifp, vpcmux_transmit); + if_settransmittxqfn(ifp, vpcmux_transmit); + if_setmbuftoqidfn(ifp, vpcmux_mbuf_to_qid); + /* + * should really be pulled from the lowest + * interface configured, but hardcode for now + */ + if_setmtu(ifp, ETHERMTU - 50); + return (0); +} + +static int +vpcmux_detach(if_ctx_t ctx) +{ + struct vpcmux_softc *vs = iflib_get_softc(ctx); + + ck_epoch_unregister(&vs->vs_record); + art_iter(&vs->vs_vxftable, vpcmux_vxftable_free_callback, NULL); + vpcmux_underlay_disconnect(vs); + atomic_add_int(&clone_count, -1); + return (0); +} + +static void +vpcmux_init(if_ctx_t ctx) +{ +} + +static void +vpcmux_stop(if_ctx_t ctx) +{ +} + +static device_method_t vpcmux_if_methods[] = { + DEVMETHOD(ifdi_cloneattach, vpcmux_cloneattach), + DEVMETHOD(ifdi_attach_post, vpcmux_attach_post), + DEVMETHOD(ifdi_detach, vpcmux_detach), + DEVMETHOD(ifdi_init, vpcmux_init), + DEVMETHOD(ifdi_stop, vpcmux_stop), + DEVMETHOD_END +}; + +static driver_t vpcmux_iflib_driver = { + "vpcmux", vpcmux_if_methods, sizeof(struct vpcmux_softc) +}; + +char vpcmux_driver_version[] = "0.0.1"; + +static struct if_shared_ctx vpcmux_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_driver_version = vpcmux_driver_version, + .isc_driver = &vpcmux_iflib_driver, + .isc_flags = IFLIB_PSEUDO, + .isc_name = "vpcmux", +}; + +if_shared_ctx_t vpcmux_sctx = &vpcmux_sctx_init; +static if_pseudo_t vpcmux_pseudo; + +static int +vpcmux_module_init(void) +{ + struct egress_cache **ecpp, *ecp; + int i, ec_size; + + vpcmux_pseudo = iflib_clone_register(vpcmux_sctx); + if (vpcmux_pseudo == NULL) + return (ENXIO); + + /* DPCPU hdr_cache init */ + ec_size = roundup(sizeof(*ecp), CACHE_LINE_SIZE); + ecp = malloc(ec_size*mp_ncpus, M_VPCMUX, M_WAITOK|M_ZERO); + + CPU_FOREACH(i) { + ecpp = DPCPU_ID_PTR(i, hdr_cache); + *ecpp = ecp; + ecp = (struct egress_cache *)(((caddr_t)ecp) + ec_size); + } + + return (0); +} + +static void +vpcmux_module_deinit(void) +{ + struct egress_cache *ecp; + + VPCMUX_LOCK(); + VPCMUX_UNLOCK(); + ecp = DPCPU_ID_GET(0, hdr_cache); + free(ecp, M_VPCMUX); + iflib_clone_deregister(vpcmux_pseudo); +} + + +static int +vpcmux_module_event_handler(module_t mod, int what, void *arg) +{ + int err; + + switch (what) { + case MOD_LOAD: + if ((err = vpcmux_module_init()) != 0) + return (err); + break; + case MOD_UNLOAD: + if (clone_count == 0) + vpcmux_module_deinit(); + else + return (EBUSY); + break; + default: + return (EOPNOTSUPP); + } + return (0); +} + +static moduledata_t vpcmux_moduledata = { + "vpcmux", + vpcmux_module_event_handler, + NULL +}; + +DECLARE_MODULE(vpcmux, vpcmux_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY); +MODULE_VERSION(vpcmux, 1); +MODULE_DEPEND(vpcmux, iflib, 1, 1, 1); Index: sys/amd64/vmm/net/if_vpcp.c =================================================================== --- /dev/null +++ sys/amd64/vmm/net/if_vpcp.c @@ -0,0 +1,919 @@ +/* + * Copyright (C) 2018 Matthew Macy + * Copyright (C) 2018 Joyent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include "ifdi_if.h" + + +static MALLOC_DEFINE(M_VPCP, "vpcp", "virtual machine interface port"); + + +static int soft_tso; +static uint64_t soft_tso_calls; +static SYSCTL_NODE(_net, OID_AUTO, vpcp, CTLFLAG_RD, 0, + "vpcp parameters"); + +SYSCTL_INT(_net_vpcp, OID_AUTO, soft_tso, CTLFLAG_RW, + &soft_tso, 0, "enable soft tso"); +SYSCTL_QUAD(_net_vpcp, OID_AUTO, soft_tso_calls, CTLFLAG_RD, + &soft_tso_calls, 0, "soft tso conversions"); + +#define VPCI_DEBUG +#ifdef VPCI_DEBUG +#define DPRINTF printf +#else +#define DPRINTF(...) +#endif + +static struct vpcp_softc * +ctx_to_vs(if_ctx_t ctx) +{ + return (iflib_get_softc(ctx)); +} +/* + * + */ +#define VS_IPFW_EGRESS 0x1 +#define VS_IPFW_INGRESS 0x2 + +struct vpcp_softc { + if_softc_ctx_t shared; + if_ctx_t vs_ctx; + struct ifnet *vs_ifswitch; + struct ifnet *vs_ifdev; + struct ifnet *vs_ifport; + uint32_t vs_mflags; + uint32_t vs_vxlanid; + uint32_t vs_flags; + uint16_t vs_vlanid; + vpc_id_t vs_devid; + enum vpc_obj_type vs_type; + struct ip_fw_chain *vs_chain; + void *vs_pcpu_cache; +}; + +static void vpcp_vxlanid_set(if_ctx_t ctx, uint32_t vxlanid); +static uint32_t vpcp_vxlanid_get(if_ctx_t ctx); +static void vpcp_vlanid_set(if_ctx_t ctx, uint16_t vlanid); +static uint16_t vpcp_vlanid_get(if_ctx_t ctx); +static int vpcp_port_type_set(if_ctx_t ctx, vpc_ctx_t vctx, vpc_obj_type_t type); +static int clone_count; + +static void +vpcp_stub_linkstate(if_t ifp __unused) +{ +} + +static int +vpcp_stub_transmit(if_t ifp __unused, struct mbuf *m) +{ + panic("switch port transmit should never be called"); + m_freechain(m); + return (0); +} + +static void +vpcp_stub_input(if_t ifp __unused, struct mbuf *m) +{ + m_freechain(m); +} + +static int +vpcp_stub_mbuf_to_qid(if_t ifp __unused, struct mbuf *m __unused) +{ + return (0); +} + +static void +vmi_txflags(struct mbuf *m, struct vpc_pkt_info *vpi, bool egress) +{ + if (egress) { + m->m_pkthdr.tso_segsz = m->m_pkthdr.flowid; + m->m_pkthdr.fibnum = 0; + m->m_pkthdr.flowid = vpi->vpi_hash; + } + m->m_pkthdr.l2hlen = vpi->vpi_l2_len; + m->m_pkthdr.l3hlen = vpi->vpi_l3_len; + m->m_pkthdr.l4hlen = vpi->vpi_l4_len; + + MPASS((m->m_pkthdr.csum_flags & CSUM_TSO) == 0 || + m->m_pkthdr.flowid); +} + +#ifdef INVARIANTS +static inline void +safe_mvec_sanity(const struct mbuf *m) +{ + if (m_ismvec(m)) + mvec_sanity(m); +} +#else +static inline void +safe_mvec_sanity(const struct mbuf *m __unused) {} +#endif + +static void +vpcp_input_process(struct ifnet *ifp, struct mbuf **m0, bool egress, vpc_obj_type_t type) +{ + struct vpcp_softc *vs; + struct vpc_pkt_info vpi; + struct mbuf *mnext, *mh, *mt, *m; + caddr_t hdr; + bool filter; + int dir = egress ? PFIL_OUT : PFIL_IN; + int action, flags; + //vni = (vs->vs_flags & VS_VXLANTAG) ? vs->vs_vni : 0; + + m = *m0; + mh = mt = NULL; + vs = iflib_get_softc(ifp->if_softc); + filter = (egress && (vs->vs_flags & VS_IPFW_EGRESS)) || + (!egress && (vs->vs_flags & VS_IPFW_INGRESS)); + flags = 0; + do { + mnext = m->m_nextpkt; + /* set mbuf flags for transmit */ + ETHER_BPF_MTAP(ifp, m); + m->m_pkthdr.rcvif = ifp; + hdr = mtod(m, caddr_t); + if (type == VPC_OBJ_VMNIC) { + vpc_parse_pkt(m, &vpi); + vmi_txflags(m, &vpi, egress); + flags = egress ? M_HOLBLOCKING : 0; + } + if (filter) { + action = ipfw_check_frame(&m, m->m_pkthdr.rcvif, dir, vs->vs_chain); + if (__predict_false(action != IP_FW_PASS)) { + m_freem(m); + goto next; + } + } + if (egress) { + m->m_flags |= vs->vs_mflags; + m->m_flags |= flags; + m->m_pkthdr.vxlanid = vs->vs_vxlanid; + m->m_pkthdr.ether_vtag = vs->vs_vlanid; + } else { + if ((m->m_pkthdr.vxlanid != vs->vs_vxlanid)|| + (m->m_pkthdr.ether_vtag != vs->vs_vlanid)) { + m_freem(m); + goto next; + } + if ((m->m_flags & M_TRUNK) == 0) + m->m_pkthdr.csum_flags |= CSUM_DATA_VALID; + } + if (mh == NULL) { + mh = mt = m; + } else { + mt->m_nextpkt = m; + mt = m; + } + safe_mvec_sanity(m); + next: + m = mnext; + } while (m != NULL); + *m0 = mh; +} + +static void +vmi_input(if_t ifp, struct mbuf *m) +{ + struct vpcp_softc *vs = iflib_get_softc(ifp->if_softc); + struct ifnet *devifp = vs->vs_ifdev; + + vpcp_input_process(vs->vs_ifport, &m, false, VPC_OBJ_VMNIC); + if (__predict_true(m != NULL)) + devifp->if_transmit_txq(devifp, m); +} + +static int +vmi_bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *s __unused, struct rtentry *r __unused) +{ + struct vpcp_softc *vs; + struct ifnet *ifswitch; + + vs = ifp->if_bridge; + ifswitch = vs->vs_ifswitch; + vpcp_input_process(vs->vs_ifport, &m, true, VPC_OBJ_VMNIC); + if (__predict_false(m == NULL)) + return (0); + return (vpcsw_transmit_ext(ifswitch, m, vs->vs_pcpu_cache)); +} + +/* + * Ingress -- from NIC + */ +static struct mbuf * +vmi_bridge_input(if_t ifp, struct mbuf *m) +{ + panic("%s should not be called\n", __func__); +} + +/* + * Egress -- from switch (i.e. switch output -> NIC input) + */ +static void +phys_input(struct ifnet *ifport, struct mbuf *m) +{ + if_ctx_t ctx = ifport->if_softc; + struct vpcp_softc *vs = iflib_get_softc(ctx); + struct ifnet *ifdev = vs->vs_ifdev; + struct mbuf *mp = m; + do { + mp->m_flags &= ~M_VPCMASK; + mp->m_pkthdr.rcvif = NULL; + ETHER_BPF_MTAP(ifport, mp); + mp = mp->m_nextpkt; + } while (mp); + ifdev->if_transmit_txq(ifdev, m); +} + +static int +phys_mbuf_to_qid(if_t ifp __unused, struct mbuf *m __unused) +{ + panic("%s should not be called\n", __func__); + return (0); +} + +static int +phys_bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *s __unused, struct rtentry *r __unused) +{ + panic("%s should not be called\n", __func__); + return (0); +} + +/* + * Ingress -- from NIC + */ +static struct mbuf * +phys_bridge_input(if_t ifp, struct mbuf *m) +{ + struct vpcp_softc *vs; + struct ifnet *ifswitch; + struct mbuf *mp; + + MPASS(ifp->if_bridge != NULL); + vs = ifp->if_bridge; + mp = m; + do { + mp->m_pkthdr.rcvif = vs->vs_ifport; + ETHER_BPF_MTAP(vs->vs_ifport, mp); + mp = m->m_nextpkt; + } while (mp); + ifswitch = vs->vs_ifswitch; + vpcsw_transmit_ext(ifswitch, m, vs->vs_pcpu_cache); + return (NULL); +} + +static void +hostif_input(if_t ifp, struct mbuf *m) +{ + struct vpcp_softc *vs = iflib_get_softc(ifp->if_softc); + struct ifnet *devifp = vs->vs_ifdev; + struct mbuf *mp = m; + struct mbuf *mh, *mt, *mnew, *mnext; + + vpcp_input_process(vs->vs_ifport, &mp, false, VPC_OBJ_HOSTIF); + mh = mt = NULL; + while (mp != NULL) { + mnext = mp->m_nextpkt; + mp->m_nextpkt = NULL; + if (m_ismvec(mp)) { + mnew = mvec_to_mchain(mp, M_NOWAIT); + m_freem(mp); + if (mnew == NULL) + goto next; + mp = mnew; + } + if (mh != NULL) { + mt->m_nextpkt = mp; + mt = mp; + } else + mh = mt = mp; + mp->m_pkthdr.rcvif = devifp; + next: + mp = mnext; + } while (mp); + if (__predict_true(mh != NULL)) + devifp->if_input(devifp, mh); +} + +static int +hostif_bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *s __unused, struct rtentry *r __unused) +{ + struct vpcp_softc *vs; + struct ifnet *ifswitch; + + vs = ifp->if_bridge; + ifswitch = vs->vs_ifswitch; + vpcp_input_process(vs->vs_ifport, &m, true, VPC_OBJ_HOSTIF); + if (__predict_false(m == NULL)) + return (0); + return (vpcsw_transmit_ext(ifswitch, m, vs->vs_pcpu_cache)); +} + +static struct mbuf * +hostif_bridge_input(if_t ifp, struct mbuf *m) +{ + return (m); +} + +static int +vpcp_port_type_set(if_ctx_t portctx, vpc_ctx_t vctx, vpc_obj_type_t type) +{ + struct ifnet *ifp, *ifdev; + struct vpcp_softc *vs; + enum vpc_obj_type prevtype; + uint64_t baudrate; + if_ctx_t switchctx; + int rc; + + ifp = iflib_get_ifp(portctx); + vs = iflib_get_softc(portctx); + switchctx = vs->vs_ifswitch->if_softc; + ifdev = NULL; + baudrate = 0; + + if (vs->vs_type != VPC_OBJ_INVALID && + type != VPC_OBJ_INVALID) { + if (bootverbose) + printf("%s can't transition directly between port types, vs_type=%x type=%x\n", + __func__, vs->vs_type, type); + return (EEXIST); + } + if (type != VPC_OBJ_INVALID) { + MPASS(vctx != NULL); + + if (bootverbose) + printf("%s transitioning to type=%x\n", + __func__, type); + + ifdev = vctx->v_ifp; + vs->vs_ifdev = ifdev; + if (ifdev->if_bridge != NULL) { + printf("%s in use\n", ifdev->if_xname); + return (EINVAL); + } + baudrate = ifdev->if_baudrate; + } else + MPASS(vctx == NULL); + + prevtype = vs->vs_type; + vs->vs_type = type; + rc = 0; + + switch (type) { + case VPC_OBJ_INVALID: + ifp->if_input = vpcp_stub_input; + ifp->if_bridge_linkstate = vpcp_stub_linkstate; + if (vs->vs_ifdev) { + ifdev = vs->vs_ifdev; + ifdev->if_capabilities &= ~IFCAP_BRIDGE_BATCH; + wmb(); + ifdev->if_bridge = NULL; + wmb(); + vpcsw_port_disconnect(switchctx, ifp); + ifdev->if_bridge_input = NULL; + ifdev->if_bridge_output = NULL; + ifdev->if_bridge_linkstate = NULL; + if_rele(vs->vs_ifdev); + vs->vs_ifdev = NULL; + } + break; + case VPC_OBJ_VPCMUX: + case VPC_OBJ_ETHLINK: + if_setmbuftoqidfn(ifp, phys_mbuf_to_qid); + ifp->if_input = phys_input; + ifdev->if_bridge_input = phys_bridge_input; + ifdev->if_bridge_output = phys_bridge_output; + break; + case VPC_OBJ_HOSTIF: + ifp->if_input = hostif_input; + ifdev->if_bridge_input = hostif_bridge_input; + ifdev->if_bridge_output = hostif_bridge_output; + break; + case VPC_OBJ_VMNIC: + ifp->if_input = vmi_input; + ifdev->if_bridge_input = vmi_bridge_input; + ifdev->if_bridge_output = vmi_bridge_output; + break; + default: + vs->vs_type = prevtype; + if (bootverbose) + device_printf(iflib_get_dev(portctx), "unknown port type %d\n", type); + rc = EINVAL; + } + if (rc == 0) { + if (type == VPC_OBJ_INVALID) { + MPASS(vs->vs_ifdev == NULL); + iflib_link_state_change(vs->vs_ctx, LINK_STATE_DOWN, IF_Gbps(100)); + } else { + ifdev->if_bridge_linkstate = vpcp_stub_linkstate; + wmb(); + ifdev->if_bridge = vs; + wmb(); + ifdev->if_capabilities |= IFCAP_BRIDGE_BATCH; + vpcsw_port_connect(switchctx, ifp, ifdev); + iflib_link_state_change(vs->vs_ctx, LINK_STATE_UP, baudrate); + if_ref(ifdev); + vs->vs_ifdev = ifdev; + memcpy(&vs->vs_devid, &vctx->v_id, sizeof(vpc_id_t)); + } + } + return (rc); +} + +static int +vpcp_port_connect(if_ctx_t ctx, const vpc_id_t *id) +{ + vpc_ctx_t vctx; + + vctx = vmmnet_lookup(id); + if (vctx == NULL) { + if (bootverbose) + printf("%s no context for %16D found\n", + __func__, id, ":"); + return (ENOENT); + } + if (vctx->v_ifp == NULL) { + if (bootverbose) + printf("%s %16D not attached\n", + __func__, id, ":"); + + return (ENXIO); + } + return (vpcp_port_type_set(ctx, vctx, vctx->v_handle_type.vht_obj_type)); +} + +static int +vpcp_port_disconnect(if_ctx_t ctx) +{ + return (vpcp_port_type_set(ctx, NULL, VPC_OBJ_INVALID)); +} + +int +vpcp_port_disconnect_ifp(struct ifnet *ifp) +{ + struct vpcp_softc *vs = ifp->if_bridge; + if_ctx_t ctx = vs->vs_ctx; + + MPASS(vs->vs_ifdev == ifp); + return (vpcp_port_type_set(ctx, NULL, VPC_OBJ_INVALID)); +} + +void +vpcp_get_id(struct ifnet *portifp, vpc_id_t *id) +{ + if_ctx_t ctx = portifp->if_softc; + struct vpcp_softc *vs; + + vs = iflib_get_softc(ctx); + memcpy(id, &vs->vs_devid, sizeof(*id)); +} + +int +vpcp_ctl(vpc_ctx_t vctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata) +{ + if_ctx_t ctx = vctx->v_ifp->if_softc; + struct vpcp_softc *vs; + int rc; + + rc = 0; + vs = iflib_get_softc(ctx); + switch (op) { + case VPC_VPCP_OP_CONNECT: + if (inlen != sizeof(vpc_id_t)) + goto fail; + rc = vpcp_port_connect(ctx, in); + break; + case VPC_VPCP_OP_DISCONNECT: + rc = vpcp_port_disconnect(ctx); + break; + case VPC_VPCP_OP_VNI_GET: { + uint32_t *out; + + out = malloc(sizeof(uint32_t), M_TEMP, M_WAITOK); + *outlen = sizeof(uint32_t); + *out = vpcp_vxlanid_get(ctx); + *outdata = out; + break; + } + case VPC_VPCP_OP_VNI_SET: { + uint32_t vni; + + if (inlen != sizeof(uint32_t)) + goto fail; + vni = *(const uint32_t *)in; + if (bootverbose) + printf("set %s vni to %d\n", vctx->v_ifp->if_xname, ntohl(vni)); + vpcp_vxlanid_set(ctx, vni>>8); + break; + } + case VPC_VPCP_OP_VTAG_GET: { + uint16_t *out; + + out = malloc(sizeof(uint16_t), M_TEMP, M_WAITOK); + *outlen = sizeof(uint16_t); + *out = vpcp_vlanid_get(ctx); + *outdata = out; + break; + } + case VPC_VPCP_OP_VTAG_SET: { + uint16_t vtag; + + if (inlen != sizeof(uint16_t)) + goto fail; + vtag = *(const uint16_t *)in; + if (vtag > ((1<<12)-1)) + goto fail; + vpcp_vlanid_set(ctx, vtag); + break; + } + case VPC_VPCP_OP_CONNECTED_ID_GET: { + vpc_id_t *id; + + if (vs->vs_type == VPC_OBJ_INVALID) + return (ENXIO); + if (*outlen < sizeof(vpc_id_t)) + return (EOVERFLOW); + *outlen = sizeof(vpc_id_t); + id = malloc(*outlen, M_TEMP, M_WAITOK); + memcpy(id, &vs->vs_devid, *outlen); + break; + } + } + return (rc); + fail: + return (EINVAL); +} + +static int +vpcp_object_info_get(if_ctx_t ctx, void *arg, int size) +{ + struct vpcp_softc *vs = iflib_get_softc(ctx); + vpc_obj_info_t *voi = arg; + + if (size != sizeof(*voi)) + return (EBADRPC); + voi->port.type = vs->vs_type; + return (0); +} + +#define VPCP_CAPS \ + IFCAP_TSO | IFCAP_HWCSUM | IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \ + IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU | IFCAP_TXCSUM_IPV6 | IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU | \ + IFCAP_LINKSTATE + +static int +vpcp_ipfw_init(struct ip_fw_chain *chain) +{ + int error; + struct ip_fw *rule = NULL; + + ipfw_init_srv(chain); + error = ipfw_init_tables(chain, 0); + + if (error) { + printf("ipfw2: setting up tables failed\n"); + free(chain->map, M_IPFW); + return (ENOSPC); + } + + IPFW_LOCK_INIT(chain); + /* fill and insert the default rule */ + rule = ipfw_alloc_rule(chain, sizeof(struct ip_fw)); + rule->cmd_len = 1; + rule->cmd[0].len = 1; + rule->cmd[0].opcode = O_ACCEPT; + chain->default_rule = rule; + ipfw_add_protected_rule(chain, rule, 0); + ipfw_eaction_init(chain, 0); + + return (0); +} + +static void +vpcp_ipfw_deinit(struct ip_fw_chain *chain) +{ + struct ip_fw *reap; + int i; + + reap = NULL; + IPFW_UH_WLOCK(chain); + IPFW_WLOCK(chain); + for (i = 0; i < chain->n_rules; i++) + ipfw_reap_add(chain, &reap, chain->map[i]); + + IPFW_WUNLOCK(chain); + IPFW_UH_WUNLOCK(chain); + + ipfw_destroy_tables(chain, 0); + ipfw_eaction_uninit(chain, 0); + if (reap != NULL) + ipfw_reap_rules(reap); + vnet_ipfw_iface_destroy(chain); + ipfw_destroy_srv(chain); + IPFW_LOCK_DESTROY(chain); +} + +static int +vpcp_cloneattach(if_ctx_t ctx, struct if_clone *ifc, const char *name, caddr_t params) +{ + struct vpcp_softc *vs = iflib_get_softc(ctx); + if_softc_ctx_t scctx; + int rc; + + atomic_add_int(&clone_count, 1); + vs->vs_ctx = ctx; + vs->vs_ifport = iflib_get_ifp(ctx); + vs->vs_chain = malloc(sizeof(struct ip_fw_chain), M_VPCP, M_WAITOK|M_ZERO); + if ((rc = vpcp_ipfw_init(vs->vs_chain))) { + free(vs->vs_chain, M_VPCP); + return (rc); + } + scctx = vs->shared = iflib_get_softc_ctx(ctx); + scctx->isc_capenable = VPCP_CAPS; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_IP6_TCP \ + | CSUM_IP6_UDP | CSUM_IP6_TCP; + return (0); +} + +static int +vpcp_attach_post(if_ctx_t ctx) +{ + struct ifnet *ifp; + + ifp = iflib_get_ifp(ctx); + if_settransmitfn(ifp, vpcp_stub_transmit); + if_settransmittxqfn(ifp, vpcp_stub_transmit); + if_setmbuftoqidfn(ifp, vpcp_stub_mbuf_to_qid); + ifp->if_input = vpcp_stub_input; + ifp->if_bridge_linkstate = vpcp_stub_linkstate; + return (0); +} + +static int +vpcp_detach(if_ctx_t ctx) +{ + struct vpcp_softc *vs = iflib_get_softc(ctx); + + if (vs->vs_ifdev != NULL) { + if (bootverbose) + printf("disconnecting port for %s\n", iflib_get_ifp(ctx)->if_xname); + vpcp_port_disconnect(ctx); + } + if (vs->vs_ifswitch != NULL) + if_rele(vs->vs_ifswitch); + vpcp_ipfw_deinit(vs->vs_chain); + free(vs->vs_chain, M_VPCP); + atomic_add_int(&clone_count, -1); + + return (0); +} + +static void +vpcp_init(if_ctx_t ctx) +{ +} + +static void +vpcp_stop(if_ctx_t ctx) +{ +} + +int +vpcp_set_ifswitch(if_ctx_t ctx, if_t ifswitch) +{ + struct vpcp_softc *vs = ctx_to_vs(ctx); + + if (vs->vs_ifswitch != NULL) + return (EEXIST); + if_ref(ifswitch); + + vs->vs_ifswitch = ifswitch; + return (0); +} + +if_t +vpcp_get_ifswitch(if_ctx_t ctx) +{ + struct vpcp_softc *vs = ctx_to_vs(ctx); + + return (vs->vs_ifswitch); +} + +void +vpcp_clear_ifswitch(if_ctx_t ctx) +{ + struct vpcp_softc *vs = ctx_to_vs(ctx); + + if (vs->vs_ifswitch == NULL) + return; + if_rele(vs->vs_ifswitch); + vs->vs_ifswitch = NULL; +} + +static void +vpcp_vxlanid_set(if_ctx_t ctx, uint32_t vxlanid) +{ + struct vpcp_softc *vs = ctx_to_vs(ctx); + + if (vxlanid) + vs->vs_mflags |= M_VXLANTAG; + else + vs->vs_mflags &= ~M_VXLANTAG; + vs->vs_vxlanid = vxlanid; +} + +static uint32_t +vpcp_vxlanid_get(if_ctx_t ctx) +{ + struct vpcp_softc *vs = ctx_to_vs(ctx); + + return (vs->vs_vxlanid); +} + +static void +vpcp_vlanid_set(if_ctx_t ctx, uint16_t vlanid) +{ + struct vpcp_softc *vs = ctx_to_vs(ctx); + + if (vlanid) + vs->vs_mflags |= M_VLANTAG; + else + vs->vs_mflags &= ~M_VLANTAG; + vs->vs_vlanid = vlanid; +} + +static uint16_t +vpcp_vlanid_get(if_ctx_t ctx) +{ + struct vpcp_softc *vs = ctx_to_vs(ctx); + + return (vs->vs_vlanid); +} + +void +vpcp_set_pcpu_cache(if_ctx_t ctx, void *cache) +{ + struct vpcp_softc *vs = iflib_get_softc(ctx); + + vs->vs_pcpu_cache = cache; +} + +void * +vpcp_get_pcpu_cache(if_ctx_t ctx) +{ + struct vpcp_softc *vs = iflib_get_softc(ctx); + + return (vs->vs_pcpu_cache); +} + +static device_method_t vpcp_if_methods[] = { + DEVMETHOD(ifdi_cloneattach, vpcp_cloneattach), + DEVMETHOD(ifdi_attach_post, vpcp_attach_post), + DEVMETHOD(ifdi_detach, vpcp_detach), + DEVMETHOD(ifdi_init, vpcp_init), + DEVMETHOD(ifdi_stop, vpcp_stop), + DEVMETHOD(ifdi_object_info_get, vpcp_object_info_get), + DEVMETHOD_END +}; + +static driver_t vpcp_iflib_driver = { + "vpcp", vpcp_if_methods, sizeof(struct vpcp_softc) +}; + +char vpcp_driver_version[] = "0.0.1"; + +static struct if_shared_ctx vpcp_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_driver_version = vpcp_driver_version, + .isc_driver = &vpcp_iflib_driver, + .isc_flags = IFLIB_PSEUDO, + .isc_name = "vpcp", +}; + +if_shared_ctx_t vpcp_sctx = &vpcp_sctx_init; + + +static if_pseudo_t vpcp_pseudo; + +static int +vpcp_module_init(void) +{ + vpcp_pseudo = iflib_clone_register(vpcp_sctx); + + return vpcp_pseudo != NULL ? 0 : ENXIO; +} + +static void +vpcp_module_deinit(void) +{ + iflib_clone_deregister(vpcp_pseudo); +} + +static int +vpcp_module_event_handler(module_t mod, int what, void *arg) +{ + int err; + + switch (what) { + case MOD_LOAD: + if ((err = vpcp_module_init()) != 0) + return (err); + break; + case MOD_UNLOAD: + if (clone_count == 0) + vpcp_module_deinit(); + else + return (EBUSY); + break; + default: + return (EOPNOTSUPP); + } + + return (0); +} + +static moduledata_t vpcp_moduledata = { + "vpcp", + vpcp_module_event_handler, + NULL +}; + +DECLARE_MODULE(vpcp, vpcp_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY); +MODULE_VERSION(vpcp, 1); +MODULE_DEPEND(vpcp, iflib, 1, 1, 1); +MODULE_DEPEND(vpcp, ipfw, 3, 3, 3); Index: sys/amd64/vmm/net/if_vpcsw.c =================================================================== --- /dev/null +++ sys/amd64/vmm/net/if_vpcsw.c @@ -0,0 +1,1047 @@ +/* + * Copyright (C) 2018 Matthew Macy + * Copyright (C) 2018 Joyent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "ifdi_if.h" + +static MALLOC_DEFINE(M_VPCSW, "vpcsw", "virtual private cloud bridge"); + +#define VCE_TRUSTED 0x0 +#define VCE_IPSEC 0x1 +#define DHCP_SPORT 68 +#define DHCP_DPORT 67 + +/* + * ifconfig vpcsw0 create + * ifconfig vpcsw0 addm vpc0 + * ifconfig vpcsw0 priority vpc0 200 + * ifconfig vpcsw0 vpc-resolver 127.0.0.1:5000 + * ifconfig vpcsw0 addm vmi7 + * ifconfig vpcsw0 pathcost vmi7 2000000 + */ +#define ARPHRD_ETHER 1 /* ethernet hardware format */ +#define ARPOP_REPLY 2 /* response to previous request */ +struct arphdr_ether { + u_short ar_hrd; /* format of hardware address */ + u_short ar_pro; /* format of protocol address */ + u_char ar_hln; /* length of hardware address */ + u_char ar_pln; /* length of protocol address */ + u_short ar_op; /* one of: */ + u_char ar_sha[ETHER_ADDR_LEN]; /* sender hardware address */ + in_addr_t ar_spa; /* sender protocol address */ + u_char ar_tha[ETHER_ADDR_LEN]; /* target hardware address */ + in_addr_t ar_tpa; /* target protocol address */ + u_char ar_pad[60 - ETHER_HDR_LEN - 24 /*arphdr*/]; +}; + +struct vpcsw_request_priv { + TAILQ_ENTRY(vpcsw_request_priv) vrp_entry; + vpc_id_t vrp_id; + struct mbuf *vrp_m; +}; + +struct vpcsw_source { + uint16_t vs_dmac[3]; /* destination mac address */ + uint16_t vs_vlanid; /* source vlanid */ + uint32_t vs_vni; /* source vni */ +}; + +struct vpcsw_cache_ent { + struct vpcsw_source vce_src; + uint16_t vce_ifindex; /* interface index */ + int vce_ticks; /* time when entry was created */ +}; + +static volatile int32_t modrefcnt; + +struct vpcsw_mcast_queue { + int vmq_mcount; + TAILQ_HEAD(vrp_head, vpcsw_request_priv) vmq_head; +}; + +struct vpcsw_softc { + if_softc_ctx_t shared; + if_ctx_t vs_ctx; + if_t vs_ifp; + volatile int32_t vs_refcnt; + struct mtx vs_lock; + + struct vpcsw_mcast_queue vs_vmq; + art_tree *vs_ftable_ro; + art_tree *vs_ftable_rw; + struct ifnet *vs_ifdefault; + vpc_id_t vs_uplink_id; + struct grouptask vs_vtep_gtask; + uint32_t vs_vni; + /* pad */ + struct vpcsw_request_priv *vs_req_pending; + struct arphdr_ether vs_arp_template; + struct vpc_copy_info vs_vci; +}; + +static int +vpcsw_knote_event(if_ctx_t ctx, struct knote *kn, int hint) +{ + struct mbuf *m; + struct kevent *kev; + struct vpcsw_softc *vs; + struct vpcsw_request_priv *vrp; + char *uaddr; + void *usize; + int32_t size; + int rc; + + vs = iflib_get_softc(ctx); + if (hint == 0) { + GROUPTASK_ENQUEUE(&vs->vs_vtep_gtask); + return (0); + } + vrp = vs->vs_req_pending; + kev = &kn->kn_kevent; + usize = (void*)kev->ext[VPCSW_SIZE_IDX]; + uaddr = (void*)kev->ext[VPCSW_ADDR_IDX]; + if (usize == NULL) + return (0); + vs->vs_vci.vci_proc = kn->kn_hook; + if (uaddr == NULL) { + size = EFAULT; + rc = vpc_async_copyout(&vs->vs_vci, &size, usize, sizeof(size)); + if (rc) + goto fail; + } + m = vrp->vrp_m; + size = m->m_len + sizeof(vrp->vrp_id); + rc = vpc_async_copyout(&vs->vs_vci, &size, usize, sizeof(size)); + if (rc) + goto fail; + rc = vpc_async_copyout(&vs->vs_vci, &vrp->vrp_id, uaddr, sizeof(vrp->vrp_id)); + if (rc) + goto fail; + rc = vpc_async_copyout(&vs->vs_vci, m->m_data, uaddr + sizeof(vrp->vrp_id), m->m_len); + if (rc) + goto fail; + vs->vs_vci.vci_proc = NULL; + return (1); + fail: + vs->vs_vci.vci_proc = NULL; + return (0); +} + +static void +_task_fn_vtep(void *arg) +{ + struct vpcsw_softc *vs; + struct vpcsw_mcast_queue *vmq; + struct vpcsw_request_priv *vrp; + if_ctx_t ctx; + + ctx = arg; + vs = iflib_get_softc(ctx); + vmq = &vs->vs_vmq; + + mtx_lock(&vs->vs_lock); + if (TAILQ_EMPTY(&vmq->vmq_head)) { + MPASS(vmq->vmq_mcount == 0); + mtx_unlock(&vs->vs_lock); + return; + } + vrp = TAILQ_FIRST(&vmq->vmq_head); + TAILQ_REMOVE(&vmq->vmq_head, vrp, vrp_entry); + vmq->vmq_mcount--; + mtx_unlock(&vs->vs_lock); + vs->vs_req_pending = vrp; + iflib_event_signal(ctx, 1); + vs->vs_req_pending = NULL; + m_freem(vrp->vrp_m); + free(vrp, M_VPCSW); +} + +#ifdef notyet +static const char *opcode_map[] = { + "", + "VPCSW_REQ_NDv4", + "VPCSW_REQ_NDv6", + "VPCSW_REQ_DHCPv4", + "VPCSW_REQ_DHCPv6", +}; +#endif + +static __inline int +hdrcmp(struct vpcsw_source *vlhs, struct vpcsw_source *vrhs) +{ + uint16_t *lhs, *rhs; + + lhs = (uint16_t *)vlhs; + rhs = (uint16_t *)vrhs; + return ((lhs[0] ^ rhs[0]) | + (lhs[1] ^ rhs[1]) | + (lhs[2] ^ rhs[2]) | + (lhs[3] ^ rhs[3]) | + (lhs[4] ^ rhs[4]) | + (lhs[5] ^ rhs[5])); +} + +static int +vpcsw_cache_lookup(struct vpcsw_cache_ent *cache, struct mbuf *m) +{ + struct vpcsw_cache_ent *vcep; + struct vpcsw_source vsrc; + struct ether_header *eh; + struct ifnet *ifp; + uint16_t *mac; + + eh = (void*)m->m_data; + mac = (uint16_t *)eh->ether_dhost; + vsrc.vs_vlanid = m->m_pkthdr.ether_vtag; + vsrc.vs_vni = m->m_pkthdr.vxlanid; + vsrc.vs_dmac[0] = mac[0]; + vsrc.vs_dmac[1] = mac[1]; + vsrc.vs_dmac[2] = mac[2]; + _critical_enter(); + vcep = &cache[curcpu]; + + if (__predict_false(vcep->vce_ticks == 0)) + goto skip; + /* + * Is still in caching window + */ + if (__predict_false(ticks - vcep->vce_ticks > hz/4)) + goto skip; + if ((ifp = vpc_if_lookup(vcep->vce_ifindex)) == NULL) + goto skip; + /* + * dmac & vxlanid match + */ + if (hdrcmp(&vcep->vce_src, &vsrc) == 0) { + /* cache hit */ + _critical_exit(); + m->m_pkthdr.rcvif = ifp; + return (1); + } + skip: + vcep->vce_ticks = 0; + _critical_exit(); + return (0); +} + +static void +vpcsw_cache_update(struct vpcsw_cache_ent *cache, struct mbuf *m) +{ + struct vpcsw_cache_ent *vcep; + struct vpcsw_source *vsrc; + struct ether_header *eh; + uint16_t *mac; + + eh = (void*)m->m_data; + mac = (uint16_t *)eh->ether_dhost; + _critical_enter(); + vcep = &cache[curcpu]; + vsrc = &vcep->vce_src; + vsrc->vs_vlanid = m->m_pkthdr.ether_vtag; + vsrc->vs_vni = m->m_pkthdr.vxlanid; + vsrc->vs_dmac[0] = mac[0]; + vsrc->vs_dmac[1] = mac[1]; + vsrc->vs_dmac[2] = mac[2]; + vcep->vce_ifindex = m->m_pkthdr.rcvif->if_index; + vcep->vce_ticks = ticks; + _critical_exit(); +} + +static int +vpc_broadcast_one(void *data, const unsigned char *key __unused, uint32_t key_len __unused, void *value) +{ + struct ifnet *ifp; + uint16_t *ifindexp = value; + struct mbuf *m, *msrc; + + msrc = (struct mbuf *)data; + if ((ifp = vpc_if_lookup(*ifindexp)) == NULL) + return (0); + MPASS(msrc->m_pkthdr.rcvif != NULL); + if (msrc->m_pkthdr.rcvif == ifp) + return (0); + m = mvec_dup(msrc, M_NOWAIT); + if (__predict_false(m == NULL)) + return (ENOMEM); + m->m_pkthdr.rcvif = ifp; + ifp->if_input(ifp, m); + return (0); +} + +static int +vpcsw_process_mcast(struct vpcsw_softc *vs, struct mbuf **msrc) +{ + struct vpcsw_mcast_queue *vmq; + struct mbuf *m, *mp; + int rc; + + vmq = &vs->vs_vmq; + m = *msrc; + if ((m->m_flags & M_HOLBLOCKING) || + (m->m_flags & (M_VXLANTAG|M_TRUNK)) == M_VXLANTAG) { + mp = mvec_dup(m, M_NOWAIT); + m_freem(m); + m = mp; + *msrc = m; + if (m == NULL) + return (ENOMEM); + } + if ((m->m_flags & (M_VXLANTAG|M_TRUNK)) == M_VXLANTAG) { + struct vpcsw_request_priv *vrp; + + vrp = malloc(sizeof(*vrp), M_VPCSW, M_NOWAIT); + if (vrp == NULL) { + m_freem(m); + *msrc = NULL; + return (ENOMEM); + } + vpcp_get_id(m->m_pkthdr.rcvif, &vrp->vrp_id); + vrp->vrp_m = m; + mtx_lock(&vs->vs_lock); + TAILQ_INSERT_HEAD(&vmq->vmq_head, vrp, vrp_entry); + if (vmq->vmq_mcount >= 128) { + vrp = TAILQ_LAST(&vmq->vmq_head, vrp_head); + TAILQ_REMOVE(&vmq->vmq_head, vrp, vrp_entry); + m_freem(vrp->vrp_m); + free(vrp, M_VPCSW); + } else + vmq->vmq_mcount++; + mtx_unlock(&vs->vs_lock); + GROUPTASK_ENQUEUE(&vs->vs_vtep_gtask); + *msrc = NULL; + rc = 0; + } else if (!(m->m_flags & M_VXLANTAG)) { + art_iter(vs->vs_ftable_ro, vpc_broadcast_one, m); + m->m_pkthdr.rcvif = NULL; + rc = 0; + } else { + m_freem(m); + rc = EINVAL; + *msrc = NULL; + } + return (rc); +} + +static int +vpcsw_process_one(struct vpcsw_softc *vs, struct vpcsw_cache_ent *cache, struct mbuf **mp) +{ + struct ether_header *eh; + struct mbuf *m; + uint16_t *vif; + struct ifnet *ifp; + + m = *mp; + eh = (void*)m->m_data; + if (__predict_false(ETHER_IS_MULTICAST(eh->ether_dhost))) + return (vpcsw_process_mcast(vs, mp)); + if (vpcsw_cache_lookup(cache, m)) + return (0); + vif = art_search(vs->vs_ftable_ro, (const unsigned char *)eh->ether_dhost); + if (vif != NULL) + ifp = vpc_if_lookup(*vif); + else + ifp = vs->vs_ifdefault; + if (__predict_false(ifp == NULL)) { + m_freem(m); + *mp = NULL; + return (ENOBUFS); + } + m->m_pkthdr.rcvif = ifp; + vpcsw_cache_update(cache, m); + return (0); +} + +static int +vpcsw_transit(struct vpcsw_softc *vs, struct vpcsw_cache_ent *cache, struct mbuf *m) +{ + struct ifnet *ifnext; + struct mbuf *mh, *mt, *mnext; + bool can_batch = true; + int rc, lasterr; + + vpc_epoch_begin(); + mh = mt = NULL; + do { + mnext = m->m_nextpkt; + m->m_nextpkt = NULL; + ETHER_BPF_MTAP(vs->vs_ifp, m); + rc = vpcsw_process_one(vs, cache, &m); + if (m == NULL) { + m = mnext; + continue; + } + if (__predict_false(rc)) + break; + if (__predict_false(m->m_pkthdr.rcvif == NULL)) { + m_freem(m); + goto next; + } + if (mh == NULL) { + mh = mt = m; + ifnext = m->m_pkthdr.rcvif; + } else { + mt->m_nextpkt = m; + mt = m; + if (__predict_false(ifnext != m->m_pkthdr.rcvif)) + can_batch = false; + } + MPASS(m != mnext); + next: + m = mnext; + } while (m != NULL); + if (__predict_false(mnext != NULL)) + m_freechain(mnext); + + if (mh == NULL) { + lasterr = rc; + goto done; + } + lasterr = 0; + if (can_batch) { + ifnext = mh->m_pkthdr.rcvif; + ifnext->if_input(ifnext, mh); + goto done; + } + m = mh; + do { + mnext = m->m_nextpkt; + m->m_nextpkt = NULL; + ifnext = m->m_pkthdr.rcvif; + ifnext->if_input(ifnext, m); + m = mnext; + } while (m != NULL); + done: + vpc_epoch_end(); + return (lasterr); +} + +static int +vpcsw_transmit(if_t ifp, struct mbuf *m) +{ + + panic("unsupported\n"); + return (EOPNOTSUPP); +} + +int +vpcsw_transmit_ext(struct ifnet *ifp, struct mbuf *m, void *cache) +{ + if_ctx_t ctx = ifp->if_softc; + struct vpcsw_softc *vs; + + vs = iflib_get_softc(ctx); + return (vpcsw_transit(vs, cache, m)); +} + +static int +vpcsw_bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *s __unused, struct rtentry *r __unused) +{ + struct vpcsw_softc *vs; + if_ctx_t ctx; + struct vpcsw_cache_ent *cache; + + MPASS(ifp->if_bridge != NULL); + vs = ifp->if_bridge; + ctx = ifp->if_softc; + cache = vpcp_get_pcpu_cache(ctx); + MPASS(cache != NULL); + return (vpcsw_transit(vs, cache, m)); +} + +static void +vpcsw_bridge_linkstate(if_t ifp __unused) +{ +} + +static struct mbuf * +vpcsw_bridge_input(if_t ifp, struct mbuf *m) +{ + struct vpcsw_softc *vs; + if_ctx_t ctx; + struct vpcsw_cache_ent *cache; + struct mbuf *mp; + + MPASS(ifp->if_bridge != NULL); + mp = NULL; + vs = ifp->if_bridge; + ctx = ifp->if_softc; + cache = vpcp_get_pcpu_cache(ctx); + MPASS(cache != NULL); + + vpcsw_transit(vs, cache, m); + return (NULL); +} + +static int +vpcsw_object_info_get(if_ctx_t ctx, void *arg, int size) +{ + struct vpcsw_softc *vs = iflib_get_softc(ctx); + vpc_obj_info_t *voi = arg; + + if (size != sizeof(*voi)) + return (EBADRPC); + voi->vswitch.vni = vs->vs_vni; + return (0); +} + +#define VPCSW_CAPS \ + IFCAP_TSO |IFCAP_HWCSUM | IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \ + IFCAP_VLAN_MTU | IFCAP_TXCSUM_IPV6 | IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU | IFCAP_LINKSTATE + +static void +vpcsw_arp_tmpl_init(struct vpcsw_softc *vs) +{ + struct arphdr_ether *ae = &vs->vs_arp_template; + + bzero(ae, sizeof(*ae)); + ae->ar_hrd = htons(ARPHRD_ETHER); + ae->ar_pro = htons(ETHERTYPE_IP); + ae->ar_hln = ETHER_ADDR_LEN; + ae->ar_pln = sizeof(in_addr_t); + ae->ar_op = htons(ARPOP_REPLY); +} + +static int +vpcsw_cloneattach(if_ctx_t ctx, struct if_clone *ifc, const char *name, caddr_t params) +{ + struct vpcsw_softc *vs = iflib_get_softc(ctx); + if_softc_ctx_t scctx; + device_t dev; + uint32_t unitno; + + dev = iflib_get_dev(ctx); + unitno = device_get_unit(dev); + + refcount_acquire(&modrefcnt); + + scctx = vs->shared = iflib_get_softc_ctx(ctx); + scctx->isc_capenable = VPCSW_CAPS; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_IP6_TCP \ + | CSUM_IP6_UDP | CSUM_IP6_TCP; + vs->vs_ctx = ctx; + vs->vs_ifp = iflib_get_ifp(ctx); + refcount_init(&vs->vs_refcnt, 0); + mtx_init(&vs->vs_lock, "vpcsw sc internal", NULL, MTX_DEF); + vs->vs_ftable_ro = malloc(sizeof(art_tree), M_VPCSW, M_WAITOK|M_ZERO); + vs->vs_ftable_rw = malloc(sizeof(art_tree), M_VPCSW, M_WAITOK|M_ZERO); + art_tree_init(vs->vs_ftable_ro, ETHER_ADDR_LEN); + art_tree_init(vs->vs_ftable_rw, ETHER_ADDR_LEN); + iflib_config_gtask_init(vs->vs_ctx, &vs->vs_vtep_gtask, _task_fn_vtep, "vtep task"); + vs->vs_vci.vci_pages = malloc(sizeof(vm_page_t *)*(ARG_MAX/PAGE_SIZE), M_VPCSW, M_WAITOK|M_ZERO); + vs->vs_vci.vci_max_count = ARG_MAX/PAGE_SIZE; + vpcsw_arp_tmpl_init(vs); + return (0); +} + + +static int +vpcsw_port_add(struct vpcsw_softc *vs, const vpc_id_t *vp_id) +{ + struct ifnet *ifp; + struct ifreq ifr; + if_ctx_t ctx; + void *cache; + int rc; + + if (vmmnet_lookup(vp_id) != NULL) { + if (bootverbose) + printf("%s can't add %16D -- already in vpc_uuid_table\n", + __func__, vp_id, ":"); + return (EEXIST); + } + snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "vpcp"); + if ((rc = if_clone_create(ifr.ifr_name, sizeof(ifr.ifr_name), NULL))) + return (rc); + if ((ifp = ifunit_ref(ifr.ifr_name)) == NULL) { + if (bootverbose) + printf("couldn't reference %s\n", ifr.ifr_name); + return (ENXIO); + } + cache = malloc(sizeof(struct vpcsw_cache_ent)*MAXCPU, M_VPCSW, M_WAITOK|M_ZERO); + ctx = ifp->if_softc; + vpcp_set_pcpu_cache(ctx, cache); + vpcp_set_ifswitch(ctx, iflib_get_ifp(vs->vs_ctx)); + ifp->if_bridge = vs; + ifp->if_bridge_input = vpcsw_bridge_input; + ifp->if_bridge_output = vpcsw_bridge_output; + ifp->if_bridge_linkstate = vpcsw_bridge_linkstate; + iflib_set_mac(ctx, vp_id->node); + vmmnet_insert(vp_id, ifp, VPC_OBJ_PORT); + vpc_ifp_cache(ifp); + ctx = ifp->if_softc; + if_rele(ifp); + return (0); +} + +static int +vpcsw_port_delete(struct vpcsw_softc *vs, const vpc_id_t *vp_id) +{ + struct ifnet *ifp; + if_ctx_t ctx; + vpc_ctx_t vctx; + void *cache; + + vctx = vmmnet_lookup(vp_id); + if (vctx == NULL) + return (ENOENT); + ifp = vctx->v_ifp; + + vmmnet_delete(vp_id); + ctx = ifp->if_softc; + cache = vpcp_get_pcpu_cache(ctx); + free(cache, M_VPCSW); + vpcp_clear_ifswitch(ctx); + ifp->if_bridge = NULL; + ifp->if_bridge_input = NULL; + ifp->if_bridge_output = NULL; + ifp->if_bridge_linkstate = NULL; + if_clone_destroy(ifp->if_xname); + return (0); +} + +static int +vpcsw_port_uplink_create(struct vpcsw_softc *vs, const vpc_id_t *vp_id) +{ + struct ifnet *ifp; + void *cache; + if_ctx_t ctx; + struct ifreq ifr; + int rc; + + if (vs->vs_ifdefault != NULL) { + if (bootverbose) + printf("%s: can't set port to %16D, vs->vs_ifdefault already set -- if: %s\n", + __func__, vp_id, ":", vs->vs_ifdefault->if_xname); + return (EEXIST); + } + if (vmmnet_lookup(vp_id) != NULL) { + if (bootverbose) + printf("%s set uplink port to %16D -- already in vpc_uuid_table\n", + __func__, vp_id, ":"); + return (EEXIST); + } + + snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "vpcp"); + if ((rc = if_clone_create(ifr.ifr_name, sizeof(ifr.ifr_name), NULL))) + return (rc); + if ((ifp = ifunit_ref(ifr.ifr_name)) == NULL) { + if (bootverbose) + printf("couldn't reference %s\n", ifr.ifr_name); + if_clone_destroy(ifr.ifr_name); + return (ENXIO); + } + rc = vmmnet_insert(vp_id, ifp, VPC_OBJ_PORT); + if (rc) { + if_rele(ifp); + if_clone_destroy(ifr.ifr_name); + return (rc); + } + cache = malloc(sizeof(struct vpcsw_cache_ent)*MAXCPU, M_VPCSW, M_WAITOK|M_ZERO); + ctx = ifp->if_softc; + if (bootverbose) + printf("switch uplink set to id: %16D - default: %s\n", + vp_id, ":", ifp->if_xname); + vpcp_set_pcpu_cache(ctx, cache); + iflib_set_mac(ctx, vp_id->node); + vpcp_set_ifswitch(ctx, iflib_get_ifp(vs->vs_ctx)); + vs->vs_ifdefault = ifp; + memcpy(&vs->vs_uplink_id, vp_id, sizeof(vpc_id_t)); + return (0); +} + +static int +vpcsw_port_uplink_get(struct vpcsw_softc *vs, vpc_id_t *vp_id) +{ + if (vs->vs_ifdefault == NULL) + return (ENOENT); + + memcpy(vp_id, &vs->vs_uplink_id, sizeof(vpc_id_t)); + return (0); +} + +static int +vpcsw_resp_send(struct vpcsw_softc *vs, const struct vpcsw_response *rsp) +{ + struct mbuf *m; + int len; + + len = rsp->vrs_context.voc_len; + m = (struct mbuf *)mvec_alloc(1, len, M_WAITOK); + if (rsp->vrs_context.voc_vni) { + m->m_pkthdr.vxlanid = rsp->vrs_context.voc_vni; + m->m_flags |= M_VXLANTAG; + } + if (rsp->vrs_context.voc_vtag) { + m->m_pkthdr.ether_vtag = rsp->vrs_context.voc_vtag; + m->m_flags |= M_VLANTAG; + } + memcpy(m->m_data, rsp->vrs_data, len); + m->m_len = len; + m->m_pkthdr.len = len; + return (vpcsw_transit(vs, NULL, m)); +} + +int +vpcsw_ctl(vpc_ctx_t vctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata) +{ + if_ctx_t ctx = vctx->v_ifp->if_softc; + struct vpcsw_softc *vs = iflib_get_softc(ctx); + int rc = 0; + vpc_id_t *out; + + switch (op) { + case VPC_VPCSW_OP_PORT_ADD: + case VPC_VPCSW_OP_PORT_DEL: + case VPC_VPCSW_OP_PORT_UPLINK_SET: + if (inlen != sizeof(vpc_id_t)) + return (EBADRPC); + break; + } + switch (op) { + case VPC_VPCSW_OP_PORT_ADD: + rc = vpcsw_port_add(vs, in); + break; + case VPC_VPCSW_OP_PORT_DEL: + rc = vpcsw_port_delete(vs, in); + break; + case VPC_VPCSW_OP_PORT_UPLINK_SET: + rc = vpcsw_port_uplink_create(vs, in); + break; + case VPC_VPCSW_OP_PORT_UPLINK_GET: + out = malloc(sizeof(*out), M_TEMP, M_WAITOK|M_ZERO); + *outdata = out; + *outlen = sizeof(*out); + rc = vpcsw_port_uplink_get(vs, out); + break; + case VPC_VPCSW_OP_STATE_GET: { + uint64_t *out; + if (*outlen < sizeof(uint64_t)) + return (EOVERFLOW); + out = malloc(sizeof(uint64_t), M_TEMP, M_WAITOK); + *out = 0x1; + *outdata = out; + } + case VPC_VPCSW_OP_STATE_SET: { + //const uint64_t *flags = in; + + if (inlen != sizeof(uint64_t)) + return (EBADRPC); + /* do something with the flags */ + } + case VPC_VPCSW_OP_RESET: + /* hrrrm.... */ + break; + case VPC_VPCSW_OP_RESPONSE: + return (vpcsw_resp_send(vs, in)); + default: + rc = ENOTSUP; + } + return (rc); +} + +static int +vpcsw_ro_update(struct vpcsw_softc *vs) +{ + art_tree *newftable, *oldftable; + int rc; + + rc = vpc_art_tree_clone(vs->vs_ftable_rw, &newftable, M_VPCSW); + if (rc) + return (rc); + oldftable = vs->vs_ftable_ro; + vs->vs_ftable_ro = newftable; + ck_epoch_synchronize(&vpc_global_record); + vpc_art_free(oldftable, M_VPCSW); + return (0); +} + +int +vpcsw_port_connect(if_ctx_t switchctx, struct ifnet *portifp, struct ifnet *devifp) +{ + struct vpcsw_softc *vs; + struct sockaddr_dl *sdl; + uint16_t *ifindexp; + + sdl = (struct sockaddr_dl *)devifp->if_addr->ifa_addr; + if (sdl->sdl_type != IFT_ETHER) + return (EINVAL); + + vs = iflib_get_softc(switchctx); + if (art_search(vs->vs_ftable_rw, LLADDR(sdl)) != NULL) { + if (bootverbose) + printf("port already in forward table can't insert %6D\n", LLADDR(sdl), ":"); + return (ENOENT); + } + + ifindexp = malloc(sizeof(uint16_t), M_VPCSW, M_WAITOK); + *ifindexp = portifp->if_index; + + vpc_ifp_cache(devifp); + vpc_ifp_cache(portifp); + if (bootverbose) + printf("storing ifindexp= %d in switch ART for %6D\n", *ifindexp, LLADDR(sdl), ":"); + art_insert(vs->vs_ftable_rw, LLADDR(sdl), ifindexp); + + return (vpcsw_ro_update(vs)); +} + +int +vpcsw_port_disconnect(if_ctx_t switchctx, struct ifnet *portifp) +{ + struct vpcsw_softc *vs; + struct sockaddr_dl *sdl; + uint16_t *ifindexp; + + vs = iflib_get_softc(switchctx); + sdl = (struct sockaddr_dl *)portifp->if_addr->ifa_addr; + if (sdl->sdl_type != IFT_ETHER) + return (EINVAL); + /* Verify ifnet in table */ + if (art_search(vs->vs_ftable_rw, LLADDR(sdl)) == NULL) { + if (bootverbose) + printf("port not found in forward table can't delete %6D\n", LLADDR(sdl), ":"); + return (ENOENT); + } + ifindexp = art_delete(vs->vs_ftable_rw, LLADDR(sdl)); + free(ifindexp, M_VPCSW); + return (vpcsw_ro_update(vs)); +} + +static int +vpcsw_mbuf_to_qid(struct ifnet *ifp, struct mbuf *m) +{ + return (0); +} + +static int +vpcsw_attach_post(if_ctx_t ctx) +{ + struct ifnet *ifp; + + ifp = iflib_get_ifp(ctx); + + ifp->if_transmit = vpcsw_transmit; + ifp->if_transmit_txq = vpcsw_transmit; + ifp->if_mbuf_to_qid = vpcsw_mbuf_to_qid; + return (0); +} + +static int +clear_bridge(void *data __unused, const unsigned char *key, uint32_t key_len __unused, void *value) +{ + uint16_t *ifindexp = value; + struct ifnet *ifp; + + printf("clearing bridge for key %6D\n", key, ":"); + if ((ifp = vpc_if_lookup(*ifindexp)) == NULL) + return (0); + ifp->if_bridge = NULL; + return (0); +} + +static int +vpcsw_detach(if_ctx_t ctx) +{ + struct vpcsw_softc *vs = iflib_get_softc(ctx); + struct vpcsw_mcast_queue *vmq; + struct vpcsw_request_priv *vrp; + + if (vs->vs_refcnt != 0) + return (EBUSY); + + ck_epoch_synchronize(&vpc_global_record); + + mtx_lock(&vs->vs_lock); + vrp = vs->vs_req_pending; + vs->vs_req_pending = NULL; + m_freem(vrp->vrp_m); + free(vrp, M_VPCSW); + + vmq = &vs->vs_vmq; + while (!TAILQ_EMPTY(&vmq->vmq_head)) { + vrp = TAILQ_FIRST(&vmq->vmq_head); + TAILQ_REMOVE(&vmq->vmq_head, vrp, vrp_entry); + m_freem(vrp->vrp_m); + free(vrp, M_VPCSW); + } + mtx_unlock(&vs->vs_lock); + iflib_config_gtask_deinit(&vs->vs_vtep_gtask); + art_iter(vs->vs_ftable_rw, clear_bridge, NULL); + if (vs->vs_ifdefault) { + if_ctx_t ifctx = vs->vs_ifdefault->if_softc; + vpc_ctx_t vctx; + void *cache; + + cache = vpcp_get_pcpu_cache(ifctx); + free(cache, M_VPCSW); + vctx = vmmnet_lookup(&vs->vs_uplink_id); + MPASS(vctx != NULL); + MPASS(vctx->v_ifp != NULL); + if_clone_destroy(vctx->v_ifp->if_xname); + vctx->v_ifp = NULL; + vmmnet_delete(&vs->vs_uplink_id); + } + mtx_destroy(&vs->vs_lock); + vpc_art_free(vs->vs_ftable_ro, M_VPCSW); + vpc_art_free(vs->vs_ftable_rw, M_VPCSW); + + refcount_release(&modrefcnt); + return (0); +} + +static void +vpcsw_init(if_ctx_t ctx) +{ +} + +static void +vpcsw_stop(if_ctx_t ctx) +{ +} + +static device_method_t vpcsw_if_methods[] = { + DEVMETHOD(ifdi_cloneattach, vpcsw_cloneattach), + DEVMETHOD(ifdi_attach_post, vpcsw_attach_post), + DEVMETHOD(ifdi_detach, vpcsw_detach), + DEVMETHOD(ifdi_init, vpcsw_init), + DEVMETHOD(ifdi_stop, vpcsw_stop), + DEVMETHOD(ifdi_knote_event, vpcsw_knote_event), + DEVMETHOD(ifdi_object_info_get, vpcsw_object_info_get), + DEVMETHOD_END +}; + +static driver_t vpcsw_iflib_driver = { + "vpcsw", vpcsw_if_methods, sizeof(struct vpcsw_softc) +}; + +char vpcsw_driver_version[] = "0.0.1"; + +static struct if_shared_ctx vpcsw_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_driver_version = vpcsw_driver_version, + .isc_driver = &vpcsw_iflib_driver, + .isc_flags = IFLIB_PSEUDO, + .isc_name = "vpcsw", +}; + +if_shared_ctx_t vpcsw_sctx = &vpcsw_sctx_init; + + +static if_pseudo_t vpcsw_pseudo; + +static int +vpcsw_module_init(void) +{ + vpcsw_pseudo = iflib_clone_register(vpcsw_sctx); + + return (vpcsw_pseudo == NULL) ? ENXIO : 0; +} + +static void +vpcsw_module_deinit(void) +{ + iflib_clone_deregister(vpcsw_pseudo); +} + + +static int +vpcsw_module_event_handler(module_t mod, int what, void *arg) +{ + int err; + + switch (what) { + case MOD_LOAD: + if ((err = vpcsw_module_init()) != 0) + return (err); + break; + case MOD_UNLOAD: + if (modrefcnt == 0) + vpcsw_module_deinit(); + else + return (EBUSY); + break; + default: + return (EOPNOTSUPP); + } + + return (0); +} + +static moduledata_t vpcsw_moduledata = { + "vpcsw", + vpcsw_module_event_handler, + NULL +}; + +DECLARE_MODULE(vpcsw, vpcsw_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY); +MODULE_VERSION(vpcsw, 1); +MODULE_DEPEND(vpcsw, vpc, 1, 1, 1); +MODULE_DEPEND(vpcsw, iflib, 1, 1, 1); Index: sys/amd64/vmm/net/vmmnet.c =================================================================== --- /dev/null +++ sys/amd64/vmm/net/vmmnet.c @@ -0,0 +1,928 @@ +/* + * Copyright (C) 2018 Matthew Macy + * Copyright (C) 2018 Joyent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +static MALLOC_DEFINE(M_VMMNET, "vmmnet", "vmm networking"); +static art_tree vpc_uuid_table; + +static struct sx vmmnet_lock; +SX_SYSINIT(vmmnet, &vmmnet_lock, "vmmnet global"); + +#define VMMNET_LOCK() sx_xlock(&vmmnet_lock) +#define VMMNET_UNLOCK() sx_xunlock(&vmmnet_lock) + +#define VPC_CTX_F_DESTROYED 0x1 +#define VPC_CTX_F_COMMITTED 0x2 + +struct vpcctx { + struct ifnet *v_ifp; + vpc_handle_type_t v_handle_type; + vpc_id_t v_id; + + volatile u_int v_refcnt; + uint32_t v_flags; +}; + +static fo_close_t vpcd_close; +static fo_stat_t vpcd_stat; +static fo_fill_kinfo_t vpcd_fill_kinfo; +static fo_poll_t vpcd_poll; +static fo_ioctl_t vpcd_ioctl; + +static struct fileops vpcd_fileops = { + .fo_close = vpcd_close, + .fo_stat = vpcd_stat, + .fo_fill_kinfo = vpcd_fill_kinfo, + .fo_poll = vpcd_poll, + .fo_ioctl = vpcd_ioctl, + .fo_flags = DFLAG_PASSABLE, +}; + +static int +vpcd_print_uuid_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + MPASS(key_len == sizeof(vpc_id_t)); + + printf("%16D\n", key, ":"); + return (0); +} + +struct typecount_info { + uint32_t count; + vpc_obj_type_t type; +}; + +static int +vpcd_typecount_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + struct typecount_info *ti = data; + struct vpcctx *ctx = value; + + if (ti->type == ctx->v_handle_type.vht_obj_type || ti->type == VPC_OBJ_TYPE_ANY) + ti->count++; + return (0); +} + + +struct objget_info { + caddr_t ptr; + uint32_t max_count; + uint32_t count; + vpc_obj_type_t type; + /* pad ?*/ +}; + +#ifdef notyet +static int +vpcd_objget_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + struct objget_info *oi = data; + struct vpcctx *ctx = value; + vpc_obj_info_t *voi; + + if (oi->type != ctx->v_handle_type && oi->type != VPC_OBJ_TYPE_ANY) + return (0); + if (oi->count == oi->max_count) + return (ENOSPC); +} +#endif + +static int +vpcd_hdrget_callback(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + struct objget_info *oi = data; + struct vpcctx *ctx = value; + vpc_obj_header_t *voh; + + + if (oi->type != ctx->v_handle_type.vht_obj_type && oi->type != VPC_OBJ_MGMT) + return (0); + if (oi->count == oi->max_count) + return (ENOSPC); + voh = (void *)oi->ptr; + voh->voh_type = ctx->v_handle_type.vht_obj_type; + if (ctx->v_ifp) + voh->voh_unit = ctx->v_ifp->if_dunit; + memcpy(&voh->voh_id, key, sizeof(vpc_id_t)); + oi->count++; + oi->ptr += sizeof(*voh); + return (0); +} + +static void +vpcd_print_uuids(void) +{ + VMMNET_LOCK(); + art_iter(&vpc_uuid_table, vpcd_print_uuid_callback, NULL); + VMMNET_UNLOCK(); +} + +static int +vpcd_close(struct file *fp, struct thread *td) +{ + struct vpcctx *ctx, *value; + + if ((ctx = fp->f_data) == NULL) + return (0); + fp->f_data = NULL; + if (ctx->v_handle_type.vht_obj_type == VPC_OBJ_PORT) + return (0); + VMMNET_LOCK(); + if (refcount_release(&ctx->v_refcnt)) { + value = art_delete(&vpc_uuid_table, (const char *)&ctx->v_id); + +#ifdef INVARIANTS + if (ctx->v_handle_type.vht_obj_type != VPC_OBJ_MGMT && value != ctx) { + printf("%16D --- vpc_id not found\n", &ctx->v_id, ":"); + vpcd_print_uuids(); + } +#endif + /* run object dtor */ + if (ctx->v_ifp) + if_clone_destroy(ctx->v_ifp->if_xname); + free(ctx, M_VMMNET); + } + VMMNET_UNLOCK(); + return (0); +} + +static int +vpcd_stat(struct file *fp, struct stat *st, struct ucred *active_cred, + struct thread *td) +{ + return (ENXIO); +} + +static int +vpcd_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp) +{ + + kif->kf_type = KF_TYPE_UNKNOWN; + return (0); +} + +static int +vpcd_poll(struct file *fp, int events, struct ucred *active_cred, + struct thread *td) +{ + return (ENXIO); +} + +static int +vpcd_ioctl(struct file *fp, u_long cmd, void *data, + struct ucred *active_cred, struct thread *td) +{ + return (ENXIO); +} + +static char *if_names[] = { + "NONE", + "vpcsw", + "vpcp", + "vpcrtr", + "vpcnat", + "vpcmux", + "vmnic", + "MGMT", + "ethlink", + "META", + "TYPE_ANY", + "hostif" +}; + +int +vmmnet_insert(const vpc_id_t *id, if_t ifp, vpc_type_t type) +{ + struct vpcctx *ctx; + + sx_assert(&vmmnet_lock, SA_XLOCKED); + if (art_search(&vpc_uuid_table, (const char*)id) != NULL) { + if (bootverbose) + printf("%s can't add %16D -- already in vpc_uuid_table\n", + __func__, id, ":"); + return (EEXIST); + } + ctx = malloc(sizeof(*ctx), M_VMMNET, M_WAITOK|M_ZERO); + if_ref(ifp); + ctx->v_ifp = ifp; + memcpy(&ctx->v_id, id, sizeof(*id)); + ctx->v_handle_type.vht_obj_type = type; + refcount_init(&ctx->v_refcnt, 1); + art_insert(&vpc_uuid_table, (const char *)id, ctx); + return (0); +} + +vpc_ctx_t +vmmnet_lookup(const vpc_id_t *id) +{ + + sx_assert(&vmmnet_lock, SA_XLOCKED); + return ((vpc_ctx_t)art_search(&vpc_uuid_table, (const char*)id)); +} + +void +vmmnet_delete(const vpc_id_t *id) +{ + void *ctx; + + sx_assert(&vmmnet_lock, SA_XLOCKED); + ctx = art_delete(&vpc_uuid_table, (const char*)id); + free(ctx, M_VMMNET); +} + +void +vmmnet_ref(vpc_ctx_t vctx) +{ + struct vpcctx *vpcctx = (void *)vctx; + + refcount_acquire(&vpcctx->v_refcnt); +} + +void +vmmnet_rele(vpc_ctx_t vctx) +{ + struct vpcctx *vpcctx = (void *)vctx; + + sx_assert(&vmmnet_lock, SA_XLOCKED); + if (refcount_release(&vpcctx->v_refcnt)) { + sx_xlock(&vmmnet_lock); + art_delete(&vpc_uuid_table, (const char*)&vpcctx->v_id); + /* run object dtor */ + if (vctx->v_ifp) + if_clone_destroy(vctx->v_ifp->if_xname); + sx_xunlock(&vmmnet_lock); + free(vctx, M_VMMNET); + } +} + +static int +kern_vpc_open(struct thread *td, const vpc_id_t *vpc_id, + vpc_type_t obj_type_full, vpc_flags_t flags, + int *vpcd) +{ + struct filedesc *fdp; + struct file *fp; + struct vpcctx *ctx; + struct ifnet *ifp; + vpc_handle_type_t *type; + char buf[IFNAMSIZ]; + int rc, fflags, fd; + uint8_t obj_type; + + type = (vpc_handle_type_t*)&obj_type_full; + obj_type = type->vht_obj_type; + ifp = NULL; + if (ETHER_IS_MULTICAST(vpc_id->node)) + return (EADDRNOTAVAIL); + if (obj_type == 0 || obj_type > VPC_OBJ_TYPE_MAX || + obj_type == VPC_OBJ_META || obj_type == VPC_OBJ_TYPE_ANY) { + printf("obj_type=%d obj_type_full=%lx\n", obj_type, obj_type_full); + return (ENOPROTOOPT); + } + if ((flags & (VPC_F_CREATE|VPC_F_OPEN)) == 0) + return (EINVAL); + if ((flags & (VPC_F_CREATE|VPC_F_OPEN)) == (VPC_F_CREATE|VPC_F_OPEN)) + return (EINVAL); + if ((flags & VPC_F_CREATE) && (obj_type != VPC_OBJ_MGMT) && + (priv_check(td, PRIV_DRIVER) != 0)) + return (EPERM); + VMMNET_LOCK(); + ctx = art_search(&vpc_uuid_table, (const unsigned char *)vpc_id); + if ((flags & VPC_F_CREATE) && (ctx != NULL)) { + rc = EEXIST; + goto unlock; + } + if ((flags & VPC_F_CREATE) && (obj_type == VPC_OBJ_PORT)) { + rc = ENODEV; + goto unlock; + } + if (flags & VPC_F_OPEN) { + if (ctx == NULL) { + rc = ENOENT; + goto unlock; + } + if (ctx->v_handle_type.vht_obj_type != obj_type) { + rc = ENODEV; + goto unlock; + } + refcount_acquire(&ctx->v_refcnt); + } else { + ctx = malloc(sizeof(*ctx), M_VMMNET, M_WAITOK|M_ZERO); + + if (obj_type != VPC_OBJ_MGMT) { + strncpy(buf, if_names[obj_type], IFNAMSIZ-1); + rc = if_clone_create(buf, sizeof(buf), NULL); + if (rc) { + printf("if_clone_create with %s failed with %d\n", + buf, rc); + goto unlock; + } + if ((ifp = ifunit_ref(buf)) == NULL) { + if (bootverbose) + printf("couldn't reference %s\n", buf); + if_clone_destroy(buf); + free(ctx, M_VMMNET); + rc = ENXIO; + goto unlock; + } + } + /* + * One reference for ART and one for descriptor + */ + refcount_init(&ctx->v_refcnt, 1); + ctx->v_ifp = ifp; + ctx->v_handle_type.vht_obj_type = obj_type; + memcpy(&ctx->v_id, vpc_id, sizeof(*vpc_id)); + if (priv_check(td, PRIV_DRIVER) == 0) { + art_insert(&vpc_uuid_table, (const char *)vpc_id, ctx); +#ifdef INVARIANTS + { + struct vpcctx *tmpctx = art_search(&vpc_uuid_table, (const char *)vpc_id); + + MPASS(tmpctx != NULL); + MPASS(tmpctx == ctx); + } +#endif + } + } + + fflags = O_CLOEXEC; + if (flags & VPC_F_WRITE) + fflags |= FWRITE; + if (priv_check(td, PRIV_DRIVER) == 0) + fflags |= O_APPEND; + + fdp = td->td_proc->p_fd; + rc = falloc(td, &fp, &fd, fflags); + if (rc) { + if (flags & VPC_F_CREATE) { + if (ifp) { + if_rele(ifp); + if_clone_destroy(buf); + } + if (priv_check(td, PRIV_DRIVER) == 0) + art_delete(&vpc_uuid_table, (const char *)vpc_id); + free(ctx, M_VMMNET); + } else + refcount_release(&ctx->v_refcnt); + goto unlock; + } + if ((flags & VPC_F_CREATE) && + (ctx->v_ifp != NULL)) { + if_ctx_t ifctx; + int macrc; + + ifctx = ctx->v_ifp->if_softc; + macrc = iflib_set_mac(ifctx, vpc_id->node); + if (macrc && bootverbose) + printf("set_mac failed: %d\n", macrc); + } + finit(fp, fflags, DTYPE_VPCFD, ctx, &vpcd_fileops); + *vpcd = fd; + unlock: + VMMNET_UNLOCK(); + if (rc == 0) + fdrop(fp, td); + return (rc); +} + +static int +vpcnat_ctl(vpc_ctx_t ctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata) +{ + return (EOPNOTSUPP); +} + +int +vpcrtr_ctl(vpc_ctx_t ctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata) +{ + return (EOPNOTSUPP); +} + + +static int +vpcmgmt_ctl(vpc_ctx_t ctx, vpc_op_t op, size_t innbyte, const void *in, + size_t *outnbyte, void **outp) +{ + int rc = 0; + + switch (op) { + case VPC_OBJ_OP_TYPE_COUNT_GET: { + const uint16_t *qtype = in; + uint16_t *typecount; + struct typecount_info ti; + + if (innbyte != sizeof(uint16_t)) + return (EBADRPC); + if (*outnbyte < sizeof(uint32_t)) + return (EOVERFLOW); + + ti.type = (vpc_obj_type_t)*qtype; + *outnbyte = sizeof(uint16_t); + typecount = malloc(sizeof(uint16_t), M_TEMP, M_WAITOK); + ti.count = 0; + art_iter(&vpc_uuid_table, vpcd_typecount_callback, &ti); + *typecount = ti.count; + *outp = typecount; + break; + } + case VPC_OBJ_OP_HDR_GET_ALL: { + const uint16_t *qtype = in; + struct objget_info oi; + void *data; + + if (innbyte != sizeof(uint16_t)) + return (EBADRPC); + if (*outnbyte < sizeof(vpc_obj_header_t)) + return (EOVERFLOW); + + oi.type = *qtype; + oi.count = 0; + oi.max_count = *outnbyte/sizeof(vpc_obj_header_t); + data = malloc(*outnbyte, M_TEMP, M_WAITOK|M_ZERO); + oi.ptr = data; + rc = art_iter(&vpc_uuid_table, vpcd_hdrget_callback, &oi); + *outnbyte = oi.count*sizeof(vpc_obj_header_t); + *outp = data; + break; + } + default: + rc = ENOTSUP; + break; + } + return (rc); +} + +static vpc_ctl_fn vpc_ctl_dispatch[] = { + NULL, + vpcsw_ctl, + vpcp_ctl, + vpcrtr_ctl, + vpcnat_ctl, + vpcmux_ctl, + vmnic_ctl, + vpcmgmt_ctl, + ethlink_ctl, + NULL, /* META */ + NULL, /* TYPE_ANY */ + hostif_ctl +}; +static int +kern_vpc_ctl(struct thread *td, int vpcd, vpc_op_t op, size_t innbyte, + const void *in, size_t *outnbyte, void **outp) +{ + struct ifnet *ifp; + cap_rights_t rights; + struct file *fp; + vpc_op_t objop; + vpc_type_t objtype; + struct vpcctx *ctx; + uint64_t caps; + int rc; + + objtype = VPC_OBJ_TYPE(op); + objop = VPC_OBJ_OP(op); + rc = 0; + if (objtype == 0 || objtype > VPC_OBJ_TYPE_MAX) + return (EOPNOTSUPP); + + caps = CAP_VPC_READ; + if (op & IOC_PRIVMUT) + caps |= CAP_VPC_PRIVWRITE; + else if (op & IOC_PRIV) + caps |= CAP_VPC_PRIVREAD; + else if (op & IOC_MUT) + caps |= CAP_VPC_WRITE; + + if (fget(td, vpcd, cap_rights_init(&rights, caps), &fp) != 0) + return (EBADF); + if ((fp->f_type != DTYPE_VPCFD) || + (fp->f_data == NULL)) { + rc = EBADF; + goto done; + } + ctx = fp->f_data; + if ((objtype != VPC_OBJ_META) && (ctx->v_handle_type.vht_obj_type != objtype)) { + rc = ENODEV; + goto done; + } + if ((op & IOC_PRIV) && ((fp->f_flag & O_APPEND) == 0)) { + rc = EPERM; + goto done; + } + if ((op & IOC_MUT) && ((fp->f_flag & FWRITE) == 0)) { + rc = EPERM; + goto done; + } + if (objtype != VPC_OBJ_META) { + VMMNET_LOCK(); + rc = vpc_ctl_dispatch[objtype]((vpc_ctx_t)ctx, op, innbyte, in, outnbyte, outp); + VMMNET_UNLOCK(); + goto done; + } + + ifp = ctx->v_ifp; + switch (op) { + case VPC_OBJ_OP_DESTROY: + if ((ctx->v_flags & (VPC_CTX_F_DESTROYED|VPC_CTX_F_COMMITTED)) != + VPC_CTX_F_COMMITTED) { + rc = EAGAIN; + goto done; + } + ctx->v_flags |= VPC_CTX_F_DESTROYED; + rc = refcount_release(&ctx->v_refcnt); + MPASS(rc == 0); + break; + case VPC_OBJ_OP_COMMIT: + if (ctx->v_flags & (VPC_CTX_F_DESTROYED|VPC_CTX_F_COMMITTED)) { + rc = EALREADY; + goto done; + } + ctx->v_flags |= VPC_CTX_F_COMMITTED; + refcount_acquire(&ctx->v_refcnt); + break; + case VPC_OBJ_OP_TYPE_GET: { + vpc_obj_type_t *typep; + + *outnbyte = 1; + typep = malloc(sizeof(vpc_obj_type_t), M_TEMP, M_WAITOK); + *typep = ctx->v_handle_type.vht_obj_type; + break; + } + case VPC_OBJ_OP_MAC_SET: { + if_ctx_t ifctx; + const uint8_t *mac = in; + + if ((ctx->v_handle_type.vht_obj_type == VPC_OBJ_ETHLINK) || + (innbyte != ETHER_ADDR_LEN)) { + rc = EBADRPC; + goto done; + } + ifctx = ifp->if_softc; + iflib_set_mac(ifctx, mac); + break; + } + case VPC_OBJ_OP_MAC_GET: { + struct sockaddr_dl *sdl; + uint8_t *mac; + + if (ctx->v_ifp == NULL) { + rc = EINPROGRESS; + goto done; + } + if (*outnbyte < ETHER_ADDR_LEN) { + rc = EOVERFLOW; + goto done; + } + *outnbyte = ETHER_ADDR_LEN; + mac = malloc(ETHER_ADDR_LEN, M_TEMP, M_WAITOK); + sdl = (struct sockaddr_dl *)ifp->if_addr->ifa_addr; + MPASS(sdl->sdl_type == IFT_ETHER); + memcpy(mac, LLADDR(sdl), ETHER_ADDR_LEN); + break; + } + case VPC_OBJ_OP_MTU_SET: { + if_ctx_t ifctx; + const uint32_t *mtu = in; + + if (innbyte != sizeof(uint32_t)) { + rc = EBADRPC; + goto done; + } + ifctx = ifp->if_softc; + iflib_set_mtu(ifctx, *mtu); + break; + } + case VPC_OBJ_OP_MTU_GET: { + uint32_t *mtu; + + if (ifp == NULL) { + rc = EINPROGRESS; + goto done; + } + if (*outnbyte < sizeof(uint32_t)) { + rc = EOVERFLOW; + goto done; + } + *outnbyte = sizeof(uint32_t); + mtu = malloc(*outnbyte, M_TEMP, M_WAITOK); + *mtu = ifp->if_mtu; + break; + } + case VPC_OBJ_OP_ID_GET: { + vpc_id_t *id; + + if (*outnbyte < sizeof(*id)) { + rc = EOVERFLOW; + goto done; + } + *outnbyte = sizeof(*id); + id = malloc(sizeof(*id), M_TEMP, M_WAITOK); + memcpy(id, &ctx->v_id, sizeof(*id)); + break; + } + default: + rc = ENOTSUP; + break; + } + done: + fdrop(fp, td); + return (rc); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vpc_open_args { + const vpc_id_t *vpc_id; + vpc_type_t obj_type; + vpc_flags_t flags; +}; +#endif +int +sys_vpc_open(struct thread *td, struct vpc_open_args *uap) +{ + vpc_id_t *vpc_id; + int rc, vpcd; + + vpc_id = malloc(sizeof(*vpc_id), M_TEMP, M_WAITOK); + if (copyin(uap->vpc_id, vpc_id, sizeof(*vpc_id))) + return (EFAULT); + if (bootverbose) + printf("%s vpc_id: %16D\n", __func__, vpc_id, ":"); + rc = kern_vpc_open(td, vpc_id, htobe64(uap->obj_type), uap->flags, &vpcd); + if (rc) + goto done; + td->td_retval[0] = vpcd; + td->td_retval[1] = 0; + done: + free(vpc_id, M_TEMP); + return (rc); +} + +#ifndef _SYS_SYSPROTO_H_ +struct vpc_ctl_args { + int vpcd; + vpc_op_t op; + size_t innbyte; + const void *in; + size_t *outnbyte; + void *out; +}; +#endif + +int +sys_vpc_ctl(struct thread *td, struct vpc_ctl_args *uap) +{ + size_t koutlen; + void *kout, *kin; + vpc_type_t objtype; + int rc; + + if (uap->innbyte > ARG_MAX) + return (E2BIG); + + kin = kout = NULL; + objtype = VPC_OBJ_TYPE(uap->op); + koutlen = 0; + if (objtype == 0 || objtype > VPC_OBJ_TYPE_MAX) + return (ENXIO); + if (uap->op & IOC_IN) { + if (uap->innbyte == 0) { + if (bootverbose) + printf("IOC_IN set but innbyte == 0\n"); + return (EFAULT); + } + kin = malloc(uap->innbyte, M_TEMP, M_WAITOK); + if (copyin(uap->in, kin, uap->innbyte)) { + if (bootverbose) + printf("copyin %p failed\n", uap->in); + rc = EFAULT; + goto done; + } + if (bootverbose && uap->innbyte == 16) + printf("ctl in: %16D\n", kin, ":"); + } + if (uap->op & IOC_OUT) { + if ((uap->outnbyte == NULL) || (uap->out == NULL)) { + if (bootverbose) + printf("IOC_OUT set but parameters are outnbyte: %p out: %p\n", + uap->outnbyte, uap->out); + rc = EFAULT; + goto done; + } + if (copyin(uap->outnbyte, &koutlen, sizeof(size_t))) { + if (bootverbose) + printf("IOC_OUT set copyin of nbyte: %p failed\n", + uap->outnbyte); + rc = EFAULT; + goto done; + } + if (koutlen == 0) { + rc = ENOSPC; + goto done; + } + } + rc = kern_vpc_ctl(td, uap->vpcd, uap->op, uap->innbyte, kin, &koutlen, &kout); + if (uap->op & IOC_OUT) { + if ((rc = copyout(&koutlen, uap->outnbyte, sizeof(size_t)))) + goto done; + if ((rc = copyout(kout, uap->out, koutlen))) + goto done; + } + done: + free(kin, M_TEMP); + free(kout, M_TEMP); + return (rc); +} + +static int filt_vpcattach(struct knote *kn); +static void filt_vpcdetach(struct knote *kn); +static int filt_vpcevent(struct knote *kn, long hint); + +static struct filterops vpc_filtops = { + .f_isfd = 1, + .f_attach = filt_vpcattach, + .f_detach = filt_vpcdetach, + .f_event = filt_vpcevent, +}; + +int +filt_vpcattach(struct knote *kn) +{ + vpc_ctx_t vctx; + if_ctx_t ctx; + + if (kn->kn_fp->f_type != DTYPE_VPCFD) + return (EBADF); + + vctx = kn->kn_fp->f_data; + if (vctx->v_handle_type.vht_obj_type == VPC_OBJ_MGMT || + vctx->v_handle_type.vht_obj_type == VPC_OBJ_ETHLINK) + return (EBADF); + ctx = vctx->v_ifp->if_softc; + return (iflib_knlist_add(ctx, kn)); +} + +static void +filt_vpcdetach(struct knote *kn) +{ + vpc_ctx_t vctx; + if_ctx_t ctx; + + MPASS(kn->kn_fp->f_type == DTYPE_VPCFD); + vctx = kn->kn_fp->f_data; + MPASS(vctx->v_handle_type.vht_obj_type != VPC_OBJ_SWITCH && + vctx->v_handle_type.vht_obj_type != VPC_OBJ_ETHLINK); + ctx = vctx->v_ifp->if_softc; + iflib_knlist_remove(ctx, kn); +} + +static int +filt_vpcevent(struct knote *kn, long hint) +{ + if_ctx_t ctx; + vpc_ctx_t vctx; + + MPASS(kn->kn_fp->f_type == DTYPE_VPCFD); + vctx = kn->kn_fp->f_data; + MPASS(vctx->v_handle_type.vht_obj_type == VPC_OBJ_SWITCH); + ctx = vctx->v_ifp->if_softc; + + return (iflib_knote_event(ctx, kn, hint)); +} + +static struct syscall_helper_data vmmnet_syscalls[] = { + SYSCALL_INIT_HELPER_F(vpc_open, SYF_CAPENABLED), + SYSCALL_INIT_HELPER_F(vpc_ctl, SYF_CAPENABLED), + SYSCALL_INIT_LAST +}; + +static int +vmmnet_module_init(void) +{ + int rc; + + if ((rc = syscall_helper_register(vmmnet_syscalls, 0))) { + printf("vmmnet syscall register failed %d\n", rc); + return (rc); + } + if ((rc = kqueue_add_filteropts(EVFILT_VPC, &vpc_filtops))) { + syscall_helper_unregister(vmmnet_syscalls); + printf("failed to register vpcsw_filtops %d\n", rc); + return (rc); + } + art_tree_init(&vpc_uuid_table, sizeof(vpc_id_t)); + return (0); +} + +static void +vmmnet_module_deinit(void) +{ + syscall_helper_unregister(vmmnet_syscalls); + kqueue_del_filteropts(EVFILT_VPC); +} + +static int +vmmnet_module_event_handler(module_t mod, int what, void *arg) +{ + switch (what) { + case MOD_LOAD: + return (vmmnet_module_init()); + break; + case MOD_UNLOAD: + vmmnet_module_deinit(); + break; + default: + return (EOPNOTSUPP); + } + + return (0); +} + +static moduledata_t vmmnet_moduledata = { + "vmmnet", + vmmnet_module_event_handler, + NULL +}; + +DECLARE_MODULE(vmmnet, vmmnet_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY); +MODULE_VERSION(vmmnet, 1); +MODULE_DEPEND(vmmnet, vpc, 1, 1, 1); Index: sys/amd64/vmm/net/vpc.c =================================================================== --- /dev/null +++ sys/amd64/vmm/net/vpc.c @@ -0,0 +1,502 @@ +/* + * Copyright (C) 2017-2018 Matthew Macy + * Copyright (C) 2017-2018 Joyent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "ifdi_if.h" + +#include + +/* + * Generic VPC services + */ +struct grouptask vpc_ifp_task; +struct ifp_cache *vpc_ic; +ck_epoch_t vpc_epoch; + +static int vpc_ifindex_target; +static bool exiting = false; + +static struct sx vpc_lock; +SX_SYSINIT(vpc, &vpc_lock, "VPC global"); + +#define VPC_LOCK() sx_xlock(&vpc_lock) +#define VPC_UNLOCK() sx_xunlock(&vpc_lock) + +DPCPU_DEFINE(ck_epoch_record_t *, vpc_epoch_record); +ck_epoch_record_t vpc_global_record; +#define IC_START_COUNT 1024 + + +static MALLOC_DEFINE(M_VPC, "vpc", "virtual private cloud utilities"); + +int +vpc_async_copyout(struct vpc_copy_info *vci, const void *kaddr, void *uaddr, size_t len) +{ + struct proc *p = vci->vci_proc; + vm_page_t *pages = vci->vci_pages; + const char *ckaddr = kaddr; + int off, count, rem, copylen; + + if (vci->vci_max_count*PAGE_SIZE < len) + return (E2BIG); + + if (len == 0) + return (0); + count = vm_fault_quick_hold_pages(&p->p_vmspace->vm_map, (vm_offset_t)uaddr, len, VM_PROT_WRITE, pages, + vci->vci_max_count); + if (count == 0) + return (0); + if (count == -1) + return (EFAULT); + rem = len; + off = 0; + do { + copylen = min(rem, PAGE_SIZE); + bcopy(ckaddr + off, (void*)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)), copylen); + off += copylen; + rem -= copylen; + pages++; + } while (rem); + pages = vci->vci_pages; + do { + vm_page_lock(*pages); + vm_page_unhold(*pages); + vm_page_unlock(*pages); + pages++; + count--; + } while (count); + return (0); +} + +static __inline int +alloc_size(void *addr) +{ + uma_slab_t slab; + int size; + + MPASS(addr); + slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK)); + if (__predict_false(slab == NULL)) + panic("free_domain: address %p(%p) has not been allocated.\n", + addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); + + if (!(slab->us_flags & UMA_SLAB_MALLOC)) { + size = slab->us_keg->uk_size; + } else { + size = slab->us_size; + } + return (size); +} + +struct art_tree_info { + art_tree *tree; + struct malloc_type *type; +}; + +static int +art_copy_one(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + struct art_tree_info *info = data; + art_tree *dst = info->tree; + void *newvalue; + int size; + + size = alloc_size(value); + newvalue = malloc(size, info->type, M_WAITOK); + memcpy(newvalue, value, size); + + art_insert(dst, key, newvalue); + return (0); +} + +static int +art_free_one(void *data, const unsigned char *key, uint32_t key_len, void *value) +{ + struct malloc_type *type = data; + + free(value, type); + return (0); +} + +void +vpc_art_free(art_tree *tree, struct malloc_type *type) +{ + art_iter(tree, art_free_one, type); + art_tree_destroy(tree); + free(tree, type); +} + +int +vpc_art_tree_clone(art_tree *src, art_tree **dstp, struct malloc_type *type) +{ + art_tree *dst; + struct art_tree_info info; + int rc; + + info.type = type; + info.tree = dst = malloc(sizeof(art_tree), type, M_WAITOK); + art_tree_init(dst, src->key_len); + rc = art_iter(src, art_copy_one, &info); + if (rc) { + vpc_art_free(dst, type); + } else + *dstp = dst; + return (rc); +} + +static void * +m_advance(struct mbuf **pm, int *poffset, int len) +{ + struct mbuf *m = *pm; + int offset = *poffset; + uintptr_t p = 0; + + MPASS(!m_ismvec(m)); + MPASS(len > 0); + + for (;;) { + if (offset + len < m->m_len) { + offset += len; + p = mtod(m, uintptr_t) + offset; + break; + } + len -= m->m_len - offset; + m = m->m_next; + offset = 0; + MPASS(m != NULL); + } + *poffset = offset; + *pm = m; + return ((void *)p); +} + +int +vpc_parse_pkt(struct mbuf *m0, struct vpc_pkt_info *tpi) +{ + const struct ether_vlan_header *evh; + const struct tcphdr *th; + const struct udphdr *uh; + struct mvec_cursor mc; + struct mbuf *m; + int eh_type, ipproto; + int l2len, l3len, offset, hashtmp, rc; + uint8_t tos; + const uint16_t *macp; + const void *l3hdr; + const void *l4hdr; + bool ismvec; + + mc.mc_off = mc.mc_idx = 0; + m = m0; + if (m0->m_len < ETHER_HDR_LEN) + return (0); + + tos = offset = 0; + ismvec = m_ismvec(m); + evh = (const void*)m0->m_data; + macp = (const void*)m0->m_data; + eh_type = ntohs(evh->evl_encap_proto); + if (eh_type == ETHERTYPE_VLAN) { + eh_type = ntohs(evh->evl_proto); + l2len = sizeof(*evh); + } else + l2len = ETHER_HDR_LEN; + hashtmp = eh_type ^ macp[0] ^ macp[1] ^ macp[2] ^ + macp[3] ^ macp[4] ^ macp[5]; + if (ismvec) + l3hdr = mvec_advance(m, &mc, l2len); + else + l3hdr = m_advance(&m, &offset, l2len); + + switch(eh_type) { +#ifdef INET6 + case ETHERTYPE_IPV6: + { + const struct ip6_hdr *ip6 = l3hdr; + + l3len = sizeof(*ip6); + ipproto = ip6->ip6_nxt; + tpi->vpi_v6 = 1; + break; + } +#endif +#ifdef INET + case ETHERTYPE_IP: + { + const struct ip *ip = l3hdr; + + hashtmp ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr; + l3len = ip->ip_hl << 2; + ipproto = ip->ip_p; + tos = ip->ip_tos; + tpi->vpi_v6 = 0; + break; + } +#endif + case ETHERTYPE_ARP: + default: + l3len = 0; + ipproto = 0; + tpi->vpi_v6 = 0; + break; + } + tpi->vpi_etype = eh_type; + tpi->vpi_proto = ipproto; + tpi->vpi_tos = tos; + m->m_pkthdr.l2hlen = tpi->vpi_l2_len = l2len; + m->m_pkthdr.l3hlen = tpi->vpi_l3_len = l3len; + if (l3len == 0) + return (0); + + if (ismvec) + l4hdr = mvec_advance(m, &mc, l3len); + else + l4hdr = m_advance(&m, &offset, l3len); + rc = 1; + if (ipproto == IPPROTO_TCP) { + th = l4hdr; + hashtmp ^= th->th_sport ^ th->th_dport; + m->m_pkthdr.l4hlen = tpi->vpi_l4_len = th->th_off << 2; + } else if (ipproto == IPPROTO_UDP) { + uh = l4hdr; + hashtmp ^= uh->uh_sport ^ uh->uh_dport; + m->m_pkthdr.l4hlen = tpi->vpi_l4_len = sizeof(struct udphdr); + } else { + rc = 0; + } + tpi->vpi_hash = hashtmp; + MPASS((rc == 0) || (l2len && l3len && tpi->vpi_l4_len)); + return (rc); +} + +static void +task_fn_ifp_update_(void *context __unused) +{ + struct ifnet **ifps, **ifps_orig; + int i, max, count; + + if (vpc_ifindex_target > vpc_ic->ic_size) { + /* grow and replace after wait */ + } + max = vpc_ic->ic_ifindex_max; + ifps = malloc(sizeof(ifps)*max, M_VPC, M_WAITOK|M_ZERO); + ifps_orig = vpc_ic->ic_ifps; + for (count = i = 0; i < max; i++) { + if (ifps_orig[i] == NULL) + continue; + if (__predict_true(!(ifps_orig[i]->if_flags & IFF_DYING) && !exiting)) + continue; + ifps[i] = ifps_orig[i]; + ifps_orig[i] = NULL; + count++; + } + if (count == 0) + goto done; + ck_epoch_synchronize(&vpc_global_record); + for (i = 0; i < max && count; i++){ + if (ifps[i] == NULL) + continue; + if_rele(ifps[i]); + count--; + } + done: + free(ifps, M_VPC); + if (__predict_false(exiting)) { + VPC_LOCK(); + free(vpc_ic, M_VPC); + vpc_ic = NULL; + wakeup(&exiting); + VPC_UNLOCK(); + } +} + +int +vpc_ifp_cache(struct ifnet *ifp) +{ + if (__predict_false(vpc_ic->ic_size -1 < ifp->if_index)) { + GROUPTASK_ENQUEUE(&vpc_ifp_task); + return (1); + } + if (vpc_ic->ic_ifps[ifp->if_index] == ifp) + return (0); + + /* XXX -- race if reference twice -- need to actually serialize with VPCMUX_LOCK */ + if (vpc_ic->ic_ifindex_max < ifp->if_index) + vpc_ic->ic_ifindex_max = ifp->if_index; + MPASS(vpc_ic->ic_ifps[ifp->if_index] == NULL); + if_ref(ifp); + vpc_ic->ic_ifps[ifp->if_index] = ifp; + return (0); +} + +struct ifnet * +vpc_if_lookup(uint32_t ifindex) +{ + struct ifnet *ifp; + + KASSERT(ifindex <= vpc_ic->ic_ifindex_max, + ("passed ifindex %d exceeds vpc_ic->ic_ifindex_max: %d", + ifindex,vpc_ic->ic_ifindex_max)); + if (__predict_false(((ifp = vpc_ic->ic_ifps[ifindex]) == NULL))) + return (NULL); + if (__predict_false(ifp->if_flags & IFF_DYING)) { + GROUPTASK_ENQUEUE(&vpc_ifp_task); + return (NULL); + } + return (ifp); +} + +static int +vpc_module_init(void) +{ + ck_epoch_record_t **erpp, *erp; + int i, er_size; + + ck_epoch_init(&vpc_epoch); + ck_epoch_register(&vpc_epoch, &vpc_global_record, NULL); + iflib_config_gtask_init(NULL, &vpc_ifp_task, task_fn_ifp_update_, "ifp update"); + + /* DPCPU vpc epoch record init */ + er_size = roundup(sizeof(*erp), CACHE_LINE_SIZE); + erp = malloc(er_size*mp_ncpus, M_VPC, M_WAITOK); + vpc_ic = malloc(sizeof(uint64_t) + (sizeof(struct ifnet *)*IC_START_COUNT), + M_VPC, M_WAITOK|M_ZERO); + vpc_ic->ic_size = IC_START_COUNT; + + CPU_FOREACH(i) { + ck_epoch_register(&vpc_epoch, erp, NULL); + erpp = DPCPU_ID_PTR(i, vpc_epoch_record); + *erpp = erp; + erp = (ck_epoch_record_t *)(((caddr_t)erp) + er_size); + } + return (0); +} + +static void +vpc_module_deinit(void) +{ + ck_epoch_record_t *erp; + + VPC_LOCK(); + exiting = true; + GROUPTASK_ENQUEUE(&vpc_ifp_task); + sx_sleep(&exiting, &vpc_lock, PDROP, "vpc exiting", 0); + + erp = DPCPU_ID_GET(0, vpc_epoch_record); + free(erp, M_VPC); + iflib_config_gtask_deinit(&vpc_ifp_task); +} + +static int +vpc_module_event_handler(module_t mod, int what, void *arg) +{ + int err; + + switch (what) { + case MOD_LOAD: + if ((err = vpc_module_init()) != 0) + return (err); + break; + case MOD_UNLOAD: + vpc_module_deinit(); + break; + default: + return (EOPNOTSUPP); + } + return (0); +} + +static moduledata_t vpc_moduledata = { + "vpc", + vpc_module_event_handler, + NULL +}; + +DECLARE_MODULE(vpc, vpc_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY); +MODULE_VERSION(vpc, 1); +MODULE_DEPEND(vpc, iflib, 1, 1, 1); Index: sys/amd64/vmm/net/vpc_route.c =================================================================== --- /dev/null +++ sys/amd64/vmm/net/vpc_route.c @@ -0,0 +1,450 @@ +/* + * Copyright (C) 2018 Matthew Macy + * Copyright (C) 2018 Joyent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_inet.h" +#include "opt_inet6.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static MALLOC_DEFINE(M_VPCROUTE, "vpcrte", "virtual private cloud routing table"); + +/* + * Do not require radix to compare more than actual IPv4/IPv6 address + */ +#define KEY_LEN_INET (offsetof(struct sockaddr_in, sin_addr) + sizeof(in_addr_t)) +#define KEY_LEN_INET6 (offsetof(struct sa_in6, sin6_addr) + sizeof(struct in6_addr)) + +#define OFF_LEN_INET (8 * offsetof(struct sockaddr_in, sin_addr)) +#define OFF_LEN_INET6 (8 * offsetof(struct sa_in6, sin6_addr)) +#define KEY_LEN(v) *((uint8_t *)&(v)) + +struct radix_addr_entry { + struct radix_node rn[2]; + struct sockaddr_in addr; + uint32_t value; + uint8_t masklen; +}; + +struct sa_in6 { + uint8_t sin6_len; + uint8_t sin6_family; + uint8_t pad[2]; + struct in6_addr sin6_addr; +}; + +struct radix_addr_xentry { + struct radix_node rn[2]; + struct sa_in6 addr6; + uint32_t value; + uint8_t masklen; +}; + +struct rt_buf +{ + void *ent_ptr; + struct sockaddr *addr_ptr; + struct sockaddr *mask_ptr; + union { + struct { + struct sockaddr_in sa; + struct sockaddr_in ma; + } a4; + struct { + struct sa_in6 sa; + struct sa_in6 ma; + } a6; + } addr; +}; + +struct rtr_ctx { + struct radix_node_head *rc_rnh4; + struct radix_node_head *rc_rnh6; + uint32_t rc_cnt4; + uint32_t rc_cnt6; + struct rt_buf rc_tb; +}; + +/* Apply ipv6 mask on ipv6 addr */ +#define APPLY_MASK(addr,mask) \ + (addr)->__u6_addr.__u6_addr32[0] &= (mask)->__u6_addr.__u6_addr32[0]; \ + (addr)->__u6_addr.__u6_addr32[1] &= (mask)->__u6_addr.__u6_addr32[1]; \ + (addr)->__u6_addr.__u6_addr32[2] &= (mask)->__u6_addr.__u6_addr32[2]; \ + (addr)->__u6_addr.__u6_addr32[3] &= (mask)->__u6_addr.__u6_addr32[3]; + +#ifdef INET6 +static inline void ipv6_writemask(struct in6_addr *addr6, uint8_t mask); + +static inline void +ipv6_writemask(struct in6_addr *addr6, uint8_t mask) +{ + uint32_t *cp; + + for (cp = (uint32_t *)addr6; mask >= 32; mask -= 32) + *cp++ = 0xFFFFFFFF; + if (mask > 0) + *cp = htonl(mask ? ~((1 << (32 - mask)) - 1) : 0); +} +#endif + +static int +addr_to_sockaddr_ent(void *paddr, struct sockaddr *sa, + struct sockaddr *ma, uint8_t mlen, int family) +{ +#ifdef INET + struct sockaddr_in *addr, *mask; +#endif +#ifdef INET6 + struct sa_in6 *addr6, *mask6; +#endif + in_addr_t a4; + + if (family == AF_INET) { +#ifdef INET + addr = (struct sockaddr_in *)sa; + mask = (struct sockaddr_in *)ma; + /* Set 'total' structure length */ + KEY_LEN(*addr) = KEY_LEN_INET; + KEY_LEN(*mask) = KEY_LEN_INET; + addr->sin_family = AF_INET; + mask->sin_addr.s_addr = + htonl(mlen ? ~((1 << (32 - mlen)) - 1) : 0); + a4 = *((in_addr_t *)paddr); + addr->sin_addr.s_addr = a4 & mask->sin_addr.s_addr; + return (mlen != 32); +#endif +#ifdef INET6 + } else if (family == AF_INET6) { + /* IPv6 case */ + addr6 = (struct sa_in6 *)sa; + mask6 = (struct sa_in6 *)ma; + /* Set 'total' structure length */ + KEY_LEN(*addr6) = KEY_LEN_INET6; + KEY_LEN(*mask6) = KEY_LEN_INET6; + addr6->sin6_family = AF_INET6; + ipv6_writemask(&mask6->sin6_addr, mlen); + memcpy(&addr6->sin6_addr, paddr, sizeof(struct in6_addr)); + APPLY_MASK(&addr6->sin6_addr, &mask6->sin6_addr); + return (mlen != 128); +#endif + } + return (0); +} + +static int +rtr_ctx_pre_del(void *paddr, + struct rt_buf *tb, uint8_t mlen, int family) +{ + struct sockaddr *addr, *mask; + int set_mask; + + if (family == AF_INET) { + if (mlen > 32) + return (EINVAL); + + addr = (struct sockaddr *)&tb->addr.a4.sa; + mask = (struct sockaddr *)&tb->addr.a4.ma; +#ifdef INET6 + } else if (family == AF_INET6) { + if (mlen > 128) + return (EINVAL); + + addr = (struct sockaddr *)&tb->addr.a6.sa; + mask = (struct sockaddr *)&tb->addr.a6.ma; +#endif + } else + return (EINVAL); + + set_mask = addr_to_sockaddr_ent(paddr, addr, mask, mlen, family); + tb->addr_ptr = addr; + if (set_mask) + tb->mask_ptr = mask; + + return (0); +} + +static int +rtr_ctx_pre_add(void *paddr, + struct rt_buf *tb, uint8_t mlen, int family) +{ + struct radix_addr_entry *ent; +#ifdef INET6 + struct radix_addr_xentry *xent; +#endif + struct sockaddr *addr, *mask; + int set_mask; + + if (family == AF_INET) { +#ifdef INET + if (mlen > 32) + return (EINVAL); + ent = malloc(sizeof(*ent), M_VPCROUTE, M_WAITOK | M_ZERO); + ent->masklen = mlen; + + addr = (struct sockaddr *)&ent->addr; + mask = (struct sockaddr *)&tb->addr.a4.ma; + tb->ent_ptr = ent; +#endif +#ifdef INET6 + } else if (family == AF_INET6) { + /* IPv6 case */ + if (mlen > 128) + return (EINVAL); + xent = malloc(sizeof(*xent), M_VPCROUTE, M_WAITOK | M_ZERO); + xent->masklen = mlen; + + addr = (struct sockaddr *)&xent->addr6; + mask = (struct sockaddr *)&tb->addr.a6.ma; + tb->ent_ptr = xent; +#endif + } else { + /* Unknown CIDR type */ + return (EINVAL); + } + + set_mask = addr_to_sockaddr_ent(paddr, addr, mask, mlen, family); + /* Set pointers */ + tb->addr_ptr = addr; + if (set_mask) + tb->mask_ptr = mask; + + return (0); +} + +static void +rtr_ctx_add(struct rtr_ctx *rc, uint32_t value, struct rt_buf *tb, int family) +{ + struct radix_node_head *rnh; + struct radix_node *rn; + + if (family == AF_INET) { + rnh = rc->rc_rnh4; + ((struct radix_addr_entry *)tb->ent_ptr)->value = value; + } else { + rnh = rc->rc_rnh6; + ((struct radix_addr_xentry *)tb->ent_ptr)->value = value; + } + rn = rn_lookup_flags(tb->addr_ptr, tb->mask_ptr, &rnh->rh, M_WAITOK); + if (rn != NULL) + return; + + rn_addroute_flags(tb->addr_ptr, tb->mask_ptr, &rnh->rh, tb->ent_ptr, M_WAITOK); + if (family == AF_INET) + rc->rc_cnt4++; + else + rc->rc_cnt6++; + tb->ent_ptr = NULL; +} + +static int +rtr_ctx_del(struct rtr_ctx *rc, struct rt_buf *tb, int family) +{ + struct radix_node_head *rnh; + struct radix_node *rn; + + if (family == AF_INET) { + rnh = rc->rc_rnh4; + } else { + rnh = rc->rc_rnh6; + } + rn = rn_delete_flags(tb->addr_ptr, tb->mask_ptr, &rnh->rh, M_WAITOK); + if (rn == NULL) + return (0); + + tb->ent_ptr = rn; + + if (family == AF_INET) + rc->rc_cnt4--; + else + rc->rc_cnt6--; + + return (0); +} + +int +vpc_rtr_add(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen, int family) +{ + struct rt_buf *tb = &rc->rc_tb; + int error; + + if ((error = rtr_ctx_pre_add(paddr, tb, mlen, family))) + return (error); + rtr_ctx_add(rc, value, tb, family); + return (0); +} + +int +vpc_rtr_addv4(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen) +{ + return (vpc_rtr_add(rc, paddr, value, mlen, AF_INET)); +} + +int +vpc_rtr_addv6(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen) +{ + return (vpc_rtr_add(rc, paddr, value, mlen, AF_INET6)); +} + +int +vpc_rtr_del(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen, int family) +{ + struct rt_buf *tb = &rc->rc_tb; + int error; + + if ((error = rtr_ctx_pre_del(paddr, tb, mlen, family))) + return (error); + rtr_ctx_del(rc, tb, family); + return (0); +} + +int +vpc_rtr_delv4(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen) +{ + return (vpc_rtr_del(rc, paddr, value, mlen, AF_INET)); +} + +int +vpc_rtr_delv6(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen) +{ + return (vpc_rtr_del(rc, paddr, value, mlen, AF_INET6)); +} + +int +vpc_rtr_lookup(rtr_ctx_t rc, void *key, uint32_t *val, int family) +{ + struct radix_node_head *rnh; + + if (family == AF_INET) { + struct radix_addr_entry *ent; + struct sockaddr_in sa; + KEY_LEN(sa) = KEY_LEN_INET; + sa.sin_addr.s_addr = *((in_addr_t *)key); + rnh = rc->rc_rnh4; + ent = (struct radix_addr_entry *)(rnh->rnh_matchaddr(&sa, &rnh->rh)); + if (ent != NULL) { + *val = ent->value; + return (1); + } + } else { + struct radix_addr_xentry *xent; + struct sa_in6 sa6; + KEY_LEN(sa6) = KEY_LEN_INET6; + memcpy(&sa6.sin6_addr, key, sizeof(struct in6_addr)); + rnh = rc->rc_rnh6; + xent = (struct radix_addr_xentry *)(rnh->rnh_matchaddr(&sa6, &rnh->rh)); + if (xent != NULL) { + *val = xent->value; + return (1); + } + } + + return (0); +} + +int +vpc_rtr_lookupv4(rtr_ctx_t rc, void *key, uint32_t *val) +{ + return (vpc_rtr_lookup(rc, key, val, AF_INET)); +} + +int +vpc_rtr_lookupv6(rtr_ctx_t rc, void *key, uint32_t *val) +{ + return (vpc_rtr_lookup(rc, key, val, AF_INET6)); +} + +rtr_ctx_t +vpc_rtr_ctx_alloc(void) +{ + rtr_ctx_t rc; + + rc = malloc(sizeof(*rc), M_VPCROUTE, M_ZERO|M_WAITOK); + + rn_inithead_flags((void **)&rc->rc_rnh4, OFF_LEN_INET, M_WAITOK); + rn_inithead_flags((void **)&rc->rc_rnh6, OFF_LEN_INET6, M_WAITOK); + return (rc); +} + +static int +free_radix_entry(struct radix_node *rn, void *arg) +{ + struct radix_node_head * const rnh = arg; + struct radix_addr_entry *ent; + + ent = (struct radix_addr_entry *) + rn_delete_flags(rn->rn_key, rn->rn_mask, &rnh->rh, M_WAITOK); + if (ent != NULL) + free(ent, M_VPCROUTE); + return (0); +} + +void +vpc_rtr_ctx_free(rtr_ctx_t rc) +{ + struct radix_node_head *rnh; + + rnh = rc->rc_rnh4; + rnh->rnh_walktree(&rnh->rh, free_radix_entry, rnh); + rn_detachhead((void**)&rc->rc_rnh4); + + rnh = rc->rc_rnh6; + rnh->rnh_walktree(&rnh->rh, free_radix_entry, rnh); + rn_detachhead((void **)&rc->rc_rnh6); + + free(rc, M_VPCROUTE); +} Index: sys/net/art.h =================================================================== --- /dev/null +++ sys/net/art.h @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2012, Armon Dadgar + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the organization nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL ARMON DADGAR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef ART_H +#define ART_H + +#define NODE4 0 +#define NODE16 1 +#define NODE48 2 +#define NODE256 3 +#define NODE_MAX NODE256 + +#define MAX_PREFIX_LEN 5 /* ETHER_ADDR_LEN-1 --- this all really needs + * to be parameterized by actual key size + * This value word aligns the art_node children + * array and thus gives better zone allocator + * density than a larger value (up to 13) + */ + + +typedef int(*art_callback)(void *data, const unsigned char *key, uint32_t key_len, void *value); + +/** + * This struct is included as part + * of all the various node sizes + */ +typedef struct { + uint8_t num_children; + uint8_t type:2; + uint8_t partial_len:6; + unsigned char partial[MAX_PREFIX_LEN]; +} art_node; + +/** + * Small node with only 4 children + */ +typedef struct { + art_node n; + unsigned char keys[4]; + art_node *children[4]; +} art_node4; + +/** + * Node with 16 children + */ +typedef struct { + art_node n; + unsigned char keys[16]; + art_node *children[16]; +} art_node16; + +/** + * Node with 48 children, but + * a full 256 byte field. + */ +typedef struct { + art_node n; + unsigned char keys[256]; + art_node *children[48]; +} art_node48; + +/** + * Full node with 256 children + */ +typedef struct { + art_node n; + art_node *children[256]; +} art_node256; + +/** + * Represents a leaf. These are + * of arbitrary size, as they include the key. + */ +typedef struct { + void *value; + unsigned char key[]; +} art_leaf; + +/** + * Main struct, points to root. + */ +typedef struct { + art_node *root; + uint64_t size; + uint32_t key_len; +} art_tree; + +/** + * Initializes an ART tree + * @return 0 on success. + */ +int art_tree_init(art_tree *t, uint32_t key_len); + +/** + * Destroys an ART tree + * @return 0 on success. + */ +int art_tree_destroy(art_tree *t); + +/** + * Returns the size of the ART tree. + */ +static inline uint64_t +art_size(art_tree *t) { + return t->size; +} + +/** + * Inserts a new value into the ART tree + * @arg t The tree + * @arg key The key + * @arg value Opaque value. + * @return NULL if the item was newly inserted, otherwise + * the old value pointer is returned. + */ +void* art_insert(art_tree *t, const unsigned char *key, void *value); + +/** + * Deletes a value from the ART tree + * @arg t The tree + * @arg key The key + * @return NULL if the item was not found, otherwise + * the value pointer is returned. + */ +void* art_delete(art_tree *t, const unsigned char *key); + +/** + * Searches for a value in the ART tree + * @arg t The tree + * @arg key The key + * @return NULL if the item was not found, otherwise + * the value pointer is returned. + */ +void* art_search(const art_tree *t, const unsigned char *key); + +/** + * Returns the minimum valued leaf + * @return The minimum leaf or NULL + */ +art_leaf* art_minimum(art_tree *t); + +/** + * Returns the maximum valued leaf + * @return The maximum leaf or NULL + */ +art_leaf* art_maximum(art_tree *t); + +/** + * Iterates through the entries pairs in the map, + * invoking a callback for each. The call back gets a + * key, value for each and returns an integer stop value. + * If the callback returns non-zero, then the iteration stops. + * @arg t The tree to iterate over + * @arg cb The callback function to invoke + * @arg data Opaque handle passed to the callback + * @return 0 on success, or the return of the callback. + */ +int art_iter(art_tree *t, art_callback cb, void *data); + +/** + * Iterates through the entries pairs in the map, + * invoking a callback for each that matches a given prefix. + * The call back gets a key, value for each and returns an integer stop value. + * If the callback returns non-zero, then the iteration stops. + * @arg t The tree to iterate over + * @arg prefix The prefix of keys to read + * @arg prefix_len The length of the prefix + * @arg cb The callback function to invoke + * @arg data Opaque handle passed to the callback + * @return 0 on success, or the return of the callback. + */ +int art_iter_prefix(art_tree *t, const unsigned char *prefix, int prefix_len, art_callback cb, void *data); + +#endif Index: sys/net/if_vpc.h =================================================================== --- /dev/null +++ sys/net/if_vpc.h @@ -0,0 +1,493 @@ +/* + * Copyright (C) 2017-2018 Matthew Macy + * Copyright (C) 2017-2018 Joyent Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef __IF_VPC_H_ +#define __IF_VPC_H_ + +#include +#include +#include + +#define VPC_VERS 0x20171228 +struct vpc_ioctl_header { + uint64_t vih_magic; + uint64_t vih_type; +}; + +struct vpcmux_fte { + uint8_t vf_hwaddr[ETHER_ADDR_LEN]; + uint16_t vf_vlanid; + uint32_t vf_vni; + struct sockaddr vf_protoaddr; +}; + +struct vpcmux_fte_list { + uint32_t vfl_count; + struct vpcmux_fte vfl_vftes[0]; +}; + +struct vpcsw_op_header { + uint32_t voh_version; + uint32_t voh_op; +}; + +struct vpcsw_op_context { + uint32_t voc_vni; + uint16_t voc_vtag; + uint16_t voc_len; +}; + +struct vpcsw_request { + vpc_id_t vrq_id; + uint8_t vrq_data[0]; +}; + +struct vpcsw_response { + struct vpcsw_op_context vrs_context; + uint8_t vrs_data[0]; +}; + +struct vpcrtr_request_v4 { + uint8_t vr_vpcsw_mac[ETHER_ADDR_LEN]; + uint16_t vr_svtag; + uint32_t vr_svni; + struct in_addr vr_saddr; + struct in_addr vr_daddr; + uint8_t vr_smac[ETHER_ADDR_LEN]; +}; + +struct vpcrtr_request_v6 { + uint8_t vr_vpcsw_smac[ETHER_ADDR_LEN]; + uint16_t vr_svtag; + uint32_t vr_svni; + struct in6_addr vr_saddr; + struct in6_addr vr_daddr; + uint8_t vr_smac[ETHER_ADDR_LEN]; +}; + +struct vpcrtr_response_v4 { + struct vpcrtr_request_v4 vr_context; + uint16_t vr_dvtag; + uint32_t vr_dvni; + struct in_addr vr_daddr; + uint8_t vr_dmac[ETHER_ADDR_LEN]; + uint8_t vr_vpcsw_dmac[ETHER_ADDR_LEN]; +}; + +typedef struct { + uint32_t voh_type; + uint32_t voh_unit; + vpc_id_t voh_id; +} vpc_obj_header_t; + +enum vpc_obj_type { + VPC_OBJ_INVALID = 0, + VPC_OBJ_SWITCH = 1, + VPC_OBJ_PORT = 2, + VPC_OBJ_ROUTER = 3, + VPC_OBJ_NAT = 4, + VPC_OBJ_VPCMUX = 5, + VPC_OBJ_VMNIC = 6, + VPC_OBJ_MGMT = 7, + VPC_OBJ_ETHLINK = 8, + VPC_OBJ_META = 9, + VPC_OBJ_TYPE_ANY = 10, + VPC_OBJ_HOSTIF = 11, + VPC_OBJ_TYPE_MAX = 11, +}; + +typedef enum vpc_obj_type vpc_obj_type_t; + +typedef struct { + vpc_obj_header_t voi_hdr; + union { + struct { + uint32_t vni; + } vswitch; + struct { + vpc_obj_type_t type; + } port; + struct { + } vmnic; + struct { + } ethlink; + struct { + } vpcmux; + }; +} vpc_obj_info_t; + + +typedef struct { + uint8_t vht_version:4; + uint8_t vht_pad1:4; + vpc_obj_type_t vht_obj_type:8; + uint64_t vht_pad2:48; +} vpc_handle_type_t; +#ifdef _KERNEL +CTASSERT(sizeof(vpc_handle_type_t) == sizeof(uint64_t)); + +#include +#include +#include +#include + +#define M_TRUNK M_PROTO1 +#define M_HOLBLOCKING M_PROTO2 + +#define M_VPCMASK (M_PROTO1|M_PROTO2) + +struct ifp_cache { + uint16_t ic_ifindex_max; + uint16_t ic_size; + uint32_t ic_pad; + struct ifnet *ic_ifps[0]; +}; + +struct vpc_pkt_info { + uint16_t vpi_etype; + uint16_t vpi_hash; + uint8_t vpi_l2_len; + uint8_t vpi_l3_len; + uint8_t vpi_l4_len; + uint8_t vpi_v6:1; + uint8_t vpi_proto:7; + uint8_t vpi_tos; +}; + +struct ck_epoch_record; +extern ck_epoch_t vpc_epoch; +extern struct ck_epoch_record vpc_global_record; +DPCPU_DECLARE(struct ck_epoch_record *, vpc_epoch_record); + + +int vpc_parse_pkt(struct mbuf *m0, struct vpc_pkt_info *tpi); +int vpc_art_tree_clone(art_tree *src, art_tree **dst, struct malloc_type *type); +void vpc_art_free(art_tree *tree, struct malloc_type *type); + +static inline void +vpc_epoch_begin(void) +{ + _critical_enter(); + sched_pin(); + ck_epoch_begin(DPCPU_GET(vpc_epoch_record), NULL); + _critical_exit(); +} + +static inline void +vpc_epoch_end(void) +{ + _critical_enter(); + sched_unpin(); + ck_epoch_end(DPCPU_GET(vpc_epoch_record), NULL); + _critical_exit(); +} + +int vpc_ifp_cache(struct ifnet *ifp); + +int vpcp_set_ifswitch(if_ctx_t ctx, if_t ifp); +if_t vpcp_get_ifswitch(if_ctx_t ctx); +void vpcp_clear_ifswitch(if_ctx_t ctx); + +struct rtr_ctx; +typedef struct rtr_ctx *rtr_ctx_t; + +typedef struct vpcctx_public { + struct ifnet *v_ifp; + vpc_handle_type_t v_handle_type; + vpc_id_t v_id; +} *vpc_ctx_t; + +struct vpc_copy_info { + struct proc *vci_proc; + vm_page_t *vci_pages; + int vci_max_count; +}; + +typedef int (*vpc_ctl_fn) (vpc_ctx_t ctx, vpc_op_t op, size_t keylen, + const void *key, size_t *vallen, void **buf); + +int vmmnet_insert(const vpc_id_t *id, if_t ifp, vpc_type_t type); +vpc_ctx_t vmmnet_lookup(const vpc_id_t *id); +void vmmnet_delete(const vpc_id_t *id); +void vmmnet_ref(vpc_ctx_t vctx); +void vmmnet_rele(vpc_ctx_t vctx); + +struct ifnet *vpc_if_lookup(uint32_t ifindex); +int vpc_async_copyout(struct vpc_copy_info *vci, const void *kaddr, void *uaddr, size_t len); +int vpcp_port_disconnect_ifp(struct ifnet *ifp); +void vpcp_set_pcpu_cache(if_ctx_t ctx, void *cache); +void *vpcp_get_pcpu_cache(if_ctx_t ctx); +void vpcp_get_id(struct ifnet *portifp, vpc_id_t *id); + +int vpcsw_transmit_ext(struct ifnet *ifp, struct mbuf *m, void *cache); +int vpcsw_port_connect(if_ctx_t switchctx, struct ifnet *portifp, struct ifnet *devifp); +int vpcsw_port_disconnect(if_ctx_t switchctx, struct ifnet *portifp); + + +int vmnic_ctl(vpc_ctx_t ctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata); + +int vpcsw_ctl(vpc_ctx_t ctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata); + +int vpcp_ctl(vpc_ctx_t ctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata); + +int vpcrtr_ctl(vpc_ctx_t ctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata); + +int vpcmux_ctl(vpc_ctx_t ctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata); + +int ethlink_ctl(vpc_ctx_t ctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata); + +int hostif_ctl(vpc_ctx_t vctx, vpc_op_t op, size_t inlen, const void *in, + size_t *outlen, void **outdata); + +struct ifnet *ethlink_ifp_get(if_ctx_t ctx); + +rtr_ctx_t vpc_rtr_ctx_alloc(void); +void vpc_rtr_ctx_free(rtr_ctx_t rc); + +int vpc_rtr_add(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen, int family); +int vpc_rtr_addv4(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen); +int vpc_rtr_addv6(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen); + +int vpc_rtr_del(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen, int family); +int vpc_rtr_delv4(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen); +int vpc_rtr_delv6(rtr_ctx_t rc, void *paddr, uint32_t value, uint8_t mlen); + +int vpc_rtr_lookup(rtr_ctx_t rc, void *key, uint32_t *val, int family); +int vpc_rtr_lookupv4(rtr_ctx_t rc, void *key, uint32_t *val); +int vpc_rtr_lookupv6(rtr_ctx_t rc, void *key, uint32_t *val); + +#endif + +enum vpc_vpcsw_op_type { + VPC_VPCSW_INVALID = 0, + VPC_VPCSW_PORT_ADD = 1, + VPC_VPCSW_PORT_DEL = 2, + VPC_VPCSW_PORT_UPLINK_SET = 3, + VPC_VPCSW_PORT_UPLINK_GET = 4, + VPC_VPCSW_STATE_GET = 5, + VPC_VPCSW_STATE_SET = 6, + VPC_VPCSW_RESET = 7, + VPC_VPCSW_RESPONSE = 8, + VPC_VPCSW_OP_TYPE_MAX = 8, +}; + +enum vpc_vpcp_op_type { + VPC_VPCP_INVALID = 0, + VPC_VPCP_CONNECT = 1, + VPC_VPCP_DISCONNECT = 2, + VPC_VPCP_VNI_GET = 3, + VPC_VPCP_VNI_SET = 4, + VPC_VPCP_VTAG_GET = 5, + VPC_VPCP_VTAG_SET = 6, + VPC_VPCP_UNUSED7 = 7, + VPC_VPCP_UNUSED8 = 8, + VPC_VPCP_CONNECTED_ID_GET = 9, + VPC_VPCP_MAX = 9, +}; + +enum vpc_vpcrtr_op_type { + VPC_VPCRTR_INVALID = 0, + // add/update + VPC_VPCRTR_ROUTE_SET = 1, + // delete + VPC_VPCRTR_ROUTE_DEL = 2, + // dump route table + VPC_VPCRTR_ROUTE_LIST = 3, + VPC_VPCRTR_MAX_OPS = 3 +}; + +enum vpc_vpcnat_op_type { + VPC_VPCNAT_INVALID = 0, + VPC_VPCNAT_MAX = 0, +}; + +enum vpc_vpcmux_op_type { + VPC_VPCMUX_INVALID = 0, + VPC_VPCMUX_LISTEN = 1, + VPC_VPCMUX_FTE_SET = 2, + VPC_VPCMUX_FTE_DEL = 3, + VPC_VPCMUX_FTE_LIST = 4, + VPC_VPCMUX_UNDERLAY_CONNECT = 5, + VPC_VPCMUX_UNDERLAY_DISCONNECT = 6, + VPC_VPCMUX_CONNECTED_ID_GET = 7, + VPC_VPCMUX_LISTEN_ADDR_GET = 8, + VPC_VPCMUX_MAX = 8, +}; + +enum vpc_vmnic_op_type { + VPC_VMNIC_INVALID = 0, + VPC_VMNIC_NQUEUES_GET = 1, + VPC_VMNIC_NQUEUES_SET = 2, + VPC_VMNIC_UNUSED3 = 3, + VPC_VMNIC_UNUSED4 = 4, + VPC_VMNIC_UNUSED5 = 5, + VPC_VMNIC_UNUSED6 = 6, + VPC_VMNIC_CONNECT = 7, + VPC_VMNIC_MSIX = 8, + VPC_VMNIC_FREEZE = 9, + VPC_VMNIC_UNFREEZE = 10, + VPC_VMNIC_OP_TYPE_MAX = 10, +}; + +enum vpc_meta_op_type { + VPC_META_INVALID = 0, + VPC_OBJ_DESTROY = 1, + VPC_OBJ_TYPE_GET = 2, + VPC_OBJ_COMMIT = 3, + VPC_OBJ_MAC_SET = 4, + VPC_OBJ_MAC_GET = 5, + VPC_OBJ_MTU_SET = 6, + VPC_OBJ_MTU_GET = 7, + VPC_OBJ_ID_GET = 8, + VPC_META_OP_TYPE_MAX = 8 +}; + +enum vpc_mgmt_op_type { + VPC_MGMT_INVALID = 0, + VPC_OBJ_TYPE_COUNT_GET = 1, + VPC_OBJ_HDR_GET_ALL = 2, + VPC_MGMT_OP_TYPE_MAX = 2 +}; + +enum vpc_ethlink_op_type { + VPC_ETHLINK_INVALID = 0, + VPC_ETHLINK_CONNECT = 1, + VPC_ETHLINK_CLONEATTACH = 2, + VPC_ETHLINK_DEVCTL = 3, + VPC_ETHLINK_DISCONNECT = 4, + VPC_ETHLINK_CONNECTED_NAME_GET = 5, + VPC_ETHLINK_VTAG_GET = 6, + VPC_ETHLINK_VTAG_SET = 7, + VPC_ETHLINK_MAX = 7, +}; + + +enum vpc_hostif_op_type { + VPC_HOSTIF_INVALID = 0, + VPC_HOSTIF_MAX = 0, +}; + + +#define IOC_MUT IOC_VOID +#define IOC_PRIV 0x10000000 +#define IOC_PRIVMUT (IOC_MUT|IOC_PRIV) + +#define VPC_OP(objtype, op) (((objtype) << 16)| (op)) +/* Modify state based on operation only -- userspace: VPC_OP_WR */ +#define VPC_OP_M(objtype, op) (IOC_MUT | ((objtype) << 16)| (op)) +#define VPC_OP_MP(objtype, op) (IOC_PRIVMUT | ((objtype) << 16)| (op)) +/* Read state based on operation only -- userspace: VPC_OP_RD */ +#define VPC_OP_O(objtype, op) (IOC_OUT | ((objtype) << 16)| (op)) +#define VPC_OP_OP(objtype, op) (IOC_OUT | IOC_PRIV | ((objtype) << 16)| (op)) +#define VPC_OP_OM(objtype, op) (IOC_OUT | IOC_MUT | ((objtype) << 16)| (op)) +#define VPC_OP_OMP(objtype, op) (IOC_OUT | IOC_PRIVMUT | ((objtype) << 16)| (op)) +#define VPC_OP_I(objtype, op) (IOC_IN | ((objtype) << 16)| (op)) +#define VPC_OP_IP(objtype, op) (IOC_IN | IOC_PRIV | ((objtype) << 16)| (op)) +/* Modify state based on operation and passed data -- userspace: VPC_OP_WR_KEY */ +#define VPC_OP_IM(objtype, op) (IOC_IN|IOC_MUT | ((objtype) << 16)| (op)) +#define VPC_OP_IMP(objtype, op) (IOC_IN|IOC_PRIVMUT | ((objtype) << 16)| (op)) +#define VPC_OP_IO(objtype, op) ((IOC_IN|IOC_OUT) | ((objtype) << 16)| (op)) +#define VPC_OP_IOP(objtype, op) ((IOC_IN|IOC_OUT|IOC_PRIV) | ((objtype) << 16)| (op)) +/* Modify state based on operation and passed data and read back results -- userspace: VPC_OP_RDWR_KEY_VAL */ +#define VPC_OP_IOM(objtype, op) ((IOC_IN|IOC_OUT|IOC_MUT) | ((objtype) << 16)| (op)) +#define VPC_OP_IOMP(objtype, op) ((IOC_IN|IOC_OUT|IOC_PRIVMUT) | ((objtype) << 16)| (op)) + +#define VPC_OBJ_TYPE(op) ((op & ~(IOC_OUT|IOC_IN|IOC_PRIVMUT)) >> 16) +#define VPC_OBJ_OP(op) ((op) & ((1<<16)-1)) + +#define VPC_OBJ_OP_TYPE_COUNT_GET VPC_OP_IO(VPC_OBJ_MGMT, VPC_OBJ_TYPE_COUNT_GET) +#define VPC_OBJ_OP_HDR_GET_ALL VPC_OP_IO(VPC_OBJ_MGMT, VPC_OBJ_HDR_GET_ALL) + +#define VPC_OBJ_OP_DESTROY VPC_OP_MP(VPC_OBJ_META, VPC_OBJ_DESTROY) +#define VPC_OBJ_OP_TYPE_GET VPC_OP_O(VPC_OBJ_META, VPC_OBJ_TYPE_GET) +#define VPC_OBJ_OP_COMMIT VPC_OP_MP(VPC_OBJ_META, VPC_OBJ_COMMIT) +#define VPC_OBJ_OP_MAC_SET VPC_OP_IMP(VPC_OBJ_META, VPC_OBJ_MAC_SET) +#define VPC_OBJ_OP_MAC_GET VPC_OP_O(VPC_OBJ_META, VPC_OBJ_MAC_GET) +#define VPC_OBJ_OP_MTU_SET VPC_OP_IMP(VPC_OBJ_META, VPC_OBJ_MTU_SET) +#define VPC_OBJ_OP_MTU_GET VPC_OP_O(VPC_OBJ_META, VPC_OBJ_MTU_GET) +#define VPC_OBJ_OP_ID_GET VPC_OP_O(VPC_OBJ_META, VPC_OBJ_ID_GET) + +#define VPC_VPCSW_OP_PORT_ADD VPC_OP_IMP(VPC_OBJ_SWITCH, VPC_VPCSW_PORT_ADD) +#define VPC_VPCSW_OP_PORT_DEL VPC_OP_IMP(VPC_OBJ_SWITCH, VPC_VPCSW_PORT_DEL) +#define VPC_VPCSW_OP_PORT_UPLINK_GET VPC_OP_O(VPC_OBJ_SWITCH, VPC_VPCSW_PORT_UPLINK_GET) +#define VPC_VPCSW_OP_PORT_UPLINK_SET VPC_OP_IMP(VPC_OBJ_SWITCH, VPC_VPCSW_PORT_UPLINK_SET) +#define VPC_VPCSW_OP_STATE_GET VPC_OP_O(VPC_OBJ_SWITCH, VPC_VPCSW_STATE_GET) +#define VPC_VPCSW_OP_STATE_SET VPC_OP_IMP(VPC_OBJ_SWITCH, VPC_VPCSW_STATE_SET) +#define VPC_VPCSW_OP_RESET VPC_OP_MP(VPC_OBJ_SWITCH, VPC_VPCSW_RESET) +#define VPC_VPCSW_OP_RESPONSE VPC_OP_IMP(VPC_OBJ_SWITCH, VPC_VPCSW_RESPONSE) + +#define VPC_VMNIC_OP_NQUEUES_GET VPC_OP_O(VPC_OBJ_VMNIC, VPC_VMNIC_NQUEUES_GET) +#define VPC_VMNIC_OP_NQUEUES_SET VPC_OP_IMP(VPC_OBJ_VMNIC, VPC_VMNIC_NQUEUES_SET) + + +#define VPC_VMNIC_OP_CONNECT VPC_OP_IMP(VPC_OBJ_VMNIC, VPC_VMNIC_CONNECT) +#define VPC_VMNIC_OP_MSIX VPC_OP_IMP(VPC_OBJ_VMNIC, VPC_VMNIC_MSIX) +#define VPC_VMNIC_OP_FREEZE VPC_OP_MP(VPC_OBJ_VMNIC, VPC_VMNIC_FREEZE) +#define VPC_VMNIC_OP_UNFREEZE VPC_OP_MP(VPC_OBJ_VMNIC, VPC_VMNIC_UNFREEZE) + +#define VPC_VPCP_OP_CONNECT VPC_OP_IMP(VPC_OBJ_PORT, VPC_VPCP_CONNECT) +#define VPC_VPCP_OP_DISCONNECT VPC_OP_MP(VPC_OBJ_PORT, VPC_VPCP_DISCONNECT) +#define VPC_VPCP_OP_VNI_GET VPC_OP_O(VPC_OBJ_PORT, VPC_VPCP_VNI_GET) +#define VPC_VPCP_OP_VNI_SET VPC_OP_IMP(VPC_OBJ_PORT, VPC_VPCP_VNI_SET) +#define VPC_VPCP_OP_VTAG_GET VPC_OP_O(VPC_OBJ_PORT, VPC_VPCP_VTAG_GET) +#define VPC_VPCP_OP_VTAG_SET VPC_OP_IMP(VPC_OBJ_PORT, VPC_VPCP_VTAG_SET) +#define VPC_VPCP_OP_CONNECTED_ID_GET VPC_OP_O(VPC_OBJ_PORT, VPC_VPCP_CONNECTED_ID_GET) + +#define VPC_ETHLINK_OP_CONNECT VPC_OP_IMP(VPC_OBJ_ETHLINK, VPC_ETHLINK_CONNECT) +#define VPC_ETHLINK_OP_DISCONNECT VPC_OP_MP(VPC_OBJ_ETHLINK, VPC_ETHLINK_DISCONNECT) +#define VPC_ETHLINK_OP_CONNECTED_NAME_GET VPC_OP_O(VPC_OBJ_ETHLINK, VPC_ETHLINK_CONNECTED_NAME_GET) +#define VPC_ETHLINK_OP_VTAG_GET VPC_OP_O(VPC_OBJ_ETHLINK, VPC_ETHLINK_VTAG_GET) +#define VPC_ETHLINK_OP_VTAG_SET VPC_OP_IMP(VPC_OBJ_ETHLINK, VPC_ETHLINK_VTAG_SET) + + +#define VPC_VPCMUX_OP_LISTEN VPC_OP_IMP(VPC_OBJ_VPCMUX, VPC_VPCMUX_LISTEN) +#define VPC_VPCMUX_OP_FTE_SET VPC_OP_IMP(VPC_OBJ_VPCMUX, VPC_VPCMUX_FTE_SET) +#define VPC_VPCMUX_OP_FTE_DEL VPC_OP_IMP(VPC_OBJ_VPCMUX, VPC_VPCMUX_FTE_DEL) +#define VPC_VPCMUX_OP_UNDERLAY_CONNECT VPC_OP_IMP(VPC_OBJ_VPCMUX, VPC_VPCMUX_UNDERLAY_CONNECT) +#define VPC_VPCMUX_OP_UNDERLAY_DISCONNECT VPC_OP_MP(VPC_OBJ_VPCMUX, VPC_VPCMUX_UNDERLAY_DISCONNECT) +#define VPC_VPCMUX_OP_CONNECTED_ID_GET VPC_OP_O(VPC_OBJ_VPCMUX, VPC_VPCMUX_CONNECTED_ID_GET) +#define VPC_VPCMUX_OP_LISTEN_ADDR_GET VPC_OP_O(VPC_OBJ_VPCMUX, VPC_VPCMUX_LISTEN_ADDR_GET) +#define VPC_VPCMUX_OP_FTE_LIST VPC_OP_O(VPC_OBJ_VPCMUX, VPC_VPCMUX_FTE_LIST) + + +#define VPC_F_CREATE (1ULL << 0) +#define VPC_F_OPEN (1ULL << 1) +#define VPC_F_READ (1ULL << 2) +#define VPC_F_WRITE (1ULL << 3) + +#define VPCSW_SIZE_IDX 0 +#define VPCSW_ADDR_IDX 1 + +#endif Index: usr.sbin/bhyve/pci_kvirtio_net.c =================================================================== --- /dev/null +++ usr.sbin/bhyve/pci_kvirtio_net.c @@ -0,0 +1,316 @@ +/* + * Copyright (C) 2017 Joyent Inc. + * Copyright (C) 2017 Matthew Macy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Proxy for the kernel vmnic pseudo device + * + * Configuration is + * + * -s ,kvirtio-net,id= + * + * e.g. -s 2:3:0,kvirtio-net,id=01234567-89ab-cdef-0123-456789abcdef + * + * Where uuid is a value associated with host side network object from a + * previous vpc_open. The attach will fail if the host side network object + * has not been marked as 'immutable' indicating that the required configuration + * parameters have been set. + * + * NB: + * - MSI-x is always used + * - The BAR will not be modified during run-time of the VM (not the + * case with UEFI) + * - currently single-queue only + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#ifndef WITHOUT_CAPSICUM +#include +#endif +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bhyverun.h" +#include "pci_emul.h" +#include "virtio.h" + +#include +#include +#include + +#ifndef max +#define max(a,b) (((a)>(b))?(a):(b)) +#endif + +#ifndef min +#define min(a,b) (((a)<(b))?(a):(b)) +#endif + + +static int +vpc_open(const vpc_id_t *vpc_id, vpc_type_t obj_type, vpc_flags_t flags) +{ + return syscall(SYS_vpc_open, vpc_id, obj_type, flags); +} + +static int +vpc_ctl(int vpcd, vpc_op_t op, size_t keylen, const void *key, size_t *vallen, void *buf) +{ + return syscall(SYS_vpc_ctl, vpcd, op, keylen, key, vallen, buf); +} + +#define VTNET_BE_REGSZ (20 + 4 + 8) /* virtio + MSI-x + config */ + +struct vtnet_be_softc { + struct pci_devinst *vbs_pi; + int vbs_fd; /* vpc descriptor for config */ + uint16_t vbs_nqs; + uint16_t vbs_nvqs; + vpc_id_t vbs_id; +}; + +static void +vtnet_be_msix(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int on) +{ + struct vtnet_be_softc *vbs; + struct vb_msix *vmsix; + int i, err, size, nvqs; + + vbs = pi->pi_arg; + nvqs = vbs->vbs_nvqs; + size = sizeof(*vmsix) + nvqs*sizeof(struct vb_msix_vector); + + vmsix = malloc(size); + vmsix->vm_status = on; + vmsix->vm_count = nvqs; + if (on) { + for (i = 0; i < nvqs; i++) { + vmsix->vm_q[i].msg = pi->pi_msix.table[i].msg_data; + vmsix->vm_q[i].addr = pi->pi_msix.table[i].addr; + } + } + err = vpc_ctl(vbs->vbs_fd, VPC_VMNIC_OP_MSIX, size, vmsix, NULL, NULL); + free(vmsix); + assert(err == 0); +} + +static uint64_t +vtnet_be_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, + int baridx, uint64_t offset, int size) +{ + + assert(baridx == 1); + return (pci_emul_msix_tread(pi, offset, size)); +} + +void +vtnet_be_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, + int baridx, uint64_t offset, int size, uint64_t value) +{ + + assert(baridx == 1); + pci_emul_msix_twrite(pi, offset, size, value); + vtnet_be_msix(ctx, vcpu, pi, 1); +} + +struct token_value { + char *token; + int type; +}; + +#define KW_ID 0x1 +#define KW_MAX KW_ID + +struct token_value token_map[] = { + {"id", KW_ID}, +}; + +static int +strtype(char *token) { + int i; + + for (i = 0; i < KW_MAX; i++) { + if (strstr(token, token_map[i].token) != NULL) + return (token_map[i].type); + } + return (-1); +} + +static char * +tokenval(char *token) +{ + + strsep(&token, "="); + return (token); +} + +static int +vtnet_be_parseopts(struct vtnet_be_softc *vbs, char *opts) +{ + char *mac, *input, *token; + int id, s; + uint32_t status; + struct uuid uuidtmp; + size_t osize; + uint16_t nqs; + uint64_t type; + vpc_handle_type_t *typep; + + if (opts == NULL) + return (1); + + mac = NULL; + input = strdup(opts); + status = uuid_s_bad_version; + while ((token = strsep(&input, ",")) != NULL) { + id = strtype(token); + switch(id) { + case KW_ID: + uuid_from_string(tokenval(token), &uuidtmp, &status); + if (status != uuid_s_ok) + return (1); + uuid_enc_be(&vbs->vbs_id, &uuidtmp); + break; + default: + printf("bad value to kvirtio %s", token); + return (1); + } + } + type = 0; + typep = (void *)&type; + typep->vht_version = 1; + typep->vht_obj_type = VPC_OBJ_VMNIC; + type = htobe64(type); + if ((s = vpc_open(&vbs->vbs_id, type, VPC_F_WRITE|VPC_F_OPEN)) < 0) + return (1); + vbs->vbs_fd = s; + osize = sizeof(nqs); + if (vpc_ctl(s, VPC_VMNIC_OP_NQUEUES_GET, 0, NULL, &osize, &nqs)) + return (1); + vbs->vbs_nqs = nqs; + vbs->vbs_nvqs = 2*nqs + 1; + + return (status != uuid_s_ok); +} + +static int +vtnet_be_clone(struct vtnet_be_softc *vbs) +{ + int s; + struct vb_vm_attach va; +#ifndef WITHOUT_CAPSICUM + cap_rights_t rights; +#endif + + s = vbs->vbs_fd; + bzero(&va, sizeof(va)); + va.vva_ioh.vih_magic = VB_MAGIC; + va.vva_io_start = vbs->vbs_pi->pi_bar[0].addr; + va.vva_io_size = VTNET_BE_REGSZ; + strncpy(va.vva_vmparent, vmname, VMNAMSIZ-1); + if (vpc_ctl(s, VPC_VMNIC_OP_CONNECT, sizeof(va), &va, NULL, NULL)) + return (errno); + +#ifndef WITHOUT_CAPSICUM + cap_rights_init(&rights, CAP_VPC_READ|CAP_VPC_PRIVWRITE); + if (cap_rights_limit(s, &rights) == -1 && errno != ENOSYS) + errx(EX_OSERR, "Unable to apply rights for sandbox"); +#endif + return (0); +} + +static int +vtnet_be_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) +{ + struct vtnet_be_softc *vbs; + int memflags; + + memflags = vm_get_memflags(ctx); + if (!(memflags & VM_MEM_F_WIRED)) { + warnx("kvirtio-net requires guest memory to be wired"); + return (1); + } + vbs = calloc(1, sizeof(struct vtnet_be_softc)); + + pi->pi_arg = vbs; + vbs->vbs_pi = pi; + + if (vtnet_be_parseopts(vbs, opts)) { + fprintf(stderr, "kvirtio-net: parse issue with \"%s\"\n", + opts ? opts : "(null)"); + return (1); + } + + pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_NET); + pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR); + pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_NETWORK); + pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_NET); + + pci_emul_alloc_bar(pi, 0, PCIBAR_IO, VTNET_BE_REGSZ); + + /* + * MSI-x BAR at 1. + * 3 IRQs - event queue (not implemented), rx, and tx + */ + if (pci_emul_add_msixcap(pi, vbs->vbs_nvqs + 1, 1)) + return (1); + + /* Attempt to open the char dev for this device */ + if (vtnet_be_clone(vbs)) + return (1); + + return (0); +} + +struct pci_devemu pci_de_vtnet_be = { + .pe_emu = "kvirtio-net", + .pe_init = vtnet_be_init, + .pe_barread = vtnet_be_read, + .pe_barwrite = vtnet_be_write, + .pe_msix = vtnet_be_msix, +}; +PCI_EMUL_SET(pci_de_vtnet_be);