diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59845efb..b107c0a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,17 +7,19 @@ permissions: jobs: package-build: - runs-on: ubuntu-latest - + strategy: + fail-fast: false + matrix: + distro: [focal, noble] steps: - uses: actions/checkout@v6 - - name: Run package build focal - run: script/cibuild-create-packages-focal + - name: Run package build ${{ matrix.distro }} + run: script/cibuild-create-packages ${{ matrix.distro }} - name: Tar files - run: tar -cvf glb-director.tar $GITHUB_WORKSPACE/tmp/build + run: tar -cvf glb-director-${{ matrix.distro }}.tar $GITHUB_WORKSPACE/tmp/build - name: Upload Artifact uses: actions/upload-artifact@v7 with: - name: glb-director - path: glb-director.tar \ No newline at end of file + name: glb-director-${{ matrix.distro }} + path: glb-director-${{ matrix.distro }}.tar \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..74215f71 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,72 @@ +name: Tests + +on: + push: + branches: + - main + - master + pull_request: + branches: + - main + - master + +permissions: + contents: read + +jobs: + build-images: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + distro: [noble, focal] + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build image + run: | + docker build --file script/Dockerfile.${{ matrix.distro }} --tag glb-director-build-${{ matrix.distro }}:latest . + docker save glb-director-build-${{ matrix.distro }}:latest --output glb-director-build-${{ matrix.distro }}.tar + + - name: Upload image artifact + uses: actions/upload-artifact@v4 + with: + name: build-${{ matrix.distro }} + path: glb-director-build-${{ matrix.distro }}.tar + retention-days: 1 + + + + test: + needs: build-images + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test-suite: [director, director-xdp, healthcheck, redirect] + distro: [noble, focal] + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Download image artifact + uses: actions/download-artifact@v4 + with: + name: build-${{ matrix.distro }} + + - name: Load Docker image + run: | + docker load --input glb-director-build-${{ matrix.distro }}.tar + + - name: Run test suite in container + run: | + docker run --rm \ + --privileged \ + --volume $(pwd):/workspace \ + --workdir /workspace \ + glb-director-build-${{ matrix.distro }}:latest \ + bash -c "cd /workspace/src/glb-${{ matrix.test-suite }} && script/test" \ No newline at end of file diff --git a/script/Dockerfile.focal b/script/Dockerfile.focal index 2208dbb5..c7e44760 100644 --- a/script/Dockerfile.focal +++ b/script/Dockerfile.focal @@ -1,35 +1,40 @@ -FROM ubuntu:focal +FROM --platform=linux/amd64 ubuntu:focal RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries -RUN apt-get update && apt-get -y install curl git - -# DPDK -RUN echo "deb http://archive.ubuntu.com/ubuntu/ bionic main universe" >> /etc/apt/sources.list ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y build-essential dpdk=17.11.1-6 dpdk-dev=17.11.1-6 libdpdk-dev=17.11.1-6 wget pkg-config libjansson-dev libsystemd-dev + +RUN apt-get update && apt-get -y install curl git software-properties-common +RUN add-apt-repository universe + +# DPDK (use 17.11 from bionic for API compatibility) +RUN echo "deb [trusted=yes arch=amd64] http://dk.archive.ubuntu.com/ubuntu/ bionic main universe" >> /etc/apt/sources.list +RUN apt-get update && apt-get install -y build-essential dpdk=17.11.1-6 dpdk-dev=17.11.1-6 libdpdk-dev=17.11.1-6 wget pkg-config libjansson-dev libsystemd-dev clang-tools-10 # iptables / DKMS -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y -f wget pkg-config libsystemd-dev dkms debhelper libxtables-dev +RUN apt-get install -y -f dkms debhelper libxtables-dev # golang -RUN wget --quiet https://golang.org/dl/go1.24.5.linux-amd64.tar.gz -O- | tar -C /usr/local -zxvf - +RUN ARCH=$(dpkg --print-architecture) && wget --quiet https://golang.org/dl/go1.24.5.linux-${ARCH}.tar.gz -O- | tar -C /usr/local -zxvf - ENV GOROOT /usr/local/go ENV GOPATH /go ENV PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" # fpm for packaging -RUN apt-get update && apt-get install -y ruby ruby-dev rubygems build-essential +RUN apt-get update && apt-get install -y ruby ruby-dev rubygems # See fpm dependency breakage issue: https://github.com/jordansissel/fpm/issues/1918 RUN gem install --version 2.7.6 dotenv RUN gem install ffi -f RUN gem install rake fpm +# Python / nosetests for scapy tests +RUN apt-get update && apt-get install -y python3 python3-pip python3-nose python3-scapy netcat-openbsd jq +RUN pip3 install siphash 'pyroute2<0.7' netaddr +RUN ln -sf /usr/bin/python3 /usr/bin/python && ln -sf $(which nosetests3) /usr/local/bin/nosetests + # XDP # linux-libc-dev must be upgraded to get a bpf.h that matches what we use. the rest match what we do in Vagrant for testing. -RUN apt-get update && apt install -y apt-transport-https curl software-properties-common RUN apt-get update && apt install -y iproute2 libbpf-dev linux-libc-dev clang-10 # Hack because the kernel headers are not installed in the right place (linuxkit vs generic) diff --git a/script/Dockerfile.noble b/script/Dockerfile.noble new file mode 100644 index 00000000..8854c446 --- /dev/null +++ b/script/Dockerfile.noble @@ -0,0 +1,67 @@ +FROM --platform=linux/amd64 ubuntu:noble + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get -y install curl git software-properties-common +RUN add-apt-repository universe + +# DPDK +RUN apt-get update +RUN apt-get install --assume-yes \ + build-essential \ + dpdk=23.11.4-0ubuntu0.24.04.2 \ + dpdk-dev=23.11.4-0ubuntu0.24.04.2 \ + libdpdk-dev=23.11.4-0ubuntu0.24.04.2 \ + wget \ + pkg-config \ + libjansson-dev \ + libsystemd-dev \ + clang-tools-18 + +# iptables / DKMS +RUN apt-get update +RUN apt-get install --assume-yes --fix-broken \ + wget \ + pkg-config \ + libsystemd-dev \ + dkms \ + dh-dkms \ + dpkg-dev \ + fakeroot \ + debhelper \ + libxtables-dev + +# golang +RUN ARCH=$(dpkg --print-architecture) && wget --quiet https://golang.org/dl/go1.24.5.linux-${ARCH}.tar.gz -O- | tar -C /usr/local -zxvf - +ENV GOROOT /usr/local/go +ENV GOPATH /go +ENV PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" + +# fpm for packaging +RUN apt-get update && apt-get install -y ruby ruby-dev rubygems build-essential + +# See fpm dependency breakage issue: https://github.com/jordansissel/fpm/issues/1918 +RUN gem install --version 2.7.6 dotenv +RUN gem install ffi -f +RUN gem install rake fpm + +# Python / nosetests for scapy tests +RUN apt-get update && apt-get install -y python3 python3-pip python3-nose python3-scapy netcat-openbsd +RUN pip3 install --break-system-packages siphash pyroute2 netaddr +RUN ln -sf /usr/bin/python3 /usr/bin/python && ln -sf $(which nosetests3) /usr/local/bin/nosetests + +# XDP +# linux-libc-dev must be upgraded to get a bpf.h that matches what we use. the rest match what we do in Vagrant for testing. +RUN apt-get update && apt install -y apt-transport-https curl software-properties-common +RUN apt-get update && apt install -y iproute2 libbpf-dev linux-libc-dev clang-20 + +# libbpf.so.0 compat symlink (noble ships libbpf.so.1 but xdp-root-shim links against .0) +# RUN ln -sf /usr/lib/x86_64-linux-gnu/libbpf.so.1 /usr/lib/x86_64-linux-gnu/libbpf.so.0 + +# Hack because the kernel headers are not installed in the right place (linuxkit vs generic) +RUN ln -s /usr/src/$(ls /usr/src/ | grep generic) /usr/src/linux-headers-$(uname -r) + +# Hack for C99 math +RUN sed -i '1s/^/#define __USE_C99_MATH\n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h +RUN sed -i '2s/^/#include \n/' /usr/src/$(ls /usr/src/ | grep generic)/include/linux/kasan-checks.h + diff --git a/script/cibuild-create-packages b/script/cibuild-create-packages index 8cf9fa34..e0f3dfd0 100755 --- a/script/cibuild-create-packages +++ b/script/cibuild-create-packages @@ -7,9 +7,14 @@ cd "$(dirname "$0")/.." . script/helpers/folding.sh +DISTRO="$1" +if [ -z "$DISTRO" ]; then + DISTRO="focal" +fi + begin_fold "Preparing Docker build environment" ( - docker build -t glb-director-build-stretch -f script/Dockerfile.stretch script + docker build -t glb-director-build-$DISTRO -f "script/Dockerfile.$DISTRO" script ) end_fold @@ -21,8 +26,8 @@ begin_fold "Building packages" docker run --rm \ --volume "$HOSTPATH":/glb-director \ - "glb-director-build-stretch" \ + "glb-director-build-$DISTRO" \ bash -c "cd /glb-director && make BUILDDIR=/glb-director/tmp/build clean mkdeb" ) -end_fold +end_fold \ No newline at end of file diff --git a/script/cibuild-create-packages-focal b/script/cibuild-create-packages-focal deleted file mode 100755 index adc26467..00000000 --- a/script/cibuild-create-packages-focal +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -e - -HOSTPATH=$(cd $(dirname "$0") && cd .. && pwd) -cd "$(dirname "$0")/.." - -. script/helpers/folding.sh - -begin_fold "Preparing Docker build environment" -( - docker build -t glb-director-build-focal -f script/Dockerfile.focal script -) -end_fold - -begin_fold "Building packages" -( - # prep - rm -rf tmp/build/ - mkdir -p tmp/build/ - - docker run --rm \ - --volume "$HOSTPATH":/glb-director \ - "glb-director-build-focal" \ - bash -c "cd /glb-director && - make BUILDDIR=/glb-director/tmp/build clean mkdeb" -) -end_fold diff --git a/script/helpers/folding.sh b/script/helpers/folding.sh index 0c387304..774bf222 100644 --- a/script/helpers/folding.sh +++ b/script/helpers/folding.sh @@ -1,9 +1,9 @@ #!/bin/bash begin_fold() { - echo "%%%FOLD {$*}%%%" + echo "::group::$*" } end_fold() { - echo "%%%END FOLD%%%" -} + echo "::endgroup::" +} \ No newline at end of file diff --git a/script/test b/script/test new file mode 100644 index 00000000..020dcb99 --- /dev/null +++ b/script/test @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Runs a Clang static analysis build (scan-build) over the project to detect +# potential bugs at compile time. Automatically detects the available version +# of scan-build installed in the environment. +set -euo pipefail + +if command -v scan-build-10 >/dev/null 2>&1; then + SCAN_BUILD=scan-build-10 +elif command -v scan-build >/dev/null 2>&1; then + SCAN_BUILD=scan-build +elif command -v scan-build-14 >/dev/null 2>&1; then + SCAN_BUILD=scan-build-14 +else + echo "scan-build is not installed" >&2 + exit 1 +fi + +"$SCAN_BUILD" make diff --git a/script/test-local b/script/test-local new file mode 100755 index 00000000..76fb2ec3 --- /dev/null +++ b/script/test-local @@ -0,0 +1,43 @@ +#!/bin/bash + +set -e + +DISTRO="${1}" + +if [[ -z "$DISTRO" ]]; then + echo "Usage: $0 " + echo " e.g. $0 focal" + echo " e.g. $0 noble" + exit 1 +fi + +HOSTPATH=$(cd "$(dirname "$0")/.." && pwd) +IMAGE="glb-director-build-${DISTRO}:latest" +DOCKERFILE="script/Dockerfile.${DISTRO}" + +if [[ ! -f "${HOSTPATH}/${DOCKERFILE}" ]]; then + echo "ERROR: Dockerfile not found: ${DOCKERFILE}" + exit 1 +fi + +cd "$HOSTPATH" + +echo "==> Building Docker image for ${DISTRO}..." +docker build --platform linux/amd64 --file "${DOCKERFILE}" --tag "${IMAGE}" . + +TEST_SUITES=(director director-xdp healthcheck redirect) + +for suite in "${TEST_SUITES[@]}"; do + echo "" + echo "==> Running test suite: glb-${suite} (${DISTRO})" + docker run --rm \ + --platform linux/amd64 \ + --privileged \ + --volume "$(pwd):/workspace" \ + --workdir /workspace \ + "${IMAGE}" \ + bash -c "cd /workspace/src/glb-${suite} && script/test" +done + +echo "" +echo "==> All test suites passed for ${DISTRO}." diff --git a/src/glb-director-xdp/Makefile b/src/glb-director-xdp/Makefile index 46d72c37..f2f6e08d 100644 --- a/src/glb-director-xdp/Makefile +++ b/src/glb-director-xdp/Makefile @@ -1,8 +1,8 @@ all: make -C xdp-root-shim/ make -C bpf/ - go build -buildvcs=false -o glb-director-xdp main.go + go build -buildvcs=false -o glb-director-xdp . clean: make -C bpf/ clean - rm -rf glb-director-xdp \ No newline at end of file + rm -rf glb-director-xdp diff --git a/src/glb-director-xdp/bpf/Makefile b/src/glb-director-xdp/bpf/Makefile index fa114682..712f9981 100644 --- a/src/glb-director-xdp/bpf/Makefile +++ b/src/glb-director-xdp/bpf/Makefile @@ -1,7 +1,7 @@ all: glb_encap.o glb_encap_trace.o passer.o tailcall.o -CLANG=clang-10 -LLC=llc-10 +CLANG ?= $(shell command -v clang-10 2>/dev/null || command -v clang-20 2>/dev/null || command -v clang-18 2>/dev/null || command -v clang 2>/dev/null) +LLC ?= $(shell command -v llc-10 2>/dev/null || command -v llc-20 2>/dev/null || command -v llc-18 2>/dev/null || command -v llc 2>/dev/null) ifeq ($(KVER),) KVER=$(shell uname -r) @@ -16,7 +16,10 @@ endif -Wno-unknown-warning-option \ -Wno-pointer-sign \ -Werror \ + -include include/compat/types.h \ -include /usr/src/linux-headers-$(KVER:-amd64=-common)/include/linux/kconfig.h \ + -I /usr/src/linux-headers-$(KVER:-amd64=-common)/arch/x86/include/generated \ + -I /usr/src/linux-headers-$(KVER:-amd64=-common)/arch/x86/include/generated/uapi \ -I /usr/src/linux-headers-$(KVER:-amd64=-common)/arch/x86/include/uapi \ -I /usr/src/linux-headers-$(KVER:-amd64=-common)/arch/x86/include/uapi \ -I /usr/src/linux-headers-$(KVER:-amd64=-common)/include/uapi \ diff --git a/src/glb-director-xdp/bpf/bpf_helpers.h b/src/glb-director-xdp/bpf/bpf_helpers.h index 864b40d2..163d9476 100644 --- a/src/glb-director-xdp/bpf/bpf_helpers.h +++ b/src/glb-director-xdp/bpf/bpf_helpers.h @@ -6,6 +6,7 @@ #define __uint(name, val) int (*name)[val] #define __type(name, val) typeof(val) *name +#define __array(name, val) typeof(val) *name[] /* Helper macro to print out debug messages */ #define bpf_printk(fmt, ...) \ diff --git a/src/glb-director-xdp/bpf/include/compat/types.h b/src/glb-director-xdp/bpf/include/compat/types.h new file mode 100644 index 00000000..76859a78 --- /dev/null +++ b/src/glb-director-xdp/bpf/include/compat/types.h @@ -0,0 +1,7 @@ +#ifndef _COMPAT_TYPES_H +#define _COMPAT_TYPES_H + +/* Provide size_t for kernel headers that reference it (e.g. kcsan-checks.h) */ +typedef unsigned long size_t; + +#endif /* _COMPAT_TYPES_H */ diff --git a/src/glb-director-xdp/bpf/tailcall.c b/src/glb-director-xdp/bpf/tailcall.c index 92b330da..e4246be3 100644 --- a/src/glb-director-xdp/bpf/tailcall.c +++ b/src/glb-director-xdp/bpf/tailcall.c @@ -19,14 +19,14 @@ #define ROOT_ARRAY_SIZE 3 -struct bpf_map_def SEC("maps") root_array = { - .type = BPF_MAP_TYPE_PROG_ARRAY, - .key_size = sizeof(__u32), - .value_size = sizeof(__u32), - .max_entries = ROOT_ARRAY_SIZE, -}; +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); + __uint(max_entries, ROOT_ARRAY_SIZE); +} root_array SEC(".maps"); -SEC("xdp-root") +SEC("xdp") int xdp_root(struct xdp_md *ctx) { #pragma clang loop unroll(full) for (__u32 i = 0; i < ROOT_ARRAY_SIZE; i++) { diff --git a/src/glb-director-xdp/debug.c b/src/glb-director-xdp/debug.c new file mode 100644 index 00000000..f3994065 --- /dev/null +++ b/src/glb-director-xdp/debug.c @@ -0,0 +1,3 @@ +#include + +bool debug = false; diff --git a/src/glb-director-xdp/script/test b/src/glb-director-xdp/script/test index f1107bfc..2e945eec 100755 --- a/src/glb-director-xdp/script/test +++ b/src/glb-director-xdp/script/test @@ -40,9 +40,28 @@ export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin . ../../script/helpers/folding.sh +# Ensure BPF filesystem is mounted (needed for XDP map pinning) +if ! mount | grep -q 'type bpf'; then + mount -t bpf bpf /sys/fs/bpf || true +fi + +# If kernel headers for the running kernel aren't available, find what we have +if [ ! -d "/usr/src/linux-headers-$(uname -r)" ]; then + AVAILABLE_HEADERS=$(ls -d /usr/src/linux-headers-*-generic 2>/dev/null | head -1 || true) + if [ -n "$AVAILABLE_HEADERS" ]; then + # Extract the version from the available headers path + AVAILABLE_KVER=$(basename "$AVAILABLE_HEADERS") + AVAILABLE_KVER="${AVAILABLE_KVER#linux-headers-}" + export KVER="$AVAILABLE_KVER" + echo "Using kernel headers version: $KVER" + fi +fi + begin_fold "Building glb-director-xdp for testing" ( make -j4 -C ../glb-director/cli + make clean + rm -f xdp-root-shim/xdp-root-shim make # scan-build bricks go ) end_fold diff --git a/src/glb-director/Makefile b/src/glb-director/Makefile index ae516f31..1a2f5fec 100644 --- a/src/glb-director/Makefile +++ b/src/glb-director/Makefile @@ -33,6 +33,32 @@ RTE_SDK ?= /usr/share/dpdk RTE_TARGET ?= x86_64-default-linuxapp-gcc +ifeq ($(wildcard $(RTE_SDK)/mk/rte.vars.mk),) +# Modern DPDK (no legacy make fragments): use pkg-config +APP = glb-director + +SRCS-y := main.c bind_classifier.c glb_kni.c glb_fwd_config.c \ +glb_encap.c glb_encap_dpdk.c glb_control_loop.c glb_processor_loop.c \ +siphash24.c glb_director_config.c statsd-client.c shared_opt.c + +CFLAGS += $(shell pkg-config --cflags libdpdk) +CFLAGS += -O3 -g -pie -fPIE -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1 -fstack-protector-strong +CFLAGS += -DSTATSD -DSYSTEMD +CFLAGS += -I$(CURDIR)/.. +LDFLAGS += $(shell pkg-config --libs libdpdk) +LDFLAGS += -z relro -z now -ljansson -lsystemd + +build/app/$(APP): $(SRCS-y) + mkdir -p build/app + $(CC) $(CFLAGS) -o $@ $(SRCS-y) $(LDFLAGS) + +all: build/app/$(APP) + +clean: + rm -rf build + +.PHONY: all clean +else include $(RTE_SDK)/mk/rte.vars.mk # binary name @@ -61,3 +87,4 @@ LDFLAGS += -lsystemd CONFIG_RTE_LIBRTE_PMD_XENVIRT = n include $(RTE_SDK)/mk/rte.extapp.mk +endif diff --git a/src/glb-director/bind_classifier.c b/src/glb-director/bind_classifier.c index d29203f9..45a41f47 100644 --- a/src/glb-director/bind_classifier.c +++ b/src/glb-director/bind_classifier.c @@ -76,8 +76,8 @@ #include #include "bind_classifier.h" -#include "bind_classifier_rules.h" #include "config.h" +#include "bind_classifier_rules.h" #include "glb_fwd_config.h" #include "glb_director_config.h" #include "log.h" diff --git a/src/glb-director/cli/Makefile b/src/glb-director/cli/Makefile index 8d5f6e6d..92378388 100644 --- a/src/glb-director/cli/Makefile +++ b/src/glb-director/cli/Makefile @@ -42,7 +42,6 @@ PCAP_SRCS = pcap_mode.c \ ../glb_fwd_config.c \ ../glb_director_config.c \ ../glb_encap.c \ - ../cmdline_parse.c \ ../cmdline_parse_etheraddr.c \ ../glb_encap_pcap.c \ ../siphash24.c \ @@ -52,7 +51,6 @@ STUB_SRCS = stub_server.c \ ../glb_fwd_config.c \ ../glb_director_config.c \ ../glb_encap.c \ - ../cmdline_parse.c \ ../cmdline_parse_etheraddr.c \ ../glb_encap_pcap.c \ ../siphash24.c \ @@ -71,6 +69,9 @@ CFLAGS += -DCLI_MODE LDFLAGS += -z relro -z now LDFLAGS += -ljansson +DPDK_CFLAGS := $(shell pkg-config --cflags libdpdk) +DPDK_LIBS := $(shell pkg-config --libs libdpdk) + glb-director-cli: main.c gcc \ $(CFLAGS) \ @@ -87,9 +88,8 @@ glb-config-check: -o glb-config-check \ -I`pwd`/.. \ -I`pwd`/../.. \ - -I/usr/include/dpdk \ - -I/usr/include/x86_64-linux-gnu \ - -ldpdk -lpcap $(LDFLAGS)\ + $(DPDK_CFLAGS) \ + -lpcap $(LDFLAGS) $(DPDK_LIBS) \ -m64 -mssse3 glb-director-pcap: @@ -99,10 +99,9 @@ glb-director-pcap: -o glb-director-pcap \ -I`pwd`/.. \ -I`pwd`/../.. \ - -I/usr/include/dpdk \ - -I/usr/include/x86_64-linux-gnu \ + $(DPDK_CFLAGS) \ -lpcap \ - -DPCAP_MODE $(LDFLAGS)\ + -DPCAP_MODE $(LDFLAGS) $(DPDK_LIBS) \ -m64 -mssse3 glb-director-stub-server: @@ -112,9 +111,8 @@ glb-director-stub-server: -o glb-director-stub-server \ -I`pwd`/.. \ -I`pwd`/../.. \ - -I/usr/include/dpdk \ - -I/usr/include/x86_64-linux-gnu \ - -DPCAP_MODE $(LDFLAGS)\ + $(DPDK_CFLAGS) \ + -DPCAP_MODE $(LDFLAGS) $(DPDK_LIBS) \ -m64 -mssse3 test-check-config: ../tests/test_check_config.c ../glb_fwd_config.c ../siphash24.c diff --git a/src/glb-director/cmdline_parse.c b/src/glb-director/cmdline_parse.c index 3ab4a020..ecb02325 100644 --- a/src/glb-director/cmdline_parse.c +++ b/src/glb-director/cmdline_parse.c @@ -71,6 +71,7 @@ #include #include +#include #include "cmdline.h" #include "cmdline_parse.h" @@ -86,6 +87,13 @@ #define CMDLINE_BUFFER_SIZE 64 +/* struct cmdline became opaque in DPDK 22.11; use the accessor if available */ +#if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0) +#define CMDLINE_CTX(cl) cmdline_get_ctx(cl) +#else +#define CMDLINE_CTX(cl) ((cl)->ctx) +#endif + /* isblank() needs _XOPEN_SOURCE >= 600 || _ISOC99_SOURCE, so use our * own. */ static int isblank2(char c) @@ -262,7 +270,7 @@ int cmdline_parse(struct cmdline *cl, const char *buf) if (!cl || !buf) return CMDLINE_PARSE_BAD_ARGS; - ctx = cl->ctx; + ctx = CMDLINE_CTX(cl); /* * - look if the buffer contains at least one line @@ -379,7 +387,7 @@ int cmdline_complete(struct cmdline *cl, const char *buf, int *state, char *dst, if (!cl || !buf || !state || !dst) return -1; - ctx = cl->ctx; + ctx = CMDLINE_CTX(cl); debug_printf("%s called\n", __func__); memset(&token_hdr, 0, sizeof(token_hdr)); diff --git a/src/glb-director/config.h b/src/glb-director/config.h index bba7752d..868e5911 100644 --- a/src/glb-director/config.h +++ b/src/glb-director/config.h @@ -31,9 +31,28 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _GLB_CONFIG_H_ +#define _GLB_CONFIG_H_ + +#include + /* Macros for printing using RTE_LOG */ #define RTE_LOGTYPE_APP RTE_LOGTYPE_USER1 +#ifndef NO_DPDK +#include +#include +#include +#include +#include +#if __has_include() && RTE_VERSION >= RTE_VERSION_NUM(23, 3, 0, 0) +#include +#define GLB_HAVE_MBUF_USERDATA_DYNFIELD 1 +#else +#define GLB_HAVE_MBUF_USERDATA_DYNFIELD 0 +#endif +#endif + /* Max size of a single packet */ #define MAX_PACKET_SZ 9220 @@ -62,3 +81,172 @@ /* default ethernet dev, used for collecting nic info */ #define DEFAULT_ETH_DEV 0 + +/* + * Compatibility aliases for DPDK API naming differences between older and + * newer releases. + */ +#ifndef NO_DPDK +#ifdef RTE_ETHER_TYPE_IPV4 +#define GLB_ETHER_HDR_DST_ADDR dst_addr +#define GLB_ETHER_HDR_SRC_ADDR src_addr + +#ifndef ether_addr +#define ether_addr rte_ether_addr +#endif + +#ifndef ether_hdr +#define ether_hdr rte_ether_hdr +#endif + +#ifndef ipv4_hdr +#define ipv4_hdr rte_ipv4_hdr +#endif + +#ifndef ipv6_hdr +#define ipv6_hdr rte_ipv6_hdr +#endif + +#if !defined(ETHER_TYPE_IPv4) && defined(RTE_ETHER_TYPE_IPV4) +#define ETHER_TYPE_IPv4 RTE_ETHER_TYPE_IPV4 +#endif + +#if !defined(ETHER_TYPE_IPv6) && defined(RTE_ETHER_TYPE_IPV6) +#define ETHER_TYPE_IPv6 RTE_ETHER_TYPE_IPV6 +#endif + +#ifndef ether_addr_octet +#define ether_addr_octet addr_bytes +#endif + +#ifndef tcp_hdr +#define tcp_hdr rte_tcp_hdr +#endif + +#ifndef udp_hdr +#define udp_hdr rte_udp_hdr +#endif + +#ifndef ether_format_addr +#define ether_format_addr rte_ether_format_addr +#endif + +/* mbuf TX flags renamed in DPDK 21.11 */ +#ifndef PKT_TX_IPV4 +#define PKT_TX_IPV4 RTE_MBUF_F_TX_IPV4 +#endif + +#ifndef PKT_TX_IP_CKSUM +#define PKT_TX_IP_CKSUM RTE_MBUF_F_TX_IP_CKSUM +#endif + +#ifndef PKT_TX_UDP_CKSUM +#define PKT_TX_UDP_CKSUM RTE_MBUF_F_TX_UDP_CKSUM +#endif + +/* Ethernet device config constants renamed in DPDK 21.11 */ +#ifndef ETH_MQ_RX_RSS +#define ETH_MQ_RX_RSS RTE_ETH_MQ_RX_RSS +#endif + +#ifndef ETH_MQ_TX_NONE +#define ETH_MQ_TX_NONE RTE_ETH_MQ_TX_NONE +#endif + +#ifndef ETH_RSS_UDP +#define ETH_RSS_UDP RTE_ETH_RSS_UDP +#endif + +#ifndef ETH_RSS_TCP +#define ETH_RSS_TCP RTE_ETH_RSS_TCP +#endif +#else +#define GLB_ETHER_HDR_DST_ADDR d_addr +#define GLB_ETHER_HDR_SRC_ADDR s_addr + +#ifndef rte_ether_addr +#define rte_ether_addr ether_addr +#endif + +#ifndef rte_ether_hdr +#define rte_ether_hdr ether_hdr +#endif + +#ifndef rte_ipv4_hdr +#define rte_ipv4_hdr ipv4_hdr +#endif + +#ifndef rte_ipv6_hdr +#define rte_ipv6_hdr ipv6_hdr +#endif + +#if !defined(RTE_ETHER_TYPE_IPV4) && defined(ETHER_TYPE_IPv4) +#define RTE_ETHER_TYPE_IPV4 ETHER_TYPE_IPv4 +#endif + +#if !defined(RTE_ETHER_TYPE_IPV6) && defined(ETHER_TYPE_IPv6) +#define RTE_ETHER_TYPE_IPV6 ETHER_TYPE_IPv6 +#endif + +#ifndef rte_tcp_hdr +#define rte_tcp_hdr tcp_hdr +#endif + +#ifndef rte_udp_hdr +#define rte_udp_hdr udp_hdr +#endif + +#ifndef rte_ether_format_addr +#define rte_ether_format_addr ether_format_addr +#endif +#endif + +#if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0) +#define GLB_ETH_DEV_COUNT() rte_eth_dev_count_avail() +#else +#define GLB_ETH_DEV_COUNT() rte_eth_dev_count() +#endif + +#ifndef RTE_LCORE_FOREACH_SLAVE +#define RTE_LCORE_FOREACH_SLAVE RTE_LCORE_FOREACH_WORKER +#endif + +#ifdef RTE_LCORE_FOREACH_WORKER +#ifndef rte_get_master_lcore +#define rte_get_master_lcore rte_get_main_lcore +#endif +#endif + +/* KNI was removed in DPDK 23.11. GLB_HAVE_KNI gates all KNI-dependent code. */ +#if __has_include() +#define GLB_HAVE_KNI 1 +#else +#define GLB_HAVE_KNI 0 +#endif + +#if GLB_HAVE_MBUF_USERDATA_DYNFIELD +extern int glb_mbuf_userdata_offset; + +static inline void glb_mbuf_set_userdata(struct rte_mbuf *mbuf, uint64_t value) +{ + *RTE_MBUF_DYNFIELD(mbuf, glb_mbuf_userdata_offset, uint64_t *) = value; +} + +static inline uint64_t glb_mbuf_get_userdata(const struct rte_mbuf *mbuf) +{ + return *RTE_MBUF_DYNFIELD(mbuf, glb_mbuf_userdata_offset, const uint64_t *); +} +#else +static inline void glb_mbuf_set_userdata(struct rte_mbuf *mbuf, uint64_t value) +{ + mbuf->udata64 = value; +} + +static inline uint64_t glb_mbuf_get_userdata(const struct rte_mbuf *mbuf) +{ + return mbuf->udata64; +} +#endif +#endif /* NO_DPDK */ + +#endif /* _GLB_CONFIG_H_ */ diff --git a/src/glb-director/glb_encap.c b/src/glb-director/glb_encap.c index a38f5ff7..042feb7c 100644 --- a/src/glb-director/glb_encap.c +++ b/src/glb-director/glb_encap.c @@ -131,8 +131,8 @@ int glb_encapsulate_packet(struct ether_hdr *eth_hdr, glb_route_context *route_c uint32_t first_hop_ip = route_context->ipv4_hops[0]; uint32_t remaining_hop_count = route_context->hop_count - 1; - eth_hdr->d_addr = g_director_config->gateway_ether_addr; - eth_hdr->s_addr = g_director_config->local_ether_addr; + eth_hdr->GLB_ETHER_HDR_DST_ADDR = g_director_config->gateway_ether_addr; + eth_hdr->GLB_ETHER_HDR_SRC_ADDR = g_director_config->local_ether_addr; eth_hdr->ether_type = htons(ETHER_TYPE_IPv4); ipv4_hdr->version = PDNET_IPV4_VERSION; diff --git a/src/glb-director/glb_kni.c b/src/glb-director/glb_kni.c index bc279b32..edc7452f 100644 --- a/src/glb-director/glb_kni.c +++ b/src/glb-director/glb_kni.c @@ -31,6 +31,10 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include "config.h" + +#if GLB_HAVE_KNI + #include #include #include @@ -366,3 +370,53 @@ static void handle_kni_to_nic(unsigned port_id, struct rte_kni *kni, "lcore-%u: -> %d packets (%d queued) burst from KNI to port %d queue %d", rte_lcore_id(), nb_rx, nb_tx, port_id, tx_queue); } + +#else /* !GLB_HAVE_KNI */ + +#include +#include +#include "glb_kni.h" +#include "log.h" + +/* KNI was removed in DPDK 23.11. Provide stubs that fail gracefully. */ + +struct glb_kni_ { + int unused; +}; + +glb_kni *glb_kni_new(uint8_t physical_port_id, uint16_t rx_tx_queue_id, + unsigned owner_lcore_id, struct rte_mempool *pktmbuf_pool) +{ + (void)physical_port_id; + (void)rx_tx_queue_id; + (void)owner_lcore_id; + (void)pktmbuf_pool; + glb_log_error("KNI support not available in this build of DPDK"); + return NULL; +} + +unsigned glb_kni_safe_tx_burst(glb_kni *gk, struct rte_mbuf **kni_tx_burst, + unsigned tx_burst_size) +{ + (void)gk; + (void)kni_tx_burst; + (void)tx_burst_size; + return 0; +} + +void glb_kni_lcore_flush(glb_kni *gk) +{ + (void)gk; +} + +void glb_kni_handle_request(glb_kni *gk) +{ + (void)gk; +} + +void glb_kni_release(glb_kni *gk) +{ + free(gk); +} + +#endif /* GLB_HAVE_KNI */ diff --git a/src/glb-director/glb_processor_loop.c b/src/glb-director/glb_processor_loop.c index 5a306994..7adf0c3d 100644 --- a/src/glb-director/glb_processor_loop.c +++ b/src/glb-director/glb_processor_loop.c @@ -129,7 +129,8 @@ static inline uint32_t processor_burst_rx_on_flows(struct glb_processor_ctx *ctx for (i = 0; i < nb_rx; i++) { // remember which flow this came from - pkts_burst[total_rx + i]->udata64 = TARGET_FLOW_PATH(f); + glb_mbuf_set_userdata(pkts_burst[total_rx + i], + TARGET_FLOW_PATH(f)); } rte_atomic64_add(&ctx->metrics.total_packet_count, nb_rx); @@ -224,7 +225,7 @@ static inline int processor_rx_dist_tx(struct glb_processor_ctx *ctx) // shard out the returned packets over their flow paths for (i = 0; i < nb_ret; i++) { struct rte_mbuf *pkt = pkts_burst[i]; - int target = pkt->udata64; + int target = glb_mbuf_get_userdata(pkt); if (unlikely(target == TARGET_KNI && perform_kni)) { if (perform_kni) { @@ -324,7 +325,7 @@ static inline int processor_worker(struct glb_processor_ctx *ctx) rte_lcore_id()); for (i = 0; i < num_pkts; i++) { // mark as destined for drop, don't forward - pkts_burst[i]->udata64 = TARGET_DROP; + glb_mbuf_set_userdata(pkts_burst[i], TARGET_DROP); } rte_atomic64_add( &ctx->metrics.classification_failures, @@ -344,7 +345,7 @@ static inline int processor_worker(struct glb_processor_ctx *ctx) rte_lcore_id(), i, num_pkts); #endif - pkts_burst[i]->udata64 = TARGET_KNI; + glb_mbuf_set_userdata(pkts_burst[i], TARGET_KNI); rte_atomic64_inc(&ctx->metrics.kni_packet_count); } else { int table = CLASSIFIED_TABLE(classifications[i]); @@ -371,7 +372,8 @@ static inline int processor_worker(struct glb_processor_ctx *ctx) #endif } else { // free the packet, we're dropping it - pkts_burst[i]->udata64 = TARGET_DROP; + glb_mbuf_set_userdata(pkts_burst[i], + TARGET_DROP); rte_atomic64_add(&ctx->metrics.encap_failures, 1); #ifdef TRACE_PACKET_FLOW glb_log_debug( @@ -383,7 +385,8 @@ static inline int processor_worker(struct glb_processor_ctx *ctx) #ifdef TRACE_PACKET_FLOW glb_log_debug( "lcore-%u: -> -> target is %d", - rte_lcore_id(), pkts_burst[i]->udata64); + rte_lcore_id(), + glb_mbuf_get_userdata(pkts_burst[i])); #endif } } diff --git a/src/glb-director/log.h b/src/glb-director/log.h index 58c4da2c..66f08317 100644 --- a/src/glb-director/log.h +++ b/src/glb-director/log.h @@ -30,13 +30,16 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#ifndef _GLB_LOG_H +#define _GLB_LOG_H + #include #include #include #include #define MAX_MESSAGE_SZ 1024 -bool debug; +extern bool debug; // outputs formatted logs to stdout or stderr @@ -165,3 +168,5 @@ inline static void glb_log_error_and_exit(const char *format, ...) exit(1); } + +#endif /* _GLB_LOG_H */ diff --git a/src/glb-director/main.c b/src/glb-director/main.c index 12c91772..afdcf092 100644 --- a/src/glb-director/main.c +++ b/src/glb-director/main.c @@ -63,7 +63,9 @@ #include #include #include +#if __has_include() #include +#endif #include #include #include @@ -94,6 +96,8 @@ #include "shared_opt.h" #include "util.h" +#include + char config_file[256]; char forwarding_table[256]; @@ -101,6 +105,10 @@ int port_num_queues[MAX_KNI_PORTS]; glb_kni *kni_ports[MAX_KNI_PORTS] = {NULL}; +#if GLB_HAVE_MBUF_USERDATA_DYNFIELD +int glb_mbuf_userdata_offset = -1; +#endif + /* Use an array of pointers rather than a contiguous array of structs * so that the pointers can be allocated separately, keeping them core-local. */ @@ -111,11 +119,13 @@ struct rte_mempool *glb_processor_msg_pool = NULL; struct rte_eth_conf port_conf = { .rxmode = { +#if RTE_VERSION < RTE_VERSION_NUM(22, 11, 0, 0) .header_split = 0, /* Header Split disabled */ .hw_ip_checksum = 1, /* IP checksum offload disabled */ .hw_vlan_filter = 0, /* VLAN filtering disabled */ .jumbo_frame = 0, /* Jumbo Frame Support disabled */ .hw_strip_crc = 1, /* CRC stripped by hardware */ +#endif .mq_mode = ETH_MQ_RX_RSS, }, .txmode = @@ -200,6 +210,21 @@ int main(int argc, char **argv) argc -= ret; argv += ret; +#if GLB_HAVE_MBUF_USERDATA_DYNFIELD + static const struct rte_mbuf_dynfield glb_mbuf_userdata_dynfield = { + .name = "glb_mbuf_userdata", + .size = sizeof(uint64_t), + .align = __alignof__(uint64_t), + .flags = 0, + }; + + glb_mbuf_userdata_offset = + rte_mbuf_dynfield_register(&glb_mbuf_userdata_dynfield); + if (glb_mbuf_userdata_offset < 0) { + glb_log_error_and_exit("Could not register mbuf userdata field"); + } +#endif + /* Find any command line options */ get_options(config_file, forwarding_table, argc, argv); @@ -220,7 +245,7 @@ int main(int argc, char **argv) } /* Find out how many NIC ports we have, validate that it's reasonable */ - nb_sys_ports = rte_eth_dev_count(); + nb_sys_ports = GLB_ETH_DEV_COUNT(); if (nb_sys_ports == 0) { glb_log_error_and_exit("No supported Ethernet device found"); return -1; @@ -275,7 +300,11 @@ int main(int argc, char **argv) } if (g_director_config->kni_enabled) { +#if __has_include() rte_kni_init(nb_sys_ports); +#else + glb_log_info("WARNING: KNI is enabled in config but not available in this DPDK version."); +#endif } /* Pre-allocate the control message mbuf pool */ diff --git a/src/glb-director/script/test b/src/glb-director/script/test index f2a44713..7774fa68 100755 --- a/src/glb-director/script/test +++ b/src/glb-director/script/test @@ -42,38 +42,60 @@ begin_fold "Building glb-director for testing" make clean make -C cli clean - scan-build-10 make - scan-build-10 make -C cli + if command -v scan-build-10 >/dev/null 2>&1; then + SCAN_BUILD=scan-build-10 + elif command -v scan-build-18 >/dev/null 2>&1; then + SCAN_BUILD=scan-build-18 + elif command -v scan-build >/dev/null 2>&1; then + SCAN_BUILD=scan-build + elif command -v scan-build-14 >/dev/null 2>&1; then + SCAN_BUILD=scan-build-14 + else + echo "scan-build is not installed" >&2 + exit 1 + fi + + "$SCAN_BUILD" make + "$SCAN_BUILD" make -C cli ) end_fold begin_fold "Running scapy packet tests against glb-director" ( - PYTHONPATH=$(pwd)/../scapy-glb-gue/:$PYTHONPATH nosetests -v -a '!director_type' -a 'director_type=dpdk' + if [ -e /dev/kni ]; then + PYTHONPATH=$(pwd)/../scapy-glb-gue/:$PYTHONPATH nosetests -v -a '!director_type' -a 'director_type=dpdk' + else + echo "Skipping DPDK director tests (no /dev/kni available, running non-DPDK tests only)" + PYTHONPATH=$(pwd)/../scapy-glb-gue/:$PYTHONPATH nosetests -v tests/test_cli_tool.py tests/test_rendezvous_table.py + fi ) end_fold -begin_fold "Running glb-director config tests" -( - bash tests/config_check.sh -) -end_fold +if [ -e /dev/kni ] || grep -q huge /proc/mounts 2>/dev/null; then + begin_fold "Running glb-director config tests" + ( + bash tests/config_check.sh + ) + end_fold -begin_fold "Running pcap tests" -( - # needs hugepages run after other tests since its already set up - bash tests/pcap_tests.sh -) -end_fold + begin_fold "Running pcap tests" + ( + # needs hugepages run after other tests since its already set up + bash tests/pcap_tests.sh + ) + end_fold -begin_fold "valgrind tests" -( - bash tests/valgrind_check.sh -) -end_fold + begin_fold "valgrind tests" + ( + bash tests/valgrind_check.sh + ) + end_fold -begin_fold "stub server tests" -( - bash tests/stub_server_tests.sh -) -end_fold + begin_fold "stub server tests" + ( + bash tests/stub_server_tests.sh + ) + end_fold +else + echo "Skipping config_check/pcap/valgrind/stub_server tests (no DPDK environment available)" +fi diff --git a/src/glb-director/shared_opt.c b/src/glb-director/shared_opt.c index 5070a65b..ce86f287 100644 --- a/src/glb-director/shared_opt.c +++ b/src/glb-director/shared_opt.c @@ -33,6 +33,8 @@ #include "shared_opt.h" #include +bool debug = false; + /* parses --config-file, --forwarding-table, and --debug cli options */ void get_options(char *config_file, char *forwarding_table, int argc, @@ -40,7 +42,6 @@ void get_options(char *config_file, char *forwarding_table, int argc, { int opt_index, opt; - debug = false; static struct option long_options[] = { {"config-file", required_argument, NULL, 'c'}, diff --git a/src/glb-director/strlcpy.h b/src/glb-director/strlcpy.h index 0f5e314a..91906e8c 100644 --- a/src/glb-director/strlcpy.h +++ b/src/glb-director/strlcpy.h @@ -1,4 +1,7 @@ -/* $OpenBSD: strlcpy.c,v 1.11 2006/05/05 15:27:38 millert Exp $ */ +/*$OpenBSD: strlcpy.c,v 1.11 2006/05/05 15:27:38 millert Exp $*/ + +#ifndef GLB_STRLCPY_H +#define GLB_STRLCPY_H /*- * Copyright (c) 1998 Todd C. Miller @@ -46,3 +49,5 @@ strlcpy(char * __restrict dst, const char * __restrict src, size_t siz) return(s - src - 1); /* count does not include NUL */ } + +#endif diff --git a/src/glb-director/tests/glb_test_utils.py b/src/glb-director/tests/glb_test_utils.py index e0bae561..17d134f1 100644 --- a/src/glb-director/tests/glb_test_utils.py +++ b/src/glb-director/tests/glb_test_utils.py @@ -17,10 +17,18 @@ import logging logging.getLogger("scapy.runtime").setLevel(logging.ERROR) +logging.getLogger("pyroute2").setLevel(logging.WARNING) +logging.getLogger("pyroute2.netlink").setLevel(logging.WARNING) +logging.getLogger("pyroute2.netlink.core").setLevel(logging.WARNING) +logging.getLogger("asyncio").setLevel(logging.WARNING) -from scapy.all import sniff, sendp, Ether, IP, IPv6, L2ListenSocket, MTU, Packet, UDP, TCP, bind_layers, ICMP, ICMPv6PacketTooBig +from scapy.all import sniff, sendp, Ether, IP, IPv6, MTU, Packet, UDP, TCP, bind_layers, ICMP, ICMPv6PacketTooBig +from scapy.arch.linux import L2ListenSocket from pyroute2 import IPRoute, NetlinkError -from nose.tools import assert_equals + +def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) + import subprocess, time import signal from contextlib import contextmanager @@ -75,7 +83,7 @@ def setup(self, iface): '--config-file', './tests/director-config.json', '--forwarding-table', './tests/test-tables.bin' ], - stdout=open('director-output.txt', 'wba'), + stdout=open('director-output.txt', 'ab'), stderr=subprocess.STDOUT, ) @@ -135,13 +143,13 @@ def updated_env(self): def wait(self): # wait for it to notify that it's actually bound to the iface. - self.notify_sock.settimeout(2) + self.notify_sock.settimeout(10) try: data, addr = self.notify_sock.recvfrom(32) - assert data == 'READY=1' # only thing it will send - except socket.timeout: + assert data == b'READY=1' # only thing it will send + except (socket.timeout, TimeoutError) as e: print('notify ready timed out') - raise Exception('Timeout while waiting for director to signal ready, did it crash?\n\n' + open('director-output.txt', 'rb').read()) + raise Exception('Timeout while waiting for director to signal ready, did it crash?\n\n' + open('director-output.txt', 'rb').read().decode('utf-8', errors='replace')) self.notify_sock.close() @@ -167,7 +175,7 @@ def setup(self, iface): '/sys/fs/bpf/root_array@' + iface, iface, ], - stdout=open('director-output.txt', 'wba'), + stdout=open('director-output.txt', 'ab'), stderr=subprocess.STDOUT, env=notify_shim.updated_env(), ) @@ -189,7 +197,7 @@ def launch_director(self): '--forwarding-table', os.path.abspath('./tests/test-tables.bin'), '--bpf-program', os.path.abspath('../glb-director-xdp/bpf/glb_encap.o'), ], - stdout=open('director-output.txt', 'wba'), + stdout=open('director-output.txt', 'ab'), stderr=subprocess.STDOUT, env=notify_director.updated_env(), ) @@ -299,7 +307,7 @@ def get_initial_director_config(cls): @classmethod def update_running_forwarding_tables(cls, config): - f = open('tests/test-tables.json', 'wb') + f = open('tests/test-tables.json', 'w') f.write(json.dumps(config, indent=4)) f.close() @@ -334,7 +342,7 @@ def setup_class(cls): GLBDirectorTestBase.py_side_mac = dict(ip.link('get', index=ip.link_lookup(ifname=cls.IFACE_NAME_PY))[0]['attrs'])['IFLA_ADDRESS'] - with open('tests/director-config.json', 'wb') as f: + with open('tests/director-config.json', 'w') as f: f.write(json.dumps(cls.get_initial_director_config(), indent=4)) # set up a statsd receiver @@ -375,7 +383,7 @@ def sendp(self, *args, **kwargs): sendp(*args, **kwargs) def wait_for_packet(self, iface, condition, timeout_seconds=5): - print('Waiting for packets on', iface.iff, 'with timeout', timeout_seconds) + print('Waiting for packets on', getattr(iface, 'iface', getattr(iface, 'iff', '?')), 'with timeout', timeout_seconds) try: with timeout(timeout_seconds): while True: @@ -388,7 +396,7 @@ def wait_for_packet(self, iface, condition, timeout_seconds=5): sys.stdout.write('-' * 50 + '\n') sys.stdout.write('Output from glb-director-ng\n') sys.stdout.write('-' * 50 + '\n') - sys.stdout.write(d.read()) + sys.stdout.write(d.read().decode('utf-8', errors='replace')) sys.stdout.write('-' * 50 + '\n') raise @@ -400,6 +408,8 @@ def stream_statsd_metrics(self, timeout=0): return # nothing more to receive, we timed out else: block, _ = s.recvfrom(4096) + if isinstance(block, bytes): + block = block.decode('utf-8') for data in block.split('\n'): metric_name, metric_data = data.split(':', 1) metric_info, metric_tags = metric_data.split('#', 1) @@ -414,7 +424,7 @@ def expect_metrics(self, spec): spec_matches = set() for metric_name, metric_value, metric_type, metric_tags in self.stream_statsd_metrics(timeout=1): metric_key = (metric_name, metric_tags) - print metric_key + print(metric_key) if metric_key in spec: assert spec[metric_key](metric_value), "Metric {} had unexpected value {}".format(metric_key, repr(metric_value)) spec_matches.add(metric_key) @@ -444,7 +454,7 @@ def pkt_hash(self, key, src_addr=None, dst_addr=None, src_port=None, dst_port=No hash_parts.append(self._encode_port(dst_port)) assert len(hash_parts) > 0 - hash_data = ''.join(hash_parts) + hash_data = b''.join(hash_parts) hash_bytes = siphash.SipHash_2_4(key, hash_data).digest() hash_num, = struct.unpack('. from rendezvous_table import GLBRendezvousTable -from nose.tools import assert_equals + +def assert_equals(a, b, msg=None): + if msg is None: + msg = "%r != %r" % (a, b) + assert a == b, msg import glob import json, subprocess, struct, socket, os, tempfile + class TestGLBBinaryCLI(): def get_example_config(self): return { @@ -63,11 +68,11 @@ def write_example_config(self): def get_example_table_reference_implementation(self, table_index): table_config = self.get_example_config()['tables'][table_index] - return GLBRendezvousTable(table_config['seed'].decode('hex')) + return GLBRendezvousTable(bytes.fromhex(table_config['seed'])) def get_example_table_hosts(self, table_index): table_config = self.get_example_config()['tables'][table_index] - return map(lambda b: b['ip'], table_config['backends']) + return list(map(lambda b: b['ip'], table_config['backends'])) def test_generate_configs(self): self.write_example_config() @@ -75,7 +80,7 @@ def test_generate_configs(self): subprocess.check_call(['cli/glb-director-cli', 'build-config', 'tests/test-config.json', 'tests/test-config.bin']) f = open('tests/test-config.bin', 'rb') - assert_equals(f.read(4), 'GLBD') + assert_equals(f.read(4), b'GLBD') num_table_entries = 0x10000 max_num_backends = 0x100 @@ -109,7 +114,7 @@ def test_generate_configs(self): assert_equals(inet_addr, socket.inet_pton(socket.AF_INET6, backend['ip'])) else: assert_equals(inet_family, 1) - assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, backend['ip']).ljust(16, '\x00')) + assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, backend['ip']).ljust(16, b'\x00')) assert_equals(be_state, 1) assert_equals(be_health, 1) @@ -128,14 +133,14 @@ def test_generate_configs(self): assert_equals(ip_bits, 128) else: assert_equals(inet_family, 1) - assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, bind['ip']).ljust(16, '\x00')) + assert_equals(inet_addr, socket.inet_pton(socket.AF_INET, bind['ip']).ljust(16, b'\x00')) assert_equals(ip_bits, 32) assert_equals(bind_port_start, bind['port']) assert_equals(bind_port_end, bind['port']) assert_equals(bind_proto, 6 if bind['proto'] == 'tcp' else 17) # validate hash key for source hashing - assert_equals(f.read(16), table['hash_key'].decode('hex').rjust(16, '\x00')) + assert_equals(f.read(16), bytes.fromhex(table['hash_key']).rjust(16, b'\x00')) # validate table entries for table_index in range(num_table_entries): diff --git a/src/glb-director/tests/test_director_classify_ranges_v4.py b/src/glb-director/tests/test_director_classify_ranges_v4.py index 799840ae..2172b87f 100644 --- a/src/glb-director/tests/test_director_classify_ranges_v4.py +++ b/src/glb-director/tests/test_director_classify_ranges_v4.py @@ -17,7 +17,8 @@ from glb_test_utils import GLBDirectorTestBase, GLBGUE from scapy.all import Ether, IP, IPv6, Packet, UDP, TCP, ICMP -from nose.tools import assert_equals +def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) from nose.plugins.attrib import attr from nose.plugins.skip import SkipTest import socket, struct, time diff --git a/src/glb-director/tests/test_director_classify_ranges_v6.py b/src/glb-director/tests/test_director_classify_ranges_v6.py index 88e4e27d..f76a5095 100644 --- a/src/glb-director/tests/test_director_classify_ranges_v6.py +++ b/src/glb-director/tests/test_director_classify_ranges_v6.py @@ -17,7 +17,8 @@ from glb_test_utils import GLBDirectorTestBase, GLBGUE from scapy.all import Ether, IP, IPv6, Packet, UDP, TCP, ICMP -from nose.tools import assert_equals +def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) from nose.plugins.attrib import attr import socket, struct, time diff --git a/src/glb-director/tests/test_director_classify_v4.py b/src/glb-director/tests/test_director_classify_v4.py index 6a368247..da065f48 100644 --- a/src/glb-director/tests/test_director_classify_v4.py +++ b/src/glb-director/tests/test_director_classify_v4.py @@ -17,7 +17,8 @@ from glb_test_utils import GLBDirectorTestBase, GLBGUE from scapy.all import Ether, IP, IPv6, Packet, UDP, TCP, ICMP -from nose.tools import assert_equals +def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) import socket, struct, time class TestGLBClassifyV4(GLBDirectorTestBase): diff --git a/src/glb-director/tests/test_director_classify_v6.py b/src/glb-director/tests/test_director_classify_v6.py index f40c6208..9ef903fc 100644 --- a/src/glb-director/tests/test_director_classify_v6.py +++ b/src/glb-director/tests/test_director_classify_v6.py @@ -17,7 +17,8 @@ from glb_test_utils import GLBDirectorTestBase, GLBGUE from scapy.all import Ether, IP, IPv6, Packet, UDP, TCP, ICMPv6PacketTooBig, ICMPv6EchoRequest -from nose.tools import assert_equals +def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) from nose.plugins.attrib import attr import socket, struct @@ -45,7 +46,7 @@ def test_01_route_classified_v6(self): assert_equals(glb_gue.private_data[0].hops, ['6.7.8.9']) inner_ip = glb_gue.payload - print repr(inner_ip) + print(repr(inner_ip)) assert isinstance(inner_ip, IPv6) # Expecting the inner IPv6 packet assert_equals(inner_ip.src, 'fd91:79d3:d621::1234') assert_equals(inner_ip.dst, 'fdb4:98ce:52d4::42') diff --git a/src/glb-director/tests/test_director_hash_fields.py b/src/glb-director/tests/test_director_hash_fields.py index 3eab328d..e5867f4e 100644 --- a/src/glb-director/tests/test_director_hash_fields.py +++ b/src/glb-director/tests/test_director_hash_fields.py @@ -17,7 +17,8 @@ from glb_test_utils import GLBDirectorTestBase, GLBGUE from scapy.all import Ether, IP, IPv6, Packet, UDP, TCP, ICMP -from nose.tools import assert_equals +def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) import socket, struct, time class GLBHashFieldsBase(GLBDirectorTestBase): diff --git a/src/glb-director/tests/test_director_kni.py b/src/glb-director/tests/test_director_kni.py index 68ee0390..e2e46b04 100644 --- a/src/glb-director/tests/test_director_kni.py +++ b/src/glb-director/tests/test_director_kni.py @@ -17,7 +17,8 @@ from glb_test_utils import GLBDirectorTestBase, GLBGUE from scapy.all import Ether, IP, IPv6, Packet, UDP, TCP -from nose.tools import assert_equals +def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) from nose.plugins.attrib import attr @attr(director_type='dpdk') diff --git a/src/glb-director/tests/test_director_metrics.py b/src/glb-director/tests/test_director_metrics.py index ec111a90..9452f4aa 100644 --- a/src/glb-director/tests/test_director_metrics.py +++ b/src/glb-director/tests/test_director_metrics.py @@ -17,7 +17,8 @@ from glb_test_utils import GLBDirectorTestBase, GLBGUE from scapy.all import Ether, IP, IPv6, Packet, UDP, TCP, ICMP -from nose.tools import assert_equals +def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) from nose.plugins.attrib import attr import socket, struct, time diff --git a/src/glb-director/tests/test_rendezvous_table.py b/src/glb-director/tests/test_rendezvous_table.py index a891831f..e92d74cd 100644 --- a/src/glb-director/tests/test_rendezvous_table.py +++ b/src/glb-director/tests/test_rendezvous_table.py @@ -16,16 +16,17 @@ # along with this project. If not, see . from rendezvous_table import GLBRendezvousTable -from nose.tools import assert_equals +def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) class TestGLBRendezvousTable(): def test_row_seeds(self): """GLBRendezvousTable correctly calculates valid row seeds""" - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) - assert_equals(table.calculate_forwarding_table_row_seed(0x0000).encode('hex'), '491c53a72df4c837') - assert_equals(table.calculate_forwarding_table_row_seed(0xffff).encode('hex'), 'f223c0cc65161620') + assert_equals(table.calculate_forwarding_table_row_seed(0x0000).hex(), '491c53a72df4c837') + assert_equals(table.calculate_forwarding_table_row_seed(0xffff).hex(), 'f223c0cc65161620') def test_order_hosts_0000(self): """ @@ -37,7 +38,7 @@ def test_order_hosts_0000(self): 1.1.1.4 6f022ce1ea607e16 """ - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] @@ -54,7 +55,7 @@ def test_order_hosts_ffff(self): 1.1.1.4 a1f610df9fbb2025 """ - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] @@ -71,7 +72,7 @@ def test_order_hosts_bb44(self): 1.1.1.4 0676eaf9cb7d2f85 """ - forwarding_table_seed = '49a3d861d661ae5ab06ed9326871a2f5'.decode('hex') + forwarding_table_seed = bytes.fromhex('49a3d861d661ae5ab06ed9326871a2f5') table = GLBRendezvousTable(forwarding_table_seed) hosts = ['1.1.1.1', '1.1.1.2', '1.1.1.3', '1.1.1.4'] diff --git a/src/glb-healthcheck/script/test b/src/glb-healthcheck/script/test index 4e412008..4246e57d 100755 --- a/src/glb-healthcheck/script/test +++ b/src/glb-healthcheck/script/test @@ -33,6 +33,7 @@ set -e ROOTDIR=$(dirname $0)/.. cd $ROOTDIR +export GOFLAGS="${GOFLAGS:+$GOFLAGS }-buildvcs=false" echo 'Building...' go build diff --git a/src/glb-redirect/Makefile b/src/glb-redirect/Makefile index 37c21e9f..b0a0e06c 100644 --- a/src/glb-redirect/Makefile +++ b/src/glb-redirect/Makefile @@ -35,12 +35,83 @@ IPT_LDFLAGS=-lxtables -shared $(CC) -o $@ $< $(IPT_CFLAGS) $(IPT_LDFLAGS) mkdeb: - rm -rf glb-redirect-iptables-dkms-mkdeb - cp -R /etc/dkms/template-dkms-mkdeb/ glb-redirect-iptables-dkms-mkdeb - chown : -R glb-redirect-iptables-dkms-mkdeb - # Works around this bug: https://ubuntuforums.org/showthread.php?t=2234906 - sed -i '/chmod 644/d' glb-redirect-iptables-dkms-mkdeb/Makefile - sed -i '/^Depends:/ s/$$/, pkg-config, libxtables12 | libxtables10, libxtables-dev | libxtables10/' glb-redirect-iptables-dkms-mkdeb/debian/control - sed -i 's/^Maintainer: .*/Maintainer: GitHub /' glb-redirect-iptables-dkms-mkdeb/debian/control - dkms mkdeb --source-only - mv ../glb-redirect-iptables-dkms_$(DKMS_MOD_VER)_*.deb $(BUILDDIR)/ + @if [ -d /etc/dkms/template-dkms-mkdeb/ ]; then \ + set -e; \ + echo "Using legacy 'dkms mkdeb' workflow"; \ + rm -rf glb-redirect-iptables-dkms-mkdeb; \ + cp -R /etc/dkms/template-dkms-mkdeb/ glb-redirect-iptables-dkms-mkdeb; \ + chown : -R glb-redirect-iptables-dkms-mkdeb; \ + sed -i '/chmod 644/d' glb-redirect-iptables-dkms-mkdeb/Makefile; \ + sed -i '/^Depends:/ s/$$/, pkg-config, libxtables12 | libxtables10, libxtables-dev | libxtables10/' glb-redirect-iptables-dkms-mkdeb/debian/control; \ + sed -i 's/^Maintainer: .*/Maintainer: GitHub /' glb-redirect-iptables-dkms-mkdeb/debian/control; \ + dkms mkdeb --source-only; \ + mv ../glb-redirect-iptables-dkms_$(DKMS_MOD_VER)_*.deb $(BUILDDIR)/; \ + elif command -v dh_dkms >/dev/null 2>&1; then \ + set -e; \ + echo "Using modern 'dh-dkms / dpkg-buildpackage' workflow"; \ + $(MAKE) mkdeb-dh-dkms; \ + else \ + echo "Error: neither 'dkms mkdeb' template nor dh-dkms available" >&2; \ + exit 1; \ + fi + +# Modern DKMS packaging via dh-dkms + dpkg-buildpackage. +# Produces an "all"-arch source-only DKMS deb that builds the kernel module +# on the installation target (same behavior as the legacy 'dkms mkdeb --source-only'). +PKG_NAME := glb-redirect-iptables +PKG_VER := $(DKMS_MOD_VER) +DH_BUILD := $(BUILDDIR)/dh-dkms-build +SRCDIR_NAME := $(PKG_NAME)-$(PKG_VER) + +.PHONY: mkdeb-dh-dkms +mkdeb-dh-dkms: + rm -rf $(DH_BUILD) + mkdir -p $(DH_BUILD)/$(SRCDIR_NAME) + # Copy the kernel module source (everything dkms.conf references). + cp -a dkms.conf ipt_GLBREDIRECT.c ipt_glbredirect.h Makefile install-ipt.sh \ + $(DH_BUILD)/$(SRCDIR_NAME)/ + # Generate a minimal debian/ tree that delegates to dh-dkms. + mkdir -p $(DH_BUILD)/$(SRCDIR_NAME)/debian/source + echo '3.0 (native)' > $(DH_BUILD)/$(SRCDIR_NAME)/debian/source/format + printf '%s\n' \ + 'Source: $(PKG_NAME)-dkms' \ + 'Section: misc' \ + 'Priority: optional' \ + 'Maintainer: GitHub ' \ + 'Build-Depends: debhelper-compat (= 13), dh-dkms' \ + 'Standards-Version: 4.6.0' \ + '' \ + 'Package: $(PKG_NAME)-dkms' \ + 'Architecture: all' \ + 'Depends: dkms, $${misc:Depends}, pkg-config, libxtables12 | libxtables10, libxtables-dev | libxtables10' \ + 'Description: GLB redirect iptables kernel module (DKMS).' \ + ' DKMS source for the ipt_GLBREDIRECT iptables target used by GLB.' \ + > $(DH_BUILD)/$(SRCDIR_NAME)/debian/control + printf '%s\n' \ + '$(PKG_NAME)-dkms ($(PKG_VER)) unstable; urgency=medium' \ + '' \ + ' * Automated build.' \ + '' \ + ' -- GitHub $(shell date -R)' \ + > $(DH_BUILD)/$(SRCDIR_NAME)/debian/changelog + # dh_dkms reads the .dkms file and installs it as dkms.conf into + # /usr/src/-/. The kernel module source files are + # installed alongside it via debian/install. + echo 'dkms.conf' > $(DH_BUILD)/$(SRCDIR_NAME)/debian/$(PKG_NAME)-dkms.dkms + printf '%s\n' \ + 'ipt_GLBREDIRECT.c usr/src/$(SRCDIR_NAME)' \ + 'ipt_glbredirect.h usr/src/$(SRCDIR_NAME)' \ + 'Makefile usr/src/$(SRCDIR_NAME)' \ + 'install-ipt.sh usr/src/$(SRCDIR_NAME)' \ + > $(DH_BUILD)/$(SRCDIR_NAME)/debian/$(PKG_NAME)-dkms.install + printf '%s\n' \ + '#!/usr/bin/make -f' \ + '%:' \ + ' dh $$@ --with dkms' \ + 'override_dh_auto_build:' \ + 'override_dh_auto_test:' \ + 'override_dh_auto_install:' \ + > $(DH_BUILD)/$(SRCDIR_NAME)/debian/rules + chmod +x $(DH_BUILD)/$(SRCDIR_NAME)/debian/rules + cd $(DH_BUILD)/$(SRCDIR_NAME) && dpkg-buildpackage -us -uc -b + mv $(DH_BUILD)/$(PKG_NAME)-dkms_$(PKG_VER)_all.deb $(BUILDDIR)/glb-redirect-iptables-dkms_$(PKG_VER)_all.deb diff --git a/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py b/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py index e8616817..8844ff88 100644 --- a/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py +++ b/src/glb-redirect/tests/test_glb_redirect_v4_on_v4.py @@ -15,7 +15,10 @@ # You should have received a copy of the GNU General Public License # along with this project. If not, see . -from nose.tools import assert_equals, assert_true +def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) +def assert_true(a): + assert a from scapy.all import IP, UDP, TCP, ICMP, sniff, send, conf from glb_scapy import GLBGUEChainedRouting, GLBGUE from glb_test_utils import GLBTestHelpers diff --git a/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py b/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py index 8a5970d6..580a2e16 100644 --- a/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py +++ b/src/glb-redirect/tests/test_glb_redirect_v6_on_v4.py @@ -15,7 +15,8 @@ # You should have received a copy of the GNU General Public License # along with this project. If not, see . -from nose.tools import assert_equals +def assert_equals(a, b): + assert a == b, "%r != %r" % (a, b) from scapy.all import IP, IPv6, UDP, TCP, ICMPv6EchoRequest, ICMPv6EchoReply, ICMPv6PacketTooBig, sniff, send, conf, L3RawSocket6 from glb_scapy import GLBGUEChainedRouting, GLBGUE from glb_test_utils import GLBTestHelpers diff --git a/src/scapy-glb-gue/glb_scapy/__init__.py b/src/scapy-glb-gue/glb_scapy/__init__.py index 213434c5..f23dcde2 100644 --- a/src/scapy-glb-gue/glb_scapy/__init__.py +++ b/src/scapy-glb-gue/glb_scapy/__init__.py @@ -15,4 +15,4 @@ # You should have received a copy of the GNU General Public License # along with scapy-glb-gue. If not, see . -from glb_gue_scapy import GLBGUEChainedRouting, GLBGUE +from .glb_gue_scapy import GLBGUEChainedRouting, GLBGUE