From d6e929081b03e9ec38c99723ac48c85dc2767da7 Mon Sep 17 00:00:00 2001 From: Andrew Hopkins Date: Thu, 13 Jul 2023 00:32:28 -0700 Subject: [PATCH 3/3] MINOR: ci: Add a weekly CI run with AWS-LC. Add support to determine latest AWS-LC release, build and cache the libcrypto/libssl, and run the tests. --- .github/matrix.py | 329 +++++++++++++++++++---------------- .github/workflows/aws-lc.yml | 65 +++++++ scripts/build-ssl.sh | 30 ++++ 3 files changed, 270 insertions(+), 154 deletions(-) create mode 100644 .github/workflows/aws-lc.yml diff --git a/.github/matrix.py b/.github/matrix.py index dbf987e93..853e3bb26 100755 --- a/.github/matrix.py +++ b/.github/matrix.py @@ -15,12 +15,6 @@ import sys import urllib.request from os import environ -if len(sys.argv) == 2: - ref_name = sys.argv[1] -else: - print("Usage: {} ".format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - # # this CI is used for both development and stable branches of HAProxy # @@ -29,34 +23,48 @@ else: # "haproxy-" - stable branches # otherwise - development branch (i.e. "latest" ssl variants, "latest" github images) # -print("Generating matrix for branch '{}'.".format(ref_name)) - def clean_ssl(ssl): return ssl.replace("_VERSION", "").lower() - -@functools.lru_cache(5) -def determine_latest_openssl(ssl): +def get_all_github_tags(url): headers = {} if environ.get("GITHUB_TOKEN") is not None: headers["Authorization"] = "token {}".format(environ.get("GITHUB_TOKEN")) - request = urllib.request.Request( - "https://api.github.com/repos/openssl/openssl/tags", headers=headers - ) + request = urllib.request.Request(url, headers=headers) try: - openssl_tags = urllib.request.urlopen(request) + tags = urllib.request.urlopen(request) except: - return "OPENSSL_VERSION=failed_to_detect" - tags = json.loads(openssl_tags.read().decode("utf-8")) + return None + tags = json.loads(tags.read().decode("utf-8")) + return [tag['name'] for tag in tags] + +@functools.lru_cache(5) +def determine_latest_openssl(ssl): + tags = get_all_github_tags("https://api.github.com/repos/openssl/openssl/tags") + if not tags: + return "OPENSSL_VERSION=failed_to_detect" latest_tag = "" for tag in tags: - name = tag["name"] - if "openssl-" in name: - if name > latest_tag: - latest_tag = name + if "openssl-" in tag: + if tag > latest_tag: + latest_tag = tag return "OPENSSL_VERSION={}".format(latest_tag[8:]) +def aws_lc_version_string_to_num(version_string): + return tuple(map(int, version_string[1:].split('.'))) + +def aws_lc_version_valid(version_string): + return re.match('^v[0-9]+(\.[0-9]+)*$', version_string) + +@functools.lru_cache(5) +def determine_latest_aws_lc(ssl): + tags = get_all_github_tags("https://api.github.com/repos/aws/aws-lc/tags") + if not tags: + return "AWS_LC_VERSION=failed_to_detect" + valid_tags = list(filter(aws_lc_version_valid, tags)) + latest_tag = max(valid_tags, key=aws_lc_version_string_to_num) + return "AWS_LC_VERSION={}".format(latest_tag[1:]) @functools.lru_cache(5) def determine_latest_libressl(ssl): @@ -86,159 +94,172 @@ def get_asan_flags(cc): ] -matrix = [] - -# Ubuntu - -if "haproxy-" in ref_name: - os = "ubuntu-22.04" # stable branch -else: - os = "ubuntu-latest" # development branch - -TARGET = "linux-glibc" -for CC in ["gcc", "clang"]: - matrix.append( - { - "name": "{}, {}, no features".format(os, CC), - "os": os, - "TARGET": TARGET, - "CC": CC, - "FLAGS": [], - } - ) - - matrix.append( - { - "name": "{}, {}, all features".format(os, CC), - "os": os, - "TARGET": TARGET, - "CC": CC, - "FLAGS": [ - "USE_ZLIB=1", - "USE_OT=1", - "OT_INC=${HOME}/opt-ot/include", - "OT_LIB=${HOME}/opt-ot/lib", - "OT_RUNPATH=1", - "USE_PCRE=1", - "USE_PCRE_JIT=1", - "USE_LUA=1", - "USE_OPENSSL=1", - "USE_SYSTEMD=1", - "USE_WURFL=1", - "WURFL_INC=addons/wurfl/dummy", - "WURFL_LIB=addons/wurfl/dummy", - "USE_DEVICEATLAS=1", - "DEVICEATLAS_SRC=addons/deviceatlas/dummy", - "USE_PROMEX=1", - "USE_51DEGREES=1", - "51DEGREES_SRC=addons/51degrees/dummy/pattern", - ], - } - ) - - # ASAN - - matrix.append( - { - "name": "{}, {}, ASAN, all features".format(os, CC), - "os": os, - "TARGET": TARGET, - "CC": CC, - "FLAGS": get_asan_flags(CC) - + [ - "USE_ZLIB=1", - "USE_OT=1", - "OT_INC=${HOME}/opt-ot/include", - "OT_LIB=${HOME}/opt-ot/lib", - "OT_RUNPATH=1", - "USE_PCRE=1", - "USE_PCRE_JIT=1", - "USE_LUA=1", - "USE_OPENSSL=1", - "USE_SYSTEMD=1", - "USE_WURFL=1", - "WURFL_INC=addons/wurfl/dummy", - "WURFL_LIB=addons/wurfl/dummy", - "USE_DEVICEATLAS=1", - "DEVICEATLAS_SRC=addons/deviceatlas/dummy", - "USE_PROMEX=1", - "USE_51DEGREES=1", - "51DEGREES_SRC=addons/51degrees/dummy/pattern", - ], - } - ) - - for compression in ["USE_ZLIB=1"]: + +def main(ref_name): + print("Generating matrix for branch '{}'.".format(ref_name)) + # Ubuntu + matrix = [] + if "haproxy-" in ref_name: + os = "ubuntu-22.04" # stable branch + else: + os = "ubuntu-latest" # development branch + + TARGET = "linux-glibc" + for CC in ["gcc", "clang"]: matrix.append( { - "name": "{}, {}, gz={}".format(os, CC, clean_compression(compression)), + "name": "{}, {}, no features".format(os, CC), "os": os, "TARGET": TARGET, "CC": CC, - "FLAGS": [compression], + "FLAGS": [], } ) - ssl_versions = [ - "stock", - "OPENSSL_VERSION=1.0.2u", - "OPENSSL_VERSION=1.1.1s", - "QUICTLS=yes", - # "BORINGSSL=yes", - ] - - if "haproxy-" not in ref_name: # development branch - ssl_versions = ssl_versions + [ - "OPENSSL_VERSION=latest", - "LIBRESSL_VERSION=latest", - ] + matrix.append( + { + "name": "{}, {}, all features".format(os, CC), + "os": os, + "TARGET": TARGET, + "CC": CC, + "FLAGS": [ + "USE_ZLIB=1", + "USE_OT=1", + "OT_INC=${HOME}/opt-ot/include", + "OT_LIB=${HOME}/opt-ot/lib", + "OT_RUNPATH=1", + "USE_PCRE=1", + "USE_PCRE_JIT=1", + "USE_LUA=1", + "USE_OPENSSL=1", + "USE_SYSTEMD=1", + "USE_WURFL=1", + "WURFL_INC=addons/wurfl/dummy", + "WURFL_LIB=addons/wurfl/dummy", + "USE_DEVICEATLAS=1", + "DEVICEATLAS_SRC=addons/deviceatlas/dummy", + "USE_PROMEX=1", + "USE_51DEGREES=1", + "51DEGREES_SRC=addons/51degrees/dummy/pattern", + ], + } + ) - for ssl in ssl_versions: - flags = ["USE_OPENSSL=1"] - if ssl == "BORINGSSL=yes" or ssl == "QUICTLS=yes" or "LIBRESSL" in ssl: - flags.append("USE_QUIC=1") - if ssl != "stock": - flags.append("SSL_LIB=${HOME}/opt/lib") - flags.append("SSL_INC=${HOME}/opt/include") - if "LIBRESSL" in ssl and "latest" in ssl: - ssl = determine_latest_libressl(ssl) - if "OPENSSL" in ssl and "latest" in ssl: - ssl = determine_latest_openssl(ssl) + # ASAN matrix.append( { - "name": "{}, {}, ssl={}".format(os, CC, clean_ssl(ssl)), + "name": "{}, {}, ASAN, all features".format(os, CC), "os": os, "TARGET": TARGET, "CC": CC, - "ssl": ssl, - "FLAGS": flags, + "FLAGS": get_asan_flags(CC) + + [ + "USE_ZLIB=1", + "USE_OT=1", + "OT_INC=${HOME}/opt-ot/include", + "OT_LIB=${HOME}/opt-ot/lib", + "OT_RUNPATH=1", + "USE_PCRE=1", + "USE_PCRE_JIT=1", + "USE_LUA=1", + "USE_OPENSSL=1", + "USE_SYSTEMD=1", + "USE_WURFL=1", + "WURFL_INC=addons/wurfl/dummy", + "WURFL_LIB=addons/wurfl/dummy", + "USE_DEVICEATLAS=1", + "DEVICEATLAS_SRC=addons/deviceatlas/dummy", + "USE_PROMEX=1", + "USE_51DEGREES=1", + "51DEGREES_SRC=addons/51degrees/dummy/pattern", + ], } ) -# macOS + for compression in ["USE_ZLIB=1"]: + matrix.append( + { + "name": "{}, {}, gz={}".format(os, CC, clean_compression(compression)), + "os": os, + "TARGET": TARGET, + "CC": CC, + "FLAGS": [compression], + } + ) + + ssl_versions = [ + "stock", + "OPENSSL_VERSION=1.0.2u", + "OPENSSL_VERSION=1.1.1s", + "QUICTLS=yes", + # "BORINGSSL=yes", + ] -if "haproxy-" in ref_name: - os = "macos-12" # stable branch -else: - os = "macos-latest" # development branch + if "haproxy-" not in ref_name: # development branch + ssl_versions = ssl_versions + [ + "OPENSSL_VERSION=latest", + "LIBRESSL_VERSION=latest", + # 'AWS_LC_VERSION=latest', + ] + + for ssl in ssl_versions: + flags = ["USE_OPENSSL=1"] + if ssl == "BORINGSSL=yes" or ssl == "QUICTLS=yes" or "LIBRESSL" in ssl: + flags.append("USE_QUIC=1") + if ssl != "stock": + flags.append("SSL_LIB=${HOME}/opt/lib") + flags.append("SSL_INC=${HOME}/opt/include") + if "LIBRESSL" in ssl and "latest" in ssl: + ssl = determine_latest_libressl(ssl) + if "OPENSSL" in ssl and "latest" in ssl: + ssl = determine_latest_openssl(ssl) + if "AWS_LC" in ssl and "latest" in ssl: + ssl = determine_latest_aws_lc(ssl) + flags.append("USE_OPENSSL_AWSLC=1") + + matrix.append( + { + "name": "{}, {}, ssl={}".format(os, CC, clean_ssl(ssl)), + "os": os, + "TARGET": TARGET, + "CC": CC, + "ssl": ssl, + "FLAGS": flags, + } + ) + + # macOS + + if "haproxy-" in ref_name: + os = "macos-12" # stable branch + else: + os = "macos-latest" # development branch + + TARGET = "osx" + for CC in ["clang"]: + matrix.append( + { + "name": "{}, {}, no features".format(os, CC), + "os": os, + "TARGET": TARGET, + "CC": CC, + "FLAGS": [], + } + ) -TARGET = "osx" -for CC in ["clang"]: - matrix.append( - { - "name": "{}, {}, no features".format(os, CC), - "os": os, - "TARGET": TARGET, - "CC": CC, - "FLAGS": [], - } - ) + # Print matrix -# Print matrix + print(json.dumps(matrix, indent=4, sort_keys=True)) -print(json.dumps(matrix, indent=4, sort_keys=True)) + if environ.get("GITHUB_OUTPUT") is not None: + with open(environ.get("GITHUB_OUTPUT"), "a") as f: + print("matrix={}".format(json.dumps({"include": matrix})), file=f) -if environ.get("GITHUB_OUTPUT") is not None: - with open(environ.get("GITHUB_OUTPUT"), "a") as f: - print("matrix={}".format(json.dumps({"include": matrix})), file=f) +if __name__ == "__main__": + if len(sys.argv) == 2: + ref_name = sys.argv[1] + main(ref_name) + else: + print("Usage: {} ".format(sys.argv[0]), file=sys.stderr) + sys.exit(1) diff --git a/.github/workflows/aws-lc.yml b/.github/workflows/aws-lc.yml new file mode 100644 index 000000000..d884e3e79 --- /dev/null +++ b/.github/workflows/aws-lc.yml @@ -0,0 +1,65 @@ +name: AWS-LC + +on: + schedule: + - cron: "0 0 * * 4" + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install VTest + run: | + scripts/build-vtest.sh + - name: Determine latest AWS-LC release + id: get_aws_lc_release + run: | + result=$(cd .github && python3 -c "from matrix import determine_latest_aws_lc; print(determine_latest_aws_lc(''))") + echo $result + echo "result=$result" >> $GITHUB_OUTPUT + - name: Cache AWS-LC + id: cache_aws_lc + uses: actions/cache@v3 + with: + path: '~/opt/' + key: ssl-${{ steps.get_aws_lc_release.outputs.result }}-Ubuntu-latest-gcc + - name: Install AWS-LC + if: ${{ steps.cache_ssl.outputs.cache-hit != 'true' }} + run: env ${{ steps.get_aws_lc_release.outputs.result }} scripts/build-ssl.sh + - name: Compile HAProxy + run: | + make -j$(nproc) CC=gcc TARGET=linux-glibc \ + USE_OPENSSL_AWSLC=1 SSL_LIB=${HOME}/opt/lib SSL_INC=${HOME}/opt/include \ + DEBUG="-DDEBUG_STRICT -DDEBUG_MEMORY_POOLS -DDEBUG_POOL_INTEGRITY" \ + ADDLIB="-Wl,-rpath,/usr/local/lib/ -Wl,-rpath,$HOME/opt/lib/" + sudo make install + - name: Show HAProxy version + id: show-version + run: | + ldd $(which haproxy) + haproxy -vv + echo "version=$(haproxy -v |awk 'NR==1{print $3}')" >> $GITHUB_OUTPUT + - name: Install problem matcher for VTest + run: echo "::add-matcher::.github/vtest.json" + - name: Run VTest for HAProxy + id: vtest + run: | + # This is required for macOS which does not actually allow to increase + # the '-n' soft limit to the hard limit, thus failing to run. + ulimit -n 65536 + make reg-tests VTEST_PROGRAM=../vtest/vtest REGTESTS_TYPES=default,bug,devel + - name: Show VTest results + if: ${{ failure() && steps.vtest.outcome == 'failure' }} + run: | + for folder in ${TMPDIR}/haregtests-*/vtc.*; do + printf "::group::" + cat $folder/INFO + cat $folder/LOG + echo "::endgroup::" + done + exit 1 diff --git a/scripts/build-ssl.sh b/scripts/build-ssl.sh index a1935dd1e..4a974f414 100755 --- a/scripts/build-ssl.sh +++ b/scripts/build-ssl.sh @@ -86,6 +86,31 @@ download_boringssl () { fi } +download_aws_lc () { + if [ ! -f "download-cache/aws-lc-${AWS_LC_VERSION}.tar.gz" ]; then + mkdir -p download-cache + wget -q -O "download-cache/aws-lc-${AWS_LC_VERSION}.tar.gz" \ + "https://github.com/aws/aws-lc/archive/refs/tags/v${AWS_LC_VERSION}.tar.gz" + fi +} + +build_aws_lc () { + if [ "$(cat ${HOME}/opt/.aws_lc-version)" != "${AWS_LC_VERSION}" ]; then + tar zxf "download-cache/aws-lc-${AWS_LC_VERSION}.tar.gz" + ( + cd "aws-lc-${AWS_LC_VERSION}/" + mkdir -p build + cd build + cmake -version + cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=1 -DDISABLE_GO=1 -DDISABLE_PERL=1 \ + -DBUILD_TESTING=0 -DCMAKE_INSTALL_PREFIX=${HOME}/opt .. + make -j$(nproc) + make install + ) + echo "${AWS_LC_VERSION}" > "${HOME}/opt/.aws_lc-version" + fi +} + download_quictls () { if [ ! -d "download-cache/quictls" ]; then git clone --depth=1 https://github.com/quictls/openssl download-cache/quictls @@ -132,6 +157,11 @@ if [ ! -z ${BORINGSSL+x} ]; then ) fi +if [ ! -z ${AWS_LC_VERSION+x} ]; then + download_aws_lc + build_aws_lc +fi + if [ ! -z ${QUICTLS+x} ]; then ( download_quictls -- 2.37.1 (Apple Git-137.1)