1Panel/scripts/openresty-modules/test-builder.sh

#!/usr/bin/env bash

set -Eeuo pipefail
export DOCKER_BUILDKIT="${DOCKER_BUILDKIT:-1}"

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)"
DEFAULT_APPSTORE_ROOT="$(cd -- "${REPO_ROOT}/.." && pwd -P)/appstore"
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"

APPSTORE_ROOT="${APPSTORE_ROOT:-${DEFAULT_APPSTORE_ROOT}}"
VERSIONS_CSV="1.27.1.2-5-1-focal,1.29.2.5-0-noble,1.31.1.1-0-noble"
MODULES_CSV=""
OUTPUT_DIR="${OUTPUT_DIR:-${PWD}/openresty-module-test-results/${RUN_ID}}"
MIRROR="${MIRROR:-}"
SKIP_PULL=0
NO_CACHE=0
KEEP_DOCKER=0
KEEP_CONTEXT=0
RUN_SOURCE_CHECKS=0
STRICT_INDIVIDUAL=0
CLEANUP_READY=0

declare -a CREATED_CONTAINERS=()
declare -a CREATED_IMAGES=()
declare -a VERSIONS=()
declare -a REQUESTED_MODULES=()

usage() {
    cat <<'EOF'
Usage: test-builder.sh [options]

Build and load-test OpenResty dynamic modules directly from the appstore tree.

Options:
  --appstore PATH       Appstore repository root (default: sibling appstore repo)
  --versions CSV        App versions to test
  --modules CSV         Module names to test (default: every catalog module)
  --output PATH         Persistent result directory
  --mirror URL          apt mirror for module build packages (CONTAINER_PACKAGE_URL)
  --skip-pull           Use local runtime images without pulling
  --no-cache            Pass --no-cache to every module Docker build
  --strict-individual   Fail when a module cannot load by itself
  --source-checks       Run Go module tests and go vet before Docker tests
  --keep-docker         Keep temporary images and containers
  --keep-context        Keep copied Docker build contexts
  -h, --help            Show this help

Environment equivalents: APPSTORE_ROOT, OUTPUT_DIR, MIRROR, DOCKER_BUILDKIT.
EOF
}

log() {
    printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*" | tee -a "${OUTPUT_DIR}/run.log"
}

die() {
    log "ERROR: $*"
    return 1
}

require_command() {
    command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
}

split_csv() {
    local value="$1"
    local -n destination="$2"
    IFS=',' read -r -a destination <<<"${value}"
}

safe_name() {
    local value="$1"
    local base digest
    base="$(printf '%s' "${value}" | sed -E 's/[^a-zA-Z0-9._-]+/-/g; s/^-+//; s/-+$//' | cut -c1-48)"
    [[ -n "${base}" ]] || base="module"
    digest="$(printf '%s' "${value}" | sha256sum | awk '{print substr($1,1,8)}')"
    printf '%s-%s' "${base}" "${digest}"
}

run_logged() {
    local log_file="$1"
    shift
    mkdir -p "$(dirname -- "${log_file}")"
    set +e
    "$@" > >(tee "${log_file}") 2>&1
    local status=$?
    set -e
    return "${status}"
}

docker_rm_container() {
    local name="$1"
    docker rm -f "${name}" >/dev/null 2>&1 || true
}

cleanup() {
    local status=$?
    if [[ "${CLEANUP_READY}" -eq 0 ]]; then
        exit "${status}"
    fi
    if [[ "${KEEP_DOCKER}" -eq 0 ]]; then
        local item
        for item in "${CREATED_CONTAINERS[@]:-}"; do
            [[ -n "${item}" ]] && docker_rm_container "${item}"
        done
        for item in "${CREATED_IMAGES[@]:-}"; do
            [[ -n "${item}" ]] && docker image rm -f "${item}" >/dev/null 2>&1 || true
        done
    fi
    if [[ "${KEEP_CONTEXT}" -eq 0 && -d "${OUTPUT_DIR}/work" ]]; then
        find "${OUTPUT_DIR}/work" -mindepth 2 -maxdepth 2 -type d -name context -prune -exec rm -rf -- {} + 2>/dev/null || true
    fi
    exit "${status}"
}

write_debug_bundle() {
    local status="$1" line="$2" command="$3"
    {
        printf 'exit_status=%s\n' "${status}"
        printf 'line=%s\n' "${line}"
        printf 'command=%s\n' "${command}"
        printf 'run_id=%s\n' "${RUN_ID}"
    } >"${OUTPUT_DIR}/failure.txt"

    docker ps -a --no-trunc >"${OUTPUT_DIR}/docker-ps.txt" 2>&1 || true
    docker image ls --digests --no-trunc >"${OUTPUT_DIR}/docker-images.txt" 2>&1 || true
    docker system df >"${OUTPUT_DIR}/docker-system-df.txt" 2>&1 || true

    local item
    for item in "${CREATED_CONTAINERS[@]:-}"; do
        [[ -n "${item}" ]] || continue
        docker inspect "${item}" >"${OUTPUT_DIR}/container-${item}.json" 2>&1 || true
        docker logs "${item}" >"${OUTPUT_DIR}/container-${item}.log" 2>&1 || true
    done

    local archive="${OUTPUT_DIR%/}.tar.gz"
    tar --exclude='*/context' -czf "${archive}" -C "$(dirname -- "${OUTPUT_DIR}")" "$(basename -- "${OUTPUT_DIR}")" 2>/dev/null || true
    printf 'Debug bundle: %s\n' "${archive}" >&2
}

on_error() {
    local status="$1" line="$2" command="$3"
    set +e
    log "FAILED at line ${line}: ${command} (exit ${status})"
    write_debug_bundle "${status}" "${line}" "${command}"
    return "${status}"
}

trap cleanup EXIT

while [[ $# -gt 0 ]]; do
    case "$1" in
        --appstore)
            APPSTORE_ROOT="$2"
            shift 2
            ;;
        --versions)
            VERSIONS_CSV="$2"
            shift 2
            ;;
        --modules)
            MODULES_CSV="$2"
            shift 2
            ;;
        --output)
            OUTPUT_DIR="$2"
            shift 2
            ;;
        --mirror)
            MIRROR="$2"
            shift 2
            ;;
        --skip-pull)
            SKIP_PULL=1
            shift
            ;;
        --no-cache)
            NO_CACHE=1
            shift
            ;;
        --strict-individual)
            STRICT_INDIVIDUAL=1
            shift
            ;;
        --source-checks)
            RUN_SOURCE_CHECKS=1
            shift
            ;;
        --keep-docker)
            KEEP_DOCKER=1
            shift
            ;;
        --keep-context)
            KEEP_CONTEXT=1
            shift
            ;;
        -h|--help)
            usage
            exit 0
            ;;
        *)
            printf 'Unknown option: %s\n' "$1" >&2
            usage >&2
            exit 2
            ;;
    esac
done

if [[ -d "${OUTPUT_DIR}" ]] && find "${OUTPUT_DIR}" -mindepth 1 -print -quit | grep -q .; then
    printf 'Output directory must be empty: %s\n' "${OUTPUT_DIR}" >&2
    exit 2
fi
mkdir -p "${OUTPUT_DIR}/work"
OUTPUT_DIR="$(cd -- "${OUTPUT_DIR}" && pwd -P)"
CLEANUP_READY=1
trap 'on_error "$?" "$LINENO" "$BASH_COMMAND"' ERR
APPSTORE_ROOT="$(cd -- "${APPSTORE_ROOT}" && pwd -P)"
split_csv "${VERSIONS_CSV}" VERSIONS
if [[ "${#VERSIONS[@]}" -eq 0 || -z "${VERSIONS[0]}" ]]; then
    die "at least one OpenResty version is required"
fi
if [[ -n "${MODULES_CSV}" ]]; then
    split_csv "${MODULES_CSV}" REQUESTED_MODULES
fi

preflight() {
    [[ "$(uname -s)" == "Linux" ]] || die "this integration test must run on Linux"
    require_command docker
    require_command jq
    require_command python3
    require_command sha256sum
    require_command sed
    require_command awk
    require_command tar
    require_command file
    require_command readelf

    docker version >"${OUTPUT_DIR}/docker-version.txt" 2>&1
    docker info >"${OUTPUT_DIR}/docker-info.txt" 2>&1
    docker compose version >"${OUTPUT_DIR}/docker-compose-version.txt" 2>&1
    uname -a >"${OUTPUT_DIR}/uname.txt"
    cp /etc/os-release "${OUTPUT_DIR}/os-release.txt" 2>/dev/null || true
    df -h >"${OUTPUT_DIR}/disk-free.txt"
    free -h >"${OUTPUT_DIR}/memory.txt" 2>&1 || true

    [[ -d "${APPSTORE_ROOT}/apps/openresty" ]] || die "invalid appstore root: ${APPSTORE_ROOT}"
    log "Results: ${OUTPUT_DIR}"
    log "Appstore: ${APPSTORE_ROOT}"
    log "Docker architecture: $(docker info --format '{{.Architecture}}')"
}

run_source_checks() {
    [[ "${RUN_SOURCE_CHECKS}" -eq 1 ]] || return 0
    require_command go
    log "Running Go dynamic-module tests"
    run_logged "${OUTPUT_DIR}/go-test.log" bash -c \
        "cd '${REPO_ROOT}/agent' && go test ./app/service -run 'NginxModule|DynamicModule' -count=1 -v"
    log "Running go vet"
    run_logged "${OUTPUT_DIR}/go-vet.log" bash -c "cd '${REPO_ROOT}/agent' && go vet ./..."
}

validate_template() {
    local version="$1"
    local app_dir="${APPSTORE_ROOT}/apps/openresty/${version}"
    local catalog="${app_dir}/build/module.catalog.json"

    [[ "${version}" =~ ^[a-zA-Z0-9._-]+$ ]] || die "unsafe version value: ${version}"
    [[ -f "${app_dir}/build/Dockerfile.modules" ]] || die "missing Dockerfile.modules for ${version}"
    [[ -f "${catalog}" ]] || die "missing module catalog for ${version}"
    jq -e 'type == "array" and length > 0 and all(.[]; .name and .params and .provider == "local")' \
        "${catalog}" >/dev/null
    bash -n "${app_dir}/scripts/init.sh"
    bash -n "${app_dir}/scripts/upgrade.sh"
    grep -Fq 'conf/modules-enabled:/usr/local/openresty/nginx/conf/modules-enabled/:ro' "${app_dir}/docker-compose.yml"
    grep -Fq './modules:/usr/local/openresty/nginx/modules/1panel/:ro' "${app_dir}/docker-compose.yml"
    grep -Fq 'include /usr/local/openresty/nginx/conf/modules-enabled/*.conf;' "${app_dir}/conf/nginx.conf"

    mkdir -p "${OUTPUT_DIR}/work/${version}/website/conf.d" "${OUTPUT_DIR}/work/${version}/website/stream.d"
    (
        cd "${app_dir}"
        CONTAINER_NAME="openresty-template-check" \
        WEBSITE_DIR="${OUTPUT_DIR}/work/${version}/website" \
        PANEL_APP_PORT_HTTP=18080 \
        docker compose config -q
    ) >"${OUTPUT_DIR}/work/${version}/compose-config.log" 2>&1
}

write_module_inputs() {
    local catalog="$1" module="$2" context="$3" input_dir="$4"
    local module_script params dynamic_params packages

    module_script="$(jq -er --arg name "${module}" '.[] | select(.name == $name) | .script' "${catalog}")"
    params="$(jq -er --arg name "${module}" '.[] | select(.name == $name) | .params' "${catalog}")"
    packages="$(jq -er --arg name "${module}" '.[] | select(.name == $name) | (.packages // []) | join(" ")' "${catalog}")"
    dynamic_params="${params//--add-module=/--add-dynamic-module=}"

    mkdir -p "${input_dir}"
    printf '%s\n' "${module_script}" >"${input_dir}/script.txt"
    printf '%s\n' "${params}" >"${input_dir}/params.original.txt"
    printf '%s\n' "${dynamic_params}" >"${input_dir}/params.dynamic.txt"
    printf '%s\n' "${packages}" >"${input_dir}/packages.txt"
    printf '#!/bin/bash\nset -e\n%s\n' "${module_script}" >"${context}/tmp/module-pre.sh"

    python3 - "${dynamic_params}" >"${context}/tmp/module-config.args" <<'PY'
import shlex
import sys

params = sys.argv[1]
args = shlex.split(params, posix=True)
if not args:
    raise SystemExit("dynamic module parameters are empty")
if not any(arg.startswith("--add-dynamic-module=") or "=dynamic" in arg for arg in args):
    raise SystemExit("module does not declare a dynamic configure option")
for arg in args:
    if not arg.startswith("--"):
        raise SystemExit(f"unsupported configure argument: {arg!r}")
    if any(char in arg for char in "\x00\r\n;&|<>"):
        raise SystemExit(f"unsafe configure argument: {arg!r}")
    print(arg)
PY
}

validate_load_directives() {
    local image="$1" modules_root="$2" directives_file="$3" config_path="$4" log_file="$5"
    {
        cat "${directives_file}"
        printf 'error_log stderr notice;\npid /tmp/nginx.pid;\nevents {}\nhttp {}\n'
    } >"${config_path}"

    run_logged "${log_file}" docker run --rm --network none \
        -v "${modules_root}:/usr/local/openresty/nginx/modules/1panel:ro" \
        -v "${config_path}:/tmp/1panel-module-test.conf:ro" \
        --entrypoint /usr/local/openresty/nginx/sbin/nginx \
        "${image}" -t -c /tmp/1panel-module-test.conf
}

build_module() {
    local version="$1" module="$2" image="$3" app_dir="$4" version_dir="$5" context="$6" sequence="$7"
    local catalog="${app_dir}/build/module.catalog.json"
    local module_key module_dir input_dir tag build_log cid packages artifact relative checksum
    local -a artifacts=()
    module_key="$(safe_name "${module}")"
    module_dir="${version_dir}/modules/${module_key}/${RUN_ID}"
    input_dir="${version_dir}/inputs/${module_key}"
    tag="1panel/openresty-module-test:${module_key}-$(safe_name "${version}")-${RUN_ID}"
    tag="${tag:0:127}"
    build_log="${version_dir}/logs/build-${module_key}.log"

    log "[${version}] building ${module}"
    write_module_inputs "${catalog}" "${module}" "${context}" "${input_dir}"
    packages="$(cat "${input_dir}/packages.txt")"

    local -a build_args=(
        build --progress=plain --target module-output
        -f "${context}/Dockerfile.modules"
        -t "${tag}"
        --build-arg "PANEL_OPENRESTY_VERSION=${version}"
        --build-arg "RESTY_ADD_PACKAGE_BUILDDEPS=${packages}"
    )
    [[ "${NO_CACHE}" -eq 0 ]] || build_args+=(--no-cache)
    [[ -z "${MIRROR}" ]] || build_args+=(--build-arg "CONTAINER_PACKAGE_URL=${MIRROR}")
    build_args+=("${context}")

    run_logged "${build_log}" docker "${build_args[@]}"
    CREATED_IMAGES+=("${tag}")

    cid="1panel-module-copy-${module_key}-${RUN_ID}"
    cid="${cid:0:63}"
    docker create --name "${cid}" "${tag}" /bin/true >"${input_dir}/container-id.txt"
    CREATED_CONTAINERS+=("${cid}")
    mkdir -p "${module_dir}"
    docker cp "${cid}:/out/." "${module_dir}"
    docker_rm_container "${cid}"

    mapfile -t artifacts < <(find "${module_dir}" -maxdepth 1 -type f -name '*.so' -print | sort)
    [[ "${#artifacts[@]}" -gt 0 ]] || die "${module} produced no top-level .so files"

    : >"${input_dir}/load-directives.conf"
    for artifact in "${artifacts[@]}"; do
        relative="${artifact#${version_dir}/modules/}"
        [[ "${relative}" =~ ^[a-zA-Z0-9_./+-]+\.so$ ]] || \
            die "unsafe module artifact path: ${relative}"
        checksum="$(sha256sum "${artifact}" | awk '{print $1}')"
        printf '%s\t%s\t%s\t%s\n' "${module}" "${relative}" "${checksum}" "$(stat -c '%s' "${artifact}")" \
            >>"${version_dir}/artifacts.tsv"
        printf 'load_module /usr/local/openresty/nginx/modules/1panel/%s;\n' "${relative}" \
            >>"${input_dir}/load-directives.conf"
        file "${artifact}" >>"${input_dir}/file.txt"
        readelf -d "${artifact}" >>"${input_dir}/readelf-dynamic.txt" 2>&1 || true
    done

    if ! validate_load_directives "${image}" "${version_dir}/modules" \
        "${input_dir}/load-directives.conf" "${input_dir}/individual-nginx.conf" \
        "${version_dir}/logs/load-${module_key}.log"; then
        printf '%s\tindividual-load-failed\n' "${module}" >>"${version_dir}/status.tsv"
        if [[ "${STRICT_INDIVIDUAL}" -eq 1 ]]; then
            die "${module} failed individual load validation"
        fi
        log "[${version}] ${module} cannot load alone; combined validation will decide"
    else
        printf '%s\tindividual-load-ok\n' "${module}" >>"${version_dir}/status.tsv"
    fi

    cat "${input_dir}/load-directives.conf" >>"${version_dir}/combined-load-directives.conf"
    cp "${input_dir}/load-directives.conf" \
        "${version_dir}/ordered-configs/$(printf '%04d' "${sequence}")-${module_key}.conf"
}

runtime_reload_test() {
    local version="$1" image="$2" version_dir="$3"
    local runtime_dir="${version_dir}/runtime" container="1panel-module-runtime-$(safe_name "${version}")-${RUN_ID}"
    local -a module_configs=()
    container="${container:0:63}"
    mkdir -p "${runtime_dir}/modules-enabled"

    cat >"${runtime_dir}/nginx.conf" <<'EOF'
error_log stderr notice;
pid /tmp/nginx.pid;
include /tmp/modules-enabled/*.conf;
events {}
http {}
EOF
    printf '# empty initial module set\n' >"${runtime_dir}/modules-enabled/0000-empty.conf"

    mapfile -t module_configs < <(find "${version_dir}/ordered-configs" -type f -name '*.conf' -print | sort)
    [[ "${#module_configs[@]}" -gt 0 ]] || die "no module configs available for runtime test"

    log "[${version}] starting runtime reload test"
    docker run -d --name "${container}" --network none \
        -v "${version_dir}/modules:/usr/local/openresty/nginx/modules/1panel:ro" \
        -v "${runtime_dir}/modules-enabled:/tmp/modules-enabled:ro" \
        -v "${runtime_dir}/nginx.conf:/tmp/1panel-runtime-nginx.conf:ro" \
        --entrypoint /usr/local/openresty/nginx/sbin/nginx \
        "${image}" -c /tmp/1panel-runtime-nginx.conf -g 'daemon off;' \
        >"${runtime_dir}/container-id.txt"
    CREATED_CONTAINERS+=("${container}")

    local attempt
    for ((attempt = 1; attempt <= 20; attempt++)); do
        if [[ "$(docker inspect --format '{{.State.Running}}' "${container}" 2>/dev/null || true)" == "true" ]]; then
            break
        fi
        sleep 1
    done
    if [[ "$(docker inspect --format '{{.State.Running}}' "${container}" 2>/dev/null || true)" != "true" ]]; then
        docker logs "${container}" >"${runtime_dir}/startup-failure.log" 2>&1 || true
        die "runtime container failed to start"
    fi

    local config
    for config in "${module_configs[@]}"; do
        cp "${config}" "${runtime_dir}/modules-enabled/$(basename -- "${config}")"
    done
    run_logged "${runtime_dir}/nginx-test.log" docker exec "${container}" \
        /usr/local/openresty/nginx/sbin/nginx -t -c /tmp/1panel-runtime-nginx.conf
    run_logged "${runtime_dir}/nginx-reload.log" docker exec "${container}" \
        /usr/local/openresty/nginx/sbin/nginx -s reload -c /tmp/1panel-runtime-nginx.conf

    printf 'load_module /usr/local/openresty/nginx/modules/1panel/not-found.so;\n' \
        >"${runtime_dir}/modules-enabled/9999-invalid.conf"
    if docker exec "${container}" /usr/local/openresty/nginx/sbin/nginx \
        -t -c /tmp/1panel-runtime-nginx.conf >"${runtime_dir}/expected-invalid.log" 2>&1; then
        die "nginx -t unexpectedly accepted a missing module"
    fi
    rm -f "${runtime_dir}/modules-enabled/9999-invalid.conf"
    run_logged "${runtime_dir}/rollback-nginx-test.log" docker exec "${container}" \
        /usr/local/openresty/nginx/sbin/nginx -t -c /tmp/1panel-runtime-nginx.conf
    [[ "$(docker inspect --format '{{.State.Running}}' "${container}")" == "true" ]] || \
        die "runtime container stopped during rollback test"

    docker logs "${container}" >"${runtime_dir}/container.log" 2>&1 || true
    docker exec "${container}" /bin/sh -c \
        'for f in /usr/local/openresty/nginx/modules/1panel/*/*/*.so; do echo "### $f"; ldd "$f" || true; done' \
        >"${runtime_dir}/ldd.txt" 2>&1 || true
    docker_rm_container "${container}"
}

test_version() {
    local version="$1"
    local app_dir="${APPSTORE_ROOT}/apps/openresty/${version}"
    local version_dir="${OUTPUT_DIR}/work/${version}"
    local context="${version_dir}/context"
    local catalog="${app_dir}/build/module.catalog.json"
    local image="1panel/openresty:${version}"
    local -a modules=()

    validate_template "${version}"
    mkdir -p "${version_dir}/logs" "${version_dir}/inputs" "${version_dir}/modules" \
        "${version_dir}/ordered-configs"
    cp -a "${app_dir}/build" "${context}"
    : >"${version_dir}/artifacts.tsv"
    : >"${version_dir}/status.tsv"
    : >"${version_dir}/combined-load-directives.conf"

    if [[ "${#REQUESTED_MODULES[@]}" -gt 0 ]]; then
        modules=("${REQUESTED_MODULES[@]}")
    else
        mapfile -t modules < <(jq -r 'sort_by([.loadOrder // 50, .name])[] | .name' "${catalog}")
    fi

    if [[ "${SKIP_PULL}" -eq 0 ]]; then
        log "[${version}] pulling ${image}"
        run_logged "${version_dir}/logs/image-pull.log" docker pull "${image}"
    fi
    docker image inspect "${image}" >"${version_dir}/image-inspect.json"
    run_logged "${version_dir}/logs/nginx-version.log" docker run --rm \
        --entrypoint /usr/local/openresty/nginx/sbin/nginx "${image}" -V
    sha256sum "${app_dir}/build/Dockerfile.modules" >"${version_dir}/builder.sha256"
    find "${app_dir}/build/tmp" -maxdepth 1 -type f -print0 | sort -z | xargs -0 sha256sum \
        >"${version_dir}/build-inputs.sha256"

    local module sequence=0
    for module in "${modules[@]}"; do
        sequence=$((sequence + 1))
        jq -e --arg name "${module}" 'any(.[]; .name == $name)' "${catalog}" >/dev/null || \
            die "module ${module} is not present in ${version} catalog"
        build_module "${version}" "${module}" "${image}" "${app_dir}" "${version_dir}" "${context}" "${sequence}"
    done

    log "[${version}] validating the combined load order"
    validate_load_directives "${image}" "${version_dir}/modules" \
        "${version_dir}/combined-load-directives.conf" "${version_dir}/combined-nginx.conf" \
        "${version_dir}/logs/load-combined.log"
    runtime_reload_test "${version}" "${image}" "${version_dir}"
    printf '%s\tPASS\n' "${version}" >>"${OUTPUT_DIR}/summary.tsv"
    log "[${version}] PASS"
}

main() {
    preflight
    run_source_checks
    printf 'version\tresult\n' >"${OUTPUT_DIR}/summary.tsv"
    local version
    for version in "${VERSIONS[@]}"; do
        test_version "${version}"
    done
    log "All requested OpenResty module tests passed"
    log "Summary: ${OUTPUT_DIR}/summary.tsv"
}

main "$@"