From e91bba759bec672f92f33b43d16607815267b984 Mon Sep 17 00:00:00 2001 From: Djeex Date: Wed, 26 Aug 2026 15:30:30 +0200 Subject: [PATCH] CI/CD hardening: lint, secret scan, coverage gate, auto CVE-fix PRs, GHCR + GitHub mirror publishing (#17) - release changelog: commits rendered as description (link), divider lines dropped - gitleaks secret scan and hadolint on every push/PR - ruff lint/format gate (Python repos) with a pytest --cov-fail-under gate - scheduled CRITICAL Trivy failures attempt an apk upgrade rebuild and open a follow-up PR if it clears the finding, instead of just failing red - images also published to ghcr.io/djeex/ - a matching GitHub Release is created on the GitHub mirror, with a notice pointing back to this repo as the source of truth --------- Co-authored-by: Djeex Reviewed-on: https://git.djeex.fr/Djeex/adguard-cidre/pulls/17 --- .gitea/workflows/ci.yml | 106 ++++++++++++++++++++++++++++-- Dockerfile | 8 ++- blocklist_scheduler.py | 68 ++++++++++++++----- ruff.toml | 10 +++ tests/test_blocklist_scheduler.py | 11 +++- 5 files changed, 177 insertions(+), 26 deletions(-) create mode 100644 ruff.toml diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 84d8030..18ea25c 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -18,6 +18,21 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Scan for secrets + run: | + # docker cp, not a build COPY: a repo's own .dockerignore (e.g. one that + # excludes .git for prod builds) would otherwise silently give an empty, + # falsely-clean scan. + CID=$(docker create zricethezav/gitleaks:v8.30.1 detect --source=/repo --no-banner -v) + docker cp . "$CID:/repo" + docker start -a "$CID" + STATUS=$? + docker rm "$CID" > /dev/null + exit $STATUS + + - name: Lint Dockerfile with hadolint + run: docker run --rm -i hadolint/hadolint:v2.15.1-alpine hadolint --failure-threshold error - < Dockerfile + - name: Build Docker image run: | docker build -t adguard-cidre:ci . 2>&1 | tee build.log @@ -42,7 +57,10 @@ jobs: - name: Run unit tests run: | docker build --target test -t adguard-cidre:test . - docker run --rm adguard-cidre:test pytest -v + docker run --rm adguard-cidre:test pytest -v --cov=. --cov-report=term-missing --cov-fail-under=75 + + - name: Lint with ruff + run: docker build --target lint -t adguard-cidre:lint . - name: Check deprecation warnings run: | @@ -52,12 +70,53 @@ jobs: fi - name: Scan with Trivy (critical - blocking) + id: trivy_critical + continue-on-error: true run: | docker run --rm \ -e DOCKER_HOST=tcp://dockerhost:2375 \ --add-host=dockerhost:host-gateway \ aquasec/trivy:0.74.0 image --exit-code 1 --severity CRITICAL adguard-cidre:ci + - name: Handle CRITICAL findings + if: steps.trivy_critical.outcome == 'failure' + run: | + if [ "${{ github.event_name }}" != "schedule" ]; then + echo "::error::CRITICAL vulnerabilities found, failing the build." + exit 1 + fi + + echo "Scheduled scan found CRITICAL vulnerabilities — attempting an automatic apk upgrade + rescan." + + sed -i '/^FROM .* AS base$/a RUN apk upgrade --no-cache' Dockerfile + docker build -t adguard-cidre:remediated . + + if docker run --rm \ + -e DOCKER_HOST=tcp://dockerhost:2375 \ + --add-host=dockerhost:host-gateway \ + aquasec/trivy:0.74.0 image --exit-code 1 --severity CRITICAL adguard-cidre:remediated; then + echo "apk upgrade clears the CRITICAL finding(s) — opening a PR for review." + + BRANCH="auto/cve-fix-$(date +%Y%m%d)-$(echo "${{ github.sha }}" | cut -c1-7)" + git config user.name "adguard-cidre-ci" + git config user.email "ci@git.djeex.fr" + git checkout -b "$BRANCH" + git add Dockerfile + git commit -m "Auto-remediate CRITICAL CVE via apk upgrade" + git config --unset-all http.https://git.djeex.fr/.extraheader || true + git push "https://Djeex:${{ secrets.CI_PUSH_TOKEN }}@git.djeex.fr/Djeex/adguard-cidre.git" "HEAD:$BRANCH" + + PR_JSON=$(curl -s -X POST \ + -H "Authorization: token ${{ secrets.CI_PUSH_TOKEN }}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg head "$BRANCH" '{title: "🔒 Auto: remediate CRITICAL CVE via apk upgrade", head: $head, base: "main", body: "Opened automatically by the scheduled CVE scan. An `apk upgrade --no-cache` cleared the CRITICAL Trivy finding(s) in a rebuild — review the diff and merge to publish the fix."}')" \ + "https://git.djeex.fr/api/v1/repos/Djeex/adguard-cidre/pulls") + echo "PR API response: $(echo "$PR_JSON" | jq -r '.html_url // .message // "unknown"')" + else + echo "::error::apk upgrade does not clear the CRITICAL finding(s) — no automatic fix available, needs manual review." + exit 1 + fi + - name: Scan with Trivy (high - informative) run: | docker run --rm \ @@ -122,6 +181,16 @@ jobs: docker push "$IMAGE:$MINOR_TAG" docker push "$IMAGE:$VERSION" + GHCR_IMAGE=ghcr.io/djeex/adguard-cidre + echo "${{ secrets.GH_TOKEN }}" | docker login ghcr.io -u Djeex --password-stdin + + docker tag adguard-cidre:ci "$GHCR_IMAGE:latest" + docker tag adguard-cidre:ci "$GHCR_IMAGE:$MINOR_TAG" + docker tag adguard-cidre:ci "$GHCR_IMAGE:$VERSION" + docker push "$GHCR_IMAGE:latest" + docker push "$GHCR_IMAGE:$MINOR_TAG" + docker push "$GHCR_IMAGE:$VERSION" + TRIGGER_MSG=$(git log -1 --format=%s "${{ github.sha }}") PR_NUM=$(echo "$TRIGGER_MSG" | grep -oE '#[0-9]+' | head -1 | tr -d '#' || true) @@ -148,7 +217,7 @@ jobs: fi REPO_URL="https://git.djeex.fr/Djeex/adguard-cidre" - COMMIT_LIST=$(git log --no-merges --format="- [%h](${REPO_URL}/commit/%H) %s" "$BASE_REF".."${{ github.sha }}") + COMMIT_LIST=$(git log --no-merges --format="- %s ([%h](${REPO_URL}/commit/%H))" "$BASE_REF".."${{ github.sha }}") SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) SOURCE_LINE="[${SHORT_SHA}](${REPO_URL}/commit/${{ github.sha }})" @@ -158,10 +227,8 @@ jobs: BODY=$(cat </dev/null || true + fi \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3400552..a079b75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,9 +15,15 @@ COPY blocklist_scheduler.py entrypoint.sh VERSION ./ RUN chmod +x entrypoint.sh FROM base AS test -RUN pip install --no-cache-dir pytest==9.1.1 +RUN pip install --no-cache-dir pytest==9.1.1 pytest-cov==7.1.0 COPY tests/ tests/ COPY pytest.ini . +FROM base AS lint +RUN pip install --no-cache-dir ruff==0.16.4 +COPY ruff.toml . +COPY tests/ tests/ +RUN ruff check . && ruff format --check . + FROM base ENTRYPOINT ["./entrypoint.sh"] diff --git a/blocklist_scheduler.py b/blocklist_scheduler.py index fe5a839..5f92290 100644 --- a/blocklist_scheduler.py +++ b/blocklist_scheduler.py @@ -1,14 +1,15 @@ #!/usr/bin/env python3 -import os -import sys import logging -import requests -import yaml -import schedule -import time +import os import re +import sys +import time from pathlib import Path +import requests +import schedule +import yaml + logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", @@ -19,7 +20,9 @@ ADGUARD_YAML = Path("/adguard/AdGuardHome.yaml") TMP_YAML = ADGUARD_YAML.parent / (ADGUARD_YAML.name + ".tmp") MANUAL_IPS_FILE = Path("/adguard/manually_blocked_ips.conf") CIDR_BASE_URL = "https://raw.githubusercontent.com/vulnebify/cidre/main/output/cidr/ipv4" -COUNTRY_LIST_URL = "https://raw.githubusercontent.com/vulnebify/cidre/refs/heads/main/cidre/countries.py" +COUNTRY_LIST_URL = ( + "https://raw.githubusercontent.com/vulnebify/cidre/refs/heads/main/cidre/countries.py" +) FIRST_BACKUP = ADGUARD_YAML.parent / "AdGuardHome.yaml.first-start.bak" LAST_UPDATE_BACKUP = ADGUARD_YAML.parent / "AdGuardHome.yaml.last-update.bak" @@ -32,6 +35,7 @@ BLOCKLIST_CRON_DAY = os.getenv("BLOCKLIST_CRON_DAY", "mon").lower() ADGUARD_CONTAINER_NAME = os.getenv("ADGUARD_CONTAINER_NAME", "adguardhome") DOCKER_API_URL = os.getenv("DOCKER_API_URL", "http://socket-proxy-adguard:2375") + def backup_first_start(): if not FIRST_BACKUP.exists(): logging.info(f"Creating first start backup: {FIRST_BACKUP}") @@ -39,10 +43,12 @@ def backup_first_start(): else: logging.info("First start backup already exists, skipping.") + def backup_last_update(): logging.info(f"Creating last update backup: {LAST_UPDATE_BACKUP}") LAST_UPDATE_BACKUP.write_text(ADGUARD_YAML.read_text()) + def fetch_all_country_codes(): try: resp = requests.get(COUNTRY_LIST_URL, timeout=15) @@ -53,6 +59,7 @@ def fetch_all_country_codes(): logging.error(f"Failed to fetch available country codes: {e}") return set() + def get_selected_countries(): if not BLOCK_COUNTRIES: logging.error("BLOCK_COUNTRIES is not set. Skipping update.") @@ -67,7 +74,9 @@ def get_selected_countries(): is_inclusion = all(not c.startswith("!") for c in raw_codes) if not (is_exclusion or is_inclusion): - logging.error("Mixed syntax in BLOCK_COUNTRIES. Use only inclusion (e.g. 'fr,de') or only exclusion (e.g. '!fr,!de').") + logging.error( + "Mixed syntax in BLOCK_COUNTRIES. Use only inclusion (e.g. 'fr,de') or only exclusion (e.g. '!fr,!de')." + ) sys.exit(1) available = fetch_all_country_codes() @@ -81,6 +90,7 @@ def get_selected_countries(): else: return sorted(selected & available) + def download_cidr_lists(countries): combined_ips = [] for code in countries: @@ -96,6 +106,7 @@ def download_cidr_lists(countries): logging.warning(f"Failed to download {code}: {e}") return combined_ips + def read_manual_ips(): if MANUAL_IPS_FILE.exists(): logging.info(f"Reading manual IPs from {MANUAL_IPS_FILE}") @@ -103,7 +114,7 @@ def read_manual_ips(): with MANUAL_IPS_FILE.open() as f: for line in f: line = line.strip() - if line and (line.count('.') == 3 or '/' in line): + if line and (line.count(".") == 3 or "/" in line): valid_ips.append(line) logging.info(f"Added {len(valid_ips)} manual IP entries") return valid_ips @@ -111,6 +122,7 @@ def read_manual_ips(): logging.info("Manual IPs file does not exist, skipping.") return [] + def update_yaml_with_ips(ips): if not ADGUARD_YAML.exists(): logging.error(f"{ADGUARD_YAML} does not exist. Cannot update.") @@ -127,15 +139,16 @@ def update_yaml_with_ips(ips): logging.error("Invalid YAML format.") return False - data['dns']['disallowed_clients'] = ips + data["dns"]["disallowed_clients"] = ips - with TMP_YAML.open('w') as f: + with TMP_YAML.open("w") as f: yaml.safe_dump(data, f) TMP_YAML.replace(ADGUARD_YAML) logging.info(f"Updated {ADGUARD_YAML} with new disallowed clients list.") return True + def restart_adguard_container(): restart_url = f"{DOCKER_API_URL}/containers/{ADGUARD_CONTAINER_NAME}/restart" logging.info(f"Restarting AdGuard container '{ADGUARD_CONTAINER_NAME}'...") @@ -148,6 +161,7 @@ def restart_adguard_container(): except Exception as e: logging.error(f"Error restarting container: {e}") + def update_blocklist(): countries = get_selected_countries() if not countries: @@ -164,11 +178,14 @@ def update_blocklist(): if success: restart_adguard_container() + def schedule_job(): try: hour, minute = [int(x) for x in BLOCKLIST_CRON_TIME.split(":")] except Exception: - logging.error(f"Invalid BLOCKLIST_CRON_TIME '{BLOCKLIST_CRON_TIME}', must be HH:MM. Defaulting to 06:00.") + logging.error( + f"Invalid BLOCKLIST_CRON_TIME '{BLOCKLIST_CRON_TIME}', must be HH:MM. Defaulting to 06:00." + ) hour, minute = 6, 0 if BLOCKLIST_CRON_TYPE == "daily": @@ -176,20 +193,34 @@ def schedule_job(): logging.info(f"Scheduled daily update at {hour:02d}:{minute:02d}") elif BLOCKLIST_CRON_TYPE == "weekly": day_names = { - "mon": "monday", "tue": "tuesday", "wed": "wednesday", "thu": "thursday", - "fri": "friday", "sat": "saturday", "sun": "sunday", + "mon": "monday", + "tue": "tuesday", + "wed": "wednesday", + "thu": "thursday", + "fri": "friday", + "sat": "saturday", + "sun": "sunday", } day = BLOCKLIST_CRON_DAY[:3] if day not in day_names: - logging.error(f"Invalid BLOCKLIST_CRON_DAY '{BLOCKLIST_CRON_DAY}', must be one of {list(day_names)}. Defaulting to Monday.") + logging.error( + f"Invalid BLOCKLIST_CRON_DAY '{BLOCKLIST_CRON_DAY}', must be one of {list(day_names)}. Defaulting to Monday." + ) day = "mon" - getattr(schedule.every(), day_names[day]).at(f"{hour:02d}:{minute:02d}").do(update_blocklist) - logging.info(f"Scheduled weekly update on {day_names[day].capitalize()} at {hour:02d}:{minute:02d}") + getattr(schedule.every(), day_names[day]).at(f"{hour:02d}:{minute:02d}").do( + update_blocklist + ) + logging.info( + f"Scheduled weekly update on {day_names[day].capitalize()} at {hour:02d}:{minute:02d}" + ) else: - logging.error(f"Invalid BLOCKLIST_CRON_TYPE '{BLOCKLIST_CRON_TYPE}', must be 'daily' or 'weekly'. Defaulting to daily.") + logging.error( + f"Invalid BLOCKLIST_CRON_TYPE '{BLOCKLIST_CRON_TYPE}', must be 'daily' or 'weekly'. Defaulting to daily." + ) schedule.every().day.at(f"{hour:02d}:{minute:02d}").do(update_blocklist) logging.info(f"Scheduled daily update at {hour:02d}:{minute:02d}") + def main(): logging.info("Starting blocklist scheduler...") backup_first_start() @@ -199,5 +230,6 @@ def main(): schedule.run_pending() time.sleep(10) + if __name__ == "__main__": main() diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..fd30ccf --- /dev/null +++ b/ruff.toml @@ -0,0 +1,10 @@ +line-length = 100 + +[lint] +select = ["E", "F", "I", "UP", "B"] +ignore = ["E501"] + +[lint.isort] +# See nvidia-stock-bot's ruff.toml for why this is pinned explicitly rather +# than left to auto-detection. +known-first-party = ["blocklist_scheduler"] diff --git a/tests/test_blocklist_scheduler.py b/tests/test_blocklist_scheduler.py index d937156..f0db295 100644 --- a/tests/test_blocklist_scheduler.py +++ b/tests/test_blocklist_scheduler.py @@ -56,6 +56,7 @@ def test_backup_first_start_raises_if_adguard_yaml_missing(tmp_path, monkeypatch # --- update_yaml_with_ips (pyyaml) --- + def test_update_yaml_with_ips_writes_disallowed_clients(tmp_path, monkeypatch): adguard_yaml = tmp_path / "AdGuardHome.yaml" adguard_yaml.write_text("dns:\n bind_hosts:\n - 0.0.0.0\n") @@ -101,8 +102,11 @@ def test_update_yaml_with_ips_missing_dns_key_raises(tmp_path, monkeypatch): # --- fetch_all_country_codes / download_cidr_lists / restart_adguard_container (requests) --- + def test_fetch_all_country_codes_parses_codes(monkeypatch): - monkeypatch.setattr(bs.requests, "get", lambda *a, **k: FakeResponse(text='COUNTRIES = ["FR", "DE", "US"]\n')) + monkeypatch.setattr( + bs.requests, "get", lambda *a, **k: FakeResponse(text='COUNTRIES = ["FR", "DE", "US"]\n') + ) assert bs.fetch_all_country_codes() == {"fr", "de", "us"} @@ -136,7 +140,9 @@ def test_restart_adguard_container_success_does_not_raise(monkeypatch): def test_restart_adguard_container_error_status_does_not_raise(monkeypatch): - monkeypatch.setattr(bs.requests, "post", lambda *a, **k: FakeResponse(status_code=500, text="err")) + monkeypatch.setattr( + bs.requests, "post", lambda *a, **k: FakeResponse(status_code=500, text="err") + ) bs.restart_adguard_container() @@ -152,6 +158,7 @@ def test_restart_adguard_container_network_error_does_not_raise(monkeypatch): # --- schedule_job (schedule) --- + @pytest.fixture(autouse=True) def clear_schedule(): yield