Merge pull request #5166 from Alexays/ci/smoke-test

ci: headless smoke + screenshot test for the bar

.github/workflows/smoke.yml

@@ -0,0 +1,80 @@
+name: smoke
+
+# Launch the real Waybar binary inside a headless sway compositor and verify it
+# starts, renders, and matches a reference screenshot. Complements the unit
+# tests and build jobs, which never actually run the bar.
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install --no-install-recommends -y \
+            sway grim imagemagick fonts-dejavu-core \
+            meson ninja-build g++ pkg-config scdoc wayland-protocols \
+            gobject-introspection libdbusmenu-gtk3-dev libevdev-dev libfmt-dev \
+            libgirepository1.0-dev libgtk-3-dev libgtkmm-3.0-dev libinput-dev \
+            libjsoncpp-dev libmpdclient-dev libnl-3-dev libnl-genl-3-dev \
+            libpulse-dev libsigc++-2.0-dev libspdlog-dev libwayland-dev \
+            upower libxkbregistry-dev libxkbcommon-dev
+
+      - name: Build & install Waybar
+        run: |
+          meson setup build -Dman-pages=disabled
+          ninja -C build
+          sudo ninja -C build install
+
+      # --- Level 1: smoke with real modules ---
+      - name: Smoke — real modules load without crashing
+        run: ./test/smoke/run.sh test/smoke/modules-config.jsonc
+
+      # --- Level 1 + 2: launch deterministic config and screenshot ---
+      - name: Launch + screenshot (deterministic config)
+        run: ./test/smoke/run.sh test/smoke/config.jsonc test/smoke/style.css bar.png
+
+      # --- Level 2: the bar actually rendered something ---
+      - name: Check the bar is not blank
+        run: |
+          sd=$(convert bar.png -crop 1920x30+0+0 +repage -colorspace Gray \
+                 -format '%[fx:standard_deviation]' info:)
+          echo "bar strip stddev = $sd"
+          awk "BEGIN{ exit !($sd > 0.01) }" || {
+            echo "::error::the top bar strip looks blank — Waybar rendered nothing"
+            exit 1
+          }
+
+      # --- Level 3: golden comparison against the committed reference ---
+      - name: Compare against reference screenshot
+        run: |
+          if [ -f test/smoke/reference.png ]; then
+            ae=$(compare -metric AE -fuzz 8% test/smoke/reference.png bar.png diff.png 2>&1 || true)
+            ae=$(printf '%s' "$ae" | grep -oE '^[0-9]+' || echo 0)
+            echo "differing pixels (fuzz 8%): $ae"
+            if [ "$ae" -gt 800 ]; then
+              echo "::error::visual regression: $ae pixels differ from the reference (threshold 800). See the 'diff.png' artifact."
+              exit 1
+            fi
+            echo "✓ within tolerance"
+          else
+            echo "::warning::No reference image yet. Download the 'waybar-screenshot' artifact from this run and commit bar.png as test/smoke/reference.png to enable golden comparison."
+          fi
+
+      - name: Upload screenshots
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: waybar-screenshot
+          path: |
+            bar.png
+            diff.png
+          if-no-files-found: warn

test/smoke/README.md

@@ -0,0 +1,34 @@
+# Smoke / screenshot test
+
+The [`smoke`](../../.github/workflows/smoke.yml) workflow launches the real
+Waybar binary inside a **headless sway** compositor (software-rendered, no GPU)
+and checks three things:
+
+1. **It starts and stays alive** — with real modules (`clock`, `cpu`, `memory`,
+   `disk`) loaded, Waybar must not crash or log a fatal error.
+   ([`modules-config.jsonc`](modules-config.jsonc))
+2. **It renders** — a deterministic config ([`config.jsonc`](config.jsonc) +
+   [`style.css`](style.css)) is captured with `grim`; the bar strip must not be
+   blank.
+3. **It looks right** — the screenshot is compared to `reference.png` with an 8%
+   fuzz and an 800-pixel tolerance. Regressions fail the job, and the screenshot
+   and diff are uploaded as artifacts.
+
+## Updating the reference image
+
+`reference.png` must be generated by the CI environment (local rendering differs
+from Ubuntu's). To (re)bless it:
+
+1. Push your change and open the run of the `smoke` workflow.
+2. Download the `waybar-screenshot` artifact.
+3. Commit its `bar.png` as `test/smoke/reference.png`.
+
+Do this whenever an **intended** visual change makes the golden comparison fail.
+
+## Run locally
+
+On a machine with `sway`, `grim` and `waybar`:
+
+```sh
+./test/smoke/run.sh test/smoke/config.jsonc test/smoke/style.css /tmp/bar.png
+```

test/smoke/config.jsonc

@@ -0,0 +1,23 @@
+// Deterministic Waybar config for the CI smoke/screenshot test.
+// Uses only static custom modules (interval "once") so the rendered bar is
+// reproducible and can be compared against a reference image.
+{
+    "layer": "top",
+    "position": "top",
+    "height": 30,
+    "modules-left": ["custom/left"],
+    "modules-center": ["custom/center"],
+    "modules-right": ["custom/right"],
+    "custom/left": {
+        "exec": "echo Waybar",
+        "interval": "once"
+    },
+    "custom/center": {
+        "exec": "echo 'CI smoke test'",
+        "interval": "once"
+    },
+    "custom/right": {
+        "exec": "echo 12:34",
+        "interval": "once"
+    }
+}

test/smoke/modules-config.jsonc

@@ -0,0 +1,18 @@
+// Waybar config for the CI smoke test: exercises real modules that work in a
+// headless environment (no hardware/daemon dependency). Its content is dynamic
+// (time, cpu, ...) so it is NOT screenshot-compared — it only checks that these
+// modules load and render without crashing the bar.
+{
+    "layer": "top",
+    "position": "top",
+    "height": 30,
+    "modules-center": ["clock", "cpu", "memory", "disk", "custom/hello"],
+    "clock": {},
+    "cpu": {},
+    "memory": {},
+    "disk": {},
+    "custom/hello": {
+        "exec": "echo ok",
+        "interval": "once"
+    }
+}

test/smoke/run.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Launch Waybar inside a headless sway compositor and verify it starts and
+# renders. Used by the `smoke` CI workflow.
+#
+# Usage: run.sh <config> [style] [screenshot.png]
+#   - always: assert waybar launches, stays alive, logs no fatal error
+#   - if <screenshot.png> given: capture the bar with grim
+#
+# Requires: sway, grim, waybar (on PATH). Software-rendered, no GPU needed.
+set -euo pipefail
+
+CONFIG="$1"
+STYLE="${2:-}"
+SHOT="${3:-}"
+
+export XDG_RUNTIME_DIR="$(mktemp -d)"
+chmod 700 "$XDG_RUNTIME_DIR"
+export WLR_BACKENDS=headless
+export WLR_RENDERER=pixman
+export WLR_LIBINPUT_NO_DEVICES=1
+export LIBGL_ALWAYS_SOFTWARE=1
+
+LOG="$(mktemp)"
+SWAYCFG="$(mktemp)"
+
+wb="waybar -c $CONFIG"
+[ -n "$STYLE" ] && wb="$wb -s $STYLE"
+
+cat > "$SWAYCFG" <<EOF
+output HEADLESS-1 resolution 1920x1080 background #000000 solid_color
+exec "$wb -l debug > $LOG 2>&1"
+EOF
+
+echo "::group::Starting sway (headless) + waybar"
+sway -c "$SWAYCFG" &
+SWAY_PID=$!
+
+cleanup() {
+    swaymsg -q exit 2>/dev/null || kill "$SWAY_PID" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# Wait for the Wayland socket to appear.
+sock=""
+for _ in $(seq 1 40); do
+    sock="$(find "$XDG_RUNTIME_DIR" -maxdepth 1 -name 'wayland-*' ! -name '*.lock' 2>/dev/null | head -1 || true)"
+    [ -n "$sock" ] && break
+    sleep 0.5
+done
+[ -n "$sock" ] || { echo "::error::compositor did not create a Wayland socket"; exit 1; }
+export WAYLAND_DISPLAY="$(basename "$sock")"
+echo "compositor up on WAYLAND_DISPLAY=$WAYLAND_DISPLAY"
+echo "::endgroup::"
+
+# Give modules time to poll/settle.
+sleep 6
+
+echo "::group::Waybar log"
+cat "$LOG" || true
+echo "::endgroup::"
+
+# --- Level 1: smoke ---
+if ! pgrep -x waybar >/dev/null; then
+    echo "::error::waybar is not running — it crashed on startup (config: $CONFIG)"
+    exit 1
+fi
+if grep -iqE 'critical|segfault|\bfatal\b|terminate called' "$LOG"; then
+    echo "::error::waybar logged a fatal error (config: $CONFIG)"
+    exit 1
+fi
+echo "✓ waybar launched and stayed alive under headless sway"
+
+# --- Level 2: screenshot ---
+if [ -n "$SHOT" ]; then
+    grim "$SHOT"
+    echo "✓ captured screenshot -> $SHOT"
+fi

test/smoke/style.css

@@ -0,0 +1,27 @@
+/* Deterministic stylesheet for the CI smoke/screenshot test.
+ * A pinned font (DejaVu Sans, installed in CI) and fixed colors keep the
+ * rendered bar reproducible across runs for the golden comparison. */
+* {
+    font-family: "DejaVu Sans";
+    font-size: 14px;
+    border: none;
+    border-radius: 0;
+    min-height: 0;
+}
+
+window#waybar {
+    background-color: #1e1e2e;
+    color: #cdd6f4;
+}
+
+#custom-left,
+#custom-center,
+#custom-right {
+    padding: 0 16px;
+    color: #cdd6f4;
+}
+
+#custom-left {
+    font-weight: bold;
+    color: #89b4fa;
+}