Try to automate update process

Creates issue if error, otherwise PR

.forgejo/workflows/auto-update.yaml

@@ -7,8 +7,11 @@ on:
   workflow_dispatch: {}
 
 jobs:
-  update:
+  check-updates:
     runs-on: nix
+    outputs:
+      changed: ${{ steps.changes.outputs.changed }}
+      hosts: ${{ steps.hosts.outputs.hosts }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -32,13 +35,92 @@ jobs:
             echo "changed=true" >> $GITHUB_OUTPUT
           fi
 
-      - name: Build all hosts
+      - name: Extract LXC hosts
+        id: hosts
         if: steps.changes.outputs.changed == 'true'
         run: |
-          nix develop -c colmena build
+          HOSTS=$(nix eval --json .#colmena --apply 'x: builtins.filter (n: n != "meta" && builtins.elem "lxc" (x.${n}.deployment.tags or [])) (builtins.attrNames x)')
+          echo "hosts=$HOSTS" >> $GITHUB_OUTPUT
+          echo "Discovered hosts: $HOSTS"
+
+      - name: Upload flake.lock
+        if: steps.changes.outputs.changed == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: flake-lock
+          path: flake.lock
+          retention-days: 1
+
+  build:
+    needs: check-updates
+    if: needs.check-updates.outputs.changed == 'true'
+    runs-on: nix
+    strategy:
+      fail-fast: false
+      matrix:
+        host: ${{ fromJson(needs.check-updates.outputs.hosts) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Download updated flake.lock
+        uses: actions/download-artifact@v4
+        with:
+          name: flake-lock
+
+      - name: Build ${{ matrix.host }}
+        id: build
+        run: |
+          echo "Building host: ${{ matrix.host }}"
+          nix build .#nixosConfigurations.${{ matrix.host }}.config.system.build.toplevel --no-link 2>&1 | tee build-output.txt
+        continue-on-error: true
+
+      - name: Upload build log on failure
+        if: failure() || steps.build.outcome == 'failure'
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-failure-${{ matrix.host }}
+          path: build-output.txt
+          retention-days: 7
+
+      - name: Check build result
+        run: |
+          if [ "${{ steps.build.outcome }}" == "failure" ]; then
+            echo "Build failed for ${{ matrix.host }}"
+            exit 1
+          fi
+
+  report:
+    needs: [check-updates, build]
+    if: always() && needs.check-updates.outputs.changed == 'true'
+    runs-on: nix
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "Flake Update Bot"
+          git config user.email "bot@noreply.local"
+
+      - name: Download updated flake.lock
+        uses: actions/download-artifact@v4
+        with:
+          name: flake-lock
+
+      - name: Download failure artifacts
+        if: needs.build.result == 'failure'
+        uses: actions/download-artifact@v4
+        with:
+          pattern: build-failure-*
+          path: failures
+          merge-multiple: false
+        continue-on-error: true
 
       - name: Create branch and commit
-        if: steps.changes.outputs.changed == 'true'
+        id: branch
         run: |
           BRANCH_NAME="auto-update/$(date +%Y-%m-%d)"
           git checkout -b "$BRANCH_NAME"
@@ -46,27 +128,68 @@ jobs:
           git commit -m "chore: update flake inputs $(date +%Y-%m-%d)"
           git push origin "$BRANCH_NAME"
           echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT
-        id: branch
 
       - name: Create Pull Request
-        if: steps.changes.outputs.changed == 'true'
+        if: needs.build.result == 'success'
         env:
           FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }}
         run: |
+          HOSTS='${{ needs.check-updates.outputs.hosts }}'
+          HOST_LIST=$(echo "$HOSTS" | jq -r '.[] | "- " + .' | tr '\n' '\n')
+          
           curl -X POST \
             -H "Authorization: token $FORGEJO_TOKEN" \
             -H "Content-Type: application/json" \
             -d '{
               "title": "chore: weekly flake update",
-              "body": "Automated flake update from CI.\n\nThis PR updates all flake inputs and has been verified to build successfully.\n\nGenerated on: '"$(date -Iseconds)"'",
+              "body": "Automated flake update from CI.\n\nThis PR updates all flake inputs and has been verified to build successfully.\n\n**Hosts built:**\n'"$(echo "$HOST_LIST" | sed 's/$/\\n/g' | tr -d '\n')"'\n\nGenerated on: '"$(date -Iseconds)"'",
               "head": "'"${{ steps.branch.outputs.branch_name }}"'",
               "base": "master"
             }' \
             "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/pulls"
 
+      - name: Create Issue on failure
+        if: needs.build.result == 'failure'
+        env:
+          FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }}
+        run: |
+          # Collect failed hosts from artifact directories
+          FAILED_HOSTS=""
+          FAILURE_DETAILS=""
+          
+          if [ -d "failures" ]; then
+            for dir in failures/build-failure-*; do
+              if [ -d "$dir" ]; then
+                HOST=$(basename "$dir" | sed 's/build-failure-//')
+                FAILED_HOSTS="$FAILED_HOSTS\n- $HOST"
+                
+                if [ -f "$dir/build-output.txt" ]; then
+                  # Get last 50 lines of build output
+                  LOG_TAIL=$(tail -50 "$dir/build-output.txt" | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\\n/g')
+                  FAILURE_DETAILS="$FAILURE_DETAILS\n\n<details>\n<summary>$HOST build log (last 50 lines)</summary>\n\n\`\`\`\n$LOG_TAIL\n\`\`\`\n</details>"
+                fi
+              fi
+            done
+          fi
+          
+          # If no failure artifacts found, list from matrix
+          if [ -z "$FAILED_HOSTS" ]; then
+            FAILED_HOSTS="\n- (Unable to determine failed hosts - check workflow logs)"
+          fi
+          
+          ISSUE_BODY="Weekly flake update failed to build some hosts.\n\n**Branch:** \`${{ steps.branch.outputs.branch_name }}\`\n**Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\n\n**Failed hosts:**$FAILED_HOSTS\n$FAILURE_DETAILS\n\nGenerated on: $(date -Iseconds)"
+          
+          curl -X POST \
+            -H "Authorization: token $FORGEJO_TOKEN" \
+            -H "Content-Type: application/json" \
+            -d "{
+              \"title\": \"Build failure: weekly flake update $(date +%Y-%m-%d)\",
+              \"body\": \"$ISSUE_BODY\"
+            }" \
+            "${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues"
+
       - name: Cleanup
         if: always()
         run: |
           nix-collect-garbage -d
-          rm -rf result .direnv
+          rm -rf result .direnv failures
-