v2: automated BIOS platform with full pipeline

Reorganized 6 branches into bios/Manufacturer/Console/. Scrapers for RetroArch, Batocera, Recalbox, and libretro core-info. Platform-aware verification replicating native logic per platform. Pack generation with dedup, alias resolution, variant support. CI/CD: weekly auto-scrape, auto-release, PR validation. Large files (>50MB) stored as GitHub Release assets, auto-fetched at build time.
2026-06-24 12:02:48 +00:00 · 2026-03-17 10:54:39 +01:00 · 2026-03-17 10:54:39 +01:00 · 13c561888d
commit 13c561888d
parent 5f96368f6d
7038 changed files with 3243612 additions and 29617 deletions
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -0,0 +1,214 @@
+name: Release BIOS Packs
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Release version (e.g., v2.0.0)"
+        required: true
+  # Auto-release when update-db finishes (after BIOS push or platform update merge)
+  workflow_run:
+    workflows: ["Update Database"]
+    types: [completed]
+    branches: [main]
+
+permissions:
+  contents: write
+
+jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    # Skip auto-release if update-db failed or was triggered by release itself
+    if: >
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
+    outputs:
+      should_release: ${{ steps.check.outputs.should_release }}
+      version: ${{ steps.version.outputs.tag }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 2
+
+      - name: Check if BIOS files changed
+        id: check
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_run" ]; then
+            # Auto-release: check if bios/ or platforms/ changed in the last commit
+            changed=$(git diff --name-only HEAD~1 HEAD | grep -cE '^(bios/|platforms/)' || true)
+            if [ "$changed" -gt 0 ]; then
+              echo "should_release=true" >> "$GITHUB_OUTPUT"
+            else
+              echo "should_release=false" >> "$GITHUB_OUTPUT"
+            fi
+          else
+            echo "should_release=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Determine version
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "tag=${{ github.event.inputs.version }}" >> "$GITHUB_OUTPUT"
+          elif [ "${{ github.event_name }}" = "push" ]; then
+            echo "tag=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
+          else
+            # Auto-release: use date-based version
+            echo "tag=auto-$(date +%Y%m%d)" >> "$GITHUB_OUTPUT"
+          fi
+
+  discover:
+    needs: check-changes
+    if: needs.check-changes.outputs.should_release == 'true'
+    runs-on: ubuntu-latest
+    outputs:
+      platforms: ${{ steps.list.outputs.platforms }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: List platforms
+        id: list
+        run: python scripts/list_platforms.py
+
+  build:
+    needs: [check-changes, discover]
+    if: needs.check-changes.outputs.should_release == 'true'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: ${{ fromJson(needs.discover.outputs.platforms) }}
+      fail-fast: false
+      max-parallel: 10
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Generate database (if not present)
+        run: |
+          if [ ! -f database.json ]; then
+            python scripts/generate_db.py --bios-dir bios --output database.json
+          fi
+
+      - name: Generate pack for ${{ matrix.platform }}
+        run: python scripts/generate_pack.py --platform ${{ matrix.platform }} --output-dir dist/
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v6
+        with:
+          name: pack-${{ matrix.platform }}
+          path: dist/*.zip
+          retention-days: 1
+
+  publish:
+    needs: [check-changes, discover, build]
+    if: needs.check-changes.outputs.should_release == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Download all pack artifacts
+        uses: actions/download-artifact@v6
+        with:
+          path: dist/
+          pattern: pack-*
+          merge-multiple: true
+
+      - name: Copy database.json
+        run: cp database.json dist/database.json 2>/dev/null || true
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Generate release notes
+        id: notes
+        run: |
+          python3 << 'PYEOF'
+          import json, os, subprocess, textwrap
+
+          with open("database.json") as f:
+              db = json.load(f)
+
+          total = db["total_files"]
+          size_mb = db["total_size"] / (1024 * 1024)
+          date = db["generated_at"]
+
+          packs = sorted(f for f in os.listdir("dist") if f.endswith(".zip"))
+          pack_list = "\n".join(f"- **{p}** ({os.path.getsize(f'dist/{p}') / 1024 / 1024:.0f} MB)" for p in packs)
+
+          # Get recent changes
+          try:
+              log = subprocess.run(
+                  ["git", "log", "--oneline", "-20", "--no-merges", "--", "bios/", "platforms/"],
+                  capture_output=True, text=True
+              ).stdout.strip()
+              changes = "\n".join(f"- {line}" for line in log.split("\n") if line) if log else "- Initial release"
+          except Exception:
+              changes = "- See commit history"
+
+          notes = textwrap.dedent(f"""\
+          {total} BIOS/firmware files, {size_mb:.0f} MB, 50+ systems, verified checksums.
+
+          ### Packs
+          {pack_list}
+
+          ### Install
+          Download, extract to your emulator's BIOS directory:
+
+          | Platform | Path |
+          |----------|------|
+          | RetroArch / Lakka | `system/` |
+          | Batocera | `/userdata/bios/` |
+          | Recalbox | `/recalbox/share/bios/` |
+
+          <details><summary>CLI & archived platforms</summary>
+
+          ```bash
+          # CLI download
+          python scripts/download.py retroarch ~/RetroArch/system/
+
+          # Archived platforms (RetroPie, etc.)
+          git clone https://github.com/Abdess/retroarch_system.git
+          cd retroarch_system && pip install pyyaml
+          python scripts/generate_pack.py --platform retropie -o ~/Downloads/
+          ```
+          </details>
+
+          ### Changes
+          {changes}
+
+          ---
+          *{date} - [Full checksums & file listing](../../blob/main/README.md)*
+          """)
+
+          with open("/tmp/release_notes.md", "w") as f:
+              f.write(notes)
+          PYEOF
+
+      - name: Create Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ needs.check-changes.outputs.version }}
+          name: "BIOS Pack ${{ needs.check-changes.outputs.version }}"
+          body_path: /tmp/release_notes.md
+          files: dist/*
+          fail_on_unmatched_files: false
+          generate_release_notes: true
+          make_latest: true
--- a/.github/workflows/update-db.yml
+++ b/.github/workflows/update-db.yml
@ -0,0 +1,41 @@
+name: Update Database
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "bios/**"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+concurrency:
+  group: update-db
+  cancel-in-progress: true
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Generate database
+        run: python scripts/generate_db.py --bios-dir bios --output database.json
+
+      - name: Generate README and CONTRIBUTING
+        run: python scripts/generate_readme.py --db database.json --platforms-dir platforms
+
+      - name: Commit changes
+        uses: stefanzweifel/git-auto-commit-action@v7
+        with:
+          commit_message: "chore: update database and documentation"
+          file_pattern: "database.json README.md CONTRIBUTING.md"
+          commit_author: "github-actions[bot] <github-actions[bot]@users.noreply.github.com>"
--- a/.github/workflows/validate.yml
+++ b/.github/workflows/validate.yml
@ -0,0 +1,146 @@
+name: Validate PR
+
+on:
+  pull_request:
+    paths:
+      - "bios/**"
+      - "platforms/**"
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  validate-bios:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Get changed BIOS files
+        id: changed
+        run: |
+          files=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} | grep '^bios/' || true)
+          echo "files=$files" >> "$GITHUB_OUTPUT"
+          echo "$files" > /tmp/changed_files.txt
+
+      - name: Validate BIOS files
+        id: validate
+        run: |
+          files=$(cat /tmp/changed_files.txt)
+          if [ -n "$files" ]; then
+            python scripts/validate_pr.py --markdown $files > /tmp/report.md 2>&1 || true
+          else
+            echo "No BIOS files changed" > /tmp/report.md
+          fi
+
+      - name: Post validation report
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const fs = require('fs');
+            const report = fs.readFileSync('/tmp/report.md', 'utf8');
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: report
+            });
+
+  validate-configs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install pyyaml jsonschema
+
+      - name: Validate platform configs
+        run: |
+          python -c "
+          import json, yaml, sys
+          from jsonschema import validate, ValidationError
+          from pathlib import Path
+
+          with open('schemas/platform.schema.json') as f:
+              schema = json.load(f)
+
+          errors = []
+          for yml_file in Path('platforms').glob('*.yml'):
+              if yml_file.name.startswith('_'):
+                  continue
+              with open(yml_file) as f:
+                  config = yaml.safe_load(f)
+              try:
+                  validate(config, schema)
+                  print(f'✅ {yml_file.name}')
+              except ValidationError as e:
+                  errors.append(f'{yml_file.name}: {e.message}')
+                  print(f'❌ {yml_file.name}: {e.message}')
+
+          if errors:
+              sys.exit(1)
+          "
+
+  label-pr:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Auto-label PR
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const { data: files } = await github.rest.pulls.listFiles({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+            });
+
+            const labels = new Set();
+
+            for (const file of files) {
+              if (file.filename.startsWith('bios/')) {
+                labels.add('bios');
+                // Extract system from path
+                const parts = file.filename.split('/');
+                if (parts.length >= 3) {
+                  labels.add(`system:${parts[1].toLowerCase()}`);
+                }
+              }
+              if (file.filename.startsWith('platforms/')) {
+                labels.add('platform-config');
+              }
+              if (file.filename.startsWith('scripts/')) {
+                labels.add('automation');
+              }
+            }
+
+            if (labels.size > 0) {
+              try {
+                await github.rest.issues.addLabels({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  labels: [...labels],
+                });
+              } catch (e) {
+                console.log('Could not add labels:', e.message);
+              }
+            }
--- a/.github/workflows/watch-updates.yml
+++ b/.github/workflows/watch-updates.yml
@ -0,0 +1,123 @@
+name: Watch Platform Updates
+
+on:
+  schedule:
+    - cron: "0 6 * * 1"  # Every Monday at 06:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+
+jobs:
+  scrape-and-update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Regenerate platform configs from live sources
+        run: |
+          python3 -c "
+          import sys, yaml
+          sys.path.insert(0, 'scripts')
+
+          def str_representer(dumper, data):
+              if any(c in data for c in '()[]{}:#'):
+                  return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='\"')
+              return dumper.represent_scalar('tag:yaml.org,2002:str', data)
+          yaml.add_representer(str, str_representer)
+
+          # Regenerate retroarch.yml from live System.dat + core-info
+          from scraper.libretro_scraper import Scraper as LS
+          config = LS().generate_platform_yaml()
+          with open('platforms/retroarch.yml', 'w') as f:
+              yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
+          print(f'RetroArch: {len(config[\"systems\"])} systems, version={config[\"version\"]}')
+
+          # Regenerate batocera.yml from live batocera-systems
+          from scraper.batocera_scraper import Scraper as BS
+          config = BS().generate_platform_yaml()
+          with open('platforms/batocera.yml', 'w') as f:
+              yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
+          print(f'Batocera: {len(config[\"systems\"])} systems, version={config[\"version\"]}')
+          "
+
+      - name: Auto-fetch missing BIOS files
+        run: |
+          python scripts/generate_db.py --bios-dir bios --output database.json
+          python scripts/auto_fetch.py --all 2>&1 | tee /tmp/fetch_report.txt || true
+
+      - name: Deduplicate BIOS files
+        run: python scripts/dedup.py
+
+      - name: Regenerate database and documentation
+        run: |
+          python scripts/generate_db.py --force --bios-dir bios --output database.json
+          python scripts/generate_readme.py --db database.json --platforms-dir platforms
+
+      - name: Check for changes
+        id: check
+        run: |
+          if git diff --quiet && [ -z "$(git ls-files --others --exclude-standard)" ]; then
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+            echo "Changes detected:"
+            git diff --stat
+            git ls-files --others --exclude-standard | head -20
+          fi
+
+      - name: Create or update PR
+        if: steps.check.outputs.has_changes == 'true'
+        run: |
+          DATE=$(date +%Y-%m-%d)
+          BRANCH="auto/update-platforms-${DATE}"
+
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          git checkout -b "$BRANCH"
+          git add -A
+          git commit -m "chore: auto-update platform data ($DATE)"
+          git push origin "$BRANCH" --force
+
+          existing_pr=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || true)
+
+          if [ -n "$existing_pr" ] && [ "$existing_pr" != "null" ]; then
+            echo "Updating existing PR #${existing_pr}"
+          else
+            gh pr create \
+              --title "Auto-update: Platform data ($DATE)" \
+              --body "$(cat <<'EOF'
+          ## Automated Platform Update
+
+          This PR was automatically generated by the weekly platform watch.
+
+          ### What changed
+          - Scraped latest BIOS requirements from libretro System.dat and batocera-systems
+          - Auto-fetched new BIOS files from public sources
+          - Deduplicated and regenerated database + documentation
+
+          ### Review Checklist
+          - [ ] New BIOS files are valid
+          - [ ] Platform configs are correct
+          - [ ] database.json and README look good
+
+          ---
+          *Auto-generated by [watch-updates](../../actions/workflows/watch-updates.yml)*
+          EOF
+          )" \
+              --label "automated,platform-update"
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}