Enhance runner configurations by adding Docker registry authentication to avoid rate limits; update example environment file with Docker Hub credentials; include new documentation for Docker rate limit solutions in structure.txt.

2025-09-13 02:24:23 +02:00
parent fefd16f5b3
commit 581bd59039
9 changed files with 640 additions and 4 deletions
--- a/docs/DOCKER_RATE_LIMIT_FIX.md
+++ b/docs/DOCKER_RATE_LIMIT_FIX.md
@@ -0,0 +1,201 @@
+# Docker Hub Rate Limit Fix
+
+## Problem
+```
+Error response from daemon: toomanyrequests: You have reached your unauthenticated pull rate limit. https://www.docker.com/increase-rate-limit
+```
+
+Docker Hub has strict rate limits:
+- **Unauthenticated**: 100 pulls per 6 hours per IP
+- **Authenticated (free)**: 200 pulls per 6 hours per user
+- **Pro/Team**: Higher limits
+
+## Solutions
+
+### Solution 1: Use Docker Hub Authentication (Recommended)
+
+#### 1.1. Create Docker Hub Account
+1. Go to [Docker Hub](https://hub.docker.com)
+2. Create a free account
+3. Note your username and password
+
+#### 1.2. Update Runner Configurations
+
+Add Docker authentication to each runner config:
+
+**`runners/config_heavy.yaml`:**
+```yaml
+container:
+  # Docker registry authentication
+  docker_username: "your_dockerhub_username"
+  docker_password: "your_dockerhub_password"
+```
+
+**`runners/config_light.yaml`:**
+```yaml
+container:
+  # Docker registry authentication
+  docker_username: "your_dockerhub_username"
+  docker_password: "your_dockerhub_password"
+```
+
+**`runners/config_docker.yaml`:**
+```yaml
+container:
+  # Docker registry authentication
+  docker_username: "your_dockerhub_username"
+  docker_password: "your_dockerhub_password"
+```
+
+**`runners/config_security.yaml`:**
+```yaml
+container:
+  # Docker registry authentication
+  docker_username: "your_dockerhub_username"
+  docker_password: "your_dockerhub_password"
+```
+
+#### 1.3. Alternative: Use Environment Variables
+
+Instead of hardcoding credentials, use environment variables:
+
+**Update `runners/.env.runners`:**
+```bash
+# Docker Hub credentials
+DOCKER_USERNAME=your_dockerhub_username
+DOCKER_PASSWORD=your_dockerhub_password
+```
+
+**Update config files:**
+```yaml
+container:
+  docker_username: ${DOCKER_USERNAME}
+  docker_password: ${DOCKER_PASSWORD}
+```
+
+### Solution 2: Use Alternative Registries
+
+#### 2.1. Use GitHub Container Registry (ghcr.io)
+
+Update image references to use GitHub's registry:
+
+**Heavy Runner:**
+```yaml
+labels:
+  - "java:docker://ghcr.io/openjdk/openjdk:17-jdk-slim"
+  - "python:docker://ghcr.io/library/python:3.11-slim"
+```
+
+**Light Runner:**
+```yaml
+labels:
+  - "nodejs:docker://ghcr.io/library/node:20-slim"
+  - "frontend:docker://ghcr.io/library/node:20-slim"
+```
+
+#### 2.2. Use Quay.io Registry
+
+```yaml
+labels:
+  - "java:docker://quay.io/eclipse/alpine_jdk17:latest"
+  - "python:docker://quay.io/python/python:3.11-slim"
+  - "nodejs:docker://quay.io/node/node:20-slim"
+```
+
+### Solution 3: Use Local Image Caching
+
+#### 3.1. Pre-pull Images on Runner Host
+
+```bash
+# On your runner host machine
+docker pull openjdk:17-jdk-slim
+docker pull python:3.11-slim
+docker pull node:20-slim
+docker pull docker:24-dind
+docker pull alpine:3.19
+
+# Tag as local images
+docker tag openjdk:17-jdk-slim localhost:5000/openjdk:17-jdk-slim
+docker tag python:3.11-slim localhost:5000/python:3.11-slim
+docker tag node:20-slim localhost:5000/node:20-slim
+docker tag docker:24-dind localhost:5000/docker:24-dind
+docker tag alpine:3.19 localhost:5000/alpine:3.19
+```
+
+#### 3.2. Update Config to Use Local Images
+
+```yaml
+labels:
+  - "java:docker://localhost:5000/openjdk:17-jdk-slim"
+  - "python:docker://localhost:5000/python:3.11-slim"
+  - "nodejs:docker://localhost:5000/node:20-slim"
+```
+
+### Solution 4: Reduce Image Pulls
+
+#### 4.1. Disable Force Pull
+
+Update all config files:
+```yaml
+container:
+  # Don't pull if image already exists
+  force_pull: false
+```
+
+#### 4.2. Use Image Caching
+
+```yaml
+container:
+  # Enable image caching
+  force_pull: false
+  force_rebuild: false
+```
+
+### Solution 5: Use Self-Hosted Registry
+
+#### 5.1. Set up Local Registry
+
+```bash
+# Run local Docker registry
+docker run -d -p 5000:5000 --name registry registry:2
+
+# Mirror images to local registry
+docker pull openjdk:17-jdk-slim
+docker tag openjdk:17-jdk-slim localhost:5000/openjdk:17-jdk-slim
+docker push localhost:5000/openjdk:17-jdk-slim
+```
+
+#### 5.2. Update Configs to Use Local Registry
+
+```yaml
+labels:
+  - "java:docker://localhost:5000/openjdk:17-jdk-slim"
+```
+
+## Recommended Approach
+
+**For immediate fix**: Use Solution 1 (Docker Hub authentication)
+**For long-term**: Combine Solutions 1 + 4 (auth + caching)
+
+## Implementation Steps
+
+1. **Create Docker Hub account** (if you don't have one)
+2. **Update `.env.runners`** with credentials
+3. **Update all config files** with authentication
+4. **Set `force_pull: false`** to reduce pulls
+5. **Test with a simple job**
+
+## Verification
+
+After implementing, test with:
+```bash
+# Check if authentication works
+docker login
+docker pull openjdk:17-jdk-slim
+```
+
+## References
+
+- [Docker Hub Rate Limits](https://www.docker.com/increase-rate-limit)
+- [Gitea Actions Documentation](https://docs.gitea.com/usage/actions/design#act-runner)
+- [Docker Registry Authentication](https://docs.docker.com/engine/reference/commandline/login/)
--- a/docs/structure.txt
+++ b/docs/structure.txt
@@ -118,4 +118,5 @@ runners/
    ├── RUNNERS.md            # Gitea runners setup and management
    ├── RUNNER_LABELS.md      # Runner labels technical documentation
    ├── OPTIMIZATION_RECOMMENDATIONS.md # CI/CD optimization recommendations
+    ├── DOCKER_RATE_LIMIT_FIX.md # Docker Hub rate limit solutions
    └── CI_CD.md              # CI/CD pipeline documentation
--- a/runners/config_docker.yaml
+++ b/runners/config_docker.yaml
@@ -98,8 +98,11 @@ container:
  # If it's "-", act_runner will find an available docker host automatically, but the docker host won't be mounted to the job containers and service containers.
  # If it's not empty or "-", the specified docker host will be used. An error will be returned if it doesn't work.
  docker_host: ""
+  # Docker registry authentication to avoid rate limits
+  docker_username: gschrooyen
+  docker_password: ${DOCKER_PASSWORD}
  # Pull docker image(s) even if already present
-  force_pull: true
+  force_pull: false
  # Rebuild docker image(s) even if already present
  force_rebuild: false
  # Always require a reachable docker daemon, even if not required by act_runner
--- a/runners/config_heavy.yaml
+++ b/runners/config_heavy.yaml
@@ -99,8 +99,11 @@ container:
  # If it's "-", act_runner will find an available docker host automatically, but the docker host won't be mounted to the job containers and service containers.
  # If it's not empty or "-", the specified docker host will be used. An error will be returned if it doesn't work.
  docker_host: ""
+  # Docker registry authentication to avoid rate limits
+  docker_username: gschrooyen
+  docker_password: ${DOCKER_PASSWORD}
  # Pull docker image(s) even if already present
-  force_pull: true
+  force_pull: false
  # Rebuild docker image(s) even if already present
  force_rebuild: false
  # Always require a reachable docker daemon, even if not required by act_runner
--- a/runners/config_light.yaml
+++ b/runners/config_light.yaml
@@ -99,8 +99,11 @@ container:
  # If it's "-", act_runner will find an available docker host automatically, but the docker host won't be mounted to the job containers and service containers.
  # If it's not empty or "-", the specified docker host will be used. An error will be returned if it doesn't work.
  docker_host: ""
+  # Docker registry authentication to avoid rate limits
+  docker_username: gschrooyen
+  docker_password: ${DOCKER_PASSWORD}
  # Pull docker image(s) even if already present
-  force_pull: true
+  force_pull: false
  # Rebuild docker image(s) even if already present
  force_rebuild: false
  # Always require a reachable docker daemon, even if not required by act_runner
--- a/runners/config_security.yaml
+++ b/runners/config_security.yaml
@@ -98,8 +98,11 @@ container:
  # If it's "-", act_runner will find an available docker host automatically, but the docker host won't be mounted to the job containers and service containers.
  # If it's not empty or "-", the specified docker host will be used. An error will be returned if it doesn't work.
  docker_host: ""
+  # Docker registry authentication to avoid rate limits
+  docker_username: gschrooyen
+  docker_password: ${DOCKER_PASSWORD}
  # Pull docker image(s) even if already present
-  force_pull: true
+  force_pull: false
  # Rebuild docker image(s) even if already present
  force_rebuild: false
  # Always require a reachable docker daemon, even if not required by act_runner
--- a/runners/env.runners.example
+++ b/runners/env.runners.example
@@ -12,3 +12,8 @@ GITEA_RUNNER_TOKEN=your_runner_registration_token_here
 # GITEA_RUNNER_NAME_LIGHT=labfusion-runner-light
 # GITEA_RUNNER_NAME_DOCKER=labfusion-runner-docker
 # GITEA_RUNNER_NAME_SECURITY=labfusion-runner-security
+
+# Docker Hub credentials (to avoid rate limits)
+# Create a free Docker Hub account at https://hub.docker.com
+DOCKER_USERNAME=your_dockerhub_username
+DOCKER_PASSWORD=your_dockerhub_password
--- a/runners/manage-runners.ps1
+++ b/runners/manage-runners.ps1
@@ -0,0 +1,208 @@
+# LabFusion Gitea Runners Management Script (PowerShell)
+# Usage: .\scripts\manage-runners.ps1 [start|stop|restart|status|logs|clean]
+
+param(
+    [Parameter(Position=0)]
+    [ValidateSet("start", "stop", "restart", "status", "logs", "clean", "help")]
+    [string]$Command = "help",
+    
+    [Parameter(Position=1)]
+    [string]$RunnerName = ""
+)
+
+$ErrorActionPreference = "Stop"
+
+$ComposeFile = "docker-compose.runners.yml"
+$EnvFile = ".env.runners"
+
+# Helper functions
+function Write-Info {
+    param([string]$Message)
+    Write-Host "[INFO] $Message" -ForegroundColor Blue
+}
+
+function Write-Success {
+    param([string]$Message)
+    Write-Host "[SUCCESS] $Message" -ForegroundColor Green
+}
+
+function Write-Warning {
+    param([string]$Message)
+    Write-Host "[WARNING] $Message" -ForegroundColor Yellow
+}
+
+function Write-Error {
+    param([string]$Message)
+    Write-Host "[ERROR] $Message" -ForegroundColor Red
+}
+
+# Check if .env.runners exists
+function Test-EnvFile {
+    if (-not (Test-Path $EnvFile)) {
+        Write-Error "Environment file $EnvFile not found!"
+        Write-Info "Please copy env.runners.example to $EnvFile and configure it:"
+        Write-Info "  Copy-Item env.runners.example $EnvFile"
+        Write-Info "  # Edit $EnvFile with your Gitea instance URL and runner token"
+        exit 1
+    }
+}
+
+# Start runners
+function Start-Runners {
+    Write-Info "Starting LabFusion Gitea runners..."
+    Test-EnvFile
+    
+    docker-compose -f $ComposeFile --env-file $EnvFile up -d
+    
+    Write-Success "Runners started successfully!"
+    Write-Info "Runners:"
+    Write-Info "  - Heavy (Java/Python): labfusion-runner-heavy"
+    Write-Info "  - Light (Node.js/Frontend): labfusion-runner-light"
+    Write-Info "  - Docker (Integration): labfusion-runner-docker"
+    Write-Info "  - Security (Scans): labfusion-runner-security"
+    
+    # Wait a moment for health checks
+    Start-Sleep -Seconds 5
+    Show-Status
+}
+
+# Stop runners
+function Stop-Runners {
+    Write-Info "Stopping LabFusion Gitea runners..."
+    
+    docker-compose -f $ComposeFile down
+    
+    Write-Success "Runners stopped successfully!"
+}
+
+# Restart runners
+function Restart-Runners {
+    Write-Info "Restarting LabFusion Gitea runners..."
+    Stop-Runners
+    Start-Sleep -Seconds 2
+    Start-Runners
+}
+
+# Show runner status
+function Show-Status {
+    Write-Info "LabFusion Gitea Runners Status:"
+    Write-Host ""
+    
+    # Check if compose file exists
+    if (-not (Test-Path $ComposeFile)) {
+        Write-Error "Docker Compose file $ComposeFile not found!"
+        exit 1
+    }
+    
+    # Show container status
+    docker-compose -f $ComposeFile ps
+    
+    Write-Host ""
+    Write-Info "Runner Health Status:"
+    
+    # Check each runner's health
+    $runners = @("labfusion-runner-heavy", "labfusion-runner-light", "labfusion-runner-docker", "labfusion-runner-security")
+    
+    foreach ($runner in $runners) {
+        $container = docker ps --format "table {{.Names}}\t{{.Status}}" | Select-String $runner
+        if ($container) {
+            try {
+                $health = docker inspect --format='{{.State.Health.Status}}' $runner 2>$null
+                if ($health) {
+                    switch ($health) {
+                        "healthy" { Write-Success "$runner`: $health" }
+                        "unhealthy" { Write-Error "$runner`: $health" }
+                        "starting" { Write-Warning "$runner`: $health" }
+                        default { Write-Warning "$runner`: $health" }
+                    }
+                } else {
+                    Write-Warning "$runner`: health check not available"
+                }
+            } catch {
+                Write-Warning "$runner`: health check failed"
+            }
+        } else {
+            Write-Error "$runner`: not running"
+        }
+    }
+}
+
+# Show logs
+function Show-Logs {
+    param([string]$Runner = "")
+    
+    if ($Runner) {
+        Write-Info "Showing logs for $Runner..."
+        docker-compose -f $ComposeFile logs -f $Runner
+    } else {
+        Write-Info "Showing logs for all runners..."
+        docker-compose -f $ComposeFile logs -f
+    }
+}
+
+# Clean up runners and data
+function Clean-Runners {
+    $response = Read-Host "This will remove all runners and their data. Are you sure? (y/N)"
+    
+    if ($response -match "^[Yy]$") {
+        Write-Info "Cleaning up runners and data..."
+        
+        # Stop and remove containers
+        docker-compose -f $ComposeFile down -v
+        
+        # Remove volumes
+        try {
+            $volumes = docker volume ls -q | Where-Object { $_ -match "runner-data" }
+            if ($volumes) {
+                docker volume rm $volumes
+            }
+            docker volume rm shared-cache 2>$null
+        } catch {
+            Write-Warning "Some volumes could not be removed (this is normal if they don't exist)"
+        }
+        
+        Write-Success "Cleanup completed!"
+    } else {
+        Write-Info "Cleanup cancelled."
+    }
+}
+
+# Show help
+function Show-Help {
+    Write-Host "LabFusion Gitea Runners Management Script (PowerShell)"
+    Write-Host ""
+    Write-Host "Usage: .\scripts\manage-runners.ps1 [COMMAND] [RUNNER_NAME]"
+    Write-Host ""
+    Write-Host "Commands:"
+    Write-Host "  start     Start all runners"
+    Write-Host "  stop      Stop all runners"
+    Write-Host "  restart   Restart all runners"
+    Write-Host "  status    Show runner status and health"
+    Write-Host "  logs      Show logs for all runners"
+    Write-Host "  logs [runner]  Show logs for specific runner"
+    Write-Host "  clean     Remove all runners and data (destructive)"
+    Write-Host "  help      Show this help message"
+    Write-Host ""
+    Write-Host "Examples:"
+    Write-Host "  .\scripts\manage-runners.ps1 start"
+    Write-Host "  .\scripts\manage-runners.ps1 status"
+    Write-Host "  .\scripts\manage-runners.ps1 logs labfusion-runner-heavy"
+    Write-Host "  .\scripts\manage-runners.ps1 clean"
+}
+
+# Main script logic
+switch ($Command) {
+    "start" { Start-Runners }
+    "stop" { Stop-Runners }
+    "restart" { Restart-Runners }
+    "status" { Show-Status }
+    "logs" { Show-Logs -Runner $RunnerName }
+    "clean" { Clean-Runners }
+    "help" { Show-Help }
+    default {
+        Write-Error "Unknown command: $Command"
+        Write-Host ""
+        Show-Help
+        exit 1
+    }
+}
--- a/runners/manage-runners.sh
+++ b/runners/manage-runners.sh
@@ -0,0 +1,209 @@
+#!/bin/bash
+
+# LabFusion Gitea Runners Management Script
+# Usage: ./scripts/manage-runners.sh [start|stop|restart|status|logs|clean]
+
+set -e
+
+COMPOSE_FILE="docker-compose.runners.yml"
+ENV_FILE=".env.runners"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Helper functions
+log_info() {
+    echo -e "${BLUE}[INFO]${NC} $1"
+}
+
+log_success() {
+    echo -e "${GREEN}[SUCCESS]${NC} $1"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARNING]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+# Check if .env.runners exists
+check_env_file() {
+    if [ ! -f "$ENV_FILE" ]; then
+        log_error "Environment file $ENV_FILE not found!"
+        log_info "Please copy env.runners.example to $ENV_FILE and configure it:"
+        log_info "  cp env.runners.example $ENV_FILE"
+        log_info "  # Edit $ENV_FILE with your Gitea instance URL and runner token"
+        exit 1
+    fi
+}
+
+# Start runners
+start_runners() {
+    log_info "Starting LabFusion Gitea runners..."
+    check_env_file
+    
+    docker-compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" up -d
+    
+    log_success "Runners started successfully!"
+    log_info "Runners:"
+    log_info "  - Heavy (Java/Python): labfusion-runner-heavy"
+    log_info "  - Light (Node.js/Frontend): labfusion-runner-light"
+    log_info "  - Docker (Integration): labfusion-runner-docker"
+    log_info "  - Security (Scans): labfusion-runner-security"
+    
+    # Wait a moment for health checks
+    sleep 5
+    show_status
+}
+
+# Stop runners
+stop_runners() {
+    log_info "Stopping LabFusion Gitea runners..."
+    
+    docker-compose -f "$COMPOSE_FILE" down
+    
+    log_success "Runners stopped successfully!"
+}
+
+# Restart runners
+restart_runners() {
+    log_info "Restarting LabFusion Gitea runners..."
+    stop_runners
+    sleep 2
+    start_runners
+}
+
+# Show runner status
+show_status() {
+    log_info "LabFusion Gitea Runners Status:"
+    echo
+    
+    # Check if compose file exists
+    if [ ! -f "$COMPOSE_FILE" ]; then
+        log_error "Docker Compose file $COMPOSE_FILE not found!"
+        exit 1
+    fi
+    
+    # Show container status
+    docker-compose -f "$COMPOSE_FILE" ps
+    
+    echo
+    log_info "Runner Health Status:"
+    
+    # Check each runner's health
+    for runner in labfusion-runner-heavy labfusion-runner-light labfusion-runner-docker labfusion-runner-security; do
+        if docker ps --format "table {{.Names}}\t{{.Status}}" | grep -q "$runner"; then
+            status=$(docker inspect --format='{{.State.Health.Status}}' "$runner" 2>/dev/null || echo "unknown")
+            case $status in
+                "healthy")
+                    log_success "$runner: $status"
+                    ;;
+                "unhealthy")
+                    log_error "$runner: $status"
+                    ;;
+                "starting")
+                    log_warning "$runner: $status"
+                    ;;
+                *)
+                    log_warning "$runner: $status"
+                    ;;
+            esac
+        else
+            log_error "$runner: not running"
+        fi
+    done
+}
+
+# Show logs
+show_logs() {
+    local runner=${2:-""}
+    
+    if [ -n "$runner" ]; then
+        log_info "Showing logs for $runner..."
+        docker-compose -f "$COMPOSE_FILE" logs -f "$runner"
+    else
+        log_info "Showing logs for all runners..."
+        docker-compose -f "$COMPOSE_FILE" logs -f
+    fi
+}
+
+# Clean up runners and data
+clean_runners() {
+    log_warning "This will remove all runners and their data. Are you sure? (y/N)"
+    read -r response
+    
+    if [[ "$response" =~ ^[Yy]$ ]]; then
+        log_info "Cleaning up runners and data..."
+        
+        # Stop and remove containers
+        docker-compose -f "$COMPOSE_FILE" down -v
+        
+        # Remove volumes
+        docker volume rm $(docker volume ls -q | grep runner-data) 2>/dev/null || true
+        docker volume rm shared-cache 2>/dev/null || true
+        
+        log_success "Cleanup completed!"
+    else
+        log_info "Cleanup cancelled."
+    fi
+}
+
+# Show help
+show_help() {
+    echo "LabFusion Gitea Runners Management Script"
+    echo
+    echo "Usage: $0 [COMMAND]"
+    echo
+    echo "Commands:"
+    echo "  start     Start all runners"
+    echo "  stop      Stop all runners"
+    echo "  restart   Restart all runners"
+    echo "  status    Show runner status and health"
+    echo "  logs      Show logs for all runners"
+    echo "  logs [runner]  Show logs for specific runner"
+    echo "  clean     Remove all runners and data (destructive)"
+    echo "  help      Show this help message"
+    echo
+    echo "Examples:"
+    echo "  $0 start"
+    echo "  $0 status"
+    echo "  $0 logs labfusion-runner-heavy"
+    echo "  $0 clean"
+}
+
+# Main script logic
+case "${1:-help}" in
+    start)
+        start_runners
+        ;;
+    stop)
+        stop_runners
+        ;;
+    restart)
+        restart_runners
+        ;;
+    status)
+        show_status
+        ;;
+    logs)
+        show_logs "$@"
+        ;;
+    clean)
+        clean_runners
+        ;;
+    help|--help|-h)
+        show_help
+        ;;
+    *)
+        log_error "Unknown command: $1"
+        echo
+        show_help
+        exit 1
+        ;;
+esac