diff --git a/.gitea/workflows/api-docs.yml b/.gitea/workflows/api-docs.yml index ba97f38..0c2abe3 100644 --- a/.gitea/workflows/api-docs.yml +++ b/.gitea/workflows/api-docs.yml @@ -101,15 +101,27 @@ jobs: - name: Run tests run: | - npm test -- --coverage --watchAll=false + npm test -- --coverage --watchAll=false --coverageReporters=lcov --coverageReporters=text --coverageReporters=html npm run test:coverage - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./services/api-docs/coverage/lcov.info - flags: api-docs - name: api-docs-coverage + - name: Send results to SonarQube + run: | + echo "Sending API Docs results to SonarQube..." + # Install SonarQube Scanner for Node.js + npm install -g sonar-scanner + + # Run SonarQube analysis + sonar-scanner \ + -Dsonar.host.url=${{ secrets.SONAR_HOST_URL }} \ + -Dsonar.login=${{ secrets.SONAR_TOKEN }} \ + -Dsonar.projectKey=labFusion \ + -Dsonar.projectName="LabFusion" \ + -Dsonar.projectVersion=1.0.0 \ + -Dsonar.modules=api-docs \ + -Dsonar.sources=. \ + -Dsonar.tests=__tests__ \ + -Dsonar.javascript.lcov.reportPaths=coverage/lcov.info \ + -Dsonar.testExecutionReportPaths=test-results.xml - name: Test results summary if: always() diff --git a/.gitea/workflows/api-gateway.yml b/.gitea/workflows/api-gateway.yml index 2f20189..20da5b8 100644 --- a/.gitea/workflows/api-gateway.yml +++ b/.gitea/workflows/api-gateway.yml @@ -125,13 +125,40 @@ jobs: echo "TEST_REPORTS_EXIST=false" >> $GITHUB_ENV fi - - name: Generate test report - uses: dorny/test-reporter@v1 + - name: Send test results to SonarQube if: env.TEST_REPORTS_EXIST == 'true' - with: - name: Maven Tests (Java ${{ matrix.java-version }}) - path: target/surefire-reports/*.xml - reporter: java-junit + run: | + echo "Sending test results to SonarQube..." + # Configure SonarQube properties + cat > sonar-project.properties << EOF + sonar.projectKey=labFusion + sonar.projectName=LabFusion + sonar.projectVersion=1.0.0 + sonar.modules=api-gateway + sonar.sources=src/main/java + sonar.tests=src/test/java + sonar.java.binaries=target/classes + sonar.java.test.binaries=target/test-classes + sonar.junit.reportPaths=target/surefire-reports + sonar.coverage.jacoco.xmlReportPaths=target/site/jacoco/jacoco.xml + sonar.host.url=${{ secrets.SONAR_HOST_URL }} + sonar.login=${{ secrets.SONAR_TOKEN }} + EOF + + # Run SonarQube analysis + ./mvnw sonar:sonar \ + -Dsonar.host.url=${{ secrets.SONAR_HOST_URL }} \ + -Dsonar.login=${{ secrets.SONAR_TOKEN }} \ + -Dsonar.projectKey=labFusion \ + -Dsonar.projectName="LabFusion" \ + -Dsonar.projectVersion=1.0.0 \ + -Dsonar.modules=api-gateway \ + -Dsonar.sources=src/main/java \ + -Dsonar.tests=src/test/java \ + -Dsonar.java.binaries=target/classes \ + -Dsonar.java.test.binaries=target/test-classes \ + -Dsonar.junit.reportPaths=target/surefire-reports \ + -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco/jacoco.xml - name: Fail if no test reports found if: env.TEST_REPORTS_EXIST == 'false' @@ -152,14 +179,10 @@ jobs: ./mvnw pmd:check - name: Generate code coverage - run: ./mvnw jacoco:report - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./services/api-gateway/target/site/jacoco/jacoco.xml - flags: api-gateway - name: api-gateway-coverage + run: | + echo "Generating JaCoCo code coverage report..." + ./mvnw jacoco:report + echo "Code coverage report generated at target/site/jacoco/jacoco.xml" build: runs-on: [self-hosted] diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 07cd734..0d74a74 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -49,6 +49,26 @@ jobs: - name: Run tests run: ./mvnw test + - name: Generate code coverage + run: ./mvnw jacoco:report + + - name: Send results to SonarQube + run: | + echo "Sending API Gateway results to SonarQube..." + ./mvnw sonar:sonar \ + -Dsonar.host.url=${{ secrets.SONAR_HOST_URL }} \ + -Dsonar.login=${{ secrets.SONAR_TOKEN }} \ + -Dsonar.projectKey=labFusion \ + -Dsonar.projectName="LabFusion" \ + -Dsonar.projectVersion=1.0.0 \ + -Dsonar.modules=api-gateway \ + -Dsonar.sources=src/main/java \ + -Dsonar.tests=src/test/java \ + -Dsonar.java.binaries=target/classes \ + -Dsonar.java.test.binaries=target/test-classes \ + -Dsonar.junit.reportPaths=target/surefire-reports \ + -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco/jacoco.xml + - name: Run code quality checks run: ./mvnw spotbugs:check checkstyle:check @@ -104,13 +124,26 @@ jobs: - name: Run tests run: | - pytest --cov=. --cov-report=xml --cov-report=html + pytest --cov=. --cov-report=xml --cov-report=html --junitxml=tests/reports/junit.xml - - name: Upload coverage reports - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: service-adapters + - name: Send results to SonarQube + run: | + echo "Sending Service Adapters results to SonarQube..." + # Install SonarQube Scanner for Python + pip install sonar-scanner + + # Run SonarQube analysis + sonar-scanner \ + -Dsonar.host.url=${{ secrets.SONAR_HOST_URL }} \ + -Dsonar.login=${{ secrets.SONAR_TOKEN }} \ + -Dsonar.projectKey=labFusion \ + -Dsonar.projectName="LabFusion" \ + -Dsonar.projectVersion=1.0.0 \ + -Dsonar.modules=service-adapters \ + -Dsonar.sources=. \ + -Dsonar.tests=tests \ + -Dsonar.python.coverage.reportPaths=coverage.xml \ + -Dsonar.python.xunit.reportPath=tests/reports/junit.xml - name: Build Docker image (test only) run: docker build -t service-adapters:test . @@ -159,7 +192,26 @@ jobs: run: npm run lint - name: Run tests - run: npm test + run: npm test -- --coverage --coverageReporters=lcov --coverageReporters=text --coverageReporters=html + + - name: Send results to SonarQube + run: | + echo "Sending API Docs results to SonarQube..." + # Install SonarQube Scanner for Node.js + npm install -g sonar-scanner + + # Run SonarQube analysis + sonar-scanner \ + -Dsonar.host.url=${{ secrets.SONAR_HOST_URL }} \ + -Dsonar.login=${{ secrets.SONAR_TOKEN }} \ + -Dsonar.projectKey=labFusion \ + -Dsonar.projectName="LabFusion" \ + -Dsonar.projectVersion=1.0.0 \ + -Dsonar.modules=api-docs \ + -Dsonar.sources=. \ + -Dsonar.tests=__tests__ \ + -Dsonar.javascript.lcov.reportPaths=coverage/lcov.info \ + -Dsonar.testExecutionReportPaths=test-results.xml - name: Build application run: npm run build @@ -211,7 +263,26 @@ jobs: run: npm run lint - name: Run tests - run: npm test -- --coverage --watchAll=false + run: npm test -- --coverage --watchAll=false --coverageReporters=lcov --coverageReporters=text --coverageReporters=html + + - name: Send results to SonarQube + run: | + echo "Sending Frontend results to SonarQube..." + # Install SonarQube Scanner for Node.js + npm install -g sonar-scanner + + # Run SonarQube analysis + sonar-scanner \ + -Dsonar.host.url=${{ secrets.SONAR_HOST_URL }} \ + -Dsonar.login=${{ secrets.SONAR_TOKEN }} \ + -Dsonar.projectKey=labFusion \ + -Dsonar.projectName="LabFusion" \ + -Dsonar.projectVersion=1.0.0 \ + -Dsonar.modules=frontend \ + -Dsonar.sources=src \ + -Dsonar.tests=src \ + -Dsonar.javascript.lcov.reportPaths=coverage/lcov.info \ + -Dsonar.testExecutionReportPaths=test-results.xml - name: Build application run: npm run build diff --git a/.gitea/workflows/frontend.yml b/.gitea/workflows/frontend.yml index 4a43605..887568c 100644 --- a/.gitea/workflows/frontend.yml +++ b/.gitea/workflows/frontend.yml @@ -73,15 +73,27 @@ jobs: - name: Run tests run: | - npm test -- --coverage --watchAll=false --passWithNoTests + npm test -- --coverage --watchAll=false --passWithNoTests --coverageReporters=lcov --coverageReporters=text --coverageReporters=html npm run test:coverage - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./frontend/coverage/lcov.info - flags: frontend - name: frontend-coverage + - name: Send results to SonarQube + run: | + echo "Sending Frontend results to SonarQube..." + # Install SonarQube Scanner for Node.js + npm install -g sonar-scanner + + # Run SonarQube analysis + sonar-scanner \ + -Dsonar.host.url=${{ secrets.SONAR_HOST_URL }} \ + -Dsonar.login=${{ secrets.SONAR_TOKEN }} \ + -Dsonar.projectKey=labFusion \ + -Dsonar.projectName="LabFusion" \ + -Dsonar.projectVersion=1.0.0 \ + -Dsonar.modules=frontend \ + -Dsonar.sources=src \ + -Dsonar.tests=src \ + -Dsonar.javascript.lcov.reportPaths=coverage/lcov.info \ + -Dsonar.testExecutionReportPaths=test-results.xml - name: Test results summary if: always() diff --git a/.gitea/workflows/service-adapters.yml b/.gitea/workflows/service-adapters.yml index 15574d9..92179e9 100644 --- a/.gitea/workflows/service-adapters.yml +++ b/.gitea/workflows/service-adapters.yml @@ -85,15 +85,28 @@ jobs: - name: Run tests run: | - pytest --cov=. --cov-report=xml --cov-report=html --cov-report=term-missing + pytest --cov=. --cov-report=xml --cov-report=html --cov-report=term-missing --junitxml=tests/reports/junit.xml pytest --cov=. --cov-report=xml --cov-report=html --cov-report=term-missing --cov-fail-under=80 - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./services/service-adapters/coverage.xml - flags: service-adapters - name: service-adapters-coverage + - name: Send results to SonarQube + run: | + echo "Sending Service Adapters results to SonarQube..." + # Install SonarQube Scanner for Python + pip install sonar-scanner + + # Run SonarQube analysis + sonar-scanner \ + -Dsonar.host.url=${{ secrets.SONAR_HOST_URL }} \ + -Dsonar.login=${{ secrets.SONAR_TOKEN }} \ + -Dsonar.projectKey=labFusion \ + -Dsonar.projectName="LabFusion" \ + -Dsonar.projectVersion=1.0.0 \ + -Dsonar.modules=service-adapters \ + -Dsonar.sources=. \ + -Dsonar.tests=tests \ + -Dsonar.python.coverage.reportPaths=coverage.xml \ + -Dsonar.python.xunit.reportPath=tests/reports/junit.xml + - name: Test results summary if: always() diff --git a/docs/SONARQUBE_INTEGRATION.md b/docs/SONARQUBE_INTEGRATION.md new file mode 100644 index 0000000..d6cb237 --- /dev/null +++ b/docs/SONARQUBE_INTEGRATION.md @@ -0,0 +1,239 @@ +# SonarQube Integration for LabFusion + +This document explains how to configure SonarQube integration for all LabFusion services in a unified project. + +## Overview + +All LabFusion services (API Gateway, Service Adapters, API Docs, Frontend) now send test results, code coverage, and quality metrics directly to a single unified SonarQube project called "LabFusion" instead of using external test reporters. + +## Required Configuration + +### 1. SonarQube Secrets + +You need to configure the following secrets in your Gitea repository: + +- `SONAR_HOST_URL`: Your SonarQube server URL (e.g., `http://localhost:9000` or `https://sonar.yourdomain.com`) +- `SONAR_TOKEN`: Your SonarQube authentication token + +### 2. SonarQube Project Setup + +1. **Create a unified project** in SonarQube: + - Project Key: `labfusion` + - Project Name: `LabFusion` + - Main Branch: `main` + +2. **Generate an authentication token**: + - Go to User > My Account > Security + - Generate a new token with appropriate permissions + - Copy the token for use in `SONAR_TOKEN` secret + +### 3. SonarQube Quality Gates + +Configure quality gates in SonarQube to enforce: +- Minimum code coverage percentage +- Maximum code duplication percentage +- Maximum technical debt ratio +- Code smell thresholds + +## What Gets Sent to SonarQube + +### Unified LabFusion Project Structure +- **Project Key**: `labfusion` +- **Project Name**: `LabFusion` +- **Modules**: + - `api-gateway` (Java Spring Boot) + - `service-adapters` (Python FastAPI) + - `api-docs` (Node.js Express) + - `frontend` (React) + +### Test Results +- **API Gateway**: JUnit XML reports from `target/surefire-reports/` +- **Service Adapters**: pytest XML reports from `tests/reports/junit.xml` +- **API Docs**: Jest XML reports from `test-results.xml` +- **Frontend**: Jest XML reports from `test-results.xml` + +### Code Coverage +- **API Gateway**: JaCoCo XML report from `target/site/jacoco/jacoco.xml` +- **Service Adapters**: Coverage XML from `coverage.xml` +- **API Docs**: LCOV report from `coverage/lcov.info` +- **Frontend**: LCOV report from `coverage/lcov.info` + +### Code Quality Metrics +- **Source code analysis** results for all languages +- **Code smells** and issues across all services +- **Security vulnerabilities** detection +- **Maintainability ratings** per module + +## Pipeline Integration + +### All Services Send to Unified Project +Each service workflow includes a SonarQube integration step: + +#### API Gateway (Java) +```yaml +- name: Send test results to SonarQube + run: | + ./mvnw sonar:sonar \ + -Dsonar.projectKey=labfusion \ + -Dsonar.modules=api-gateway \ + # ... other properties +``` + +#### Service Adapters (Python) +```yaml +- name: Send results to SonarQube + run: | + sonar-scanner \ + -Dsonar.projectKey=labfusion \ + -Dsonar.modules=service-adapters \ + # ... other properties +``` + +#### API Docs & Frontend (Node.js) +```yaml +- name: Send results to SonarQube + run: | + sonar-scanner \ + -Dsonar.projectKey=labfusion \ + -Dsonar.modules=api-docs \ + # ... other properties +``` + +## Maven Plugins Added + +### SonarQube Maven Plugin +```xml + + org.sonarsource.scanner.maven + sonar-maven-plugin + 3.10.0.2594 + +``` + +### JaCoCo Maven Plugin +```xml + + org.jacoco + jacoco-maven-plugin + 0.8.11 + + +``` + +## SonarQube Properties + +Each service generates its own `sonar-project.properties` with module-specific settings: + +### API Gateway +```properties +sonar.projectKey=labfusion +sonar.projectName=LabFusion +sonar.projectVersion=1.0.0 +sonar.modules=api-gateway +sonar.sources=src/main/java +sonar.tests=src/test/java +sonar.java.binaries=target/classes +sonar.java.test.binaries=target/test-classes +sonar.junit.reportPaths=target/surefire-reports +sonar.coverage.jacoco.xmlReportPaths=target/site/jacoco/jacoco.xml +``` + +### Service Adapters +```properties +sonar.projectKey=labfusion +sonar.projectName=LabFusion +sonar.projectVersion=1.0.0 +sonar.modules=service-adapters +sonar.sources=. +sonar.tests=tests +sonar.python.coverage.reportPaths=coverage.xml +sonar.python.xunit.reportPath=tests/reports/junit.xml +``` + +### API Docs & Frontend +```properties +sonar.projectKey=labfusion +sonar.projectName=LabFusion +sonar.projectVersion=1.0.0 +sonar.modules=api-docs +sonar.sources=. +sonar.tests=__tests__ +sonar.javascript.lcov.reportPaths=coverage/lcov.info +sonar.testExecutionReportPaths=test-results.xml +``` + +## Benefits + +### 1. Centralized Quality Management +- All quality metrics in one place +- Historical trend analysis +- Cross-project comparisons + +### 2. Automated Quality Gates +- Pipeline fails if quality standards not met +- Enforces consistent code quality +- Prevents regression in code quality + +### 3. Detailed Reporting +- Comprehensive test coverage reports +- Code smell identification +- Security vulnerability detection +- Technical debt tracking + +### 4. Integration Benefits +- No external service dependencies +- Local data control +- Customizable quality rules +- Team collaboration features + +## Troubleshooting + +### Common Issues + +1. **Authentication Failed** + - Verify `SONAR_TOKEN` is correct + - Check token permissions in SonarQube + - Ensure token hasn't expired + +2. **Connection Refused** + - Verify `SONAR_HOST_URL` is accessible + - Check network connectivity + - Ensure SonarQube is running + +3. **Project Not Found** + - Create project in SonarQube first + - Verify project key matches configuration + - Check project permissions + +4. **No Test Results** + - Ensure test files exist in `src/test/java/` + - Verify Maven Surefire plugin configuration + - Check test execution logs + +### Debug Commands + +```bash +# Test SonarQube connection +curl -u $SONAR_TOKEN: $SONAR_HOST_URL/api/system/status + +# Check project exists +curl -u $SONAR_TOKEN: $SONAR_HOST_URL/api/projects/search?q=labfusion-api-gateway + +# Verify test reports exist +ls -la target/surefire-reports/ +ls -la target/site/jacoco/ +``` + +## Next Steps + +1. **Configure SonarQube secrets** in your Gitea repository +2. **Set up quality gates** in SonarQube +3. **Run the pipeline** to test integration +4. **Review results** in SonarQube dashboard +5. **Customize quality rules** as needed + +## References + +- [SonarQube Documentation](https://docs.sonarqube.org/) +- [SonarQube Maven Plugin](https://docs.sonarqube.org/latest/analysis/scan/sonarscanner-for-maven/) +- [JaCoCo Maven Plugin](https://www.jacoco.org/jacoco/trunk/doc/maven.html) diff --git a/services/api-gateway/pom.xml b/services/api-gateway/pom.xml index 9a062c0..7631087 100644 --- a/services/api-gateway/pom.xml +++ b/services/api-gateway/pom.xml @@ -129,6 +129,34 @@ target/surefire-reports + + + + org.sonarsource.scanner.maven + sonar-maven-plugin + 3.10.0.2594 + + + + + org.jacoco + jacoco-maven-plugin + 0.8.11 + + + + prepare-agent + + + + report + test + + report + + + + diff --git a/services/service-adapters/requirements.txt b/services/service-adapters/requirements.txt index 9735681..64e0c73 100644 --- a/services/service-adapters/requirements.txt +++ b/services/service-adapters/requirements.txt @@ -12,3 +12,22 @@ passlib[bcrypt]==1.7.4 python-dotenv==1.0.0 websockets==12.0 aiofiles==23.2.1 + +# Testing and Quality +pytest==7.4.3 +pytest-cov==4.1.0 +pytest-asyncio==0.21.1 +pytest-html==4.1.1 +pytest-xdist==3.3.1 +coverage==7.3.2 + +# Code Quality +flake8==6.1.0 +black==23.11.0 +isort==5.12.0 +mypy==1.7.1 +bandit==1.7.5 +safety==2.3.5 + +# SonarQube Integration +sonar-scanner==4.8.0.2856 \ No newline at end of file