Part 2: Industrialize and secure an advanced Python CI/CD pipeline

@startuml
!theme plain
title Python CI/CD pipeline to PyPI
skinparam backgroundColor #f8f9fa
skinparam componentStyle rectangle

rectangle "Developer" as dev
rectangle "GitHub Repository" as repo {
  rectangle ".github/workflows/" as workflows
  rectangle "tests/" as tests
  rectangle "setup.py / pyproject.toml" as setup
  rectangle "requirements.txt" as req
}

rectangle "GitHub Actions" as actions {
  rectangle "Test Runner" as test_runner
  rectangle "Build" as build
  rectangle "Security Scan" as security
  rectangle "(No output, as there is no French text provided to translate.)" as quality
}

rectangle "PyPI" as pypi {
  rectangle "Test PyPI" as test_pypi
  rectangle "Production PyPI" as prod_pypi
}

rectangle "Users" as users

dev --> repo : push/PR
repo --> actions : trigger workflow
actions --> test_runner : run tests
actions --> security : security checks
actions --> quality : code quality
actions --> build : build package
build --> test_pypi : deploy (pre-release)
build --> prod_pypi : deploy (release)
prod_pypi --> users : install package

@enduml

playlist-downloader/
├── .github/
│   └── workflows/
│       ├── ci.yml
│       ├── release.yml
│       └── security.yml
├── src/
│   └── playlist_downloader/
│       ├── __init__.py
│       ├── cli.py
│       ├── core/
│       └── adapters/
├── tests/
│   ├── unit/
│   ├── integration/
│   └── conftest.py
├── docs/
├── pyproject.toml
├── requirements.txt
├── requirements-dev.txt
├── MANIFEST.in
├── README.md
├── LICENSE
└── CHANGELOG.md

[build-system]
requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2"]
build-backend = "setuptools.build_meta"

[project]
name = "playlist-downloader"
authors = [
    {name = "Christophe Hérolivier", email = "[email protected]"},
]
description = "CLI tool for YouTube playlist management"
readme = "README.md"
requires-python = ">=3.8"
keywords = ["youtube", "playlist", "cli", "downloader"]
license = {text = "MIT"}
classifiers = [
    "Development Status :: 4 - Beta",
    "Environment :: Console",
    "Intended Audience :: End Users/Desktop",
    "License :: OSI Approved :: MIT License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.8",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Topic :: Multimedia :: Sound/Audio",
    "Topic :: Utilities",
]
dependencies = [
    "typer>=0.9.0",
    "yt-dlp>=2023.7.6",
    "google-api-python-client>=2.0.0",
    "google-auth-oauthlib>=1.0.0",
    "pyyaml>=6.0",
    "rich>=13.0.0",
]
dynamic = ["version"]

[project.optional-dependencies]
dev = [
    "pytest>=7.0.0",
    "pytest-cov>=4.0.0",
    "pytest-mock>=3.10.0",
    "black>=23.0.0",
    "flake8>=6.0.0",
    "mypy>=1.0.0",
    "pre-commit>=3.0.0",
    "tox>=4.0.0",
]
test = [
    "pytest>=7.0.0",
    "pytest-cov>=4.0.0",
    "pytest-mock>=3.10.0",
]

[project.urls]
Homepage = "https://github.com/cheroliv/playlist-downloader"
Documentation = "https://github.com/cheroliv/playlist-downloader#readme"
Repository = "https://github.com/cheroliv/playlist-downloader.git"
"Bug Tracker" = "https://github.com/cheroliv/playlist-downloader/issues"

[project.scripts]
playlist-downloader = "playlist_downloader.cli:main"

[tool.setuptools_scm]
write_to = "src/playlist_downloader/_version.py"

[tool.setuptools.packages.find]
where = ["src"]

[tool.pytest.ini_options]
testpaths = ["tests"]
python_files = ["test_*.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]
addopts = [
    "--cov=src/playlist_downloader",
    "--cov-report=html",
    "--cov-report=term-missing",
    "--cov-fail-under=85",
]

[tool.black]
line-length = 88
target-version = ['py38']
include = '\.pyi?$'
extend-exclude = '''
/(
  \.eggs
  | \.git
  | \.hg
  | \.mypy_cache
  | \.tox
  | \.venv
  | _build
  | buck-out
  | build
  | dist
)/
'''

[tool.mypy]
python_version = "3.8"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = true
disallow_incomplete_defs = true
check_untyped_defs = true
disallow_untyped_decorators = true
no_implicit_optional = true
warn_redundant_casts = true
warn_unused_ignores = true
warn_no_return = true
warn_unreachable = true
strict_equality = true

[[tool.mypy.overrides]]
module = [
    "yt_dlp.*",
    "googleapiclient.*",
    "google_auth_oauthlib.*",
]
ignore_missing_imports = true

name: CI

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main ]

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache dependencies
      uses: actions/cache@v3
      with:
        path: |
          ~/.cache/pip
          ~/.cache/pre-commit
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e ".[dev]"

    - name: Lint with flake8
      run: |
        flake8 src tests --count --select=E9,F63,F7,F82 --show-source --statistics
        flake8 src tests --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics

    - name: Check code formatting with Black
      run: black --check src tests

    - name: Type checking with mypy
      run: mypy src

    - name: Run tests with pytest
      run: |
        pytest tests/ -v --cov=src/playlist_downloader \
          --cov-report=xml --cov-report=term-missing

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3
      if: matrix.python-version == '3.11'
      with:
        file: ./coverage.xml
        flags: unittests
        name: codecov-umbrella

  security:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install bandit[toml] safety

    - name: Run security checks with bandit
      run: bandit -r src/ -f json -o bandit-report.json

    - name: Check dependencies with safety
      run: safety check --json --output safety-report.json

    - name: Upload security reports
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: security-reports
        path: |
          bandit-report.json
          safety-report.json

  build:
    needs: [test, security]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: "3.11"

    - name: Install build dependencies
      run: |
        python -m pip install --upgrade pip
        pip install build twine

    - name: Build package
      run: python -m build

    - name: Check package with twine
      run: twine check dist/*

    - name: Upload build artifacts
      uses: actions/upload-artifact@v3
      with:
        name: dist
        path: dist/

name: Release

on:
  push:
    tags:
      - 'v*.*.*'
  workflow_dispatch:
    inputs:
      environment:
        description: 'Deployment environment'
        required: true
        default: 'test'
        type: choice
        options:
        - test
        - production

env:
  PYTHON_VERSION: "3.11"

jobs:
  release:
    runs-on: ubuntu-latest
    environment:
      name: ${{ github.event.inputs.environment || (startsWith(github.ref, 'refs/tags/') && 'production' || 'test') }}

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ env.PYTHON_VERSION }}

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install build twine

    - name: Build package
      run: python -m build

    - name: Check package
      run: twine check dist/*

    - name: Publish to Test PyPI
      if: github.event.inputs.environment == 'test' || (startsWith(github.ref, 'refs/tags/') && contains(github.ref, 'rc'))
      env:
        TWINE_USERNAME: __token__
        TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
      run: |
        twine upload --repository testpypi dist/*

    - name: Publish to PyPI
      if: github.event.inputs.environment == 'production' || (startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc'))
      env:
        TWINE_USERNAME: __token__
        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
      run: |
        twine upload dist/*

    - name: Create GitHub Release
      if: startsWith(github.ref, 'refs/tags/')
      uses: actions/create-release@v1
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
        tag_name: ${{ github.ref }}
        release_name: Release ${{ github.ref }}
        draft: false
        prerelease: ${{ contains(github.ref, 'rc') }}

  post-release:
    needs: release
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/')

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ env.PYTHON_VERSION }}

    - name: Test installation from PyPI
      run: |
        sleep 60  # Attendre la propagation sur PyPI
        pip install playlist-downloader
        playlist-downloader --version

    - name: Update documentation
      run: |
        # Script pour mettre à jour la documentation
        echo "Documentation updated for version ${GITHUB_REF#refs/tags/}"