Archestack auf Kubernetes, der kurze Weg

Azure DevOps

1. Füge archestack/azure-pipelines.yml als Pipeline hinzu.
2. Lege unter Pipelines, Environments archestack-dev an und füge ihr den Namespace archestack des Clusters als Kubernetes-Ressource hinzu. Lege archestack-tst, -acc und -prd genauso an, wenn du sie brauchst.
3. Lege unter Pipelines, Library eine Variablengruppe mit demselben Namen an, archestack-dev, mit den fünf Werten, die Passwörter als Secrets.

# archestack/azure-pipelines.yml: Archestack on a Kubernetes cluster, under /archestack of a hostname the cluster's
# ingress already serves. Run it by hand, pick the environment and the Archestack version.
trigger:
  branches:
    include: [main]
  paths:
    include:
      - archestack/**

parameters:
  - name: environment
    displayName: Environment
    type: string
    default: dev
    values: [dev, tst, acc, prd]
  - name: imageTag
    displayName: Archestack version (image tag)
    type: string
    default: latest
  - name: monitoring
    displayName: Deploy Grafana with Prometheus, Tempo and Loki
    type: boolean
    default: true

variables:
  # One variable group per environment, named archestack-<environment>, holding:
  #   HOST                the hostname the cluster's ingress serves, such as apps.example.com
  #   ADMIN_EMAIL         the first Archestack admin, created on the first start
  #   POSTGRES_PASSWORD   secret; set once, see the note at "Secrets" below
  #   ADMIN_PASSWORD      secret; the first admin's password
  #   GRAFANA_PASSWORD    secret; Grafana's user admin
  - group: archestack-${{ parameters.environment }}
  - name: namespace
    value: archestack
  # The public Docker Hub images of Archestack.
  - name: registry
    value: docker.io/nickpeelman

stages:
  - stage: deploy
    displayName: Deploy ${{ parameters.environment }}
    jobs:
      - deployment: deploy
        displayName: Deploy Archestack
        pool:
          vmImage: ubuntu-latest
        # An Azure DevOps environment with the cluster's namespace added as a Kubernetes resource.
        environment:
          name: archestack-${{ parameters.environment }}
          resourceName: archestack
          resourceType: Kubernetes
        strategy:
          runOnce:
            deploy:
              steps:
                - checkout: self

                # Only these six names are replaced, so the $(POSTGRES_PASSWORD) the API reads and
                # Grafana's $${...} stay as they are.
                - bash: |
                    set -e
                    if [ -z "$HOST" ] || [ -z "$ADMIN_EMAIL" ]; then
                      echo "##vso[task.logissue type=error]Set HOST and ADMIN_EMAIL in the variable group archestack-${{ parameters.environment }}."
                      exit 1
                    fi
                    mkdir -p "$(Agent.TempDirectory)/manifests"
                    for f in archestack/manifests/*.yml; do
                      envsubst '${NAMESPACE} ${REGISTRY} ${IMAGE_TAG} ${ADMIN_EMAIL} ${HOST} ${MONITORING}' < "$f" > "$(Agent.TempDirectory)/manifests/$(basename "$f")"
                    done
                    if [ "$MONITORING" != "true" ]; then rm "$(Agent.TempDirectory)/manifests/observability.yml"; fi
                  displayName: Fill in the manifests
                  env:
                    NAMESPACE: $(namespace)
                    REGISTRY: $(registry)
                    IMAGE_TAG: ${{ parameters.imageTag }}
                    ADMIN_EMAIL: $(ADMIN_EMAIL)
                    HOST: $(HOST)
                    MONITORING: ${{ lower(parameters.monitoring) }}

                # Secrets. The database password is fixed when PostgreSQL first starts on an empty disk:
                # changing it afterwards does not change it in the database, and the API can no longer
                # connect. The admin password is used only to create the first admin.
                - bash: |
                    set -e
                    for value in "$PG_PASSWORD" "$ADMIN_PASSWORD_VALUE" "$GRAFANA_PASSWORD_VALUE"; do
                      case "$value" in
                        "" | '$('*)
                          echo "##vso[task.logissue type=error]Set the secrets POSTGRES_PASSWORD, ADMIN_PASSWORD and GRAFANA_PASSWORD in the variable group archestack-${{ parameters.environment }}."
                          exit 1
                          ;;
                      esac
                    done
                    kubectl create secret generic archestack-secrets \
                      --from-literal=postgres-password="$PG_PASSWORD" \
                      --from-literal=admin-password="$ADMIN_PASSWORD_VALUE" \
                      --from-literal=grafana-admin-password="$GRAFANA_PASSWORD_VALUE" \
                      -n $(namespace) --dry-run=client -o yaml > "$(Agent.TempDirectory)/manifests/secrets.yml"
                  displayName: Make the secret
                  env:
                    PG_PASSWORD: $(POSTGRES_PASSWORD)
                    ADMIN_PASSWORD_VALUE: $(ADMIN_PASSWORD)
                    GRAFANA_PASSWORD_VALUE: $(GRAFANA_PASSWORD)

                # Waits for every rollout. The API's first start creates and migrates its databases.
                - task: KubernetesManifest@1
                  displayName: Deploy Archestack
                  inputs:
                    action: deploy
                    namespace: $(namespace)
                    rolloutStatusTimeout: 1800
                    manifests: $(Agent.TempDirectory)/manifests/*.yml

                - bash: |
                    echo "Builder:       https://$(HOST)/archestack/"
                    echo "Published app: https://$(HOST)/archestack/app/"
                    echo "Grafana:       https://$(HOST)/archestack-grafana/"
                  displayName: Archestack addresses

• Fill in the manifests ersetzt darin sechs Namen und sonst nichts, das Datenbankpasswort, das die API zur Laufzeit liest, bleibt also ein Verweis auf das Secret.
• Make the secret macht aus den drei Passwörtern ein Kubernetes-Secret. Fehlt eines, stoppt es den Lauf, statt mit einem leeren Passwort zu deployen.
• Deploy Archestack wendet den Ordner an und wartet, bis jeder Pod bereit ist. Ist monitoring aus, bleiben Grafana und sein Stack weg.

GitHub Actions

1. Verschiebe archestack/github-actions.yml nach .github/workflows/archestack.yml: GitHub führt Workflows nur aus diesem Ordner aus.
2. Lege unter Settings, Environments archestack-dev an, mit HOST und ADMIN_EMAIL als Variablen und den drei Passwörtern als Secrets, dazu das Secret KUBECONFIG: das, was kubeconfig.sh ausgibt. In einem privaten Repository brauchen Umgebungen GitHub Pro, Team oder Enterprise.
3. Starte den Workflow unter Actions, Archestack, Run workflow.

# archestack/github-actions.yml: Archestack on a Kubernetes cluster, from GitHub Actions. GitHub runs
# workflows from one folder only: move this file to .github/workflows/archestack.yml. A push to main
# that changes archestack/ deploys to dev; under Actions, Archestack, Run workflow, you pick the
# environment and the Archestack version.
name: Archestack

on:
  push:
    branches: [main]
    paths: ["archestack/**"]
  workflow_dispatch:
    inputs:
      environment:
        description: Environment
        type: choice
        options: [dev, tst, acc, prd]
        default: dev
      imageTag:
        description: Archestack version (image tag)
        default: latest
      monitoring:
        description: Deploy Grafana with Prometheus, Tempo and Loki
        type: boolean
        default: true

permissions:
  contents: read

jobs:
  deploy:
    runs-on: ubuntu-latest
    # One GitHub environment per environment, named archestack-<environment>, holding the variables
    #   HOST                the hostname the cluster's ingress serves, such as apps.example.com
    #   ADMIN_EMAIL         the first Archestack admin, created on the first start
    # and the secrets
    #   POSTGRES_PASSWORD   set once: PostgreSQL keeps the password it first started with
    #   ADMIN_PASSWORD      the first admin's password
    #   GRAFANA_PASSWORD    Grafana's user admin
    #   KUBECONFIG          the cluster sign-in archestack/kubeconfig.sh prints
    environment: archestack-${{ inputs.environment || 'dev' }}
    concurrency: archestack-${{ inputs.environment || 'dev' }}    # one rollout per environment at a time
    steps:
      - uses: actions/checkout@v4
      - uses: azure/setup-kubectl@v4
        with:
          version: v1.35.9                                       # within one minor version of the cluster's
      - name: Sign in to the cluster
        run: |
          if [ -z "$KUBECONFIG_FILE" ]; then
            echo "::error::Set the secret KUBECONFIG in the environment archestack-${{ inputs.environment || 'dev' }}."
            exit 1
          fi
          install -m 600 /dev/null "$RUNNER_TEMP/kubeconfig"
          printf '%s\n' "$KUBECONFIG_FILE" > "$RUNNER_TEMP/kubeconfig"
          echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV"
        env:
          KUBECONFIG_FILE: ${{ secrets.KUBECONFIG }}
      - name: Deploy Archestack
        run: bash archestack/deploy.sh
        env:
          IMAGE_TAG: ${{ inputs.imageTag || 'latest' }}
          # A push has no inputs, so it deploys with monitoring on.
          MONITORING: ${{ (github.event_name == 'workflow_dispatch' && !inputs.monitoring) && 'false' || 'true' }}
          HOST: ${{ vars.HOST }}
          ADMIN_EMAIL: ${{ vars.ADMIN_EMAIL }}
          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
          ADMIN_PASSWORD: ${{ secrets.ADMIN_PASSWORD }}
          GRAFANA_PASSWORD: ${{ secrets.GRAFANA_PASSWORD }}

• Ein Push auf main, der archestack/ ändert, deployt mit den Images latest in die Entwicklung, wie es die Azure DevOps-Pipeline tut.
• concurrency verhindert, dass zwei Läufe gleichzeitig in dieselbe Umgebung ausrollen.
• kubectl hat eine feste Version. Halte sie höchstens eine Minor-Version von der deines Clusters entfernt.

GitLab CI

1. Binde die Datei des Kits aus der eigenen .gitlab-ci.yml des Repositorys ein, oder lege eine an, die nur dies enthält:

   include:
     - local: archestack/gitlab-ci.yml

2. Füge unter Settings, CI/CD, Variables die fünf Werte hinzu, beschränkt auf die Umgebung archestack-dev, die Passwörter maskiert, dazu KUBECONFIG vom Typ File: das, was kubeconfig.sh ausgibt. Schütze sie alle, sodass nur geschützte Branches sie bekommen.
3. Starte sie unter Build, Pipelines, New pipeline und wähle dort die Umgebung.

# archestack/gitlab-ci.yml: Archestack on a Kubernetes cluster, from GitLab CI. Include it from the
# repository's own .gitlab-ci.yml (a file holding just these two lines when there is none):
#   include:
#     - local: archestack/gitlab-ci.yml
# A push to the default branch that changes archestack/ deploys to dev; under Build, Pipelines,
# New pipeline, you pick the environment and the Archestack version.
variables:
  ARCHESTACK_ENVIRONMENT:
    value: dev
    options: [dev, tst, acc, prd]
    description: Environment
  ARCHESTACK_IMAGE_TAG:
    value: latest
    description: Archestack version (image tag)
  ARCHESTACK_MONITORING:
    value: "true"
    options: ["true", "false"]
    description: Deploy Grafana with Prometheus, Tempo and Loki

archestack:
  stage: deploy
  image: alpine:3.22
  rules:
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      changes: [archestack/**/*]
  # Variables scoped to this environment (Settings, CI/CD, Variables), all protected:
  #   HOST                the hostname the cluster's ingress serves, such as apps.example.com
  #   ADMIN_EMAIL         the first Archestack admin, created on the first start
  #   POSTGRES_PASSWORD   masked; set once: PostgreSQL keeps the password it first started with
  #   ADMIN_PASSWORD      masked; the first admin's password
  #   GRAFANA_PASSWORD    masked; Grafana's user admin
  #   KUBECONFIG          type File: the cluster sign-in archestack/kubeconfig.sh prints. With the
  #                       GitLab agent for Kubernetes, set KUBE_CONTEXT to the agent's context instead.
  environment:
    name: archestack-$ARCHESTACK_ENVIRONMENT
  resource_group: archestack-$ARCHESTACK_ENVIRONMENT      # one rollout per environment at a time
  variables:
    KUBECTL_VERSION: v1.35.9                             # within one minor version of the cluster's
  before_script:
    - apk add --no-cache bash curl gettext-envsubst
    - curl -fsSLo /usr/local/bin/kubectl "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
    - chmod +x /usr/local/bin/kubectl
    - 'if [ -n "${KUBE_CONTEXT:-}" ]; then kubectl config use-context "$KUBE_CONTEXT"; fi'
  script:
    - IMAGE_TAG=$ARCHESTACK_IMAGE_TAG MONITORING=$ARCHESTACK_MONITORING bash archestack/deploy.sh

• Ein Push auf den Standard-Branch, der archestack/ ändert, deployt mit den Images latest in die Entwicklung.
• Der Job läuft in der Stage deploy, die GitLab hat, es sei denn, deine .gitlab-ci.yml führt eigene Stages ohne sie auf.
• Der GitLab Agent für Kubernetes funktioniert auch: Setze statt KUBECONFIG die Variable KUBE_CONTEXT auf den Kontext des Agents.
• resource_group verhindert, dass zwei Pipelines gleichzeitig in dieselbe Umgebung ausrollen.
• Das Image und kubectl haben feste Versionen. Halte kubectl höchstens eine Minor-Version von der deines Clusters entfernt.

Was GitHub Actions und GitLab CI ausführen

Beide führen deploy.sh aus, die drei Schritte der Azure DevOps-Pipeline als ein Skript:

#!/usr/bin/env bash
# archestack/deploy.sh: what github-actions.yml and gitlab-ci.yml run, the same three steps
# azure-pipelines.yml takes with Azure DevOps' own tasks. kubectl has to reach the cluster already.
# Reads HOST, ADMIN_EMAIL, POSTGRES_PASSWORD, ADMIN_PASSWORD and GRAFANA_PASSWORD, and optionally
# IMAGE_TAG (latest) and MONITORING (true):
#   IMAGE_TAG=1.4.23 bash archestack/deploy.sh
set -euo pipefail
cd "$(dirname "$0")"

for name in HOST ADMIN_EMAIL POSTGRES_PASSWORD ADMIN_PASSWORD GRAFANA_PASSWORD; do
  if [ -z "${!name:-}" ]; then echo "Set $name for this environment" >&2; exit 1; fi
done

export NAMESPACE=${NAMESPACE:-archestack}
export REGISTRY=${REGISTRY:-docker.io/nickpeelman}    # the public Docker Hub images of Archestack
export IMAGE_TAG=${IMAGE_TAG:-latest}
export MONITORING=${MONITORING:-true}
export HOST ADMIN_EMAIL

# 1. Fill in the manifests. Only these six names are replaced, so the $(POSTGRES_PASSWORD) the API
#    reads and Grafana's $${...} stay as they are.
out=$(mktemp -d)
trap 'rm -rf "$out"' EXIT
for f in manifests/*.yml; do
  envsubst '${NAMESPACE} ${REGISTRY} ${IMAGE_TAG} ${ADMIN_EMAIL} ${HOST} ${MONITORING}' < "$f" > "$out/$(basename "$f")"
done
if [ "$MONITORING" != true ]; then rm "$out/observability.yml"; fi

# 2. The secret, handed from one kubectl to the next and never written to a file. The database
#    password is fixed when PostgreSQL first starts on an empty disk; the admin password only makes
#    the first admin.
kubectl create secret generic archestack-secrets -n "$NAMESPACE" \
  --from-literal=postgres-password="$POSTGRES_PASSWORD" \
  --from-literal=admin-password="$ADMIN_PASSWORD" \
  --from-literal=grafana-admin-password="$GRAFANA_PASSWORD" \
  --dry-run=client -o yaml | kubectl apply -f -

# 3. Deploy, then wait for every rollout. The API's first start creates and migrates its databases.
applied=$(kubectl apply -n "$NAMESPACE" -f "$out" -o name)
for workload in $(grep -E '^(deployment|statefulset)\.apps/' <<< "$applied"); do
  kubectl rollout status -n "$NAMESPACE" "$workload" --timeout=30m
done

echo "Builder:       https://$HOST/archestack/"
echo "Published app: https://$HOST/archestack/app/"
if [ "$MONITORING" = true ]; then echo "Grafana:       https://$HOST/archestack-grafana/"; fi

Am Cluster anmelden

GitHub Actions und GitLab CI melden sich mit der kubeconfig an, die kubeconfig.sh ausgibt: ein Service Account, der in den Namespace archestack darf und sonst nirgendwohin, mit einem Token, das nicht abläuft. Führe das Skript einmal pro Cluster aus, als jemand, der dort Rollen anlegen darf:

#!/usr/bin/env bash
# archestack/kubeconfig.sh: the cluster sign-in for the GitHub Actions or GitLab CI pipeline, allowed
# into the archestack namespace and nothing else. Run it once per cluster, as someone who may create
# roles there, with kubectl pointing at that cluster:
#   bash archestack/kubeconfig.sh > kubeconfig-dev
# Paste the file into the environment's KUBECONFIG, then delete it. To take the access away:
#   kubectl -n archestack delete secret archestack-deploy-token
set -euo pipefail
NAMESPACE=${NAMESPACE:-archestack}

kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >&2
kubectl apply -n "$NAMESPACE" -f - >&2 <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: archestack-deploy
---
# Everything inside the namespace (the built-in admin role), nothing outside it.
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: archestack-deploy
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: admin
subjects:
  - kind: ServiceAccount
    name: archestack-deploy
    namespace: $NAMESPACE
---
# A token that does not expire. Deleting this secret revokes it.
apiVersion: v1
kind: Secret
metadata:
  name: archestack-deploy-token
  annotations:
    kubernetes.io/service-account.name: archestack-deploy
type: kubernetes.io/service-account-token
EOF

# Kubernetes fills the token in a moment later.
token=""
for _ in $(seq 30); do
  token=$(kubectl get secret archestack-deploy-token -n "$NAMESPACE" -o jsonpath='{.data.token}' | base64 --decode)
  if [ -n "$token" ]; then break; fi
  sleep 1
done
if [ -z "$token" ]; then echo "The token did not appear in archestack-deploy-token" >&2; exit 1; fi
ca=$(kubectl get secret archestack-deploy-token -n "$NAMESPACE" -o jsonpath='{.data.ca\.crt}')
server=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')

cat <<EOF
apiVersion: v1
kind: Config
clusters:
  - name: cluster
    cluster:
      server: $server
      certificate-authority-data: $ca
users:
  - name: archestack-deploy
    user:
      token: $token
contexts:
  - name: archestack
    context:
      cluster: cluster
      user: archestack-deploy
      namespace: $NAMESPACE
current-context: archestack
EOF

• Nicht deine eigene kubeconfig. Sie trägt deine Rechte auf dem ganzen Cluster, und auf AKS mit Entra ID-Anmeldung verlangt sie, dass sich eine Person anmeldet, was eine Pipeline nicht kann.
• Den Zugang entziehen heißt, das Secret archestack-deploy-token im Namespace zu löschen. Führe das Skript erneut aus, um ein neues zu bekommen.