O Archestack no Kubernetes, a forma curta

Azure DevOps

1. Acrescente archestack/azure-pipelines.yml como pipeline.
2. Em Pipelines, Environments, crie archestack-dev e acrescente-lhe o namespace archestack do cluster como recurso Kubernetes. Crie archestack-tst, -acc e -prd da mesma forma quando precisar deles.
3. Em Pipelines, Library, crie um grupo de variáveis com o mesmo nome, archestack-dev, com as cinco, as palavras-passe como segredos.

# archestack/azure-pipelines.yml: Archestack on a Kubernetes cluster, under /archestack of a hostname the cluster's
# ingress already serves. Run it by hand, pick the environment and the Archestack version.
trigger:
  branches:
    include: [main]
  paths:
    include:
      - archestack/**

parameters:
  - name: environment
    displayName: Environment
    type: string
    default: dev
    values: [dev, tst, acc, prd]
  - name: imageTag
    displayName: Archestack version (image tag)
    type: string
    default: latest
  - name: monitoring
    displayName: Deploy Grafana with Prometheus, Tempo and Loki
    type: boolean
    default: true

variables:
  # One variable group per environment, named archestack-<environment>, holding:
  #   HOST                the hostname the cluster's ingress serves, such as apps.example.com
  #   ADMIN_EMAIL         the first Archestack admin, created on the first start
  #   POSTGRES_PASSWORD   secret; set once, see the note at "Secrets" below
  #   ADMIN_PASSWORD      secret; the first admin's password
  #   GRAFANA_PASSWORD    secret; Grafana's user admin
  - group: archestack-${{ parameters.environment }}
  - name: namespace
    value: archestack
  # The public Docker Hub images of Archestack.
  - name: registry
    value: docker.io/nickpeelman

stages:
  - stage: deploy
    displayName: Deploy ${{ parameters.environment }}
    jobs:
      - deployment: deploy
        displayName: Deploy Archestack
        pool:
          vmImage: ubuntu-latest
        # An Azure DevOps environment with the cluster's namespace added as a Kubernetes resource.
        environment:
          name: archestack-${{ parameters.environment }}
          resourceName: archestack
          resourceType: Kubernetes
        strategy:
          runOnce:
            deploy:
              steps:
                - checkout: self

                # Only these six names are replaced, so the $(POSTGRES_PASSWORD) the API reads and
                # Grafana's $${...} stay as they are.
                - bash: |
                    set -e
                    if [ -z "$HOST" ] || [ -z "$ADMIN_EMAIL" ]; then
                      echo "##vso[task.logissue type=error]Set HOST and ADMIN_EMAIL in the variable group archestack-${{ parameters.environment }}."
                      exit 1
                    fi
                    mkdir -p "$(Agent.TempDirectory)/manifests"
                    for f in archestack/manifests/*.yml; do
                      envsubst '${NAMESPACE} ${REGISTRY} ${IMAGE_TAG} ${ADMIN_EMAIL} ${HOST} ${MONITORING}' < "$f" > "$(Agent.TempDirectory)/manifests/$(basename "$f")"
                    done
                    if [ "$MONITORING" != "true" ]; then rm "$(Agent.TempDirectory)/manifests/observability.yml"; fi
                  displayName: Fill in the manifests
                  env:
                    NAMESPACE: $(namespace)
                    REGISTRY: $(registry)
                    IMAGE_TAG: ${{ parameters.imageTag }}
                    ADMIN_EMAIL: $(ADMIN_EMAIL)
                    HOST: $(HOST)
                    MONITORING: ${{ lower(parameters.monitoring) }}

                # Secrets. The database password is fixed when PostgreSQL first starts on an empty disk:
                # changing it afterwards does not change it in the database, and the API can no longer
                # connect. The admin password is used only to create the first admin.
                - bash: |
                    set -e
                    for value in "$PG_PASSWORD" "$ADMIN_PASSWORD_VALUE" "$GRAFANA_PASSWORD_VALUE"; do
                      case "$value" in
                        "" | '$('*)
                          echo "##vso[task.logissue type=error]Set the secrets POSTGRES_PASSWORD, ADMIN_PASSWORD and GRAFANA_PASSWORD in the variable group archestack-${{ parameters.environment }}."
                          exit 1
                          ;;
                      esac
                    done
                    kubectl create secret generic archestack-secrets \
                      --from-literal=postgres-password="$PG_PASSWORD" \
                      --from-literal=admin-password="$ADMIN_PASSWORD_VALUE" \
                      --from-literal=grafana-admin-password="$GRAFANA_PASSWORD_VALUE" \
                      -n $(namespace) --dry-run=client -o yaml > "$(Agent.TempDirectory)/manifests/secrets.yml"
                  displayName: Make the secret
                  env:
                    PG_PASSWORD: $(POSTGRES_PASSWORD)
                    ADMIN_PASSWORD_VALUE: $(ADMIN_PASSWORD)
                    GRAFANA_PASSWORD_VALUE: $(GRAFANA_PASSWORD)

                # Waits for every rollout. The API's first start creates and migrates its databases.
                - task: KubernetesManifest@1
                  displayName: Deploy Archestack
                  inputs:
                    action: deploy
                    namespace: $(namespace)
                    rolloutStatusTimeout: 1800
                    manifests: $(Agent.TempDirectory)/manifests/*.yml

                - bash: |
                    echo "Builder:       https://$(HOST)/archestack/"
                    echo "Published app: https://$(HOST)/archestack/app/"
                    echo "Grafana:       https://$(HOST)/archestack-grafana/"
                  displayName: Archestack addresses

• Fill in the manifests substitui seis nomes nos manifestos e nada mais, por isso a palavra-passe da base de dados que a API lê em execução continua a ser uma referência ao Secret.
• Make the secret transforma as três palavras-passe num só Secret do Kubernetes. Para a execução quando falta uma, em vez de fazer deploy com uma palavra-passe vazia.
• Deploy Archestack aplica a pasta e espera até todos os pods estarem prontos. Com monitoring desligado, o Grafana e a sua stack ficam de fora.

GitHub Actions

1. Mova archestack/github-actions.yml para .github/workflows/archestack.yml: o GitHub só corre workflows a partir dessa pasta.
2. Em Settings, Environments, crie archestack-dev, com HOST e ADMIN_EMAIL como variáveis e as três palavras-passe como segredos, mais o segredo KUBECONFIG: o que o kubeconfig.sh imprime. Num repositório privado, os ambientes precisam do GitHub Pro, Team ou Enterprise.
3. Corra-o em Actions, Archestack, Run workflow.

# archestack/github-actions.yml: Archestack on a Kubernetes cluster, from GitHub Actions. GitHub runs
# workflows from one folder only: move this file to .github/workflows/archestack.yml. A push to main
# that changes archestack/ deploys to dev; under Actions, Archestack, Run workflow, you pick the
# environment and the Archestack version.
name: Archestack

on:
  push:
    branches: [main]
    paths: ["archestack/**"]
  workflow_dispatch:
    inputs:
      environment:
        description: Environment
        type: choice
        options: [dev, tst, acc, prd]
        default: dev
      imageTag:
        description: Archestack version (image tag)
        default: latest
      monitoring:
        description: Deploy Grafana with Prometheus, Tempo and Loki
        type: boolean
        default: true

permissions:
  contents: read

jobs:
  deploy:
    runs-on: ubuntu-latest
    # One GitHub environment per environment, named archestack-<environment>, holding the variables
    #   HOST                the hostname the cluster's ingress serves, such as apps.example.com
    #   ADMIN_EMAIL         the first Archestack admin, created on the first start
    # and the secrets
    #   POSTGRES_PASSWORD   set once: PostgreSQL keeps the password it first started with
    #   ADMIN_PASSWORD      the first admin's password
    #   GRAFANA_PASSWORD    Grafana's user admin
    #   KUBECONFIG          the cluster sign-in archestack/kubeconfig.sh prints
    environment: archestack-${{ inputs.environment || 'dev' }}
    concurrency: archestack-${{ inputs.environment || 'dev' }}    # one rollout per environment at a time
    steps:
      - uses: actions/checkout@v4
      - uses: azure/setup-kubectl@v4
        with:
          version: v1.35.9                                       # within one minor version of the cluster's
      - name: Sign in to the cluster
        run: |
          if [ -z "$KUBECONFIG_FILE" ]; then
            echo "::error::Set the secret KUBECONFIG in the environment archestack-${{ inputs.environment || 'dev' }}."
            exit 1
          fi
          install -m 600 /dev/null "$RUNNER_TEMP/kubeconfig"
          printf '%s\n' "$KUBECONFIG_FILE" > "$RUNNER_TEMP/kubeconfig"
          echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV"
        env:
          KUBECONFIG_FILE: ${{ secrets.KUBECONFIG }}
      - name: Deploy Archestack
        run: bash archestack/deploy.sh
        env:
          IMAGE_TAG: ${{ inputs.imageTag || 'latest' }}
          # A push has no inputs, so it deploys with monitoring on.
          MONITORING: ${{ (github.event_name == 'workflow_dispatch' && !inputs.monitoring) && 'false' || 'true' }}
          HOST: ${{ vars.HOST }}
          ADMIN_EMAIL: ${{ vars.ADMIN_EMAIL }}
          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
          ADMIN_PASSWORD: ${{ secrets.ADMIN_PASSWORD }}
          GRAFANA_PASSWORD: ${{ secrets.GRAFANA_PASSWORD }}

• Um push para main que altere archestack/ faz deploy para desenvolvimento com as imagens latest, como faz o pipeline do Azure DevOps.
• concurrency impede que duas execuções façam rollout para o mesmo ambiente ao mesmo tempo.
• O kubectl está fixado. Mantenha-o a não mais de uma versão minor de distância da do seu cluster.

GitLab CI

1. Inclua o ficheiro do kit a partir do .gitlab-ci.yml do próprio repositório, ou crie um só com isto:

   include:
     - local: archestack/gitlab-ci.yml

2. Em Settings, CI/CD, Variables, acrescente as cinco com o âmbito de ambiente archestack-dev, as palavras-passe mascaradas, mais KUBECONFIG do tipo File: o que o kubeconfig.sh imprime. Proteja-as todas, para que só as branches protegidas as recebam.
3. Corra-o em Build, Pipelines, New pipeline, e escolha lá o ambiente.

# archestack/gitlab-ci.yml: Archestack on a Kubernetes cluster, from GitLab CI. Include it from the
# repository's own .gitlab-ci.yml (a file holding just these two lines when there is none):
#   include:
#     - local: archestack/gitlab-ci.yml
# A push to the default branch that changes archestack/ deploys to dev; under Build, Pipelines,
# New pipeline, you pick the environment and the Archestack version.
variables:
  ARCHESTACK_ENVIRONMENT:
    value: dev
    options: [dev, tst, acc, prd]
    description: Environment
  ARCHESTACK_IMAGE_TAG:
    value: latest
    description: Archestack version (image tag)
  ARCHESTACK_MONITORING:
    value: "true"
    options: ["true", "false"]
    description: Deploy Grafana with Prometheus, Tempo and Loki

archestack:
  stage: deploy
  image: alpine:3.22
  rules:
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      changes: [archestack/**/*]
  # Variables scoped to this environment (Settings, CI/CD, Variables), all protected:
  #   HOST                the hostname the cluster's ingress serves, such as apps.example.com
  #   ADMIN_EMAIL         the first Archestack admin, created on the first start
  #   POSTGRES_PASSWORD   masked; set once: PostgreSQL keeps the password it first started with
  #   ADMIN_PASSWORD      masked; the first admin's password
  #   GRAFANA_PASSWORD    masked; Grafana's user admin
  #   KUBECONFIG          type File: the cluster sign-in archestack/kubeconfig.sh prints. With the
  #                       GitLab agent for Kubernetes, set KUBE_CONTEXT to the agent's context instead.
  environment:
    name: archestack-$ARCHESTACK_ENVIRONMENT
  resource_group: archestack-$ARCHESTACK_ENVIRONMENT      # one rollout per environment at a time
  variables:
    KUBECTL_VERSION: v1.35.9                             # within one minor version of the cluster's
  before_script:
    - apk add --no-cache bash curl gettext-envsubst
    - curl -fsSLo /usr/local/bin/kubectl "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
    - chmod +x /usr/local/bin/kubectl
    - 'if [ -n "${KUBE_CONTEXT:-}" ]; then kubectl config use-context "$KUBE_CONTEXT"; fi'
  script:
    - IMAGE_TAG=$ARCHESTACK_IMAGE_TAG MONITORING=$ARCHESTACK_MONITORING bash archestack/deploy.sh

• Um push para a branch predefinida que altere archestack/ faz deploy para desenvolvimento com as imagens latest.
• O job corre no stage deploy, que o GitLab tem, a não ser que o seu .gitlab-ci.yml liste stages próprios sem ele.
• O agente do GitLab para Kubernetes também funciona: defina KUBE_CONTEXT com o contexto do agente em vez de KUBECONFIG.
• resource_group impede que dois pipelines façam rollout para o mesmo ambiente ao mesmo tempo.
• A imagem e o kubectl estão fixados. Mantenha o kubectl a não mais de uma versão minor de distância da do seu cluster.

O que o GitHub Actions e o GitLab CI correm

Ambos correm o deploy.sh, os três passos do pipeline do Azure DevOps num só script:

#!/usr/bin/env bash
# archestack/deploy.sh: what github-actions.yml and gitlab-ci.yml run, the same three steps
# azure-pipelines.yml takes with Azure DevOps' own tasks. kubectl has to reach the cluster already.
# Reads HOST, ADMIN_EMAIL, POSTGRES_PASSWORD, ADMIN_PASSWORD and GRAFANA_PASSWORD, and optionally
# IMAGE_TAG (latest) and MONITORING (true):
#   IMAGE_TAG=1.4.23 bash archestack/deploy.sh
set -euo pipefail
cd "$(dirname "$0")"

for name in HOST ADMIN_EMAIL POSTGRES_PASSWORD ADMIN_PASSWORD GRAFANA_PASSWORD; do
  if [ -z "${!name:-}" ]; then echo "Set $name for this environment" >&2; exit 1; fi
done

export NAMESPACE=${NAMESPACE:-archestack}
export REGISTRY=${REGISTRY:-docker.io/nickpeelman}    # the public Docker Hub images of Archestack
export IMAGE_TAG=${IMAGE_TAG:-latest}
export MONITORING=${MONITORING:-true}
export HOST ADMIN_EMAIL

# 1. Fill in the manifests. Only these six names are replaced, so the $(POSTGRES_PASSWORD) the API
#    reads and Grafana's $${...} stay as they are.
out=$(mktemp -d)
trap 'rm -rf "$out"' EXIT
for f in manifests/*.yml; do
  envsubst '${NAMESPACE} ${REGISTRY} ${IMAGE_TAG} ${ADMIN_EMAIL} ${HOST} ${MONITORING}' < "$f" > "$out/$(basename "$f")"
done
if [ "$MONITORING" != true ]; then rm "$out/observability.yml"; fi

# 2. The secret, handed from one kubectl to the next and never written to a file. The database
#    password is fixed when PostgreSQL first starts on an empty disk; the admin password only makes
#    the first admin.
kubectl create secret generic archestack-secrets -n "$NAMESPACE" \
  --from-literal=postgres-password="$POSTGRES_PASSWORD" \
  --from-literal=admin-password="$ADMIN_PASSWORD" \
  --from-literal=grafana-admin-password="$GRAFANA_PASSWORD" \
  --dry-run=client -o yaml | kubectl apply -f -

# 3. Deploy, then wait for every rollout. The API's first start creates and migrates its databases.
applied=$(kubectl apply -n "$NAMESPACE" -f "$out" -o name)
for workload in $(grep -E '^(deployment|statefulset)\.apps/' <<< "$applied"); do
  kubectl rollout status -n "$NAMESPACE" "$workload" --timeout=30m
done

echo "Builder:       https://$HOST/archestack/"
echo "Published app: https://$HOST/archestack/app/"
if [ "$MONITORING" = true ]; then echo "Grafana:       https://$HOST/archestack-grafana/"; fi

Iniciar sessão no cluster

O GitHub Actions e o GitLab CI iniciam sessão com o kubeconfig que o kubeconfig.sh imprime: uma service account com acesso ao namespace archestack e a nada mais, com um token que não expira. Corra-o uma vez por cluster, como alguém que lá possa criar roles:

#!/usr/bin/env bash
# archestack/kubeconfig.sh: the cluster sign-in for the GitHub Actions or GitLab CI pipeline, allowed
# into the archestack namespace and nothing else. Run it once per cluster, as someone who may create
# roles there, with kubectl pointing at that cluster:
#   bash archestack/kubeconfig.sh > kubeconfig-dev
# Paste the file into the environment's KUBECONFIG, then delete it. To take the access away:
#   kubectl -n archestack delete secret archestack-deploy-token
set -euo pipefail
NAMESPACE=${NAMESPACE:-archestack}

kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >&2
kubectl apply -n "$NAMESPACE" -f - >&2 <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: archestack-deploy
---
# Everything inside the namespace (the built-in admin role), nothing outside it.
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: archestack-deploy
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: admin
subjects:
  - kind: ServiceAccount
    name: archestack-deploy
    namespace: $NAMESPACE
---
# A token that does not expire. Deleting this secret revokes it.
apiVersion: v1
kind: Secret
metadata:
  name: archestack-deploy-token
  annotations:
    kubernetes.io/service-account.name: archestack-deploy
type: kubernetes.io/service-account-token
EOF

# Kubernetes fills the token in a moment later.
token=""
for _ in $(seq 30); do
  token=$(kubectl get secret archestack-deploy-token -n "$NAMESPACE" -o jsonpath='{.data.token}' | base64 --decode)
  if [ -n "$token" ]; then break; fi
  sleep 1
done
if [ -z "$token" ]; then echo "The token did not appear in archestack-deploy-token" >&2; exit 1; fi
ca=$(kubectl get secret archestack-deploy-token -n "$NAMESPACE" -o jsonpath='{.data.ca\.crt}')
server=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')

cat <<EOF
apiVersion: v1
kind: Config
clusters:
  - name: cluster
    cluster:
      server: $server
      certificate-authority-data: $ca
users:
  - name: archestack-deploy
    user:
      token: $token
contexts:
  - name: archestack
    context:
      cluster: cluster
      user: archestack-deploy
      namespace: $NAMESPACE
current-context: archestack
EOF

• Não o seu próprio kubeconfig. Leva os seus direitos sobre todo o cluster, e no AKS com início de sessão via Entra ID pede a uma pessoa que inicie sessão, o que um pipeline não consegue fazer.
• Retirar o acesso é apagar o Secret archestack-deploy-token no namespace. Corra o script outra vez para obter um novo.