# 1 -
cd ~
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
cat > kps-values.yaml <<'YAML'
prometheus:
  prometheusSpec:
    retention: 2d
    serviceMonitorSelectorNilUsesHelmValues: false   # watch ALL ServiceMonitors
    ruleSelectorNilUsesHelmValues: false             # and ALL PrometheusRules
# kubeadm binds these to localhost - skip them in the lab
kubeEtcd: { enabled: false }
kubeScheduler: { enabled: false }
kubeControllerManager: { enabled: false }
YAML
helm install kps prometheus-community/kube-prometheus-stack -n monitoring \
  --create-namespace -f kps-values.yaml --wait --timeout 10m
kubectl get pods -n monitoring

# 2 - what arrived?
kubectl get crd | grep monitoring.coreos.com
kubectl get ds,deploy,sts -n monitoring
kubectl get servicemonitors -n monitoring

# 3 - password, then open Grafana and Prometheus to your PC via support01
kubectl get svc -n monitoring
kubectl get secret -n monitoring kps-grafana -o jsonpath='{.data.admin-password}' | base64 -d; echo
kubectl port-forward -n monitoring svc/kps-grafana 3000:80 --address 0.0.0.0 &
kubectl port-forward -n monitoring svc/kps-kube-prometheus-stack-prometheus 9090 --address 0.0.0.0 &

sum by (namespace) (rate(container_cpu_usage_seconds_total{container!=""}[5m]))
sum by (namespace) (container_memory_working_set_bytes{container!=""}) / 1024 / 1024
kube_pod_container_status_restarts_total > 0
kube_deployment_status_replicas_available != kube_deployment_spec_replicas

# 4 - podinfo exposes /metrics and ships a ServiceMonitor in its chart
helm repo add podinfo https://stefanprodan.github.io/podinfo
helm install podinfo podinfo/podinfo -n shop --create-namespace \
  --set replicaCount=2 --set serviceMonitor.enabled=true
kubectl get servicemonitor -n shop

# 5 - generate a little traffic
kubectl run load -n shop --image busybox:1.36 --restart=Never -- \
  sh -c 'while true; do wget -qO- http://podinfo:9898/ >/dev/null; wget -qO- http://podinfo:9898/status/500 >/dev/null 2>&1; sleep 0.2; done'

sum by (status) (rate(http_request_duration_seconds_count{namespace="shop"}[1m]))

# 6 -
cat > alert.yaml <<'YAML'
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: shop-alerts
  namespace: shop
spec:
  groups:
    - name: shop
      rules:
        - alert: ShopHighErrorRate
          expr: |
            sum(rate(http_request_duration_seconds_count{namespace="shop",status=~"5.."}[2m]))
              / sum(rate(http_request_duration_seconds_count{namespace="shop"}[2m])) > 0.2
          for: 1m
          labels: { severity: warning }
          annotations:
            summary: "More than 20 % of shop requests fail"
YAML
kubectl apply -f alert.yaml

# 7 - stop the errors, watch it resolve
kubectl delete pod load -n shop