diff --git a/.helmignore b/.helmignore new file mode 100644 index 0000000..8c15628 --- /dev/null +++ b/.helmignore @@ -0,0 +1,13 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.swp +*.bak +*.tmp +*.orig +*~ +.idea/ +.vscode/ +# Argo CD examples live next to the chart but are not rendered by it. +argocd/ diff --git a/Chart.yaml b/Chart.yaml new file mode 100644 index 0000000..fd30810 --- /dev/null +++ b/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: redis-cluster +description: >- + Sharded, highly available Redis Cluster with multiple writable primaries, + per-shard replicas, automatic failover, and Prometheus metrics. +type: application +version: 0.4.0 +appVersion: "7.4" +kubeVersion: ">=1.25.0-0" +home: https://git.mulas.me/corrado/redis-cluster +sources: + - https://git.mulas.me/corrado/redis-cluster + - https://github.com/redis/redis +keywords: + - redis + - redis-cluster + - sharding + - high-availability + - cache +maintainers: + - name: corrado.mulas diff --git a/LICENSE b/LICENSE index a31bd62..eafc2fa 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2026 corrado. +Copyright (c) 2026 Corrado Mulas . Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index c6c68ca..dd12924 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,988 @@ # redis-cluster -Helm chart for Kubernetes Redis sharded cluster with cross-readers \ No newline at end of file +A production-oriented Helm chart for a native, sharded Redis Cluster on +Kubernetes. It creates multiple independently writable shard primaries, one or +more readable failover replicas for every shard, strict cross-node placement, +persistent storage, optional external cluster-aware access, Prometheus metrics, +and an optional RedisInsight UI. + +This chart deploys Redis Cluster mode. It does not deploy Sentinel, a proxy, or +an active-active database. Every hash slot has exactly one writable primary; +write capacity scales because different slot ranges belong to different +primaries. + +## Contents + +- [Architecture](#architecture) +- [Requirements](#requirements) +- [Quick start](#quick-start) +- [Connecting clients](#connecting-clients) +- [Authentication](#authentication) +- [Persistence](#persistence) +- [External access](#external-access) +- [RedisInsight](#redisinsight) +- [Metrics](#metrics) +- [Network policy and service mesh](#network-policy-and-service-mesh) +- [Scheduling and availability](#scheduling-and-availability) +- [Safe upgrades](#safe-upgrades) +- [Scaling and topology changes](#scaling-and-topology-changes) +- [Backup and restore](#backup-and-restore) +- [Argo CD](#argo-cd) +- [Configuration reference](#configuration-reference) +- [Troubleshooting](#troubleshooting) +- [Uninstalling](#uninstalling) + +## Architecture + +The default topology is three shards with one replica per shard: + +```text +Kubernetes worker A Kubernetes worker B Kubernetes worker C +------------------- ------------------- ------------------- +shard 0 member 0 shard 1 member 0 shard 2 member 0 +canonical primary canonical primary canonical primary + +shard 2 member 1 shard 0 member 1 shard 1 member 1 +replica replica replica +``` + +That produces six Redis instances: + +```text +3 writable primaries + 3 readable/failover replicas = 6 Redis pods +``` + +Each persistent member is a separate one-replica StatefulSet with its own PVC. +This design allows Kubernetes to enforce all of the following: + +- all canonical primaries occupy different `kubernetes.io/hostname` domains; +- the members of one shard never occupy the same node; +- each replica is crossed onto the node containing another shard's primary; +- member identities and Redis node IDs survive pod replacement. + +At first boot, the shard-0 member-0 pod coordinates cluster creation. It waits +for every pristine member, introduces all nodes, divides all 16,384 hash slots +between the primaries, and attaches each replica to its intended primary. It +refuses to overwrite partially initialized cluster state. + +Redis can promote a replica after a primary failure. A reconciliation loop later +returns the cluster to its canonical crossed layout after the failed node comes +back. Redis Cluster replication is asynchronous; a narrow failure or partition +window can still lose an acknowledged write. + +## Requirements + +- Kubernetes 1.25 or newer. +- Helm 3. +- At least `cluster.shards` schedulable worker nodes with distinct + `kubernetes.io/hostname` labels. The default requires three workers. +- Enough CPU and memory on those workers for every Redis and exporter pod. +- A default StorageClass, or an explicit `persistence.storageClass`, when + persistence is enabled. +- Working pod DNS and unrestricted Redis client and cluster-bus traffic between + members. + +Optional features have additional requirements: + +- external access requires a controller that implements `type: LoadBalancer` + Services and populates `.status.loadBalancer.ingress`; +- `metrics.serviceMonitor.enabled` requires the Prometheus Operator CRDs; +- RedisInsight Istio ingress requires the Istio networking CRDs; +- automatic RedisInsight certificate creation requires cert-manager. + +## Quick start + +Clone the repository and install an internal-only cluster: + +```sh +git clone https://git.mulas.me/corrado/redis-cluster.git +cd redis-cluster + +helm install redis . \ + --namespace redis \ + --create-namespace +``` + +Wait for all six default members: + +```sh +kubectl get pods -n redis -w +``` + +Check cluster health and slot coverage: + +```sh +kubectl exec -n redis redis-redis-cluster-s0-n0-0 -c redis -- \ + redis-cli cluster info + +kubectl exec -n redis redis-redis-cluster-s0-n0-0 -c redis -- \ + redis-cli --cluster check 127.0.0.1:6379 +``` + +`cluster_state:ok`, `cluster_slots_ok:16384`, and zero failed slots indicate a +healthy cluster. + +### Recommended production values + +Create a values file: + +```yaml +auth: + enabled: true + existingSecret: redis-auth + existingSecretKey: redis-password + +persistence: + storageClass: fast-storage + size: 100Gi + +redis: + maxmemory: 80gb + resources: + requests: + cpu: "1" + memory: 84Gi + limits: + memory: 88Gi + +metrics: + serviceMonitor: + enabled: true + labels: + release: kube-prometheus-stack + +networkPolicy: + enabled: true + allowExternal: false + extraIngress: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: applications +``` + +Create the password Secret without committing the password: + +```sh +kubectl create namespace redis +kubectl create secret generic redis-auth \ + --namespace redis \ + --from-literal=redis-password="$(openssl rand -base64 32)" +``` + +Install: + +```sh +helm upgrade --install redis . \ + --namespace redis \ + --create-namespace \ + --values production-values.yaml +``` + +Keep `redis.maxmemory` below the container memory limit. Redis needs headroom +for replication buffers, client buffers, fork/copy-on-write activity, AOF +rewrites, allocator overhead, and the operating process. + +## Connecting clients + +Redis Cluster requires a cluster-aware client. A standalone or Sentinel client +will not correctly follow `MOVED` and `ASK` redirects. + +The internal balanced bootstrap endpoint is: + +```text +-redis-cluster..svc: +``` + +For the example release: + +```text +redis-redis-cluster.redis.svc:6379 +``` + +The bootstrap Service only provides an initial node. The client downloads the +slot map and then connects directly to the member endpoints advertised by +Redis. + +### redis-cli + +```sh +kubectl run redis-cli --rm -it \ + --namespace redis \ + --image=redis:7.4-alpine \ + -- redis-cli -c -h redis-redis-cluster -p 6379 +``` + +Add `-a "$REDIS_PASSWORD"` when authentication is enabled. + +### Python + +```python +from redis.cluster import RedisCluster + +client = RedisCluster( + host="redis-redis-cluster.redis.svc", + port=6379, + password="replace-me", + decode_responses=True, +) + +client.set("hello", "world") +print(client.get("hello")) +``` + +Set `read_from_replicas=True` only if stale replica reads are acceptable. + +### Multi-key operations + +Transactions, Lua scripts, and multi-key commands can only span keys in one hash +slot. Use a shared hash tag when keys must be colocated: + +```text +cart:{customer-42} +cart-items:{customer-42} +``` + +## Authentication + +Authentication is disabled by default so the chart can be evaluated without +secret bootstrapping. Enable it for production. + +### Explicit password + +```yaml +auth: + enabled: true + password: replace-me +``` + +This places the password in Helm release data and is not recommended for Git. + +### Existing Secret + +```yaml +auth: + enabled: true + existingSecret: redis-auth + existingSecretKey: redis-password +``` + +The Secret must exist in the release namespace: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: redis-auth + namespace: redis +type: Opaque +stringData: + redis-password: CHANGE-ME +``` + +Use SOPS, Sealed Secrets, External Secrets, or another secret manager rather +than committing the cleartext example. + +### Generated password + +When `auth.enabled=true` and neither `password` nor `existingSecret` is set, the +chart generates a password. A normal Helm upgrade uses `lookup` to reuse the +existing Secret. + +Retrieve it with: + +```sh +kubectl get secret -n redis redis-redis-cluster \ + -o jsonpath='{.data.redis-password}' | base64 -d +echo +``` + +Argo CD renders charts without live-cluster access, so generated passwords need +the `ignoreDifferences` treatment shown in +[`argocd/application.yaml`](argocd/application.yaml), or preferably an +externally managed Secret. + +The chart configures Redis `requirepass` and `masterauth`. It does not currently +manage Redis ACL users or Redis TLS. + +## Persistence + +Persistence is enabled by default. Every Redis member receives an independent +PVC and writes an append-only file using `appendfsync everysec`. + +```yaml +persistence: + enabled: true + storageClass: "" + accessModes: + - ReadWriteOnce + size: 3Gi +``` + +An empty `storageClass` uses the cluster default. Any valid Kubernetes storage +quantity is accepted, including `500Mi`, `100Gi`, and `2Ti`. The chart imposes +no upper size limit. Actual capacity is controlled by the StorageClass, +provisioner, namespace quotas, and infrastructure. + +`persistence.size` is per member, not total capacity. With three shards and one +replica, `100Gi` requests six 100 Gi volumes. + +When persistence is disabled, Redis uses `emptyDir` and all data on a member is +lost when its pod is rescheduled. + +### Expanding existing volumes + +Increasing the value for a new installation is straightforward. Existing +StatefulSet `volumeClaimTemplates` are immutable, so changing +`persistence.size` alone may be rejected and does not reliably resize existing +claims. + +To expand an existing deployment: + +1. Confirm the StorageClass has `allowVolumeExpansion: true`. +2. Back up the cluster. +3. Patch each existing PVC to the new requested size. +4. Wait for the storage controller and filesystem resize to complete. +5. Keep `persistence.size` at the new value for future members. + +Never shrink Redis PVCs in place. + +## External access + +External access is disabled by default because Redis Cluster cannot be exposed +correctly through a single generic TCP LoadBalancer. + +When enabled, the chart creates: + +- one balanced bootstrap LoadBalancer Service; and +- one LoadBalancer Service for every Redis member. + +The default 3x1 topology therefore creates seven LoadBalancers. This can incur +cloud-provider cost and consume seven addresses. + +```yaml +externalAccess: + enabled: true + port: 6379 + loadBalancerClass: "" + externalTrafficPolicy: Cluster + loadBalancerSourceRanges: + - 203.0.113.0/24 +``` + +The chart does not assume MetalLB or a particular cloud provider. It waits for +each member Service to publish either an IP or hostname under +`.status.loadBalancer.ingress`, then makes that member advertise the discovered +endpoint. All Services use the same Redis port; there is no fixed IP list and +no port fan-out. + +Get the bootstrap endpoint: + +```sh +kubectl get service -n redis redis-redis-cluster-external \ + -o jsonpath='{range .status.loadBalancer.ingress[*]}{.ip}{.hostname}{"\n"}{end}' +``` + +External clients need only that bootstrap endpoint in their startup +configuration, but they must be able to route to every per-member address +returned in the slot map. A client that can reach the bootstrap address but not +the member addresses will connect initially and then fail on redirected keys. + +Keep the LoadBalancer Services instead of deleting and recreating them. If the +controller assigns a new member address, restart that member's one-pod +StatefulSet so its discovery init container records and advertises the new +endpoint. + +External Redis traffic is unencrypted. Restrict +`loadBalancerSourceRanges`, use upstream firewalls/private routing, enable +authentication, or provide network-layer encryption. Do not expose an +unauthenticated cluster to the public Internet. + +### Provider annotations + +Bootstrap and member annotations are separate: + +```yaml +externalAccess: + bootstrap: + annotations: + service.beta.kubernetes.io/example: bootstrap-value + memberServices: + annotations: + service.beta.kubernetes.io/example: member-value +``` + +Use these for provider-specific address pools, internal LoadBalancers, health +checks, or DNS controllers. + +## RedisInsight + +RedisInsight is optional and disabled by default: + +```yaml +redisInsight: + enabled: true +``` + +It creates one Deployment, one ClusterIP Service, and optionally a dedicated +PVC. RedisInsight stores its SQLite state, saved connections, and logs under +`/data`. + +### Automatic cluster connection + +No IP address is required. With `redisInsight.connection.host=""`, the chart +preconfigures RedisInsight from the release-aware balanced bootstrap endpoint: + +- with external access disabled, it uses the internal + `-redis-cluster` Service DNS; +- with external access enabled, an init container reads the published IP or + hostname from the `-redis-cluster-external` LoadBalancer Service and + injects that address before RedisInsight starts. + +Kubernetes and the LoadBalancer controller therefore own the VIP lifecycle. +RedisInsight follows the correct release and namespace without a hard-coded +address. Port `0` automatically follows `redis.port` or +`externalAccess.port`. + +```yaml +redisInsight: + enabled: true + connection: + host: "" + port: 0 + alias: redis-cluster + tls: false +``` + +Set `host` and `port` when RedisInsight should inspect another cluster, or when +pods cannot hairpin through the external LoadBalancer endpoint. When chart +authentication is enabled, RedisInsight reads the same password Secret +automatically. + +Port-forward the UI: + +```sh +kubectl port-forward -n redis \ + service/redis-redis-cluster-insight 5540:80 +``` + +Open . + +### Istio HTTP ingress + +The chart can create an Istio Gateway and VirtualService for RedisInsight: + +```yaml +redisInsight: + enabled: true + ingress: + enabled: true + host: redisinsight.example.com + gatewaySelector: + istio: ingressgateway + requestTimeout: 60s + tls: + enabled: false +``` + +The host is required when ingress is enabled. Hosting RedisInsight under a +rewritten path prefix is not supported; give it a host and route `/`. + +### Istio TLS with an existing Secret + +```yaml +redisInsight: + ingress: + enabled: true + host: redisinsight.example.com + tls: + enabled: true + credentialName: redisinsight-tls + certificate: + create: false +``` + +The credential Secret must be available to the selected Istio ingress gateway. +Exact Secret placement depends on the Istio installation. + +### Istio TLS with cert-manager + +```yaml +redisInsight: + ingress: + enabled: true + host: redisinsight.example.com + tls: + enabled: true + credentialName: redisinsight-tls + certificate: + create: true + issuerName: letsencrypt-production + issuerKind: ClusterIssuer + secretNamespace: istio-system +``` + +Set `secretNamespace` to the namespace from which the ingress gateway reads +credential Secrets. An empty value uses the Helm release namespace. + +RedisInsight is an administrative interface and does not provide ingress +authentication. Put an identity-aware proxy or Istio authorization policy in +front of it and restrict network exposure. + +## Metrics + +The Redis exporter sidecar is enabled by default on every member: + +```yaml +metrics: + enabled: true + port: 9121 +``` + +Enable a ServiceMonitor when the Prometheus Operator CRDs are installed: + +```yaml +metrics: + serviceMonitor: + enabled: true + interval: 30s + scrapeTimeout: 10s + labels: + release: kube-prometheus-stack +``` + +Use labels selected by your Prometheus installation. + +## Network policy and service mesh + +### NetworkPolicy + +NetworkPolicy creation is disabled by default because policy enforcement and +default-deny behavior vary between CNIs. + +```yaml +networkPolicy: + enabled: true + allowExternal: false + extraIngress: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: applications + metricsFromNamespaces: + - monitoring +``` + +The generated policy always allows Redis client, replication, migration, and +cluster-bus traffic between release pods. `allowExternal` refers to Redis client +ingress from outside the selected pods; it does not create a LoadBalancer. + +### Istio sidecars + +Redis pods disable Istio injection by default: + +```yaml +istio: + injectSidecar: false +``` + +Redis Cluster uses direct node-to-node client and cluster-bus connections. +Namespace-wide strict mTLS can break this traffic. Either keep the Redis +workload outside strict mesh enforcement, create a scoped PERMISSIVE +PeerAuthentication, or deliberately enable and validate sidecars before +production use. + +## Scheduling and availability + +The chart uses required pod affinity and anti-affinity, not preferences. It will +leave pods Pending rather than silently place a primary beside its own replica. + +For the default topology: + +- three workers are required; +- losing one worker removes one primary and a replica of another shard; +- the lost primary's replica is on a surviving worker; +- two surviving primaries form the majority needed to authorize promotion. + +Inspect placement: + +```sh +kubectl get pods -n redis \ + -L redis-cluster.shard,redis-cluster.member \ + -o wide +``` + +The two members of each shard must have different `NODE` values. All member-0 +pods must also occupy different nodes. + +The global PodDisruptionBudget defaults to `maxUnavailable: 1`. It limits +cooperating voluntary disruptions but cannot prevent node crashes or an +administrator from deleting multiple pods. + +### Failure behavior + +- Failure detection begins after approximately + `cluster.nodeTimeoutMilliseconds`. +- A healthy cross-node replica can be elected primary. +- Cluster-aware clients refresh the slot map and reconnect. +- Losing both members of one shard loses that slot range. +- With `cluster.requireFullCoverage=true`, loss of any slot range makes the + cluster unavailable until coverage returns. +- Automatic failover requires a majority of current primaries to communicate. + +## Safe upgrades + +Every member StatefulSet uses `updateStrategy: OnDelete`. A Helm or Argo update +changes the pod template but deliberately does not restart all independent +members at once. + +Restart exactly one member at a time: + +1. Inspect `CLUSTER NODES` and choose a current replica first. +2. Delete that one pod. +3. Wait for the replacement to become Ready. +4. Confirm `cluster_state:ok`. +5. Continue with the next replica, then primaries one by one. + +Example: + +```sh +kubectl exec -n redis redis-redis-cluster-s0-n0-0 -c redis -- \ + redis-cli cluster nodes + +kubectl delete pod -n redis redis-redis-cluster-s0-n1-0 + +kubectl wait -n redis \ + --for=condition=Ready \ + pod/redis-redis-cluster-s0-n1-0 \ + --timeout=300s + +kubectl exec -n redis redis-redis-cluster-s0-n0-0 -c redis -- \ + redis-cli cluster info +``` + +The preStop hook asks a healthy replica to take over before an intentional +primary restart. Never delete both members of one shard together. + +## Scaling and topology changes + +`cluster.shards` and `cluster.replicasPerShard` define the first-boot topology. +Changing them in Helm on a populated cluster does not reshard existing data and +is not a supported scaling procedure. + +Adding a shard requires an explicit operational workflow: + +1. create and persist the new members; +2. join them with `CLUSTER MEET`; +3. assign replicas; +4. move slots with `redis-cli --cluster reshard`; +5. verify slot coverage and failure placement. + +Removing a shard requires moving all of its slots away before removing its +nodes. Adding replicas requires joining the nodes and issuing +`CLUSTER REPLICATE`. Back up first. + +The chart enforces at least three shards, at least one replica per shard, and +fewer replicas per shard than shards so the crossed placement remains valid. + +## Backup and restore + +This chart does not schedule backups. Redis Cluster stores different slot ranges +on different primaries, so a backup strategy must cover every shard. + +Common approaches include: + +- crash-consistent snapshots of all member volumes; +- coordinated AOF/RDB copies from one healthy member of every shard; +- a Redis-aware backup product that understands cluster topology. + +Test restore procedures. Restoring only one shard or one arbitrary member does +not reconstruct the complete keyspace. Keep backups outside the cluster and +verify both data and all 16,384 slots after recovery. + +## Argo CD + +[`argocd/application.yaml`](argocd/application.yaml) is a portable example using +the public repository. Apply it once: + +```sh +kubectl apply -f argocd/application.yaml +``` + +The file is outside `templates/`, so it is not part of the Helm release. If you +edit its inline `valuesObject`, a normal sync of the Redis Application cannot +update the Application's own spec. Reapply the file, manage it from an +app-of-apps, or use an ApplicationSet: + +```sh +kubectl apply -f argocd/application.yaml +``` + +For authentication, prefer an externally managed Secret. If the chart generates +the password, retain `RespectIgnoreDifferences=true` and the Secret +`ignoreDifferences` rule from the example. + +Argo applies new StatefulSet templates, but `OnDelete` intentionally prevents +automatic Redis pod rolls. Perform the one-member-at-a-time procedure after +reviewing each change. Additional authentication, external access, and Istio +examples are in [`argocd/README.md`](argocd/README.md). + +## Configuration reference + +The authoritative defaults are in [`values.yaml`](values.yaml), and +[`values.schema.json`](values.schema.json) validates supported values. + +### Release and image + +| Value | Default | Description | +| --- | --- | --- | +| `nameOverride` | `""` | Override the chart name portion of resource names. | +| `fullnameOverride` | `""` | Override the complete release resource prefix; maximum 53 characters. | +| `image.repository` | `redis` | Redis image repository. | +| `image.tag` | `7.4-alpine` | Redis image tag. | +| `image.pullPolicy` | `IfNotPresent` | Redis image pull policy. | +| `imagePullSecrets` | `[]` | Pod image pull Secret references. | + +### Cluster + +| Value | Default | Description | +| --- | --- | --- | +| `cluster.shards` | `3` | Independently writable primaries and slot ranges. | +| `cluster.replicasPerShard` | `1` | Readable failover replicas per shard. | +| `cluster.busPort` | `16379` | Redis node-to-node cluster bus port. | +| `cluster.nodeTimeoutMilliseconds` | `10000` | Failure-detection timeout. | +| `cluster.requireFullCoverage` | `true` | Stop serving when any slot is unavailable. | +| `cluster.allowReadsWhenDown` | `false` | Permit reads while cluster state is down. | +| `cluster.migrationBarrier` | `1` | Minimum healthy replicas retained during replica migration. | + +### Redis and authentication + +| Value | Default | Description | +| --- | --- | --- | +| `auth.enabled` | `false` | Enable password authentication. | +| `auth.password` | `""` | Explicit password; generated when empty and no Secret is supplied. | +| `auth.existingSecret` | `""` | Existing password Secret name. | +| `auth.existingSecretKey` | `redis-password` | Password key in the existing Secret. | +| `redis.port` | `6379` | Redis client port. | +| `redis.maxmemory` | `384mb` | Redis memory ceiling. Empty disables the explicit ceiling. | +| `redis.maxmemoryPolicy` | `noeviction` | Redis eviction policy. | +| `redis.extraConfig` | `""` | Additional lines appended to `redis.conf`. | +| `redis.resources` | see values | Redis container requests and limits. | + +Do not use `redis.extraConfig` to override cluster identity, announce endpoints, +data directory, authentication, or other settings managed by the chart. + +### Probes + +| Value | Default | Description | +| --- | --- | --- | +| `probes.redis.startup.initialDelaySeconds` | `10` | Delay before startup checks. | +| `probes.redis.startup.periodSeconds` | `5` | Startup check interval. | +| `probes.redis.startup.timeoutSeconds` | `4` | Startup check timeout. | +| `probes.redis.startup.failureThreshold` | `180` | Failures allowed before restart. | +| `probes.redis.liveness.periodSeconds` | `15` | Liveness check interval. | +| `probes.redis.liveness.timeoutSeconds` | `8` | Liveness check timeout. | +| `probes.redis.liveness.failureThreshold` | `10` | Liveness failures before restart. | +| `probes.redis.readiness.periodSeconds` | `5` | Readiness check interval. | +| `probes.redis.readiness.timeoutSeconds` | `4` | Readiness check timeout. | +| `probes.redis.readiness.failureThreshold` | `3` | Readiness failures before removal from Services. | + +### Persistence + +| Value | Default | Description | +| --- | --- | --- | +| `persistence.enabled` | `true` | Create one PVC per Redis member. | +| `persistence.storageClass` | `""` | StorageClass; empty uses the cluster default. | +| `persistence.accessModes` | `[ReadWriteOnce]` | PVC access modes. | +| `persistence.size` | `3Gi` | Capacity per member; no chart-side maximum. | +| `persistence.annotations` | `{}` | PVC template annotations. | + +### External access + +| Value | Default | Description | +| --- | --- | --- | +| `externalAccess.enabled` | `false` | Create bootstrap and per-member LoadBalancers. | +| `externalAccess.port` | `6379` | External Redis port on every Service. | +| `externalAccess.bootstrap.annotations` | `{}` | Bootstrap Service annotations. | +| `externalAccess.memberServices.annotations` | `{}` | Per-member Service annotations. | +| `externalAccess.loadBalancerClass` | `""` | Optional LoadBalancer implementation class. | +| `externalAccess.externalTrafficPolicy` | `Cluster` | Kubernetes external traffic policy. | +| `externalAccess.loadBalancerSourceRanges` | `[]` | Allowed client CIDRs. | +| `externalAccess.allocateLoadBalancerNodePorts` | `true` | Allocate backing NodePorts. | +| `externalAccess.endpointDiscovery.*` | see values | Image/resources for endpoint discovery. | + +### RedisInsight + +| Value | Default | Description | +| --- | --- | --- | +| `redisInsight.enabled` | `false` | Deploy RedisInsight. | +| `redisInsight.image.repository` | `redis/redisinsight` | RedisInsight image repository. | +| `redisInsight.image.tag` | `3.8.0` | RedisInsight image tag. | +| `redisInsight.image.pullPolicy` | `IfNotPresent` | RedisInsight image pull policy. | +| `redisInsight.port` | `5540` | RedisInsight container port. | +| `redisInsight.connection.host` | `""` | Empty discovers the balanced release bootstrap endpoint. | +| `redisInsight.connection.port` | `0` | Zero follows the selected Redis Service port. | +| `redisInsight.connection.alias` | `redis-cluster` | Display name in RedisInsight. | +| `redisInsight.connection.tls` | `false` | Enable TLS for the Redis connection. | +| `redisInsight.persistence.enabled` | `true` | Persist RedisInsight state. | +| `redisInsight.persistence.storageClass` | `""` | StorageClass; empty uses default. | +| `redisInsight.persistence.accessModes` | `[ReadWriteOnce]` | RedisInsight PVC access modes. | +| `redisInsight.persistence.size` | `1Gi` | RedisInsight PVC size; no chart-side maximum. | +| `redisInsight.persistence.annotations` | `{}` | RedisInsight PVC annotations. | +| `redisInsight.resources` | see values | RedisInsight requests and limits. | +| `redisInsight.podAnnotations` | `{}` | RedisInsight pod annotations. | +| `redisInsight.nodeSelector` | `{}` | RedisInsight node selector. | +| `redisInsight.tolerations` | `[]` | RedisInsight pod tolerations. | +| `redisInsight.ingress.enabled` | `false` | Create Istio Gateway and VirtualService. | +| `redisInsight.ingress.host` | `""` | Required hostname when ingress is enabled. | +| `redisInsight.ingress.gatewaySelector` | `{istio: ingressgateway}` | Select the Istio gateway workload. | +| `redisInsight.ingress.requestTimeout` | `60s` | Istio route timeout. | +| `redisInsight.ingress.tls.enabled` | `false` | Serve HTTPS on the Gateway. | +| `redisInsight.ingress.tls.credentialName` | `""` | Istio TLS credential Secret. | +| `redisInsight.ingress.tls.certificate.create` | `false` | Create the Secret using cert-manager. | +| `redisInsight.ingress.tls.certificate.*` | see values | Issuer and Secret namespace settings. | + +### Metrics, policy, and pod settings + +| Value | Default | Description | +| --- | --- | --- | +| `metrics.enabled` | `true` | Add redis_exporter to every member pod. | +| `metrics.image.repository` | `ghcr.io/oliver006/redis_exporter` | Exporter image repository. | +| `metrics.image.tag` | `v1.87.0-alpine` | Exporter image tag. | +| `metrics.image.pullPolicy` | `IfNotPresent` | Exporter pull policy. | +| `metrics.port` | `9121` | Exporter port. | +| `metrics.resources` | see values | Exporter requests and limits. | +| `metrics.serviceMonitor.enabled` | `false` | Create a Prometheus Operator ServiceMonitor. | +| `metrics.serviceMonitor.interval` | `30s` | Prometheus scrape interval. | +| `metrics.serviceMonitor.scrapeTimeout` | `10s` | Prometheus scrape timeout. | +| `metrics.serviceMonitor.labels` | `{}` | Labels used by Prometheus to select the ServiceMonitor. | +| `networkPolicy.enabled` | `false` | Create an ingress NetworkPolicy. | +| `networkPolicy.allowExternal` | `true` | Allow Redis client ingress from all sources. | +| `networkPolicy.extraIngress` | `[]` | Allowed peers when unrestricted ingress is disabled. | +| `networkPolicy.metricsFromNamespaces` | `[]` | Namespaces allowed to scrape metrics. | +| `podDisruptionBudget.enabled` | `true` | Create a global Redis PDB. | +| `podDisruptionBudget.maxUnavailable` | `1` | Maximum voluntary unavailable Redis pods. | +| `serviceAccount.create` | `true` | Create the endpoint-discovery ServiceAccount. | +| `serviceAccount.name` | `""` | Existing or generated ServiceAccount name. | +| `serviceAccount.annotations` | `{}` | ServiceAccount annotations. | +| `nodeSelector` | `{}` | Redis pod node selector. | +| `tolerations` | `[]` | Redis pod tolerations. | +| `topologySpreadConstraints` | `[]` | Additional spread constraints. | +| `priorityClassName` | `""` | Redis pod priority class. | +| `podAnnotations` / `podLabels` | `{}` | Additional Redis pod metadata. | +| `podSecurityContext` | see values | Redis pod-level security context. | +| `containerSecurityContext` | see values | Redis/exporter container security context. | +| `istio.injectSidecar` | `false` | Enable sidecars on Redis member pods. | +| `terminationGracePeriodSeconds` | `30` | Pod shutdown grace period. | + +## Troubleshooting + +### Pods remain Pending + +Inspect scheduling events: + +```sh +kubectl describe pod -n redis +``` + +Common causes are: + +- fewer schedulable workers than `cluster.shards`; +- insufficient CPU or memory; +- node taints without matching tolerations; +- node selectors excluding required workers; +- existing pods violating the required crossed affinity layout; +- unbound PVCs or a missing default StorageClass. + +The scheduler message `didn't match pod affinity/anti-affinity rules` means the +availability invariant cannot currently be satisfied. Do not weaken it unless +you accept a primary and its replica sharing a failure domain. + +### External-access pods wait in init + +```sh +kubectl logs -n redis -c discover-external-endpoint +kubectl get services -n redis +``` + +The LoadBalancer controller must populate an IP or hostname for every member +Service. Check controller events, address-pool capacity, provider annotations, +quota, and `loadBalancerClass`. + +### `cluster_state:fail` + +```sh +kubectl exec -n redis redis-redis-cluster-s0-n0-0 -c redis -- \ + redis-cli cluster nodes + +kubectl exec -n redis redis-redis-cluster-s0-n0-0 -c redis -- \ + redis-cli cluster info +``` + +Check for missing members, failed slot ranges, DNS failures, blocked port 16379, +or a member that cannot read its PVC. Readiness intentionally remains false +until the complete cluster is healthy; liveness does not restart Redis merely +for `CLUSTERDOWN`. + +### Authentication errors + +Confirm the client is using the current Secret and that all pods reference the +same password: + +```sh +kubectl get secret -n redis redis-redis-cluster \ + -o jsonpath='{.data.redis-password}' | base64 -d +echo +``` + +When using an existing Secret, verify its name/key match the Helm values. Restart +members one at a time if a password Secret was deliberately changed. + +### External client connects, then fails + +This almost always means the bootstrap address is reachable but the member +addresses in `CLUSTER SLOTS`/`CLUSTER SHARDS` are not. Route and permit every +per-member LoadBalancer endpoint. + +### RedisInsight is not rendered by Argo CD + +Confirm the live Application contains `redisInsight.enabled: true`: + +```sh +kubectl get application -n argocd redis-cluster -o yaml +``` + +Changing `argocd/application.yaml` in this repository does not update the live +Application during its own Helm sync because the file is outside `templates/`. +Reapply it or manage it from an app-of-apps. + +### Inspect rendered manifests + +```sh +helm lint . +helm template redis . --namespace redis --debug +``` + +## Uninstalling + +Back up first, then remove the release: + +```sh +helm uninstall redis --namespace redis +``` + +PVCs created from StatefulSet volume claim templates are normally retained when +the StatefulSets are deleted. Verify them before deleting anything: + +```sh +kubectl get pvc -n redis +``` + +The directly managed RedisInsight PVC may be removed with the Helm release, +depending on Helm and storage-policy behavior. Preserve or snapshot it if its UI +state matters. + +Delete retained Redis data PVCs only when data destruction is intentional. + +## License + +This project is distributed under the BSD 3-Clause License. See +[`LICENSE`](LICENSE). diff --git a/argocd/README.md b/argocd/README.md new file mode 100644 index 0000000..bb13dcb --- /dev/null +++ b/argocd/README.md @@ -0,0 +1,84 @@ +# Argo CD examples + +This directory is excluded from the packaged Helm chart. Its files are examples +for managing the chart from Argo CD. + +## Application + +[`application.yaml`](application.yaml) deploys: + +- three Redis shards; +- one replica per shard; +- authenticated Redis with a chart-generated initial password; +- persistent volumes using the cluster's default StorageClass; +- internal-only Redis Services; +- RedisInsight connected automatically to the balanced bootstrap endpoint. + +Apply it once: + +```sh +kubectl apply -f argocd/application.yaml +``` + +Edit the destination namespace, target revision, and inline `valuesObject` as +needed. Because this file is not a Helm template, changing it in the Redis +Application's own repository does not change the live Application spec during a +normal sync. Reapply it, or manage it from an app-of-apps/ApplicationSet. + +## Password Secret + +For production GitOps, use an externally managed Secret and set: + +```yaml +auth: + enabled: true + existingSecret: redis-auth + existingSecretKey: redis-password +``` + +[`redis-auth-secret.example.yaml`](redis-auth-secret.example.yaml) documents the +required Secret shape. Do not commit an actual cleartext password. + +If the chart generates the password, retain +`RespectIgnoreDifferences=true` and the Secret ignore rule from +`application.yaml`; Argo's Helm renderer cannot use `lookup` to read the live +Secret while comparing desired state. + +## Enabling external access + +Add this to `valuesObject`: + +```yaml +externalAccess: + enabled: true + loadBalancerClass: "" + loadBalancerSourceRanges: + - 203.0.113.0/24 +``` + +This creates one balanced bootstrap LoadBalancer and one LoadBalancer per Redis +member. Every advertised member address must be routable from external clients. + +## Exposing RedisInsight with Istio + +Add: + +```yaml +redisInsight: + enabled: true + ingress: + enabled: true + host: redisinsight.example.com + tls: + enabled: true + credentialName: redisinsight-tls + certificate: + create: true + issuerName: letsencrypt-production + issuerKind: ClusterIssuer + secretNamespace: istio-system +``` + +Replace the hostname, issuer, gateway selector, and Secret namespace with values +from your environment. RedisInsight has no built-in ingress authentication; add +an identity-aware proxy or Istio authorization policy before exposing it. diff --git a/argocd/application.yaml b/argocd/application.yaml new file mode 100644 index 0000000..08cba89 --- /dev/null +++ b/argocd/application.yaml @@ -0,0 +1,77 @@ +# Public Argo CD example. Adjust the destination namespace, revision and values +# for your environment, then apply this file once (or manage it from an +# app-of-apps). Files under argocd/ are examples and are not rendered by Helm. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: redis-cluster + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://git.mulas.me/corrado/redis-cluster.git + targetRevision: master + path: . + helm: + releaseName: redis + valuesObject: + cluster: + shards: 3 + replicasPerShard: 1 + + # Production deployments should authenticate even when Redis is only + # reachable inside the cluster. This example lets the chart create the + # initial password Secret; see the ignore rule below. + auth: + enabled: true + + persistence: + # Empty selects the cluster's default StorageClass. + storageClass: "" + # This is per Redis member. The chart imposes no maximum. + size: 10Gi + + # Disabled by default because this creates one LoadBalancer per member + # plus one balanced bootstrap LoadBalancer. + externalAccess: + enabled: false + + metrics: + serviceMonitor: + enabled: false + + # RedisInsight automatically discovers the release's balanced + # bootstrap endpoint. No IP address is required. + redisInsight: + enabled: true + persistence: + storageClass: "" + size: 2Gi + ingress: + enabled: false + + destination: + server: https://kubernetes.default.svc + namespace: redis + + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - RespectIgnoreDifferences=true + + # Argo CD renders Helm without API access, so Helm's lookup cannot recover a + # previously generated password during comparison. Ignore the generated + # Secret data to keep the first password stable. Prefer an existing Secret + # managed by SOPS, Sealed Secrets or External Secrets for serious GitOps use. + ignoreDifferences: + - group: "" + kind: Secret + name: redis-redis-cluster + namespace: redis + jsonPointers: + - /data/redis-password diff --git a/argocd/redis-auth-secret.example.yaml b/argocd/redis-auth-secret.example.yaml new file mode 100644 index 0000000..36bcc85 --- /dev/null +++ b/argocd/redis-auth-secret.example.yaml @@ -0,0 +1,19 @@ +# OPTIONAL — the clean GitOps way to handle the password: create the secret +# out-of-band (or via SealedSecrets/SOPS/ExternalSecrets), then set +# auth.existingSecret: redis-auth in the Application's valuesObject. +# +# Do NOT commit a real password to git. Either apply this once by hand with a +# real value: +# +# kubectl create secret generic redis-auth -n redis \ +# --from-literal=redis-password="$(openssl rand -base64 24)" +# +# ...or encrypt this file with your secret-management tool of choice. +apiVersion: v1 +kind: Secret +metadata: + name: redis-auth + namespace: redis +type: Opaque +stringData: + redis-password: CHANGE-ME diff --git a/templates/NOTES.txt b/templates/NOTES.txt new file mode 100644 index 0000000..3ce2c34 --- /dev/null +++ b/templates/NOTES.txt @@ -0,0 +1,63 @@ +Native Redis Cluster deployed: {{ .Values.cluster.shards }} writable shards, +{{ .Values.cluster.replicasPerShard }} replica(s) per shard, +{{ include "redis-cluster.nodeCount" . }} total Redis instances. + +Every shard's persistent members have REQUIRED hostname anti-affinity. A shard +cannot place two copies on one Kubernetes worker. + +{{- if .Values.auth.enabled }} + +Get the password: + + kubectl get secret -n {{ .Release.Namespace }} {{ include "redis-cluster.secretName" . }} \ + -o jsonpath='{.data.{{ include "redis-cluster.secretKey" . }}}' | base64 -d +{{- end }} +{{- if .Values.redisInsight.enabled }} + +RedisInsight: +{{- if .Values.redisInsight.ingress.enabled }} + {{ ternary "https" "http" .Values.redisInsight.ingress.tls.enabled }}://{{ .Values.redisInsight.ingress.host }} +{{- else }} + kubectl port-forward -n {{ .Release.Namespace }} service/{{ include "redis-cluster.redisInsightName" . }} 5540:80 + http://127.0.0.1:5540 +{{- end }} +{{- end }} + +Cluster-aware client startup endpoint: + + {{ include "redis-cluster.fullname" . }}.{{ .Release.Namespace }}.svc:{{ .Values.redis.port }} +{{- if .Values.externalAccess.enabled }} + +External cluster-aware client startup endpoint: + + kubectl get service -n {{ .Release.Namespace }} \ + {{ include "redis-cluster.fullname" . }}-external \ + -o jsonpath='{.status.loadBalancer.ingress[0].ip}{.status.loadBalancer.ingress[0].hostname}' + +Use that returned address on port {{ .Values.externalAccess.port }}. + +The external bootstrap LoadBalancer is discovery only. Redis advertises one +dynamic LoadBalancer VIP per member on the same port, so MOVED/ASK redirects +remain routable from outside Kubernetes without fixed addresses or port fan-out. +{{- end }} + +Check all 16384 slots and primary/replica assignments: + + kubectl exec -n {{ .Release.Namespace }} \ + {{ include "redis-cluster.fullname" . }}-s0-n0-0 -c redis -- \ + redis-cli --cluster check 127.0.0.1:{{ .Values.redis.port }} + +Do not use a standalone or Sentinel client. The client must support Redis +Cluster redirects and connect to the per-node hostnames returned in the slot map. + +StatefulSets use updateStrategy=OnDelete so independent members cannot all roll +at once. Follow the one-member-at-a-time upgrade procedure in README.md. + +{{- if .Values.metrics.enabled }} + +Metrics: http://:{{ .Values.metrics.port }}/metrics (redis_exporter). +{{- if not .Values.metrics.serviceMonitor.enabled }} +Set metrics.serviceMonitor.enabled=true (plus the correct selector labels) when +using Prometheus Operator. +{{- end }} +{{- end }} diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl new file mode 100644 index 0000000..99fcfeb --- /dev/null +++ b/templates/_helpers.tpl @@ -0,0 +1,154 @@ +{{/* Chart name */}} +{{- define "redis-cluster.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Fully qualified app name */}} +{{- define "redis-cluster.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Chart label */}} +{{- define "redis-cluster.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Common labels */}} +{{- define "redis-cluster.labels" -}} +helm.sh/chart: {{ include "redis-cluster.chart" . }} +{{ include "redis-cluster.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* Selector labels */}} +{{- define "redis-cluster.selectorLabels" -}} +app.kubernetes.io/name: {{ include "redis-cluster.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Headless service DNS name. Deliberately stop at .svc: the cluster DNS suffix is +not necessarily cluster.local, and Kubernetes expands this portable name using +the pod's search domains. +*/}} +{{- define "redis-cluster.headlessFqdn" -}} +{{ include "redis-cluster.fullname" . }}-headless.{{ .Release.Namespace }}.svc +{{- end -}} + +{{/* Service account name */}} +{{- define "redis-cluster.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} +{{- default (include "redis-cluster.fullname" .) .Values.serviceAccount.name -}} +{{- else -}} +{{- default "default" .Values.serviceAccount.name -}} +{{- end -}} +{{- end -}} + +{{/* Secret name holding the password */}} +{{- define "redis-cluster.secretName" -}} +{{- if .Values.auth.existingSecret -}} +{{- .Values.auth.existingSecret -}} +{{- else -}} +{{- include "redis-cluster.fullname" . -}} +{{- end -}} +{{- end -}} + +{{/* Secret key holding the password */}} +{{- define "redis-cluster.secretKey" -}} +{{- if .Values.auth.existingSecret -}} +{{- .Values.auth.existingSecretKey -}} +{{- else -}} +redis-password +{{- end -}} +{{- end -}} + +{{/* +Password: explicit value wins; otherwise reuse the one already in the cluster +(so upgrades don't rotate it); otherwise generate. +*/}} +{{- define "redis-cluster.password" -}} +{{- if .Values.auth.password -}} +{{- .Values.auth.password -}} +{{- else -}} +{{- $existing := lookup "v1" "Secret" .Release.Namespace (include "redis-cluster.fullname" .) -}} +{{- if and $existing $existing.data (index $existing.data "redis-password") -}} +{{- index $existing.data "redis-password" | b64dec -}} +{{- else -}} +{{- randAlphaNum 24 -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Total Redis nodes: shards * (one primary + replicas per shard). */}} +{{- define "redis-cluster.nodeCount" -}} +{{- mul (int .Values.cluster.shards) (add (int .Values.cluster.replicasPerShard) 1) -}} +{{- end -}} + +{{/* Balanced bootstrap Service used by cluster-aware clients. */}} +{{- define "redis-cluster.bootstrapServiceName" -}} +{{- if .Values.externalAccess.enabled -}} +{{- printf "%s-external" (include "redis-cluster.fullname" .) -}} +{{- else -}} +{{- include "redis-cluster.fullname" . -}} +{{- end -}} +{{- end -}} + +{{/* RedisInsight resource name. */}} +{{- define "redis-cluster.redisInsightName" -}} +{{- printf "%s-insight" (include "redis-cluster.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* RedisInsight labels deliberately do not match the Redis pod selector. */}} +{{- define "redis-cluster.redisInsightLabels" -}} +helm.sh/chart: {{ include "redis-cluster.chart" . }} +app.kubernetes.io/name: redisinsight +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: insight +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{- define "redis-cluster.redisInsightSelectorLabels" -}} +app.kubernetes.io/name: redisinsight +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: insight +{{- end -}} + +{{/* Environment shared by Redis lifecycle scripts. */}} +{{- define "redis-cluster.commonEnv" -}} +- name: STS_NAME + value: {{ include "redis-cluster.fullname" . | quote }} +- name: HEADLESS_SERVICE + value: {{ include "redis-cluster.headlessFqdn" . | quote }} +- name: NODE_COUNT + value: {{ include "redis-cluster.nodeCount" . | quote }} +- name: SHARD_COUNT + value: {{ .Values.cluster.shards | quote }} +- name: REPLICAS_PER_SHARD + value: {{ .Values.cluster.replicasPerShard | quote }} +- name: REDIS_PORT + value: {{ .Values.redis.port | quote }} +- name: CLUSTER_BUS_PORT + value: {{ .Values.cluster.busPort | quote }} +{{- if .Values.auth.enabled }} +- name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "redis-cluster.secretName" . }} + key: {{ include "redis-cluster.secretKey" . }} +- name: REDISCLI_AUTH + valueFrom: + secretKeyRef: + name: {{ include "redis-cluster.secretName" . }} + key: {{ include "redis-cluster.secretKey" . }} +{{- end }} +{{- end -}} diff --git a/templates/configmap.yaml b/templates/configmap.yaml new file mode 100644 index 0000000..8872537 --- /dev/null +++ b/templates/configmap.yaml @@ -0,0 +1,381 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "redis-cluster.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.labels" . | nindent 4 }} +data: + redis.conf: | + port {{ .Values.redis.port }} + dir /data + appendonly yes + appendfsync everysec + save "" + protected-mode no + tcp-keepalive 60 + repl-diskless-sync yes + replica-read-only yes + cluster-enabled yes + cluster-config-file nodes.conf + cluster-port {{ .Values.cluster.busPort }} + cluster-node-timeout {{ .Values.cluster.nodeTimeoutMilliseconds }} + cluster-require-full-coverage {{ ternary "yes" "no" .Values.cluster.requireFullCoverage }} + cluster-allow-reads-when-down {{ ternary "yes" "no" .Values.cluster.allowReadsWhenDown }} + cluster-migration-barrier {{ .Values.cluster.migrationBarrier }} + cluster-preferred-endpoint-type hostname + {{- if .Values.redis.maxmemory }} + maxmemory {{ .Values.redis.maxmemory }} + maxmemory-policy {{ .Values.redis.maxmemoryPolicy }} + {{- end }} + {{- with .Values.redis.extraConfig }} + {{- . | nindent 4 }} + {{- end }} + + start-redis.sh: | + #!/bin/sh + set -eu + + POD_FQDN="$(hostname).${HEADLESS_SERVICE}" + if [ "$EXTERNAL_ACCESS_ENABLED" = "true" ]; then + ANNOUNCE_HOSTNAME="$(cat /external-endpoint/hostname)" + fi + /scripts/repair-node-addresses.sh + CONF=/etc/redis-runtime/redis.conf + cp /etc/redis-ro/redis.conf "$CONF" + { + echo "cluster-announce-hostname $ANNOUNCE_HOSTNAME" + echo "cluster-announce-human-nodename $(hostname)" + echo "cluster-announce-port $ANNOUNCE_PORT" + echo "cluster-announce-bus-port $CLUSTER_BUS_PORT" + if [ -n "${REDIS_PASSWORD:-}" ]; then + echo "requirepass $REDIS_PASSWORD" + echo "masterauth $REDIS_PASSWORD" + fi + } >> "$CONF" + + # Only the designated coordinator attempts first-time cluster creation. + # The bootstrap runs beside Redis because every node must be listening + # before CLUSTER MEET can form the topology. + if [ "$POD_FQDN" = "$COORDINATOR_HOST" ]; then + /scripts/bootstrap-cluster.sh & + /scripts/reconcile-topology.sh & + fi + + exec redis-server "$CONF" + + discover-external-endpoint.sh: | + #!/bin/sh + set -eu + + token_file=/var/run/secrets/kubernetes.io/serviceaccount/token + ca_file=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + api="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}" + url="${api}/api/v1/namespaces/${POD_NAMESPACE}/services/${EXTERNAL_SERVICE_NAME}" + + while :; do + response="$(curl --fail --silent --show-error --cacert "$ca_file" \ + -H "Authorization: Bearer $(cat "$token_file")" "$url" 2>/dev/null || true)" + endpoint="$(printf '%s\n' "$response" | + sed -n 's/.*"ip":[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1)" + if [ -z "$endpoint" ]; then + endpoint="$(printf '%s\n' "$response" | + sed -n 's/.*"hostname":[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1)" + fi + if [ -n "$endpoint" ]; then + printf '%s\n' "$endpoint" > /external-endpoint/hostname + echo "Discovered external endpoint $endpoint for $EXTERNAL_SERVICE_NAME" + exit 0 + fi + echo "Waiting for LoadBalancer endpoint on $EXTERNAL_SERVICE_NAME" + sleep 2 + done + + start-redisinsight.sh: | + #!/bin/sh + set -eu + + RI_REDIS_HOST="$(cat /bootstrap-endpoint/hostname)" + export RI_REDIS_HOST + echo "Starting RedisInsight with discovered Redis bootstrap endpoint $RI_REDIS_HOST" + exec "$@" + + repair-node-addresses.sh: | + #!/bin/sh + set -u + # nodes.conf persists node IDs and membership on the PVC, but its cluster + # bus endpoints include pod IPs. Refresh those IPs using each member's + # unique human nodename before Redis starts so a full reschedule recovers + # independently of the client endpoint Redis advertises. + nodes=/data/nodes.conf + [ -s "$nodes" ] || exit 0 + + resolve_host() { + host=$1 + if command -v getent >/dev/null 2>&1; then + getent hosts "$host" 2>/dev/null | awk 'NR == 1 { print $1 }' + else + nslookup "$host" 2>/dev/null | + awk '/^Address: / && $2 !~ /#/ { address=$2 } END { print address }' + fi + } + + while :; do + resolved=0 + old_ifs=$IFS; IFS=',' + for host in $ALL_NODE_HOSTS; do + [ -n "$(resolve_host "$host")" ] && resolved=$((resolved + 1)) + done + IFS=$old_ifs + [ "$resolved" = "$NODE_COUNT" ] && break + echo "Waiting for all $NODE_COUNT persisted member hostnames to resolve ($resolved ready)" + sleep 2 + done + + old_ifs=$IFS; IFS=',' + for assignment in $NODE_ADDRESS_ASSIGNMENTS; do + pod_name=${assignment%%=*} + internal_host=${assignment#*=} + ip="$(resolve_host "$internal_host")" + [ -n "$ip" ] || continue + awk -v target="$pod_name" -v new_ip="$ip" ' + { + metadata_count = split($2, metadata, ",") + nodename = "" + for (i = 3; i <= metadata_count; i++) { + split(metadata[i], auxiliary, "=") + if (auxiliary[1] == "nodename") nodename = auxiliary[2] + } + if (nodename == target) { + split(metadata[1], bus, "@") + split(bus[1], endpoint, ":") + suffix = substr($2, length(metadata[1]) + 1) + $2 = new_ip ":" endpoint[2] "@" bus[2] suffix + } + print + } + ' "$nodes" > "${nodes}.new" && mv "${nodes}.new" "$nodes" + done + IFS=$old_ifs + + bootstrap-cluster.sh: | + #!/bin/sh + set -u + + cli() { + redis-cli --no-auth-warning "$@" + } + + resolve_host() { + host=$1 + if command -v getent >/dev/null 2>&1; then + getent hosts "$host" 2>/dev/null | awk 'NR == 1 { print $1 }' + else + nslookup "$host" 2>/dev/null | + awk '/^Address: / && $2 !~ /#/ { address=$2 } END { print address }' + fi + } + + node_value() { + host=$1 field=$2 + cli -h "$host" -p "$REDIS_PORT" cluster info 2>/dev/null | + tr -d '\r' | awk -F: -v field="$field" '$1 == field { print $2 }' + } + + echo "Redis Cluster bootstrap coordinator: $COORDINATOR_HOST" + while :; do + all_ready=true + old_ifs=$IFS; IFS=',' + for host in $ALL_NODE_HOSTS; do + if [ "$(cli -h "$host" -p "$REDIS_PORT" ping 2>/dev/null)" != PONG ]; then + all_ready=false + break + fi + done + IFS=$old_ifs + $all_ready && break + sleep 2 + done + + if [ "$(node_value "$COORDINATOR_HOST" cluster_state)" = ok ]; then + echo "Redis Cluster already healthy; bootstrap is a no-op" + exit 0 + fi + + # Never overwrite a partially configured or previously initialized + # cluster. That state needs repair, not a destructive second bootstrap. + pristine=true + old_ifs=$IFS; IFS=',' + for host in $ALL_NODE_HOSTS; do + known="$(node_value "$host" cluster_known_nodes)" + assigned="$(node_value "$host" cluster_slots_assigned)" + if [ "$known" != 1 ] || [ "$assigned" != 0 ]; then + pristine=false + break + fi + done + IFS=$old_ifs + if ! $pristine; then + echo "Cluster has existing membership or slots but is not healthy; refusing to reinitialize it" >&2 + exit 0 + fi + + echo "Creating a ${SHARD_COUNT}-shard cluster from ${NODE_COUNT} pristine nodes" + old_ifs=$IFS; IFS=',' + for host in $ALL_NODE_HOSTS; do + [ "$host" = "$COORDINATOR_HOST" ] && continue + ip="$(resolve_host "$host")" + if [ -z "$ip" ]; then + echo "Could not resolve $host" >&2 + exit 1 + fi + cli -h "$COORDINATOR_HOST" -p "$REDIS_PORT" \ + cluster meet "$ip" "$REDIS_PORT" "$CLUSTER_BUS_PORT" >/dev/null || exit 1 + done + IFS=$old_ifs + + # Wait until every node has learned the complete membership before roles + # are assigned. This avoids CLUSTER REPLICATE racing gossip propagation. + while :; do + complete=true + old_ifs=$IFS; IFS=',' + for host in $ALL_NODE_HOSTS; do + [ "$(node_value "$host" cluster_known_nodes)" = "$NODE_COUNT" ] || complete=false + done + IFS=$old_ifs + $complete && break + sleep 1 + done + + shard=0 + old_ifs=$IFS; IFS=',' + for primary in $PRIMARY_HOSTS; do + start=$((16384 * shard / SHARD_COUNT)) + end=$((16384 * (shard + 1) / SHARD_COUNT - 1)) + echo "Assigning slots $start-$end to shard $shard ($primary)" + cli -h "$primary" -p "$REDIS_PORT" cluster addslotsrange "$start" "$end" >/dev/null || exit 1 + shard=$((shard + 1)) + done + IFS=$old_ifs + + old_ifs=$IFS; IFS=',' + for assignment in $REPLICA_ASSIGNMENTS; do + replica=${assignment%%=*} + primary=${assignment#*=} + primary_id="$(cli -h "$primary" -p "$REDIS_PORT" cluster myid 2>/dev/null | tr -d '\r')" + [ -n "$primary_id" ] || exit 1 + echo "Attaching $replica as a replica of $primary" + cli -h "$replica" -p "$REDIS_PORT" cluster replicate "$primary_id" >/dev/null || exit 1 + done + IFS=$old_ifs + + # Do not declare success merely because slots are covered. Wait for every + # intended replica relationship and for every member to see a healthy + # cluster, which prevents an early restart from racing topology gossip. + while :; do + converged=true + old_ifs=$IFS; IFS=',' + for assignment in $REPLICA_ASSIGNMENTS; do + replica=${assignment%%=*} + role="$(cli -h "$replica" -p "$REDIS_PORT" role 2>/dev/null | head -n1)" + [ "$role" = slave ] || converged=false + done + for host in $ALL_NODE_HOSTS; do + [ "$(node_value "$host" cluster_state)" = ok ] || converged=false + done + IFS=$old_ifs + $converged && break + sleep 1 + done + echo "Redis Cluster created: all 16384 slots covered" + + reconcile-topology.sh: | + #!/bin/sh + set -u + # Redis correctly promotes replicas but does not automatically fail back. + # Restore the canonical member-0 primaries after a failed worker returns, + # keeping the three current primaries distributed across workers. + cli() { redis-cli --no-auth-warning "$@"; } + cluster_state() { + cli -h "$COORDINATOR_HOST" -p "$REDIS_PORT" cluster info 2>/dev/null | + tr -d '\r' | awk -F: '$1 == "cluster_state" { print $2 }' + } + + while :; do + [ -f /etc/redis-runtime/terminating ] && exit 0 + if [ "$(cluster_state)" = ok ]; then + old_ifs=$IFS; IFS=',' + for primary_identity in $PRIMARY_HOSTS; do + role="$(cli -h "$primary_identity" -p "$REDIS_PORT" role 2>/dev/null | head -n1)" + if [ "$role" = slave ]; then + link="$(cli -h "$primary_identity" -p "$REDIS_PORT" info replication 2>/dev/null | + tr -d '\r' | awk -F: '$1 == "master_link_status" { print $2 }')" + if [ "$link" = up ]; then + echo "Restoring canonical primary role to $primary_identity" + cli -h "$primary_identity" -p "$REDIS_PORT" cluster failover >/dev/null 2>&1 || true + i=0 + while [ "$i" -lt 20 ]; do + role="$(cli -h "$primary_identity" -p "$REDIS_PORT" role 2>/dev/null | head -n1)" + [ "$role" = master ] && break + sleep 1 + i=$((i + 1)) + done + while [ "$(cluster_state)" != ok ]; do sleep 1; done + fi + fi + done + IFS=$old_ifs + fi + sleep 30 + done + + startup-redis.sh: | + #!/bin/sh + [ "$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning ping 2>/dev/null)" = PONG ] + + liveness-redis.sh: | + #!/bin/sh + resp="$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning ping 2>&1 || true)" + case "$resp" in + PONG|*LOADING*|*CLUSTERDOWN*) exit 0 ;; + *) echo "$resp"; exit 1 ;; + esac + + readiness-redis.sh: | + #!/bin/sh + [ "$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning ping 2>/dev/null)" = PONG ] || exit 1 + state="$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning cluster info 2>/dev/null | + tr -d '\r' | awk -F: '$1 == "cluster_state" { print $2 }')" + [ "$state" = ok ] || { echo "cluster_state=$state"; exit 1; } + + prestop-redis.sh: | + #!/bin/sh + # During an intentional restart, ask a healthy replica to take over first. + # Replica identities remain on a different Kubernetes node because shard + # anti-affinity is tied to persistent member identity, not current role. + : > /etc/redis-runtime/terminating + role="$(redis-cli -p "$REDIS_PORT" --no-auth-warning role 2>/dev/null | head -n1 || true)" + [ "$role" = master ] || exit 0 + node_id="$(redis-cli -p "$REDIS_PORT" --no-auth-warning cluster myid 2>/dev/null | tr -d '\r')" + [ -n "$node_id" ] || exit 0 + endpoint="$(redis-cli -p "$REDIS_PORT" --no-auth-warning cluster replicas "$node_id" 2>/dev/null | + awk '$3 !~ /fail/ && $8 == "connected" { print $2; exit }')" + [ -n "$endpoint" ] || exit 0 + address=${endpoint%%@*} + metadata=${endpoint#*,} + if [ "$metadata" != "$endpoint" ] && [ -n "$metadata" ]; then + replica_host=$metadata + else + replica_host=${address%:*} + fi + replica_port=${address##*:} + echo "Requesting graceful failover to $replica_host:$replica_port" + redis-cli -h "$replica_host" -p "$replica_port" --no-auth-warning cluster failover >/dev/null 2>&1 || exit 0 + i=0 + while [ "$i" -lt 15 ]; do + role="$(redis-cli -p "$REDIS_PORT" --no-auth-warning role 2>/dev/null | head -n1 || true)" + [ "$role" != master ] && exit 0 + sleep 1 + i=$((i + 1)) + done + exit 0 diff --git a/templates/external-endpoint-rbac.yaml b/templates/external-endpoint-rbac.yaml new file mode 100644 index 0000000..97056e3 --- /dev/null +++ b/templates/external-endpoint-rbac.yaml @@ -0,0 +1,37 @@ +{{- if .Values.externalAccess.enabled }} +{{- $fullname := include "redis-cluster.fullname" . }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $fullname }}-endpoint-reader + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["services"] + verbs: ["get"] + resourceNames: + - {{ $fullname }}-external + {{- range $shard := until (int .Values.cluster.shards) }} + {{- range $member := until (int (add (int $.Values.cluster.replicasPerShard) 1)) }} + - {{ printf "%s-s%d-n%d-x" $fullname $shard $member | trunc 63 | trimSuffix "-" }} + {{- end }} + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $fullname }}-endpoint-reader + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "redis-cluster.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $fullname }}-endpoint-reader +{{- end }} diff --git a/templates/networkpolicy.yaml b/templates/networkpolicy.yaml new file mode 100644 index 0000000..4c0f0d0 --- /dev/null +++ b/templates/networkpolicy.yaml @@ -0,0 +1,51 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "redis-cluster.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + {{- include "redis-cluster.selectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + ingress: + # Intra-release traffic: client protocol is also used for replication and + # key migration; the cluster bus carries gossip and failover elections. + - from: + - podSelector: + matchLabels: + {{- include "redis-cluster.selectorLabels" . | nindent 14 }} + ports: + - port: {{ .Values.redis.port }} + - port: {{ .Values.cluster.busPort }} + # Client traffic. + {{- if .Values.networkPolicy.allowExternal }} + - ports: + - port: {{ .Values.redis.port }} + {{- else if .Values.networkPolicy.extraIngress }} + - from: + {{- toYaml .Values.networkPolicy.extraIngress | nindent 8 }} + ports: + - port: {{ .Values.redis.port }} + {{- end }} + {{- if .Values.metrics.enabled }} + # Metrics scraping. + {{- if .Values.networkPolicy.metricsFromNamespaces }} + - from: + {{- range .Values.networkPolicy.metricsFromNamespaces }} + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ . }} + {{- end }} + ports: + - port: {{ .Values.metrics.port }} + {{- else }} + - ports: + - port: {{ .Values.metrics.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/templates/pdb.yaml b/templates/pdb.yaml new file mode 100644 index 0000000..f5b040f --- /dev/null +++ b/templates/pdb.yaml @@ -0,0 +1,14 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "redis-cluster.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.labels" . | nindent 4 }} +spec: + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + selector: + matchLabels: + {{- include "redis-cluster.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/templates/redisinsight-istio.yaml b/templates/redisinsight-istio.yaml new file mode 100644 index 0000000..c22ce11 --- /dev/null +++ b/templates/redisinsight-istio.yaml @@ -0,0 +1,87 @@ +{{- if and .Values.redisInsight.enabled .Values.redisInsight.ingress.enabled }} +{{- if not .Values.redisInsight.ingress.host }} +{{- fail "redisInsight.ingress.host is required when RedisInsight ingress is enabled" }} +{{- end }} +{{- if and .Values.redisInsight.ingress.tls.enabled (not .Values.redisInsight.ingress.tls.credentialName) }} +{{- fail "redisInsight.ingress.tls.credentialName is required when TLS is enabled" }} +{{- end }} +{{- if and .Values.redisInsight.ingress.tls.certificate.create (not .Values.redisInsight.ingress.tls.enabled) }} +{{- fail "redisInsight.ingress.tls.enabled must be true when certificate.create is true" }} +{{- end }} +{{- if and .Values.redisInsight.ingress.tls.certificate.create (not .Values.redisInsight.ingress.tls.certificate.issuerName) }} +{{- fail "redisInsight.ingress.tls.certificate.issuerName is required when certificate.create is true" }} +{{- end }} +{{- if and .Values.redisInsight.ingress.tls.enabled .Values.redisInsight.ingress.tls.certificate.create }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "redis-cluster.redisInsightName" . }}-tls + namespace: {{ default .Release.Namespace .Values.redisInsight.ingress.tls.certificate.secretNamespace }} + labels: + {{- include "redis-cluster.redisInsightLabels" . | nindent 4 }} +spec: + secretName: {{ .Values.redisInsight.ingress.tls.credentialName }} + issuerRef: + name: {{ .Values.redisInsight.ingress.tls.certificate.issuerName }} + kind: {{ .Values.redisInsight.ingress.tls.certificate.issuerKind }} + commonName: {{ .Values.redisInsight.ingress.host | quote }} + dnsNames: + - {{ .Values.redisInsight.ingress.host | quote }} +--- +{{- end }} +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: {{ include "redis-cluster.redisInsightName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.redisInsightLabels" . | nindent 4 }} +spec: + selector: + {{- toYaml .Values.redisInsight.ingress.gatewaySelector | nindent 4 }} + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - {{ .Values.redisInsight.ingress.host | quote }} + {{- if .Values.redisInsight.ingress.tls.enabled }} + tls: + httpsRedirect: true + {{- end }} + {{- if .Values.redisInsight.ingress.tls.enabled }} + - port: + number: 443 + name: https + protocol: HTTPS + hosts: + - {{ .Values.redisInsight.ingress.host | quote }} + tls: + mode: SIMPLE + credentialName: {{ .Values.redisInsight.ingress.tls.credentialName }} + {{- end }} +--- +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: {{ include "redis-cluster.redisInsightName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.redisInsightLabels" . | nindent 4 }} +spec: + hosts: + - {{ .Values.redisInsight.ingress.host | quote }} + gateways: + - {{ include "redis-cluster.redisInsightName" . }} + http: + - match: + - uri: + prefix: / + timeout: {{ .Values.redisInsight.ingress.requestTimeout }} + route: + - destination: + host: {{ include "redis-cluster.redisInsightName" . }} + port: + number: 80 +{{- end }} diff --git a/templates/redisinsight.yaml b/templates/redisinsight.yaml new file mode 100644 index 0000000..5f2fe43 --- /dev/null +++ b/templates/redisinsight.yaml @@ -0,0 +1,220 @@ +{{- if .Values.redisInsight.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "redis-cluster.redisInsightName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.redisInsightLabels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + selector: + {{- include "redis-cluster.redisInsightSelectorLabels" . | nindent 4 }} +{{- if .Values.redisInsight.persistence.enabled }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "redis-cluster.redisInsightName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.redisInsightLabels" . | nindent 4 }} + {{- with .Values.redisInsight.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + {{- toYaml .Values.redisInsight.persistence.accessModes | nindent 4 }} + {{- with .Values.redisInsight.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.redisInsight.persistence.size }} +{{- end }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "redis-cluster.redisInsightName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.redisInsightLabels" . | nindent 4 }} +spec: + replicas: 1 + # RedisInsight stores its connection database under /data. Recreate avoids + # two pods contending for the same ReadWriteOnce volume during an upgrade. + strategy: + type: Recreate + selector: + matchLabels: + {{- include "redis-cluster.redisInsightSelectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "redis-cluster.redisInsightSelectorLabels" . | nindent 8 }} + annotations: + sidecar.istio.io/inject: "false" + {{- with .Values.redisInsight.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if and .Values.externalAccess.enabled (not .Values.redisInsight.connection.host) }} + serviceAccountName: {{ include "redis-cluster.serviceAccountName" . }} + automountServiceAccountToken: false + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + {{- if and .Values.externalAccess.enabled (not .Values.redisInsight.connection.host) }} + initContainers: + - name: discover-bootstrap-endpoint + image: "{{ .Values.externalAccess.endpointDiscovery.image.repository }}:{{ .Values.externalAccess.endpointDiscovery.image.tag }}" + imagePullPolicy: {{ .Values.externalAccess.endpointDiscovery.image.pullPolicy }} + command: ["/bin/sh", "/scripts/discover-external-endpoint.sh"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: EXTERNAL_SERVICE_NAME + value: {{ include "redis-cluster.bootstrapServiceName" . | quote }} + resources: + {{- toYaml .Values.externalAccess.endpointDiscovery.resources | nindent 12 }} + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: bootstrap-endpoint + mountPath: /external-endpoint + - name: kube-api-access + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + {{- end }} + containers: + - name: redisinsight + image: "{{ .Values.redisInsight.image.repository }}:{{ .Values.redisInsight.image.tag }}" + imagePullPolicy: {{ .Values.redisInsight.image.pullPolicy }} + {{- if and .Values.externalAccess.enabled (not .Values.redisInsight.connection.host) }} + # Preserve the image CMD as arguments; this wrapper only injects the + # dynamically discovered LoadBalancer endpoint before exec. + command: ["/scripts/start-redisinsight.sh"] + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + env: + - name: RI_APP_HOST + value: "0.0.0.0" + - name: RI_APP_PORT + value: {{ .Values.redisInsight.port | quote }} + # These variables register a connection inside RedisInsight only; + # they do not alter the Redis cluster or its Kubernetes resources. + - name: RI_REDIS_HOST + value: {{ default (include "redis-cluster.bootstrapServiceName" .) .Values.redisInsight.connection.host | quote }} + - name: RI_REDIS_PORT + value: {{ default (ternary .Values.externalAccess.port .Values.redis.port .Values.externalAccess.enabled) .Values.redisInsight.connection.port | quote }} + - name: RI_REDIS_ALIAS + value: {{ .Values.redisInsight.connection.alias | quote }} + - name: RI_REDIS_TLS + value: {{ .Values.redisInsight.connection.tls | quote }} + {{- if .Values.auth.enabled }} + - name: RI_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "redis-cluster.secretName" . }} + key: {{ include "redis-cluster.secretKey" . }} + {{- end }} + - name: RI_STDOUT_LOGGER + value: "true" + ports: + - name: http + containerPort: {{ .Values.redisInsight.port }} + protocol: TCP + startupProbe: + httpGet: + path: /api/health/ + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /api/health/ + port: http + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /api/health/ + port: http + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 5 + resources: + {{- toYaml .Values.redisInsight.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /data + {{- if and .Values.externalAccess.enabled (not .Values.redisInsight.connection.host) }} + - name: scripts + mountPath: /scripts + readOnly: true + - name: bootstrap-endpoint + mountPath: /bootstrap-endpoint + readOnly: true + {{- end }} + volumes: + - name: data + {{- if .Values.redisInsight.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ include "redis-cluster.redisInsightName" . }} + {{- else }} + emptyDir: {} + {{- end }} + {{- if and .Values.externalAccess.enabled (not .Values.redisInsight.connection.host) }} + - name: scripts + configMap: + name: {{ include "redis-cluster.fullname" . }} + defaultMode: 0555 + - name: bootstrap-endpoint + emptyDir: {} + - name: kube-api-access + projected: + defaultMode: 0444 + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt + {{- end }} + {{- with .Values.redisInsight.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.redisInsight.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/templates/secret.yaml b/templates/secret.yaml new file mode 100644 index 0000000..b8af0bb --- /dev/null +++ b/templates/secret.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.auth.enabled (not .Values.auth.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "redis-cluster.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.labels" . | nindent 4 }} +type: Opaque +data: + redis-password: {{ include "redis-cluster.password" . | b64enc | quote }} +{{- end }} diff --git a/templates/service-external.yaml b/templates/service-external.yaml new file mode 100644 index 0000000..98f42bc --- /dev/null +++ b/templates/service-external.yaml @@ -0,0 +1,75 @@ +{{- if .Values.externalAccess.enabled }} +{{- $root := . }} +{{- $fullname := include "redis-cluster.fullname" . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $fullname }}-external + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.labels" . | nindent 4 }} + redis-cluster.external-role: bootstrap + {{- with .Values.externalAccess.bootstrap.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: LoadBalancer + externalTrafficPolicy: {{ .Values.externalAccess.externalTrafficPolicy }} + allocateLoadBalancerNodePorts: {{ .Values.externalAccess.allocateLoadBalancerNodePorts }} + {{- with .Values.externalAccess.loadBalancerClass }} + loadBalancerClass: {{ . | quote }} + {{- end }} + {{- with .Values.externalAccess.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + {{- include "redis-cluster.selectorLabels" . | nindent 4 }} + ports: + - name: tcp-redis + port: {{ .Values.externalAccess.port }} + targetPort: tcp-redis + appProtocol: redis +{{- range $shard := until (int .Values.cluster.shards) }} +{{- range $member := until (int (add (int $.Values.cluster.replicasPerShard) 1)) }} +{{- $memberName := printf "%s-s%d-n%d" $fullname $shard $member | trunc 63 | trimSuffix "-" }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $memberName }}-x + namespace: {{ $.Release.Namespace }} + labels: + {{- include "redis-cluster.labels" $root | nindent 4 }} + redis-cluster.external-role: member + redis-cluster.shard: {{ $shard | quote }} + redis-cluster.member: {{ $member | quote }} + {{- with $.Values.externalAccess.memberServices.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: LoadBalancer + externalTrafficPolicy: {{ $.Values.externalAccess.externalTrafficPolicy }} + allocateLoadBalancerNodePorts: {{ $.Values.externalAccess.allocateLoadBalancerNodePorts }} + {{- with $.Values.externalAccess.loadBalancerClass }} + loadBalancerClass: {{ . | quote }} + {{- end }} + {{- with $.Values.externalAccess.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + {{- include "redis-cluster.selectorLabels" $root | nindent 4 }} + redis-cluster.shard: {{ $shard | quote }} + redis-cluster.member: {{ $member | quote }} + ports: + - name: tcp-redis + port: {{ $.Values.externalAccess.port }} + targetPort: tcp-redis + appProtocol: redis +{{- end }} +{{- end }} +{{- end }} diff --git a/templates/service-headless.yaml b/templates/service-headless.yaml new file mode 100644 index 0000000..53efd23 --- /dev/null +++ b/templates/service-headless.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "redis-cluster.fullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.labels" . | nindent 4 }} +spec: + clusterIP: None + # Pods must resolve each other during bootstrap, before they are Ready. + publishNotReadyAddresses: true + selector: + {{- include "redis-cluster.selectorLabels" . | nindent 4 }} + ports: + - name: tcp-redis + port: {{ .Values.redis.port }} + targetPort: tcp-redis + appProtocol: redis + - name: tcp-cluster + port: {{ .Values.cluster.busPort }} + targetPort: tcp-cluster + appProtocol: redis diff --git a/templates/service.yaml b/templates/service.yaml new file mode 100644 index 0000000..2938700 --- /dev/null +++ b/templates/service.yaml @@ -0,0 +1,24 @@ +{{/* Any node is a valid startup node. Cluster-aware clients fetch the slot map +and then connect directly to the stable per-member hostnames Redis advertises. */}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "redis-cluster.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.labels" . | nindent 4 }} +spec: + type: ClusterIP + selector: + {{- include "redis-cluster.selectorLabels" . | nindent 4 }} + ports: + - name: tcp-redis + port: {{ .Values.redis.port }} + targetPort: tcp-redis + appProtocol: redis + {{- if .Values.metrics.enabled }} + - name: http-metrics + port: {{ .Values.metrics.port }} + targetPort: http-metrics + appProtocol: http + {{- end }} diff --git a/templates/serviceaccount.yaml b/templates/serviceaccount.yaml new file mode 100644 index 0000000..170a13e --- /dev/null +++ b/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "redis-cluster.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: false +{{- end }} diff --git a/templates/servicemonitor.yaml b/templates/servicemonitor.yaml new file mode 100644 index 0000000..1a5be33 --- /dev/null +++ b/templates/servicemonitor.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "redis-cluster.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "redis-cluster.labels" . | nindent 4 }} + {{- with .Values.metrics.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "redis-cluster.selectorLabels" . | nindent 6 }} + endpoints: + - port: http-metrics + interval: {{ .Values.metrics.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} +{{- end }} diff --git a/templates/statefulset.yaml b/templates/statefulset.yaml new file mode 100644 index 0000000..c203dba --- /dev/null +++ b/templates/statefulset.yaml @@ -0,0 +1,311 @@ +{{- $root := . -}} +{{- $fullname := include "redis-cluster.fullname" . -}} +{{- if gt (len $fullname) 53 -}} + {{- fail (printf "release fullname %q is too long; use fullnameOverride with at most 53 characters" $fullname) -}} +{{- end -}} +{{- if ge (int .Values.cluster.replicasPerShard) (int .Values.cluster.shards) -}} + {{- fail "cluster.replicasPerShard must be lower than cluster.shards so crossed placement never puts a shard replica with its own primary" -}} +{{- end -}} +{{- $headless := include "redis-cluster.headlessFqdn" . -}} +{{- $nodeCount := include "redis-cluster.nodeCount" . -}} +{{- $allHosts := list -}} +{{- $primaryHosts := list -}} +{{- $assignments := list -}} +{{- $addressAssignments := list -}} +{{- range $shard := until (int .Values.cluster.shards) -}} + {{- range $member := until (int (add (int $.Values.cluster.replicasPerShard) 1)) -}} + {{- $memberName := printf "%s-s%d-n%d" $fullname $shard $member | trunc 63 | trimSuffix "-" -}} + {{- $podName := printf "%s-0" $memberName -}} + {{- $host := printf "%s-0.%s" $memberName $headless -}} + {{- $allHosts = append $allHosts $host -}} + {{- $addressAssignments = append $addressAssignments (printf "%s=%s" $podName $host) -}} + {{- if eq $member 0 -}} + {{- $primaryHosts = append $primaryHosts $host -}} + {{- else -}} + {{- $primaryName := printf "%s-s%d-n0" $fullname $shard | trunc 63 | trimSuffix "-" -}} + {{- $primaryHost := printf "%s-0.%s" $primaryName $headless -}} + {{- $assignments = append $assignments (printf "%s=%s" $host $primaryHost) -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- $coordinator := first $primaryHosts -}} +{{- range $shard := until (int .Values.cluster.shards) }} +{{- range $member := until (int (add (int $.Values.cluster.replicasPerShard) 1)) }} +{{- $memberName := printf "%s-s%d-n%d" $fullname $shard $member | trunc 63 | trimSuffix "-" }} +{{- $memberIndex := add (mul $shard (add (int $.Values.cluster.replicasPerShard) 1)) $member }} +{{- $announceHostname := printf "%s-0.%s" $memberName $headless }} +{{- $externalServiceName := printf "%s-x" $memberName }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $memberName }} + namespace: {{ $.Release.Namespace }} + labels: + {{- include "redis-cluster.labels" $root | nindent 4 }} + redis-cluster.shard: {{ $shard | quote }} + redis-cluster.member: {{ $member | quote }} +spec: + serviceName: {{ $fullname }}-headless + replicas: 1 + podManagementPolicy: Parallel + # Separate one-member StatefulSets would otherwise all roll concurrently. + # OnDelete makes upgrades explicit and safe; see README.md. + updateStrategy: + type: OnDelete + selector: + matchLabels: + {{- include "redis-cluster.selectorLabels" $root | nindent 6 }} + redis-cluster.shard: {{ $shard | quote }} + redis-cluster.member: {{ $member | quote }} + template: + metadata: + labels: + {{- include "redis-cluster.selectorLabels" $root | nindent 8 }} + redis-cluster.shard: {{ $shard | quote }} + redis-cluster.member: {{ $member | quote }} + {{- with $.Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") $root | sha256sum }} + {{- if not $.Values.istio.injectSidecar }} + sidecar.istio.io/inject: "false" + {{- else }} + proxy.istio.io/config: '{ "holdApplicationUntilProxyStarts": true }' + {{- end }} + {{- with $.Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "redis-cluster.serviceAccountName" $root }} + automountServiceAccountToken: false + terminationGracePeriodSeconds: {{ $.Values.terminationGracePeriodSeconds }} + {{- with $.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + securityContext: + {{- toYaml $.Values.podSecurityContext | nindent 8 }} + affinity: + {{- if gt $member 0 }} + # Cross replicas onto the worker holding another shard's canonical + # primary. For 3x1 this produces A:P0+R2, B:P1+R0, C:P2+R1. + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + {{- include "redis-cluster.selectorLabels" $root | nindent 18 }} + redis-cluster.shard: {{ mod (add $shard $member) (int $.Values.cluster.shards) | quote }} + redis-cluster.member: "0" + topologyKey: kubernetes.io/hostname + {{- end }} + podAntiAffinity: + # Hard invariants: copies of one shard never share a worker, and + # equivalent member identities are spread across workers. In the + # canonical topology, this permits at most one primary per worker. + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + {{- include "redis-cluster.selectorLabels" $root | nindent 18 }} + redis-cluster.shard: {{ $shard | quote }} + topologyKey: kubernetes.io/hostname + - labelSelector: + matchLabels: + {{- include "redis-cluster.selectorLabels" $root | nindent 18 }} + redis-cluster.member: {{ $member | quote }} + topologyKey: kubernetes.io/hostname + {{- with $.Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $.Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if $.Values.externalAccess.enabled }} + initContainers: + - name: discover-external-endpoint + image: "{{ $.Values.externalAccess.endpointDiscovery.image.repository }}:{{ $.Values.externalAccess.endpointDiscovery.image.tag }}" + imagePullPolicy: {{ $.Values.externalAccess.endpointDiscovery.image.pullPolicy }} + command: ["/bin/sh", "/scripts/discover-external-endpoint.sh"] + securityContext: + {{- toYaml $.Values.containerSecurityContext | nindent 12 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: EXTERNAL_SERVICE_NAME + value: {{ $externalServiceName | quote }} + resources: + {{- toYaml $.Values.externalAccess.endpointDiscovery.resources | nindent 12 }} + volumeMounts: + - name: scripts + mountPath: /scripts + - name: external-endpoint + mountPath: /external-endpoint + - name: kube-api-access + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + {{- end }} + containers: + - name: redis + image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} + command: ["/bin/sh", "/scripts/start-redis.sh"] + securityContext: + {{- toYaml $.Values.containerSecurityContext | nindent 12 }} + env: + {{- include "redis-cluster.commonEnv" $root | nindent 12 }} + - name: COORDINATOR_HOST + value: {{ $coordinator | quote }} + - name: ALL_NODE_HOSTS + value: {{ join "," $allHosts | quote }} + - name: PRIMARY_HOSTS + value: {{ join "," $primaryHosts | quote }} + - name: REPLICA_ASSIGNMENTS + value: {{ join "," $assignments | quote }} + - name: NODE_ADDRESS_ASSIGNMENTS + value: {{ join "," $addressAssignments | quote }} + - name: ANNOUNCE_HOSTNAME + value: {{ $announceHostname | quote }} + - name: ANNOUNCE_PORT + value: {{ ternary $.Values.externalAccess.port $.Values.redis.port $.Values.externalAccess.enabled | quote }} + - name: EXTERNAL_ACCESS_ENABLED + value: {{ $.Values.externalAccess.enabled | quote }} + ports: + - name: tcp-redis + containerPort: {{ $.Values.redis.port }} + - name: tcp-cluster + containerPort: {{ $.Values.cluster.busPort }} + startupProbe: + exec: + command: ["/bin/sh", "/scripts/startup-redis.sh"] + initialDelaySeconds: {{ $.Values.probes.redis.startup.initialDelaySeconds }} + periodSeconds: {{ $.Values.probes.redis.startup.periodSeconds }} + timeoutSeconds: {{ $.Values.probes.redis.startup.timeoutSeconds }} + failureThreshold: {{ $.Values.probes.redis.startup.failureThreshold }} + livenessProbe: + exec: + command: ["/bin/sh", "/scripts/liveness-redis.sh"] + periodSeconds: {{ $.Values.probes.redis.liveness.periodSeconds }} + timeoutSeconds: {{ $.Values.probes.redis.liveness.timeoutSeconds }} + failureThreshold: {{ $.Values.probes.redis.liveness.failureThreshold }} + readinessProbe: + exec: + command: ["/bin/sh", "/scripts/readiness-redis.sh"] + periodSeconds: {{ $.Values.probes.redis.readiness.periodSeconds }} + timeoutSeconds: {{ $.Values.probes.redis.readiness.timeoutSeconds }} + failureThreshold: {{ $.Values.probes.redis.readiness.failureThreshold }} + lifecycle: + preStop: + exec: + command: ["/bin/sh", "/scripts/prestop-redis.sh"] + resources: + {{- toYaml $.Values.redis.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /data + - name: scripts + mountPath: /scripts + - name: config + mountPath: /etc/redis-ro + - name: redis-runtime + mountPath: /etc/redis-runtime + {{- if $.Values.externalAccess.enabled }} + - name: external-endpoint + mountPath: /external-endpoint + readOnly: true + {{- end }} + {{- if $.Values.metrics.enabled }} + - name: metrics + image: "{{ $.Values.metrics.image.repository }}:{{ $.Values.metrics.image.tag }}" + imagePullPolicy: {{ $.Values.metrics.image.pullPolicy }} + securityContext: + {{- toYaml $.Values.containerSecurityContext | nindent 12 }} + env: + - name: REDIS_ADDR + value: "redis://localhost:{{ $.Values.redis.port }}" + {{- if $.Values.auth.enabled }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "redis-cluster.secretName" $root }} + key: {{ include "redis-cluster.secretKey" $root }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ $.Values.metrics.port }} + livenessProbe: + tcpSocket: + port: http-metrics + initialDelaySeconds: 10 + periodSeconds: 15 + readinessProbe: + tcpSocket: + port: http-metrics + periodSeconds: 15 + resources: + {{- toYaml $.Values.metrics.resources | nindent 12 }} + {{- end }} + volumes: + - name: scripts + configMap: + name: {{ $fullname }} + defaultMode: 0555 + - name: config + configMap: + name: {{ $fullname }} + items: + - key: redis.conf + path: redis.conf + - name: redis-runtime + emptyDir: {} + {{- if $.Values.externalAccess.enabled }} + - name: external-endpoint + emptyDir: {} + - name: kube-api-access + projected: + defaultMode: 0444 + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt + {{- end }} + {{- if not $.Values.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- if $.Values.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: data + {{- with $.Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- toYaml $.Values.persistence.accessModes | nindent 10 }} + {{- with $.Values.persistence.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ $.Values.persistence.size }} + {{- end }} +{{- end }} +{{- end }} diff --git a/values.schema.json b/values.schema.json new file mode 100644 index 0000000..17018f8 --- /dev/null +++ b/values.schema.json @@ -0,0 +1,230 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "cluster": { + "type": "object", + "required": ["shards", "replicasPerShard", "busPort"], + "properties": { + "shards": { "type": "integer", "minimum": 3, "maximum": 99 }, + "replicasPerShard": { "type": "integer", "minimum": 1, "maximum": 9 }, + "busPort": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "nodeTimeoutMilliseconds": { "type": "integer", "minimum": 1000 }, + "requireFullCoverage": { "type": "boolean" }, + "allowReadsWhenDown": { "type": "boolean" }, + "migrationBarrier": { "type": "integer", "minimum": 0 } + } + }, + "auth": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "password": { "type": "string" }, + "existingSecret": { "type": "string" }, + "existingSecretKey": { "type": "string" } + } + }, + "redis": { + "type": "object", + "properties": { + "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "maxmemory": { "type": "string" }, + "maxmemoryPolicy": { + "enum": [ + "noeviction", + "allkeys-lru", "volatile-lru", + "allkeys-lfu", "volatile-lfu", + "allkeys-random", "volatile-random", + "volatile-ttl" + ] + }, + "extraConfig": { "type": "string" } + } + }, + "probes": { + "type": "object", + "properties": { + "redis": { + "type": "object", + "properties": { + "startup": { "$ref": "#/definitions/probe" }, + "liveness": { "$ref": "#/definitions/probe" }, + "readiness": { "$ref": "#/definitions/probe" } + } + } + } + }, + "externalAccess": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "bootstrap": { + "type": "object", + "properties": { + "annotations": { "type": "object" } + } + }, + "memberServices": { + "type": "object", + "properties": { + "annotations": { "type": "object" } + } + }, + "loadBalancerClass": { "type": "string" }, + "externalTrafficPolicy": { "enum": ["Cluster", "Local"] }, + "loadBalancerSourceRanges": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "allocateLoadBalancerNodePorts": { "type": "boolean" }, + "endpointDiscovery": { + "type": "object", + "properties": { + "image": { + "type": "object", + "properties": { + "repository": { "type": "string", "minLength": 1 }, + "tag": { "type": "string", "minLength": 1 }, + "pullPolicy": { "enum": ["Always", "IfNotPresent", "Never"] } + } + }, + "resources": { "type": "object" } + } + } + } + }, + "metrics": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "port": { "type": "integer", "minimum": 1, "maximum": 65535 } + } + }, + "redisInsight": { + "type": "object", + "required": ["enabled", "image", "port", "connection", "persistence", "ingress"], + "properties": { + "enabled": { "type": "boolean" }, + "image": { + "type": "object", + "required": ["repository", "tag", "pullPolicy"], + "properties": { + "repository": { "type": "string", "minLength": 1 }, + "tag": { "type": "string", "minLength": 1 }, + "pullPolicy": { "enum": ["Always", "IfNotPresent", "Never"] } + } + }, + "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "connection": { + "type": "object", + "required": ["host", "port", "alias", "tls"], + "properties": { + "host": { "type": "string" }, + "port": { "type": "integer", "minimum": 0, "maximum": 65535 }, + "alias": { "type": "string", "minLength": 1 }, + "tls": { "type": "boolean" } + } + }, + "persistence": { + "type": "object", + "required": ["enabled", "size"], + "properties": { + "enabled": { "type": "boolean" }, + "storageClass": { "type": "string" }, + "accessModes": { + "type": "array", + "minItems": 1, + "items": { "enum": ["ReadWriteOnce", "ReadOnlyMany", "ReadWriteMany", "ReadWriteOncePod"] } + }, + "size": { + "type": "string", + "minLength": 1, + "pattern": "^([0-9]+([.][0-9]+)?|[.][0-9]+)([eE][+-]?[0-9]+|[num]|[kKMGTP]i?|Ei?)?$" + }, + "annotations": { "type": "object" } + } + }, + "resources": { "type": "object" }, + "podAnnotations": { "type": "object" }, + "nodeSelector": { "type": "object" }, + "tolerations": { "type": "array" }, + "ingress": { + "type": "object", + "required": ["enabled", "host", "gatewaySelector", "requestTimeout", "tls"], + "properties": { + "enabled": { "type": "boolean" }, + "host": { "type": "string" }, + "gatewaySelector": { "type": "object", "minProperties": 1 }, + "requestTimeout": { "type": "string", "minLength": 1 }, + "tls": { + "type": "object", + "required": ["enabled", "credentialName", "certificate"], + "properties": { + "enabled": { "type": "boolean" }, + "credentialName": { "type": "string" }, + "certificate": { + "type": "object", + "required": ["create", "issuerName", "issuerKind", "secretNamespace"], + "properties": { + "create": { "type": "boolean" }, + "issuerName": { "type": "string" }, + "issuerKind": { "enum": ["Issuer", "ClusterIssuer"] }, + "secretNamespace": { "type": "string" } + } + } + } + } + } + } + } + }, + "persistence": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "storageClass": { "type": "string" }, + "size": { + "type": "string", + "minLength": 1, + "pattern": "^([0-9]+([.][0-9]+)?|[.][0-9]+)([eE][+-]?[0-9]+|[num]|[kKMGTP]i?|Ei?)?$" + } + } + }, + "networkPolicy": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "allowExternal": { "type": "boolean" }, + "extraIngress": { "type": "array" }, + "metricsFromNamespaces": { "type": "array", "items": { "type": "string" } } + } + }, + "podDisruptionBudget": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "maxUnavailable": { "type": "integer", "minimum": 0 } + } + }, + "istio": { + "type": "object", + "properties": { + "injectSidecar": { "type": "boolean" } + } + }, + "terminationGracePeriodSeconds": { "type": "integer", "minimum": 0 } + }, + "definitions": { + "probe": { + "type": "object", + "required": ["periodSeconds", "timeoutSeconds", "failureThreshold"], + "properties": { + "initialDelaySeconds": { "type": "integer", "minimum": 0 }, + "periodSeconds": { "type": "integer", "minimum": 1 }, + "timeoutSeconds": { "type": "integer", "minimum": 1 }, + "failureThreshold": { "type": "integer", "minimum": 1 } + } + } + } +} diff --git a/values.yaml b/values.yaml new file mode 100644 index 0000000..4255744 --- /dev/null +++ b/values.yaml @@ -0,0 +1,247 @@ +# Native Redis Cluster topology. The member StatefulSets contain +# shards * (replicasPerShard + 1) pods. Changing either value after the cluster +# contains data requires an explicit Redis reshard/add-node operation; a Helm +# upgrade alone intentionally does not mutate the slot map. +cluster: + # -- Independently writable hash-slot primaries. Redis requires at least 3 + # for automatic failover elections. + shards: 3 + # -- Readable, failover-capable replicas attached to every shard primary. + replicasPerShard: 1 + # -- Dedicated node-to-node cluster bus port. + busPort: 16379 + nodeTimeoutMilliseconds: 10000 + requireFullCoverage: true + allowReadsWhenDown: false + migrationBarrier: 1 + +image: + repository: redis + tag: 7.4-alpine + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +auth: + enabled: false + # -- Password. Leave empty to auto-generate one (persisted across upgrades + # via helm lookup). + password: "" + # -- Use an existing secret instead of creating one. + existingSecret: "" + existingSecretKey: redis-password + +redis: + port: 6379 + # -- Cap Redis memory usage; keep it below the container memory limit. + maxmemory: 384mb + maxmemoryPolicy: noeviction + # -- Extra lines appended verbatim to redis.conf. + extraConfig: "" + resources: + requests: + cpu: 70m + memory: 256Mi + limits: + memory: 512Mi + +probes: + redis: + # -- Redis may need to recover AOF and rejoin the cluster after a worker or + # storage outage. Startup failures are tolerated for 15 minutes. + startup: + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 4 + failureThreshold: 180 + # -- Ten consecutive failed checks over 150 seconds are required before + # Kubernetes restarts Redis. LOADING and CLUSTERDOWN remain live states. + liveness: + periodSeconds: 15 + timeoutSeconds: 8 + failureThreshold: 10 + # -- Readiness only controls traffic; it never restarts the container. + readiness: + periodSeconds: 5 + timeoutSeconds: 4 + failureThreshold: 3 + +# Redis Cluster clients must reach the exact member named by MOVED/ASK replies. +# When enabled, the Kubernetes LoadBalancer implementation allocates one +# endpoint per member plus a balanced bootstrap endpoint. Pods discover their +# assigned endpoint from the Kubernetes API before Redis starts. +externalAccess: + # -- Disabled by default because this creates one LoadBalancer per Redis + # member plus one bootstrap LoadBalancer and can incur provider charges. + enabled: false + port: 6379 + bootstrap: + annotations: {} + memberServices: + annotations: {} + loadBalancerClass: "" + externalTrafficPolicy: Cluster + # -- Strongly recommended: restrict this to the client networks. An empty + # list leaves reachability to the LoadBalancer/firewall implementation. + loadBalancerSourceRanges: [] + allocateLoadBalancerNodePorts: true + endpointDiscovery: + image: + repository: curlimages/curl + tag: 8.21.0 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 5m + memory: 8Mi + limits: + memory: 32Mi + +metrics: + enabled: true + image: + # -- redis_exporter's own release registry (upstream project; also mirrored + # at quay.io/oliver006/redis_exporter and docker.io/oliver006/redis_exporter). + # GHCR avoids Docker Hub pull rate limits. + repository: ghcr.io/oliver006/redis_exporter + tag: v1.87.0-alpine + pullPolicy: IfNotPresent + port: 9121 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + memory: 64Mi + serviceMonitor: + # -- Requires the Prometheus Operator CRDs (kube-prometheus-stack). + enabled: false + interval: 30s + scrapeTimeout: 10s + # -- Extra labels so your Prometheus instance selects this ServiceMonitor, + # e.g. release: kube-prometheus-stack + labels: {} + +# Optional browser UI for inspecting and operating this Redis Cluster. The +# component is disabled by default; enabling it creates one RedisInsight pod, +# its Service and storage, and (when requested) an Istio HTTP route. +redisInsight: + enabled: false + image: + repository: redis/redisinsight + tag: 3.8.0 + pullPolicy: IfNotPresent + port: 5540 + connection: + # -- Empty automatically uses this release's balanced bootstrap endpoint: + # internal Service DNS normally, or the discovered LoadBalancer ingress + # address when external access is enabled. Set a hostname/IP to override. + host: "" + # -- Zero automatically follows externalAccess.port or redis.port. + port: 0 + alias: redis-cluster + tls: false + persistence: + enabled: true + # -- Empty uses the cluster's default StorageClass. + storageClass: "" + accessModes: + - ReadWriteOnce + size: 1Gi + annotations: {} + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi + podAnnotations: {} + nodeSelector: {} + tolerations: [] + ingress: + enabled: false + host: "" + gatewaySelector: + istio: ingressgateway + requestTimeout: 60s + tls: + enabled: false + # -- Name of an existing TLS Secret available to the Istio ingress + # gateway. Required when TLS is enabled. + credentialName: "" + certificate: + # -- Ask cert-manager to create credentialName. Leave false when the + # Secret is managed outside this chart. + create: false + issuerName: "" + issuerKind: ClusterIssuer + # -- Empty creates the Certificate in the release namespace. Some + # Istio installations require gateway credentials in istio-system; + # set that namespace explicitly when needed. + secretNamespace: "" + +persistence: + enabled: true + # -- Empty uses the cluster's default StorageClass. + storageClass: "" + accessModes: + - ReadWriteOnce + # -- Requested capacity for every Redis member PVC. The chart imposes no + # upper bound; the StorageClass and cluster quota decide what is available. + size: 3Gi + annotations: {} + +istio: + # -- Keep the Envoy sidecar OFF the Redis pods (recommended). Redis Cluster + # uses direct pod-to-pod client and cluster-bus connections. Client pods can reach + # Redis from inside the mesh (mTLS in PERMISSIVE mode, or add a + # PeerAuthentication exception for this workload if you run STRICT). + injectSidecar: false + +networkPolicy: + # -- Standard Kubernetes NetworkPolicy. Enforcement depends on the installed + # CNI. Pod-to-pod client, replication, key-migration, and cluster-bus traffic + # is always allowed. + enabled: false + # -- Allow any pod in the cluster to reach the Redis client port. + # Set to false and fill extraIngress to lock down clients. + allowExternal: true + # -- Extra "from" peers allowed on the client ports when allowExternal=false, + # e.g. [{namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: myapp}}}] + extraIngress: [] + # -- Namespaces allowed to scrape metrics (empty = any). + metricsFromNamespaces: [] + +podDisruptionBudget: + enabled: true + maxUnavailable: 1 + +serviceAccount: + create: true + name: "" + annotations: {} + +nodeSelector: {} +tolerations: [] +topologySpreadConstraints: [] +priorityClassName: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: + fsGroup: 999 + runAsNonRoot: true + +containerSecurityContext: + runAsUser: 999 + runAsGroup: 999 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + +terminationGracePeriodSeconds: 30