apiVersion: v1 kind: ConfigMap metadata: name: {{ include "redis-cluster.fullname" . }} namespace: {{ .Release.Namespace }} labels: {{- include "redis-cluster.labels" . | nindent 4 }} data: redis.conf: | port {{ .Values.redis.port }} dir /data appendonly yes appendfsync everysec save "" protected-mode no tcp-keepalive 60 repl-diskless-sync yes replica-read-only yes cluster-enabled yes cluster-config-file nodes.conf cluster-port {{ .Values.cluster.busPort }} cluster-node-timeout {{ .Values.cluster.nodeTimeoutMilliseconds }} cluster-require-full-coverage {{ ternary "yes" "no" .Values.cluster.requireFullCoverage }} cluster-allow-reads-when-down {{ ternary "yes" "no" .Values.cluster.allowReadsWhenDown }} cluster-migration-barrier {{ .Values.cluster.migrationBarrier }} cluster-preferred-endpoint-type hostname {{- if .Values.redis.maxmemory }} maxmemory {{ .Values.redis.maxmemory }} maxmemory-policy {{ .Values.redis.maxmemoryPolicy }} {{- end }} {{- with .Values.redis.extraConfig }} {{- . | nindent 4 }} {{- end }} start-redis.sh: | #!/bin/sh set -eu POD_FQDN="$(hostname).${HEADLESS_SERVICE}" if [ "$EXTERNAL_ACCESS_ENABLED" = "true" ]; then ANNOUNCE_HOSTNAME="$(cat /external-endpoint/hostname)" fi /scripts/repair-node-addresses.sh CONF=/etc/redis-runtime/redis.conf cp /etc/redis-ro/redis.conf "$CONF" { echo "cluster-announce-hostname $ANNOUNCE_HOSTNAME" echo "cluster-announce-human-nodename $(hostname)" echo "cluster-announce-port $ANNOUNCE_PORT" echo "cluster-announce-bus-port $CLUSTER_BUS_PORT" if [ -n "${REDIS_PASSWORD:-}" ]; then echo "requirepass $REDIS_PASSWORD" echo "masterauth $REDIS_PASSWORD" fi } >> "$CONF" # Only the designated coordinator attempts first-time cluster creation. # The bootstrap runs beside Redis because every node must be listening # before CLUSTER MEET can form the topology. if [ "$POD_FQDN" = "$COORDINATOR_HOST" ]; then /scripts/bootstrap-cluster.sh & /scripts/reconcile-topology.sh & fi exec redis-server "$CONF" discover-external-endpoint.sh: | #!/bin/sh set -eu token_file=/var/run/secrets/kubernetes.io/serviceaccount/token ca_file=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt api="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}" url="${api}/api/v1/namespaces/${POD_NAMESPACE}/services/${EXTERNAL_SERVICE_NAME}" while :; do response="$(curl --fail --silent --show-error --cacert "$ca_file" \ -H "Authorization: Bearer $(cat "$token_file")" "$url" 2>/dev/null || true)" endpoint="$(printf '%s\n' "$response" | sed -n 's/.*"ip":[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1)" if [ -z "$endpoint" ]; then endpoint="$(printf '%s\n' "$response" | sed -n 's/.*"hostname":[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1)" fi if [ -n "$endpoint" ]; then printf '%s\n' "$endpoint" > /external-endpoint/hostname echo "Discovered external endpoint $endpoint for $EXTERNAL_SERVICE_NAME" exit 0 fi echo "Waiting for LoadBalancer endpoint on $EXTERNAL_SERVICE_NAME" sleep 2 done start-redisinsight.sh: | #!/bin/sh set -eu RI_REDIS_HOST="$(cat /bootstrap-endpoint/hostname)" export RI_REDIS_HOST echo "Starting RedisInsight with discovered Redis bootstrap endpoint $RI_REDIS_HOST" exec "$@" repair-node-addresses.sh: | #!/bin/sh set -u # nodes.conf persists node IDs and membership on the PVC, but its cluster # bus endpoints include pod IPs. Refresh those IPs using each member's # unique human nodename before Redis starts so a full reschedule recovers # independently of the client endpoint Redis advertises. nodes=/data/nodes.conf [ -s "$nodes" ] || exit 0 resolve_host() { host=$1 if command -v getent >/dev/null 2>&1; then getent hosts "$host" 2>/dev/null | awk 'NR == 1 { print $1 }' else nslookup "$host" 2>/dev/null | awk '/^Address: / && $2 !~ /#/ { address=$2 } END { print address }' fi } while :; do resolved=0 old_ifs=$IFS; IFS=',' for host in $ALL_NODE_HOSTS; do [ -n "$(resolve_host "$host")" ] && resolved=$((resolved + 1)) done IFS=$old_ifs [ "$resolved" = "$NODE_COUNT" ] && break echo "Waiting for all $NODE_COUNT persisted member hostnames to resolve ($resolved ready)" sleep 2 done old_ifs=$IFS; IFS=',' for assignment in $NODE_ADDRESS_ASSIGNMENTS; do pod_name=${assignment%%=*} internal_host=${assignment#*=} ip="$(resolve_host "$internal_host")" [ -n "$ip" ] || continue awk -v target="$pod_name" -v new_ip="$ip" ' { metadata_count = split($2, metadata, ",") nodename = "" for (i = 3; i <= metadata_count; i++) { split(metadata[i], auxiliary, "=") if (auxiliary[1] == "nodename") nodename = auxiliary[2] } if (nodename == target) { split(metadata[1], bus, "@") split(bus[1], endpoint, ":") suffix = substr($2, length(metadata[1]) + 1) $2 = new_ip ":" endpoint[2] "@" bus[2] suffix } print } ' "$nodes" > "${nodes}.new" && mv "${nodes}.new" "$nodes" done IFS=$old_ifs bootstrap-cluster.sh: | #!/bin/sh set -u cli() { redis-cli --no-auth-warning "$@" } resolve_host() { host=$1 if command -v getent >/dev/null 2>&1; then getent hosts "$host" 2>/dev/null | awk 'NR == 1 { print $1 }' else nslookup "$host" 2>/dev/null | awk '/^Address: / && $2 !~ /#/ { address=$2 } END { print address }' fi } node_value() { host=$1 field=$2 cli -h "$host" -p "$REDIS_PORT" cluster info 2>/dev/null | tr -d '\r' | awk -F: -v field="$field" '$1 == field { print $2 }' } echo "Redis Cluster bootstrap coordinator: $COORDINATOR_HOST" while :; do all_ready=true old_ifs=$IFS; IFS=',' for host in $ALL_NODE_HOSTS; do if [ "$(cli -h "$host" -p "$REDIS_PORT" ping 2>/dev/null)" != PONG ]; then all_ready=false break fi done IFS=$old_ifs $all_ready && break sleep 2 done if [ "$(node_value "$COORDINATOR_HOST" cluster_state)" = ok ]; then echo "Redis Cluster already healthy; bootstrap is a no-op" exit 0 fi # Never overwrite a partially configured or previously initialized # cluster. That state needs repair, not a destructive second bootstrap. pristine=true old_ifs=$IFS; IFS=',' for host in $ALL_NODE_HOSTS; do known="$(node_value "$host" cluster_known_nodes)" assigned="$(node_value "$host" cluster_slots_assigned)" if [ "$known" != 1 ] || [ "$assigned" != 0 ]; then pristine=false break fi done IFS=$old_ifs if ! $pristine; then echo "Cluster has existing membership or slots but is not healthy; refusing to reinitialize it" >&2 exit 0 fi echo "Creating a ${SHARD_COUNT}-shard cluster from ${NODE_COUNT} pristine nodes" old_ifs=$IFS; IFS=',' for host in $ALL_NODE_HOSTS; do [ "$host" = "$COORDINATOR_HOST" ] && continue ip="$(resolve_host "$host")" if [ -z "$ip" ]; then echo "Could not resolve $host" >&2 exit 1 fi cli -h "$COORDINATOR_HOST" -p "$REDIS_PORT" \ cluster meet "$ip" "$REDIS_PORT" "$CLUSTER_BUS_PORT" >/dev/null || exit 1 done IFS=$old_ifs # Wait until every node has learned the complete membership before roles # are assigned. This avoids CLUSTER REPLICATE racing gossip propagation. while :; do complete=true old_ifs=$IFS; IFS=',' for host in $ALL_NODE_HOSTS; do [ "$(node_value "$host" cluster_known_nodes)" = "$NODE_COUNT" ] || complete=false done IFS=$old_ifs $complete && break sleep 1 done shard=0 old_ifs=$IFS; IFS=',' for primary in $PRIMARY_HOSTS; do start=$((16384 * shard / SHARD_COUNT)) end=$((16384 * (shard + 1) / SHARD_COUNT - 1)) echo "Assigning slots $start-$end to shard $shard ($primary)" cli -h "$primary" -p "$REDIS_PORT" cluster addslotsrange "$start" "$end" >/dev/null || exit 1 shard=$((shard + 1)) done IFS=$old_ifs old_ifs=$IFS; IFS=',' for assignment in $REPLICA_ASSIGNMENTS; do replica=${assignment%%=*} primary=${assignment#*=} primary_id="$(cli -h "$primary" -p "$REDIS_PORT" cluster myid 2>/dev/null | tr -d '\r')" [ -n "$primary_id" ] || exit 1 echo "Attaching $replica as a replica of $primary" cli -h "$replica" -p "$REDIS_PORT" cluster replicate "$primary_id" >/dev/null || exit 1 done IFS=$old_ifs # Do not declare success merely because slots are covered. Wait for every # intended replica relationship and for every member to see a healthy # cluster, which prevents an early restart from racing topology gossip. while :; do converged=true old_ifs=$IFS; IFS=',' for assignment in $REPLICA_ASSIGNMENTS; do replica=${assignment%%=*} role="$(cli -h "$replica" -p "$REDIS_PORT" role 2>/dev/null | head -n1)" [ "$role" = slave ] || converged=false done for host in $ALL_NODE_HOSTS; do [ "$(node_value "$host" cluster_state)" = ok ] || converged=false done IFS=$old_ifs $converged && break sleep 1 done echo "Redis Cluster created: all 16384 slots covered" reconcile-topology.sh: | #!/bin/sh set -u # Redis correctly promotes replicas but does not automatically fail back. # Restore the canonical member-0 primaries after a failed worker returns, # keeping the three current primaries distributed across workers. cli() { redis-cli --no-auth-warning "$@"; } cluster_state() { cli -h "$COORDINATOR_HOST" -p "$REDIS_PORT" cluster info 2>/dev/null | tr -d '\r' | awk -F: '$1 == "cluster_state" { print $2 }' } while :; do [ -f /etc/redis-runtime/terminating ] && exit 0 if [ "$(cluster_state)" = ok ]; then old_ifs=$IFS; IFS=',' for primary_identity in $PRIMARY_HOSTS; do role="$(cli -h "$primary_identity" -p "$REDIS_PORT" role 2>/dev/null | head -n1)" if [ "$role" = slave ]; then link="$(cli -h "$primary_identity" -p "$REDIS_PORT" info replication 2>/dev/null | tr -d '\r' | awk -F: '$1 == "master_link_status" { print $2 }')" if [ "$link" = up ]; then echo "Restoring canonical primary role to $primary_identity" cli -h "$primary_identity" -p "$REDIS_PORT" cluster failover >/dev/null 2>&1 || true i=0 while [ "$i" -lt 20 ]; do role="$(cli -h "$primary_identity" -p "$REDIS_PORT" role 2>/dev/null | head -n1)" [ "$role" = master ] && break sleep 1 i=$((i + 1)) done while [ "$(cluster_state)" != ok ]; do sleep 1; done fi fi done IFS=$old_ifs fi sleep 30 done startup-redis.sh: | #!/bin/sh [ "$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning ping 2>/dev/null)" = PONG ] liveness-redis.sh: | #!/bin/sh resp="$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning ping 2>&1 || true)" case "$resp" in PONG|*LOADING*|*CLUSTERDOWN*) exit 0 ;; *) echo "$resp"; exit 1 ;; esac readiness-redis.sh: | #!/bin/sh [ "$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning ping 2>/dev/null)" = PONG ] || exit 1 state="$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning cluster info 2>/dev/null | tr -d '\r' | awk -F: '$1 == "cluster_state" { print $2 }')" [ "$state" = ok ] || { echo "cluster_state=$state"; exit 1; } prestop-redis.sh: | #!/bin/sh # During an intentional restart, ask a healthy replica to take over first. # Replica identities remain on a different Kubernetes node because shard # anti-affinity is tied to persistent member identity, not current role. : > /etc/redis-runtime/terminating role="$(redis-cli -p "$REDIS_PORT" --no-auth-warning role 2>/dev/null | head -n1 || true)" [ "$role" = master ] || exit 0 node_id="$(redis-cli -p "$REDIS_PORT" --no-auth-warning cluster myid 2>/dev/null | tr -d '\r')" [ -n "$node_id" ] || exit 0 endpoint="$(redis-cli -p "$REDIS_PORT" --no-auth-warning cluster replicas "$node_id" 2>/dev/null | awk '$3 !~ /fail/ && $8 == "connected" { print $2; exit }')" [ -n "$endpoint" ] || exit 0 address=${endpoint%%@*} metadata=${endpoint#*,} if [ "$metadata" != "$endpoint" ] && [ -n "$metadata" ]; then replica_host=$metadata else replica_host=${address%:*} fi replica_port=${address##*:} echo "Requesting graceful failover to $replica_host:$replica_port" redis-cli -h "$replica_host" -p "$replica_port" --no-auth-warning cluster failover >/dev/null 2>&1 || exit 0 i=0 while [ "$i" -lt 15 ]; do role="$(redis-cli -p "$REDIS_PORT" --no-auth-warning role 2>/dev/null | head -n1 || true)" [ "$role" != master ] && exit 0 sleep 1 i=$((i + 1)) done exit 0