382 lines
13 KiB
YAML
382 lines
13 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: {{ include "redis-cluster.fullname" . }}
|
|
namespace: {{ .Release.Namespace }}
|
|
labels:
|
|
{{- include "redis-cluster.labels" . | nindent 4 }}
|
|
data:
|
|
redis.conf: |
|
|
port {{ .Values.redis.port }}
|
|
dir /data
|
|
appendonly yes
|
|
appendfsync everysec
|
|
save ""
|
|
protected-mode no
|
|
tcp-keepalive 60
|
|
repl-diskless-sync yes
|
|
replica-read-only yes
|
|
cluster-enabled yes
|
|
cluster-config-file nodes.conf
|
|
cluster-port {{ .Values.cluster.busPort }}
|
|
cluster-node-timeout {{ .Values.cluster.nodeTimeoutMilliseconds }}
|
|
cluster-require-full-coverage {{ ternary "yes" "no" .Values.cluster.requireFullCoverage }}
|
|
cluster-allow-reads-when-down {{ ternary "yes" "no" .Values.cluster.allowReadsWhenDown }}
|
|
cluster-migration-barrier {{ .Values.cluster.migrationBarrier }}
|
|
cluster-preferred-endpoint-type hostname
|
|
{{- if .Values.redis.maxmemory }}
|
|
maxmemory {{ .Values.redis.maxmemory }}
|
|
maxmemory-policy {{ .Values.redis.maxmemoryPolicy }}
|
|
{{- end }}
|
|
{{- with .Values.redis.extraConfig }}
|
|
{{- . | nindent 4 }}
|
|
{{- end }}
|
|
|
|
start-redis.sh: |
|
|
#!/bin/sh
|
|
set -eu
|
|
|
|
POD_FQDN="$(hostname).${HEADLESS_SERVICE}"
|
|
if [ "$EXTERNAL_ACCESS_ENABLED" = "true" ]; then
|
|
ANNOUNCE_HOSTNAME="$(cat /external-endpoint/hostname)"
|
|
fi
|
|
/scripts/repair-node-addresses.sh
|
|
CONF=/etc/redis-runtime/redis.conf
|
|
cp /etc/redis-ro/redis.conf "$CONF"
|
|
{
|
|
echo "cluster-announce-hostname $ANNOUNCE_HOSTNAME"
|
|
echo "cluster-announce-human-nodename $(hostname)"
|
|
echo "cluster-announce-port $ANNOUNCE_PORT"
|
|
echo "cluster-announce-bus-port $CLUSTER_BUS_PORT"
|
|
if [ -n "${REDIS_PASSWORD:-}" ]; then
|
|
echo "requirepass $REDIS_PASSWORD"
|
|
echo "masterauth $REDIS_PASSWORD"
|
|
fi
|
|
} >> "$CONF"
|
|
|
|
# Only the designated coordinator attempts first-time cluster creation.
|
|
# The bootstrap runs beside Redis because every node must be listening
|
|
# before CLUSTER MEET can form the topology.
|
|
if [ "$POD_FQDN" = "$COORDINATOR_HOST" ]; then
|
|
/scripts/bootstrap-cluster.sh &
|
|
/scripts/reconcile-topology.sh &
|
|
fi
|
|
|
|
exec redis-server "$CONF"
|
|
|
|
discover-external-endpoint.sh: |
|
|
#!/bin/sh
|
|
set -eu
|
|
|
|
token_file=/var/run/secrets/kubernetes.io/serviceaccount/token
|
|
ca_file=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
|
api="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}"
|
|
url="${api}/api/v1/namespaces/${POD_NAMESPACE}/services/${EXTERNAL_SERVICE_NAME}"
|
|
|
|
while :; do
|
|
response="$(curl --fail --silent --show-error --cacert "$ca_file" \
|
|
-H "Authorization: Bearer $(cat "$token_file")" "$url" 2>/dev/null || true)"
|
|
endpoint="$(printf '%s\n' "$response" |
|
|
sed -n 's/.*"ip":[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1)"
|
|
if [ -z "$endpoint" ]; then
|
|
endpoint="$(printf '%s\n' "$response" |
|
|
sed -n 's/.*"hostname":[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1)"
|
|
fi
|
|
if [ -n "$endpoint" ]; then
|
|
printf '%s\n' "$endpoint" > /external-endpoint/hostname
|
|
echo "Discovered external endpoint $endpoint for $EXTERNAL_SERVICE_NAME"
|
|
exit 0
|
|
fi
|
|
echo "Waiting for LoadBalancer endpoint on $EXTERNAL_SERVICE_NAME"
|
|
sleep 2
|
|
done
|
|
|
|
start-redisinsight.sh: |
|
|
#!/bin/sh
|
|
set -eu
|
|
|
|
RI_REDIS_HOST="$(cat /bootstrap-endpoint/hostname)"
|
|
export RI_REDIS_HOST
|
|
echo "Starting RedisInsight with discovered Redis bootstrap endpoint $RI_REDIS_HOST"
|
|
exec "$@"
|
|
|
|
repair-node-addresses.sh: |
|
|
#!/bin/sh
|
|
set -u
|
|
# nodes.conf persists node IDs and membership on the PVC, but its cluster
|
|
# bus endpoints include pod IPs. Refresh those IPs using each member's
|
|
# unique human nodename before Redis starts so a full reschedule recovers
|
|
# independently of the client endpoint Redis advertises.
|
|
nodes=/data/nodes.conf
|
|
[ -s "$nodes" ] || exit 0
|
|
|
|
resolve_host() {
|
|
host=$1
|
|
if command -v getent >/dev/null 2>&1; then
|
|
getent hosts "$host" 2>/dev/null | awk 'NR == 1 { print $1 }'
|
|
else
|
|
nslookup "$host" 2>/dev/null |
|
|
awk '/^Address: / && $2 !~ /#/ { address=$2 } END { print address }'
|
|
fi
|
|
}
|
|
|
|
while :; do
|
|
resolved=0
|
|
old_ifs=$IFS; IFS=','
|
|
for host in $ALL_NODE_HOSTS; do
|
|
[ -n "$(resolve_host "$host")" ] && resolved=$((resolved + 1))
|
|
done
|
|
IFS=$old_ifs
|
|
[ "$resolved" = "$NODE_COUNT" ] && break
|
|
echo "Waiting for all $NODE_COUNT persisted member hostnames to resolve ($resolved ready)"
|
|
sleep 2
|
|
done
|
|
|
|
old_ifs=$IFS; IFS=','
|
|
for assignment in $NODE_ADDRESS_ASSIGNMENTS; do
|
|
pod_name=${assignment%%=*}
|
|
internal_host=${assignment#*=}
|
|
ip="$(resolve_host "$internal_host")"
|
|
[ -n "$ip" ] || continue
|
|
awk -v target="$pod_name" -v new_ip="$ip" '
|
|
{
|
|
metadata_count = split($2, metadata, ",")
|
|
nodename = ""
|
|
for (i = 3; i <= metadata_count; i++) {
|
|
split(metadata[i], auxiliary, "=")
|
|
if (auxiliary[1] == "nodename") nodename = auxiliary[2]
|
|
}
|
|
if (nodename == target) {
|
|
split(metadata[1], bus, "@")
|
|
split(bus[1], endpoint, ":")
|
|
suffix = substr($2, length(metadata[1]) + 1)
|
|
$2 = new_ip ":" endpoint[2] "@" bus[2] suffix
|
|
}
|
|
print
|
|
}
|
|
' "$nodes" > "${nodes}.new" && mv "${nodes}.new" "$nodes"
|
|
done
|
|
IFS=$old_ifs
|
|
|
|
bootstrap-cluster.sh: |
|
|
#!/bin/sh
|
|
set -u
|
|
|
|
cli() {
|
|
redis-cli --no-auth-warning "$@"
|
|
}
|
|
|
|
resolve_host() {
|
|
host=$1
|
|
if command -v getent >/dev/null 2>&1; then
|
|
getent hosts "$host" 2>/dev/null | awk 'NR == 1 { print $1 }'
|
|
else
|
|
nslookup "$host" 2>/dev/null |
|
|
awk '/^Address: / && $2 !~ /#/ { address=$2 } END { print address }'
|
|
fi
|
|
}
|
|
|
|
node_value() {
|
|
host=$1 field=$2
|
|
cli -h "$host" -p "$REDIS_PORT" cluster info 2>/dev/null |
|
|
tr -d '\r' | awk -F: -v field="$field" '$1 == field { print $2 }'
|
|
}
|
|
|
|
echo "Redis Cluster bootstrap coordinator: $COORDINATOR_HOST"
|
|
while :; do
|
|
all_ready=true
|
|
old_ifs=$IFS; IFS=','
|
|
for host in $ALL_NODE_HOSTS; do
|
|
if [ "$(cli -h "$host" -p "$REDIS_PORT" ping 2>/dev/null)" != PONG ]; then
|
|
all_ready=false
|
|
break
|
|
fi
|
|
done
|
|
IFS=$old_ifs
|
|
$all_ready && break
|
|
sleep 2
|
|
done
|
|
|
|
if [ "$(node_value "$COORDINATOR_HOST" cluster_state)" = ok ]; then
|
|
echo "Redis Cluster already healthy; bootstrap is a no-op"
|
|
exit 0
|
|
fi
|
|
|
|
# Never overwrite a partially configured or previously initialized
|
|
# cluster. That state needs repair, not a destructive second bootstrap.
|
|
pristine=true
|
|
old_ifs=$IFS; IFS=','
|
|
for host in $ALL_NODE_HOSTS; do
|
|
known="$(node_value "$host" cluster_known_nodes)"
|
|
assigned="$(node_value "$host" cluster_slots_assigned)"
|
|
if [ "$known" != 1 ] || [ "$assigned" != 0 ]; then
|
|
pristine=false
|
|
break
|
|
fi
|
|
done
|
|
IFS=$old_ifs
|
|
if ! $pristine; then
|
|
echo "Cluster has existing membership or slots but is not healthy; refusing to reinitialize it" >&2
|
|
exit 0
|
|
fi
|
|
|
|
echo "Creating a ${SHARD_COUNT}-shard cluster from ${NODE_COUNT} pristine nodes"
|
|
old_ifs=$IFS; IFS=','
|
|
for host in $ALL_NODE_HOSTS; do
|
|
[ "$host" = "$COORDINATOR_HOST" ] && continue
|
|
ip="$(resolve_host "$host")"
|
|
if [ -z "$ip" ]; then
|
|
echo "Could not resolve $host" >&2
|
|
exit 1
|
|
fi
|
|
cli -h "$COORDINATOR_HOST" -p "$REDIS_PORT" \
|
|
cluster meet "$ip" "$REDIS_PORT" "$CLUSTER_BUS_PORT" >/dev/null || exit 1
|
|
done
|
|
IFS=$old_ifs
|
|
|
|
# Wait until every node has learned the complete membership before roles
|
|
# are assigned. This avoids CLUSTER REPLICATE racing gossip propagation.
|
|
while :; do
|
|
complete=true
|
|
old_ifs=$IFS; IFS=','
|
|
for host in $ALL_NODE_HOSTS; do
|
|
[ "$(node_value "$host" cluster_known_nodes)" = "$NODE_COUNT" ] || complete=false
|
|
done
|
|
IFS=$old_ifs
|
|
$complete && break
|
|
sleep 1
|
|
done
|
|
|
|
shard=0
|
|
old_ifs=$IFS; IFS=','
|
|
for primary in $PRIMARY_HOSTS; do
|
|
start=$((16384 * shard / SHARD_COUNT))
|
|
end=$((16384 * (shard + 1) / SHARD_COUNT - 1))
|
|
echo "Assigning slots $start-$end to shard $shard ($primary)"
|
|
cli -h "$primary" -p "$REDIS_PORT" cluster addslotsrange "$start" "$end" >/dev/null || exit 1
|
|
shard=$((shard + 1))
|
|
done
|
|
IFS=$old_ifs
|
|
|
|
old_ifs=$IFS; IFS=','
|
|
for assignment in $REPLICA_ASSIGNMENTS; do
|
|
replica=${assignment%%=*}
|
|
primary=${assignment#*=}
|
|
primary_id="$(cli -h "$primary" -p "$REDIS_PORT" cluster myid 2>/dev/null | tr -d '\r')"
|
|
[ -n "$primary_id" ] || exit 1
|
|
echo "Attaching $replica as a replica of $primary"
|
|
cli -h "$replica" -p "$REDIS_PORT" cluster replicate "$primary_id" >/dev/null || exit 1
|
|
done
|
|
IFS=$old_ifs
|
|
|
|
# Do not declare success merely because slots are covered. Wait for every
|
|
# intended replica relationship and for every member to see a healthy
|
|
# cluster, which prevents an early restart from racing topology gossip.
|
|
while :; do
|
|
converged=true
|
|
old_ifs=$IFS; IFS=','
|
|
for assignment in $REPLICA_ASSIGNMENTS; do
|
|
replica=${assignment%%=*}
|
|
role="$(cli -h "$replica" -p "$REDIS_PORT" role 2>/dev/null | head -n1)"
|
|
[ "$role" = slave ] || converged=false
|
|
done
|
|
for host in $ALL_NODE_HOSTS; do
|
|
[ "$(node_value "$host" cluster_state)" = ok ] || converged=false
|
|
done
|
|
IFS=$old_ifs
|
|
$converged && break
|
|
sleep 1
|
|
done
|
|
echo "Redis Cluster created: all 16384 slots covered"
|
|
|
|
reconcile-topology.sh: |
|
|
#!/bin/sh
|
|
set -u
|
|
# Redis correctly promotes replicas but does not automatically fail back.
|
|
# Restore the canonical member-0 primaries after a failed worker returns,
|
|
# keeping the three current primaries distributed across workers.
|
|
cli() { redis-cli --no-auth-warning "$@"; }
|
|
cluster_state() {
|
|
cli -h "$COORDINATOR_HOST" -p "$REDIS_PORT" cluster info 2>/dev/null |
|
|
tr -d '\r' | awk -F: '$1 == "cluster_state" { print $2 }'
|
|
}
|
|
|
|
while :; do
|
|
[ -f /etc/redis-runtime/terminating ] && exit 0
|
|
if [ "$(cluster_state)" = ok ]; then
|
|
old_ifs=$IFS; IFS=','
|
|
for primary_identity in $PRIMARY_HOSTS; do
|
|
role="$(cli -h "$primary_identity" -p "$REDIS_PORT" role 2>/dev/null | head -n1)"
|
|
if [ "$role" = slave ]; then
|
|
link="$(cli -h "$primary_identity" -p "$REDIS_PORT" info replication 2>/dev/null |
|
|
tr -d '\r' | awk -F: '$1 == "master_link_status" { print $2 }')"
|
|
if [ "$link" = up ]; then
|
|
echo "Restoring canonical primary role to $primary_identity"
|
|
cli -h "$primary_identity" -p "$REDIS_PORT" cluster failover >/dev/null 2>&1 || true
|
|
i=0
|
|
while [ "$i" -lt 20 ]; do
|
|
role="$(cli -h "$primary_identity" -p "$REDIS_PORT" role 2>/dev/null | head -n1)"
|
|
[ "$role" = master ] && break
|
|
sleep 1
|
|
i=$((i + 1))
|
|
done
|
|
while [ "$(cluster_state)" != ok ]; do sleep 1; done
|
|
fi
|
|
fi
|
|
done
|
|
IFS=$old_ifs
|
|
fi
|
|
sleep 30
|
|
done
|
|
|
|
startup-redis.sh: |
|
|
#!/bin/sh
|
|
[ "$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning ping 2>/dev/null)" = PONG ]
|
|
|
|
liveness-redis.sh: |
|
|
#!/bin/sh
|
|
resp="$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning ping 2>&1 || true)"
|
|
case "$resp" in
|
|
PONG|*LOADING*|*CLUSTERDOWN*) exit 0 ;;
|
|
*) echo "$resp"; exit 1 ;;
|
|
esac
|
|
|
|
readiness-redis.sh: |
|
|
#!/bin/sh
|
|
[ "$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning ping 2>/dev/null)" = PONG ] || exit 1
|
|
state="$(redis-cli -t 3 -p "$REDIS_PORT" --no-auth-warning cluster info 2>/dev/null |
|
|
tr -d '\r' | awk -F: '$1 == "cluster_state" { print $2 }')"
|
|
[ "$state" = ok ] || { echo "cluster_state=$state"; exit 1; }
|
|
|
|
prestop-redis.sh: |
|
|
#!/bin/sh
|
|
# During an intentional restart, ask a healthy replica to take over first.
|
|
# Replica identities remain on a different Kubernetes node because shard
|
|
# anti-affinity is tied to persistent member identity, not current role.
|
|
: > /etc/redis-runtime/terminating
|
|
role="$(redis-cli -p "$REDIS_PORT" --no-auth-warning role 2>/dev/null | head -n1 || true)"
|
|
[ "$role" = master ] || exit 0
|
|
node_id="$(redis-cli -p "$REDIS_PORT" --no-auth-warning cluster myid 2>/dev/null | tr -d '\r')"
|
|
[ -n "$node_id" ] || exit 0
|
|
endpoint="$(redis-cli -p "$REDIS_PORT" --no-auth-warning cluster replicas "$node_id" 2>/dev/null |
|
|
awk '$3 !~ /fail/ && $8 == "connected" { print $2; exit }')"
|
|
[ -n "$endpoint" ] || exit 0
|
|
address=${endpoint%%@*}
|
|
metadata=${endpoint#*,}
|
|
if [ "$metadata" != "$endpoint" ] && [ -n "$metadata" ]; then
|
|
replica_host=$metadata
|
|
else
|
|
replica_host=${address%:*}
|
|
fi
|
|
replica_port=${address##*:}
|
|
echo "Requesting graceful failover to $replica_host:$replica_port"
|
|
redis-cli -h "$replica_host" -p "$replica_port" --no-auth-warning cluster failover >/dev/null 2>&1 || exit 0
|
|
i=0
|
|
while [ "$i" -lt 15 ]; do
|
|
role="$(redis-cli -p "$REDIS_PORT" --no-auth-warning role 2>/dev/null | head -n1 || true)"
|
|
[ "$role" != master ] && exit 0
|
|
sleep 1
|
|
i=$((i + 1))
|
|
done
|
|
exit 0
|