commands:
base [ create | boot ] ...
- cluster create
+ cluster [ build |
+ destroy | undefine |
+ create | update_hosts | boot | setup ] ...
create base
create a base image
fi
}
+announce ()
+{
+ echo "######################################################################"
+ printf "# %-66s #\n" "$*"
+ echo "######################################################################"
+ echo ""
+}
+
+waitfor ()
+{
+ local file="$1"
+ local msg="$2"
+ local timeout="$3"
+
+ local tmpfile=$(mktemp)
+
+ cat <<EOF >"$tmpfile"
+spawn tail -n 10000 -f $file
+expect -timeout $timeout -re "$msg"
+EOF
+
+ export LANG=C
+ expect "$tmpfile"
+ rm -f "$tmpfile"
+
+ if ! grep -E "$msg" "$file" > /dev/null; then
+ echo "Failed to find \"$msg\" in \"$file\""
+ return 1
+ fi
+
+ return 0
+}
+
###############################
# Indirectly call a function named by ${1}_${2}
done
}
+node_is_ctdb_node_DEFAULT ()
+{
+ echo 0
+}
+
hack_one_node_with ()
{
local filter="$1" ; shift
NODES="$nodes"
}
+list_all_cluster_nodes ()
+{
+ # Local function only defined in subshell
+ (
+ print_node_name ()
+ {
+ echo "$3"
+ }
+ for_each_node print_node_name
+ ) | sort
+}
+
+list_all_virsh_domains ()
+{
+ local pattern="${CLUSTER_PATTERN:-${CLUSTER}[a-z]*[0-9]}"
+
+ local domains=$(virsh list --all | awk '{print $2}' | tail -n +3)
+ local d
+ for d in $domains ; do
+ case "$d" in
+ ($pattern) echo "$d" ;;
+ esac
+ done | sort
+}
+
+virsh_cluster ()
+{
+ local command="$1"
+ shift
+
+ local nodes=$(list_all_cluster_nodes)
+ local domains=$(list_all_virsh_domains)
+
+ if [ "$nodes" != "$domains" ] ; then
+ echo "WARNING: Found matching virsh domains that are not part of this cluster!"
+ echo
+ fi
+
+ local ret=0
+ local n
+ for n in $nodes ; do
+ virsh "$command" "$n" "$@" 2>&1 || ret=$?
+ done
+
+ return $ret
+}
+
register_hook ()
{
local hook_var="$1"
# the same disk.
hack_disk_hooks=""
+create_node_DEFAULT ()
+{
+ local type="$1"
+ local ip_offset="$2"
+ local name="$3"
+ local ctdb_node="$4"
+
+ echo "Creating node \"$name\" (of type \"${type}\")"
+
+ create_node_COMMON "$name" "$ip_offset" "$type"
+}
+
# common node creation stuff
create_node_COMMON ()
{
diskimage unmount
}
-# Provides an easy way of removing nodes from $NODE.
-create_node_null () {
- :
-}
-
hack_network_map_hooks=""
# Uses: CLUSTER, NAME, NETWORKS, FIRSTIP, ip_offset
local net="${ip_bits%/*}"
local netname="acnet_${net//./_}"
- local ip="${net%.*}.${IPNUM}"
- local mask="255.255.255.0"
+ local ip="${net%.*}.${IPNUM}/${ip_bits#*/}"
+
+ local ipv6="fc00:${net//./:}::${IPNUM}/64"
# This can be used to override the variables in the echo
# statement below. The hook can use any other variables
# available in this function.
run_hooks hack_network_map_hooks
- echo "${netname} ${dev} ${ip} ${mask} ${mac} ${opts}"
+ echo "${netname} ${dev} ${ip} ${ipv6} ${mac} ${opts}"
count=$(($count + 1))
done >"$network_map"
}
##############################
-hack_nodes_functions=
-
-expand_nodes () {
+expand_nodes ()
+{
# Expand out any abbreviations in NODES.
local ns=""
local n
done
NODES="$ns"
- # Apply nodes hacks. Some of this is about backward compatibility
- # but the hacks also fill in the node names and whether they're
- # part of the CTDB cluster. The order is the order that
- # configuration modules register their hacks.
- run_hooks hack_nodes_functions
-
- if [ -n "$NUMNODES" ] ; then
- # Attempt to respect NUMNODES. Reduce the number of CTDB
- # nodes to NUMNODES.
- local numnodes=$NUMNODES
-
- hack_filter ()
- {
- if [ "$ctdb_node" = 1 ] ; then
- if [ $numnodes -gt 0 ] ; then
- numnodes=$(($numnodes - 1))
- else
- node_type="null"
- ctdb_node=0
- fi
- fi
- }
-
- hack_all_nodes_with hack_filter
-
- [ $numnodes -gt 0 ] && \
- die "Can't not use NUMNODES to increase the number of nodes over that specified by NODES. You need to set NODES instead - please read the documentation."
- fi
-
# Check IP addresses for duplicates.
local ip_offsets=":"
# This function doesn't modify anything...
ip_offsets="${ip_offsets}${ip_offset}:"
}
hack_all_nodes_with get_ip_offset
+
+ # Determine node names and whether they're in the CTDB cluster
+ declare -A node_count
+ _get_name_ctdb_node ()
+ {
+ local count=$((${node_count[$node_type]:-0} + 1))
+ node_count[$node_type]=$count
+ name=$(call_func node_name_format "$node_type" "$CLUSTER" $count) || {
+ echo "ERROR: Node type \"${node_type}\" not defined!"
+ echo "Valid node types are:"
+ set | sed -n 's@^node_name_format_\(.*\) ().*@ \1@p'
+ exit 1
+ }
+ ctdb_node=$(call_func node_is_ctdb_node "$node_type")
+ }
+ hack_all_nodes_with _get_name_ctdb_node
}
##############################
hosts_file=
-common_nodelist_hacking ()
+cluster_nodelist_hacking ()
{
# Rework the NODES list
expand_nodes
# Build /etc/hosts and hack the names of the ctdb nodes
hosts_line_hack_name ()
{
- # Ignore nodes without names (e.g. "null")
- [ "$node_type" != "null" -a -n "$name" ] || return 0
-
local sname=""
local hosts_line
local ip_addr="${NETWORK_PRIVATE_PREFIX}.$(($FIRSTIP + $ip_offset))"
-
+
+ # Primary name for CTDB nodes is <CLUSTER>n<num>
if [ "$ctdb_node" = 1 ] ; then
num_ctdb_nodes=$(($num_ctdb_nodes + 1))
sname="${CLUSTER}n${num_ctdb_nodes}"
nodes_file="tmp/nodes.$CLUSTER"
local num_nodes=0
hack_all_nodes_with ctdb_nodes_line >$nodes_file
- : "${NUMNODES:=${num_nodes}}" # Set $NUMNODES if necessary
+
+ # Build /etc/ctdb/nodes.ipv6
+ ctdb_nodes_line_ipv6 ()
+ {
+ [ "$ctdb_node" = 1 ] || return 0
+ echo "fc00:${NETWORK_PRIVATE_PREFIX//./:}::$(($FIRSTIP + $ip_offset))"
+ num_nodes=$(($num_nodes + 1))
+ }
+ nodes_file_ipv6="tmp/nodes.$CLUSTER.ipv6"
+ local num_nodes=0
+ hack_all_nodes_with ctdb_nodes_line_ipv6 >$nodes_file_ipv6
# Build UUID map
uuid_map="tmp/uuid_map.$CLUSTER"
{
# Use $1. If not set then use value from configuration file.
CLUSTER="${1:-${CLUSTER}}"
+ announce "cluster create \"${CLUSTER}\""
[ -n "$CLUSTER" ] || die "\$CLUSTER not set"
sanity_check_cluster_name
# Run hooks before doing anything else.
run_hooks create_cluster_hooks
- common_nodelist_hacking
-
for_each_node call_func create_node
echo "Cluster $CLUSTER created"
register_hook cluster_created_hooks cluster_created_hosts_message
+cluster_destroy ()
+{
+ announce "cluster destroy \"${CLUSTER}\""
+ [ -n "$CLUSTER" ] || die "\$CLUSTER not set"
+
+ virsh_cluster destroy || true
+}
+
+cluster_undefine ()
+{
+ announce "cluster undefine \"${CLUSTER}\""
+ [ -n "$CLUSTER" ] || die "\$CLUSTER not set"
+
+ virsh_cluster undefine --managed-save || true
+}
+
+cluster_update_hosts ()
+{
+ announce "cluster update_hosts \"${CLUSTER}\""
+ [ -n "$CLUSTER" ] || die "\$CLUSTER not set"
+
+ [ -n "$hosts_file" ] || hosts_file="tmp/hosts.${CLUSTER}"
+ [ -r "$hosts_file" ] || die "Missing hosts file \"${hosts_file}\""
+
+ # Building a general node name regexp is a bit cumbersome. :-)
+ local name_regexp="("
+ for i in $(set | sed -n -e "s@^\(node_name_format_.*\) ().*@\1@p") ; do
+ # Format node name with placeholders (remembering that "_" is
+ # not valid in a real cluster name)
+ local t=$("$i" "_" "0")
+ # now replace the placeholders with regexps - order is
+ # important here, since the cluster name can contain digits
+ t=$(sed -r -e "s@[[:digit:]]+@[[:digit:]]+@" -e "s@_@${CLUSTER}@" <<<"$t")
+ # add to the regexp
+ name_regexp="${name_regexp}${t}|"
+ done
+ name_regexp="${name_regexp}${CLUSTER}n[[:digit:]]+)"
+
+ local pat="# autocluster ${CLUSTER}\$|[[:space:]]${name_regexp}"
+
+ local t="/etc/hosts.${CLUSTER}"
+ grep -E "$pat" /etc/hosts >"$t" || true
+ if diff -B "$t" "$hosts_file" >/dev/null ; then
+ rm "$t"
+ return
+ fi
+
+ local old=/etc/hosts.old.autocluster
+ cp /etc/hosts "$old"
+ local new=/etc/hosts.new
+ grep -Ev "$pat" "$old" |
+ cat -s - "$hosts_file" >"$new"
+
+ mv "$new" /etc/hosts
+
+ echo "Made these changes to /etc/hosts:"
+ diff -u "$old" /etc/hosts || true
+}
+
+cluster_boot ()
+{
+ announce "cluster boot \"${CLUSTER}\""
+ [ -n "$CLUSTER" ] || die "\$CLUSTER not set"
+
+ virsh_cluster start || return $?
+
+ local nodes=$(list_all_cluster_nodes)
+
+ # Wait for each node
+ local i
+ for i in $nodes ; do
+ waitfor "${KVMLOG}/serial.$i" "login:" 300 || {
+ vircmd destroy "$CLUSTER_PATTERN"
+ die "Failed to create cluster"
+ }
+ done
+
+ # Move past the last line of log output
+ echo ""
+}
+
+cluster_setup_tasks_DEFAULT ()
+{
+ local stage="$1"
+
+ # By default nodes have no tasks
+ case "$stage" in
+ install_packages) echo "" ;;
+ setup_clusterfs) echo "" ;;
+ setup_node) echo "" ;;
+ setup_cluster) echo "" ;;
+ esac
+}
+
+cluster_setup ()
+{
+ announce "cluster setup \"${CLUSTER}\""
+ [ -n "$CLUSTER" ] || die "\$CLUSTER not set"
+
+ local ssh="ssh -n -o StrictHostKeyChecking=no"
+ local setup_clusterfs_done=false
+ local setup_cluster_done=false
+
+ _cluster_setup_do_stage ()
+ {
+ local stage="$1"
+ local type="$2"
+ local ip_offset="$3"
+ local name="$4"
+ local ctdb_node="$5"
+
+ local tasks=$(call_func cluster_setup_tasks "$type" "$stage")
+
+ if [ -n "$tasks" ] ; then
+ # These tasks are only done on 1 node
+ case "$stage" in
+ setup_clusterfs)
+ if $setup_clusterfs_done ; then
+ return
+ else
+ setup_clusterfs_done=true
+ fi
+ ;;
+ setup_cluster)
+ if $setup_cluster_done ; then
+ return
+ else
+ setup_cluster_done=true
+ fi
+ ;;
+ esac
+
+ $ssh "$name" ./scripts/cluster_setup.sh "$stage" $tasks
+ fi
+
+ }
+
+ local stages="install_packages setup_clusterfs setup_node setup_cluster"
+ local stage
+ for stage in $stages ; do
+ for_each_node _cluster_setup_do_stage "$stage"
+ done
+}
+
create_one_node ()
{
CLUSTER="$1"
mkdir -p $VIRTBASE/$CLUSTER $KVMLOG tmp
- common_nodelist_hacking
-
for n in $NODES ; do
set -- $(IFS=: ; echo $n)
[ $single_node_ip_offset -eq $2 ] || continue
# specified then use the IP address associated with it.
INSTALL_IP=""
INSTALL_GW=""
- local netname dev ip mask mac opts
- while read netname dev ip mask mac opts; do
+ local netname dev ip ipv6 mac opts
+ while read netname dev ip ipv6 mac opts; do
local o
for o in $opts ; do
case "$o" in
gw\=*)
INSTALL_GW="${o#gw=}"
- INSTALL_IP="${ip}${FIRSTIP}"
+ INSTALL_IP="$ip"
esac
done
[ -n "$INSTALL_IP" ] || INSTALL_IP="$ip"
setup_timezone
+ IPNUM=$FIRSTIP
make_network_map
guess_install_network
run_hooks setup_base_hooks
}
+ipv4_prefix_to_netmask ()
+{
+ local prefix="$1"
+
+ local div=$(($prefix / 8))
+ local mod=$(($prefix % 8))
+
+ local octet
+ for octet in 1 2 3 4 ; do
+ if [ $octet -le $div ] ; then
+ echo -n "255"
+ elif [ $mod -ne 0 -a $octet -eq $(($div + 1)) ] ; then
+ local shift=$((8 - $mod))
+ echo -n $(( (255 >> $shift << $shift) ))
+ else
+ echo -n 0
+ fi
+ if [ $octet -lt 4 ] ; then
+ echo -n '.'
+ fi
+ done
+
+ echo
+}
+
# setup various networking components
setup_network()
{
echo "Setting up /etc/ctdb/nodes"
diskimage mkdir_p "/etc/ctdb"
- diskimage put "$nodes_file" "/etc/ctdb/nodes"
+ if [ "$NETWORK_STACK" = "ipv4" ] ; then
+ diskimage put "$nodes_file" "/etc/ctdb/nodes"
+ elif [ "$NETWORK_STACK" = "ipv6" ] ; then
+ diskimage put "$nodes_file_ipv6" "/etc/ctdb/nodes"
+ elif [ "$NETWORK_STACK" = "dual" ] ; then
+ diskimage put "$nodes_file" "/etc/ctdb/nodes.ipv4"
+ diskimage put "$nodes_file_ipv6" "/etc/ctdb/nodes.ipv6"
+ diskimage put "$nodes_file" "/etc/ctdb/nodes"
+ else
+ die "Error: Invalid NETWORK_STACK value \"$NETWORK_STACK\"."
+ fi
[ "$WEBPROXY" = "" ] || {
diskimage append_text "export http_proxy=$WEBPROXY" "/etc/bashrc"
diskimage rm_rf "/etc/udev/rules.d/70-persistent-net.rules"
echo "Setting up network interfaces: "
- local netname dev ip mask mac opts
- while read netname dev ip mask mac opts; do
+ local netname dev ip ipv6 mac opts
+ while read netname dev ip ipv6 mac opts; do
echo " $dev"
- local o gw
+ local o gw addr mask
gw=""
for o in $opts ; do
case "$o" in
esac
done
+ addr=${ip%/*}
+ mask=$(ipv4_prefix_to_netmask ${ip#*/})
+
cat <<EOF | \
diskimage put - "/etc/sysconfig/network-scripts/ifcfg-${dev}"
DEVICE=$dev
ONBOOT=yes
TYPE=Ethernet
-IPADDR=$ip
+IPADDR=$addr
NETMASK=$mask
HWADDR=$mac
+IPV6INIT=yes
+IPV6ADDR=$ipv6
${gw:+GATEWAY=}${gw}
EOF
register_hook setup_base_hooks setup_network
+setup_base_cluster_setup_config ()
+{
+ local f
+ {
+ echo "# Generated by autocluster"
+ echo
+ # This is a bit of a hack. Perhaps these script belong
+ # elsewhere, since they no longer have templates?
+ for f in $(find "${BASE_TEMPLATES}/all/root/scripts" -type f |
+ xargs grep -l '^#config:') ; do
+
+ b=$(basename "$f")
+ echo "# $b"
+ local vs v
+ vs=$(sed -n 's@^#config: *@@p' "$f")
+ for v in $vs ; do
+ # This could substitute the values in directly using
+ # ${!v} but then no sanity checking is done to make
+ # sure variables are set.
+ echo "${v}=\"@@${v}@@\""
+ done
+ echo
+ done
+ } | diskimage substitute_vars - "/root/scripts/cluster_setup.config"
+}
+
+register_hook setup_base_hooks setup_base_cluster_setup_config
+
setup_timezone() {
[ -z "$TIMEZONE" ] && {
[ -r /etc/timezone ] && {
actions_init
for t in "$@" ; do
case "$t" in
- create) actions_add "cluster_${t}" ;;
+ destroy|undefine|create|update_hosts|boot|setup)
+ actions_add "cluster_${t}" ;;
+ build)
+ for t in destroy undefine create update_hosts boot setup ; do
+ actions_add "cluster_${t}"
+ done
+ ;;
*) usage ;;
esac
done
+ cluster_nodelist_hacking
actions_run
;;
git.samba.org / autocluster.git / blobdiff