#!/bin/bash
# main autocluster script
#
# Copyright (C) Andrew Tridgell 2008
# Copyright (C) Martin Schwenke 2008
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see .
##BEGIN-INSTALLDIR-MAGIC##
# There are better ways of doing this but not if you still want to be
# able to run straight out of a git tree. :-)
if [ -f "$0" ]; then
autocluster="$0"
else
autocluster=$(which "$0")
fi
if [ -L "$autocluster" ] ; then
autocluster=$(readlink "$autocluster")
fi
installdir=$(dirname "$autocluster")
##END-INSTALLDIR-MAGIC##
####################
# show program usage
usage ()
{
cat <
options:
-c specify config file (default is "config")
EOF
releases=$(list_releases)
usage_smart_display \
defconf "WITH_RELEASE" "" \
"" "specify preset options for a release using a version string. Possible values are: ${releases}."
cat < execute and exit
-E execute and continue
-x enable script debugging
--dump dump config settings and exit
configuration options:
EOF
usage_config_options
cat <&2
exit 1
}
###############################
# Indirectly call a function named by ${1}_${2}
call_func () {
local func="$1" ; shift
local type="$1" ; shift
local f="${func}_${type}"
if type -t "$f" >/dev/null && ! type -P "$f" >/dev/null ; then
"$f" "$@"
else
f="${func}_DEFAULT"
if type -t "$f" >/dev/null && ! type -P "$f" >/dev/null ; then
"$f" "$type" "$@"
else
die "No function defined for \"${func}\" \"${type}\""
fi
fi
}
# Note that this will work if you pass "call_func f" because the first
# element of the node tuple is the node type. Nice... :-)
for_each_node ()
{
local n
for n in $NODES ; do
"$@" $(IFS=: ; echo $n)
done
}
hack_one_node_with ()
{
local filter="$1" ; shift
local node_type="$1"
local ip_offset="$2"
local name="$3"
local ctdb_node="$4"
$filter
local item="${node_type}:${ip_offset}${name:+:}${name}${ctdb_node:+:}${ctdb_node}"
nodes="${nodes}${nodes:+ }${item}"
}
# This also gets used for non-filtering iteration.
hack_all_nodes_with ()
{
local filter="$1"
local nodes=""
for_each_node hack_one_node_with "$filter"
NODES="$nodes"
}
register_hook ()
{
local hook_var="$1"
local new_hook="$2"
eval "$hook_var=\"${!hook_var}${!hook_var:+ }${new_hook}\""
}
run_hooks ()
{
local hook_var="$1"
local i
for i in ${!hook_var} ; do
$i
done
}
# Use with care, since this may clear some autocluster defaults.!
clear_hooks ()
{
local hook_var="$1"
eval "$hook_var=\"\""
}
##############################
# common node creation stuff
create_node_COMMON ()
{
local NAME="$1"
local ip_offset="$2"
local type="$3"
local template_file="${4:-$NODE_TEMPLATE}"
if [ "$SYSTEM_DISK_FORMAT" != "qcow2" -a "$BASE_FORMAT" = "qcow2" ] ; then
die "Error: if BASE_FORMAT is \"qcow2\" then SYSTEM_DISK_FORMAT must also be \"qcow2\"."
fi
IPNUM=$(($FIRSTIP + $ip_offset))
DISK="${VIRTBASE}/${CLUSTER}/${NAME}.${SYSTEM_DISK_FORMAT}"
local base_disk="${VIRTBASE}/${BASENAME}.${BASE_FORMAT}"
if [ "$BASE_PER_NODE_TYPE" = "yes" ] ; then
base_disk="${VIRTBASE}/${BASENAME}-${type}.${BASE_FORMAT}"
fi
mkdir -p $VIRTBASE/$CLUSTER tmp
local di="$DISK"
if [ "$DISK_FOLLOW_SYMLINKS" = "yes" -a -L "$DISK" ] ; then
di=$(readlink "$DISK")
fi
rm -f "$di"
case "$SYSTEM_DISK_FORMAT" in
qcow2)
echo "Creating the disk..."
qemu-img create -b "$base_disk" -f qcow2 "$di"
create_node_configure_image "$DISK" "$type"
;;
raw)
echo "Creating the disk..."
cp -v --sparse=always "$base_disk" "$di"
create_node_configure_image "$DISK" "$type"
;;
reflink)
echo "Creating the disk..."
cp -v --reflink=always "$base_disk" "$di"
create_node_configure_image "$DISK" "$type"
;;
mmclone)
echo "Creating the disk (using mmclone)..."
local base_snap="${base_disk}.snap"
[ -f "$base_snap" ] || mmclone snap "$base_disk" "$base_snap"
mmclone copy "$base_snap" "$di"
create_node_configure_image "$DISK" "$type"
;;
none)
echo "Skipping disk image creation as requested"
;;
*)
die "Error: unknown SYSTEM_DISK_FORMAT=\"${SYSTEM_DISK_FORMAT}\"."
esac
set_macaddrs $CLUSTER $ip_offset
# Pull the UUID for this node out of the map.
UUID=$(awk "\$1 == $ip_offset {print \$2}" $uuid_map)
echo "Creating $NAME.xml"
substitute_vars $template_file tmp/$NAME.xml
# install the XML file
$VIRSH undefine $NAME > /dev/null 2>&1 || true
$VIRSH define tmp/$NAME.xml
}
create_node_configure_image ()
{
local disk="$1"
local type="$2"
diskimage mount "$disk"
setup_base "$type"
diskimage unmount
}
# Provides an easy way of removing nodes from $NODE.
create_node_null () {
:
}
##############################
hack_nodes_functions=
expand_nodes () {
# Expand out any abbreviations in NODES.
local ns=""
local n
for n in $NODES ; do
local t="${n%:*}"
local ips="${n#*:}"
case "$ips" in
*,*)
local i
for i in ${ips//,/ } ; do
ns="${ns}${ns:+ }${t}:${i}"
done
;;
*-*)
local i
for i in $(seq ${ips/-/ }) ; do
ns="${ns}${ns:+ }${t}:${i}"
done
;;
*)
ns="${ns}${ns:+ }${n}"
esac
done
NODES="$ns"
# Apply nodes hacks. Some of this is about backward compatibility
# but the hacks also fill in the node names and whether they're
# part of the CTDB cluster. The order is the order that
# configuration modules register their hacks.
run_hooks hack_nodes_functions
if [ -n "$NUMNODES" ] ; then
# Attempt to respect NUMNODES. Reduce the number of CTDB
# nodes to NUMNODES.
local numnodes=$NUMNODES
hack_filter ()
{
if [ "$ctdb_node" = 1 ] ; then
if [ $numnodes -gt 0 ] ; then
numnodes=$(($numnodes - 1))
else
node_type="null"
ctdb_node=0
fi
fi
}
hack_all_nodes_with hack_filter
[ $numnodes -gt 0 ] && \
die "Can't not use NUMNODES to increase the number of nodes over that specified by NODES. You need to set NODES instead - please read the documentation."
fi
# Check IP addresses for duplicates.
local ip_offsets=":"
# This function doesn't modify anything...
get_ip_offset ()
{
[ "${ip_offsets/${ip_offset}}" != "$ip_offsets" ] && \
die "Duplicate IP offset in NODES - ${node_type}:${ip_offset}"
ip_offsets="${ip_offsets}${ip_offset}:"
}
hack_all_nodes_with get_ip_offset
}
##############################
sanity_check_cluster_name ()
{
[ -z "${CLUSTER//[A-Za-z0-9]}" ] || \
die "Cluster names should be restricted to the characters A-Za-z0-9. \
Some cluster filesystems have problems with other characters."
}
hosts_file=
common_nodelist_hacking ()
{
# Rework the NODES list
expand_nodes
# Build /etc/hosts and hack the names of the ctdb nodes
hosts_line_hack_name ()
{
# Ignore nodes without names (e.g. "null")
[ "$node_type" != "null" -a -n "$name" ] || return 0
local sname=""
local hosts_line
local ip_addr="$IPBASE.$IPNET0.$(($FIRSTIP + $ip_offset))"
if [ "$ctdb_node" = 1 ] ; then
num_ctdb_nodes=$(($num_ctdb_nodes + 1))
sname="${CLUSTER}n${num_ctdb_nodes}"
hosts_line="$ip_addr ${sname}.${ld} ${name}.${ld} $name $sname"
name="$sname"
else
hosts_line="$ip_addr ${name}.${ld} $name"
fi
# This allows you to add a function to your configuration file
# to modify hostnames (and other aspects of nodes). This
# function can access/modify $name (the existing name),
# $node_type and $ctdb_node (1, if the node is a member of the
# CTDB cluster, 0 otherwise).
if [ -n "$HOSTNAME_HACKING_FUNCTION" ] ; then
local old_name="$name"
$HOSTNAME_HACKING_FUNCTION
if [ "$name" != "$old_name" ] ; then
hosts_line="$ip_addr ${name}.${ld} $name"
fi
fi
echo "$hosts_line"
}
hosts_file="tmp/hosts.$CLUSTER"
{
local num_ctdb_nodes=0
local ld=$(echo $DOMAIN | tr A-Z a-z)
echo "# autocluster $CLUSTER"
hack_all_nodes_with hosts_line_hack_name
echo
} >$hosts_file
# Build /etc/ctdb/nodes
ctdb_nodes_line ()
{
[ "$ctdb_node" = 1 ] || return 0
echo "$IPBASE.$IPNET0.$(($FIRSTIP + $ip_offset))"
num_nodes=$(($num_nodes + 1))
}
nodes_file="tmp/nodes.$CLUSTER"
local num_nodes=0
hack_all_nodes_with ctdb_nodes_line >$nodes_file
: "${NUMNODES:=${num_nodes}}" # Set $NUMNODES if necessary
# Build UUID map
uuid_map="tmp/uuid_map.$CLUSTER"
uuid_map_line ()
{
echo "${ip_offset} $(uuidgen) ${node_type}"
}
hack_all_nodes_with uuid_map_line >$uuid_map
}
create_cluster_hooks=
cluster_created_hooks=
create_cluster ()
{
CLUSTER="$1"
sanity_check_cluster_name
mkdir -p $VIRTBASE/$CLUSTER $KVMLOG tmp
# Run hooks before doing anything else.
run_hooks create_cluster_hooks
common_nodelist_hacking
for_each_node call_func create_node
echo "Cluster $CLUSTER created"
echo ""
run_hooks cluster_created_hooks
}
cluster_created_hosts_message ()
{
echo "You may want to add this to your /etc/hosts file:"
cat $hosts_file
}
register_hook cluster_created_hooks cluster_created_hosts_message
create_one_node ()
{
CLUSTER="$1"
local single_node_ip_offset="$2"
sanity_check_cluster_name
mkdir -p $VIRTBASE/$CLUSTER $KVMLOG tmp
common_nodelist_hacking
for n in $NODES ; do
set -- $(IFS=: ; echo $n)
[ $single_node_ip_offset -eq $2 ] || continue
call_func create_node "$@"
echo "Requested node created"
echo ""
echo "You may want to update your /etc/hosts file:"
cat $hosts_file
break
done
}
###############################
# test the proxy setup
test_proxy() {
export http_proxy=$WEBPROXY
wget -O /dev/null $INSTALL_SERVER || \
die "Your WEBPROXY setting \"$WEBPROXY\" is not working"
echo "Proxy OK"
}
###################
kickstart_floppy_create_hooks=
# create base image
create_base() {
NAME="$BASENAME"
DISK="${VIRTBASE}/${NAME}.${BASE_FORMAT}"
mkdir -p $KVMLOG
echo "Testing WEBPROXY $WEBPROXY"
test_proxy
local di="$DISK"
if [ "$DISK_FOLLOW_SYMLINKS" = "yes" -a -L "$DISK" ] ; then
di=$(readlink "$DISK")
fi
echo "Creating the disk"
qemu-img create -f $BASE_FORMAT "$di" $DISKSIZE
rm -rf tmp
mkdir -p mnt tmp tmp/ISO
setup_timezone
echo "Creating kickstart file from template"
substitute_vars "$KICKSTART" "tmp/ks.cfg"
if [ $INSTALLKEY = "--skip" ]; then
cat <"$tmp_out"
# Handle any indirects by looping until nothing changes.
# However, only handle 10 levels of recursion.
count=0
while : ; do
if ! _substitute_vars "$tmp_out" "@@@" ; then
rm -f "$tmp_out"
die "Failed to expand template $infile"
fi
# No old version of file means no changes made.
if [ ! -f "${tmp_out}.old" ] ; then
break
fi
rm -f "${tmp_out}.old"
count=$(($count + 1))
if [ $count -ge 10 ] ; then
rm -f "$tmp_out"
die "Recursion too deep in $infile - only 10 levels allowed!"
fi
done
# Now regular variables.
if ! _substitute_vars "$tmp_out" "@@" ; then
rm -f "$tmp_out"
die "Failed to expand template $infile"
fi
rm -f "${tmp_out}.old"
if [ -n "$outfile" ] ; then
mv "$tmp_out" "$outfile"
else
cat "$tmp_out"
rm -f "$tmp_out"
fi
)}
# Delimiter @@ means to substitute contents of variable.
# Delimiter @@@ means to substitute contents of file named by variable.
# @@@ supports leading '|' in variable value, which means to excute a
# command.
_substitute_vars() {(
tmp_out="$1"
delimiter="${2:-@@}"
# Get the list of variables used in the template. The grep
# gets rid of any blank lines and lines with extraneous '@'s
# next to template substitutions.
VARS=$(sed -n -e "s#[^@]*${delimiter}\([A-Z0-9_][A-Z0-9_]*\)${delimiter}[^@]*#\1\n#gp" "$tmp_out" |
grep '^[A-Z0-9_][A-Z0-9_]*$' |
sort -u)
tmp=$(mktemp)
for v in $VARS; do
# variable variables are fun .....
[ "${!v+x}" ] || {
rm -f $tmp
die "No substitution given for ${delimiter}$v${delimiter} in $infile"
}
s=${!v}
if [ "$delimiter" = "@@@" ] ; then
f=${s:-/dev/null}
c="${f#|}" # Is is a command, signified by a leading '|'?
if [ "$c" = "$f" ] ; then
# No leading '|', cat file.
s=$(cat -- "$f")
[ $? -eq 0 ] || {
rm -f $tmp
die "Could not substitute contents of file $f"
}
else
# Leading '|', execute command.
# Quoting problems here - using eval "$c" doesn't help.
s=$($c)
[ $? -eq 0 ] || {
rm -f $tmp
die "Could not execute command $c"
}
fi
fi
# escape some pesky chars
# This first one can be too slow if done using a bash
# variable pattern subsitution.
s=$(echo -n "$s" | tr '\n' '\001' | sed -e 's/\o001/\\n/g')
s=${s//#/\\#}
s=${s//&/\\&}
echo "s#${delimiter}${v}${delimiter}#${s}#g"
done > $tmp
# Get the in-place sed to make a backup of the old file.
# Remove the backup if it is the same as the resulting file -
# this acts as a flag to the caller that no changes were made.
sed -i.old -f $tmp "$tmp_out"
if cmp -s "${tmp_out}.old" "$tmp_out" ; then
rm -f "${tmp_out}.old"
fi
rm -f $tmp
)}
check_command() {
which $1 > /dev/null || die "Please install $1 to continue"
}
# Set a variable if it isn't already set. This allows environment
# variables to override default config settings.
defconf() {
local v="$1"
local e="$2"
[ "${!v+x}" ] || eval "$v=\"$e\""
}
load_config () {
local i
for i in "${installdir}/config.d/"*.defconf ; do
. "$i"
done
}
# Print the list of config variables defined in config.d/.
get_config_options () {( # sub-shell for local declaration of defconf()
local options=
defconf() { options="$options $1" ; }
load_config
echo $options
)}
# Produce a list of long options, suitable for use with getopt, that
# represent the config variables defined in config.d/.
getopt_config_options () {
local x=$(get_config_options | tr 'A-Z_' 'a-z-')
echo "${x// /:,}:"
}
# Unconditionally set the config variable associated with the given
# long option.
setconf_longopt () {
local longopt="$1"
local e="$2"
local v=$(echo "${longopt#--}" | tr 'a-z-' 'A-Z_')
# unset so defconf will set it
eval "unset $v"
defconf "$v" "$e"
}
# Dump all of the current config variables.
dump_config() {
local o
for o in $(get_config_options) ; do
echo "${o}=\"${!o}\""
done
exit 0
}
# $COLUMNS is set in interactive bash shells. It probably isn't set
# in this shell, so let's set it if it isn't.
: ${COLUMNS:=$(stty size 2>/dev/null | sed -e 's@.* @@')}
: ${COLUMNS:=80}
export COLUMNS
# Print text assuming it starts after other text in $startcol and
# needs to wrap before $COLUMNS - 2. Subsequent lines start at $startcol.
# Long "words" will extend past $COLUMNS - 2.
fill_text() {
local startcol="$1"
local text="$2"
local width=$(($COLUMNS - 2 - $startcol))
[ $width -lt 0 ] && width=$((78 - $startcol))
local out=""
local padding
if [ $startcol -gt 0 ] ; then
padding=$(printf "\n%${startcol}s" " ")
else
padding="
"
fi
while [ -n "$text" ] ; do
local orig="$text"
# If we already have output then arrange padding on the next line.
[ -n "$out" ] && out="${out}${padding}"
# Break the text at $width.
out="${out}${text:0:${width}}"
text="${text:${width}}"
# If we have left over text then the line break may be ugly,
# so let's check and try to break it on a space.
if [ -n "$text" ] ; then
# The 'x's stop us producing a special character like '(',
# ')' or '!'. Yuck - there must be a better way.
if [ "x${text:0:1}" != "x " -a "x${text: -1:1}" != "x " ] ; then
# We didn't break on a space. Arrange for the
# beginning of the broken "word" to appear on the next
# line but not if it will make us loop infinitely.
if [ "${orig}" != "${out##* }${text}" ] ; then
text="${out##* }${text}"
out="${out% *}"
else
# Hmmm, doing that would make us loop, so add the
# rest of the word from the remainder of the text
# to this line and let it extend past $COLUMNS - 2.
out="${out}${text%% *}"
if [ "${text# *}" != "$text" ] ; then
# Remember the text after the next space for next time.
text="${text# *}"
else
# No text after next space.
text=""
fi
fi
else
# We broke on a space. If it will be at the beginning
# of the next line then remove it.
text="${text# }"
fi
fi
done
echo "$out"
}
# Display usage text, trying these approaches in order.
# 1. See if it all fits on one line before $COLUMNS - 2.
# 2. See if splitting before the default value and indenting it
# to $startcol means that nothing passes $COLUMNS - 2.
# 3. Treat the message and default value as a string and just us fill_text()
# to format it.
usage_display_text () {
local startcol="$1"
local desc="$2"
local default="$3"
local width=$(($COLUMNS - 2 - $startcol))
[ $width -lt 0 ] && width=$((78 - $startcol))
default="(default \"$default\")"
if [ $((${#desc} + 1 + ${#default})) -le $width ] ; then
echo "${desc} ${default}"
else
local padding=$(printf "%${startcol}s" " ")
if [ ${#desc} -lt $width -a ${#default} -lt $width ] ; then
echo "$desc"
echo "${padding}${default}"
else
fill_text $startcol "${desc} ${default}"
fi
fi
}
# Display usage information for long config options.
usage_smart_display () {( # sub-shell for local declaration of defconf()
local startcol=33
defconf() {
local local longopt=$(echo "$1" | tr 'A-Z_' 'a-z-')
printf " --%-25s " "${longopt}=${3}"
usage_display_text $startcol "$4" "$2"
}
"$@"
)}
# Display usage information for long config options.
usage_config_options (){
usage_smart_display load_config
}
list_releases () {
local releases=$(cd $installdir/releases && echo *.release)
releases="${releases//.release}"
releases="${releases// /\", \"}"
echo "\"$releases\""
}
with_release () {
local release="$1"
shift # subsequent args are passed to release file
# This simply loads an extra config file from $installdir/releases
f="${installdir}/releases/${release}.release"
if [ -r "$f" ] ; then
. "$f"
else
f="${installdir}/releases/${release%%-*}.release"
if [ -r "$f" ] ; then
. "$f" "${release#*-}"
else
echo "Unknown release \"${release}\" specified to --with-release"
printf "%-25s" "Supported releases are: "
fill_text 25 "$(list_releases)"
exit 1
fi
fi
}
has_public_addresses_DEFAULT ()
{
false
}
make_public_addresses() {
local firstip="${1:-$[${FIRSTIP} + ${PUBLIC_IP_OFFSET}]}"
local num_addrs="${2:-${NUMNODES}}"
if [ $(( $firstip + $num_addrs - 1 )) -gt 254 ]; then
die "make_public_addresses: last octet > 254 - change PUBLIC_IP_OFFSET"
fi
local e
for e in $IPNET1 $IPNET2 ; do
echo -ne "${IPBASE}.${e}.${firstip},${num_addrs},eth${e} "
done
echo
}
######################################################################
post_config_hooks=
######################################################################
load_config
############################
# parse command line options
long_opts=$(getopt_config_options)
getopt_output=$(getopt -n autocluster -o "c:e:E:xh" -l help,dump,with-release: -l "$long_opts" -- "$@")
[ $? != 0 ] && usage
use_default_config=true
# We do 2 passes of the options. The first time we just handle usage
# and check whether -c is being used.
eval set -- "$getopt_output"
while true ; do
case "$1" in
-c) shift 2 ; use_default_config=false ;;
-e) shift 2 ;;
-E) shift 2 ;;
--) shift ; break ;;
--with-release) shift 2 ;; # Don't set use_default_config=false!!!
--dump|-x) shift ;;
-h|--help) usage ;; # Usage should be shown here for real defaults.
--*) shift 2 ;; # Assume other long opts are valid and take an arg.
*) usage ;; # shouldn't happen, so this is reasonable.
esac
done
config="./config"
$use_default_config && [ -r "$config" ] && . "$config"
eval set -- "$getopt_output"
while true ; do
case "$1" in
# force at least ./local_file to avoid accidental file from $PATH
-c) . "$(dirname $2)/$(basename $2)" ; shift 2 ;;
-e) run_hooks post_config_hooks ; eval "$2" ; exit ;;
-E) eval "$2" ; shift 2 ;;
--with-release)
with_release "$2"
shift 2
;;
-x) set -x; shift ;;
--dump) run_hooks post_config_hooks ; dump_config ;;
--) shift ; break ;;
-h|--help) usage ;; # Redundant.
--*)
# Putting --opt1|opt2|... into a variable and having case
# match against it as a pattern doesn't work. The | is
# part of shell syntax, so we need to do this. Look away
# now to stop your eyes from bleeding! :-)
x=",${long_opts}" # Now each option is surrounded by , and :
if [ "$x" != "${x#*,${1#--}:}" ] ; then
# Our option, $1, surrounded by , and : was in $x, so is legal.
setconf_longopt "$1" "$2"; shift 2
else
usage
fi
;;
*) usage ;; # shouldn't happen, so this is reasonable.
esac
done
run_hooks post_config_hooks
# catch errors
set -e
set -E
trap 'es=$?;
echo ERROR: failed in function \"${FUNCNAME}\" at line ${LINENO} of ${BASH_SOURCE[0]} with code $es;
exit $es' ERR
# check for needed programs
check_command expect
[ $# -lt 1 ] && usage
command="$1"
shift
case $command in
create)
type=$1
shift
case $type in
base)
[ $# != 0 ] && usage
create_base
;;
cluster)
[ $# != 1 ] && usage
create_cluster "$1"
;;
node)
[ $# != 2 ] && usage
create_one_node "$1" "$2"
;;
*)
usage;
;;
esac
;;
mount)
[ $# != 1 ] && usage
diskimage mount "$1"
;;
unmount|umount)
[ $# != 0 ] && usage
diskimage unmount
;;
bootbase)
boot_base;
;;
testproxy)
test_proxy;
;;
*)
usage;
;;
esac