-#!/bin/bash
+#!/bin/sh
# Automatically setup GPFS. This is a quick way to get setup with an
# autocluster system. It finds NSDs, does various pieces of GPFS
set -e
-####################
-
-# $nsd_servers is setup in here
-. $(dirname $0)/nsd_server_common.bash "$@"
-
-####################
-
-usage ()
-{
- cat <<EOF
-Usage: $0 [ OPTION ]
- options:
-
- --gpfs-num-nsds=<num> Specify number of NSDs to use for GPFS
- filesystem. Defaults to value of
- autocluster variable GPFS_DEFAULT_NSDS.
- If this is unset then all NSDs found
- are used.
-
- --cluster-name=<string> Specify prefix for GPFS cluster name. The
- domain will be appended to this.
- Defaults to value of autocluster
- variable CLUSTER.
-
- --mountpoint=<dir> The directory where the GPFS filesystem
- should be mounted.
-
-EOF
- exit 1
-}
-
gpfs_num_nsds="@@GPFS_DEFAULT_NSDS@@"
cluster_name="@@CLUSTER@@"
mountpoint="@@CLUSTERFS_DEFAULT_MOUNTPOINT@@"
-
-opts="gpfs-num-nsds:,cluster-name:,mountpoint:"
-getopt_output=$(getopt -n autocluster -o h -l "$opts" -- "$@")
-[ $? != 0 ] && usage
-
-eval set -- "$getopt_output"
-while true ; do
- case "$1" in
- --gpfs-num-nsds) gpfs_num_nsds="$2" ; shift 2 ;;
- --cluster-name) cluster_name="$2" ; shift 2 ;;
- --mountpoint) mountpoint="$2" ; shift 2 ;;
- --) shift ; break ;;
- -h) usage ;;
- *) echo "Unknown option \"$1\"" ; usage ;;
- esac
-done
-
-[ -z "$1" ] || usage
-
-####################
-
-domain=$(dnsdomainname)
-nodes=$(onnode -q all hostname | grep -i $domain | tr 'A-Z\012' 'a-z\040')
-first="${nodes%% *}"
-primary="$first"
-if [ "$first" != "$nodes" ] ; then
- # More than 1 node...
- rest="${nodes#* }"
- secondary="${rest%% *}"
-else
- secondary=""
+nsd_servers="@@NSD_SERVERS@@"
+
+dir=$(dirname "$0")
+
+##################################################
+
+# If there are sofs_storage nodes in the cluster (meaning that
+# other sofs_* nodes will not have direct-attached storage) then
+# scripts that include this snippet must be run on one of the
+# storage nodes. Therefore, in the case, this snippet tries to
+# determine if it is running on the 1st NSD server and, if not,
+# attempts to run the script there.
+if [ -n "$nsd_servers" -a \
+ "${HOSTNAME%%.*}" != "${nsd_servers%%[.,]*}" ] ; then
+ if [ "${0#/}" != "$0" ] ; then
+ script="$0"
+ else
+ script="${PWD}/${0}"
+ fi
+ re_exec_node="${nsd_servers%%[.,]*}"
+ echo
+ echo "Creating NSDs on node \"${re_exec_node}\""
+ exec ssh "$re_exec_node" "$script" "$@"
fi
-if [ -n "$nsd_servers" ] ; then
- nodes="${nodes} ${nsd_servers//,/ }"
+##################################################
- # Determine secondary GPFS server. Is there a 2nd NSD server? If
- # not then use node 0 of the CTDB cluster.
- secondary="${nsd_servers#*,}"
- secondary="${secondary%%,*}"
- [ -n "$secondary" -a "$secondary" != "${nsd_servers%%,*}" ] || \
- secondary="$primary"
+# Uses: cluster_name
+gpfs_setup ()
+{
+ _domain=$(dnsdomainname)
+ _nodes=$(onnode -q all hostname | grep -i "$_domain" | tr 'A-Z\012' 'a-z\040')
- # Primary is just the 1st NSD server.
- primary="${nsd_servers%%,*}"
-fi
+ _first="${_nodes%% *}"
-nodefile=/tmp/nodes.$$
-for n in $nodes; do
- designation="manager"
+ # Determine primary and secondary nodes. Give preference to NSD
+ # servers, falling back to regular nodes if there aren't any or
+ # aren't enough.
if [ -n "$nsd_servers" ] ; then
- # This designates the $nsd_servers and $first as quorum nodes.
- foo="${nsd_servers},"
- if [ "$n" = "$first" -o "$foo" != "${foo/${n},/}" ] ; then
- designation="${designation}-quorum"
- fi
- else
- designation="${designation}-quorum"
+ _primary="${nsd_servers%%,*}"
+ _rest="${nsd_servers#*,}"
+ _secondary="${_rest%%,*}"
+ fi
+ if [ -z "$_primary" ] ; then
+ _primary="$_first"
+ _rest="${_nodes#* }"
+ _secondary="${_rest%% *}"
+ elif [ -z "$_secondary" ] ; then
+ _secondary="$_first"
fi
- echo "$n:${designation}:"
-done > $nodefile
-echo "Creating cluster"
-mmcrcluster -N $nodefile -p $primary ${secondary:+-s} $secondary -r /usr/bin/ssh -R /usr/bin/scp -C $cluster_name.$domain
+ # Create the node description file for mmcrcluster. If there are
+ # dedicated storage nodes then they are quorum nodes, along with
+ # the first node. If there are no dedicated storage nodes then
+ # all nodes are quorum nodes.
+ _nodefile="${dir}/gpfs_nodes.${cluster_name}"
+ {
+ for _n in $_nodes ; do
+ if [ "$_n" = "$_first" ] ; then
+ echo "${_n}:manager-quorum:"
+ elif [ -n "$nsd_servers" ] ; then
+ echo "${_n}:manager:"
+ else
+ echo "${_n}:manager-quorum:"
+ fi
+ done
+ for _n in $(echo "$nsd_servers" | sed -e 's@,@ @g') ; do
+ echo "${_n}:manager-quorum:"
+ done
+ } >"$_nodefile"
+
+ echo "Creating cluster"
+ # Don't quote secondary, since it might not exist
+ mmcrcluster -N "$_nodefile" \
+ -p "$_primary" ${_secondary:+-s} $_secondary \
+ -r /usr/bin/ssh -R /usr/bin/scp -C "${cluster_name}.${_domain}"
+
+ # GPFS >= 3.3 needs this. Earlier versions don't have
+ # mmchlicense, so be careful.
+ if type mmchlicense >/dev/null 2>&1 ; then
+ echo
+ echo "Attempting to set server license mode for all nodes"
+ mmchlicense server --accept -N all
+ fi
-# GPFS 3.3 needs this... and GPFS 3.2 doesn't have mmchlicense, so
-# this seems like a good option for now.
-if type mmchlicense >/dev/null 2>&1 ; then
- mmchlicense server --accept -N all
-fi
+ echo
+ echo "Attempting to set adminMode=allToAll"
+ mmchconfig adminMode=allToAll </dev/null || true
+
+ echo
+ echo "Generating auth key"
+ mmauth genkey new
+
+ echo
+ echo "Setting GPFS config options"
+ mmchconfig autoload=yes,leaseRecoveryWait=3,maxFilesToCache=20000,failureDetectionTime=10,maxMBpS=500,unmountOnDiskFail=yes,pagepool=64M,allowSambaCaseInsensitiveLookup=no,cipherList=AUTHONLY
+
+ echo "Starting gpfs"
+ mmstartup -a
+
+ echo "Waiting for gpfs to become active"
+ _count=0
+ while mmgetstate -a | tail -n +4 | grep -v " active" > /dev/null; do
+ echo -n "."
+ _count=$(($_count + 1))
+ if [ $_count -gt 60 ] ; then
+ echo "TIMEOUT: gpfs didn't become active"
+ exit 1
+ fi
+ sleep 1
+ done
+ echo
+}
+
+nsdfile="${dir}/gpfs_nsds_all.${cluster_name}"
+
+# Uses: nsd_servers
+# Sets: nsdfile
+gpfs_mknsd ()
+{
+ echo
+ echo "Setting up NSDs"
-echo
-echo "Attempting to set adminMode=allToAll"
-mmchconfig adminMode=allToAll </dev/null || true
+ _idfile="${dir}/shared_disk_ids.${cluster_name}"
+ if [ ! -r "$_idfile" ] ; then
+ echo "ERROR: missing disk ID file \"${_idfile}\""
+ exit 1
+ fi
-. $(dirname $0)/mknsd.sh
+ # Create an extended regexp that matches any of the IDs
+ pat=$(tr '\n' '|' <"$_idfile" | sed -e 's@|$@@')
-echo
-echo "Generating auth key"
-mmauth genkey new
+ # Now get devices and names from multipath
+ multipath -dl |
+ sed -r -n -e "s@^[^[:space:]]+[[:space:]]+\(($pat)\)[[:space:]](dm-[^[:space:]]+).*@\1 \2@p" |
+ while read _name _disk ; do
+ _name=$(echo "$_name" | tr -d -c '[:alnum:]')
+ echo "${_disk}:${nsd_servers}::dataAndMetadata:1:${_name}:"
+ done >"$nsdfile"
-echo
-echo "Setting GPFS config options"
-mmchconfig autoload=yes,leaseRecoveryWait=3,maxFilesToCache=20000,failureDetectionTime=10,maxMBpS=500,unmountOnDiskFail=yes,pagepool=64M,allowSambaCaseInsensitiveLookup=no,cipherList=AUTHONLY
+ mmcrnsd -F "$nsdfile"
+ mmlsnsd -m
+}
-echo "Starting gpfs"
-mmstartup -a
+# Uses: mountpoint, gpfs_num_nsds, nsdfile
+gpfs_mkfs ()
+{
+ echo
+ echo "Creating filesystem"
-echo "Waiting for gpfs to become active"
-count=0
-while mmgetstate -a | tail -n +4 | grep -v " active" > /dev/null; do
- echo -n "."
- count=$(($count + 1))
- if [ $count -gt 60 ] ; then
- echo "TIMEOUT: gpfs didn't become active"
+ if [ ! -r "$nsdfile" ] ; then
+ echo "ERROR: missing NSD file \"${nsdfile}\""
exit 1
fi
- sleep 1
-done
-echo
-echo
-echo "Creating filesystem"
-mkdir -p "${mountpoint}/automountdir"
+ mkdir -p "${mountpoint}/automountdir"
-nsdfile2=/tmp/nsd2.$$
-if [ -n "$gpfs_num_nsds" ] ; then
- head -n $(($gpfs_num_nsds * 2))
-else
- cat
-fi <$nsdfile >$nsdfile2
+ nsdfile2="${dir}/gpfs_nsds_defaultfs.${cluster_name}"
+ if [ -n "$gpfs_num_nsds" ] ; then
+ head -n $(($gpfs_num_nsds * 2))
+ else
+ cat
+ fi <"$nsdfile" >"$nsdfile2"
+
+ chattr +i "$mountpoint"
+
+ mmcrfs gpfs0 -F "$nsdfile2" \
+ -A yes -Q yes -D nfs4 -B 64k -k nfs4 -n 32 -E yes -S no \
+ -T "$mountpoint" -i 512
+
+ rm -f "$nsdfile2"
+}
+
+gpfs_mount ()
+{
+ echo
+ echo "Mounting filesystem"
+ mmmount gpfs0 -a
-chattr +i "$mountpoint"
-mmcrfs gpfs0 -F $nsdfile2 -A yes -Q yes -D nfs4 -B 64k -k nfs4 -n 32 -E yes -S no -T "$mountpoint" -i 512
+ echo "Waiting for gpfs to mount"
+ while ! mount | grep /dev/gpfs0 > /dev/null; do
+ echo -n "."
+ sleep 1
+ done
-echo
-echo "Mounting filesystem"
-mmmount gpfs0 -a
+ echo
+}
-echo "Waiting for gpfs to mount"
-while ! mount | grep /dev/gpfs0 > /dev/null; do echo -n "."; sleep 1; done
-echo
+gpfs_complete ()
+{
+ echo "GPFS setup complete"
+}
-echo "GPFS setup complete"
+[ -n "$1" ] || set -- setup mknsd mkfs mount complete
-# LocalWords: DEF
+for action ; do
+ gpfs_$action
+done
git.samba.org / martins / autocluster.git / commitdiff