:provider => {
:libvirt => {
:prefix => 'vagrant',
+ :box => 'fedora/23-cloud-base',
+ },
+ :virtualbox => {
+ :prefix => 'vagrant',
+ :box => 'fedora/23-cloud-base',
},
},
}
#:container_name => 'node1',
},
:libvirt => {
- :box => 'local-fedora-21.2',
:prefix => 'gluster',
},
},
},
]
+
#
# Load the config, if it exists,
# possibly override with commandline args,
#print cluster_internal_ips
#print "\n"
-#PROVISION_SCRIPT = <<SCRIPT
-#yum -y install make samba
-#SCRIPT
-
-
-SELINUX_SCRIPT = <<SCRIPT
-set -e
-setenforce permissive
-
-BACKUP_SUFFIX=".orig.$(date +%Y%m%d-%H%M%S)"
-
-FILE=/etc/selinux/config
-test -f ${FILE} && {
- sed -i${BACKUP_SUFFIX} -e 's/^SELINUX=.*$/SELINUX=disabled/g' ${FILE}
-} || {
- cat <<EOF > ${FILE}
-SELINUX=disabled
-EOF
-}
- touch ${FILE}
-SCRIPT
NET_FIX_ALWAYS_SCRIPT = <<SCRIPT
set -e
# eth1 is not brought up automatically
-# by 'vagrant up' of the existing vm
+# by 'vagrant up' of the existing vm.
# because eth1 is not up, glusterd can
# not be started and gluster volumes can
-# not be mountd. fix it all up here until
-# we have a correctly working environment
+# not be mounted. fix it all up here until
+# we have a correctly working environment.
ifdown eth1
ifup eth1
echo "Mounting ${MOUNTPT}."
mount ${MOUNTPT}
}
+
+ systemctl start ctdb
} || {
# not provisioned yet
echo "${MOUNTPT} not set up yet. Not mounting."
ifup eth1
SCRIPT
-INSTALL_SCRIPT = <<SCRIPT
-set -e
-
-echo "Installing software..."
-
-yum -y install xfsprogs
-yum -y install glusterfs{,-server,-fuse,-geo-replication}
-yum -y install ctdb samba{,-client,-vfs-glusterfs}
-SCRIPT
-
-XFS_SCRIPT = <<SCRIPT
-set -e
-
-DEVICE=$1
-PARTDEV=${DEVICE}1
-DISKDEV="/dev/${DEVICE}"
-DISKPARTDEV="/dev/${PARTDEV}"
-##MOUNTP=$2
-MOUNTP=/export/${PARTDEV}
-BRICKD=${MOUNTP}/brick
-
-BACKUP_SUFFIX=".orig.$(date +%Y%m%d-%H%M%S)"
-
-parted -s ${DISKDEV} print > /dev/null 2>&1 && {
- echo "Label exists on ${DISKDEV}."
-} || {
- echo "Creating label on ${DISKDEV}."
- parted -s ${DISKDEV} mklabel msdos
-}
-
-parted -s ${DISKDEV} print 1 > /dev/null 2>&1 && {
- echo "Partition ${DISKPARTDEV} exists."
-} || {
- echo "Creating partition ${DISKPARTDEV}."
- parted -s ${DISKDEV} mkpart primary 1 100%
-}
-
-blkid -s TYPE ${DISKPARTDEV} | grep -q -s 'TYPE="xfs"' && {
- echo "Partition ${DISKPARTDEV} contains xfs file system."
-} || {
- echo "Creating xfs filesystem on ${DISKPARTDEV}."
- mkfs.xfs -f ${DISKPARTDEV}
-}
-
-mkdir -p ${MOUNTP}
-
-FILE=/etc/fstab
-
-grep -q -s ${DISKPARTDEV} ${FILE} && {
- echo "Mount entry for ${DISKPARTDEV} is present in ${FILE}."
-} || {
- echo "Creating mount entry for ${DISKPARTDEV} in ${FILE}."
- test -f ${FILE} || touch ${FILE}
- cp -f -a ${FILE} ${FILE}${BACKUP_SUFFIX}
- cat <<EOF >> ${FILE}
-${DISKPARTDEV} ${MOUNTP} xfs defaults 0 0
-EOF
-}
-
-mount | grep ${MOUNTP} && {
- echo "${MOUNTP} is already mounted."
-} || {
- echo "Mounting ${MOUNTP}."
- mount ${MOUNTP}
-}
-
-mkdir -p ${BRICKD}
-SCRIPT
-
-GLUSTER_START_SCRIPT = <<SCRIPT
-set -e
-systemctl start glusterd.service
-SCRIPT
-
-#GLUSTER_PROBE_SCRIPT = <<SCRIPT
-#set -e
-#
-#PEER_IP=$1
-#
-#gluster peer probe ${PEER_IP}
-#SCRIPT
-
-GLUSTER_PROBE_SCRIPT = <<SCRIPT
-set -e
-
-PEER_IPS="$@"
-
-for PEER_IP in ${PEER_IPS}
-do
- # try for some time to reach the other node:
- for COUNT in $(seq 1 20)
- do
- gluster peer probe ${PEER_IP} 2> /dev/null && {
- break
- } || {
- sleep 1
- }
- done
-done
-SCRIPT
-
-GLUSTER_WAIT_PEERS_SCRIPT = <<SCRIPT
-set -e
-
-NUM_NODES="$1"
-
-echo "Waiting for $NUM_NODES peers."
-
-for count in $(seq 1 60)
-do
- PEERS=$(gluster pool list | grep -v ^UUID | wc -l)
- [ "$PEERS" = "$NUM_NODES" ] && {
- echo "Done waiting: $NUM_NODES peers connected."
- exit 0
- } || {
- sleep 1
- }
-done
-
-echo "TIMEOUT waiting for $NUM_NODES peers."
-exit 1
-
-SCRIPT
-
-GLUSTER_CREATEVOL_SCRIPT = <<SCRIPT
-#set -e
-
-VOLNAME=$1
-shift
-REP=$1
-shift
-
-while true; do
- MSG="$(gluster volume status ${VOLNAME} 2>&1 1>/dev/null)"
- RET=$?
- [ $RET -eq 0 ] && break
- [ "$MSG" = 'Another transaction is in progress. Please try again after sometime.' ] || break
- sleep 1
-done
-
-[ $RET -eq 0 ] && {
- echo "gluster volume ${VOLNAME} already exists and is active."
- exit 0
-}
-
-[ "$MSG" = "Volume ${VOLNAME} does not exist" ] && {
- echo "Creating gluster volume ${VOLNAME}."
- echo "cmd: gluster volume create $VOLNAME rep $REP transport tcp $@"
- while true; do
- MSG=$(gluster volume create $VOLNAME rep $REP transport tcp $@ 2>&1 1>/dev/null)
- RET=$?
- [ $RET -eq 0 ] && break
- [ "$MSG" = "volume create: ${VOLNAME}: failed: Volume ${VOLNAME} already exists" ] && {
- RET=0
- break
- }
- [ "$MSG" = "volume create: $VOLNAME: failed: Another transaction is in progress. Please try again after sometime." ] || break
- done
-
- [ $RET -eq 0 ] || {
- echo "gluster volume create $VOLNAME failed ('$MSG')- trying to force."
-
- while true; do
- MSG=$(gluster volume create $VOLNAME rep $REP transport tcp $@ force 2>&1 1>/dev/null)
- RET=$?
- [ $RET -eq 0 ] && break
- [ "$MSG" = "volume create: ${VOLNAME}: failed: Volume ${VOLNAME} already exists" ] && {
- RET=0
- break
- }
- [ "$MSG" = "volume create: $VOLNAME: failed: Another transaction is in progress. Please try again after sometime." ] || break
- done
- }
-
- [ $RET -eq 0 ] || {
- echo "gluster volume create $VOLNAME failed with force ('$MSG')- giving up"
- exit 1
- }
-
- while true; do
- MSG="$(gluster volume status ${VOLNAME} 2>&1 1>/dev/null)"
- RET=$?
- [ $RET -eq 0 ] && break
- [ "${MSG}" != "${MSG#Another transaction}" ] || break
- sleep 1
- done
-
- [ $RET -eq 0 ] && {
- echo "gluster volume ${VOLNAME} is already started."
- exit 0
- }
-}
-
-[ "$MSG" = "Volume ${VOLNAME} is not started" ] && {
- echo "starting gluster volume ${VOLNAME}."
- while true; do
- MSG=$(gluster volume start ${VOLNAME} 2>&1 1> /dev/null)
- RET=$?
- [ $RET -eq 0 ] && break
- [ "$MSG" = "volume start: ${VOLNAME}: failed: Volume ${VOLNAME} already started" ] && {
- RET=0
- break
- }
- [ "$MSG" = "volume start: ${VOLNAME}: failed: Another transaction is in progress. Please try again after sometime." ] || break
- done
-
- [ $RET -eq 0 ] || {
- echo "gluster volume start ${VOLNAME} failed ('$MSG')."
- exit 1
- }
-} || {
- echo "Error: 'gluster volume status ${VOLNAME}' gave '$MSG' ($RET)"
- exit 1
-}
-
-exit 0
-
-SCRIPT
-
-GLUSTER_MOUNT_SCRIPT = <<SCRIPT
-set -e
-
-VOLNAME=$1
-shift
-MOUNTPT=$1
-shift
-
-MOUNTDEV="127.0.0.1:/${VOLNAME}"
-
-mkdir -p ${MOUNTPT}
-
-#mount -t glusterfs ${MOUNTDEV} ${MOUNTPT}
-
-BACKUP_SUFFIX=".orig.$(date +%Y%m%d-%H%M%S)"
-
-FILE=/etc/fstab
-
-grep -q -s "${MOUNTPT}" ${FILE} || {
- test -f ${FILE} || touch ${FILE}
- cp -f -a ${FILE} ${FILE}${BACKUP_SUFFIX}
-
- cat <<EOF >> ${FILE}
-${MOUNTDEV} ${MOUNTPT} glusterfs defaults,selinux 0 0
-EOF
-}
-
-mount | grep -q -s ${MOUNTPT} && {
- echo "${MOUNTPT} is already mounted."
-} || {
- echo "Mounting ${MOUNTPT}."
- mount ${MOUNTPT}
-}
-
-SCRIPT
-
-
-CTDB_STOP_SCRIPT = <<SCRIPT
-set -e
-systemctl stop ctdb.service
-SCRIPT
-
-CTDB_CREATE_NODES_SCRIPT = <<SCRIPT
-set -e
-
-BACKUP_SUFFIX=".orig.$(date +%Y%m%d-%H%M%S)"
-
-NODES_IPS="$@"
-
-FILE=/etc/ctdb/nodes
-test -f ${FILE} || touch ${FILE}
-cp -f -a ${FILE} ${FILE}${BACKUP_SUFFIX}
-
-echo -n > ${FILE}
-for IP in ${NODES_IPS}
-do
- echo "$IP" >> ${FILE}
-done
-SCRIPT
-
-CTDB_CREATE_PUBADDRS_SCRIPT = <<SCRIPT
-set -e
-
-BACKUP_SUFFIX=".orig.$(date +%Y%m%d-%H%M%S)"
-
-PUB_IPS="$@"
-
-FILE=/etc/ctdb/public_addresses
-test -f ${FILE} || touch ${FILE}
-cp -f -a ${FILE} ${FILE}${BACKUP_SUFFIX}
-
-echo -n > ${FILE}
-for IP in ${PUB_IPS}
-do
- echo ${IP} >> ${FILE}
-done
-SCRIPT
-
-CTDB_CREATE_CONF_SCRIPT = <<SCRIPT
-set -e
-
-BACKUP_SUFFIX=".orig.$(date +%Y%m%d-%H%M%S)"
-
-RECLOCKDIR=/gluster/gv0/ctdb
-mkdir -p ${RECLOCKDIR}
-RECLOCKFILE=${RECLOCKDIR}/reclock
-
-PUBLIC_ADDRESSES_FILE=/etc/ctdb/public_addresses
-NODES_FILE=/etc/ctdb/nodes
-
-FILE=/etc/sysconfig/ctdb
-test -f ${FILE} || touch ${FILE}
-cp -f -a ${FILE} ${FILE}${BACKUP_SUFFIX}
-
-echo -n > ${FILE}
-cat <<EOF >> ${FILE}
-CTDB_NODES=${NODES_FILE}
-#CTDB_PUBLIC_ADDRESSES=${PUBLIC_ADDRESSES_FILE}
-CTDB_RECOVERY_LOCK=${RECLOCKFILE}
-CTDB_MANAGES_SAMBA="yes"
-CTDB_SAMBA_SKIP_SHARE_CHECK="yes"
-#CTDB_MANAGES_WINBIND="yes"
-EOF
-SCRIPT
SAMBA_CREATE_CONF_SCRIPT = <<SCRIPT
set -e
EOF
SCRIPT
-CTDB_START_SCRIPT = <<SCRIPT
-set -e
-systemctl start ctdb.service
-SCRIPT
+
+#
+# disks: hard-coded for all nodes for now:
+# TODO: make (some of) these configurable ...
+#
+disks = [
+ {
+ :size => 1, # in GB
+ #:volname => "gv0",
+ },
+ {
+ :size => 10,
+ #:volname => "gv1",
+ },
+]
+
+driveletters = ('b'..'z').to_a
+
+#brick_mount_prefix = "/export"
+brick_mount_prefix = "/bricks"
+brick_path_suffix = "brick"
+gluster_volume_prefix = "gv"
+gluster_mount_prefix = "/gluster"
+
+disks.each_with_index do |disk,disk_num|
+ disk[:number] = disk_num
+ disk[:volume_name] = "#{gluster_volume_prefix}#{disk[:number]}"
+ disk[:volume_mount_point] = "#{gluster_mount_prefix}/#{disk[:volume_name]}"
+ disk[:dev_names] = {
+ :libvirt => "vd#{driveletters[disk[:number]]}",
+ :virtualbox => "sd#{driveletters[disk[:number]]}",
+ }
+ disk[:dev_name] = "sd#{driveletters[disk[:number]]}"
+ disk[:brick_name] = "brick0"
+ disk[:label] = "#{disk[:volume_name]}-#{disk[:brick_name]}"
+ disk[:brick_mount_point] = "#{brick_mount_prefix}/#{disk[:label]}"
+ disk[:brick_path] = "#{disk[:brick_mount_point]}/#{brick_path_suffix}"
+end
+
+# /dev/{sv}db --> xfs filesys (on /dev/{sv}db1)
+# --> mount unter /bricks/gv0
+# --> dir /bricks/gv0/brick --> dir for gluster createvol gv0
+# --> gluster/fuse mount /gluster/gv0
+
+
+my_config = {
+ :provider => :libvirt,
+}
+
#
# The vagrant machine definitions
#
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
- if Vagrant.has_plugin?("vagrant-cachier")
- config.cache.scope = :box
- end
+ config.vm.synced_folder ".", "/vagrant", disabled: true
+ #config.vm.synced_folder './', '/vagrant', type: '9p', disabled: false, accessmode: "squash", owner: "vagrant"
+
+ #if Vagrant.has_plugin?("vagrant-cachier")
+ # config.cache.scope = :machine
+ # #config.cache.scope = :box
+
+ # config.cache.synced_folder_opts = {
+ # type: :nfs,
+ # # The nolock option can be useful for an NFSv3 client that wants to avoid the
+ # # NLM sideband protocol. Without this option, apt-get might hang if it tries
+ # # to lock files needed for /var/cache/* operations. All of this can be avoided
+ # # by using NFSv4 everywhere. Please note that the tcp option is not the default.
+ # #mount_options: ['rw', 'vers=3', 'tcp', 'nolock']
+ # }
+ #end
# just let one node do the probing
probing = false
- vms.each do |machine|
+ vms.each_with_index do |machine,machine_num|
config.vm.define machine[:hostname] do |node|
node.vm.box = machine[:provider][:libvirt][:box]
node.vm.hostname = machine[:hostname]
- node.vm.provider :libvirt do |libvirt|
- libvirt.default_prefix = machine[:provider][:libvirt][:prefix]
- libvirt.memory = 1024
- libvirt.storage :file, :size => '64M', :device => 'vdb'
- libvirt.storage :file, :size => '10G', :device => 'vdc'
+ print "machine #{machine_num}: #{machine[:hostname]}\n"
+
+ node.vm.provider :libvirt do |lv|
+ lv.default_prefix = machine[:provider][:libvirt][:prefix]
+ lv.memory = 1024
+ end
+
+ node.vm.provider :virtualbox do |vb|
+ vb.memory = 1024
+ end
- machine[:networks].each do |net|
- if not net[:ipv4] == ''
- node.vm.network :private_network, :ip => net[:ipv4]
- end
+ disks.each do |disk|
+ node.vm.provider :libvirt do |lv|
+ print " [libvirt] attaching disk ##{disk[:number]}: #{disk[:dev_name]}\n"
+ lv.storage :file, :size => "#{disk[:size]}G", :device => "#{disk[:dev_names][:libvirt]}"
+ #lv.storage :file, :size => "#{disk[:size]}G", :bus => "sata" , :device => "#{disk[:dev_name]}"
+ end
+ node.vm.provider :virtualbox do |vb|
+ disk_size = disk[:size]*1024
+ #disk_file = "disk-#{machine_num}-#{disk[:dev_names][:virtualbox]}.vdi"
+ #print " [virtualbox] disk ##{disk[:number]}: #{disk[:dev_names][:virtualbox]}\n"
+ disk_file = "disk-#{machine_num}-#{disk[:dev_name]}.vdi"
+ print " [virtualbox] attaching disk ##{disk[:number]}: #{disk[:dev_name]}\n"
+ vb.customize [ "createhd", "--filename", disk_file, "--size", disk_size ]
+ vb.customize [ "storageattach", :id, "--storagectl", "SATA Controller", "--port", 3+disk[:number], "--device", 0, "--type", "hdd", "--medium", disk_file ]
end
end
+ machine[:networks].each do |net|
+ if not net[:ipv4] == ''
+ node.vm.network :private_network, :ip => net[:ipv4]
+ end
+ end
node.vm.provision "selinux", type: "shell" do |s|
- s.inline = SELINUX_SCRIPT
+ s.path = "provision/shell/sys/selinux-off.sh"
end
# There is some problem with the fedora base box:
# the interface is not sufficient. We need to restart
# NetworkManager in order to teach it to not feel
# responsible for the interface any more.
- node.vm.provision "net_fix_initial", type: "shell" do |s|
- s.inline = NET_FIX_INITIAL_SCRIPT
- end
+ ###node.vm.provision "net_fix_initial", type: "shell" do |s|
+ ### s.inline = NET_FIX_INITIAL_SCRIPT
+ ###end
node.vm.provision "install", type: "shell" do |s|
- s.inline = INSTALL_SCRIPT
+ s.path = "provision/shell/sys/install-yum.sh"
+ s.args = [ "xfsprogs",
+ "glusterfs",
+ "glusterfs-server",
+ "glusterfs-fuse",
+ "glusterfs-geo-replication",
+ "ctdb",
+ "samba",
+ "samba-client",
+ "samba-vfs-glusterfs" ]
end
# There is some problem with the fedora base box:
# We need to up the interface on reboots.
# It does not come up automatically.
- node.vm.provision "net_fix_always", type: "shell", run: "always" do |s|
- s.inline = NET_FIX_ALWAYS_SCRIPT
- s.args = [ '/gluster/gv0', '/gluster/gv1' ]
- end
-
- # multiple privisioners with same name possible?
- node.vm.provision "xfs_0", type: "shell" do |s|
- s.inline = XFS_SCRIPT
- #s.args = [ "vdb", "/export/gluster/brick1" ]
- s.args = [ "vdb" ]
- end
-
- node.vm.provision "xfs_1", type: "shell" do |s|
- s.inline = XFS_SCRIPT
- #s.args = [ "vdc" , "/export/gluster/brick2" ]
- s.args = [ "vdc" ]
+ ###node.vm.provision "net_fix_always", type: "shell", run: "always" do |s|
+ ### s.inline = NET_FIX_ALWAYS_SCRIPT
+ ### s.args = [ '/gluster/gv0', '/gluster/gv1' ]
+ ###end
+
+ disks.each do |disk|
+ print " create_brick: size #{disk[:size]}G, label #{disk[:label]} under #{disk[:brick_mount_point]}\n"
+ node.vm.provision "create_brick_#{disk[:number]}", type: "shell" do |s|
+ s.path = "provision/shell/gluster/create-brick.v2.sh"
+ s.args = [ "#{disk[:size]}G", disk[:label], disk[:brick_mount_point], brick_path_suffix ]
+ end
end
+
node.vm.provision "gluster_start", type: "shell" do |s|
- s.inline = GLUSTER_START_SCRIPT
+ s.path = "provision/shell/gluster/gluster-start.sh"
end
if !probing
probing = true
node.vm.provision "gluster_probe", type: "shell" do |s|
- s.inline = GLUSTER_PROBE_SCRIPT
+ s.path = "provision/shell/gluster/gluster-probe.sh"
s.args = cluster_internal_ips
end
end
node.vm.provision "gluster_wait_peers", type: "shell" do |s|
- s.inline = GLUSTER_WAIT_PEERS_SCRIPT
- s.args = [ cluster_internal_ips.length ]
+ s.path = "provision/shell/gluster/gluster-wait-peers.sh"
+ s.args = [ cluster_internal_ips.length, 300 ]
end
- node.vm.provision "gluster_createvol_0", type: "shell" do |s|
- mount_points = cluster_internal_ips.map do |ip|
- "#{ip}:/export/vdb1/brick"
- end
- s.inline = GLUSTER_CREATEVOL_SCRIPT
- s.args = [ "gv0", "3" ] + mount_points
- end
- node.vm.provision "gluster_mount_0", type: "shell" do |s|
- s.inline = GLUSTER_MOUNT_SCRIPT
- s.args = [ "gv0", "/gluster/gv0" ]
- end
+ disks.each do |disk|
+ brick_mount_points = cluster_internal_ips.map do |ip|
+ "#{ip}:#{disk[:brick_path]}"
+ end
+
+ print " brick directories: #{brick_mount_points}\n"
- node.vm.provision "gluster_createvol_1", type: "shell" do |s|
- mount_points = cluster_internal_ips.map do |ip|
- "#{ip}:/export/vdc1/brick"
+ node.vm.provision "gluster_createvol_#{disk[:number]}", type: "shell" do |s|
+ s.path = "provision/shell/gluster/gluster-create-volume.sh"
+ s.args = [ disk[:volume_name], "3" ] + brick_mount_points
end
- s.inline = GLUSTER_CREATEVOL_SCRIPT
- s.args = [ "gv1", "3" ] + mount_points
- end
- node.vm.provision "gluster_mount_1", type: "shell" do |s|
- s.inline = GLUSTER_MOUNT_SCRIPT
- s.args = [ "gv1", "/gluster/gv1" ]
+ node.vm.provision "gluster_mount_#{disk[:number]}", type: "shell" do |s|
+ s.path = "provision/shell/gluster/gluster-mount-volume.sh"
+ s.args = [ disk[:volume_name], disk[:volume_mount_point] ]
+ end
end
#
#
node.vm.provision "ctdb_stop", type: "shell" do |s|
- s.inline = CTDB_STOP_SCRIPT
+ s.path = "provision/shell/ctdb/ctdb-stop.sh"
end
node.vm.provision "ctdb_create_nodes", type: "shell" do |s|
- s.inline = CTDB_CREATE_NODES_SCRIPT
+ s.path = "provision/shell/ctdb/ctdb-create-nodes.sh"
s.args = cluster_internal_ips
end
#node.vm.provision "ctdb_create_pubaddrs", type: "shell" do |s|
- # s.inline = CTDB_CREATE_PUBADDRS_SCRIPT
+ # s.path = "provision/shell/ctdb/ctdb-create-pubaddrs.sh"
# s.arg =
#end
node.vm.provision "ctdb_create_conf", type: "shell" do |s|
- s.inline = CTDB_CREATE_CONF_SCRIPT
+ s.path = "provision/shell/ctdb/ctdb-create-conf.sh"
+ s.args = [ "/gluster/gv0/ctdb" ]
end
node.vm.provision "samba_create_conf", type: "shell" do |s|
end
node.vm.provision "ctdb_start", type: "shell" do |s|
- s.inline = CTDB_START_SCRIPT
+ s.path = "provision/shell/ctdb/ctdb-start.sh"
end
end
git.samba.org / obnox / vagrant / vagrant-gluster-samba-cluster.git / blobdiff