#!/usr/bin/env bash
if [ ! -f /home/resume-after-reboot ]; then
function sqlkiller {
while :
do
sqlstatus=$(systemctl status mysql | awk 'FNR == 3 {print $2}')
if [[ "${sqlstatus}" == "active" ]]; then
break
else
/etc/init.d/mysql start
sleep 5s
continue
fi
done
}
function createnewvol {
while :
do
progress=$(aws ec2 describe-snapshots --snapshot-id $snapid --query "Snapshots[*].{Cond:State}" --output text --region us-east-1)
if [[ "${progress}" == "completed" ]]; then
freshvol=$(aws ec2 create-volume --region us-east-1 --availability-zone us-east-1d --snapshot-id $snapid --volume-type gp2 --output text | awk '{print $8}')
touch /home/freshvol.txt
echo $freshvol > /home/freshvol.txt
sleep 2m
break
else
continue
fi
done
}
function searchsnap {
for ((i=0;i<5;i++))
do
current=$(date +%Y%m%d -d "-$i days")
snapid=$(aws ec2 describe-snapshots --filters Name=description,Values=""$value"_$current" --query "Snapshots[*].{SD:SnapshotId}" --region us-east-1 --output text)
if [[ $snapid == *"snap-"* ]]; then
echo "Snapshot found - continuing with ID: "$snapid" " >> /var/log/ebs-update.log
createnewvol
break
elif [[ $i -ne 4 ]]; then
continue
else
echo "Snapshot not found - exiting" >> /var/log/ebs-update.log
echo "--" >> /var/log/ebs-update.log
exit 1
fi
exit 1
done
}
while :
do
status=$(pidof mysqld)
if [[ $status -eq 0 ]]; then
echo "Mysql is off - $(date) - Proceeding with updating Database" >> /var/log/ebs-update.log
break
else
sqlkiller
/etc/init.d/mysql stop
pkill -9 mysql
pkill -9 mysqld
pkill -9 mysqld_safe
continue
fi
done
prefix=$(hostname)
value=${prefix#*-}
value="$value-snapshot"
value=$(echo "$value" | sed -r 's/master/slave/g')
instanceid=$(ec2metadata --instance-id)
for letter in /dev/xvdj xvdj /dev/sdj sdj; do
volumeid=$(aws ec2 describe-volumes --filters Name=attachment.instance-id,Values=$instanceid Name=attachment.device,Values=$letter --query "Volumes[*].{ID:VolumeId}" --output text --region us-east-1)
if [ -z "$volumeid" ]; then
continue
else
break
fi
done
fuser -km /dev/xvdj
umount -d /dev/xvdj
fuser -km /dev/sdj
umount -d /dev/sdj
fuser -km /dev/mapper/mysql--product--master-mysql
umount -d /dev/mapper/mysql--product--master-mysql
aws ec2 detach-volume --volume-id $volumeid --region us-east-1 --force
while :
do
status=$(aws ec2 describe-volumes --volume-ids $volumeid --query "Volumes[*].{OP:State}" --output text --region us-east-1)
if [[ "${status}" == "available" ]]; then
searchsnap
break
else
sleep 2m
continue
fi
done
sed -i 's/server.*/server = puppet-master.srv.fish.1/' /etc/puppetlabs/puppet/puppet.conf
script="@reboot root /opt/fishpond/bin/ebs-update"
echo "$script" >> /etc/crontab
touch /home/resume-after-reboot
/sbin/reboot
else
sed -i '/@reboot/d' /etc/crontab
rm -f /home/resume-after-reboot
while :
do
freshvolafter=$(cat /home/freshvol.txt)
instanceidafter=$(ec2metadata --instance-id)
newstatus=$(aws ec2 describe-volumes --volume-ids $freshvolafter --query "Volumes[*].{OP:State}" --output text --region us-east-1)
if [[ "${newstatus}" == "available" ]]; then
aws ec2 attach-volume --volume-id $freshvolafter --instance-id $instanceidafter --device /dev/sdj --region us-east-1
sleep 5m
break
else
continue
fi
done
while :
do
freshvolafterattach=$(cat /home/freshvol.txt)
instanceidafterattach=$(ec2metadata --instance-id)
newstatusattach=$(aws ec2 describe-volumes --volume-ids $freshvolafterattach --query "Volumes[*].{OP:State}" --output text --region us-east-1)
if [[ "${newstatusattach}" == "in-use" ]]; then
break
else
continue
fi
done
mount /dev/mapper/mysql--product--master-mysql /mnt/mysql
mount /dev/xvdj /mnt/mysql
mount /dev/sdj /mnt/mysql
if grep -qs '/mnt/mysql' /proc/mounts; then
logline=$(tail -n2 /var/log/mysql/mysql-error.log | head -1)
if [[ "${logline}" == *"Shutdown complete"* ]]; then
/etc/init.d/mysql start
else
/etc/init.d/mysql restart
fi
sleep 10s
else
mount /dev/mapper/mysql--product--master-mysql /mnt/mysql
mount /dev/xvdj /mnt/mysql
mount /dev/sdj /mnt/mysql
fi
rm -f /home/freshvol.txt
sed -i 's/server.*/server = puppet-master.srv.fish/' /etc/puppetlabs/puppet/puppet.conf
/opt/puppetlabs/bin/puppet agent -t
sleep 5s
echo "Script ran correctly at $(date)" >> /var/log/ebs-update.log
echo "--" >> /var/log/ebs-update.log
fi
Monday, 29 April 2019
Reboot and Volume detach/attach
Subscribe to:
Post Comments (Atom)
No comments:
Post a Comment