#!/usr/bin/env bash if [ ! -f /home/resume-after-reboot ]; then function sqlkiller { while : do sqlstatus=$(systemctl status mysql | awk 'FNR == 3 {print $2}') if [[ "${sqlstatus}" == "active" ]]; then break else /etc/init.d/mysql start sleep 5s continue fi done } function createnewvol { while : do progress=$(aws ec2 describe-snapshots --snapshot-id $snapid --query "Snapshots[*].{Cond:State}" --output text --region us-east-1) if [[ "${progress}" == "completed" ]]; then freshvol=$(aws ec2 create-volume --region us-east-1 --availability-zone us-east-1d --snapshot-id $snapid --volume-type gp2 --output text | awk '{print $8}') touch /home/freshvol.txt echo $freshvol > /home/freshvol.txt sleep 2m break else continue fi done } function searchsnap { for ((i=0;i<5;i++)) do current=$(date +%Y%m%d -d "-$i days") snapid=$(aws ec2 describe-snapshots --filters Name=description,Values=""$value"_$current" --query "Snapshots[*].{SD:SnapshotId}" --region us-east-1 --output text) if [[ $snapid == *"snap-"* ]]; then echo "Snapshot found - continuing with ID: "$snapid" " >> /var/log/ebs-update.log createnewvol break elif [[ $i -ne 4 ]]; then continue else echo "Snapshot not found - exiting" >> /var/log/ebs-update.log echo "--" >> /var/log/ebs-update.log exit 1 fi exit 1 done } while : do status=$(pidof mysqld) if [[ $status -eq 0 ]]; then echo "Mysql is off - $(date) - Proceeding with updating Database" >> /var/log/ebs-update.log break else sqlkiller /etc/init.d/mysql stop pkill -9 mysql pkill -9 mysqld pkill -9 mysqld_safe continue fi done prefix=$(hostname) value=${prefix#*-} value="$value-snapshot" value=$(echo "$value" | sed -r 's/master/slave/g') instanceid=$(ec2metadata --instance-id) for letter in /dev/xvdj xvdj /dev/sdj sdj; do volumeid=$(aws ec2 describe-volumes --filters Name=attachment.instance-id,Values=$instanceid Name=attachment.device,Values=$letter --query "Volumes[*].{ID:VolumeId}" --output text --region us-east-1) if [ -z "$volumeid" ]; then continue else break fi done fuser -km /dev/xvdj umount -d /dev/xvdj fuser -km /dev/sdj umount -d /dev/sdj fuser -km /dev/mapper/mysql--product--master-mysql umount -d /dev/mapper/mysql--product--master-mysql aws ec2 detach-volume --volume-id $volumeid --region us-east-1 --force while : do status=$(aws ec2 describe-volumes --volume-ids $volumeid --query "Volumes[*].{OP:State}" --output text --region us-east-1) if [[ "${status}" == "available" ]]; then searchsnap break else sleep 2m continue fi done sed -i 's/server.*/server = puppet-master.srv.fish.1/' /etc/puppetlabs/puppet/puppet.conf script="@reboot root /opt/fishpond/bin/ebs-update" echo "$script" >> /etc/crontab touch /home/resume-after-reboot /sbin/reboot else sed -i '/@reboot/d' /etc/crontab rm -f /home/resume-after-reboot while : do freshvolafter=$(cat /home/freshvol.txt) instanceidafter=$(ec2metadata --instance-id) newstatus=$(aws ec2 describe-volumes --volume-ids $freshvolafter --query "Volumes[*].{OP:State}" --output text --region us-east-1) if [[ "${newstatus}" == "available" ]]; then aws ec2 attach-volume --volume-id $freshvolafter --instance-id $instanceidafter --device /dev/sdj --region us-east-1 sleep 5m break else continue fi done while : do freshvolafterattach=$(cat /home/freshvol.txt) instanceidafterattach=$(ec2metadata --instance-id) newstatusattach=$(aws ec2 describe-volumes --volume-ids $freshvolafterattach --query "Volumes[*].{OP:State}" --output text --region us-east-1) if [[ "${newstatusattach}" == "in-use" ]]; then break else continue fi done mount /dev/mapper/mysql--product--master-mysql /mnt/mysql mount /dev/xvdj /mnt/mysql mount /dev/sdj /mnt/mysql if grep -qs '/mnt/mysql' /proc/mounts; then logline=$(tail -n2 /var/log/mysql/mysql-error.log | head -1) if [[ "${logline}" == *"Shutdown complete"* ]]; then /etc/init.d/mysql start else /etc/init.d/mysql restart fi sleep 10s else mount /dev/mapper/mysql--product--master-mysql /mnt/mysql mount /dev/xvdj /mnt/mysql mount /dev/sdj /mnt/mysql fi rm -f /home/freshvol.txt sed -i 's/server.*/server = puppet-master.srv.fish/' /etc/puppetlabs/puppet/puppet.conf /opt/puppetlabs/bin/puppet agent -t sleep 5s echo "Script ran correctly at $(date)" >> /var/log/ebs-update.log echo "--" >> /var/log/ebs-update.log fi
Monday, 29 April 2019
Reboot and Volume detach/attach
Subscribe to:
Post Comments (Atom)
No comments:
Post a Comment