Monday 29 April 2019

Reboot and Volume detach/attach

#!/usr/bin/env bash

if [ ! -f /home/resume-after-reboot ]; then

function sqlkiller {
while :
do
  sqlstatus=$(systemctl status mysql | awk 'FNR == 3 {print $2}')
  if [[ "${sqlstatus}" == "active" ]]; then
  break
  else
  /etc/init.d/mysql start
  sleep 5s
  continue
  fi
done
}

function createnewvol {
while :
do
 progress=$(aws ec2 describe-snapshots --snapshot-id $snapid --query "Snapshots[*].{Cond:State}" --output text --region us-east-1)
 if [[ "${progress}" == "completed" ]]; then
 freshvol=$(aws ec2 create-volume --region us-east-1 --availability-zone us-east-1d --snapshot-id $snapid --volume-type gp2 --output text | awk '{print $8}')
 touch /home/freshvol.txt
 echo $freshvol > /home/freshvol.txt
 sleep 2m
 break
 else
 continue
 fi
done
}

function searchsnap {
for ((i=0;i<5;i++))
 do
  current=$(date +%Y%m%d -d "-$i days")
  snapid=$(aws ec2 describe-snapshots --filters Name=description,Values=""$value"_$current" --query "Snapshots[*].{SD:SnapshotId}" --region us-east-1 --output text)
  if [[ $snapid == *"snap-"* ]]; then
  echo "Snapshot found - continuing with ID: "$snapid" " >> /var/log/ebs-update.log
  createnewvol
  break
  elif [[ $i -ne 4 ]]; then
  continue
  else
  echo "Snapshot not found - exiting" >> /var/log/ebs-update.log
  echo "--" >> /var/log/ebs-update.log
  exit 1
  fi
exit 1
done
}

while :
do
 status=$(pidof mysqld)
 if [[ $status -eq 0 ]]; then
 echo "Mysql is off - $(date) - Proceeding with updating Database" >> /var/log/ebs-update.log
 break
 else
 sqlkiller
 /etc/init.d/mysql stop
 pkill -9 mysql
 pkill -9 mysqld
 pkill -9 mysqld_safe
 continue
 fi
done

prefix=$(hostname)
value=${prefix#*-}
value="$value-snapshot"
value=$(echo "$value" | sed -r 's/master/slave/g')

instanceid=$(ec2metadata --instance-id)
for letter in /dev/xvdj xvdj /dev/sdj sdj; do
  volumeid=$(aws ec2 describe-volumes --filters Name=attachment.instance-id,Values=$instanceid Name=attachment.device,Values=$letter --query "Volumes[*].{ID:VolumeId}" --output text --region us-east-1)
  if [ -z "$volumeid" ]; then
  continue
  else
  break
  fi
done

fuser -km /dev/xvdj
umount -d /dev/xvdj
fuser -km /dev/sdj
umount -d /dev/sdj
fuser -km /dev/mapper/mysql--product--master-mysql
umount -d /dev/mapper/mysql--product--master-mysql

aws ec2 detach-volume --volume-id $volumeid --region us-east-1 --force

while :
do
 status=$(aws ec2 describe-volumes --volume-ids $volumeid --query "Volumes[*].{OP:State}" --output text --region us-east-1)
 if [[ "${status}" == "available" ]]; then
 searchsnap
 break
 else
 sleep 2m
 continue
 fi
done

sed -i 's/server.*/server = puppet-master.srv.fish.1/' /etc/puppetlabs/puppet/puppet.conf
script="@reboot root /opt/fishpond/bin/ebs-update"
echo "$script" >> /etc/crontab
touch /home/resume-after-reboot
/sbin/reboot

else
sed -i '/@reboot/d' /etc/crontab
rm -f /home/resume-after-reboot
while :
do
 freshvolafter=$(cat /home/freshvol.txt)
 instanceidafter=$(ec2metadata --instance-id)
 newstatus=$(aws ec2 describe-volumes --volume-ids $freshvolafter --query "Volumes[*].{OP:State}" --output text --region us-east-1)
 if [[ "${newstatus}" == "available" ]]; then
 aws ec2 attach-volume --volume-id $freshvolafter --instance-id $instanceidafter --device /dev/sdj --region us-east-1
 sleep 5m
 break
 else
 continue
 fi
done

while :
do
 freshvolafterattach=$(cat /home/freshvol.txt)
 instanceidafterattach=$(ec2metadata --instance-id)
 newstatusattach=$(aws ec2 describe-volumes --volume-ids $freshvolafterattach --query "Volumes[*].{OP:State}" --output text --region us-east-1)
 if [[ "${newstatusattach}" == "in-use" ]]; then
 break
 else
 continue
 fi
done

mount /dev/mapper/mysql--product--master-mysql /mnt/mysql
mount /dev/xvdj /mnt/mysql
mount /dev/sdj /mnt/mysql

 if grep -qs '/mnt/mysql' /proc/mounts; then
   logline=$(tail -n2 /var/log/mysql/mysql-error.log | head -1)
    if [[ "${logline}" == *"Shutdown complete"* ]]; then
    /etc/init.d/mysql start
    else
    /etc/init.d/mysql restart
    fi
  sleep 10s
 else
   mount /dev/mapper/mysql--product--master-mysql /mnt/mysql
   mount /dev/xvdj /mnt/mysql
   mount /dev/sdj /mnt/mysql
 fi

rm -f /home/freshvol.txt
sed -i 's/server.*/server = puppet-master.srv.fish/' /etc/puppetlabs/puppet/puppet.conf
/opt/puppetlabs/bin/puppet agent -t
sleep 5s
echo "Script ran correctly at $(date)" >> /var/log/ebs-update.log
echo "--"  >> /var/log/ebs-update.log
fi

No comments:

Post a Comment