Reboot and Volume detach/attach

#!/usr/bin/env bash

if [ ! -f /home/resume-after-reboot ]; then

function sqlkiller {
while :
do
  sqlstatus=$(systemctl status mysql | awk 'FNR == 3 {print $2}')
  if [[ "${sqlstatus}" == "active" ]]; then
  break
  else
  /etc/init.d/mysql start
  sleep 5s
  continue
  fi
done
}

function createnewvol {
while :
do
 progress=$(aws ec2 describe-snapshots --snapshot-id $snapid --query "Snapshots[*].{Cond:State}" --output text --region us-east-1)
 if [[ "${progress}" == "completed" ]]; then
 freshvol=$(aws ec2 create-volume --region us-east-1 --availability-zone us-east-1d --snapshot-id $snapid --volume-type gp2 --output text | awk '{print $8}')
 touch /home/freshvol.txt
 echo $freshvol > /home/freshvol.txt
 sleep 2m
 break
 else
 continue
 fi
done
}

function searchsnap {
for ((i=0;i<5;i++))
 do
  current=$(date +%Y%m%d -d "-$i days")
  snapid=$(aws ec2 describe-snapshots --filters Name=description,Values=""$value"_$current" --query "Snapshots[*].{SD:SnapshotId}" --region us-east-1 --output text)
  if [[ $snapid == *"snap-"* ]]; then
  echo "Snapshot found - continuing with ID: "$snapid" " >> /var/log/ebs-update.log
  createnewvol
  break
  elif [[ $i -ne 4 ]]; then
  continue
  else
  echo "Snapshot not found - exiting" >> /var/log/ebs-update.log
  echo "--" >> /var/log/ebs-update.log
  exit 1
  fi
exit 1
done
}

while :
do
 status=$(pidof mysqld)
 if [[ $status -eq 0 ]]; then
 echo "Mysql is off - $(date) - Proceeding with updating Database" >> /var/log/ebs-update.log
 break
 else
 sqlkiller
 /etc/init.d/mysql stop
 pkill -9 mysql
 pkill -9 mysqld
 pkill -9 mysqld_safe
 continue
 fi
done

prefix=$(hostname)
value=${prefix#*-}
value="$value-snapshot"
value=$(echo "$value" | sed -r 's/master/slave/g')

instanceid=$(ec2metadata --instance-id)
for letter in /dev/xvdj xvdj /dev/sdj sdj; do
  volumeid=$(aws ec2 describe-volumes --filters Name=attachment.instance-id,Values=$instanceid Name=attachment.device,Values=$letter --query "Volumes[*].{ID:VolumeId}" --output text --region us-east-1)
  if [ -z "$volumeid" ]; then
  continue
  else
  break
  fi
done

fuser -km /dev/xvdj
umount -d /dev/xvdj
fuser -km /dev/sdj
umount -d /dev/sdj
fuser -km /dev/mapper/mysql--product--master-mysql
umount -d /dev/mapper/mysql--product--master-mysql

aws ec2 detach-volume --volume-id $volumeid --region us-east-1 --force

while :
do
 status=$(aws ec2 describe-volumes --volume-ids $volumeid --query "Volumes[*].{OP:State}" --output text --region us-east-1)
 if [[ "${status}" == "available" ]]; then
 searchsnap
 break
 else
 sleep 2m
 continue
 fi
done

sed -i 's/server.*/server = puppet-master.srv.fish.1/' /etc/puppetlabs/puppet/puppet.conf
script="@reboot root /opt/fishpond/bin/ebs-update"
echo "$script" >> /etc/crontab
touch /home/resume-after-reboot
/sbin/reboot

else
sed -i '/@reboot/d' /etc/crontab
rm -f /home/resume-after-reboot
while :
do
 freshvolafter=$(cat /home/freshvol.txt)
 instanceidafter=$(ec2metadata --instance-id)
 newstatus=$(aws ec2 describe-volumes --volume-ids $freshvolafter --query "Volumes[*].{OP:State}" --output text --region us-east-1)
 if [[ "${newstatus}" == "available" ]]; then
 aws ec2 attach-volume --volume-id $freshvolafter --instance-id $instanceidafter --device /dev/sdj --region us-east-1
 sleep 5m
 break
 else
 continue
 fi
done

while :
do
 freshvolafterattach=$(cat /home/freshvol.txt)
 instanceidafterattach=$(ec2metadata --instance-id)
 newstatusattach=$(aws ec2 describe-volumes --volume-ids $freshvolafterattach --query "Volumes[*].{OP:State}" --output text --region us-east-1)
 if [[ "${newstatusattach}" == "in-use" ]]; then
 break
 else
 continue
 fi
done

mount /dev/mapper/mysql--product--master-mysql /mnt/mysql
mount /dev/xvdj /mnt/mysql
mount /dev/sdj /mnt/mysql

 if grep -qs '/mnt/mysql' /proc/mounts; then
   logline=$(tail -n2 /var/log/mysql/mysql-error.log | head -1)
    if [[ "${logline}" == *"Shutdown complete"* ]]; then
    /etc/init.d/mysql start
    else
    /etc/init.d/mysql restart
    fi
  sleep 10s
 else
   mount /dev/mapper/mysql--product--master-mysql /mnt/mysql
   mount /dev/xvdj /mnt/mysql
   mount /dev/sdj /mnt/mysql
 fi

rm -f /home/freshvol.txt
sed -i 's/server.*/server = puppet-master.srv.fish/' /etc/puppetlabs/puppet/puppet.conf
/opt/puppetlabs/bin/puppet agent -t
sleep 5s
echo "Script ran correctly at $(date)" >> /var/log/ebs-update.log
echo "--"  >> /var/log/ebs-update.log
fi

Sync directory - encrypt and move

Backup Script:

#!/usr/bin/env bash

echo "Rstudio backup has been started on $(date)" >> /var/log/rstudiobackup.log

dir1="/media/somedirectory/backup-dir"

now=$(date +"%m_%d_%Y_%H")
if [ -d "dir1" ]; then
 :
else
 mkdir -p /media/somedirectory/backup-dir
fi

rsync -avhz /media/dironserver/dirtobackup/ /media/somedirectory/backup-dir
cd /media/somedirectory
tar -I pigz -cf $now.tar.gz backup-dir
gpg --recipient naveed@nasheikh.com --trust-model always --encrypt --armor $now.tar.gz
mv $now.tar.gz.asc /srv/some-remote-dir

echo "Rstudio backup has been Completed on $(date)" >> /var/log/rstudiobackup.log
echo "--" >> /var/log/rstudiobackup.log

exit 0

Place this in /usr/local/bin ( hence no PATH problems )

Packages to install :

apt install pigz

Install cron job

crontab -e

30 3 * * SUN /usr/local/bin/backupper.sh

30 3 * * 1-6  /usr/bin/rsync -avhz /media/dironserver/dirtobackup/ /media/somedirectory/backup-dir

Job Finish

Mount checker with email capability

This is done in 3 scripts:

1st Script - the init service:

Mount the directory:

mount -t cifs //192.168.02.02/somedirectory /srv/somedirectory -o vers=3.0,credentials=/root/creds

#! /bin/sh

### BEGIN INIT INFO
# Provides:          cifchecker
# Required-Start:    $remote_fs $syslog
# Required-Stop:     $remote_fs $syslog
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Simple script to start a program at boot
# Description:       A simple script which will start / stop a program a boot / shutdown.
### END INIT INFO

# If you want a command to always run, put it here

# Carry out specific functions when asked to by the system
case "$1" in
  start)
    echo "Starting cifchecker"
    # run application you want to start
    /usr/local/bin/sendemail.sh &
    ;;
  stop)
    echo "Stopping cifchecker"
    # kill application you want to stop
    dead=$(ps -o pgid,cmd -U root | grep -v grep | grep sendemail | awk '{print $1}')
    kill -- -$dead
    ;;
  *)
    echo "Usage: /etc/init.d/cifchecker {start|stop}"
    exit 1
    ;;
esac

exit 0

Place this in /etc/init.d/ directory and run:

chmod 755 cifchecker

update-rc.d cifchecker defaults

2nd Script: This keeps an eye on the Partition and calls email script if needed:

#!/usr/bin/env bash

while true;

do

 /bin/findmnt /srv/directoryname

 status=$(echo $?)

 sleep 1m

 if [ $status -ne 0 ]; then

  /usr/local/bin/mailer.py 'Mount partition has been lost'

  while true;

  do

   /bin/findmnt /srv/directoryname

   statusafter=$(echo $?)

   sleep 1m

   if [ $statusafter -ne 0 ]; then

    continue

   else

    /usr/local/bin/mailer.py 'Mount partition restored'

    break

   fi

  done

 else

  continue

 fi

done

3rd Script: This sends the email as per directive:

#!/usr/bin/env python3

import os, fnmatch, subprocess,datetime,time,smtplib,sys

from email.message import EmailMessage

icomm1 = sys.argv[1]

msg = EmailMessage()

msg.set_content(icomm1)

msg['Subject'] = 'cifs mount issue'

msg['From'] = 'naveed@nasheikh.com'

msg['To'] = 'adrian@nasheikh.com','tamy@nasheikh.com'

s = smtplib.SMTP('smtp.stats.govt.nz')

s.send_message(msg)

s.quit()

Good practice is to place both executable scripts in /usr/local/bin

-------------------------------------------------------------Fi----------------------------------------------------------

Nagios Script

#!/bin/bash

response=$(curl -s http://search-orders.srv.fish:8080/binlog-webapp/binlog?type=status)
tstamp=$(curl -s http://search-orders.srv.fish:8080/binlog-webapp/binlog?type=status| jq '.status' |  awk -F'"' '$2=="currentTimestamp"{print $4}')
status=$(curl -s http://search-orders.srv.fish:8080/binlog-webapp/binlog?type=status| jq '.status' |  awk -F'"' '$2=="running"{print $4}')

oldstamp=$(date +%s -d "-24 hours")
respinsec=$(date -d "${tstamp}" +"%s")

if [ -z "$response" ]; then
    echo "CRITICAL status - No API response"
    exit 2

elif [[ "${status}" != "true" ]]; then
    echo "CRITICAL status - Search replication is not running."
    exit 2

elif (( respinsec < oldstamp )); then
    echo "CRITICAL status - Timestamp is over 24 hours."
    exit 2

else
    echo "OK - Search Replication is running correctly"
    exit 0
fi

DotNet DockerFile

FROM mcr.microsoft.com/dotnet/core/aspnet:2.2

WORKDIR /app

COPY ./src/AspMVC/publish .

ENTRYPOINT ["dotnet", "AspMVC.dll"]

docker run -p 80:80 myimage

Mysql DB backup with lock table system

#!/usr/bin/env bash

WAITFORLOCK=/root/waitlock

WAITFORSNAPSHOT=/root/waitforsnapshot

LOCKTABLERUN=/root/locktables.pid

function locktable {
(
    echo "FLUSH TABLES WITH READ LOCK;" && \
    sleep 5 && \
    touch ${WAITFORSNAPSHOT} && \
    rm -f ${WAITFORLOCK} && \
    while [ -e ${WAITFORSNAPSHOT} ]; do sleep 1; done && \
    echo "SHOW MASTER STATUS;" && \
    echo "UNLOCK TABLES;" && \
    echo "\quit" \
) | mysql --defaults-file=/root/.my.cnf

rm -f ${LOCKTABLERUN}
}

function prefreeze {

if [ -e ${WAITFORLOCK} ]; then
 echo Previous backup failed, waitforlock file still present && exit 1
fi

if [ -e ${WAITFORSNAPSHOT} ]; then
 echo Previous backup failed, WAITFORSNAPSHOT file still present && exit 1
fi

if [ -e ${LOCKTABLERUN} ]; then
 ps -p `cat ${LOCKTABLERUN}` > /dev/null 2>&1;
 if [ $? -eq 0 ]; then
  echo Panic, locktables script still running && exit 1
 else
  rm -f ${LOCKTABLERUN}
 fi
fi

touch ${WAITFORLOCK}

locktable &

LOCKTABLEPID=$!
echo ${LOCKTABLEPID} > ${LOCKTABLERUN}

while [ -e ${WAITFORLOCK} ]; do
 ps -p ${LOCKTABLEPID} > /dev/null 2>&1;
 if [ $? -eq 1 ]; then
  break
 fi
 sleep 1
done

if [ -e ${WAITFORLOCK} ]; then
 echo Tablelock script exited without removing waitforlock file, something went wrong
else
 echo Tables are locked
fi
}

prefreeze &&

server=$(hostname)
 if [[ "${server}" == *"product"* ]]; then
   server="db-product-slave-snapshot"
 elif [[ "${server}" == *"customer"* ]]; then
   server="db-customer-slave-snapshot"
 elif [[ "${server}" == *"finance"* ]]; then
   server="db-finance-slave-snapshot"
 else
   server=$(hostname)
 fi
instanceid=$(ec2metadata --instance-id)
for letter in /dev/xvdj xvdj /dev/sdj sdj; do
  volumeid=$(aws ec2 describe-volumes --filters Name=attachment.instance-id,Values=$instanceid Name=attachment.device,Values=$letter --query "Volumes[*].{ID:VolumeId}" --output text --region us-east-1)
  if [ -z "$volumeid" ]; then
  continue
  else
  break
  fi
done
snapid=$(aws ec2 create-snapshot --volume-id $volumeid --description ""$server"_$(date +%Y%m%d)" --output text --region us-east-1 | awk '{print $4}')
echo "Backup initiated with SnapshotID: "$snapid"" >> /var/log/ebs-snapshot.log

while :
do
 progress=$(aws ec2 describe-snapshots --snapshot-id $snapid --query "Snapshots[*].{Cond:State}" --output text --region us-east-1)
 if [[ "${progress}" == "pending" ]]; then
 sleep 5m
 continue
 else
 result=$(aws ec2 describe-snapshots --snapshot-id $snapid --query "Snapshots[*].{Cond:State}" --output text --region us-east-1)
 echo "Snap has been "$result" and Mysql has been started on $(date)" >> /var/log/ebs-snapshot.log
 echo "--" >> /var/log/ebs-snapshot.log
 break
 fi
done

rm ${WAITFORSNAPSHOT}

exit 0