Kubernetes Deployment Via Ansible

/etc/ansible/playbook/kube-dependencies.yaml

- hosts: all
  become: yes
  tasks:
   - name: install gpg
     apt:
       name: gpg
       state: present
       update_cache: true

   - name: install Docker
     apt:
       name: docker.io
       state: present
       update_cache: true

   - name: Enable service
     service:
       name: docker
       enabled: yes

   - name: start service
     service:
       name: docker
       state: started

   - name: install APT Transport HTTPS
     apt:
       name: apt-transport-https
       state: present

   - name: add Kubernetes apt-key
     apt_key:
       url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
       state: present

   - name: add Kubernetes' APT repository
     apt_repository:
      repo: deb http://apt.kubernetes.io/ kubernetes-xenial main
      state: present
      filename: 'kubernetes'

   - name: install kubelet
     apt:
       name: kubelet
       state: present
       update_cache: true

   - name: install kubeadm
     apt:
       name: kubeadm
       state: present

- hosts: master
  become: yes
  tasks:
   - name: install kubectl
     apt:
       name: kubectl
       state: present
       force: yes

/etc/ansible/playbook/master.yaml

- hosts: master
  become: yes
  tasks:
    - name: Disable SWAP since kubernetes can't work with swap enabled (1/2)
      shell: |
        swapoff -a
      when: ansible_swaptotal_mb > 0

    - name: Disable SWAP in fstab since kubernetes can't work with swap enabled (2/2)
      replace:
        path: /etc/fstab
        regexp: '^(.+?\sswap\s+sw\s+.*)$'
        replace: '# \1'

    - name: initialize the cluster
      shell: kubeadm init --pod-network-cidr=10.244.0.0/16 >> cluster_initialized.txt
      args:
        chdir: $HOME
        creates: cluster_initialized.txt

    - name: create .kube directory
      become: yes
      file:
        path: $HOME/.kube
        state: directory
        mode: 0755

    - name: copy admin.conf to user's kube config
      copy:
        src: /etc/kubernetes/admin.conf
        dest: $HOME/.kube/config
        remote_src: yes

    - name: install Pod network
      become: yes
      shell: kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"
      args:
        chdir: $HOME
        creates: pod_network_setup.txt

/etc/ansible/playbook/workers.yaml

- hosts: master

  become: yes

  gather_facts: false

  tasks:

    - name: get join command

      shell: kubeadm token create --print-join-command

      register: join_command_raw

    - name: set join command

      set_fact:

        join_command: "{{ join_command_raw.stdout_lines[0] }}"

- hosts: workers

  become: yes

  tasks:

    - name: Remove swapfile from /etc/fstab

      mount:

        name: swap

        fstype: swap

        state: absent

    - name: Disable swap

      command: swapoff -a

      when: ansible_swaptotal_mb > 0

    

    - name: join cluster

      shell: "{{ hostvars[groups['master'][0]].join_command }} >> node_joined.txt"

      args:

        chdir: $HOME

        creates: node_joined.txt

/etc/ansible/hosts

[master]

192.168.109.150

[workers]

192.168.109.151

[all:vars]

ansible_python_interpreter=/usr/bin/python3

Reverse Engineer a docker image for Dockerfile

#!/bin/bash

if [[ "$(docker images -q chenzj/dfimage:latest 2> /dev/null)" == "" ]]; then
  docker pull chenzj/dfimage
fi

read -e -p "Enter Image ID: " IMAGE_ID

if grep -q dfimage /etc/profile;
 then
 :
 else
  echo "" >> /etc/profile
  echo alias dfimage="'docker run -v /var/run/docker.sock:/var/run/docker.sock --rm chenzj/dfimage'" >> /etc/profile
  source /etc/profile
fi

docker run -v /var/run/docker.sock:/var/run/docker.sock --rm chenzj/dfimage $IMAGE_ID > Dockerfile1

docker history --no-trunc $IMAGE_ID  | tac | tr -s ' ' | cut -d " " -f 5- | sed 's,^/bin/sh -c #(nop) ,,g' | sed 's,^/bin/sh -c,RUN,g' | sed 's, && ,\n  & ,g' | sed 's,\s*[0-9]*[\.]*[0-9]*[kMG]*B\s*$,,g' | head -n -1 > Dockerfile2

Systemd Service only initiates at shutdown ( no reboot ) and keeps network intact

# the %H is the env var for systemd file which is $(hostname) for bash

[Unit]
Description=Run a Bash script at shutdown
DefaultDependencies=no
Wants=network-online.target
After=network-online.target
Before=poweroff.target halt.target

[Service]
ExecStart=/usr/bin/curl -X POST 'http://box21.ara.ac.nz:32446/query?db=telegraf' --data-urlencode "q=DROP SERIES WHERE host = '%H'"
Type=oneshot
RemainAfterExit=yes

[Install]
WantedBy=poweroff.target halt.target 

Reboot and Volume detach/attach

#!/usr/bin/env bash

if [ ! -f /home/resume-after-reboot ]; then

function sqlkiller {
while :
do
  sqlstatus=$(systemctl status mysql | awk 'FNR == 3 {print $2}')
  if [[ "${sqlstatus}" == "active" ]]; then
  break
  else
  /etc/init.d/mysql start
  sleep 5s
  continue
  fi
done
}

function createnewvol {
while :
do
 progress=$(aws ec2 describe-snapshots --snapshot-id $snapid --query "Snapshots[*].{Cond:State}" --output text --region us-east-1)
 if [[ "${progress}" == "completed" ]]; then
 freshvol=$(aws ec2 create-volume --region us-east-1 --availability-zone us-east-1d --snapshot-id $snapid --volume-type gp2 --output text | awk '{print $8}')
 touch /home/freshvol.txt
 echo $freshvol > /home/freshvol.txt
 sleep 2m
 break
 else
 continue
 fi
done
}

function searchsnap {
for ((i=0;i<5;i++))
 do
  current=$(date +%Y%m%d -d "-$i days")
  snapid=$(aws ec2 describe-snapshots --filters Name=description,Values=""$value"_$current" --query "Snapshots[*].{SD:SnapshotId}" --region us-east-1 --output text)
  if [[ $snapid == *"snap-"* ]]; then
  echo "Snapshot found - continuing with ID: "$snapid" " >> /var/log/ebs-update.log
  createnewvol
  break
  elif [[ $i -ne 4 ]]; then
  continue
  else
  echo "Snapshot not found - exiting" >> /var/log/ebs-update.log
  echo "--" >> /var/log/ebs-update.log
  exit 1
  fi
exit 1
done
}

while :
do
 status=$(pidof mysqld)
 if [[ $status -eq 0 ]]; then
 echo "Mysql is off - $(date) - Proceeding with updating Database" >> /var/log/ebs-update.log
 break
 else
 sqlkiller
 /etc/init.d/mysql stop
 pkill -9 mysql
 pkill -9 mysqld
 pkill -9 mysqld_safe
 continue
 fi
done

prefix=$(hostname)
value=${prefix#*-}
value="$value-snapshot"
value=$(echo "$value" | sed -r 's/master/slave/g')

instanceid=$(ec2metadata --instance-id)
for letter in /dev/xvdj xvdj /dev/sdj sdj; do
  volumeid=$(aws ec2 describe-volumes --filters Name=attachment.instance-id,Values=$instanceid Name=attachment.device,Values=$letter --query "Volumes[*].{ID:VolumeId}" --output text --region us-east-1)
  if [ -z "$volumeid" ]; then
  continue
  else
  break
  fi
done

fuser -km /dev/xvdj
umount -d /dev/xvdj
fuser -km /dev/sdj
umount -d /dev/sdj
fuser -km /dev/mapper/mysql--product--master-mysql
umount -d /dev/mapper/mysql--product--master-mysql

aws ec2 detach-volume --volume-id $volumeid --region us-east-1 --force

while :
do
 status=$(aws ec2 describe-volumes --volume-ids $volumeid --query "Volumes[*].{OP:State}" --output text --region us-east-1)
 if [[ "${status}" == "available" ]]; then
 searchsnap
 break
 else
 sleep 2m
 continue
 fi
done

sed -i 's/server.*/server = puppet-master.srv.fish.1/' /etc/puppetlabs/puppet/puppet.conf
script="@reboot root /opt/fishpond/bin/ebs-update"
echo "$script" >> /etc/crontab
touch /home/resume-after-reboot
/sbin/reboot

else
sed -i '/@reboot/d' /etc/crontab
rm -f /home/resume-after-reboot
while :
do
 freshvolafter=$(cat /home/freshvol.txt)
 instanceidafter=$(ec2metadata --instance-id)
 newstatus=$(aws ec2 describe-volumes --volume-ids $freshvolafter --query "Volumes[*].{OP:State}" --output text --region us-east-1)
 if [[ "${newstatus}" == "available" ]]; then
 aws ec2 attach-volume --volume-id $freshvolafter --instance-id $instanceidafter --device /dev/sdj --region us-east-1
 sleep 5m
 break
 else
 continue
 fi
done

while :
do
 freshvolafterattach=$(cat /home/freshvol.txt)
 instanceidafterattach=$(ec2metadata --instance-id)
 newstatusattach=$(aws ec2 describe-volumes --volume-ids $freshvolafterattach --query "Volumes[*].{OP:State}" --output text --region us-east-1)
 if [[ "${newstatusattach}" == "in-use" ]]; then
 break
 else
 continue
 fi
done

mount /dev/mapper/mysql--product--master-mysql /mnt/mysql
mount /dev/xvdj /mnt/mysql
mount /dev/sdj /mnt/mysql

 if grep -qs '/mnt/mysql' /proc/mounts; then
   logline=$(tail -n2 /var/log/mysql/mysql-error.log | head -1)
    if [[ "${logline}" == *"Shutdown complete"* ]]; then
    /etc/init.d/mysql start
    else
    /etc/init.d/mysql restart
    fi
  sleep 10s
 else
   mount /dev/mapper/mysql--product--master-mysql /mnt/mysql
   mount /dev/xvdj /mnt/mysql
   mount /dev/sdj /mnt/mysql
 fi

rm -f /home/freshvol.txt
sed -i 's/server.*/server = puppet-master.srv.fish/' /etc/puppetlabs/puppet/puppet.conf
/opt/puppetlabs/bin/puppet agent -t
sleep 5s
echo "Script ran correctly at $(date)" >> /var/log/ebs-update.log
echo "--"  >> /var/log/ebs-update.log
fi

Sync directory - encrypt and move

Backup Script:

#!/usr/bin/env bash

echo "Rstudio backup has been started on $(date)" >> /var/log/rstudiobackup.log

dir1="/media/somedirectory/backup-dir"

now=$(date +"%m_%d_%Y_%H")
if [ -d "dir1" ]; then
 :
else
 mkdir -p /media/somedirectory/backup-dir
fi

rsync -avhz /media/dironserver/dirtobackup/ /media/somedirectory/backup-dir
cd /media/somedirectory
tar -I pigz -cf $now.tar.gz backup-dir
gpg --recipient naveed@nasheikh.com --trust-model always --encrypt --armor $now.tar.gz
mv $now.tar.gz.asc /srv/some-remote-dir

echo "Rstudio backup has been Completed on $(date)" >> /var/log/rstudiobackup.log
echo "--" >> /var/log/rstudiobackup.log

exit 0

Place this in /usr/local/bin ( hence no PATH problems )

Packages to install :

apt install pigz

Install cron job

crontab -e

30 3 * * SUN /usr/local/bin/backupper.sh

30 3 * * 1-6  /usr/bin/rsync -avhz /media/dironserver/dirtobackup/ /media/somedirectory/backup-dir

Job Finish

Mount checker with email capability

This is done in 3 scripts:

1st Script - the init service:

Mount the directory:

mount -t cifs //192.168.02.02/somedirectory /srv/somedirectory -o vers=3.0,credentials=/root/creds

#! /bin/sh

### BEGIN INIT INFO
# Provides:          cifchecker
# Required-Start:    $remote_fs $syslog
# Required-Stop:     $remote_fs $syslog
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Simple script to start a program at boot
# Description:       A simple script which will start / stop a program a boot / shutdown.
### END INIT INFO

# If you want a command to always run, put it here

# Carry out specific functions when asked to by the system
case "$1" in
  start)
    echo "Starting cifchecker"
    # run application you want to start
    /usr/local/bin/sendemail.sh &
    ;;
  stop)
    echo "Stopping cifchecker"
    # kill application you want to stop
    dead=$(ps -o pgid,cmd -U root | grep -v grep | grep sendemail | awk '{print $1}')
    kill -- -$dead
    ;;
  *)
    echo "Usage: /etc/init.d/cifchecker {start|stop}"
    exit 1
    ;;
esac

exit 0

Place this in /etc/init.d/ directory and run:

chmod 755 cifchecker

update-rc.d cifchecker defaults

2nd Script: This keeps an eye on the Partition and calls email script if needed:

#!/usr/bin/env bash

while true;

do

 /bin/findmnt /srv/directoryname

 status=$(echo $?)

 sleep 1m

 if [ $status -ne 0 ]; then

  /usr/local/bin/mailer.py 'Mount partition has been lost'

  while true;

  do

   /bin/findmnt /srv/directoryname

   statusafter=$(echo $?)

   sleep 1m

   if [ $statusafter -ne 0 ]; then

    continue

   else

    /usr/local/bin/mailer.py 'Mount partition restored'

    break

   fi

  done

 else

  continue

 fi

done

3rd Script: This sends the email as per directive:

#!/usr/bin/env python3

import os, fnmatch, subprocess,datetime,time,smtplib,sys

from email.message import EmailMessage

icomm1 = sys.argv[1]

msg = EmailMessage()

msg.set_content(icomm1)

msg['Subject'] = 'cifs mount issue'

msg['From'] = 'naveed@nasheikh.com'

msg['To'] = 'adrian@nasheikh.com','tamy@nasheikh.com'

s = smtplib.SMTP('smtp.stats.govt.nz')

s.send_message(msg)

s.quit()

Good practice is to place both executable scripts in /usr/local/bin

-------------------------------------------------------------Fi----------------------------------------------------------

Nagios Script

#!/bin/bash

response=$(curl -s http://search-orders.srv.fish:8080/binlog-webapp/binlog?type=status)
tstamp=$(curl -s http://search-orders.srv.fish:8080/binlog-webapp/binlog?type=status| jq '.status' |  awk -F'"' '$2=="currentTimestamp"{print $4}')
status=$(curl -s http://search-orders.srv.fish:8080/binlog-webapp/binlog?type=status| jq '.status' |  awk -F'"' '$2=="running"{print $4}')

oldstamp=$(date +%s -d "-24 hours")
respinsec=$(date -d "${tstamp}" +"%s")

if [ -z "$response" ]; then
    echo "CRITICAL status - No API response"
    exit 2

elif [[ "${status}" != "true" ]]; then
    echo "CRITICAL status - Search replication is not running."
    exit 2

elif (( respinsec < oldstamp )); then
    echo "CRITICAL status - Timestamp is over 24 hours."
    exit 2

else
    echo "OK - Search Replication is running correctly"
    exit 0
fi