Monday, 26 October 2015

Getting a DESKTOP on centos!

Do a minimal install of centos 6.5 and start doing this:

# yum -y groupinstall "Desktop" "Desktop Platform" "X Window System" "Fonts" "General Purpose Desktop"

# init 5

To have CentOS boot into runlevel 5 “X11” instead of runlevel 3 “Full multiuser mode”, modify the /etc/inittab file to change start up level like this :

# nano /etc/inittab

id:3:initdefault:

to

id:5:initdefault:

Job Done!

Friday, 2 October 2015

AWS Auto Scaling - Complete CLI Solution

#! /bin/bash

function ASG {
read -e -p "Enter Min no. of instances(Integer Value)...: " min
echo ""
read -e -p "Enter Max no. of instances(Integer Value)...: " max
echo ""
read -e -p "Enter Desired no. of instances(Integer Value)...: " desired
echo ""
aws autoscaling create-auto-scaling-group --auto-scaling-group-name $Cluster-$COMPONENT-ASG --launch-configuration-name RTB-$COMPONENT-ASG --min-size $min --max-size $max --desired-capacity $desired --availability-zones us-east-1e --health-check-type ELB --health-check-grace-period 240 --default-cooldown 300 --load-balancer-names "RTB" --vpc-zone-identifier subnet-fea1f0d5 ; check;
}

function config {
echo ""
aws autoscaling create-launch-configuration --launch-configuration-name $Cluster-$COMPONENT-ASG --image-id ami-b7f68bd2 --instance-type $COMPONENT --spot-price "0.27" --no-associate-public-ip-address  --security-groups sg-665df302 --user-data $File --key-name rtbservers --instance-monitoring Enabled=false --no-ebs-optimized --block-device-mappings "[{\"DeviceName\":\"/dev/xvda\",\"Ebs\":{\"SnapshotId\":\"snap-68dc2c09\",\"VolumeSize\":10,\"VolumeType\": \"gp2\",\"DeleteOnTermination\": true}},{\"DeviceName\": \"/dev/sdb\",\"VirtualName\":\"ephemeral0\"}]" ; check && ASG;
}

function check {
if [ $? -eq 0 ]
then
echo "Command Executed Successfully"
echo ""
else
echo "Error, Please try again!"
echo ""
exit 1
fi
}

function type {
echo ""
echo "Please select the desired Instance type"
echo ""

select COMPONENT in c3.2xlarge c3.4xlarge m3.2xlarge r3.2xlarge r3.4xlarge g2.2xlarge
do
case $COMPONENT in

c3.2xlarge) config;
            break;;
c3.4xlarge) config;
            break;;
m3.2xlarge) config;
            break;;
r3.2xlarge) config;
            break;;
r3.4xlarge) config;
            break;;
g2.2xlarge) config;
            break;;
esac
done
}

echo ""
echo "Please select the desired Cluster type"
echo ""

select Cluster in RTB RTB3 RTB4
do
case $Cluster in
RTB) File="file:///operations/users/naveed/bootscriptrtb.txt"
     type;
     break;;
RTB3) File="file:///operations/users/naveed/bootscriptrtb3.txt"
      type;
      break;;
RTB4) File="file:///operations/users/naveed/bootscriptrtb4.txt"
      type;
      break;;
esac
done

echo "Please configure AS Policy"
echo ""
read -e -p "Enter No. of Instance to be affected (Integer Value)...: " aff
echo ""

aws autoscaling put-scaling-policy --policy-name increase-policy --auto-scaling-group-name $Cluster-$COMPONENT-ASG --scaling-adjustment $aff --adjustment-type ChangeInCapacity --output text > increase.txt
inc=$(cat increase.txt)
aws autoscaling put-scaling-policy --policy-name decrease-policy --auto-scaling-group-name $Cluster-$COMPONENT-ASG --scaling-adjustment -$aff --adjustment-type ChangeInCapacity --output text > decrease.txt
dec=$(cat decrease.txt)

echo "Please configure AS Alarm"
echo ""
read -e -p "Enter High CPU threshold (Integer Value)...: " high
echo ""
read -e -p "Enter Lower CPU threshold (Integer Value)...: " low
echo ""

aws cloudwatch put-metric-alarm --alarm-name $Cluster-$COMPONENT-AddCapacity --metric-name $Cluster-$COMPONENT-CPU-HIGH --namespace AWS/EC2 --statistic Average --period 300 --threshold $high --comparison-operator GreaterThanOrEqualToThreshold --dimensions "Name=AutoScalingGroupName,Value=$Cluster-$COMPONENT-ASG" --evaluation-periods 2 --alarm-actions $inc
aws cloudwatch put-metric-alarm --alarm-name $Cluster-$COMPONENT-RemoveCapacity --metric-name $Cluster-$COMPONENT-CPU-LOW --namespace AWS/EC2 --statistic Average --period 300 --threshold $low --comparison-operator LessThanOrEqualToThreshold --dimensions "Name=AutoScalingGroupName,Value=$Cluster-$COMPONENT-ASG" --evaluation-periods 2 --alarm-actions $dec

rm -fr increase.txt decrease.txt

echo "Job Finished!"
echo ""

Friday, 25 September 2015

Fixing the GPG missing keys error

Sometimes we face the missing gpg key issue when we add diffrent sources to sources.list files, well this can be fixed easily:
Im gonna show an example where im using ubuntu 14 and try to first add sources of debian jessie 8.

first go to http://debgen.simplylinux.ch/

get your source list, for eg:

deb http://ftp.us.debian.org/debian testing main contrib non-free
deb-src http://ftp.us.debian.org/debian testing main contrib non-free

deb http://security.debian.org/ jessie/updates main contrib non-free
deb-src http://security.debian.org/ jessie/updates main contrib non-free


paste this in

nano /etc/apt/sources.list

then run this script:

#!/bin/bash

apt-get update 2> /tmp/keymissing; for key in $(grep "NO_PUBKEY" /tmp/keymissing |sed "s/.*NO_PUBKEY //"); do echo -e "\nProcessing key: $key"; gpg --keyserver pgpkeys.mit.edu --recv $key && gpg --export --armor $key | apt-key add -; done
apt-get update 2> /tmp/keymissing; for key in $(grep "NO_PUBKEY" /tmp/keymissing |sed "s/.*NO_PUBKEY //"); do echo -e "\nProcessing key: $key"; gpg --keyserver subkeys.pgp.net --recv $key && gpg --export --armor $key | apt-key add -; done

# it works if the command finished and says things like:
#  gpg: requesting key 46925553 from hkp server pgpkeys.mit.edu
#  gpg: key 46925553: "Debian Archive Automatic Signing Key (7.0/wheezy)
#  <ftpmaster@debian.org>" not changed
#  gpg: Total number processed: 1
#  gpg:              unchanged: 1

# If doesnt work and stuck on
# gpg: requesting key 46925553 from hkp server pgpkeys.mit.edu

# Then copy key from this line
# "Processing key: 8B48AD6246925553"
# and google search "8B48AD6246925553" this will return keyserver, like below, put those in for --keyserver line

# And look for keyserver

# Works for debian: pgpkeys.mit.edu
# Works for ubuntu: subkeys.pgp.net


This will work for both, ubuntu and debian - vice versa.

Special thanks to :  http://ram.kossboss.com/fix-missing-gpg-key-apt-get/

Monday, 7 September 2015

Init Script Alternative

Install upstart package from repo, avaiable in both debian and centos.

To create a job to be started automatically when Ubuntu starts. As written example, suppose create the following file /etc/init/testservice.conf with sudo:

# testservice - test service job file

description "my service description"
author "Me <myself@i.com>"

# Stanzas
#
# Stanzas control when and how a process is started and stopped
# See a list of stanzas here: http://upstart.ubuntu.com/wiki/Stanzas

# When to start the service
start on runlevel [2345]

# When to stop the service
stop on runlevel [016]

# Automatically restart process if crashed
respawn

# Essentially lets upstart know the process will detach itself to the background
# This option does not seem to be of great importance, so it does not need to be set.
#expect fork

# Specify working directory
chdir /home/user/testcode

# Specify the process/command to start, e.g.
exec python mycommand.py arg1 arg2

To 'manually' start or stop the process use

sudo start testservice
sudo stop testservice

The above example works great for everything , but however, there is short cut too :

The solution is to use a shutdown-hook for the system. In Ubuntu it is very easy to add a shutdown-hook by using upstart.

The upstart job config file (/etc/init/shutdown-hook.conf) should look something like this:

description "run at shutdown"

start on starting rc
task
exec /bin/bash /etc/my_service/upload_log_to_s3.sh log

Finally I just needed the script (/etc/my_service/upload_log_to_s3.sh) for uploading a log file to S3:

#!/bin/bash

log_file_ext=$1

gzip -c /var/log/my_service/*.$log_file_ext > /tmp/log.gz

aws s3 cp /tmp/log.gz s3://my_service_bucket/logs/`date +%Y-%m-%dT%H:%M:%SZ`.log.gz

The upload script will just gzip the log file (needed as I’m using delaycompress), rename the log file to the current timestamp, and upload the file using aws-cli. The argument sets the file extension of the log file, which is necessary to be able to upload both the current (.log) as well as the previous log file (.log.1).

Sunday, 6 September 2015

Post HTTP API grafana Query



The details are as follows:


URL : http://www.nasheikh.com:3000/api/datasources?=
Option : Post
Headers :  Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
Payload : 

{
        "name":"test3",
        "type":"influxdb_08",
        "url":"http://localhost:8086",
        "access":"proxy",
        "basicAuth":false,
        "database":"server1",
        "user":"root",
        "password":"root"
}


Do this in one liner in curl :

curl -H "Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk" -H "Accept: application/json" -H "Content-type: application/json" -X POST -d @datainject.json http://www.nasheikh.com:3000/api/datasources?=

Here datainject.json is the file which is pasted above just saved as a json file and passed in curl.

Watch a file for its content, and perform an action accordingly:


Parent Script

#!/bin/bash

while true; do
  change=$(inotifywait -e close_write,moved_to,create .)
  change=${change#./ * }
  if [ "$change" = "changingfile.txt" ]; then ./child.sh; fi
done

Child Script

#!/bin/bash

rm -fr middle*
count=0
eraser=0
while read server
do
let count=$count+1
sed 's/server1/'${server}'/g;s/Tracker/'${server}'/g' center > middle.$count
sed 's/test1/'${server}'/g' datainject.json > newdata.$count.json
curl -H "Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk" -H "Accept: application/json" -H "Content-type: application/json" -X POST -d @newdata.$count.json http://www.nasheikh.com:3000/api/datasources?=
sleep 2
done < network.txt
eraser=$(echo middle.$count)
sed -i '$ s/.$//' $eraser
sleep 2
cat prepend middle* final > merged.json
curl -H "Authorization: Bearer eyJrIjoiNzNjc2s5amxkZW9aUE81UXV3dVVOcmRHNTJhRkVOY1oiLCJuIjoiYWRtaW4iLCJpZCI6MX0=" -H "Accept: application/json" -H "Content-type: application/json" -X POST -d @merged.json http://nasheikh.com:3000/api/dashboards/db
rm -fr middle* merged.json newdata*

========================================================================

Saturday, 5 September 2015

Update Server List from Amazon VPC

#!/bin/bash

aws ec2 describe-instances --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text --profile nash >> mainusa.txt

while true

do

aws ec2 describe-instances --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text --profile nash >> freshusa.txt

diff -Naur mainusa.txt freshusa.txt > my.patch

check=$(cat my.patch | wc -l)

echo $check

if [ "$check" -gt 0 ]

then

patch mainusa.txt < my.patch

else

echo "no chnage"

fi

rm -fr freshusa.txt my.patch

sleep 30

done

Wednesday, 19 August 2015

Random Stuff 2

Calculate Response time of web page

#!/bin/sh

while true
do
script -c "time -p wget http://52.20.204.148/ -q --output-document=/dev/null 2>/tmp/tmp.log" -f /tmp/tmp.log > /dev/null
sleep 5
NUM=$(cat /tmp/tmp.log | grep "real" | cut -d " " -f2 | tr -cd '[:digit:].')
curl -X POST -d '[{"name":"foo","columns":["val"],"points":[['$NUM']]}]' "http://52.20.204.148:8086/db/server1/series?u=root&p=root"
done

========================================================================

Type of web server response per minute

#!/bin/bash

while true
do
#set -x
mkdir -p /home/ec2-user/tmp/percfile
rm -f /home/ec2-user/tmp/percfile/*
dd=`date +%d`
mon=`date +%m`
yea=`date +%Y`
for (( i = 1; i >0; i-- ))
do
grep $(date +%d/%b/%Y:%H:%M -d "-$i  min") /var/log/httpd/access_log >> /home/ec2-user/tmp/percfile/log1mins
GOOD=$(grep "200" /home/ec2-user/tmp/percfile/log1mins | wc -l)
BAD=$(grep -v "200" /home/ec2-user/tmp/percfile/log1mins | wc -l)
curl -X POST -d '[{"name":"response","columns":["good","bad"],"points":[['$GOOD','$BAD']]}]' "http://localhost:8086/db/server1/series?u=root&p=root"
done
sleep 1m
done

========================================================================

No of hits to a web server per second

#!/bin/sh
frequency=1
lastCount=`wc -l /var/log/httpd/access_log | sed 's/\([0-9]*\).*/\1/'`
while true
do
newCount=`wc -l /var/log/httpd/access_log | sed 's/\([0-9]*\).*/\1/'`
diff=$(( newCount - lastCount ))
rate=$(echo "$diff / $frequency" |bc -l)
#echo $rate
curl -X POST -d '[{"name":"hps","columns":["val"],"points":[['$rate']]}]' "http://localhost:8086/db/server1/series?u=root&p=root"
lastCount=$newCount
sleep $frequency
done

========================================================================

Gets your public IP

#!/bin/bash
while true
do
sleep 1
myip=$(curl wgetip.com 2> /dev/null) > /dev/null
sleep 1
curl -X POST -d '[{"name":"response","columns":["val1"],"points":[["'$myip'"]]}]' "http://$myip:8086/db/server1/series?u=root&p=root"
sleep 1
done

========================================================================

Service status in boolean

#!/bin/bash
while true
do
sleep 1
pidof  httpd >/dev/null
if [[ $? -eq 0 ]] ; then
curl -X POST -d '[{"name":"response","columns":["val2"],"points":[[0]]}]' "http://localhost:8086/db/server1/series?u=root&p=root"
else
curl -X POST -d '[{"name":"response","columns":["val2"],"points":[[1]]}]' "http://localhost:8086/db/server1/series?u=root&p=root"
sleep 1
fi
done

========================================================================

Type of web server response per minute

#!/bin/sh

#while true
#do
NOW=$( tail -n1 /var/log/httpd/access_log | head -1 | awk -F'[][]' '{ gsub(/\//," ",$2); sub(/:/," ",$2); "date +%s -d \""$2"\""|getline d; print d;}' )
REF=$(( $NOW - 60 ))
NUM=$REF
LINE=1
COUNT=0
BAD=0
while [ $NUM -ge $REF ]
do
NUM=$( tail -n$LINE /var/log/httpd/access_log | head -1 | awk -F'[][]' '{ gsub(/\//," ",$2); sub(/:/," ",$2); "date +%s -d \""$2"\""|getline d; print d;}' )
tail -n$LINE /var/log/httpd/access_log | head -1 | grep -i "200" >/dev/null && let COUNT=$COUNT+1
tail -n$LINE /var/log/httpd/access_log | head -1 | grep -v "200" >/dev/null && let BAD=$BAD+1
let LINE=$LINE+1
done
let COUNT=$COUNT-1
let BAD=BAD-1
curl -X POST -d '[{"name":"response","columns":["good","bad"],"points":[['$COUNT','$BAD']]}]' "http://localhost:8086/db/server1/series?u=root&p=root"
#sleep 15m
#done

========================================================================

Sql access through php

<?php
$servername = "localhost";
$username = "root";
$password = "qwe";
$dbname = "grafana";
$homepage = file_get_contents("/home/comb.txt");

// Create connection
$conn = mysqli_connect($servername, $username, $password, $dbname);
// Check connection
if (!$conn) {
    die("Connection failed: " . mysqli_connect_error());
}

$sql = "UPDATE dashboard SET data='$homepage' WHERE slug='dash2'";

if (mysqli_query($conn, $sql)) {
    echo "Record updated successfully";
} else {
    echo "Error updating record: " . mysqli_error($conn);
}

mysqli_close($conn);
?>

========================================================================


Run local script on remote server ( with root priviliges )

Parent Script :

#!/bin/bash
SOURCEFILE=$1
SOURCEIP=$2
TARGETIP=$3
FILE=$(basename $1)
echo "Enter the path"
read path
scp -3 -r -i /root/asw/Admaxim-23-06-2015.pem ec2-user@$SOURCEIP:$SOURCEFILE ec2-user@$TARGETIP:/tmp    
ssh -t -t -i /root/asw/Admaxim-23-06-2015.pem ec2-user@$TARGETIP sudo -i "bash -s" -- < ./remold.sh "$FILE" "$path"
exit


Child Script :

#!/bin/bash
cd /tmp
yes | cp -r $1 $2
rm -fr $1
exit
exit

========================================================================

SSH into a remote server with non root account with login key and go root automatically :


#!/bin/bash
ssh -t -t -i /root/asw/keyfile.pem user@$1 sudo -i "bash -s"

========================================================================

Pass defined column value to loop  :


#!/bin/bash

while read server
do
echo ${server}
done < final | awk '{print $1}'

========================================================================

DB creation :


curl -X POST "http://localhost:8086/db?u=root&p=root" -d '{"name": "'${server}'"}'

========================================================================