Data Integrity Kafka

#!/bin/bash

brokers1=localhost-src:9092

brokers2=localhost-dest:9092

numberofpart=$(kafka-topics.sh --bootstrap-server $brokers1 --describe --topic $1 | grep PartitionCount | awk '{print $6}')

echo ""

echo "Topic $1 has total $numberofpart partition(s)"

echo ""

echo "Now Checking newest msg from every partition"

echo ""

echo "------------------------------------------------------------------------------------------------------------------"

printf "| Partition no |     Src Cluster Hash |      Dest Cluster Hash | Match/No Match | \n"

echo "------------------------------------------------------------------------------------------------------------------"

for (( c=0; c<$numberofpart; c++ ))

do

 srcvalue=$(kcat -q -C -b $brokers1 -t $1 -p $c -o -1 -e | md5sum | sed s/'\s'//g)

 destvalue=$(kcat -q -C -b $brokers2 -t $1 -p $c -o -1 -e | md5sum| sed s/'\s'//g)

 if [ "$srcvalue" = "$destvalue" ]; then

     echo "       $c     $srcvalue     $destvalue        Match"

 else

     echo "       $c     $srcvalue     $destvalue        No Match"

 fi

done

echo ""

echo "Now Checking oldest msg from every partition"

echo ""

echo "------------------------------------------------------------------------------------------------------------------"

printf "| Partition no |     Src Cluster Hash |      Dest Cluster Hash | Match/No Match | \n"

echo "------------------------------------------------------------------------------------------------------------------"

for (( c=0; c<$numberofpart; c++ ))

do

 srcvalue=$(kcat -q -C -b $brokers1 -t $1 -p $c -o beginning -c 1 -e | md5sum | sed s/'\s'//g)

 destvalue=$(kcat -q -C -b $brokers2 -t $1 -p $c -o beginning -c 1 -e | md5sum| sed s/'\s'//g)

 if [ "$srcvalue" = "$destvalue" ]; then

     echo "       $c    $srcvalue     $destvalue        Match"

 else

     echo "       $c    $srcvalue     $destvalue        No Match"

 fi

done

echo ""

Waf-v2 as module

module "aws_wafv2_web_acl_waf" {

  source  = "umotif-public/waf-webaclv2/aws"

  version = "~> 3.0.0"

  name_prefix            = "${var.environment}_WAF"

  description            = "${var.environment}_WAF"

  scope                  = var.scope

  create_alb_association = false

  allow_default_action   = true # set to allow if not specified

  tags = {

    product-name = "common"

  }

  visibility_config = {

    cloudwatch_metrics_enabled = true

    metric_name                = "${var.environment}_WAF"

    sampled_requests_enabled   = true

  }

  rules = [

    for rule in var.managed_rules :

    {

      name     = rule.rule_name

      priority = index(var.managed_rules[*].rule_name, rule.rule_name)

      override_action = rule.mode == "allow" ? "count" : "none"

      visibility_config = {

        cloudwatch_metrics_enabled = true

        metric_name                = rule.rule_name

        sampled_requests_enabled   = true

      }

      managed_rule_group_statement = {

        name          = rule.rule_name

        vendor_name   = rule.provider_name

        excluded_rule = rule.exceptions

      }

    }

  ]

}

-----------------------------------------------------------------------------------------------------------------------------

managed_rules = [

  {

    provider_name = "F5"

    rule_name     = "OWASP_Managed"

    exceptions    = [

      "rule_XSS_script_tag__Parameter__AllQueryArguments_Body",

      "rule_div_tag__behavior__Parameter__AllQueryArguments_Body",

      "rule_Java_code_injection___org_apache_commons_collections_AllQueryArguments_Body"

    ]

    mode          = "allow"

  },

   {

     provider_name = "AWS"

     rule_name     = "AWSManagedRulesAnonymousIpList"

     exceptions    = ["HostingProviderIPList"]

     mode          = "block"

   },

   {

     provider_name = "AWS"

     rule_name     = "AWSManagedRulesAmazonIpReputationList"

     exceptions    = []

     mode          = "block"

   }

]

-----------------------------------------------------------------------------------------------------------------------------

variable "environment" {

  description = "The name of the environment"

  type        = string

}

variable "scope" {

  description = "The name of resource CLOUDFRONT/REGIONAL only options (if regional declare in tf vars file)"

  type        = string

  default     = "CLOUDFRONT"

}

variable "managed_rules" {

  description = "the managed rules name in order (make sure not to cross 1500 wcu's)"

  type = list(object({

    provider_name = string

    rule_name     = string

    exceptions    = list(string)

    mode          = string

  }))

}

Waf-v2 as resource

resource "aws_wafv2_web_acl" "web_acl_rules" {

  name        = var.web_acl_name

  description = var.web_acl_name

  scope       = var.scope

  tags = {

    product-name = "common"

  }

  default_action {

    allow {

    }

  }

  visibility_config {

    cloudwatch_metrics_enabled = true

    metric_name                = var.web_acl_name

    sampled_requests_enabled   = true

  }

  dynamic "rule" {

    for_each = var.managed_rules

    content {

      name     = "${var.web_acl_name}-${rule.value.rule_name}"

      priority = rule.key

      override_action {

        dynamic "none" {

          for_each = rule.value.mode == "block" ? [1] : []

          content {}

        }

        dynamic "count" {

          for_each = rule.value.mode == "count" ? [1] : []

          content {}

        }

      }

      statement {

        managed_rule_group_statement {

          name        = rule.value.rule_name

          vendor_name = rule.value.provider_name

          dynamic "excluded_rule" {

            for_each = rule.value.exceptions

            content {

              name = excluded_rule.value

            }

          }

        }

      }

      visibility_config {

        cloudwatch_metrics_enabled = true

        metric_name                = "${var.web_acl_name}-${rule.value.rule_name}"

        sampled_requests_enabled   = true

      }

    }

  }

}

-----------------------------------------------------------------------------------------------------------------------------

scope = "REGIONAL"

managed_rules = [

  {

    provider_name = "F5"

    rule_name     = "OWASP_Managed"

    exceptions    = ["rule_XML_External_Entity__XXE__injection_attempt__Content__AllQueryArguments_Body"]

    mode          = "allow"

  },

  {

    provider_name = "AWS"

    rule_name     = "AWSManagedRulesAnonymousIpList"

    exceptions    = ["HostingProviderIPList"]

    mode          = "block"

  },

  {

    provider_name = "AWS"

    rule_name     = "AWSManagedRulesAmazonIpReputationList"

    exceptions    = []

    mode          = "block"

  }

]

-----------------------------------------------------------------------------------------------------------------------------

variable "environment" {

  description = "The name of the environment"

  type        = string

}

variable "scope" {

  description = "The name of resource CLOUDFRONT/REGIONAL only options (if regional declare in tf vars file)"

  type        = string

  default     = "CLOUDFRONT"

}

variable "managed_rules" {

  description = "the managed rules name in order (make sure not to cross 1500 wcu's)"

  type = list(object({

    provider_name = string

    rule_name     = string

    exceptions    = list(string)

    mode          = string

  }))

}

CircleCI Orbs

version: '2.1'

orbs:

  aws-s3: circleci/aws-s3@3.0

jobs:

  s3_sync_dev:

    docker:

      - image: 'cimg/python:3.10'

    resource_class: small

    steps:

      - checkout

      - aws-s3/sync:

          aws-access-key-id: ACCESS_ID_DEV

          aws-secret-access-key: SECRET_KEY_DEV

          aws-region: AWS_REGION_DEV

          from: .

          to: 's3://ansible-bucket-test'

      - run: aws --version

  s3_sync_prod:

    docker:

      - image: 'cimg/python:3.10'

    resource_class: small

    steps:

      - checkout

      - aws-s3/sync:

          aws-access-key-id: ACCESS_ID_PROD

          aws-secret-access-key: SECRET_KEY_PROD

          aws-region: AWS_REGION_PROD

          from: .

          to: 's3://ansible-bucket-prod'

      - run: aws --version

workflows:

  s3-execution:

    jobs:

      - s3_sync_dev:

          filters:

            branches:

              only: dev

      - s3_sync_prod:

          filters:

            branches:

              only: prod

Terraform user with secrets in aws secret manager

 data "aws_iam_policy_document" "ci_user_s3_policy" {

  statement {

    actions = [

      "s3:DeleteObject",

      "s3:DeleteObjectTagging",

      "s3:DeleteObjectVersion",

      "s3:DeleteObjectVersionTagging",

      "s3:ListBucket",

      "s3:GetObject*",

      "s3:PutObject*",

      "s3:ReplicateObject",

      "s3:RestoreObject"

    ]

    resources = [

      "arn:aws:s3:::ansible-bucket-${var.environment}/*",

      "arn:aws:s3:::ansible-bucket-${var.environment}"

    ]

  }

}

variable "environment" {

  type = string

}

variable "vaultpass" {

  type = string

}

resource "aws_s3_bucket" "ansible_bucket" {

  bucket        = "ansible-bucket-${var.environment}"

  acl           = "private"

  force_destroy = true

}

resource "aws_iam_user" "user" {

  name = "ansible-ci-upload"

}

resource "aws_iam_access_key" "ansible_repo" {

  user = aws_iam_user.user.name

}

resource "aws_iam_policy" "ci_user_s3_policy" {

  policy = data.aws_iam_policy_document.ci_user_s3_policy.json

}

resource "aws_iam_user_policy_attachment" "attach-policy" {

  user       = aws_iam_user.user.name

  policy_arn = aws_iam_policy.ci_user_s3_policy.arn

}

resource "aws_secretsmanager_secret" "ansible_credentials" {

  name = "ansible-circleci-user-creds"

}

resource "aws_secretsmanager_secret" "ansible_git_credentials" {

  name = "ansible-git-creds"

}

resource "aws_secretsmanager_secret_version" "ansible_credentials" {

  secret_id = aws_secretsmanager_secret.ansible_credentials.id

  secret_string = jsonencode({

    access_key    = aws_iam_access_key.ansible_repo.id

    access_secret = aws_iam_access_key.ansible_repo.secret

    vault_pass    = var.vaultpass

  })

}

resource "aws_secretsmanager_secret_version" "ansible_credentials_key" {

  secret_id     = aws_secretsmanager_secret.ansible_git_credentials.id

  secret_string = file("/mnt/workspace/AnsibleMaster.pem")

}

Move Existing data to Glacier

 #!/bin/bash

> filelist

aws sts get-caller-identity

TARGETBUCKET=$1

echo ''

echo $TARGETBUCKET

echo ''

aws s3 ls $TARGETBUCKET --recursive | awk '{ print $4 }' >> filelist

while read objname

do

 aws s3api copy-object --copy-source $TARGETBUCKET/${objname} --bucket $TARGETBUCKET --storage-class GLACIER --key ${objname}

done < filelist

aws s3api list-objects --bucket $TARGETBUCKET --query 'Contents[].{Key: Key, SC: StorageClass}' --output table

LVM Shorthand

 LVM Creation

sudo pvcreate /dev/sda /dev/sdb

sudo vgcreate LVMVolGroup /dev/sda /dev/sdb

sudo lvcreate -L 10G -n test1 LVMVolGroup

sudo lvcreate -l 100%FREE -n test2 LVMVolGroup

sudo mkfs -t ext4 /dev/LVMVolGroup/test1

sudo mkfs -t ext4 /dev/LVMVolGroup/test2

sudo mkdir /vol1

sudo mkdir /vol2

sudo mount /dev/LVMVolGroup/test1 /vol1

sudo mount /dev/LVMVolGroup/test2 /vol2

echo "/dev/LVMVolGroup/test1 /vol1 auto noatime 0 0" | sudo tee -a /etc/fstab

echo "/dev/LVMVolGroup/test2 /vol2 auto noatime 0 0" | sudo tee -a /etc/fstab

sudo mount -a

sudo reboot

Useful Commands:

pvdisplay

vgdiaplay

lvdisplay