Chapter 1: Linux Kernel — The Heart of DevOps

1.1 Essential System Information Commands

$ uname -r
5.15.0-91-generic
$ cat /etc/os-release
NAME="Ubuntu" VERSION="22.04.3 LTS"
$ hostnamectl
Static hostname: ubuntu-server

$ cat server-audit.sh
#!/bin/bash
echo "=== SERVER AUDIT REPORT ==="
echo "Hostname: $(hostname)"
echo "Kernel: $(uname -r)"
echo "OS: $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)"
echo "CPU Cores: $(nproc)"
echo "RAM: $(free -h | awk '/^Mem:/ {print $2}')"
echo "Disk /: $(df -h / | awk 'NR==2 {print $5}')"
echo "Uptime: $(uptime -p)"

$ cat kernel-compliance.sh
#!/bin/bash
KERNEL=$(uname -r | cut -d'-' -f1)
REQUIRED="5.15"
if [[ "$(printf '%s\n' "$REQUIRED" "$KERNEL" | sort -V | head -n1)" != "$REQUIRED" ]]; then
  echo "❌ FAIL: Kernel $KERNEL is below $REQUIRED"
  exit 1
else
  echo "✅ PASS: Kernel $KERNEL meets requirement"
  exit 0
fi

Chapter 2: Linux Filesystem Hierarchy & Permission Octal System

$ sudo find /var/www/html -type f -exec chmod 644 {} \;
$ sudo find /var/www/html -type d -exec chmod 755 {} \;
$ sudo chown -R www-data:www-data /var/www/html
$ sudo chmod 600 /var/www/html/wp-config.php
$ sudo chmod 640 /var/www/html/.htaccess

$ sudo groupadd devteam
$ sudo usermod -aG devteam alice
$ sudo usermod -aG devteam bob
$ sudo usermod -aG devteam carol
$ sudo mkdir -p /opt/project
$ sudo chown -R :devteam /opt/project
$ sudo chmod 2775 /opt/project
$ sudo chmod 664 /opt/project/*
$ umask 002 # Add to each user's .bashrc

Chapter 3: Bash Scripting — Variables, Conditionals, Loops, Functions

$ cat deploy-manager.sh
#!/bin/bash
set -euo pipefail
APP_DIR="/opt/myapp"
BACKUP_DIR="/opt/backups"
VERSION=$(date +%Y%m%d_%H%M%S)

# Backup current version before deployment
backup() {
  tar -czf "$BACKUP_DIR/app_$VERSION.tar.gz" -C "$APP_DIR" .
  echo "✅ Backup created: app_$VERSION.tar.gz"
}

# Keep only last 5 backups
cleanup() {
  ls -t $BACKUP_DIR/app_*.tar.gz | tail -n +6 | xargs -r rm -f
}

# Rollback to specific version
rollback() {
  local backup_file=$1
  if [ ! -f "$BACKUP_DIR/$backup_file" ]; then
    echo "❌ Backup $backup_file not found!"
    return 1
  fi
  rm -rf $APP_DIR/*
  tar -xzf "$BACKUP_DIR/$backup_file" -C "$APP_DIR"
  sudo systemctl restart myapp
  echo "✅ Rollback to $backup_file complete"
}

case ${1:-} in
  backup) backup; cleanup;;
  rollback) rollback "$2";;
  *) echo "Usage: $0 {backup|rollback }"; exit 1;;
esac

$ cat log-rotator.sh
#!/bin/bash
LOG_FILE="/var/log/myapp/application.log"
MAX_SIZE_MB=100
MAX_SIZE_BYTES=$((MAX_SIZE_MB * 1024 * 1024))

if [ -f "$LOG_FILE" ]; then
  CURRENT_SIZE=$(stat -c%s "$LOG_FILE")
  if [ $CURRENT_SIZE -gt $MAX_SIZE_BYTES ]; then
    TIMESTAMP=$(date +%Y%m%d_%H%M%S)
    mv "$LOG_FILE" "${LOG_FILE}.${TIMESTAMP}"
    gzip "${LOG_FILE}.${TIMESTAMP}"
    touch "$LOG_FILE"
    chown appuser:appuser "$LOG_FILE"
    chmod 644 "$LOG_FILE"
    # Send SIGHUP to reopen log file (for apps that support it)
    kill -HUP $(pgrep -f "myapp") 2>/dev/null || true
    echo "$(date): Log rotated (was ${CURRENT_SIZE} bytes)" >> /var/log/log-rotator.log
  fi
fi

# Delete logs older than 30 days
find /var/log/myapp/ -name "*.gz" -mtime +30 -delete

Chapter 4: User Management, Sudo Access & SSH Hardening

# On Bastion Host:
$ sudo useradd -m -s /bin/bash jumpser
$ sudo mkdir -p /home/jumpser/.ssh
$ sudo chmod 700 /home/jumpser/.ssh
$ echo "AllowUsers jumpser" | sudo tee -a /etc/ssh/sshd_config
$ sudo systemctl restart sshd

# On Client Machine (~/.ssh/config):
Host bastion
    HostName bastion.company.com
    User jumpser
    IdentityFile ~/.ssh/id_ed25519

Host 10.0.*.*
    ProxyJump bastion
    User ubuntu
    IdentityFile ~/.ssh/id_ed25519

# Now connect directly:
$ ssh 10.0.1.10 # Automatically tunnels through bastion!

$ cat rotate-ssh-keys.sh
#!/bin/bash
set -e
KEY_COMMENT="devops-$(date +%Y%m%d)"
KEY_PATH="$HOME/.ssh/id_ed25519_$KEY_COMMENT"

# Generate new key pair
ssh-keygen -t ed25519 -f "$KEY_PATH" -N "" -C "$KEY_COMMENT"

# Deploy to all servers
for server in $(cat servers.txt); do
  echo "Rotating keys on $server..."
  ssh-copy-id -i "${KEY_PATH}.pub" "$server"
  # Remove old keys (older than 30 days by comment pattern)
  ssh "$server" "sed -i '/devops-/d' ~/.ssh/authorized_keys"
  # Verify we can still connect
  ssh -i "$KEY_PATH" "$server" "echo '✅ Connected with new key'"
done

# Update local SSH config to use new key
sed -i "s|IdentityFile.*id_ed25519|IdentityFile $KEY_PATH|g" ~/.ssh/config
echo "✅ Key rotation complete. Old keys removed from all servers."

Chapter 5: Process Management, Signals & systemd Services

$ sudo nano /etc/systemd/system/myapp.service
[Unit]
Description=My Node.js Application
After=network.target
Wants=docker.service

[Service]
Type=simple
User=appuser
Group=appuser
WorkingDirectory=/opt/myapp
ExecStart=/usr/bin/node /opt/myapp/server.js
Environment="NODE_ENV=production"
Environment="PORT=8080"
Restart=on-failure
RestartSec=10
StartLimitBurst=5
StartLimitIntervalSec=60
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

$ sudo systemctl daemon-reload
$ sudo systemctl enable --now myapp
$ sudo systemctl status myapp

$ cat /opt/scripts/memory-monitor.sh
#!/bin/bash
SERVICE_NAME="myapp"
MEMORY_LIMIT_MB=2048
PID=$(pgrep -f "node.*server.js" | head -1)

if [ -n "$PID" ]; then
  MEM_KB=$(ps -o rss= -p $PID 2>/dev/null)
  if [ -n "$MEM_KB" ]; then
    MEM_MB=$((MEM_KB / 1024))
    if [ $MEM_MB -gt $MEMORY_LIMIT_MB ]; then
      echo "$(date): Memory limit exceeded (${MEM_MB}MB > ${MEMORY_LIMIT_MB}MB), restarting..."
      sudo systemctl restart $SERVICE_NAME
    fi
  fi
fi

# Schedule with cron every 5 minutes
$ sudo crontab -e
*/5 * * * * /opt/scripts/memory-monitor.sh >> /var/log/memory-monitor.log 2>&1

Chapter 6: Networking — IP, Ports, DNS, Firewall (UFW/iptables)

# Basic UFW rate limiting for SSH
$ sudo ufw limit ssh

# Advanced iptables rate limiting for HTTP/HTTPS
$ sudo iptables -A INPUT -p tcp --dport 80 -m limit --limit 25/minute --limit-burst 50 -j ACCEPT
$ sudo iptables -A INPUT -p tcp --dport 80 -j DROP
$ sudo iptables -A INPUT -p tcp --dport 443 -m limit --limit 25/minute --limit-burst 50 -j ACCEPT
$ sudo iptables -A INPUT -p tcp --dport 443 -j DROP

# Protect against SYN flood specifically
$ sudo iptables -A INPUT -p tcp --syn -m limit --limit 12/s --limit-burst 24 -j ACCEPT
$ sudo iptables -A INPUT -p tcp --syn -j DROP

# Save iptables rules persistently
$ sudo apt install iptables-persistent
$ sudo netfilter-persistent save

# Add internal service mappings
$ echo "10.0.1.10 db.postgres.internal" | sudo tee -a /etc/hosts
$ echo "10.0.1.20 cache.redis.internal" | sudo tee -a /etc/hosts
$ echo "10.0.1.30 api.auth.internal" | sudo tee -a /etc/hosts

# Test connectivity
$ ping -c 2 db.postgres.internal
$ nc -zv db.postgres.internal 5432

# Ansible playbook to distribute hosts file across cluster
$ cat distribute-hosts.yml
- hosts: all
  tasks:
    - name: Add internal service mappings
      lineinfile:
        path: /etc/hosts
        line: "{{ item }}"
      loop:
        - "10.0.1.10 db.postgres.internal"
        - "10.0.1.20 cache.redis.internal"

Chapter 7: Package Management — apt, yum, dnf, and DevOps Toolchain

# On internet-connected machine (download only)
$ sudo apt-get update
$ sudo apt-get install --download-only nginx docker.io postgresql-14 redis
$ sudo tar -czf offline-packages.tar.gz /var/cache/apt/archives/*.deb

# Transfer tarball to air-gapped server (via USB/DVD)
$ sudo tar -xzf offline-packages.tar.gz -C /var/cache/apt/archives/
$ sudo dpkg -i /var/cache/apt/archives/*.deb

# Handle dependencies automatically
$ sudo apt-get install --fix-broken
$ sudo apt-get install -f

# For RPM-based systems (CentOS/RHEL)
$ sudo yum install --downloadonly --downloaddir=/tmp/packages nginx docker
$ sudo rpm -ivh /tmp/packages/*.rpm

$ cat provision-devops-tools.sh
#!/bin/bash
set -e

# Update system
sudo apt update && sudo apt upgrade -y
sudo apt install -y curl wget git vim jq unzip htop net-tools

# Install Docker
curl -fsSL https://get.docker.com | bash
sudo usermod -aG docker $USER

# Install kubectl
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

# Install Terraform
wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp.gpg
echo "deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
sudo apt update && sudo apt install terraform -y

# Install Helm
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

# Install AWS CLI v2
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip && sudo ./aws/install

# Install Ansible
sudo apt install -y python3-pip
pip3 install ansible

# Verify installations
echo "✅ Docker: $(docker --version)"
echo "✅ kubectl: $(kubectl version --client)"
echo "✅ Terraform: $(terraform version)"
echo "✅ Helm: $(helm version)"
echo "✅ AWS CLI: $(aws --version)"
echo "✅ Ansible: $(ansible --version | head -1)"

echo "🎉 DevOps workstation provisioned! Please log out and back in for Docker permissions."

Chapter 8: Cron Jobs — Automating Scheduled Tasks

$ cat /opt/scripts/postgres-backup.sh
#!/bin/bash
BACKUP_DIR="/backups/postgres"
DB_NAME="production_db"
DB_USER="backup_user"
DATE=$(date +%Y%m%d_%H%M%S)
mkdir -p "$BACKUP_DIR"

# Perform backup
PGPASSWORD="secret" pg_dump -U $DB_USER -h localhost $DB_NAME | gzip > "$BACKUP_DIR/db_${DATE}.sql.gz"

# Delete backups older than 7 days
find "$BACKUP_DIR" -name "db_*.sql.gz" -mtime +7 -delete

# Optional: Upload to S3 (requires AWS CLI configured)
# aws s3 cp "$BACKUP_DIR/db_${DATE}.sql.gz" s3://mybucket/backups/

echo "$(date): Backup completed - db_${DATE}.sql.gz" >> /var/log/backup.log

# Add to crontab
$ sudo crontab -e
# Daily backup at 2:00 AM
0 2 * * * /opt/scripts/postgres-backup.sh >> /var/log/backup.log 2>&1

# Health check every 5 minutes (monitoring)
*/5 * * * * curl -fsS https://hc-ping.com/your-uuid || /opt/scripts/alert.sh

$ sudo crontab -e
# Check certificate renewal twice daily (at 12 AM and 12 PM)
0 0,12 * * * /usr/bin/certbot renew --quiet --deploy-hook "systemctl reload nginx" --pre-hook "echo 'Renewal starting' >> /var/log/certbot.log" >> /var/log/certbot-renew.log 2>&1

# Alternative: Weekly renewal check with email notification
0 2 * * 0 /usr/bin/certbot renew --quiet && echo "SSL renewed on $(date)" | mail -s "SSL Certificate Status" admin@company.com

# For wildcard certificates (DNS challenge)
$ cat /opt/scripts/dns-renew.sh
#!/bin/bash
certbot renew --manual --preferred-challenges dns --quiet
systemctl reload nginx
systemctl reload haproxy

Chapter 9: Log Analysis — grep, sed, awk Mastery

$ cat /opt/scripts/nginx-error-analyzer.sh
#!/bin/bash
LOG_FILE="/var/log/nginx/access.log"
REPORT_DIR="/var/reports"
DATE=$(date +%Y%m%d_%H%M%S)
mkdir -p "$REPORT_DIR"

# Total 5xx errors in last hour
ERROR_COUNT=$(awk -v date="$(date --date='1 hour ago' '+%d/%b/%Y:%H')" '$4 ~ date && $9 >= 500' $LOG_FILE | wc -l)

if [ $ERROR_COUNT -gt 100 ]; then
  echo "ALERT: High 5xx error rate: $ERROR_COUNT errors in last hour" | mail -s "⚠️ High Error Rate" oncall@company.com
fi

# Top 10 failing endpoints (5xx)
awk '$9 >= 500 {print $7, $9}' $LOG_FILE | sort | uniq -c | sort -rn | head -10 > "$REPORT_DIR/failing_endpoints_$DATE.txt"

# Top 10 IP addresses causing 5xx
awk '$9 >= 500 {print $1}' $LOG_FILE | sort | uniq -c | sort -rn | head -10 > "$REPORT_DIR/top_ips_$DATE.txt"

# Real-time monitoring with tail -f and grep
# Use: tail -f /var/log/nginx/access.log | grep --line-buffered "HTTP/1.1\" 5[0-9][0-9]"

# One-liner for quick analysis
$ awk '$9 == 404 {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head -20

$ cat /opt/scripts/ssh-attack-detector.sh
#!/bin/bash
AUTH_LOG="/var/log/auth.log"
THRESHOLD=10
TIME_WINDOW=5 # minutes

# Extract failed SSH attempts in last 5 minutes
FAILED_IPS=$(grep "Failed password" $AUTH_LOG | grep "$(date --date="$TIME_WINDOW minutes ago" '+%b %d %H:%M')" | awk '{print $(NF-3)}' | sort | uniq -c | awk -v t=$THRESHOLD '$1 > t {print $2}')

# Block attacking IPs using ufw
for IP in $FAILED_IPS; do
  if ! sudo ufw status | grep -q "$IP"; then
    sudo ufw deny from $IP to any
    echo "$(date): Blocked attacking IP $IP" >> /var/log/ssh-block.log
  fi
done

# Alternative using sed to anonymize IPs in logs
sed -i 's/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/REDACTED/g' $AUTH_LOG

# Count unique attackers per day
grep "Failed password" $AUTH_LOG | awk '{print $(NF-3)}' | sort | uniq -c | sort -rn | head -20

# Schedule to run every 5 minutes
$ sudo crontab -e
*/5 * * * * /opt/scripts/ssh-attack-detector.sh

Chapter 10: LVM (Logical Volume Manager) — Flexible Storage Management

# Step 1: Add new disk to VM (e.g., /dev/sdb from cloud console)
# Step 2: Create physical volume
$ sudo pvcreate /dev/sdb
$ sudo pvs # Verify PV created

# Step 3: Extend volume group (assuming existing VG named 'vg_data')
$ sudo vgextend vg_data /dev/sdb
$ sudo vgs # Verify free space in VG

# Step 4: Extend logical volume (add 20GB)
$ sudo lvextend -L +20G /dev/vg_data/lv_postgres
$ # Or extend to use all free space: lvextend -l +100%FREE /dev/vg_data/lv_postgres

# Step 5: Resize filesystem online (no unmount needed!)
$ sudo resize2fs /dev/vg_data/lv_postgres # For ext4
# For XFS: sudo xfs_growfs /mount/point

# Step 6: Verify new size
$ df -h /var/lib/postgresql
$ sudo lvdisplay /dev/vg_data/lv_postgres

# Create LVM snapshot for backup (point-in-time)
$ sudo lvcreate -L 10G -s -n lv_postgres_snap /dev/vg_data/lv_postgres
$ sudo mount /dev/vg_data/lv_postgres_snap /mnt/snapshot
$ # Backup from snapshot, then remove: sudo lvremove vg_data/lv_postgres_snap

# Create thin pool (allocate metadata and data)
$ sudo pvcreate /dev/sdb /dev/sdc
$ sudo vgcreate vg_container /dev/sdb /dev/sdc
$ sudo lvcreate -L 1G -n thin_pool_meta vg_container
$ sudo lvcreate -L 100G -n thin_pool_data vg_container
$ sudo lvconvert --type thin-pool --poolmetadata vg_container/thin_pool_meta vg_container/thin_pool_data

# Create thin volumes (over-provisioned)
$ sudo lvcreate -V 50G -T vg_container/thin_pool_data -n container_vol1
$ sudo lvcreate -V 50G -T vg_container/thin_pool_data -n container_vol2
$ sudo lvcreate -V 50G -T vg_container/thin_pool_data -n container_vol3

# Format and mount
$ sudo mkfs.ext4 /dev/vg_container/container_vol1
$ sudo mount /dev/vg_container/container_vol1 /var/lib/docker

# Monitor thin pool usage
$ sudo lvs -a vg_container
$ sudo lvdisplay --maps vg_container/thin_pool_data

# Extend thin pool when data usage > 80%
$ sudo pvcreate /dev/sdd
$ sudo vgextend vg_container /dev/sdd
$ sudo lvextend -L +50G vg_container/thin_pool_data

Chapter 11: Environment Variables & Configuration Management (12-Factor App)

# Directory structure
/opt/myapp/
├── .env.base
├── .env.development
├── .env.staging
├── .env.production
└── load-config.sh

$ cat .env.base
APP_NAME=MyApplication
LOG_FORMAT=json

$ cat .env.development
APP_ENV=development
DATABASE_URL=postgresql://localhost/dev_db
DEBUG=true
LOG_LEVEL=debug

$ cat .env.production
APP_ENV=production
DATABASE_URL=postgresql://prod-db.internal:5432/app_prod
DEBUG=false
LOG_LEVEL=warn
API_KEY=prod-secret-key

$ cat load-config.sh
#!/bin/bash
ENV=${1:-development}
CONFIG_DIR="/opt/myapp"

if [ ! -f "$CONFIG_DIR/.env.$ENV" ]; then
  echo "❌ Environment file .env.$ENV not found!"
  exit 1
fi

# Load base then environment-specific (override)
set -a
source "$CONFIG_DIR/.env.base"
source "$CONFIG_DIR/.env.$ENV"
set +a

# Validate required variables
for var in DATABASE_URL APP_ENV; do
  if [ -z "${!var}" ]; then
    echo "❌ Required variable $var is not set for $ENV environment!"
    exit 1
  fi
done

echo "✅ Loaded configuration for $ENV environment"
echo "   DATABASE_URL: ${DATABASE_URL}"

# Usage: source load-config.sh production
# Then run: node server.js

# Docker run with env file
$ docker run --env-file .env.production myapp:latest

# Kubernetes secret from env file
$ kubectl create secret generic app-config --from-env-file=.env.production

# Start Vault dev server (testing only)
$ vault server -dev
$ export VAULT_ADDR='http://127.0.0.1:8200'

# Store a secret
$ vault kv put secret/myapp DATABASE_PASSWORD="s3cr3t" API_KEY="abc123"

# Retrieve secret
$ vault kv get -field=DATABASE_PASSWORD secret/myapp

# Application script to fetch secrets
$ cat /opt/scripts/fetch-secrets.sh
#!/bin/bash
export DATABASE_PASSWORD=$(vault kv get -field=DATABASE_PASSWORD secret/myapp)
export API_KEY=$(vault kv get -field=API_KEY secret/myapp)
exec node /opt/myapp/server.js

# systemd service with Vault integration
$ sudo nano /etc/systemd/system/myapp.service
[Service]
Environment="VAULT_ADDR=http://vault.internal:8200"
ExecStartPre=/usr/local/bin/vault login -method=token token=$(cat /etc/vault-token)
ExecStart=/opt/scripts/fetch-secrets.sh

# Environment variable best practices
$ # Never hardcode secrets in scripts
$ # Never commit .env files to git
$ echo ".env" >> .gitignore
$ echo "*.env" >> .gitignore
$ echo "secrets/" >> .gitignore

# Validate no secrets in git history
$ git log -p | grep -i "password\|secret\|key\|token"