Bits of .NET

🔍 ELK Stack = Log Intelligence

Raw logs are useless. ELK Stack turns container logs into actionable intelligence for monitoring and debugging.

📝 Docker Compose ELK Setup

# docker-compose-elk.yml
version: '3.8'

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0
    environment:
      - discovery.type=single-node
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - xpack.security.enabled=false
    volumes:
      - elasticsearch-data:/usr/share/elasticsearch/data
    ports:
      - 9200:9200

  logstash:
    image: docker.elastic.co/logstash/logstash:8.10.0
    volumes:
      - ./logstash/pipeline:/usr/share/logstash/pipeline
      - /var/run/docker.sock:/var/run/docker.sock
    ports:
      - 5000:5000/tcp
      - 5000:5000/udp
      - 9600:9600
    depends_on:
      - elasticsearch

  kibana:
    image: docker.elastic.co/kibana/kibana:8.10.0
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    ports:
      - 5601:5601
    depends_on:
      - elasticsearch

  filebeat:
    image: docker.elastic.co/beats/filebeat:8.10.0
    user: root
    volumes:
      - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - /var/run/docker.sock:/var/run/docker.sock
    command: filebeat -e -strict.perms=false
    depends_on:
      - logstash

🎯 Logstash Pipeline

# logstash/pipeline/logstash.conf
input {
    beats { port => 5044 }
    tcp { port => 5000 codec => json_lines }
    udp { port => 5000 codec => json_lines }
}

filter {
    if [@metadata][beat] == filebeat {
        json { source => message target => docker }
        mutate { add_field => { container_name => %{[docker][container][name]} } }
        date { match => [ [docker][time], ISO8601 ] target => @timestamp }
    }
    
    if [container_name] =~ /^prod-/ {
        mutate { add_tag => production }
    } else if [container_name] =~ /^staging-/ {
        mutate { add_tag => staging }
    } else {
        mutate { add_tag => development }
    }
    
    grok {
        match => { 
            message => %{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:loglevel} \[%{DATA:thread}\] %{DATA:class} - %{GREEDYDATA:log_message}
        }
    }
    
    mutate { remove_field => [host, ecs, agent] }
}

output {
    elasticsearch {
        hosts => [elasticsearch:9200]
        index => logstash-%{+YYYY.MM.dd}
    }
    stdout { codec => rubydebug }
}

💡 ELK Best Practices

• Use index patterns for efficient searching
• Implement index lifecycle management
• Set up email alerts for critical errors
• Use Kibana dashboards for monitoring
• Implement log retention policies