Streamlining Kubernetes Log Retrieval with Python

Working with Kubernetes can often involve sifting through logs to troubleshoot or monitor applications. This can be cumbersome, especially when dealing with multiple pods across various namespaces. To simplify this process, I've developed a Python script that enhances the experience of fetching and managing Kubernetes pod logs.

The Challenge

Retrieving logs in Kubernetes typically requires using kubectl commands with specific flags for each pod and namespace. This manual process becomes inefficient and time-consuming, particularly when you need to access logs from multiple pods or wish to monitor logs in real time.

The Solution

To address these challenges, I've created a Python script that automates log retrieval, allowing for interactive selection of namespaces and pods, real-time log tailing, and the option to export logs with automatic naming and organization. Additionally, for users operating within Windows Subsystem for Linux (WSL2), the script includes functionality to open the file explorer directly to the log file's location, bridging the gap between the Linux command line and the Windows graphical interface.

Key Features

• Interactive Namespace and Pod Selection: Choose which pod's logs to fetch without manually typing namespace and pod names.
• Real-time Log Tailing: Option to tail logs for ongoing monitoring.
• Automatic Log Export: Export logs to a timestamped file for easy organization and later review.
• File Explorer Integration: For WSL2 users, the script can open Windows Explorer to the directory containing the exported log files, streamlining access to logs.

Requirements

• Python 3
• Kubernetes Python client (kubernetes)
• questionary for interactive prompts

Before running the script, ensure you have the necessary packages installed:

pip install kubernetes questionary

The Script

# Import necessary libraries
import os
import sys
import subprocess
from datetime import datetime
from kubernetes import client, config
import questionary

# Function definitions for loading kube config, listing namespaces and pods,
# and the enhanced get_pod_logs function

def load_kube_config(custom_kubeconfig=None):
    # Load Kubernetes configuration
    if custom_kubeconfig:
        config.load_kube_config(config_file=custom_kubeconfig)
    else:
        config.load_kube_config()

def list_namespaces(api_instance):
    # Return a list of namespaces in the cluster
    try:
        namespaces = api_instance.list_namespace()
        return [namespace.metadata.name for namespace in namespaces.items]
    except client.rest.ApiException as e:
        print(f"Failed to list namespaces: {e}")
        exit(1)

def list_pods(api_instance, namespace):
    # Return a list of pods in a specified namespace
    try:
        pods = api_instance.list_namespaced_pod(namespace)
        return [pod.metadata.name for pod in pods.items]
    except client.rest.ApiException as e:
        print(f"Failed to list pods in namespace '{namespace}': {e}")
        exit(1)

def get_pod_logs(api_instance, namespace, pod_name, tail=False, export=False):
    # Fetch and print or export logs from a specific pod
    try:
        logs = api_instance.read_namespaced_pod_log(name=pod_name, namespace=namespace, follow=tail, _preload_content=False)
        if export:
            # Define the directory to save logs
            log_directory = os.path.expanduser('~/K8s_Logs')
            os.makedirs(log_directory, exist_ok=True)
            # Generate a file name
            timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
            file_name = f"{namespace}_{pod_name}_{timestamp}.log"
            export_path = os.path.join(log_directory, file_name)
            with open(export_path, 'w') as file:
                for line in logs.stream():
                    file.write(line.decode('utf-8'))
            print(f"Logs have been exported to {export_path}")
            # Open file explorer to the log directory (Adjustment for WSL2 users included)
            if os.name == 'nt':  # Windows
                subprocess.run(['explorer', os.path.normpath(log_directory)])
            elif 'microsoft' in os.uname().release:  # WSL2
                linux_path = log_directory.replace('/home', 'home')
                wsl_path = '\\\\wsl$\\Ubuntu\\' + linux_path.replace('/', '\\')
                subprocess.run(['explorer.exe', wsl_path])
            else:
                try:
                    subprocess.run(['xdg-open', log_directory])
                except Exception:
                    print("Could not automatically open the file explorer.")
        elif tail:
            print(f"Tailing logs for pod {pod_name} in namespace {namespace}:")
            for line in logs.stream():
                print(line.decode('utf-8'), end='')
        else:
            print(f"Logs for pod {pod_name} in namespace {namespace}:\n{logs.read().decode('utf-

8')}")
    except client.rest.ApiException as e:
        print(f"Failed to retrieve logs: {e}")
        exit(1)

# Main script execution for interactive selections and log management

if __name__ == "__main__":
    custom_kubeconfig = questionary.text("Enter custom kubeconfig path (leave blank for default):").ask()
    load_kube_config(custom_kubeconfig)
    v1 = client.CoreV1Api()
    namespaces = list_namespaces(v1)
    namespace = questionary.select("Select a namespace:", choices=namespaces).ask()
    pods = list_pods(v1, namespace)
    pod_name = questionary.select("Select a pod:", choices=pods).ask()
    tail_logs = questionary.confirm("Tail logs?").ask()
    export_logs = not tail_logs and questionary.confirm("Export logs to a file?").ask()
    get_pod_logs(v1, namespace, pod_name, tail=tail_logs, export=export_logs)

Conclusion

This Python script represents a significant step forward in simplifying Kubernetes log management. By automating and enhancing the process of fetching logs, it saves time and increases efficiency, allowing developers and system administrators to focus on more critical tasks. Whether you're troubleshooting an application or simply need to keep an eye on your services, this tool provides a streamlined and user-friendly approach to handling Kubernetes logs.