找出占用Kubernetes节点磁盘的Pod

有部分开发不规范导致日志写入容器, 在 k8s 环境下导致节点磁盘空间占用过高, 解决方案有:

1. 通过监控告警来提前预防
2. 挂载磁盘, 容器日志写入磁盘
3. 使用自动清理脚本

kubectl 查询每个 pod 占用磁盘空间

使用 python 处理所有节点

import subprocess
import json
import sys
from collections import defaultdict

def get_all_nodes():
    """
    获取集群中所有节点名称
    """
    cmd = ["kubectl", "get", "nodes", "-o", "json"]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, check=True)
        data = json.loads(result.stdout)
        nodes = [item["metadata"]["name"] for item in data.get("items", [])]
        return nodes
    except subprocess.CalledProcessError as e:
        print(f"获取节点列表失败: {e.stderr}", file=sys.stderr)
        sys.exit(1)
    except json.JSONDecodeError as e:
        print(f"解析节点列表 JSON 失败: {e}", file=sys.stderr)
        sys.exit(1)

def get_node_stats_summary(node_name):
    """
    通过 kubectl 获取指定节点的 stats summary
    """
    cmd = [
        "kubectl",
        "get",
        "--raw",
        f"/api/v1/nodes/{node_name}/proxy/stats/summary"
    ]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, check=True)
        return json.loads(result.stdout)
    except subprocess.CalledProcessError as e:
        print(f"节点 {node_name} 获取失败: {e.stderr}", file=sys.stderr)
        return None
    except json.JSONDecodeError as e:
        print(f"节点 {node_name} 解析 JSON 失败: {e}", file=sys.stderr)
        return None

def format_pod_stats(data, node_name):
    """
    提取并格式化输出每个 Pod 的名称和 rootfs.usedBytes
    """
    pods = data.get("pods", [])
    if not pods:
        return []

    pod_stats = []
    for pod in pods:
        pod_name = pod.get("podRef", {}).get("name", "unknown")
        namespace = pod.get("podRef", {}).get("namespace", "unknown")
        containers = pod.get("containers", [])

        total_used_bytes = 0
        container_details = []
        
        for container in containers:
            rootfs = rootfs = container.get("rootfs", {})
            used_bytes = rootfs.get("usedBytes", 0)
            total_used_bytes += used_bytes
            container_details.append({
                "container_name": container.get("name", "unknown"),
                "used_bytes": used_bytes
            })

        pod_stats.append({
            "node": node_name,
            "namespace": namespace,
            "pod_name": pod_name,
            "total_used_bytes": total_used_bytes,
            "containers": container_details
        })

    return pod_stats

def format_bytes(bytes_value):
    """
    将字节数转换为人类可读的格式
    """
    if bytes_value is None:
        return "N/A"
    
    units = ["B", "KiB", "MiB", "GiB", "TiB"]
    index = 0
    size = float(bytes_value)
    
    while size >= 1024 and index < len(units) - 1:
        size /= 1024
        index += 1
    
    return f"{size:.2f} {units[index]}"

def print_summary(all_pod_stats):
    """
    打印汇总统计信息
    """
    if not all_pod_stats:
        print("没有找到任何 Pod 信息")
        return

    # 按节点分组统计
    node_summary = defaultdict(lambda: {"pod_count": 0, "total_storage": 0})
    
    print("\n" + "=" * 100)
    print("所有节点 Pod 存储使用统计")
    print("=" * 100)
    
    for pod_stat in all_pod_stats:
        node = pod_stat["node"]
        pod_name = pod_stat["pod_name"]
        namespace = pod_stat["namespace"]
        total_used = pod_stat["total_used_bytes"]
        
        node_summary[node]["pod_count"] += 1
        node_summary[node]["total_storage"] += total_used
        
        # 打印每个 Pod 的详细信息
        print(f"\n📦 Pod: {pod_name}")
        print(f"   Namespace: {namespace}")
        print(f"   Node: {node}")
        print(f"   Total Storage Used: {format_bytes(total_used)}")
        
        # 打印容器详细信息
        for container in pod_stat["containers"]:
            print(f"   └─ Container: {container['container_name']}")
            print(f"      Used Bytes: {format_bytes(container['used_bytes'])}")
        
        print("-" * 80)
    
    # 打印节点汇总
    print("\n" + "=" * 100)
    print("节点汇总统计")
    print("=" * 100)
    
    total_pods = 0
    total_storage = 0
    
    for node, summary in sorted(node_summary.items()):
        print(f"\n🖥️  Node: {node}")
        print(f"   Pod 数量: {summary['pod_count']}")
        print(f"   总存储使用: {format_bytes(summary['total_storage'])}")
        
        total_pods += summary['pod_count']
        total_storage += summary['total_storage']
    
    # 打印全局汇总
    print("\n" + "=" * 100)
    print("全局汇总")
    print("=" * 100)
    print(f"📊 总节点数: {len(node_summary)}")
    print(f"📊 总 Pod 数: {total_pods}")
    print(f"📊 总存储使用: {format_bytes(total_storage)}")
    print("=" * 100)

def main():
    print("正在获取集群节点列表...")
    nodes = get_all_nodes()
    print(f"找到 {len(nodes)} 个节点: {', '.join(nodes)}\n")
    
    all_pod_stats = []
    
    for i, node in enumerate(nodes, 1):
        print(f"[{i}/{len(nodes)}] 正在获取节点 {node} 的统计数据...")
        data = get_node_stats_summary(node)
        
        if data:
            pod_stats = format_pod_stats(data, node)
            all_pod_stats.extend(pod_stats)
            print(f"   找到 {len(pod_stats)} 个 Pod")
        else:
            print(f"   节点 {node} 数据获取失败，跳过")
    
    print_summary(all_pod_stats)

if __name__ == "__main__":
    main()

执行命令方式

# 需要先安装 jq
kubectl get --raw /api/v1/nodes/10.20.20.12/proxy/stats/summary | jq '.pods[] | "PodName:  \(.podRef.name)", "usedBytes:   \(.containers[].rootfs.usedBytes)", "======================================================"'

输出结果如图:

Docker 相关操作

查看容器磁盘占用

# 节点上查看磁盘使用情况,并安装从大到小排序,可以看到各个容器的占用磁盘空间。
docker ps -a --format "table {{.Size}}\t{{.Names}}" | sort -hr

# 查看磁盘总体使用情况
docker system df