容器核心技术（九） cgroup

# GRUB_CMDLINE_LINUX_DEFAULT="quiet"
GRUB_CMDLINE_LINUX_DEFAULT="quiet systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false"

sudo grub-mkconfig -o /boot/grub/grub.cfg

sudo reboot

tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,size=4096k,nr_inodes=1024,mode=755)
cgroup2 on /sys/fs/cgroup/unified type cgroup2 (rw,nosuid,nodev,noexec,relatime,nsdelegate)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,name=systemd)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/rdma type cgroup (rw,nosuid,nodev,noexec,relatime,rdma)

12:rdma:/
11:devices:/user.slice
10:perf_event:/
9:hugetlb:/
8:memory:/user.slice/user-1000.slice/session-3.scope
7:cpuset:/
6:pids:/user.slice/user-1000.slice/session-3.scope
5:net_cls,net_prio:/
4:cpu,cpuacct:/
3:blkio:/
2:freezer:/
1:name=systemd:/user.slice/user-1000.slice/session-3.scope
0::/user.slice/user-1000.slice/session-3.scope

/sys/fs/cgroup/
    cpu,cpuacct/           -+                         -+
        cgroup.procs        +--> <cgroup>              |
        ...                -+                          |
        docker/            -+                          |
            cgroup.procs    +--> <cgroup>              +----> <hierarchy> (cpu,cpuacct)
            ...            -+                          |
            container1/    -+--> <cgroup>              |
            container2/    -+--> <cgroup>              |
            container3/    -+--> <cgroup>             -+
    memory/                -+--> <cgroup>             -+----> <hierarchy> (memory)
    ...

package main

import (
	"fmt"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/shirou/gopsutil/v3/process"
	"golang.org/x/sys/unix"
)

func usage100PercentCPU() {
	i := 0
	for {
		i = i + 1
	}
}

func printSelfCPUPercent(p *process.Process) {
	cpuPercent, err := p.Percent(1 * time.Second)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%.2f%s\n", cpuPercent, "%")
}

func attachSelfToCgroup(cgroupPath string) error {
	return os.WriteFile(path.Join(cgroupPath, "cgroup.procs"), []byte(fmt.Sprint(os.Getpid())), 0644)
}

func printSelfCgroup(subsys string) {
	selfCgroupBytes, err := os.ReadFile("/proc/self/cgroup")
	if err != nil {
		panic(err)
	}
	for _, line := range strings.Split(string(selfCgroupBytes), "\n") {
		subSystems := strings.Split(strings.Split(line, ":")[1], ",")
		for _, now := range subSystems {
			if now == subsys {
				fmt.Println(line)
				return
			}
		}
	}
	panic(fmt.Errorf("subsys not found: %s", subsys))
}

func main() {

	// 启动 CPU 负载
	go usage100PercentCPU()

	p, err := process.NewProcess(int32(os.Getpid()))
	if err != nil {
		panic(err)
	}

	// 打印默认的情况
	fmt.Printf("当前进程的 cpu cgroup 为: ")
	printSelfCgroup("cpu")
	fmt.Printf("当前进程的 cpu 使用率为: ")
	printSelfCPUPercent(p)
	fmt.Println()

	// 在默认 cpu 层级，根 cgroup 创建一个名为 demo 的 cgroup。
	cpuDemoCgroupDir := "/sys/fs/cgroup/cpu/demo"
	err = os.Mkdir(cpuDemoCgroupDir, 777)
	if err != nil {
		panic(err)
	}
	defer func() {
		// 使用 unix 的 rmdir 系统调用。
		// 参考 https://github.com/opencontainers/runc/blob/main/libcontainer/cgroups/utils.go#L231
		err = unix.Rmdir(cpuDemoCgroupDir)
		if err != nil && err != unix.ENOENT {
			panic(err)
		}
	}()
	// 配置 CPU quota 为 0.2 核，0.2 * 100000 = 20000
	cpuCFSQuotaPath := path.Join(cpuDemoCgroupDir, "cpu.cfs_quota_us")
	cpuCFSPeriodPath := path.Join(cpuDemoCgroupDir, "cpu.cfs_period_us")
	cpuCFSPeriod, err := os.ReadFile(cpuCFSPeriodPath) // 100000
	if err != nil {
		panic(err)
	}
	cpuCFSPeriodInt, err := strconv.ParseInt(strings.TrimSpace(string(cpuCFSPeriod)), 10, 32)
	if err != nil {
		panic(err)
	}
	cpuCore := 0.2
	cpuCFSQuotaInt := int(cpuCore * float64(cpuCFSPeriodInt))
	err = os.WriteFile(cpuCFSQuotaPath, []byte(fmt.Sprint(cpuCFSQuotaInt)), 0644)
	if err != nil {
		panic(err)
	}
	cpuCFSQuota, err := os.ReadFile(cpuCFSQuotaPath)
	if err != nil {
		panic(err)
	}
	fmt.Printf("创建 demo cpu cgroup, 配置 core: %f, 即\n  cpu.cfs_quota_us: %s\n  cpu.cfs_quota_us: %s\n",
		cpuCore,
		strings.TrimSpace(string(cpuCFSQuota)),
		strings.TrimSpace(string(cpuCFSPeriod)),
	)
	// 将当前进程加入到当前 cgroup
	err = attachSelfToCgroup(cpuDemoCgroupDir)
	if err != nil {
		panic(err)
	}
	defer func() {
		// 进程退出前，将当前进程脱离 demo cgroup，如果不处理，删除 cgroup 将报错 device or resource busy
		err = attachSelfToCgroup("/sys/fs/cgroup/cpu")
		if err != nil {
			panic(err)
		}
	}()
	fmt.Println("当前进程已加入 demo cpu cgroup")
	fmt.Println()

	fmt.Printf("当前进程的 cpu cgroup 为: ")
	printSelfCgroup("cpu")
	fmt.Printf("当前进程的 cpu 使用率为: ")
	printSelfCPUPercent(p)
}

当前进程的 cpu cgroup 为: 4:cpu,cpuacct:/
当前进程的 cpu 使用率为: 93.24%

创建 demo cpu cgroup, 配置 core: 0.200000, 即
  cpu.cfs_quota_us: 20000
  cpu.cfs_quota_us: 100000
当前进程已加入 demo cpu cgroup

当前进程的 cpu cgroup 为: 4:cpu,cpuacct:/demo
当前进程的 cpu 使用率为: 21.37%

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/shirou/gopsutil/v3/process"
	"golang.org/x/sys/unix"
)

const (
	MB                  = 1024 * 1024
	memoryDemoCgroupDir = "/sys/fs/cgroup/memory/demo"
)

const (
	SubCommandMonitor = "monitor"
	SubCommandAlloc   = "alloc"
)

var PinnedAllocMemory [][]byte

func allocMemory(size int64) {
	b := make([]byte, size)
	for i := int64(0); i < size; i++ {
		b[i] = byte(i)
	}
	PinnedAllocMemory = append(PinnedAllocMemory, b)
}

func printRSSMemory(p *process.Process) {
	m, err := p.MemoryInfo()
	if err != nil {
		panic(err)
	}
	s := 0
	for i := 0; i < len(PinnedAllocMemory[0]); i++ {
		s += int(PinnedAllocMemory[0][i])
	}
	fmt.Printf("  pid %d: %s\n", p.Pid, m.String())
}

func handleOOMEvent() {
	fmt.Println("=== handle oom event ...")
	usage_in_bytes, err := os.ReadFile(path.Join(memoryDemoCgroupDir, `memory.usage_in_bytes`))
	if err != nil {
		panic(err)
	}
	fmt.Printf("memory.usage_in_bytes: \n%s\n", string(usage_in_bytes))
	ommControlBytes, err := os.ReadFile(path.Join(memoryDemoCgroupDir, `memory.oom_control`))
	if err != nil {
		panic(err)
	}
	fmt.Printf("memory.oom_control: \n%s\n", string(ommControlBytes))
	procsBytes, err := os.ReadFile(path.Join(memoryDemoCgroupDir, "cgroup.procs"))
	if err != nil {
		panic(err)
	}
	procs := strings.Split(string(procsBytes), "\n")
	for _, pidStr := range procs {
		pidStr = strings.TrimSpace(pidStr)
		if pidStr == "" {
			break
		}
		pid64, _ := strconv.ParseInt(pidStr, 10, 32)
		p, err := process.NewProcess(int32(pid64))
		if err != nil {
			panic(err)
		}
		status, err := p.Status()
		if err != nil {
			panic(err)
		}
		oom_score, _ := os.ReadFile(fmt.Sprintf("/proc/%d/oom_score", p.Pid))
		oom_adj, _ := os.ReadFile(fmt.Sprintf("/proc/%d/oom_adj ", p.Pid))
		oom_score_adj, _ := os.ReadFile(fmt.Sprintf("/proc/%d/oom_score_adj", p.Pid))
		fmt.Printf("pid %d state is: %v (oom_adj=%s, oom_score=%s, oom_score_adj=%s)\n", p.Pid, status, strings.TrimSpace(string(oom_adj)), strings.TrimSpace(string(oom_score)), strings.TrimSpace(string(oom_score_adj)))
		// 如果要模拟内核的 oom-killer 的逻辑： oom_score + oom_adj + oom_score_adj 排序，并给最大的那个发送 SIGKILL(9) 信号。
		//
		// 但是 oom-killer 作为 Linux 内存分配的最后的保证，非常不建议禁用 oom-killer。
		//
		// 因此推荐如下：
		//   1. 通过调整 `oom_score_adj` 来调整进程的内存优先级，以自定义 oom 时 killer 的顺序。
		//   2. 可以通过监听 oom killer 事件，统计 oom 发生的频率，以辅助调度。
		//   2. 如需收集记录进程被 kill 的日志，只能通过 `dmesg` 或 /var/log/syslog 内核日志获取信息 （killed process）。
	}
}

func createOOMEventHandler() {
	// https://www.jianshu.com/p/f2403e33c766
	// https://access.redhat.com/documentation/zh-cn/red_hat_enterprise_linux/7/html/resource_management_guide/sec-memory
	var events [128]unix.EpollEvent
	var buf [8]byte
	// 创建epoll实例
	epollFd, err := unix.EpollCreate1(unix.EPOLL_CLOEXEC)
	if err != nil {
		panic(err)
	}
	// 创建eventfd实例
	efd, _ := unix.Eventfd(0, unix.EFD_CLOEXEC)

	event := unix.EpollEvent{
		Fd:     int32(efd),
		Events: unix.EPOLLHUP | unix.EPOLLIN | unix.EPOLLERR,
	}
	// 将eventfd添加到epoll中进行监听
	unix.EpollCtl(epollFd, unix.EPOLL_CTL_ADD, int(efd), &event)

	// 打开oom_control文件
	evtFile, _ := os.Open(path.Join(memoryDemoCgroupDir, "memory.oom_control"))

	// 注册oom事件，当有oom事件时，eventfd将会有数据可读
	data := fmt.Sprintf("%d %d", efd, evtFile.Fd())
	ioutil.WriteFile(path.Join(memoryDemoCgroupDir, "cgroup.event_control"), []byte(data), 0o700)

	for {
		// 开始监听oom事件
		n, err := unix.EpollWait(epollFd, events[:], -1)
		if err == nil {
			for i := 0; i < n; i++ {
				// 消费掉eventfd的数据
				unix.Read(int(events[i].Fd), buf[:])
				// 处理 oom envent
				handleOOMEvent()
			}
		}
	}
}

func monitor() {
	// 1.1 在默认 memory 层级，根 cgroup 创建一个名为 demo 的 cgroup。
	if err := os.Mkdir(memoryDemoCgroupDir, 0o755); err != nil {
		panic(err)
	}
	defer func() {
		// 使用 unix 的 rmdir 系统调用。
		// 参考 https://github.com/opencontainers/runc/blob/main/libcontainer/cgroups/utils.go#L231
		if err := unix.Rmdir(memoryDemoCgroupDir); err != nil && err != unix.ENOENT {
			panic(err)
		}
	}()
	// 1.2 配置该 cgroup 的 memory.limit_in_bytes 为 100 MB
	if err := os.WriteFile(filepath.Join(memoryDemoCgroupDir, "memory.limit_in_bytes"), []byte("100M"), 0o644); err != nil {
		panic(err)
	}

	// 2.1 创建两个进程，先加入 cgroup，再分配 70MB 左右内存（注意：实测先分配内存再加入不会被 cgroup 感知？）。
	proc1 := exec.Command("/proc/self/exe", fmt.Sprint(70*MB))
	proc1.Stdout = os.Stdout
	proc1.Stderr = os.Stderr
	if err := proc1.Start(); err != nil {
		panic(err)
	}
	proc1Done := make(chan error, 1)
	go func() { proc1Done <- proc1.Wait(); close(proc1Done) }()
	defer func() { proc1.Process.Kill(); proc1.Wait() }()
	fmt.Printf("proc1 %d start ...\n", proc1.Process.Pid)
	time.Sleep(1 * time.Second)

	proc2 := exec.Command("/proc/self/exe", fmt.Sprint(70*MB))
	proc2.Stdout = os.Stdout
	proc2.Stderr = os.Stderr
	if err := proc2.Start(); err != nil {
		panic(err)
	}
	proc2Done := make(chan error, 1)
	go func() { proc2Done <- proc2.Wait(); close(proc2Done) }()
	defer func() { proc2.Process.Kill(); proc2.Wait() }()
	fmt.Printf("proc2 %d start ...\n", proc2.Process.Pid)
	time.Sleep(1 * time.Second)

	// 2.3 观察两个进程的存活状态
	select {
	case <-proc1Done:
		fmt.Printf("proc1 %d has exited: %s\n", proc1.Process.Pid, proc1.ProcessState.String())
	default:
		fmt.Printf("proc1 %d running\n", proc1.Process.Pid)
	}
	select {
	case <-proc2Done:
		fmt.Printf("proc2 %d has exited: %s\n", proc2.Process.Pid, proc2.ProcessState.String())
	default:
		fmt.Printf("proc2 %d is running\n", proc2.Process.Pid)
	}
	fmt.Println()

	// 3.1 再创建一个子进程，加入上面创建的 demo memory cgroup，并设置该进程的 oom_score_adj 为 1000，并申请 50 MB 内存
	proc3 := exec.Command("/proc/self/exe", fmt.Sprint(50*MB), "1000")
	proc3.Stdout = os.Stdout
	proc3.Stderr = os.Stderr
	if err := proc3.Start(); err != nil {
		panic(err)
	}
	proc3Done := make(chan error, 1)
	go func() { proc3Done <- proc3.Wait(); close(proc3Done) }()
	defer func() { proc3.Process.Kill(); proc3.Wait() }()
	fmt.Printf("proc3 %d start ...\n", proc3.Process.Pid)
	time.Sleep(1 * time.Second)

	select {
	case <-proc1Done:
		fmt.Printf("proc1 %d has exited: %s\n", proc1.Process.Pid, proc1.ProcessState.String())
	default:
		fmt.Printf("proc1 %d running\n", proc1.Process.Pid)
	}
	select {
	case <-proc2Done:
		fmt.Printf("proc2 %d has exited: %s\n", proc2.Process.Pid, proc2.ProcessState.String())
	default:
		fmt.Printf("proc2 %d is running\n", proc2.Process.Pid)
	}
	select {
	case <-proc3Done:
		fmt.Printf("proc3 %d has exited: %s\n", proc3.Process.Pid, proc3.ProcessState.String())
	default:
		fmt.Printf("proc3 %d is running\n", proc3.Process.Pid)
	}
	fmt.Println()

	// ! 注意：这里只是一个演示，在生产环境不建议禁用 oom-killer
	// 4.1 监控进程写入 1 到 `memory.oom_control` 禁用内核 oom killer，并通过 `cgroup.event_control` 文件配置当前进程接受 OOM 事件。
	if err := os.WriteFile(filepath.Join(memoryDemoCgroupDir, "memory.oom_control"), []byte("1"), 0o644); err != nil {
		panic(err)
	}
	go createOOMEventHandler()
	// 5.1 监控再创建一个子进程，加入上面创建的 demo memory cgroup，并申请 50 MB 内存
	proc4 := exec.Command("/proc/self/exe", fmt.Sprint(50*MB))
	proc4.Stdout = os.Stdout
	proc4.Stderr = os.Stderr
	if err := proc4.Start(); err != nil {
		panic(err)
	}
	proc4Done := make(chan error, 1)
	go func() { proc4Done <- proc4.Wait(); close(proc4Done) }()
	defer func() { proc4.Process.Kill(); proc4.Wait() }()
	fmt.Printf("proc4 %d start ...\n", proc4.Process.Pid)
	time.Sleep(1 * time.Second)
	// time.Sleep(60 * time.Second)
}

// sudo swapoff -a
// go build ./src/go/01-namespace/07-cgroup/v1/02-memory && sudo ./02-memory; rm -rf ./02-memory
// sudo dmesg -T | egrep -i 'killed process'
// sudo swapon -a

func main() {
	if len(os.Args) == 1 {
		monitor()
	} else {
		s, err := strconv.ParseInt(os.Args[1], 10, 64)
		if err != nil {
			panic(err)
		}
		// 2.1.a 加入 Cgroup。
		if err = os.WriteFile(path.Join(memoryDemoCgroupDir, "cgroup.procs"), []byte(fmt.Sprint(os.Getpid())), 0o644); err != nil {
			panic(err)
		}
		// 3.1.a 配置进程的 `oom_score_adj`
		if len(os.Args) == 3 {
			adj, err := strconv.ParseInt(os.Args[2], 10, 64)
			if err != nil {
				panic(err)
			}
			if err = os.WriteFile("/proc/self/oom_score_adj", []byte(fmt.Sprint(adj)), 0o644); err != nil {
				panic(err)
			}
		}
		// 2.2.b 申请内存
		allocMemory(s)
		p, err := process.NewProcess(int32(os.Getpid()))
		if err != nil {
			panic(err)
		}
		printRSSMemory(p)
		time.Sleep(60 * time.Second)
	}
}

proc1 380232 start ...
  pid 380232: {"rss":80896000,"vms":1247854592,"hwm":0,"data":0,"stack":0,"locked":0,"swap":0}
proc2 380256 start ...
  pid 380256: {"rss":82972672,"vms":1247789056,"hwm":0,"data":0,"stack":0,"locked":0,"swap":0}
proc1 380232 has exited: signal: killed
proc2 380256 is running

proc3 380270 start ...
proc1 380232 has exited: signal: killed
proc2 380256 is running
proc3 380270 has exited: signal: killed

proc4 380295 start ...
=== handle oom event ...
memory.usage_in_bytes: 
104857600

memory.oom_control: 
oom_kill_disable 1
under_oom 1
oom_kill 2

pid 380256 state is: [sleep] (oom_adj=, oom_score=669, oom_score_adj=0)
pid 380295 state is: [sleep] (oom_adj=, oom_score=668, oom_score_adj=0)