Skip to content

Commit

Permalink
Fix kubelet stuck issue due to hung fs #125298
Browse files Browse the repository at this point in the history
  - kubelet is stuck when restarted with hung fs
  - timeout the context when checking for getvfsstats

Fix go formatting
  • Loading branch information
pragvija committed Jul 9, 2024
1 parent 54dff2b commit 2133709
Showing 1 changed file with 37 additions and 10 deletions.
47 changes: 37 additions & 10 deletions fs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package fs

import (
"bufio"
"context"
"fmt"
"os"
"os/exec"
Expand All @@ -29,6 +30,7 @@ import (
"strconv"
"strings"
"syscall"
"time"

zfs "github.com/mistifyio/go-zfs"
mount "github.com/moby/sys/mountinfo"
Expand Down Expand Up @@ -716,16 +718,41 @@ func (i *RealFsInfo) GetDirUsage(dir string) (UsageInfo, error) {
}

func getVfsStats(path string) (total uint64, free uint64, avail uint64, inodes uint64, inodesFree uint64, err error) {
var s syscall.Statfs_t
if err = syscall.Statfs(path, &s); err != nil {
return 0, 0, 0, 0, 0, err
}
total = uint64(s.Frsize) * s.Blocks
free = uint64(s.Frsize) * s.Bfree
avail = uint64(s.Frsize) * s.Bavail
inodes = uint64(s.Files)
inodesFree = uint64(s.Ffree)
return total, free, avail, inodes, inodesFree, nil
// timeout the context with, default is 2sec
timeout := 2
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
defer cancel()

type result struct {
total uint64
free uint64
avail uint64
inodes uint64
inodesFree uint64
err error
}

resultChan := make(chan result, 1)

go func() {
var s syscall.Statfs_t
if err = syscall.Statfs(path, &s); err != nil {
total, free, avail, inodes, inodesFree = 0, 0, 0, 0, 0
}
total = uint64(s.Frsize) * s.Blocks
free = uint64(s.Frsize) * s.Bfree
avail = uint64(s.Frsize) * s.Bavail
inodes = uint64(s.Files)
inodesFree = uint64(s.Ffree)
resultChan <- result{total: total, free: free, avail: avail, inodes: inodes, inodesFree: inodesFree, err: err}
}()

select {
case <-ctx.Done():
return 0, 0, 0, 0, 0, ctx.Err()
case res := <-resultChan:
return res.total, res.free, res.avail, res.inodes, res.inodesFree, res.err
}
}

// Devicemapper thin provisioning is detailed at
Expand Down

0 comments on commit 2133709

Please sign in to comment.