perf(local): eliminate per-entry allocations in stat scheduler worker path

Reduce heap allocation churn in the hot stat-worker path by introducing
per-worker reusable state and removing unnecessary os.File wrappers.

Per-worker scratch buffers:
- Result slice: workers reuse a goroutine-local []statFileInfo scratch
  instead of allocating per-microbatch. Publish still copies under mutex
  with lease validation, preserving stale-completion safety.
- Name buffer: custom fstatat using unix.Syscall6(SYS_NEWFSTATAT) with a
  per-worker [256]byte buffer replaces unix.Fstatat's per-call
  ByteSliceFromString allocation. Uses Syscall6 (not RawSyscall6) for
  CephFS scheduler awareness.
- ownedEntries: scratch-recycled on listController following the existing
  resultsScratch/partsScratch pattern.

Raw fd lifecycle:
- openDirAtReadFD returns raw int fd instead of *os.File, eliminating
  per-batch File struct, finalizer registration, and poll init overhead.
- closeStatDirFD uses syscall.Close for cross-platform compatibility.
- All close/drain/error paths updated with idempotent guard.

Scratch capture timing:
- Moved resultsScratch/partsScratch captures to after Wait() returns,
  matching ownedEntriesScratch and eliminating a latent aliasing window
  during the Schedule-to-Wait period.

Estimated savings: ~500-1000 MB cumulative allocation reduction and
~3-8M fewer heap objects per listing run on CephFS workloads.

Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>

Author: darthShadow

backend/local/list.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"runtime"
 	"slices"
+	"syscall"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -31,19 +32,30 @@ type cachedDirEntry struct {
 	useStatFD bool
 }
 
+const invalidStatDirFD = -1
+
 type readResult struct {
 	entries []os.DirEntry
 	err     error
-	statDir *os.File
+	statDir int
+}
+
+func newReadResult() readResult {
+	return readResult{statDir: invalidStatDirFD}
+}
+
+func closeStatDirFD(fd *int) {
+	if fd == nil || *fd < 0 {
+		return
+	}
+	_ = syscall.Close(*fd)
+	*fd = invalidStatDirFD
 }
 
 // closeReadResultStatDir releases the batch-scoped stat fd when a ReadDir
 // batch exits before handing ownership to a batchController.
 func closeReadResultStatDir(batch *readResult) {
-	if batch.statDir != nil {
-		_ = batch.statDir.Close()
-		batch.statDir = nil
-	}
+	closeStatDirFD(&batch.statDir)
 }
 
 func stopTimer(timer *time.Timer) {
@@ -186,8 +198,8 @@ func (f *Fs) listFileInfos(ctx context.Context, fd *os.File, openDir func() (*os
 		}
 		return cachedDirEntry{DirEntry: entry}, true
 	}
-	cachedStatFunc := func(entry *cachedDirEntry) (os.FileInfo, error) {
-		return statFunc(entry.DirEntry), nil
+	cachedStatFunc := func(entry *cachedDirEntry, nameBuf []byte) (os.FileInfo, []byte, error) {
+		return statFunc(entry.DirEntry), nameBuf, nil
 	}
 	fis, err := f.listCachedFileInfos(ctx, fd, openDir, cachedPreFilter, cachedStatFunc)
 	if err != nil {
@@ -207,7 +219,7 @@ func (f *Fs) listFileInfos(ctx context.Context, fd *os.File, openDir func() (*os
 // returns exactly readDirBatchSize entries (indicating more may follow), later
 // batches are read through prefetchReader so ReadDir for batch N+1 overlaps
 // stat work for batch N.
-func (f *Fs) listCachedFileInfos(ctx context.Context, fd *os.File, openDir func() (*os.File, error), preFilter func(os.DirEntry) (cachedDirEntry, bool), statFunc func(entry *cachedDirEntry) (os.FileInfo, error)) (allFis []statFileInfo, err error) {
+func (f *Fs) listCachedFileInfos(ctx context.Context, fd *os.File, openDir func() (*os.File, error), preFilter func(os.DirEntry) (cachedDirEntry, bool), statFunc func(entry *cachedDirEntry, nameBuf []byte) (os.FileInfo, []byte, error)) (allFis []statFileInfo, err error) {
 	const readDirBatchSize = 1024
 
 	defer func() {
@@ -301,7 +313,7 @@ func (f *Fs) listCachedFileInfos(ctx context.Context, fd *os.File, openDir func(
 	}
 
 	var entries []cachedDirEntry
-	batch := readResult{}
+	batch := newReadResult()
 	batch.entries, batch.err = fd.ReadDir(readDirBatchSize)
 	if len(batch.entries) > 0 {
 		// Give this ReadDir batch its own stat handle tied to the same opened
@@ -429,30 +441,30 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		return cachedEntry, slices.Contains(filter.Opt.ExcludeFile, entry.Name())
 	}
 
-	statFunc := func(entry *cachedDirEntry) (os.FileInfo, error) {
+	statFunc := func(entry *cachedDirEntry, nameBuf []byte) (os.FileInfo, []byte, error) {
 		entryType := entry.Type()
 		if includeDirFn != nil && entryType.IsDir() {
 			include, inclErr := includeDirFn(entry.newRemote)
 			if inclErr != nil {
 				fs.Infof(entry.newRemote, "directory exclusion check failed: %v", inclErr)
 			}
 			if !include {
-				return nil, nil
+				return nil, nameBuf, nil
 			}
 		}
-		fi, fierr := statDirEntry(entry)
+		fi, nextNameBuf, fierr := statDirEntry(entry, nameBuf)
 		if os.IsNotExist(fierr) {
-			return nil, nil
+			return nil, nextNameBuf, nil
 		}
 		if fierr != nil {
 			if useFilter && !filter.IncludeRemote(entry.newRemote) {
-				return nil, nil
+				return nil, nextNameBuf, nil
 			}
 			namepath := join.FilePathJoin(fsDirPath, entry.Name())
 			fierr = fmt.Errorf("failed to get info about directory entry %q: %w", namepath, fierr)
 			fs.Errorf(dir, "%v", fierr)
 			_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr))
-			return nil, nil
+			return nil, nextNameBuf, nil
 		}
 		if entryType == 0 && f.opt.SkipRecent {
 			if directoryRecentlyChanged {
@@ -461,12 +473,12 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					!fileCTime.After(timeNow.Add(1*time.Hour)) &&
 					fileCTime.Add(5*time.Minute).After(timeNow)
 				if fileRecentlyChanged {
-					return nil, nil
+					return nil, nextNameBuf, nil
 				}
 			}
 		}
 
-		return fi, nil
+		return fi, nextNameBuf, nil
 	}
 	fis, err := f.listCachedFileInfos(ctx, fd, openDir, preFilter, statFunc)
 	if err != nil {

backend/local/list_fstatat_linux.go

@@ -2,17 +2,20 @@
 
 // This file implements the linux/amd64 fast-stat path used by the sequential
 // listing loop. openDirAtReadFD opens "." relative to the listing read fd to
-// create a batch-scoped stat handle, listStatDirFD snapshots that handle's raw
-// dirfd once per batch, and statDirEntry uses fstatat(dirfd, name,
+// create a batch-scoped raw stat dirfd, listStatDirFD validates that raw fd
+// once per batch, and statDirEntry uses fstatat(dirfd, name,
 // AT_SYMLINK_NOFOLLOW) without constructing absolute paths. Other platforms use
 // list_fstatat_other.go and fall back to entry.Info().
 
 package local
 
 import (
 	"os"
+	"runtime"
+	"strings"
 	"syscall"
 	"time"
+	"unsafe"
 
 	"golang.org/x/sys/unix"
 )
@@ -89,15 +92,38 @@ func syscallStatFromUnix(st unix.Stat_t) syscall.Stat_t {
 	}
 }
 
-func fstatatNoFollow(dirfd int, name string) (os.FileInfo, error) {
+func fstatatNoFollow(dirfd int, name string, nameBuf []byte) (os.FileInfo, []byte, error) {
+	if strings.IndexByte(name, 0) >= 0 {
+		return nil, nameBuf, syscall.EINVAL
+	}
+
+	need := len(name) + 1
+	if cap(nameBuf) < need {
+		nameBuf = make([]byte, need)
+	} else {
+		nameBuf = nameBuf[:need]
+	}
+	copy(nameBuf, name)
+	nameBuf[len(name)] = 0
+
 	var st unix.Stat_t
 	for {
-		err := unix.Fstatat(dirfd, name, &st, unix.AT_SYMLINK_NOFOLLOW)
-		if err == syscall.EINTR {
+		_, _, errno := unix.Syscall6(
+			unix.SYS_NEWFSTATAT,
+			uintptr(dirfd),
+			uintptr(unsafe.Pointer(&nameBuf[0])),
+			uintptr(unsafe.Pointer(&st)),
+			uintptr(unix.AT_SYMLINK_NOFOLLOW),
+			0,
+			0,
+		)
+		runtime.KeepAlive(nameBuf)
+		runtime.KeepAlive(&st)
+		if errno == syscall.EINTR {
 			continue
 		}
-		if err != nil {
-			return nil, err
+		if errno != 0 {
+			return nil, nameBuf[:0], errno
 		}
 		break
 	}
@@ -110,19 +136,19 @@ func fstatatNoFollow(dirfd int, name string) (os.FileInfo, error) {
 		size:    st.Size,
 		modTime: modTime,
 		stat:    syscallStatFromUnix(st),
-	}, nil
+	}, nameBuf[:0], nil
 }
 
 // openDirAtReadFD opens "." relative to the active listing fd so one ReadDir
 // batch gets its own directory handle for fstatat work.
-func openDirAtReadFD(fd *os.File) (*os.File, error) {
+func openDirAtReadFD(fd *os.File) (int, error) {
 	if fd == nil {
-		return nil, os.ErrClosed
+		return invalidStatDirFD, os.ErrClosed
 	}
 
 	rawfd := fd.Fd()
 	if rawfd == ^uintptr(0) {
-		return nil, os.ErrClosed
+		return invalidStatDirFD, os.ErrClosed
 	}
 
 	for {
@@ -131,30 +157,27 @@ func openDirAtReadFD(fd *os.File) (*os.File, error) {
 			continue
 		}
 		if err != nil {
-			return nil, err
+			return invalidStatDirFD, err
 		}
-		return os.NewFile(uintptr(statFD), fd.Name()), nil
+		return statFD, nil
 	}
 }
 
-// listStatDirFD snapshots the raw dirfd from the batch-owned stat handle so the
-// batch can reuse it for every entry without reacquiring it per stat call.
-func listStatDirFD(fd *os.File) (int, bool) {
-	if fd == nil {
-		return 0, false
-	}
-	rawfd := fd.Fd()
-	if rawfd == ^uintptr(0) {
-		return 0, false
+// listStatDirFD validates the batch-owned raw dirfd before the batch snapshots
+// it into each cached entry.
+func listStatDirFD(fd int) (int, bool) {
+	if fd < 0 {
+		return invalidStatDirFD, false
 	}
-	return int(rawfd), true
+	return fd, true
 }
 
 // statDirEntry stats one cached entry through the batch-scoped dirfd when
 // available, otherwise it falls back to the DirEntry.Info path.
-func statDirEntry(entry *cachedDirEntry) (os.FileInfo, error) {
+func statDirEntry(entry *cachedDirEntry, nameBuf []byte) (os.FileInfo, []byte, error) {
 	if entry.useStatFD {
-		return fstatatNoFollow(entry.statDirFD, entry.Name())
+		return fstatatNoFollow(entry.statDirFD, entry.Name(), nameBuf)
 	}
-	return entry.Info()
+	fi, err := entry.Info()
+	return fi, nameBuf, err
 }