From 47a622a903ac1f9341ea31b623302b101aee9c01 Mon Sep 17 00:00:00 2001 From: Dan Kegel Date: Sat, 8 Jan 2022 16:22:25 -0800 Subject: [PATCH] Implement os.RemoveAll() TODO: enable test on windows once Readdir is implemented there --- src/os/file.go | 5 - src/os/path.go | 24 ++- src/os/read_test.go | 102 ++++++++++ src/os/removeall_noat.go | 143 ++++++++++++++ src/os/removeall_other.go | 16 ++ src/os/removeall_test.go | 390 ++++++++++++++++++++++++++++++++++++++ src/os/tempfile_test.go | 3 + 7 files changed, 676 insertions(+), 7 deletions(-) create mode 100644 src/os/read_test.go create mode 100644 src/os/removeall_noat.go create mode 100644 src/os/removeall_other.go create mode 100644 src/os/removeall_test.go diff --git a/src/os/file.go b/src/os/file.go index 10f77f45..d10e4afb 100644 --- a/src/os/file.go +++ b/src/os/file.go @@ -64,11 +64,6 @@ func Remove(path string) error { return nil } -// RemoveAll is a stub, it is not implemented. -func RemoveAll(path string) error { - return ErrNotImplemented -} - // Name returns the name of the file with which it was opened. func (f *File) Name() string { return f.name diff --git a/src/os/path.go b/src/os/path.go index 3cdc91a0..66d738e5 100644 --- a/src/os/path.go +++ b/src/os/path.go @@ -1,5 +1,5 @@ -//go:build !baremetal && !js -// +build !baremetal,!js +//go:build !baremetal +// +build !baremetal // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style @@ -60,3 +60,23 @@ func MkdirAll(path string, perm FileMode) error { } return nil } + +// RemoveAll removes path and any children it contains. +// It removes everything it can but returns the first error +// it encounters. If the path does not exist, RemoveAll +// returns nil (no error). +// If there is an error, it will be of type *PathError. +func RemoveAll(path string) error { + return removeAll(path) +} + +// endsWithDot reports whether the final component of path is ".". +func endsWithDot(path string) bool { + if path == "." { + return true + } + if len(path) >= 2 && path[len(path)-1] == '.' && IsPathSeparator(path[len(path)-2]) { + return true + } + return false +} diff --git a/src/os/read_test.go b/src/os/read_test.go new file mode 100644 index 00000000..0051f976 --- /dev/null +++ b/src/os/read_test.go @@ -0,0 +1,102 @@ +//go:build !baremetal && !js && !wasi +// +build !baremetal,!js,!wasi + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "bytes" + . "os" + "path/filepath" + "testing" +) + +func checkNamedSize(t *testing.T, path string, size int64) { + // TODO: this statement fails on wasi, possibly it objects to reading Stat on a symlink + dir, err := Stat(path) + if err != nil { + t.Fatalf("Stat %q (looking for size %d): %s", path, size, err) + return + } + if dir.Size() != size { + t.Errorf("Stat %q: size %d want %d", path, dir.Size(), size) + } +} + +func TestReadFile(t *testing.T) { + filename := "rumpelstilzchen" + contents, err := ReadFile(filename) + if err == nil { + t.Fatalf("ReadFile %s: error expected, none found", filename) + } + + filename = "read_test.go" + contents, err = ReadFile(filename) + if err != nil { + t.Fatalf("ReadFile %s: %v", filename, err) + } + + checkNamedSize(t, filename, int64(len(contents))) +} + +func TestWriteFile(t *testing.T) { + f, err := CreateTemp("", "ioutil-test") + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer Remove(f.Name()) + + msg := "Programming today is a race between software engineers striving to " + + "build bigger and better idiot-proof programs, and the Universe trying " + + "to produce bigger and better idiots. So far, the Universe is winning." + + if err := WriteFile(f.Name(), []byte(msg), 0644); err != nil { + t.Fatalf("WriteFile %s: %v", f.Name(), err) + } + + data, err := ReadFile(f.Name()) + if err != nil { + t.Fatalf("ReadFile %s: %v", f.Name(), err) + } + + if string(data) != msg { + t.Fatalf("ReadFile: wrong data:\nhave %q\nwant %q", string(data), msg) + } +} + +func TestReadOnlyWriteFile(t *testing.T) { + // TODO: also skip on wasi, where file permissions are ignored + if Getuid() == 0 { + t.Skipf("Root can write to read-only files anyway, so skip the read-only test.") + } + + // We don't want to use CreateTemp directly, since that opens a file for us as 0600. + tempDir, err := MkdirTemp("", t.Name()) + if err != nil { + t.Fatal(err) + } + defer RemoveAll(tempDir) + filename := filepath.Join(tempDir, "blurp.txt") + + shmorp := []byte("shmorp") + florp := []byte("florp") + err = WriteFile(filename, shmorp, 0444) + if err != nil { + t.Fatalf("WriteFile %s: %v", filename, err) + } + err = WriteFile(filename, florp, 0444) + if err == nil { + t.Fatalf("Expected an error when writing to read-only file %s", filename) + } + got, err := ReadFile(filename) + if err != nil { + t.Fatalf("ReadFile %s: %v", filename, err) + } + if !bytes.Equal(got, shmorp) { + t.Fatalf("want %s, got %s", shmorp, got) + } +} diff --git a/src/os/removeall_noat.go b/src/os/removeall_noat.go new file mode 100644 index 00000000..8b41282b --- /dev/null +++ b/src/os/removeall_noat.go @@ -0,0 +1,143 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !baremetal && !js && !wasi +// +build !baremetal,!js,!wasi + +package os + +import ( + "io" + "runtime" + "syscall" +) + +func removeAll(path string) error { + if path == "" { + // fail silently to retain compatibility with previous behavior + // of RemoveAll. See issue 28830. + return nil + } + + // The rmdir system call permits removing "." on Plan 9, + // so we don't permit it to remain consistent with the + // "at" implementation of RemoveAll. + if endsWithDot(path) { + return &PathError{Op: "RemoveAll", Path: path, Err: syscall.EINVAL} + } + + // Simple case: if Remove works, we're done. + err := Remove(path) + if err == nil || IsNotExist(err) { + return nil + } + + // Otherwise, is this a directory we need to recurse into? + dir, serr := Lstat(path) + if serr != nil { + if serr, ok := serr.(*PathError); ok && (IsNotExist(serr.Err) || serr.Err == syscall.ENOTDIR) { + return nil + } + return serr + } + if !dir.IsDir() { + // Not a directory; return the error from Remove. + return err + } + + // Remove contents & return first error. + err = nil + for { + fd, err := Open(path) + if err != nil { + if IsNotExist(err) { + // Already deleted by someone else. + return nil + } + return err + } + + const reqSize = 1024 + var names []string + var readErr error + + for { + numErr := 0 + names, readErr = fd.Readdirnames(reqSize) + + for _, name := range names { + err1 := RemoveAll(path + string(PathSeparator) + name) + if err == nil { + err = err1 + } + if err1 != nil { + numErr++ + } + } + + // If we can delete any entry, break to start new iteration. + // Otherwise, we discard current names, get next entries and try deleting them. + if numErr != reqSize { + break + } + } + + // Removing files from the directory may have caused + // the OS to reshuffle it. Simply calling Readdirnames + // again may skip some entries. The only reliable way + // to avoid this is to close and re-open the + // directory. See issue 20841. + fd.Close() + + if readErr == io.EOF { + break + } + // If Readdirnames returned an error, use it. + if err == nil { + err = readErr + } + if len(names) == 0 { + break + } + + // We don't want to re-open unnecessarily, so if we + // got fewer than request names from Readdirnames, try + // simply removing the directory now. If that + // succeeds, we are done. + if len(names) < reqSize { + err1 := Remove(path) + if err1 == nil || IsNotExist(err1) { + return nil + } + + if err != nil { + // We got some error removing the + // directory contents, and since we + // read fewer names than we requested + // there probably aren't more files to + // remove. Don't loop around to read + // the directory again. We'll probably + // just get the same error. + return err + } + } + } + + // Remove directory. + err1 := Remove(path) + if err1 == nil || IsNotExist(err1) { + return nil + } + if runtime.GOOS == "windows" && IsPermission(err1) { + if fs, err := Stat(path); err == nil { + if err = Chmod(path, FileMode(0200|int(fs.Mode()))); err == nil { + err1 = Remove(path) + } + } + } + if err == nil { + err = err1 + } + return err +} diff --git a/src/os/removeall_other.go b/src/os/removeall_other.go new file mode 100644 index 00000000..41cb0f54 --- /dev/null +++ b/src/os/removeall_other.go @@ -0,0 +1,16 @@ +//go:build baremetal || js || wasi +// +build baremetal js wasi + +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os + +import ( + "syscall" +) + +func removeAll(path string) error { + return &PathError{Op: "RemoveAll", Path: path, Err: syscall.ENOSYS} +} diff --git a/src/os/removeall_test.go b/src/os/removeall_test.go new file mode 100644 index 00000000..81e7f58f --- /dev/null +++ b/src/os/removeall_test.go @@ -0,0 +1,390 @@ +//go:build darwin || (linux && !baremetal && !js && !wasi) +// +build darwin linux,!baremetal,!js,!wasi + +// TODO: implement ReadDir on windows + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "fmt" + "os" + . "os" + "path/filepath" + "runtime" + "testing" +) + +func TestRemoveAll(t *testing.T) { + tmpDir, _ := os.MkdirTemp("", "TestRemoveAll") + if err := RemoveAll(""); err != nil { + t.Errorf("RemoveAll(\"\"): %v; want nil", err) + } + + file := filepath.Join(tmpDir, "file") + path := filepath.Join(tmpDir, "_TestRemoveAll_") + fpath := filepath.Join(path, "file") + dpath := filepath.Join(path, "dir") + + // Make a regular file and remove + fd, err := Create(file) + if err != nil { + t.Fatalf("create %q: %s", file, err) + } + fd.Close() + if err = RemoveAll(file); err != nil { + t.Fatalf("RemoveAll %q (first): %s", file, err) + } + if _, err = Lstat(file); err == nil { + t.Fatalf("Lstat %q succeeded after RemoveAll (first)", file) + } + + // Make directory with 1 file and remove. + if err := MkdirAll(path, 0777); err != nil { + t.Fatalf("MkdirAll %q: %s", path, err) + } + fd, err = Create(fpath) + if err != nil { + t.Fatalf("create %q: %s", fpath, err) + } + fd.Close() + if err = RemoveAll(path); err != nil { + t.Fatalf("RemoveAll %q (second): %s", path, err) + } + if _, err = Lstat(path); err == nil { + t.Fatalf("Lstat %q succeeded after RemoveAll (second)", path) + } + + // Make directory with file and subdirectory and remove. + if err = MkdirAll(dpath, 0777); err != nil { + t.Fatalf("MkdirAll %q: %s", dpath, err) + } + fd, err = Create(fpath) + if err != nil { + t.Fatalf("create %q: %s", fpath, err) + } + fd.Close() + fd, err = Create(dpath + "/file") + if err != nil { + t.Fatalf("create %q: %s", fpath, err) + } + fd.Close() + if err = RemoveAll(path); err != nil { + t.Fatalf("RemoveAll %q (third): %s", path, err) + } + if _, err := Lstat(path); err == nil { + t.Fatalf("Lstat %q succeeded after RemoveAll (third)", path) + } + + // Chmod is not supported under Windows and test fails as root. + if runtime.GOOS != "windows" && Getuid() != 0 { + // Make directory with file and subdirectory and trigger error. + if err = MkdirAll(dpath, 0777); err != nil { + t.Fatalf("MkdirAll %q: %s", dpath, err) + } + + for _, s := range []string{fpath, dpath + "/file1", path + "/zzz"} { + fd, err = Create(s) + if err != nil { + t.Fatalf("create %q: %s", s, err) + } + fd.Close() + } + if err = Chmod(dpath, 0); err != nil { + t.Fatalf("Chmod %q 0: %s", dpath, err) + } + + // No error checking here: either RemoveAll + // will or won't be able to remove dpath; + // either way we want to see if it removes fpath + // and path/zzz. Reasons why RemoveAll might + // succeed in removing dpath as well include: + // * running as root + // * running on a file system without permissions (FAT) + RemoveAll(path) + Chmod(dpath, 0777) + + for _, s := range []string{fpath, path + "/zzz"} { + if _, err = Lstat(s); err == nil { + t.Fatalf("Lstat %q succeeded after partial RemoveAll", s) + } + } + } + if err = RemoveAll(path); err != nil { + t.Fatalf("RemoveAll %q after partial RemoveAll: %s", path, err) + } + if _, err = Lstat(path); err == nil { + t.Fatalf("Lstat %q succeeded after RemoveAll (final)", path) + } +} + +// Test RemoveAll on a large directory. +func TestRemoveAllLarge(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + + tmpDir, _ := os.MkdirTemp("", "TestRemoveAllLarge") + path := filepath.Join(tmpDir, "_TestRemoveAllLarge_") + + // Make directory with 1000 files and remove. + if err := MkdirAll(path, 0777); err != nil { + t.Fatalf("MkdirAll %q: %s", path, err) + } + for i := 0; i < 1000; i++ { + fpath := fmt.Sprintf("%s/file%d", path, i) + fd, err := Create(fpath) + if err != nil { + t.Fatalf("create %q: %s", fpath, err) + } + fd.Close() + } + if err := RemoveAll(path); err != nil { + t.Fatalf("RemoveAll %q: %s", path, err) + } + if _, err := Lstat(path); err == nil { + t.Fatalf("Lstat %q succeeded after RemoveAll", path) + } +} + +func TestRemoveAllDot(t *testing.T) { + prevDir, err := Getwd() + if err != nil { + t.Fatalf("Could not get wd: %s", err) + } + tempDir, err := os.MkdirTemp("", "TestRemoveAllDot-") + if err != nil { + t.Fatalf("Could not create TempDir: %s", err) + } + defer RemoveAll(tempDir) + + err = Chdir(tempDir) + if err != nil { + t.Fatalf("Could not chdir to tempdir: %s", err) + } + + err = RemoveAll(".") + if err == nil { + t.Errorf("RemoveAll succeed to remove .") + } + + err = Chdir(prevDir) + if err != nil { + t.Fatalf("Could not chdir %s: %s", prevDir, err) + } +} + +func TestRemoveAllDotDot(t *testing.T) { + t.Parallel() + + tempDir, _ := os.MkdirTemp("", "TestRemoveAllDotDot") + subdir := filepath.Join(tempDir, "x") + subsubdir := filepath.Join(subdir, "y") + if err := MkdirAll(subsubdir, 0777); err != nil { + t.Fatal(err) + } + if err := RemoveAll(filepath.Join(subsubdir, "..")); err != nil { + t.Error(err) + } + for _, dir := range []string{subsubdir, subdir} { + if _, err := Stat(dir); err == nil { + t.Errorf("%s: exists after RemoveAll", dir) + } + } +} + +// Issue #29178. +func TestRemoveReadOnlyDir(t *testing.T) { + t.Parallel() + + tempDir, _ := os.MkdirTemp("", "TestRemoveReadOnlyDir") + subdir := filepath.Join(tempDir, "x") + if err := Mkdir(subdir, 0); err != nil { + t.Fatal(err) + } + + // If an error occurs make it more likely that removing the + // temporary directory will succeed. + defer Chmod(subdir, 0777) + + if err := RemoveAll(subdir); err != nil { + t.Fatal(err) + } + + if _, err := Stat(subdir); err == nil { + t.Error("subdirectory was not removed") + } +} + +// Issue #29983. +func TestRemoveAllButReadOnlyAndPathError(t *testing.T) { + switch runtime.GOOS { + case "js", "windows": + t.Skipf("skipping test on %s", runtime.GOOS) + } + + if Getuid() == 0 { + t.Skip("skipping test when running as root") + } + + t.Parallel() + + tempDir, _ := os.MkdirTemp("", "TestRemoveAllButReadOnlyAndPathError") + dirs := []string{ + "a", + "a/x", + "a/x/1", + "b", + "b/y", + "b/y/2", + "c", + "c/z", + "c/z/3", + } + readonly := []string{ + "b", + } + inReadonly := func(d string) bool { + for _, ro := range readonly { + if d == ro { + return true + } + dd, _ := filepath.Split(d) + if filepath.Clean(dd) == ro { + return true + } + } + return false + } + + for _, dir := range dirs { + if err := Mkdir(filepath.Join(tempDir, dir), 0777); err != nil { + t.Fatal(err) + } + } + for _, dir := range readonly { + d := filepath.Join(tempDir, dir) + if err := Chmod(d, 0555); err != nil { + t.Fatal(err) + } + + // Defer changing the mode back so that the deferred + // RemoveAll(tempDir) can succeed. + defer Chmod(d, 0777) + } + + err := RemoveAll(tempDir) + if err == nil { + t.Fatal("RemoveAll succeeded unexpectedly") + } + + // The error should be of type *PathError. + // see issue 30491 for details. + if pathErr, ok := err.(*PathError); ok { + want := filepath.Join(tempDir, "b", "y") + if pathErr.Path != want { + t.Errorf("RemoveAll(%q): err.Path=%q, want %q", tempDir, pathErr.Path, want) + } + } else { + t.Errorf("RemoveAll(%q): error has type %T, want *fs.PathError", tempDir, err) + } + + for _, dir := range dirs { + _, err := Stat(filepath.Join(tempDir, dir)) + if inReadonly(dir) { + if err != nil { + t.Errorf("file %q was deleted but should still exist", dir) + } + } else { + if err == nil { + t.Errorf("file %q still exists but should have been deleted", dir) + } + } + } +} + +func TestRemoveUnreadableDir(t *testing.T) { + switch runtime.GOOS { + case "js": + t.Skipf("skipping test on %s", runtime.GOOS) + } + + if Getuid() == 0 { + t.Skip("skipping test when running as root") + } + + t.Parallel() + + tempDir, _ := os.MkdirTemp("", "TestRemoveUnreadableDir") + target := filepath.Join(tempDir, "d0", "d1", "d2") + if err := MkdirAll(target, 0755); err != nil { + t.Fatal(err) + } + if err := Chmod(target, 0300); err != nil { + t.Fatal(err) + } + if err := RemoveAll(filepath.Join(tempDir, "d0")); err != nil { + t.Fatal(err) + } +} + +// Issue 29921 +func TestRemoveAllWithMoreErrorThanReqSize(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + + tmpDir, _ := os.MkdirTemp("", "TestRemoveAllWithMoreErrorThanReqSize") + path := filepath.Join(tmpDir, "_TestRemoveAllWithMoreErrorThanReqSize_") + + // Make directory with 1025 read-only files. + if err := MkdirAll(path, 0777); err != nil { + t.Fatalf("MkdirAll %q: %s", path, err) + } + for i := 0; i < 1025; i++ { + fpath := filepath.Join(path, fmt.Sprintf("file%d", i)) + fd, err := Create(fpath) + if err != nil { + t.Fatalf("create %q: %s", fpath, err) + } + fd.Close() + } + + // Make the parent directory read-only. On some platforms, this is what + // prevents os.Remove from removing the files within that directory. + if err := Chmod(path, 0555); err != nil { + t.Fatal(err) + } + defer Chmod(path, 0755) + + // This call should not hang, even on a platform that disallows file deletion + // from read-only directories. + err := RemoveAll(path) + + if Getuid() == 0 { + // On many platforms, root can remove files from read-only directories. + return + } + if err == nil { + if runtime.GOOS == "windows" { + // Marking a directory as read-only in Windows does not prevent the RemoveAll + // from creating or removing files within it. + return + } + t.Fatal("RemoveAll() = nil; want error") + } + + dir, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer dir.Close() + + names, _ := dir.Readdirnames(1025) + if len(names) < 1025 { + t.Fatalf("RemoveAll() unexpectedly removed %d read-only files from that directory", 1025-len(names)) + } +} diff --git a/src/os/tempfile_test.go b/src/os/tempfile_test.go index 61f1fb3b..72444934 100644 --- a/src/os/tempfile_test.go +++ b/src/os/tempfile_test.go @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !baremetal && !js && !wasi +// +build !baremetal,!js,!wasi + package os_test import (