Implement os.RemoveAll()

TODO: enable test on windows once Readdir is implemented there

src/os/file.go

@@ -64,11 +64,6 @@ func Remove(path string) error {
 	return nil
 }
 
-// RemoveAll is a stub, it is not implemented.
-func RemoveAll(path string) error {
-	return ErrNotImplemented
-}
-
 // Name returns the name of the file with which it was opened.
 func (f *File) Name() string {
 	return f.name

src/os/path.go

@@ -1,5 +1,5 @@
-//go:build !baremetal && !js
-// +build !baremetal,!js
+//go:build !baremetal
+// +build !baremetal
 
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
@@ -60,3 +60,23 @@ func MkdirAll(path string, perm FileMode) error {
 	}
 	return nil
 }
+
+// RemoveAll removes path and any children it contains.
+// It removes everything it can but returns the first error
+// it encounters. If the path does not exist, RemoveAll
+// returns nil (no error).
+// If there is an error, it will be of type *PathError.
+func RemoveAll(path string) error {
+	return removeAll(path)
+}
+
+// endsWithDot reports whether the final component of path is ".".
+func endsWithDot(path string) bool {
+	if path == "." {
+		return true
+	}
+	if len(path) >= 2 && path[len(path)-1] == '.' && IsPathSeparator(path[len(path)-2]) {
+		return true
+	}
+	return false
+}

src/os/read_test.go

@@ -0,0 +1,102 @@
+//go:build !baremetal && !js && !wasi
+// +build !baremetal,!js,!wasi
+
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package os_test
+
+import (
+	"bytes"
+	. "os"
+	"path/filepath"
+	"testing"
+)
+
+func checkNamedSize(t *testing.T, path string, size int64) {
+	// TODO: this statement fails on wasi, possibly it objects to reading Stat on a symlink
+	dir, err := Stat(path)
+	if err != nil {
+		t.Fatalf("Stat %q (looking for size %d): %s", path, size, err)
+		return
+	}
+	if dir.Size() != size {
+		t.Errorf("Stat %q: size %d want %d", path, dir.Size(), size)
+	}
+}
+
+func TestReadFile(t *testing.T) {
+	filename := "rumpelstilzchen"
+	contents, err := ReadFile(filename)
+	if err == nil {
+		t.Fatalf("ReadFile %s: error expected, none found", filename)
+	}
+
+	filename = "read_test.go"
+	contents, err = ReadFile(filename)
+	if err != nil {
+		t.Fatalf("ReadFile %s: %v", filename, err)
+	}
+
+	checkNamedSize(t, filename, int64(len(contents)))
+}
+
+func TestWriteFile(t *testing.T) {
+	f, err := CreateTemp("", "ioutil-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	defer Remove(f.Name())
+
+	msg := "Programming today is a race between software engineers striving to " +
+		"build bigger and better idiot-proof programs, and the Universe trying " +
+		"to produce bigger and better idiots. So far, the Universe is winning."
+
+	if err := WriteFile(f.Name(), []byte(msg), 0644); err != nil {
+		t.Fatalf("WriteFile %s: %v", f.Name(), err)
+	}
+
+	data, err := ReadFile(f.Name())
+	if err != nil {
+		t.Fatalf("ReadFile %s: %v", f.Name(), err)
+	}
+
+	if string(data) != msg {
+		t.Fatalf("ReadFile: wrong data:\nhave %q\nwant %q", string(data), msg)
+	}
+}
+
+func TestReadOnlyWriteFile(t *testing.T) {
+	// TODO: also skip on wasi, where file permissions are ignored
+	if Getuid() == 0 {
+		t.Skipf("Root can write to read-only files anyway, so skip the read-only test.")
+	}
+
+	// We don't want to use CreateTemp directly, since that opens a file for us as 0600.
+	tempDir, err := MkdirTemp("", t.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer RemoveAll(tempDir)
+	filename := filepath.Join(tempDir, "blurp.txt")
+
+	shmorp := []byte("shmorp")
+	florp := []byte("florp")
+	err = WriteFile(filename, shmorp, 0444)
+	if err != nil {
+		t.Fatalf("WriteFile %s: %v", filename, err)
+	}
+	err = WriteFile(filename, florp, 0444)
+	if err == nil {
+		t.Fatalf("Expected an error when writing to read-only file %s", filename)
+	}
+	got, err := ReadFile(filename)
+	if err != nil {
+		t.Fatalf("ReadFile %s: %v", filename, err)
+	}
+	if !bytes.Equal(got, shmorp) {
+		t.Fatalf("want %s, got %s", shmorp, got)
+	}
+}

src/os/removeall_noat.go

@@ -0,0 +1,143 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !baremetal && !js && !wasi
+// +build !baremetal,!js,!wasi
+
+package os
+
+import (
+	"io"
+	"runtime"
+	"syscall"
+)
+
+func removeAll(path string) error {
+	if path == "" {
+		// fail silently to retain compatibility with previous behavior
+		// of RemoveAll. See issue 28830.
+		return nil
+	}
+
+	// The rmdir system call permits removing "." on Plan 9,
+	// so we don't permit it to remain consistent with the
+	// "at" implementation of RemoveAll.
+	if endsWithDot(path) {
+		return &PathError{Op: "RemoveAll", Path: path, Err: syscall.EINVAL}
+	}
+
+	// Simple case: if Remove works, we're done.
+	err := Remove(path)
+	if err == nil || IsNotExist(err) {
+		return nil
+	}
+
+	// Otherwise, is this a directory we need to recurse into?
+	dir, serr := Lstat(path)
+	if serr != nil {
+		if serr, ok := serr.(*PathError); ok && (IsNotExist(serr.Err) || serr.Err == syscall.ENOTDIR) {
+			return nil
+		}
+		return serr
+	}
+	if !dir.IsDir() {
+		// Not a directory; return the error from Remove.
+		return err
+	}
+
+	// Remove contents & return first error.
+	err = nil
+	for {
+		fd, err := Open(path)
+		if err != nil {
+			if IsNotExist(err) {
+				// Already deleted by someone else.
+				return nil
+			}
+			return err
+		}
+
+		const reqSize = 1024
+		var names []string
+		var readErr error
+
+		for {
+			numErr := 0
+			names, readErr = fd.Readdirnames(reqSize)
+
+			for _, name := range names {
+				err1 := RemoveAll(path + string(PathSeparator) + name)
+				if err == nil {
+					err = err1
+				}
+				if err1 != nil {
+					numErr++
+				}
+			}
+
+			// If we can delete any entry, break to start new iteration.
+			// Otherwise, we discard current names, get next entries and try deleting them.
+			if numErr != reqSize {
+				break
+			}
+		}
+
+		// Removing files from the directory may have caused
+		// the OS to reshuffle it. Simply calling Readdirnames
+		// again may skip some entries. The only reliable way
+		// to avoid this is to close and re-open the
+		// directory. See issue 20841.
+		fd.Close()
+
+		if readErr == io.EOF {
+			break
+		}
+		// If Readdirnames returned an error, use it.
+		if err == nil {
+			err = readErr
+		}
+		if len(names) == 0 {
+			break
+		}
+
+		// We don't want to re-open unnecessarily, so if we
+		// got fewer than request names from Readdirnames, try
+		// simply removing the directory now. If that
+		// succeeds, we are done.
+		if len(names) < reqSize {
+			err1 := Remove(path)
+			if err1 == nil || IsNotExist(err1) {
+				return nil
+			}
+
+			if err != nil {
+				// We got some error removing the
+				// directory contents, and since we
+				// read fewer names than we requested
+				// there probably aren't more files to
+				// remove. Don't loop around to read
+				// the directory again. We'll probably
+				// just get the same error.
+				return err
+			}
+		}
+	}
+
+	// Remove directory.
+	err1 := Remove(path)
+	if err1 == nil || IsNotExist(err1) {
+		return nil
+	}
+	if runtime.GOOS == "windows" && IsPermission(err1) {
+		if fs, err := Stat(path); err == nil {
+			if err = Chmod(path, FileMode(0200|int(fs.Mode()))); err == nil {
+				err1 = Remove(path)
+			}
+		}
+	}
+	if err == nil {
+		err = err1
+	}
+	return err
+}

src/os/removeall_other.go

@@ -0,0 +1,16 @@
+//go:build baremetal || js || wasi
+// +build baremetal js wasi
+
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package os
+
+import (
+	"syscall"
+)
+
+func removeAll(path string) error {
+	return &PathError{Op: "RemoveAll", Path: path, Err: syscall.ENOSYS}
+}

src/os/removeall_test.go

@@ -0,0 +1,390 @@
+//go:build darwin || (linux && !baremetal && !js && !wasi)
+// +build darwin linux,!baremetal,!js,!wasi
+
+// TODO: implement ReadDir on windows
+
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package os_test
+
+import (
+	"fmt"
+	"os"
+	. "os"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+func TestRemoveAll(t *testing.T) {
+	tmpDir, _ := os.MkdirTemp("", "TestRemoveAll")
+	if err := RemoveAll(""); err != nil {
+		t.Errorf("RemoveAll(\"\"): %v; want nil", err)
+	}
+
+	file := filepath.Join(tmpDir, "file")
+	path := filepath.Join(tmpDir, "_TestRemoveAll_")
+	fpath := filepath.Join(path, "file")
+	dpath := filepath.Join(path, "dir")
+
+	// Make a regular file and remove
+	fd, err := Create(file)
+	if err != nil {
+		t.Fatalf("create %q: %s", file, err)
+	}
+	fd.Close()
+	if err = RemoveAll(file); err != nil {
+		t.Fatalf("RemoveAll %q (first): %s", file, err)
+	}
+	if _, err = Lstat(file); err == nil {
+		t.Fatalf("Lstat %q succeeded after RemoveAll (first)", file)
+	}
+
+	// Make directory with 1 file and remove.
+	if err := MkdirAll(path, 0777); err != nil {
+		t.Fatalf("MkdirAll %q: %s", path, err)
+	}
+	fd, err = Create(fpath)
+	if err != nil {
+		t.Fatalf("create %q: %s", fpath, err)
+	}
+	fd.Close()
+	if err = RemoveAll(path); err != nil {
+		t.Fatalf("RemoveAll %q (second): %s", path, err)
+	}
+	if _, err = Lstat(path); err == nil {
+		t.Fatalf("Lstat %q succeeded after RemoveAll (second)", path)
+	}
+
+	// Make directory with file and subdirectory and remove.
+	if err = MkdirAll(dpath, 0777); err != nil {
+		t.Fatalf("MkdirAll %q: %s", dpath, err)
+	}
+	fd, err = Create(fpath)
+	if err != nil {
+		t.Fatalf("create %q: %s", fpath, err)
+	}
+	fd.Close()
+	fd, err = Create(dpath + "/file")
+	if err != nil {
+		t.Fatalf("create %q: %s", fpath, err)
+	}
+	fd.Close()
+	if err = RemoveAll(path); err != nil {
+		t.Fatalf("RemoveAll %q (third): %s", path, err)
+	}
+	if _, err := Lstat(path); err == nil {
+		t.Fatalf("Lstat %q succeeded after RemoveAll (third)", path)
+	}
+
+	// Chmod is not supported under Windows and test fails as root.
+	if runtime.GOOS != "windows" && Getuid() != 0 {
+		// Make directory with file and subdirectory and trigger error.
+		if err = MkdirAll(dpath, 0777); err != nil {
+			t.Fatalf("MkdirAll %q: %s", dpath, err)
+		}
+
+		for _, s := range []string{fpath, dpath + "/file1", path + "/zzz"} {
+			fd, err = Create(s)
+			if err != nil {
+				t.Fatalf("create %q: %s", s, err)
+			}
+			fd.Close()
+		}
+		if err = Chmod(dpath, 0); err != nil {
+			t.Fatalf("Chmod %q 0: %s", dpath, err)
+		}
+
+		// No error checking here: either RemoveAll
+		// will or won't be able to remove dpath;
+		// either way we want to see if it removes fpath
+		// and path/zzz. Reasons why RemoveAll might
+		// succeed in removing dpath as well include:
+		//	* running as root
+		//	* running on a file system without permissions (FAT)
+		RemoveAll(path)
+		Chmod(dpath, 0777)
+
+		for _, s := range []string{fpath, path + "/zzz"} {
+			if _, err = Lstat(s); err == nil {
+				t.Fatalf("Lstat %q succeeded after partial RemoveAll", s)
+			}
+		}
+	}
+	if err = RemoveAll(path); err != nil {
+		t.Fatalf("RemoveAll %q after partial RemoveAll: %s", path, err)
+	}
+	if _, err = Lstat(path); err == nil {
+		t.Fatalf("Lstat %q succeeded after RemoveAll (final)", path)
+	}
+}
+
+// Test RemoveAll on a large directory.
+func TestRemoveAllLarge(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+
+	tmpDir, _ := os.MkdirTemp("", "TestRemoveAllLarge")
+	path := filepath.Join(tmpDir, "_TestRemoveAllLarge_")
+
+	// Make directory with 1000 files and remove.
+	if err := MkdirAll(path, 0777); err != nil {
+		t.Fatalf("MkdirAll %q: %s", path, err)
+	}
+	for i := 0; i < 1000; i++ {
+		fpath := fmt.Sprintf("%s/file%d", path, i)
+		fd, err := Create(fpath)
+		if err != nil {
+			t.Fatalf("create %q: %s", fpath, err)
+		}
+		fd.Close()
+	}
+	if err := RemoveAll(path); err != nil {
+		t.Fatalf("RemoveAll %q: %s", path, err)
+	}
+	if _, err := Lstat(path); err == nil {
+		t.Fatalf("Lstat %q succeeded after RemoveAll", path)
+	}
+}
+
+func TestRemoveAllDot(t *testing.T) {
+	prevDir, err := Getwd()
+	if err != nil {
+		t.Fatalf("Could not get wd: %s", err)
+	}
+	tempDir, err := os.MkdirTemp("", "TestRemoveAllDot-")
+	if err != nil {
+		t.Fatalf("Could not create TempDir: %s", err)
+	}
+	defer RemoveAll(tempDir)
+
+	err = Chdir(tempDir)
+	if err != nil {
+		t.Fatalf("Could not chdir to tempdir: %s", err)
+	}
+
+	err = RemoveAll(".")
+	if err == nil {
+		t.Errorf("RemoveAll succeed to remove .")
+	}
+
+	err = Chdir(prevDir)
+	if err != nil {
+		t.Fatalf("Could not chdir %s: %s", prevDir, err)
+	}
+}
+
+func TestRemoveAllDotDot(t *testing.T) {
+	t.Parallel()
+
+	tempDir, _ := os.MkdirTemp("", "TestRemoveAllDotDot")
+	subdir := filepath.Join(tempDir, "x")
+	subsubdir := filepath.Join(subdir, "y")
+	if err := MkdirAll(subsubdir, 0777); err != nil {
+		t.Fatal(err)
+	}
+	if err := RemoveAll(filepath.Join(subsubdir, "..")); err != nil {
+		t.Error(err)
+	}
+	for _, dir := range []string{subsubdir, subdir} {
+		if _, err := Stat(dir); err == nil {
+			t.Errorf("%s: exists after RemoveAll", dir)
+		}
+	}
+}
+
+// Issue #29178.
+func TestRemoveReadOnlyDir(t *testing.T) {
+	t.Parallel()
+
+	tempDir, _ := os.MkdirTemp("", "TestRemoveReadOnlyDir")
+	subdir := filepath.Join(tempDir, "x")
+	if err := Mkdir(subdir, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	// If an error occurs make it more likely that removing the
+	// temporary directory will succeed.
+	defer Chmod(subdir, 0777)
+
+	if err := RemoveAll(subdir); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := Stat(subdir); err == nil {
+		t.Error("subdirectory was not removed")
+	}
+}
+
+// Issue #29983.
+func TestRemoveAllButReadOnlyAndPathError(t *testing.T) {
+	switch runtime.GOOS {
+	case "js", "windows":
+		t.Skipf("skipping test on %s", runtime.GOOS)
+	}
+
+	if Getuid() == 0 {
+		t.Skip("skipping test when running as root")
+	}
+
+	t.Parallel()
+
+	tempDir, _ := os.MkdirTemp("", "TestRemoveAllButReadOnlyAndPathError")
+	dirs := []string{
+		"a",
+		"a/x",
+		"a/x/1",
+		"b",
+		"b/y",
+		"b/y/2",
+		"c",
+		"c/z",
+		"c/z/3",
+	}
+	readonly := []string{
+		"b",
+	}
+	inReadonly := func(d string) bool {
+		for _, ro := range readonly {
+			if d == ro {
+				return true
+			}
+			dd, _ := filepath.Split(d)
+			if filepath.Clean(dd) == ro {
+				return true
+			}
+		}
+		return false
+	}
+
+	for _, dir := range dirs {
+		if err := Mkdir(filepath.Join(tempDir, dir), 0777); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, dir := range readonly {
+		d := filepath.Join(tempDir, dir)
+		if err := Chmod(d, 0555); err != nil {
+			t.Fatal(err)
+		}
+
+		// Defer changing the mode back so that the deferred
+		// RemoveAll(tempDir) can succeed.
+		defer Chmod(d, 0777)
+	}
+
+	err := RemoveAll(tempDir)
+	if err == nil {
+		t.Fatal("RemoveAll succeeded unexpectedly")
+	}
+
+	// The error should be of type *PathError.
+	// see issue 30491 for details.
+	if pathErr, ok := err.(*PathError); ok {
+		want := filepath.Join(tempDir, "b", "y")
+		if pathErr.Path != want {
+			t.Errorf("RemoveAll(%q): err.Path=%q, want %q", tempDir, pathErr.Path, want)
+		}
+	} else {
+		t.Errorf("RemoveAll(%q): error has type %T, want *fs.PathError", tempDir, err)
+	}
+
+	for _, dir := range dirs {
+		_, err := Stat(filepath.Join(tempDir, dir))
+		if inReadonly(dir) {
+			if err != nil {
+				t.Errorf("file %q was deleted but should still exist", dir)
+			}
+		} else {
+			if err == nil {
+				t.Errorf("file %q still exists but should have been deleted", dir)
+			}
+		}
+	}
+}
+
+func TestRemoveUnreadableDir(t *testing.T) {
+	switch runtime.GOOS {
+	case "js":
+		t.Skipf("skipping test on %s", runtime.GOOS)
+	}
+
+	if Getuid() == 0 {
+		t.Skip("skipping test when running as root")
+	}
+
+	t.Parallel()
+
+	tempDir, _ := os.MkdirTemp("", "TestRemoveUnreadableDir")
+	target := filepath.Join(tempDir, "d0", "d1", "d2")
+	if err := MkdirAll(target, 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := Chmod(target, 0300); err != nil {
+		t.Fatal(err)
+	}
+	if err := RemoveAll(filepath.Join(tempDir, "d0")); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Issue 29921
+func TestRemoveAllWithMoreErrorThanReqSize(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+
+	tmpDir, _ := os.MkdirTemp("", "TestRemoveAllWithMoreErrorThanReqSize")
+	path := filepath.Join(tmpDir, "_TestRemoveAllWithMoreErrorThanReqSize_")
+
+	// Make directory with 1025 read-only files.
+	if err := MkdirAll(path, 0777); err != nil {
+		t.Fatalf("MkdirAll %q: %s", path, err)
+	}
+	for i := 0; i < 1025; i++ {
+		fpath := filepath.Join(path, fmt.Sprintf("file%d", i))
+		fd, err := Create(fpath)
+		if err != nil {
+			t.Fatalf("create %q: %s", fpath, err)
+		}
+		fd.Close()
+	}
+
+	// Make the parent directory read-only. On some platforms, this is what
+	// prevents os.Remove from removing the files within that directory.
+	if err := Chmod(path, 0555); err != nil {
+		t.Fatal(err)
+	}
+	defer Chmod(path, 0755)
+
+	// This call should not hang, even on a platform that disallows file deletion
+	// from read-only directories.
+	err := RemoveAll(path)
+
+	if Getuid() == 0 {
+		// On many platforms, root can remove files from read-only directories.
+		return
+	}
+	if err == nil {
+		if runtime.GOOS == "windows" {
+			// Marking a directory as read-only in Windows does not prevent the RemoveAll
+			// from creating or removing files within it.
+			return
+		}
+		t.Fatal("RemoveAll(<read-only directory>) = nil; want error")
+	}
+
+	dir, err := Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dir.Close()
+
+	names, _ := dir.Readdirnames(1025)
+	if len(names) < 1025 {
+		t.Fatalf("RemoveAll(<read-only directory>) unexpectedly removed %d read-only files from that directory", 1025-len(names))
+	}
+}

src/os/tempfile_test.go

@@ -2,6 +2,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+//go:build !baremetal && !js && !wasi
+// +build !baremetal,!js,!wasi
+
 package os_test
 
 import (