tinygo/src/runtime/gc_extalloc.go

// +build gc.extalloc

package runtime

import (
	"internal/task"
	"runtime/interrupt"
	"unsafe"
)

// This garbage collector implementation allows TinyGo to use an external memory allocator.
// It appends a header to the end of every allocation which the garbage collector uses for tracking purposes.
// This is also a conservative collector.

const (
	gcDebug   = false
	gcAsserts = false
)

func initHeap() {}

// memTreap is a treap which is used to track allocations for the garbage collector.
type memTreap struct {
	root *memTreapNode
}

// printNode recursively prints a subtree at a given indentation depth.
func (t *memTreap) printNode(n *memTreapNode, depth int) {
	for i := 0; i < depth; i++ {
		print(" ")
	}
	println(n, n.priority())
	if n == nil {
		return
	}
	if gcAsserts && n.parent == nil && t.root != n {
		runtimePanic("parent missing")
	}
	t.printNode(n.left, depth+1)
	t.printNode(n.right, depth+1)
}

// print the treap.
func (t *memTreap) print() {
	println("treap:")
	t.printNode(t.root, 1)
}

// empty returns whether the treap contains any nodes.
func (t *memTreap) empty() bool {
	return t.root == nil
}

// minAddr returns the lowest address contained in an allocation in the treap.
func (t *memTreap) minAddr() uintptr {
	// Find the rightmost node.
	n := t.root
	for n.right != nil {
		n = n.right
	}

	// The lowest address is the base of the rightmost node.
	return uintptr(unsafe.Pointer(&n.base))
}

// maxAddr returns the highest address contained in an allocation in the treap.
func (t *memTreap) maxAddr() uintptr {
	// Find the leftmost node.
	n := t.root
	for n.left != nil {
		n = n.left
	}

	// The highest address is the end of the leftmost node.
	return uintptr(unsafe.Pointer(&n.base)) + n.size
}

// rotateRight does a right rotation of p and q.
// https://en.wikipedia.org/wiki/Tree_rotation#/media/File:Tree_rotation.png
func (t *memTreap) rotateRight(p, q *memTreapNode) {
	if t.root == q {
		t.root = p
	} else {
		*q.parentSlot() = p
	}

	//a := p.left
	b := p.right
	//c := q.right

	p.parent = q.parent
	p.right = q

	q.parent = p
	q.left = b

	if b != nil {
		b.parent = q
	}
}

// rotateLeft does a left rotation of p and q.
// https://en.wikipedia.org/wiki/Tree_rotation#/media/File:Tree_rotation.png
func (t *memTreap) rotateLeft(p, q *memTreapNode) {
	if t.root == p {
		t.root = q
	} else {
		*p.parentSlot() = q
	}

	//a := p.left
	b := q.left
	//c := q.right

	q.parent = p.parent
	q.left = p

	p.parent = q
	p.right = b

	if b != nil {
		b.parent = p
	}
}

// rotate rotates a lower node up to its parent.
// The node n must be a child of m, and will be the parent of m after the rotation.
func (t *memTreap) rotate(n, m *memTreapNode) {
	// https://en.wikipedia.org/wiki/Tree_rotation#/media/File:Tree_rotation.png
	if uintptr(unsafe.Pointer(n)) > uintptr(unsafe.Pointer(m)) {
		t.rotateRight(n, m)
	} else {
		t.rotateLeft(m, n)
	}
}

// insert a node into the treap.
func (t *memTreap) insert(n *memTreapNode) {
	if gcAsserts && (n.parent != nil || n.left != nil || n.right != nil) {
		runtimePanic("tried to insert unzeroed treap node")
	}

	if t.root == nil {
		// This is the first node, and can be inserted directly into the root.
		t.root = n
		return
	}

	// Insert like a regular binary search tree.
	for n.parent = t.root; *n.parentSlot() != nil; n.parent = *n.parentSlot() {
	}
	*n.parentSlot() = n

	// Rotate the tree to restore the heap invariant.
	priority := n.priority()
	for n.parent != nil && priority > n.parent.priority() {
		t.rotate(n, n.parent)
	}
}

// lookupAddr finds the treap node with the allocation containing the specified address.
// If the address is not contained in any allocations in this treap, nil is returned.
// NOTE: fields of memTreapNodes are not considered part of the allocations.
func (t *memTreap) lookupAddr(addr uintptr) *memTreapNode {
	n := t.root
	for n != nil && !n.contains(addr) {
		if addr > uintptr(unsafe.Pointer(n)) {
			n = n.left
		} else {
			n = n.right
		}
	}

	return n
}

// replace a node with another node on the treap.
func (t *memTreap) replace(old, new *memTreapNode) {
	if gcAsserts && (old == nil || new == nil) {
		if gcDebug {
			println("tried to replace:", old, "->", new)
		}
		runtimePanic("invalid replacement")
	}
	if gcAsserts && old.parent == nil && old != t.root {
		if gcDebug {
			println("tried to replace:", old, "->", new)
			t.print()
		}
		runtimePanic("corrupted tree")
	}
	new.parent = old.parent
	if old == t.root {
		t.root = new
	} else {
		*new.parentSlot() = new
	}
}

// remove a node from the treap.
// This does not free the allocation.
func (t *memTreap) remove(n *memTreapNode) {
scan:
	for {
		switch {
		case n.left == nil && n.right == nil && n.parent == nil:
			// This is the only node - uproot it.
			t.root = nil
			break scan
		case n.left == nil && n.right == nil:
			// There are no nodes beneath here, so just remove this node from the parent.
			*n.parentSlot() = nil
			break scan
		case n.left != nil && n.right == nil:
			t.replace(n, n.left)
			break scan
		case n.right != nil && n.left == nil:
			t.replace(n, n.right)
			break scan
		default:
			// Rotate this node downward.
			if n.left.priority() > n.right.priority() {
				t.rotate(n.left, n)
			} else {
				t.rotate(n.right, n)
			}
		}
	}

	n.left = nil
	n.right = nil
	n.parent = nil
}

// memTreapNode is a treap node used to track allocations for the garbage collector.
// This struct is prepended to every allocation.
type memTreapNode struct {
	parent, left, right *memTreapNode
	size                uintptr
	base                struct{}
}

// priority computes a pseudo-random priority value for this treap node.
// This value is a fibonacci hash (https://en.wikipedia.org/wiki/Hash_function#Fibonacci_hashing) of the node's memory address.
func (n *memTreapNode) priority() uintptr {
	// Select fibonacci multiplier for this bit-width.
	var fibonacciMultiplier uint64
	switch 8 * unsafe.Sizeof(uintptr(0)) {
	case 16:
		fibonacciMultiplier = 40503
	case 32:
		fibonacciMultiplier = 2654435769
	case 64:
		fibonacciMultiplier = 11400714819323198485
	default:
		runtimePanic("invalid size of uintptr")
	}

	// Hash the pointer.
	return uintptr(fibonacciMultiplier) * uintptr(unsafe.Pointer(n))
}

// contains returns whether this allocation contains a given address.
func (n *memTreapNode) contains(addr uintptr) bool {
	return addr >= uintptr(unsafe.Pointer(&n.base)) && addr < uintptr(unsafe.Pointer(&n.base))+n.size
}

// parentSlot returns a pointer to the parent's reference to this node.
func (n *memTreapNode) parentSlot() **memTreapNode {
	if uintptr(unsafe.Pointer(n)) > uintptr(unsafe.Pointer(n.parent)) {
		return &n.parent.left
	} else {
		return &n.parent.right
	}
}

// memScanQueue is a queue of memTreapNodes.
type memScanQueue struct {
	head, tail *memTreapNode
}

// push adds an allocation onto the queue.
func (q *memScanQueue) push(n *memTreapNode) {
	if gcAsserts && (n.left != nil || n.right != nil || n.parent != nil) {
		runtimePanic("tried to push a treap node that is in use")
	}

	if q.head == nil {
		q.tail = n
	} else {
		q.head.left = n
	}
	n.right = q.head
	q.head = n
}

// pop removes the next allocation from the queue.
func (q *memScanQueue) pop() *memTreapNode {
	n := q.tail
	q.tail = n.left
	if q.tail == nil {
		q.head = nil
	}
	n.left = nil
	n.right = nil
	return n
}

// empty returns whether the queue contains any allocations.
func (q *memScanQueue) empty() bool {
	return q.tail == nil
}

// allocations is a treap containing all allocations.
var allocations memTreap

// usedMem is the total amount of allocated memory (including the space taken up by memory treap nodes).
var usedMem uintptr

// firstPtr and lastPtr are the bounds of memory used by the heap.
// They are computed before the collector starts marking, and are used to quickly eliminate false positives.
var firstPtr, lastPtr uintptr

// scanQueue is a queue of marked allocations to scan.
var scanQueue memScanQueue

// mark searches for an allocation containing the given address and marks it if found.
func mark(addr uintptr) bool {
	if addr < firstPtr || addr > lastPtr {
		// Pointer is outside of allocated bounds.
		return false
	}

	node := allocations.lookupAddr(addr)
	if node != nil {
		if gcDebug {
			println("mark:", addr)
		}
		allocations.remove(node)
		scanQueue.push(node)
	}

	return node != nil
}

func markRoot(addr uintptr, root uintptr) {
	marked := mark(root)
	if gcDebug {
		if marked {
			println("marked root:", root, "at", addr)
		} else if addr != 0 {
			println("did not mark root:", root, "at", addr)
		}
	}
}

func markRoots(start uintptr, end uintptr) {
	scan(start, end)
}

// scan loads all pointer-aligned words and marks any pointers that it finds.
func scan(start uintptr, end uintptr) {
	// Align start and end pointers.
	start = (start + unsafe.Alignof(unsafe.Pointer(nil)) - 1) &^ (unsafe.Alignof(unsafe.Pointer(nil)) - 1)
	end &^= unsafe.Alignof(unsafe.Pointer(nil)) - 1

	// Mark all pointers.
	for ptr := start; ptr < end; ptr += unsafe.Alignof(unsafe.Pointer(nil)) {
		mark(*(*uintptr)(unsafe.Pointer(ptr)))
	}
}

// scan marks all allocations referenced by this allocation.
// This should only be invoked by the garbage collector.
func (n *memTreapNode) scan() {
	start := uintptr(unsafe.Pointer(&n.base))
	end := start + n.size
	scan(start, end)
}

// destroy removes and frees all allocations in the treap.
func (t *memTreap) destroy() {
	n := t.root
	for n != nil {
		switch {
		case n.left != nil:
			// Destroy the left subtree.
			n = n.left
		case n.right != nil:
			// Destroy the right subtree.
			n = n.right
		default:
			// This is a leaf node, so delete it and jump back to the parent.

			// Save the parent to jump back to.
			parent := n.parent

			if parent != nil {
				*n.parentSlot() = nil
			} else {
				t.root = nil
			}

			// Update used memory.
			usedMem -= unsafe.Sizeof(memTreapNode{}) + n.size
			if gcDebug {
				println("collecting:", &n.base, "size:", n.size)
				println("used memory:", usedMem)
			}

			// Free the node.
			extfree(unsafe.Pointer(n))

			// Jump back to the parent node.
			n = parent
		}
	}
}

// gcrunning is used by gcAsserts to determine whether the garbage collector is running.
// This is used to detect if the collector is invoking itself or trying to allocate memory.
var gcrunning bool

// activeMem is a queue used to store marked allocations which have already been scanned.
// This is only used when the garbage collector is running.
var activeMem memScanQueue

func GC() {
	if gcDebug {
		println("running GC")
	}
	if allocations.empty() {
		// Skip collection because the heap is empty.
		if gcDebug {
			println("nothing to collect")
		}
		return
	}

	if gcAsserts {
		if gcrunning {
			runtimePanic("GC called itself")
		}
		gcrunning = true
	}

	if gcDebug {
		println("pre-GC allocations:")
		allocations.print()
	}

	// Before scanning, find the lowest and highest allocated pointers.
	// These can be quickly compared against to eliminate most false positives.
	firstPtr, lastPtr = allocations.minAddr(), allocations.maxAddr()

	// Start by scanning the stack.
	markStack()

	// Scan all globals.
	markGlobals()

	// Channel operations in interrupts may move task pointers around while we are marking.
	// Therefore we need to scan the runqueue seperately.
	var markedTaskQueue task.Queue
runqueueScan:
	for !runqueue.Empty() {
		// Pop the next task off of the runqueue.
		t := runqueue.Pop()

		// Mark the task if it has not already been marked.
		markRoot(uintptr(unsafe.Pointer(&runqueue)), uintptr(unsafe.Pointer(t)))

		// Push the task onto our temporary queue.
		markedTaskQueue.Push(t)
	}

	// Scan all referenced allocations, building a new treap with marked allocations.
	// The marking process deletes the allocations from the old allocations treap, so they are only queued once.
	for !scanQueue.empty() {
		// Pop a marked node off of the scan queue.
		n := scanQueue.pop()

		// Scan and mark all nodes that this references.
		n.scan()

		// Insert this node into the active memory queue.
		activeMem.push(n)
	}

	i := interrupt.Disable()
	if !runqueue.Empty() {
		// Something new came in while finishing the mark.
		interrupt.Restore(i)
		goto runqueueScan
	}
	runqueue = markedTaskQueue
	interrupt.Restore(i)

	// The allocations treap now only contains unreferenced nodes. Destroy them all.
	allocations.destroy()
	if gcAsserts && !allocations.empty() {
		runtimePanic("failed to fully destroy allocations")
	}

	// Treapify the active memory queue.
	for !activeMem.empty() {
		allocations.insert(activeMem.pop())
	}

	if gcDebug {
		println("GC finished")
	}

	if gcAsserts {
		gcrunning = false
	}
}

// heapBound is used to control the growth of the heap.
// When the heap exceeds this size, the garbage collector is run.
// If the garbage collector cannot free up enough memory, the bound is doubled until the allocation fits.
var heapBound uintptr = 4 * unsafe.Sizeof(memTreapNode{})

// zeroSizedAlloc is just a sentinel that gets returned when allocating 0 bytes.
var zeroSizedAlloc uint8

// alloc tries to find some free space on the heap, possibly doing a garbage
// collection cycle if needed. If no space is free, it panics.
//go:noinline
func alloc(size uintptr) unsafe.Pointer {
	if size == 0 {
		return unsafe.Pointer(&zeroSizedAlloc)
	}

	if gcAsserts && gcrunning {
		runtimePanic("allocated inside the garbage collector")
	}

	// Calculate size of allocation including treap node.
	allocSize := unsafe.Sizeof(memTreapNode{}) + size

	var gcRan bool
	for {
		// Try to bound heap growth.
		if usedMem+allocSize < usedMem {
			if gcDebug {
				println("current mem:", usedMem, "alloc size:", allocSize)
			}
			runtimePanic("target heap size exceeds address space size")
		}
		if usedMem+allocSize > heapBound {
			if !gcRan {
				// Run the garbage collector before growing the heap.
				if gcDebug {
					println("heap reached size limit")
				}
				GC()
				gcRan = true
				continue
			} else {
				// Grow the heap bound to fit the allocation.
				for heapBound != 0 && usedMem+allocSize > heapBound {
					heapBound <<= 1
				}
				if heapBound == 0 {
					// This is only possible on hosted 32-bit systems.
					// Allow the heap bound to encompass everything.
					heapBound = ^uintptr(0)
				}
				if gcDebug {
					println("raising heap size limit to", heapBound)
				}
			}
		}

		// Allocate the memory.
		ptr := extalloc(allocSize)
		if ptr == nil {
			if gcDebug {
				println("extalloc failed")
			}
			if gcRan {
				// Garbage collector was not able to free up enough memory.
				runtimePanic("out of memory")
			} else {
				// Run the garbage collector and try again.
				GC()
				gcRan = true
				continue
			}
		}

		// Initialize the memory treap node.
		node := (*memTreapNode)(ptr)
		*node = memTreapNode{
			size: size,
		}

		// Insert allocation into the allocations treap.
		allocations.insert(node)

		// Extract the user's section of the allocation.
		ptr = unsafe.Pointer(&node.base)
		if gcAsserts && !node.contains(uintptr(ptr)) {
			runtimePanic("node is not self-contained")
		}
		if gcAsserts {
			check := allocations.lookupAddr(uintptr(ptr))
			if check == nil {
				if gcDebug {
					println("failed to find:", ptr)
					allocations.print()
				}
				runtimePanic("bad insert")
			}
		}

		// Zero the allocation.
		memzero(ptr, size)

		// Update used memory.
		usedMem += allocSize

		if gcDebug {
			println("allocated:", uintptr(ptr), "size:", size)
			println("used memory:", usedMem)
		}

		return ptr
	}
}

func free(ptr unsafe.Pointer) {
	// Currently unimplemented due to bugs in coroutine lowering.
}

func KeepAlive(x interface{}) {
	// Unimplemented. Only required with SetFinalizer().
}

func SetFinalizer(obj interface{}, finalizer interface{}) {
	// Unimplemented.
}