ci: only build the necessary LLVM libraries and tools

This should improve rebuild time, but perhaps more importantly massively
reduces cache size which then reduces incremental build time.

.circleci/config.yml

@@ -80,12 +80,12 @@ commands:
     steps:
       - restore_cache:
           keys:
-            - wasi-libc-sysroot-v3
+            - wasi-libc-sysroot-v4
       - run:
           name: "Build wasi-libc"
           command: make wasi-libc
       - save_cache:
-          key: wasi-libc-sysroot-v3
+          key: wasi-libc-sysroot-v4
           paths:
             - lib/wasi-libc/sysroot
   test-linux:
@@ -108,10 +108,10 @@ commands:
       - run: go install -tags=llvm<<parameters.llvm>> .
       - restore_cache:
           keys:
-            - wasi-libc-sysroot-systemclang-v2
+            - wasi-libc-sysroot-systemclang-v3
       - run: make wasi-libc
       - save_cache:
-          key: wasi-libc-sysroot-systemclang-v2
+          key: wasi-libc-sysroot-systemclang-v3
           paths:
             - lib/wasi-libc/sysroot
       - run: go test -v -tags=llvm<<parameters.llvm>> ./cgo ./compileopts ./compiler ./interp ./transform .
@@ -153,7 +153,7 @@ commands:
       - llvm-source-linux
       - restore_cache:
           keys:
-            - llvm-build-11-linux-v1-assert
+            - llvm-build-11-linux-v2-assert
       - run:
           name: "Build LLVM"
           command: |
@@ -168,7 +168,7 @@ commands:
               make ASSERT=1 llvm-build
             fi
       - save_cache:
-          key: llvm-build-11-linux-v1-assert
+          key: llvm-build-11-linux-v2-assert
           paths:
             llvm-build
       - run: make ASSERT=1
@@ -211,7 +211,7 @@ commands:
       - llvm-source-linux
       - restore_cache:
           keys:
-            - llvm-build-11-linux-v1-noassert
+            - llvm-build-11-linux-v2-noassert
       - run:
           name: "Build LLVM"
           command: |
@@ -226,7 +226,7 @@ commands:
               make llvm-build
             fi
       - save_cache:
-          key: llvm-build-11-linux-v1-noassert
+          key: llvm-build-11-linux-v2-noassert
           paths:
             llvm-build
       - build-wasi-libc
@@ -290,7 +290,7 @@ commands:
             - llvm-project
       - restore_cache:
           keys:
-            - llvm-build-11-macos-v1
+            - llvm-build-11-macos-v2
       - run:
           name: "Build LLVM"
           command: |
@@ -302,17 +302,17 @@ commands:
               make llvm-build
             fi
       - save_cache:
-          key: llvm-build-11-macos-v1
+          key: llvm-build-11-macos-v2
           paths:
             llvm-build
       - restore_cache:
           keys:
-            - wasi-libc-sysroot-macos-v2
+            - wasi-libc-sysroot-macos-v3
       - run:
           name: "Build wasi-libc"
           command: make wasi-libc
       - save_cache:
-          key: wasi-libc-sysroot-macos-v2
+          key: wasi-libc-sysroot-macos-v3
           paths:
             - lib/wasi-libc/sysroot
       - run:

Makefile

@@ -54,32 +54,46 @@ ifeq ($(OS),Windows_NT)
     CGO_LDFLAGS += -static -static-libgcc -static-libstdc++
     CGO_LDFLAGS_EXTRA += -lversion
 
-    LIBCLANG_PATH = $(abspath $(LLVM_BUILDDIR))/lib/liblibclang.a
+    LIBCLANG_NAME = libclang
 
 else ifeq ($(shell uname -s),Darwin)
     MD5SUM = md5
-    LIBCLANG_PATH = $(abspath $(LLVM_BUILDDIR))/lib/libclang.a
+    LIBCLANG_NAME = clang
 else ifeq ($(shell uname -s),FreeBSD)
     MD5SUM = md5
-    LIBCLANG_PATH = $(abspath $(LLVM_BUILDDIR))/lib/libclang.a
+    LIBCLANG_NAME = clang
     START_GROUP = -Wl,--start-group
     END_GROUP = -Wl,--end-group
 else
-    LIBCLANG_PATH = $(abspath $(LLVM_BUILDDIR))/lib/libclang.a
+    LIBCLANG_NAME = clang
     START_GROUP = -Wl,--start-group
     END_GROUP = -Wl,--end-group
 endif
 
-CLANG_LIBS = $(START_GROUP) -lclangAnalysis -lclangARCMigrate -lclangAST -lclangASTMatchers -lclangBasic -lclangCodeGen -lclangCrossTU -lclangDriver -lclangDynamicASTMatchers -lclangEdit -lclangFormat -lclangFrontend -lclangFrontendTool -lclangHandleCXX -lclangHandleLLVM -lclangIndex -lclangLex -lclangParse -lclangRewrite -lclangRewriteFrontend -lclangSema -lclangSerialization -lclangStaticAnalyzerCheckers -lclangStaticAnalyzerCore -lclangStaticAnalyzerFrontend -lclangTooling -lclangToolingASTDiff -lclangToolingCore -lclangToolingInclusions $(END_GROUP) -lstdc++
+# Libraries that should be linked in for the statically linked Clang.
+CLANG_LIB_NAMES = clangAnalysis clangARCMigrate clangAST clangASTMatchers clangBasic clangCodeGen clangCrossTU clangDriver clangDynamicASTMatchers clangEdit clangFormat clangFrontend clangFrontendTool clangHandleCXX clangHandleLLVM clangIndex clangLex clangParse clangRewrite clangRewriteFrontend clangSema clangSerialization clangStaticAnalyzerCheckers clangStaticAnalyzerCore clangStaticAnalyzerFrontend clangTooling clangToolingASTDiff clangToolingCore clangToolingInclusions
+CLANG_LIBS = $(START_GROUP) $(addprefix -l,$(CLANG_LIB_NAMES)) $(END_GROUP) -lstdc++
 
-LLD_LIBS = $(START_GROUP) -llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML $(END_GROUP)
+# Libraries that should be linked in for the statically linked LLD.
+LLD_LIB_NAMES = lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML
+LLD_LIBS = $(START_GROUP) $(addprefix -l,$(LLD_LIB_NAMES)) $(END_GROUP)
 
+# Other libraries that are needed to link TinyGo.
+EXTRA_LIB_NAMES = LLVMInterpreter
+
+# These build targets appear to be the only ones necessary to build all TinyGo
+# dependencies. Only building a subset significantly speeds up rebuilding LLVM.
+# The Makefile rules convert a name like lldELF to lib/liblldELF.a to match the
+# library path (for ninja).
+# This list also includes a few tools that are necessary as part of the full
+# TinyGo build.
+NINJA_BUILD_TARGETS = clang llvm-config llvm-ar llvm-nm $(addprefix lib/lib,$(addsuffix .a,$(LIBCLANG_NAME) $(CLANG_LIB_NAMES) $(LLD_LIB_NAMES) $(EXTRA_LIB_NAMES)))
 
 # For static linking.
 ifneq ("$(wildcard $(LLVM_BUILDDIR)/bin/llvm-config*)","")
     CGO_CPPFLAGS+=$(shell $(LLVM_BUILDDIR)/bin/llvm-config --cppflags) -I$(abspath $(LLVM_BUILDDIR))/tools/clang/include -I$(abspath $(CLANG_SRC))/include -I$(abspath $(LLD_SRC))/include
     CGO_CXXFLAGS=-std=c++14
-    CGO_LDFLAGS+=$(LIBCLANG_PATH) -L$(abspath $(LLVM_BUILDDIR)/lib) $(CLANG_LIBS) $(LLD_LIBS) $(shell $(LLVM_BUILDDIR)/bin/llvm-config --ldflags --libs --system-libs $(LLVM_COMPONENTS)) -lstdc++ $(CGO_LDFLAGS_EXTRA)
+    CGO_LDFLAGS+=$(abspath $(LLVM_BUILDDIR))/lib/lib$(LIBCLANG_NAME).a -L$(abspath $(LLVM_BUILDDIR)/lib) $(CLANG_LIBS) $(LLD_LIBS) $(shell $(LLVM_BUILDDIR)/bin/llvm-config --ldflags --libs --system-libs $(LLVM_COMPONENTS)) -lstdc++ $(CGO_LDFLAGS_EXTRA)
 endif
 
 
@@ -146,7 +160,7 @@ $(LLVM_BUILDDIR)/build.ninja: llvm-source
 
 # Build LLVM.
 $(LLVM_BUILDDIR): $(LLVM_BUILDDIR)/build.ninja
-	cd $(LLVM_BUILDDIR); ninja
+	cd $(LLVM_BUILDDIR); ninja $(NINJA_BUILD_TARGETS)
 
 
 # Build wasi-libc sysroot

azure-pipelines.yml

@@ -32,7 +32,7 @@ jobs:
     - task: CacheBeta@0
       displayName: Cache LLVM build
       inputs:
-        key: llvm-build-11-windows-v3
+        key: llvm-build-11-windows-v4
         path: llvm-build
     - task: Bash@3
       displayName: Build LLVM
@@ -56,7 +56,7 @@ jobs:
     - task: CacheBeta@0
       displayName: Cache wasi-libc sysroot
       inputs:
-        key: wasi-libc-sysroot-v3
+        key: wasi-libc-sysroot-v4
         path: lib/wasi-libc/sysroot
     - task: Bash@3
       displayName: Build wasi-libc