shard sema and codegen across N llvm modules

- ZIG_PARALLEL_SEMA: Sema runs concurrently across worker threads with
  per-unit claim/wait, retry-on-dependency-cycle, and per-map mutexes
  replacing the global sema_lock for the non-incremental fast path.
- InternPool: thread-safe writers (locked single-field setters, seqlock
  on getNav, sorted-shard prelocking for getFunc*Ies, 256 hash shards).
- llvm backend: PartitionSet emits N independent llvm modules in parallel;
  cross-shard refs are linkonce_odr; --llvm-codegen-threads=N partitions
  by file path; --llvm-no-merge-shards leaves shard .o files unmerged.
- link.MachO -r: handle N shard inputs; emit hidden defs as private-extern;
  convert tentatives so Apple ld_new accepts the merged object.
- link.Elf: handle N shard inputs; batch preads in writeRelocatable to
  avoid per-atom syscall storm under heavy COMDAT section counts.
- link.Lld: pass all shard paths to lld for elf/coff/wasm.
- std.Build.Step.Compile: llvm_codegen_threads, llvm_no_merge_shards.

Author: alii

.gitignore

@@ -20,3 +20,4 @@ zig-out/
 # Although this was renamed to .zig-cache, let's leave it here for a few
 # releases to make it less annoying to work with multiple branches.
 zig-cache/
+bun-cache/

build.zig

@@ -123,6 +123,11 @@ pub fn build(b: *std.Build) !void {
         "llvm-has-xtensa",
         "Whether LLVM has the experimental target xtensa enabled",
     ) orelse false;
+    const llvm_has_polly = b.option(
+        bool,
+        "llvm-has-polly",
+        "Whether LLVM was built with Polly and requires linking it",
+    ) orelse false;
     const enable_ios_sdk = b.option(bool, "enable-ios-sdk", "Run tests requiring presence of iOS SDK and frameworks") orelse false;
     const enable_macos_sdk = b.option(bool, "enable-macos-sdk", "Run tests requiring presence of macOS SDK and frameworks") orelse enable_ios_sdk;
     const enable_symlinks_windows = b.option(bool, "enable-symlinks-windows", "Run tests requiring presence of symlinks on Windows") orelse false;
@@ -332,6 +337,7 @@ pub fn build(b: *std.Build) !void {
                 .llvm_has_csky = llvm_has_csky,
                 .llvm_has_arc = llvm_has_arc,
                 .llvm_has_xtensa = llvm_has_xtensa,
+                .llvm_has_polly = llvm_has_polly,
             });
         }
         if (target.result.os.tag == .windows) {
@@ -739,7 +745,7 @@ fn addCompilerMod(b: *std.Build, options: AddCompilerModOptions) *std.Build.Modu
 fn addCompilerStep(b: *std.Build, options: AddCompilerModOptions) *std.Build.Step.Compile {
     const exe = b.addExecutable(.{
         .name = "zig",
-        .max_rss = 10_000_000_000,
+        .max_rss = 11_000_000_000,
         .root_module = addCompilerMod(b, options),
     });
     exe.stack_size = stack_size;
@@ -858,6 +864,7 @@ fn addStaticLlvmOptionsToModule(mod: *std.Build.Module, options: struct {
     llvm_has_csky: bool,
     llvm_has_arc: bool,
     llvm_has_xtensa: bool,
+    llvm_has_polly: bool,
 }) !void {
     // Adds the Zig C++ sources which both stage1 and stage2 need.
     //
@@ -898,6 +905,10 @@ fn addStaticLlvmOptionsToModule(mod: *std.Build.Module, options: struct {
         mod.linkSystemLibrary(lib_name, .{});
     };
 
+    if (options.llvm_has_polly) for (llvm_libs_polly) |lib_name| {
+        mod.linkSystemLibrary(lib_name, .{});
+    };
+
     mod.linkSystemLibrary("z", .{});
     mod.linkSystemLibrary("zstd", .{});
 
@@ -1419,6 +1430,10 @@ const llvm_libs_xtensa = [_][]const u8{
     "LLVMXtensaDesc",
     "LLVMXtensaInfo",
 };
+const llvm_libs_polly = [_][]const u8{
+    "Polly",
+    "PollyISL",
+};
 
 fn generateLangRef(b: *std.Build) std.Build.LazyPath {
     const doctest_exe = b.addExecutable(.{

lib/std/Build/Step/Compile.zig

@@ -162,6 +162,11 @@ dead_strip_dylibs: bool = false,
 /// When enabled, outputs multiple .o files: filename.0.o, filename.1.o, etc.
 llvm_codegen_threads: u32 = 0,
 
+/// Skip the relocatable -r merge of partitioned LLVM output. The shard
+/// objects are emitted directly to `{emit}.{i}.o` for the downstream linker
+/// to consume. Only meaningful when `llvm_codegen_threads > 1`.
+llvm_no_merge_shards: bool = false,
+
 /// Skip linker step for build-obj - outputs raw LLVM object file(s).
 /// Saves time by avoiding parse/resolve/write cycle.
 no_link_obj: bool = false,
@@ -1532,6 +1537,9 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
     if (compile.llvm_codegen_threads > 0) {
         try zig_args.append(b.fmt("--llvm-codegen-threads={d}", .{compile.llvm_codegen_threads}));
     }
+    if (compile.llvm_no_merge_shards) {
+        try zig_args.append("--llvm-no-merge-shards");
+    }
     if (compile.no_link_obj) {
         try zig_args.append("--no-link");
     }

src/Air.zig

@@ -2154,6 +2154,7 @@ pub fn unwrapShuffleTwo(air: *const Air, zcu: *const Zcu, inst_index: Inst.Index
 }
 
 pub const typesFullyResolved = types_resolved.typesFullyResolved;
+pub const resolveTypesFully = types_resolved.resolveTypesFully;
 pub const typeFullyResolved = types_resolved.checkType;
 pub const valFullyResolved = types_resolved.checkVal;
 pub const legalize = Legalize.legalize;

src/Air/types_resolved.zig

@@ -10,6 +10,21 @@ pub fn typesFullyResolved(air: Air, zcu: *Zcu) bool {
     return checkBody(air, air.getMainBody(), zcu);
 }
 
+/// Under parallel Sema, `resolve_type_fully` and `codegen_func` run
+/// concurrently, so types may be mid-resolution rather than failed. Walk the
+/// same AIR shape as `typesFullyResolved` but force-resolve each struct/union
+/// (blocking on `claimOrWait`-gated resolution). Returns false only if
+/// resolution itself errors.
+pub fn resolveTypesFully(air: Air, pt: Zcu.PerThread) bool {
+    tls_resolve_pt = pt;
+    defer tls_resolve_pt = null;
+    return checkBody(air, air.getMainBody(), pt.zcu);
+}
+
+/// `checkType` is reached via a long instruction walk; thread the optional
+/// PerThread via tls instead of plumbing it through every switch arm.
+threadlocal var tls_resolve_pt: ?Zcu.PerThread = null;
+
 fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
     const tags = air.instructions.items(.tag);
     const datas = air.instructions.items(.data);
@@ -513,6 +528,10 @@ pub fn checkType(ty: Type, zcu: *Zcu) bool {
         },
         .@"struct" => switch (ip.indexToKey(ty.toIntern())) {
             .struct_type => {
+                if (tls_resolve_pt) |pt| {
+                    ty.resolveFully(pt) catch return false;
+                    return true;
+                }
                 const struct_obj = zcu.typeToStruct(ty).?;
                 return switch (struct_obj.layout) {
                     .@"packed" => struct_obj.backingIntTypeUnordered(ip) != .none,
@@ -530,6 +549,12 @@ pub fn checkType(ty: Type, zcu: *Zcu) bool {
             },
             else => unreachable,
         },
-        .@"union" => return zcu.typeToUnion(ty).?.flagsUnordered(ip).status == .fully_resolved,
+        .@"union" => {
+            if (tls_resolve_pt) |pt| {
+                ty.resolveFully(pt) catch return false;
+                return true;
+            }
+            return zcu.typeToUnion(ty).?.flagsUnordered(ip).status == .fully_resolved;
+        },
     };
 }