build(darwin): fix macOS arm64 compilation and enable VideoToolbox

Resolve three blocking issues for building FFmpeg and 20 dependencies on
macOS arm64 within Nix development environment:
- Force clang compiler via FFmpeg configure flags and build environment
- Disable x264 assembly on Apple Silicon (GNU macro incompatibility with
LLVM)
- Build rav1e without git_version feature to avoid libgit2/libiconv
dependency
- Configure C++ header search order with -nostdinc++ and explicit libc++
path
- Filter NIX_CFLAGS_COMPILE to prevent incorrect -isystem injection
- Use Apple's /usr/bin/libtool instead of GNU libtool for library
combining
- Add VideoToolbox hardware encoder support (H.264, HEVC, ProRes)
- Export LIBCXX_INCLUDE from flake.nix for C++ dependency builds
Root cause: Nix dev shell provides both GCC and Clang, but CGO_CFLAGS
points to Clang's builtin headers containing preprocessor features
(__has_feature, __building_module) that GCC doesn't recognise.
Verified: All 17 platform-applicable libraries build successfully,
combine into lib/darwin_arm64/libffmpeg.a (63MB), tests pass,
VideoToolbox encoders confirmed present.

Signed-off-by: Martin Wimpress <martin@wimpress.org>

Author: flexiondotorg

flake.nix

@@ -53,6 +53,11 @@
               cargo-c
               rustc
             ]
+            ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [
+              # C++ standard library headers for building C++ dependencies (zimg, etc.)
+              # The .dev output contains include/c++/v1/ with <algorithm>, <iostream>, etc.
+              llvmPackages_18.libcxx
+            ]
             ++ pkgs.lib.optionals pkgs.stdenv.isLinux [
               # Hardware acceleration runtime (Linux only)
               vulkan-loader # Required for Vulkan accelerated encoders
@@ -73,6 +78,9 @@
             # CGO needs both the SDK path and clang's builtin headers (stdarg.h, stddef.h, etc.)
             export CGO_CFLAGS="-isysroot ''${SDKROOT:-$(xcrun --show-sdk-path)} -I${pkgs.llvmPackages_18.libclang.lib}/lib/clang/18/include"
             export CPATH="${pkgs.llvmPackages_18.libclang.dev}/include:$CPATH"
+            # C++ standard library headers path for building C++ dependencies (zimg, etc.)
+            # The builder uses this to add -I flag for <algorithm>, <iostream>, etc.
+            export LIBCXX_INCLUDE="${pkgs.llvmPackages_18.libcxx.dev}/include/c++/v1"
             # Set deployment target to match ffmpeg-statigo build (macOS 13.0+)
             export MACOSX_DEPLOYMENT_TARGET="13.0"
             # macOS uses DYLD_LIBRARY_PATH instead of LD_LIBRARY_PATH

internal/builder/buildsystems.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 )
 
 // stagingDir derives the staging/install directory from a build directory path.
@@ -63,11 +64,49 @@ func (a *AutoconfBuild) Configure(lib *Library, srcPath, buildDir, installDir st
 
 		cflags := fmt.Sprintf("-O3 -I%s", incDir)
 		cppflags := fmt.Sprintf("-I%s", incDir)
+		cxxflags := "-O3" // Start with optimization flag
 		ldflags := fmt.Sprintf("-L%s", libDir)
 
+		// On macOS, configure proper SDK and header paths
+		// This is required for CGO compilation and C++ builds
+		if runtime.GOOS == "darwin" {
+			// CGO_CFLAGS format: -isysroot /path/to/sdk -I/nix/store/.../lib/clang/18/include
+			cgoCflags := os.Getenv("CGO_CFLAGS")
+			libcxxInclude := os.Getenv("LIBCXX_INCLUDE")
+
+			// For C: add SDK path and clang builtins
+			if cgoCflags != "" {
+				cflags = fmt.Sprintf("%s %s", cflags, cgoCflags)
+			}
+
+			// For CPPFLAGS: only add -isysroot, NOT the clang builtin include path
+			// This prevents C++ from finding clang's stddef.h before libc++'s wrapper
+			if cgoCflags != "" {
+				// Extract just the -isysroot part from CGO_CFLAGS
+				// CGO_CFLAGS = "-isysroot /path/to/sdk -I/path/to/clang/include"
+				// We only want the -isysroot portion for CPPFLAGS
+				parts := strings.Fields(cgoCflags)
+				for i, part := range parts {
+					if part == "-isysroot" && i+1 < len(parts) {
+						cppflags = fmt.Sprintf("%s -isysroot %s", cppflags, parts[i+1])
+						break
+					}
+				}
+			}
+
+			// For C++: use -nostdinc++ to disable built-in paths, then add libc++ first,
+			// then clang builtins (for stddef.h, stdarg.h), then SDK path
+			if libcxxInclude != "" && cgoCflags != "" {
+				cxxflags = fmt.Sprintf("%s -nostdinc++ -I%s %s", cxxflags, libcxxInclude, cgoCflags)
+			} else if cgoCflags != "" {
+				cxxflags = fmt.Sprintf("%s %s", cxxflags, cgoCflags)
+			}
+		}
+
 		args = append(args,
 			fmt.Sprintf("CFLAGS=%s", cflags),
 			fmt.Sprintf("CPPFLAGS=%s", cppflags),
+			fmt.Sprintf("CXXFLAGS=%s", cxxflags),
 			fmt.Sprintf("LDFLAGS=%s", ldflags),
 		)
 	}

internal/builder/ffmpeg_args.go

@@ -393,5 +393,32 @@ func FFmpegArgsCommon(os string) []string {
 		)
 	}
 
+	// macOS-specific hardware acceleration (VideoToolbox)
+	if os == "darwin" {
+		args = append(args,
+			// H.264 VideoToolbox encoder and hwaccel
+			"--enable-encoder=h264_videotoolbox",
+			"--enable-hwaccel=h264_videotoolbox",
+			// H.265/HEVC VideoToolbox encoder and hwaccel
+			"--enable-encoder=hevc_videotoolbox",
+			"--enable-hwaccel=hevc_videotoolbox",
+			// ProRes VideoToolbox encoder and hwaccel
+			"--enable-encoder=prores_videotoolbox",
+			"--enable-hwaccel=prores_videotoolbox",
+			// AV1 VideoToolbox hwaccel (M3+ decode support)
+			"--enable-hwaccel=av1_videotoolbox",
+			// VP9 VideoToolbox hwaccel
+			"--enable-hwaccel=vp9_videotoolbox",
+			// MPEG-2 VideoToolbox hwaccel
+			"--enable-hwaccel=mpeg2_videotoolbox",
+			// MPEG-4 VideoToolbox hwaccel
+			"--enable-hwaccel=mpeg4_videotoolbox",
+			// H.263 VideoToolbox hwaccel
+			"--enable-hwaccel=h263_videotoolbox",
+			// MPEG-1 VideoToolbox hwaccel
+			"--enable-hwaccel=mpeg1_videotoolbox",
+		)
+	}
+
 	return args
 }

internal/builder/libraries.go

@@ -537,7 +537,7 @@ var x264 = &Library{
 	BuildSystem:   &AutoconfBuild{},
 	SkipAutoFlags: true, // x264 has a custom configure script that rejects CFLAGS/LDFLAGS
 	ConfigureArgs: func(os string) []string {
-		return []string{
+		args := []string{
 			"--disable-avs",
 			"--disable-cli",
 			"--disable-ffms",
@@ -548,6 +548,12 @@ var x264 = &Library{
 			"--enable-static",
 			"--enable-strip",
 		}
+		// x264's aarch64 assembly uses GNU assembler macros (const, endconst, T())
+		// that aren't compatible with LLVM's integrated assembler on macOS
+		if os == "darwin" && runtime.GOARCH == "arm64" {
+			args = append(args, "--disable-asm")
+		}
+		return args
 	},
 	PostExtract: func(srcPath string) error {
 		// x264 needs to find nasm explicitly on x86/x86_64
@@ -605,16 +611,41 @@ var rav1e = &Library{
 	BuildSystem: &CargoBuild{
 		InstallFunc: func(srcPath, installDir string) error {
 			// Set RUSTFLAGS for native CPU optimization
-			os.Setenv("RUSTFLAGS", "-C target-cpu=native")
+			rustflags := "-C target-cpu=native"
+
+			// On macOS, add SDK library path for any native dependencies
+			if runtime.GOOS == "darwin" {
+				cgoCflags := os.Getenv("CGO_CFLAGS")
+				// Extract SDK path from CGO_CFLAGS (-isysroot <path>)
+				var sdkPath string
+				parts := strings.Fields(cgoCflags)
+				for i, p := range parts {
+					if p == "-isysroot" && i+1 < len(parts) {
+						sdkPath = parts[i+1]
+						break
+					}
+				}
+
+				if sdkPath != "" {
+					sdkLibPath := filepath.Join(sdkPath, "usr", "lib")
+					rustflags += " -C link-arg=-L" + sdkLibPath
+				}
+			}
+
+			os.Setenv("RUSTFLAGS", rustflags)
 			os.Setenv("CARGO_PROFILE_RELEASE_DEBUG", "false")
 
 			// cargo cinstall for C library installation
+			// Use --no-default-features to avoid git_version which pulls in libgit2
+			// Re-enable asm and threading for performance
 			return runCommand(srcPath, os.Stdout, installDir, "cargo", "cinstall",
 				fmt.Sprintf("--prefix=%s", installDir),
 				"--libdir=lib",
 				"--library-type=staticlib",
 				"--crt-static",
-				"--release")
+				"--release",
+				"--no-default-features",
+				"--features=asm,threading")
 		},
 	},
 	LinkLibs: []string{"librav1e"},
@@ -767,7 +798,7 @@ var ffmpeg = &Library{
 		fmt.Printf("Applied OpenSSL 3.6 compatibility patch to tls_openssl.c\n")
 		return nil
 	},
-	ConfigureArgs: func(os string) []string {
+	ConfigureArgs: func(targetOS string) []string {
 		// FFmpeg needs explicit paths to headers and libraries
 		stagingDir, _ := filepath.Abs(".build/staging")
 		incDir := filepath.Join(stagingDir, "include")
@@ -782,8 +813,16 @@ var ffmpeg = &Library{
 			fmt.Sprintf("--extra-ldflags=%s", extraLdflags),
 		}
 
+		// On macOS, force clang as the compiler
+		// The Nix dev shell includes both gcc and clang, but our CFLAGS include
+		// paths to Clang's builtin headers (stddef.h, stdarg.h) which use Clang-specific
+		// features like __has_feature and __building_module that gcc doesn't understand
+		if targetOS == "darwin" {
+			args = append(args, "--cc=clang", "--cxx=clang++")
+		}
+
 		// Add common FFmpeg arguments (platform-specific)
-		args = append(args, FFmpegArgsCommon(os)...)
+		args = append(args, FFmpegArgsCommon(targetOS)...)
 
 		return args
 	},

internal/builder/library.go

@@ -207,5 +207,100 @@ func buildEnv(installDir string) []string {
 		env = append(env, "PATH="+binPath)
 	}
 
+	// On macOS, remove NIX_CFLAGS_COMPILE which interferes with C++ header search order
+	// The Nix clang wrapper injects -isystem paths that cause libc++ headers to be
+	// searched after C standard library headers, breaking <cstddef> and similar wrappers
+	if runtime.GOOS == "darwin" {
+		filtered := env[:0]
+		for _, e := range env {
+			if !strings.HasPrefix(e, "NIX_CFLAGS_COMPILE=") {
+				filtered = append(filtered, e)
+			}
+		}
+		env = filtered
+
+		// Force clang as the compiler on macOS
+		// The Nix dev shell includes both gcc and clang, but our CFLAGS include
+		// paths to Clang's builtin headers (stddef.h, stdarg.h) which use Clang-specific
+		// features like __has_feature and __building_module that gcc doesn't understand
+		env = append(env, "CC=clang", "CXX=clang++")
+	}
+
+	// On macOS, ensure CFLAGS/CXXFLAGS include SDK path and clang builtin headers
+	// This is required for both C (stdarg.h, stddef.h) and C++ (<algorithm>, <cstring>) compilation
+	if runtime.GOOS == "darwin" {
+		cgoCflags := os.Getenv("CGO_CFLAGS")
+		if cgoCflags != "" {
+			// Set CFLAGS with full CGO_CFLAGS (includes -isysroot and -I.../clang/18/include)
+			updatedCflags := false
+			for i, e := range env {
+				if strings.HasPrefix(e, "CFLAGS=") {
+					existing := strings.TrimPrefix(e, "CFLAGS=")
+					env[i] = "CFLAGS=" + existing + " " + cgoCflags
+					updatedCflags = true
+					break
+				}
+			}
+			if !updatedCflags {
+				env = append(env, "CFLAGS="+cgoCflags)
+			}
+
+			// Build CXXFLAGS with -nostdinc++ and explicit libcxx include path
+			// Use -nostdinc++ to disable built-in C++ paths, preventing NIX_CFLAGS_COMPILE
+			// from interfering with header search order
+			// Then add libc++ headers before clang builtins
+			var cxxExtra string
+			libcxxInclude := os.Getenv("LIBCXX_INCLUDE")
+			if libcxxInclude != "" {
+				cxxExtra = "-nostdinc++ -I" + libcxxInclude + " " + cgoCflags
+			} else {
+				cxxExtra = cgoCflags
+			}
+
+			// Set CXXFLAGS with same flags for C++ builds
+			updatedCxxflags := false
+			for i, e := range env {
+				if strings.HasPrefix(e, "CXXFLAGS=") {
+					existing := strings.TrimPrefix(e, "CXXFLAGS=")
+					env[i] = "CXXFLAGS=" + existing + " " + cxxExtra
+					updatedCxxflags = true
+					break
+				}
+			}
+			if !updatedCxxflags {
+				env = append(env, "CXXFLAGS="+cxxExtra)
+			}
+
+			// Extract SDK path from CGO_CFLAGS (-isysroot <path>) for LDFLAGS
+			// This ensures cargo/rustc can find SDK libraries like libiconv
+			var sdkPath string
+			parts := strings.Fields(cgoCflags)
+			for i, p := range parts {
+				if p == "-isysroot" && i+1 < len(parts) {
+					sdkPath = parts[i+1]
+					break
+				}
+			}
+			if sdkPath != "" {
+				ldExtra := "-L" + filepath.Join(sdkPath, "usr", "lib")
+				updatedLdflags := false
+				for i, e := range env {
+					if strings.HasPrefix(e, "LDFLAGS=") {
+						existing := strings.TrimPrefix(e, "LDFLAGS=")
+						env[i] = "LDFLAGS=" + existing + " " + ldExtra
+						updatedLdflags = true
+						break
+					}
+				}
+				if !updatedLdflags {
+					env = append(env, "LDFLAGS="+ldExtra)
+				}
+
+				// Also set LIBRARY_PATH for cargo/rustc which may not use LDFLAGS
+				env = append(env, "LIBRARY_PATH="+filepath.Join(sdkPath, "usr", "lib"))
+			}
+		}
+	}
+
 	return env
 }

internal/builder/main.go

@@ -210,10 +210,10 @@ func combineLibraries(libs []*Library, stagingDir, output string) error {
 	return combineLinux(libFiles, output)
 }
 
-// combineMac uses libtool to combine static libraries on macOS
+// combineMac uses Apple's libtool to combine static libraries on macOS
 // This is more efficient than ar as it doesn't require extracting all object files
 func combineMac(libFiles []string, output string) error {
-	log.Println("Using libtool -static approach (macOS)")
+	log.Println("Using Apple libtool -static approach (macOS)")
 
 	// Ensure output directory exists
 	outputDir := filepath.Dir(output)
@@ -224,10 +224,10 @@ func combineMac(libFiles []string, output string) error {
 	// Remove existing output file if present
 	os.Remove(output)
 
-	// Use libtool to combine libraries directly
-	// libtool -static is Apple's tool specifically designed for this purpose
+	// Use Apple's libtool (not GNU libtool from Nix) to combine libraries directly
+	// Apple's libtool -static is specifically designed for this purpose
 	args := append([]string{"-static", "-o", output}, libFiles...)
-	libtoolCmd := exec.Command("libtool", args...)
+	libtoolCmd := exec.Command("/usr/bin/libtool", args...)
 
 	// Capture output for debugging
 	var stdout, stderr bytes.Buffer