Add sparse_checkout option to git source

Author: ianks

bundler/lib/bundler/dependency.rb

@@ -58,6 +58,12 @@ def glob
       @glob = @options["glob"]
     end
 
+    def sparse_checkout
+      return @sparse_checkout if defined?(@sparse_checkout)
+
+      @sparse_checkout = @options["sparse_checkout"]
+    end
+
     def platforms
       @platforms ||= Array(@options["platforms"])
     end

bundler/lib/bundler/dsl.rb

@@ -17,7 +17,7 @@ def self.evaluate(gemfile, lockfile, unlock)
     VALID_PLATFORMS = Bundler::CurrentRuby::PLATFORM_MAP.keys.freeze
 
     VALID_KEYS = %w[group groups git path glob name branch ref tag require submodules
-                    platform platforms source install_if force_ruby_platform].freeze
+                    platform platforms source install_if force_ruby_platform sparse_checkout].freeze
 
     GITHUB_PULL_REQUEST_URL = %r{\Ahttps://github\.com/([A-Za-z0-9_\-\.]+/[A-Za-z0-9_\-\.]+)/pull/(\d+)\z}
     GITLAB_MERGE_REQUEST_URL = %r{\Ahttps://gitlab\.com/([A-Za-z0-9_\-\./]+)/-/merge_requests/(\d+)\z}

bundler/lib/bundler/lockfile_parser.rb

@@ -47,7 +47,7 @@ def to_s
     PATH         = "PATH"
     PLUGIN       = "PLUGIN SOURCE"
     SPECS        = "  specs:"
-    OPTIONS      = /^  ([a-z]+): (.*)$/i
+    OPTIONS      = /^  ([a-z_]+): (.*)$/i
     SOURCE       = [GIT, GEM, PATH, PLUGIN].freeze
 
     SECTIONS_BY_VERSION_INTRODUCED = {

bundler/lib/bundler/man/gemfile.5.ronn

@@ -352,6 +352,24 @@ Git repositories support a number of additional options.
     Specify `submodules: true` to cause bundler to expand any
     submodules included in the git repository
 
+  * `sparse_checkout`:
+    Specify directories to include when checking out a git repository.
+    Uses git's sparse-checkout feature (requires git 2.25+) to only
+    download specified directories, reducing disk usage for large repos.
+
+      gem "foo", git: "https://github.com/org/monorepo.git",
+                 sparse_checkout: "packages/foo",
+                 glob: "packages/foo/*.gemspec"
+
+    Multiple directories can be specified as an array:
+
+      gem "bar", github: "org/mono",
+                 sparse_checkout: ["gems/bar", "shared/lib"],
+                 glob: "gems/bar/*.gemspec"
+
+    On git versions below 2.25, the entire repository will be cloned
+    with a warning.
+
 If a git repository contains multiple `.gemspecs`, each `.gemspec`
 represents a gem located at the same place in the file system as
 the `.gemspec`.

bundler/lib/bundler/source/git.rb

@@ -7,7 +7,7 @@ class Source
     class Git < Path
       autoload :GitProxy, File.expand_path("git/git_proxy", __dir__)
 
-      attr_reader :uri, :ref, :branch, :options, :glob, :submodules
+      attr_reader :uri, :ref, :branch, :options, :glob, :submodules, :sparse_checkout
 
       def initialize(options)
         @options = options
@@ -25,6 +25,7 @@ def initialize(options)
         @branch     = options["branch"]
         @ref        = options["ref"] || options["branch"] || options["tag"]
         @submodules = options["submodules"]
+        @sparse_checkout = options["sparse_checkout"]
         @name       = options["name"]
         @version    = options["version"].to_s.strip.gsub("-", ".pre.")
 
@@ -57,27 +58,29 @@ def to_lock
         %w[ref branch tag submodules].each do |opt|
           out << "  #{opt}: #{options[opt]}\n" if options[opt]
         end
+        out << "  sparse_checkout: #{@sparse_checkout}\n" if @sparse_checkout
         out << "  glob: #{@glob}\n" unless default_glob?
         out << "  specs:\n"
       end
 
       def to_gemfile
-        specifiers = %w[ref branch tag submodules glob].map do |opt|
+        specifiers = %w[ref branch tag submodules glob sparse_checkout].map do |opt|
           "#{opt}: #{options[opt]}" if options[opt]
         end
 
         uri_with_specifiers(specifiers)
       end
 
       def hash
-        [self.class, uri, ref, branch, name, glob, submodules].hash
+        [self.class, uri, ref, branch, name, glob, submodules, sparse_checkout].hash
       end
 
       def eql?(other)
         other.is_a?(Git) && uri == other.uri && ref == other.ref &&
           branch == other.branch && name == other.name &&
           glob == other.glob &&
-          submodules == other.submodules
+          submodules == other.submodules &&
+          sparse_checkout == other.sparse_checkout
       end
 
       alias_method :==, :eql?
@@ -86,7 +89,8 @@ def include?(other)
         other.is_a?(Git) && uri == other.uri &&
           name == other.name &&
           glob == other.glob &&
-          submodules == other.submodules
+          submodules == other.submodules &&
+          sparse_checkout == other.sparse_checkout
       end
 
       def to_s
@@ -126,17 +130,13 @@ def name
       # checkout of the git repository. When using local git
       # repos, this is set to the local repo.
       def install_path
-        @install_path ||= begin
-          git_scope = "#{base_name}-#{shortref_for_path(revision)}"
-
-          Bundler.install_path.join(git_scope)
-        end
+        @install_path ||= Bundler.install_path.join("#{base_name}-#{shortref_for_path(revision, sparse_checkout: @sparse_checkout)}")
       end
 
       alias_method :path, :install_path
 
       def extension_dir_name
-        "#{base_name}-#{shortref_for_path(revision)}"
+        "#{base_name}-#{shortref_for_path(revision, sparse_checkout: @sparse_checkout)}"
       end
 
       def unlock!
@@ -246,7 +246,7 @@ def cache_path
       end
 
       def app_cache_dirname
-        "#{base_name}-#{shortref_for_path(locked_revision || revision)}"
+        "#{base_name}-#{shortref_for_path(locked_revision || revision, sparse_checkout: @sparse_checkout)}"
       end
 
       def revision
@@ -375,8 +375,10 @@ def shortref_for_display(ref)
         ref[0..6]
       end
 
-      def shortref_for_path(ref)
-        ref[0..11]
+      def shortref_for_path(ref, sparse_checkout: nil)
+        scope = ref[0..11]
+        scope += "-#{Bundler::Digest.sha1(sparse_checkout)[0..7]}" if sparse_checkout
+        scope
       end
 
       def glob_for_display
@@ -432,7 +434,9 @@ def load_gemspec(file)
       end
 
       def git_scope
-        "#{base_name}-#{uri_hash}"
+        scope = "#{base_name}-#{uri_hash}"
+        scope += "-#{Bundler::Digest.sha1(@sparse_checkout)[0..7]}" if @sparse_checkout
+        scope
       end
 
       def extension_cache_slug(_)

bundler/lib/bundler/source/git/git_proxy.rb

@@ -54,7 +54,7 @@ def initialize(options)
       # All actions required by the Git source is encapsulated in this
       # object.
       class GitProxy
-        attr_accessor :path, :uri, :branch, :tag, :ref, :explicit_ref
+        attr_accessor :path, :uri, :branch, :tag, :ref, :explicit_ref, :sparse_checkout
         attr_writer :revision
 
         def initialize(path, uri, options = {}, revision = nil, git = nil)
@@ -72,6 +72,7 @@ def initialize(path, uri, options = {}, revision = nil, git = nil)
           @revision = revision
           @git      = git
           @commit_ref = nil
+          @sparse_checkout = options["sparse_checkout"]
         end
 
         def revision
@@ -130,6 +131,8 @@ def copy_to(destination, submodules = false)
             end
           end
 
+          setup_sparse_checkout(destination)
+
           ref = @commit_ref || (locked_to_full_sha? && @revision)
           if ref
             git "config", "uploadpack.allowAnySHA1InWant", "true", dir: path.to_s if @commit_ref.nil? && needs_allow_any_sha1_in_want?
@@ -458,6 +461,21 @@ def supports_fetching_unreachable_refs?
         def supports_cloning_with_no_tags?
           @supports_cloning_with_no_tags ||= Gem::Version.new(version) >= Gem::Version.new("2.14.0-rc0")
         end
+
+        def supports_sparse_checkout?
+          @supports_sparse_checkout ||= Gem::Version.new(version) >= Gem::Version.new("2.25.0")
+        end
+
+        def setup_sparse_checkout(destination)
+          return unless @sparse_checkout
+
+          unless supports_sparse_checkout?
+            Bundler.ui.warn "Git #{version} doesn't support sparse-checkout (requires 2.25+). Cloning full repository."
+            return
+          end
+
+          git "sparse-checkout", "set", "--cone", @sparse_checkout, dir: destination
+        end
       end
     end
   end

bundler/spec/bundler/source/git/git_proxy_spec.rb

@@ -334,4 +334,68 @@
       end
     end
   end
+
+  describe "#supports_sparse_checkout?" do
+    it "returns true for git 2.25+" do
+      allow(git_proxy).to receive(:git_local).with("--version").and_return("git version 2.25.0")
+      expect(git_proxy.send(:supports_sparse_checkout?)).to be true
+    end
+
+    it "returns true for git 2.30+" do
+      allow(git_proxy).to receive(:git_local).with("--version").and_return("git version 2.30.0")
+      expect(git_proxy.send(:supports_sparse_checkout?)).to be true
+    end
+
+    it "returns false for git < 2.25" do
+      allow(git_proxy).to receive(:git_local).with("--version").and_return("git version 2.24.0")
+      expect(git_proxy.send(:supports_sparse_checkout?)).to be false
+    end
+
+    it "returns false for git 2.20" do
+      allow(git_proxy).to receive(:git_local).with("--version").and_return("git version 2.20.0")
+      expect(git_proxy.send(:supports_sparse_checkout?)).to be false
+    end
+  end
+
+  describe "#setup_sparse_checkout" do
+    let(:destination) { Pathname("destination") }
+
+    context "with sparse_checkout option" do
+      let(:options) { { "sparse_checkout" => "packages/foo" } }
+
+      context "with git 2.25+" do
+        it "runs sparse-checkout set with cone mode" do
+          allow(git_proxy).to receive(:git_local).with("--version").and_return("git version 2.30.0")
+          expect(git_proxy).to receive(:git).with("sparse-checkout", "set", "--cone", "packages/foo", dir: destination)
+          git_proxy.send(:setup_sparse_checkout, destination)
+        end
+      end
+
+      context "with git < 2.25" do
+        it "skips sparse checkout and warns" do
+          allow(git_proxy).to receive(:git_local).with("--version").and_return("git version 2.20.0")
+          expect(Bundler.ui).to receive(:warn).with(/doesn't support sparse-checkout/)
+          expect(git_proxy).not_to receive(:git).with("sparse-checkout", any_args)
+          git_proxy.send(:setup_sparse_checkout, destination)
+        end
+      end
+    end
+
+    context "without sparse_checkout option" do
+      let(:options) { {} }
+
+      it "does nothing" do
+        expect(git_proxy).not_to receive(:git)
+        git_proxy.send(:setup_sparse_checkout, destination)
+      end
+    end
+  end
+
+  context "with sparse_checkout option" do
+    let(:options) { { "sparse_checkout" => "packages/foo" } }
+
+    it "stores sparse_checkout from options" do
+      expect(git_proxy.sparse_checkout).to eq("packages/foo")
+    end
+  end
 end