From cba1d5db07e79c5d5717ab468264890d63b8ee67 Mon Sep 17 00:00:00 2001 From: Jon Morrow Date: Wed, 6 Nov 2019 09:16:49 -0800 Subject: [PATCH] Implement deep signing for OSX Notarization * Adds deep signing of libraries and binaries to the pakacking process for the pkg packager. * Enables the hardened runtime for binaries. * Adds --preserve-xattr flag to pkgbuild so signing is preserved through packaging and install.* In order to deep sign we have to know where a software definition will install binaries and libraries. To facilite this lib_dirs and bin_dirs functions have been added to the software definition. These two functions return defaults that match standard omnibus locations, but allow individual software defs to override if they do something different. Bumps major version since signing deep signing is a major change. Signed-off-by: Jon Morrow --- RELEASE_NOTES.md | 5 + VERSION | 2 +- lib/omnibus/packagers/pkg.rb | 124 +++++++++++- lib/omnibus/software.rb | 40 ++++ lib/omnibus/version.rb | 2 +- spec/unit/packagers/pkg_spec.rb | 348 ++++++++++++++++++++++++++++++++ spec/unit/software_spec.rb | 30 +++ 7 files changed, 546 insertions(+), 5 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c6ffdb5bd..6e6e399e6 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,8 @@ +## Omnibus 7.0: + +### Deep Signing and Hardened Runtime + +When packaging using the pkg packager omnibus will now deep sign all binaries and libraries in the package based of each software definition's bin_dirs and lib_dirs. When siging binaries the hardened runtime is enabled. ## Omnibus 6.0: diff --git a/VERSION b/VERSION index 27a717ab0..66ce77b7e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.1.21 \ No newline at end of file +7.0.0 diff --git a/lib/omnibus/packagers/pkg.rb b/lib/omnibus/packagers/pkg.rb index 858811502..b2fc40812 100644 --- a/lib/omnibus/packagers/pkg.rb +++ b/lib/omnibus/packagers/pkg.rb @@ -64,6 +64,8 @@ class Packager::PKG < Packager::Base build do write_scripts + sign_software_libs_and_bins + build_component_pkg write_distribution_file @@ -177,6 +179,67 @@ def write_scripts end end + def sign_software_libs_and_bins + if signing_identity + log.info(log_key) { "Finding libraries and binaries that require signing." } + + bin_dirs = Set[] + lib_dirs = Set[] + binaries = Set[] + libraries = Set[] + + # Capture lib_dirs and bin_dirs from each software + project.softwares.each do |software| + lib_dirs.merge(software.lib_dirs) + bin_dirs.merge(software.bin_dirs) + end + + # Find all binaries in each bind_dir + bin_dirs.each do |dir| + binaries.merge Dir["#{dir}/*"] + end + # Filter out symlinks, non-files, and non-executables + log.debug(log_key) { " Filtering non-binary files:" } + binaries.select! { |bin| is_binary?(bin) } + + # Use otool to find all libries that are used by our binaries + binaries.each do |bin| + libraries.merge find_linked_libs bin + end + + # Find all libraries in each lib_dir and add any we missed with otool + lib_dirs.each do |dir| + libraries.merge Dir["#{dir}/*"] + end + + # Filter Mach-O libraries and bundles + log.debug(log_key) { " Filtering non-library files:" } + libraries.select! { |lib| is_macho?(lib) } + + # Use otool to find all libries that are used by our libraries + otool_libs = Set[] + libraries.each do |lib| + otool_libs.merge find_linked_libs lib + end + + # Filter Mach-O libraries and bundles + otool_libs.select! { |lib| is_macho?(lib) } + libraries.merge otool_libs + + log.info(log_key) { " Signing libraries:" } unless libraries.empty? + libraries.each do |library| + log.debug(log_key) { " Signing: #{library}" } + sign_library(library) + end + + log.info(log_key) { " Signing binaries:" } unless binaries.empty? + binaries.each do |binary| + log.debug(log_key) { " Signing: #{binary}" } + sign_binary(binary, true) + end + end + end + # # Construct the intermediate build product. It can be installed with the # Installer.app, but doesn't contain the data needed to customize the @@ -185,16 +248,20 @@ def write_scripts # @return [void] # def build_component_pkg - command = <<-EOH.gsub(/^ {8}/, "") + command = <<~EOH pkgbuild \\ --identifier "#{safe_identifier}" \\ --version "#{safe_version}" \\ --scripts "#{scripts_dir}" \\ --root "#{project.install_dir}" \\ --install-location "#{project.install_dir}" \\ - "#{component_pkg}" + --preserve-xattr \\ EOH + command << %Q{ --sign "#{signing_identity}" \\\n} if signing_identity + command << %Q{ "#{component_pkg}"} + command << %Q{\n} + Dir.chdir(staging_dir) do shellout!(command) end @@ -229,7 +296,7 @@ def write_distribution_file # @return [void] # def build_product_pkg - command = <<-EOH.gsub(/^ {8}/, "") + command = <<~EOH productbuild \\ --distribution "#{staging_dir}/Distribution" \\ --resources "#{resources_dir}" \\ @@ -320,5 +387,56 @@ def safe_version converted end end + + # + # Given a file path return any linked libraries. + # + # @param [String] file_path + # The path to a file + # @return [Array] + # The linked libs + # + def find_linked_libs(file_path) + # Find all libaries for each bin + command = "otool -L #{file_path}" + + stdout = shellout!(command).stdout + stdout.slice!(file_path) + stdout.scan(/#{install_dir}\S*/) + end + + def sign_library(lib) + sign_binary(lib) + end + + def sign_binary(bin, hardened_runtime = false) + command = "codesign -s '#{signing_identity}' '#{bin}'" + command << %q{ --options=runtime} if hardened_runtime + ## Force re-signing to deal with binaries that have the same sha. + command << %q{ --force} + command << %Q{\n} + + shellout!(command) + end + + def is_binary?(bin) + is_binary = File.file?(bin) && + File.executable?(bin) && + !File.symlink?(bin) + log.debug(log_key) { " removing from signing: #{bin}" } unless is_binary + is_binary + end + + def is_macho?(lib) + is_macho = false + if is_binary?(lib) + command = "file #{lib}" + + stdout = shellout!(command).stdout + is_macho = stdout.match?(/Mach-O.*library/) || stdout.match?(/Mach-O.*bundle/) + end + log.debug(log_key) { " removing from signing: #{lib}" } unless is_macho + is_macho + end end end diff --git a/lib/omnibus/software.rb b/lib/omnibus/software.rb index 0f63eb2b3..f284cd0a8 100644 --- a/lib/omnibus/software.rb +++ b/lib/omnibus/software.rb @@ -205,6 +205,46 @@ def maintainer(val = NULL) end expose :maintainer + # + # Sets the bin_dirs where this software installs bins. + # + # @example + # bin_dirs ['/opt/chef-workstation/bin'] + # + # @param [Array] val + # the bin_dirs of the software + # + # @return [Array] + # + def bin_dirs(val = NULL) + if null?(val) + @bin_dirs || [windows_safe_path("#{install_dir}/bin"), windows_safe_path("#{install_dir}/embedded/bin")] + else + @bin_dirs = val + end + end + expose :bin_dirs + + # + # Sets the lib_dirs where this software installs libs. + # + # @example + # lib_dirs ['/opt/chef-workstation/bin'] + # + # @param [Array] val + # the lib_dirs of the software + # + # @return [Array] + # + def lib_dirs(val = NULL) + if null?(val) + @lib_dirs || [windows_safe_path("#{install_dir}/embedded/lib")] + else + @lib_dirs = val + end + end + expose :lib_dirs + # # Add a software dependency to this software. # diff --git a/lib/omnibus/version.rb b/lib/omnibus/version.rb index 838e2ad7c..1f8d7d562 100644 --- a/lib/omnibus/version.rb +++ b/lib/omnibus/version.rb @@ -15,5 +15,5 @@ # module Omnibus - VERSION = "6.1.21".freeze + VERSION = "7.0.0".freeze end diff --git a/spec/unit/packagers/pkg_spec.rb b/spec/unit/packagers/pkg_spec.rb index aedb92f49..7755c15a9 100644 --- a/spec/unit/packagers/pkg_spec.rb +++ b/spec/unit/packagers/pkg_spec.rb @@ -109,6 +109,158 @@ module Omnibus end end + describe "#sign_software_libs_and_bins" do + context "when pkg signing is disabled" do + it "does not sign anything" do + expect(subject).not_to receive(:sign_binary) + expect(subject).not_to receive(:sign_library) + subject.sign_software_libs_and_bins + end + + it "returns an empty set" do + expect(subject.sign_software_libs_and_bins).to be_nil + end + end + + context "when pkg signing is enabled" do + before do + subject.signing_identity("My Special Identity") + end + + context "without software" do + it "does not sign anything" do + expect(subject).not_to receive(:sign_binary) + expect(subject).not_to receive(:sign_library) + subject.sign_software_libs_and_bins + end + + it "returns an empty set" do + expect(subject.sign_software_libs_and_bins).to eq(Set.new) + end + end + + context "project with software" do + let(:software) do + Software.new(project).tap do |software| + software.name("software-full-name") + end + end + + before do + allow(project).to receive(:softwares).and_return([software]) + end + + context "with empty bin_dirs and lib_dirs" do + before do + allow(software).to receive(:lib_dirs).and_return([]) + allow(software).to receive(:bin_dirs).and_return([]) + end + + it "does not sign anything" do + expect(subject).not_to receive(:sign_binary) + expect(subject).not_to receive(:sign_library) + subject.sign_software_libs_and_bins + end + + it "returns an empty set" do + expect(subject.sign_software_libs_and_bins).to eq(Set.new) + end + end + + context "with default bin_dirs and lib_dirs" do + context "with binaries" do + let(:bin) { "/opt/#{project.name}/bin/test_bin" } + let(:embedded_bin) { "/opt/#{project.name}/embedded/bin/test_bin" } + before do + allow(Dir).to receive(:[]).with("/opt/#{project.name}/bin/*").and_return([bin]) + allow(Dir).to receive(:[]).with("/opt/#{project.name}/embedded/bin/*").and_return([embedded_bin]) + allow(Dir).to receive(:[]).with("/opt/#{project.name}/embedded/lib/*").and_return([]) + allow(subject).to receive(:is_binary?).with(bin).and_return(true) + allow(subject).to receive(:is_binary?).with(embedded_bin).and_return(true) + allow(subject).to receive(:find_linked_libs).with(bin).and_return([]) + allow(subject).to receive(:find_linked_libs).with(embedded_bin).and_return([]) + allow(subject).to receive(:sign_binary).with(bin, true) + allow(subject).to receive(:sign_binary).with(embedded_bin, true) + end + + it "signs the binaries" do + expect(subject).to receive(:sign_binary).with(bin, true) + expect(subject).to receive(:sign_binary).with(embedded_bin, true) + subject.sign_software_libs_and_bins + end + + it "returns a set with the signed binaries" do + expect(subject.sign_software_libs_and_bins).to eq(Set.new [bin, embedded_bin]) + end + end + + context "with library" do + let(:lib) { "/opt/#{project.name}/embedded/lib/test_lib" } + before do + allow(Dir).to receive(:[]).with("/opt/#{project.name}/bin/*").and_return([]) + allow(Dir).to receive(:[]).with("/opt/#{project.name}/embedded/bin/*").and_return([]) + allow(Dir).to receive(:[]).with("/opt/#{project.name}/embedded/lib/*").and_return([lib]) + allow(subject).to receive(:is_macho?).with(lib).and_return(true) + allow(subject).to receive(:find_linked_libs).with(lib).and_return([]) + allow(subject).to receive(:sign_library).with(lib) + end + + it "signs the library" do + expect(subject).to receive(:sign_library).with(lib) + subject.sign_software_libs_and_bins + end + end + + context "with binaries and libraries with linked libs" do + let(:bin) { "/opt/#{project.name}/bin/test_bin" } + let(:bin2) { "/opt/#{project.name}/bin/test_bin2" } + let(:embedded_bin) { "/opt/#{project.name}/embedded/bin/test_bin" } + let(:lib) { "/opt/#{project.name}/embedded/lib/test_lib" } + let(:lib2) { "/opt/#{project.name}/embedded/lib/test_lib2" } + before do + allow(Dir).to receive(:[]).with("/opt/#{project.name}/bin/*").and_return([bin, bin2]) + allow(Dir).to receive(:[]).with("/opt/#{project.name}/embedded/bin/*").and_return([embedded_bin]) + allow(Dir).to receive(:[]).with("/opt/#{project.name}/embedded/lib/*").and_return([lib]) + allow(subject).to receive(:is_binary?).with(bin).and_return(true) + allow(subject).to receive(:is_binary?).with(bin2).and_return(true) + allow(subject).to receive(:is_binary?).with(embedded_bin).and_return(true) + allow(subject).to receive(:is_macho?).with(lib).and_return(true) + allow(subject).to receive(:is_macho?).with(lib2).and_return(true) + allow(subject).to receive(:find_linked_libs).with(bin).and_return([lib2]) + allow(subject).to receive(:find_linked_libs).with(bin2).and_return([]) + allow(subject).to receive(:find_linked_libs).with(embedded_bin).and_return([]) + allow(subject).to receive(:find_linked_libs).with(lib).and_return([]) + allow(subject).to receive(:find_linked_libs).with(lib2).and_return([]) + allow(subject).to receive(:sign_binary).with(bin, true) + allow(subject).to receive(:sign_binary).with(bin2, true) + allow(subject).to receive(:sign_binary).with(embedded_bin, true) + allow(subject).to receive(:sign_library).with(lib) + allow(subject).to receive(:sign_library).with(lib2) + allow(Digest::SHA256).to receive(:file).with(bin).and_return(Digest::SHA256.new.update(bin)) + allow(Digest::SHA256).to receive(:file).with(bin2).and_return(Digest::SHA256.new.update(bin2)) + allow(Digest::SHA256).to receive(:file).with(embedded_bin).and_return(Digest::SHA256.new.update(embedded_bin)) + allow(Digest::SHA256).to receive(:file).with(lib).and_return(Digest::SHA256.new.update(lib)) + allow(Digest::SHA256).to receive(:file).with(lib2).and_return(Digest::SHA256.new.update(lib2)) + end + + it "signs the binaries" do + expect(subject).to receive(:sign_binary).with(bin, true) + expect(subject).to receive(:sign_binary).with(bin2, true) + expect(subject).to receive(:sign_binary).with(embedded_bin, true) + subject.sign_software_libs_and_bins + end + + it "signs the libraries" do + expect(subject).to receive(:sign_library).with(lib) + expect(subject).to receive(:sign_library).with(lib2) + subject.sign_software_libs_and_bins + end + end + end + end + end + end + describe "#build_component_pkg" do it "executes the pkgbuild command" do expect(subject).to receive(:shellout!).with <<-EOH.gsub(/^ {10}/, "") @@ -118,6 +270,7 @@ module Omnibus --scripts "#{staging_dir}/Scripts" \\ --root "/opt/project-full-name" \\ --install-location "/opt/project-full-name" \\ + --preserve-xattr \\ "project-full-name-core.pkg" EOH @@ -267,5 +420,200 @@ module Omnibus end end end + + describe "#find_linked_libs" do + context "with linked libs" do + let(:file) { "/opt/#{project.name}/embedded/bin/test_bin" } + let(:stdout) do + <<~EOH + /opt/#{project.name}/embedded/bin/test_bin: + /opt/#{project.name}/embedded/lib/lib.dylib (compatibility version 7.0.0, current version 7.4.0) + /opt/#{project.name}/embedded/lib/lib.6.dylib (compatibility version 7.0.0, current version 7.4.0) + /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1281.0.0) + EOH + end + let(:shellout) { Mixlib::ShellOut.new } + + before do + allow(shellout).to receive(:run_command) + allow(shellout).to receive(:stdout) + .and_return(stdout) + allow(subject).to receive(:shellout!) + .with("otool -L #{file}") + .and_return(shellout) + end + + it "returns empty array" do + expect(subject.find_linked_libs(file)).to eq([ + "/opt/#{project.name}/embedded/lib/lib.dylib", + "/opt/#{project.name}/embedded/lib/lib.6.dylib", + ]) + end + end + + context "with only system linked libs" do + let(:file) { "/opt/#{project.name}/embedded/lib/lib.dylib" } + let(:stdout) do + <<~EOH + /opt/#{project.name}/embedded/lib/lib.dylib: + /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1281.0.0) + EOH + end + let(:shellout) { Mixlib::ShellOut.new } + before do + allow(shellout).to receive(:run_command) + allow(shellout).to receive(:stdout) + .and_return(stdout) + allow(subject).to receive(:shellout!) + .with("otool -L #{file}") + .and_return(shellout) + end + + it "returns empty array" do + expect(subject.find_linked_libs(file)).to eq([]) + end + end + + context "file is just a file" do + let(:file) { "/opt/#{project.name}/embedded/lib/file.rb" } + let(:shellout) { Mixlib::ShellOut.new } + before do + allow(shellout).to receive(:run_command) + allow(shellout).to receive(:stdout) + .and_return("#{file}: is not an object file") + allow(subject).to receive(:shellout!) + .with("otool -L #{file}") + .and_return(shellout) + end + + it "returns empty array" do + expect(subject.find_linked_libs(file)).to eq([]) + end + end + end + + describe "#is_binary?" do + context "when is a file, executable, and not a symlink" do + before do + allow(File).to receive(:file?).with("file").and_return(true) + allow(File).to receive(:executable?).with("file").and_return(true) + allow(File).to receive(:symlink?).with("file").and_return(false) + end + + it "returns true" do + expect(subject.is_binary?("file")).to be true + end + end + + context "when not a file" do + before do + allow(File).to receive(:file?).with("file").and_return(false) + allow(File).to receive(:executable?).with("file").and_return(true) + allow(File).to receive(:symlink?).with("file").and_return(false) + end + + it "returns false" do + expect(subject.is_binary?("file")).to be false + end + end + + context "when not an executable" do + before do + allow(File).to receive(:file?).with("file").and_return(true) + allow(File).to receive(:executable?).with("file").and_return(false) + allow(File).to receive(:symlink?).with("file").and_return(false) + end + + it "returns false" do + expect(subject.is_binary?("file")).to be false + end + end + + context "when is symlink" do + before do + allow(File).to receive(:file?).with("file").and_return(true) + allow(File).to receive(:executable?).with("file").and_return(true) + allow(File).to receive(:symlink?).with("file").and_return(true) + end + + it "returns false" do + expect(subject.is_binary?("file")).to be false + end + end + end + + describe "#is_macho?" do + let(:shellout) { Mixlib::ShellOut.new } + + context "when is a Mach-O library" do + before do + allow(subject).to receive(:is_binary?).with("file").and_return(true) + expect(subject).to receive(:shellout!).with("file file").and_return(shellout) + allow(shellout).to receive(:stdout) + .and_return("file: Mach-O 64-bit dynamically linked shared library x86_64") + end + + it "returns true" do + expect(subject.is_macho?("file")).to be true + end + end + + context "when is a Mach-O Bundle" do + before do + allow(subject).to receive(:is_binary?).with("file").and_return(true) + expect(subject).to receive(:shellout!).with("file file").and_return(shellout) + allow(shellout).to receive(:stdout) + .and_return("file: Mach-O 64-bit bundle x86_64") + end + + it "returns true" do + expect(subject.is_macho?("file")).to be true + end + end + + context "when is not a Mach-O Bundle or Mach-O library" do + before do + allow(subject).to receive(:is_binary?).with("file").and_return(true) + expect(subject).to receive(:shellout!).with("file file").and_return(shellout) + allow(shellout).to receive(:stdout) + .and_return("file: ASCII text") + end + + it "returns true" do + expect(subject.is_macho?("file")).to be false + end + end + end + + describe "#sign_library" do + before do + subject.signing_identity("My Special Identity") + end + + it "calls sign_binary without hardened runtime" do + expect(subject).to receive(:sign_binary).with("file") + subject.sign_library("file") + end + end + + describe "#sign_binary" do + before do + subject.signing_identity("My Special Identity") + end + + it "it signs the binary without hardened runtime" do + expect(subject).to receive(:shellout!) + .with("codesign -s '#{subject.signing_identity}' 'file' --force\n") + subject.sign_binary("file") + end + + context "with hardened runtime" do + it "it signs the binary with hardened runtime" do + expect(subject).to receive(:shellout!) + .with("codesign -s '#{subject.signing_identity}' 'file' --options=runtime --force\n") + subject.sign_binary("file", true) + end + end + end end end diff --git a/spec/unit/software_spec.rb b/spec/unit/software_spec.rb index 5395dbf7f..b3a78637e 100644 --- a/spec/unit/software_spec.rb +++ b/spec/unit/software_spec.rb @@ -558,6 +558,36 @@ module Omnibus end end + describe "#bin_dirs" do + it "sets bin_dirs" do + subject.bin_dirs ["my_bin_dir"] + expect(subject.bin_dirs).to eq(["my_bin_dir"]) + expect(subject.bin_dirs).to be_kind_of(Array) + end + + context "bin_dirs is not set" do + it "returns default values" do + expect(subject.bin_dirs).to eq(["/opt/project/bin", "/opt/project/embedded/bin"]) + expect(subject.bin_dirs).to be_kind_of(Array) + end + end + end + + describe "#lib_dirs" do + it "sets lib_dirs" do + subject.lib_dirs ["my_lib_dir"] + expect(subject.lib_dirs).to eq(["my_lib_dir"]) + expect(subject.lib_dirs).to be_kind_of(Array) + end + + context "lib_dirs is not set" do + it "returns default values" do + expect(subject.lib_dirs).to eq(["/opt/project/embedded/lib"]) + expect(subject.lib_dirs).to be_kind_of(Array) + end + end + end + context "testing repo-level version overrides" do context "without overrides" do it "returns the original values" do