From bfd0b64a6db4784565a45fef79795ae637469753 Mon Sep 17 00:00:00 2001 From: Andrei Filipovici Date: Tue, 18 Aug 2020 11:59:20 +0300 Subject: [PATCH] (FACT-2740) Add Gce fact --- lib/facter/facts/linux/gce.rb | 16 +++ lib/facter/facts/windows/gce.rb | 16 +++ lib/facter/resolvers/ec2.rb | 22 +--- lib/facter/resolvers/gce.rb | 54 ++++++++ lib/facter/resolvers/utils/http.rb | 67 ++++++++++ spec/facter/facts/linux/gce_spec.rb | 55 +++++++++ spec/facter/resolvers/ec2_spec.rb | 40 ++---- spec/facter/resolvers/gce_spec.rb | 150 +++++++++++++++++++++++ spec/facter/resolvers/utils/http_spec.rb | 66 ++++++++++ spec/fixtures/gce | 71 +++++++++++ 10 files changed, 506 insertions(+), 51 deletions(-) create mode 100644 lib/facter/facts/linux/gce.rb create mode 100644 lib/facter/facts/windows/gce.rb create mode 100644 lib/facter/resolvers/gce.rb create mode 100644 lib/facter/resolvers/utils/http.rb create mode 100644 spec/facter/facts/linux/gce_spec.rb create mode 100644 spec/facter/resolvers/gce_spec.rb create mode 100644 spec/facter/resolvers/utils/http_spec.rb create mode 100644 spec/fixtures/gce diff --git a/lib/facter/facts/linux/gce.rb b/lib/facter/facts/linux/gce.rb new file mode 100644 index 0000000000..a98fd50efd --- /dev/null +++ b/lib/facter/facts/linux/gce.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Facts + module Linux + class Gce + FACT_NAME = 'gce' + + def call_the_resolver + bios_vendor = Facter::Resolvers::Linux::DmiBios.resolve(:bios_vendor) + + fact_value = bios_vendor&.include?('Google') ? Facter::Resolvers::Gce.resolve(:metadata) : nil + Facter::ResolvedFact.new(FACT_NAME, fact_value) + end + end + end +end diff --git a/lib/facter/facts/windows/gce.rb b/lib/facter/facts/windows/gce.rb new file mode 100644 index 0000000000..0023fe0528 --- /dev/null +++ b/lib/facter/facts/windows/gce.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Facts + module Windows + class Gce + FACT_NAME = 'gce' + + def call_the_resolver + virtualization = Facter::Resolvers::Virtualization.resolve(:virtual) + + fact_value = virtualization&.include?('gce') ? Facter::Resolvers::Gce.resolve(:metadata) : nil + Facter::ResolvedFact.new(FACT_NAME, fact_value) + end + end + end +end diff --git a/lib/facter/resolvers/ec2.rb b/lib/facter/resolvers/ec2.rb index 2c969f0b47..5c98119aa0 100644 --- a/lib/facter/resolvers/ec2.rb +++ b/lib/facter/resolvers/ec2.rb @@ -7,13 +7,13 @@ class Ec2 < BaseResolver @fact_list ||= {} EC2_METADATA_ROOT_URL = 'http://169.254.169.254/latest/meta-data/' EC2_USERDATA_ROOT_URL = 'http://169.254.169.254/latest/user-data/' - EC2_CONNECTION_TIMEOUT = 0.6 EC2_SESSION_TIMEOUT = 5 class << self private def post_resolve(fact_name) + log.debug('Querying Ec2 metadata') @fact_list.fetch(fact_name) { read_facts(fact_name) } end @@ -29,7 +29,7 @@ def query_for_metadata(url, container) metadata.each_line do |line| next if line.empty? - http_path_component = build_path_compoent(line) + http_path_component = build_path_component(line) next if http_path_component == 'security-credentials/' if http_path_component.end_with?('/') @@ -44,27 +44,13 @@ def query_for_metadata(url, container) end end - def build_path_compoent(line) + def build_path_component(line) array_match = /^(\d+)=.*$/.match(line) array_match ? "#{array_match[1]}/" : line.strip end def get_data_from(url) - require 'net/http' - - parsed_url = URI.parse(url) - http = Net::HTTP.new(parsed_url.host) - http.read_timeout = determine_session_timeout - http.open_timeout = EC2_CONNECTION_TIMEOUT - resp = http.get(parsed_url.path) - response_code_valid?(resp.code) ? resp.body : '' - rescue StandardError => e - log.debug("Trying to connect to #{url} but got: #{e.message}") - '' - end - - def response_code_valid?(http_code) - http_code.to_i.equal?(200) + Utils::Http.get_request(url, {}, { session: determine_session_timeout }) end def determine_session_timeout diff --git a/lib/facter/resolvers/gce.rb b/lib/facter/resolvers/gce.rb new file mode 100644 index 0000000000..60a4e64675 --- /dev/null +++ b/lib/facter/resolvers/gce.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Facter + module Resolvers + class Gce < BaseResolver + @semaphore = Mutex.new + @fact_list ||= {} + METADATA_URL = 'http://metadata.google.internal/computeMetadata/v1/?recursive=true&alt=json' + HEADERS = { "Metadata-Flavor": 'Google', "Accept": 'application/json' }.freeze + + class << self + private + + def post_resolve(fact_name) + log.debug('reading Gce metadata') + @fact_list.fetch(fact_name) { read_facts(fact_name) } + end + + def read_facts(fact_name) + @fact_list[:metadata] = query_for_metadata + @fact_list[fact_name] + end + + def query_for_metadata + gce_data = extract_to_hash(Utils::Http.get_request(METADATA_URL, HEADERS)) + parse_instance(gce_data) + + gce_data.empty? ? nil : gce_data + end + + def extract_to_hash(metadata) + JSON.parse(metadata) + rescue JSON::ParserError => e + log.debug("Trying to parse result but got: #{e.message}") + {} + end + + def parse_instance(gce_data) + instance_data = gce_data['instance'] + return if instance_data.nil? || instance_data.empty? + + %w[image machineType zone].each do |key| + instance_data[key] = instance_data[key].split('/').last if instance_data[key] + end + + network = instance_data.dig('networkInterfaces', 0, 'network') + instance_data['networkInterfaces'][0]['network'] = network.split('/').last unless network.nil? + + gce_data['instance'] = instance_data + end + end + end + end +end diff --git a/lib/facter/resolvers/utils/http.rb b/lib/facter/resolvers/utils/http.rb new file mode 100644 index 0000000000..c30e4f0767 --- /dev/null +++ b/lib/facter/resolvers/utils/http.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Facter + module Resolvers + module Utils + module Http + class << self + CONNECTION_TIMEOUT = 0.6 + SESSION_TIMEOUT = 5 + @log = Facter::Log.new(self) + + # Makes a GET http request and returns it's response. + # + # Params: + # url: String which contains the address to which the request will be made + # headers: Hash which contains the headers you need to add to your request. + # Default headers is an empty hash + # Example: { "Accept": 'application/json' } + # timeouts: Hash that includes the values for the session and connection timeouts. + # Example: { session: 2.4. connection: 5 } + # + # Return value: + # is a string with the response body if the response code is 200. + # If the response code is not 200, an empty string is returned. + def get_request(url, headers = {}, timeouts = {}) + make_request(url, headers, timeouts, 'GET') + end + + private + + def make_request(url, headers, timeouts, request_type) + require 'net/http' + + uri = URI.parse(url) + http = http_obj(uri, timeouts) + request = request_obj(headers, uri, request_type) + + # Make the request + resp = http.request(request) + response_code_valid?(resp.code.to_i) ? resp.body : '' + rescue StandardError => e + @log.debug("Trying to connect to #{url} but got: #{e.message}") + '' + end + + def http_obj(parsed_url, timeouts) + http = Net::HTTP.new(parsed_url.host) + http.read_timeout = timeouts[:session] || SESSION_TIMEOUT + http.open_timeout = timeouts[:connection] || CONNECTION_TIMEOUT + http + end + + def request_obj(headers, parsed_url, request_type) + return Net::HTTP::Get.new(parsed_url.request_uri, headers) if request_type == 'GET' + + raise StandardError("Unknown http request type: #{request_type}") + end + + def response_code_valid?(http_code) + @log.debug("Request failed with error code #{http_code}") unless http_code.equal?(200) + http_code.equal?(200) + end + end + end + end + end +end diff --git a/spec/facter/facts/linux/gce_spec.rb b/spec/facter/facts/linux/gce_spec.rb new file mode 100644 index 0000000000..21858d1caa --- /dev/null +++ b/spec/facter/facts/linux/gce_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +describe Facts::Linux::Gce do + describe '#call_the_resolver' do + subject(:fact) { Facts::Linux::Gce.new } + + before do + allow(Facter::Resolvers::Gce).to receive(:resolve).with(:metadata).and_return(value) + allow(Facter::Resolvers::Linux::DmiBios).to receive(:resolve).with(:bios_vendor).and_return(vendor) + end + + context 'when hypervisor is Gce' do + let(:vendor) { 'Google' } + let(:value) do + { + 'oslogin' => { + 'authenticate' => { + 'sessions' => { + } + } + }, + 'project' => { + 'numericProjectId' => 728_618_928_092, + 'projectId' => 'facter-performance-history' + } + } + end + + it 'calls Facter::Resolvers::Linux::Gce' do + fact.call_the_resolver + expect(Facter::Resolvers::Gce).to have_received(:resolve).with(:metadata) + end + + it 'calls Facter::Resolvers::Linux::DmiBios' do + fact.call_the_resolver + expect(Facter::Resolvers::Linux::DmiBios).to have_received(:resolve).with(:bios_vendor) + end + + it 'returns gce fact' do + expect(fact.call_the_resolver).to be_an_instance_of(Facter::ResolvedFact).and \ + have_attributes(name: 'gce', value: value) + end + end + + context 'when hypervisor is not Gce' do + let(:vendor) { 'unknown' } + let(:value) { nil } + + it 'returns nil' do + expect(fact.call_the_resolver).to be_an_instance_of(Facter::ResolvedFact).and \ + have_attributes(name: 'gce', value: nil) + end + end + end +end diff --git a/spec/facter/resolvers/ec2_spec.rb b/spec/facter/resolvers/ec2_spec.rb index e130d56ee5..f45d05593f 100644 --- a/spec/facter/resolvers/ec2_spec.rb +++ b/spec/facter/resolvers/ec2_spec.rb @@ -3,18 +3,14 @@ describe Facter::Resolvers::Ec2 do subject(:ec2) { Facter::Resolvers::Ec2 } - let(:uri) { URI.parse('http://169.254.169.254/latest/meta-data/') } - let(:userdata_uri) { URI.parse('http://169.254.169.254/latest/user-data/') } - let(:http_spy) { instance_spy(Net::HTTP) } - let(:response) { instance_spy(Net::HTTPResponse) } + let(:uri) { 'http://169.254.169.254/latest/meta-data/' } + let(:userdata_uri) { 'http://169.254.169.254/latest/user-data/' } let(:log_spy) { instance_spy(Facter::Log) } before do + allow(Facter::Resolvers::Utils::Http).to receive(:get_request).with(uri, {}, { session: 5 }).and_return(output) + allow(Facter::Resolvers::Utils::Http).to receive(:get_request).with(userdata_uri, {}, { session: 5 }).and_return('') ec2.instance_variable_set(:@log, log_spy) - allow(Net::HTTP).to receive(:new).with(uri.host).and_return(http_spy) - allow(http_spy).to receive(:get).with(uri.path).and_return(response) - allow(Net::HTTP).to receive(:new).with(userdata_uri.host).and_return(http_spy) - allow(http_spy).to receive(:get).with(userdata_uri.path).and_return(response_userdata) end after do @@ -23,20 +19,12 @@ context 'when no exception is thrown' do let(:output) { "security-credentials/\nami-id" } - let(:ami_uri) { URI.parse('http://169.254.169.254/latest/meta-data/ami-id') } + let(:ami_uri) { 'http://169.254.169.254/latest/meta-data/ami-id' } let(:ami_id) { 'some_id_123' } - let(:response2) { instance_spy(Net::HTTPResponse) } - let(:response_userdata) { instance_spy(Net::HTTPResponse) } before do - allow(response).to receive(:code).and_return(200) - allow(response).to receive(:body).and_return(output) - allow(Net::HTTP).to receive(:new).with(ami_uri.host).and_return(http_spy) - allow(http_spy).to receive(:get).with(ami_uri.path).and_return(response2) - allow(response2).to receive(:code).and_return(200) - allow(response2).to receive(:body).and_return(ami_id) - - allow(response_userdata).to receive(:code).and_return(404) + allow(Facter::Resolvers::Utils::Http).to receive(:get_request) + .with(ami_uri, {}, { session: 5 }).and_return(ami_id) end it 'returns ec2 metadata' do @@ -50,13 +38,6 @@ context 'when an exception is thrown' do let(:output) { 'security-credentials/' } - let(:response_userdata) { instance_spy(Net::HTTPResponse) } - - before do - allow(response).to receive(:code).and_return(200) - allow(response).to receive(:body).and_return(output) - allow(http_spy).to receive(:get).with(userdata_uri.path).and_raise(Net::OpenTimeout) - end it 'returns empty ec2 metadata' do expect(ec2.resolve(:metadata)).to eq({}) @@ -65,12 +46,5 @@ it 'returns empty ec2 userdata' do expect(ec2.resolve(:userdata)).to eq('') end - - it 'logs timeout error' do - ec2.resolve(:userdata) - - expect(log_spy).to have_received(:debug) - .with('Trying to connect to http://169.254.169.254/latest/user-data/ but got: Net::OpenTimeout') - end end end diff --git a/spec/facter/resolvers/gce_spec.rb b/spec/facter/resolvers/gce_spec.rb new file mode 100644 index 0000000000..62a05e53a2 --- /dev/null +++ b/spec/facter/resolvers/gce_spec.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +describe Facter::Resolvers::Gce do + let(:gce_metadata_url) { 'http://metadata.google.internal/computeMetadata/v1/?recursive=true&alt=json' } + let(:gce_url_headers) { { "Metadata-Flavor": 'Google', "Accept": 'application/json' } } + + before do + allow(Facter::Resolvers::Utils::Http).to receive(:get_request) + .with(gce_metadata_url, gce_url_headers) + .and_return(http_response_body) + end + + after do + Facter::Resolvers::Gce.invalidate_cache + end + + context 'when http request is successful' do + let(:http_response_body) { load_fixture('gce').read } + let(:value) do + { + 'instance' => { + 'attributes' => { + }, + 'cpuPlatform' => 'Intel Broadwell', + 'description' => '', + 'disks' => [ + { + 'deviceName' => 'instance-3', + 'index' => 0, + 'interface' => 'SCSI', + 'mode' => 'READ_WRITE', + 'type' => 'PERSISTENT' + } + ], + 'guestAttributes' => { + }, + 'hostname' => 'instance-3.c.facter-performance-history.internal', + 'id' => 2_206_944_706_428_651_580, + 'image' => 'ubuntu-2004-focal-v20200810', + 'legacyEndpointAccess' => { + '0.1' => 0, + 'v1beta1' => 0 + }, + 'licenses' => [ + { + 'id' => '2211838267635035815' + } + ], + 'machineType' => 'n1-standard-2', + 'maintenanceEvent' => 'NONE', + 'name' => 'instance-3', + 'networkInterfaces' => [ + { + 'accessConfigs' => [ + { + 'externalIp' => '34.89.230.102', + 'type' => 'ONE_TO_ONE_NAT' + } + ], + 'dnsServers' => [ + '169.254.169.254' + ], + 'forwardedIps' => [], + 'gateway' => '10.156.0.1', + 'ip' => '10.156.0.4', + 'ipAliases' => [], + 'mac' => '42:01:0a:9c:00:04', + 'mtu' => 1460, + 'network' => 'default', + 'subnetmask' => '255.255.240.0', + 'targetInstanceIps' => [] + } + ], + 'preempted' => 'FALSE', + 'remainingCpuTime' => -1, + 'scheduling' => { + 'automaticRestart' => 'TRUE', + 'onHostMaintenance' => 'MIGRATE', + 'preemptible' => 'FALSE' + }, + 'serviceAccounts' => { + '728618928092-compute@developer.gserviceaccount.com' => { + 'aliases' => [ + 'default' + ], + 'email' => '728618928092-compute@developer.gserviceaccount.com', + 'scopes' => [ + 'https://www.googleapis.com/auth/devstorage.read_only', + 'https://www.googleapis.com/auth/logging.write', + 'https://www.googleapis.com/auth/monitoring.write', + 'https://www.googleapis.com/auth/servicecontrol', + 'https://www.googleapis.com/auth/service.management.readonly', + 'https://www.googleapis.com/auth/trace.append' + ] + }, + 'default' => { + 'aliases' => [ + 'default' + ], + 'email' => '728618928092-compute@developer.gserviceaccount.com', + 'scopes' => [ + 'https://www.googleapis.com/auth/devstorage.read_only', + 'https://www.googleapis.com/auth/logging.write', + 'https://www.googleapis.com/auth/monitoring.write', + 'https://www.googleapis.com/auth/servicecontrol', + 'https://www.googleapis.com/auth/service.management.readonly', + 'https://www.googleapis.com/auth/trace.append' + ] + } + }, + 'tags' => [], + 'virtualClock' => { + 'driftToken' => '0' + }, + 'zone' => 'europe-west3-c' + }, + 'oslogin' => { + 'authenticate' => { + 'sessions' => { + } + } + }, + 'project' => { + 'attributes' => { + 'ssh-keys' => 'john_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B' \ + " google-ssh {\"userName\":\"john.doe@puppet.com\",\"expireOn\":\"2020-08-13T12:17:19+0000\"}\n" + }, + 'numericProjectId' => 728_618_928_092, + 'projectId' => 'facter-performance-history' + } + } + end + + it 'returns gce data' do + result = Facter::Resolvers::Gce.resolve(:metadata) + + expect(result).to eq(value) + end + end + + context 'when http request fails' do + let(:http_response_body) { 'Request failed with error code: 404' } + + it 'returns nil' do + result = Facter::Resolvers::Gce.resolve(:metadata) + + expect(result).to be(nil) + end + end +end diff --git a/spec/facter/resolvers/utils/http_spec.rb b/spec/facter/resolvers/utils/http_spec.rb new file mode 100644 index 0000000000..2ed3030700 --- /dev/null +++ b/spec/facter/resolvers/utils/http_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +describe Facter::Resolvers::Utils::Http do + subject(:http) { Facter::Resolvers::Utils::Http } + + describe '#get_request' do + let(:url) { 'http://169.254.169.254/meta-data/' } + let(:uri) { URI.parse(url) } + let(:http_spy) { instance_spy(Net::HTTP) } + let(:http_get_spy) { instance_spy(Net::HTTP::Get) } + let(:response_spy) { instance_spy(Net::HTTPResponse) } + let(:log_spy) { instance_spy(Facter::Log) } + + before do + http.instance_variable_set(:@log, log_spy) + + allow(Net::HTTP).to receive(:new).with(uri.host).and_return(http_spy) + allow(Net::HTTP::Get).to receive(:new).with(uri.request_uri, {}).and_return(http_get_spy) + allow(http_spy).to receive(:request).with(http_get_spy).and_return(response_spy) + + allow(response_spy).to receive(:code).and_return(200) + allow(response_spy).to receive(:body).and_return(output) + end + + context 'when http get request is successful' do + let(:output) { 'request output' } + + it 'returns the output' do + expect(http.get_request(url)).to eq(output) + end + end + + shared_examples 'logs error and output is empty string' do + let(:output) { '' } + + it 'returns empty string' do + expect(http.get_request(url)).to eq(output) + end + + it 'logs error code' do + http.get_request(url) + expect(log_spy).to have_received(:debug).with(log_message) + end + end + + context 'when http get request has error code' do + let(:log_message) { 'Request failed with error code 404' } + + before do + allow(response_spy).to receive(:code).and_return(404) + end + + it_behaves_like 'logs error and output is empty string' + end + + context 'when http get request fails due to timeout' do + let(:log_message) { 'Trying to connect to http://169.254.169.254/meta-data/ but got: Net::OpenTimeout' } + + before do + allow(http_spy).to receive(:request).with(http_get_spy).and_raise(Net::OpenTimeout) + end + + it_behaves_like 'logs error and output is empty string' + end + end +end diff --git a/spec/fixtures/gce b/spec/fixtures/gce new file mode 100644 index 0000000000..ddf34476d3 --- /dev/null +++ b/spec/fixtures/gce @@ -0,0 +1,71 @@ +{ +"instance": + { + "attributes":{}, + "cpuPlatform":"Intel Broadwell", + "description":"", + "disks":[{"deviceName":"instance-3","index":0,"interface":"SCSI","mode":"READ_WRITE","type":"PERSISTENT"}], + "guestAttributes":{}, + "hostname":"instance-3.c.facter-performance-history.internal", + "id":2206944706428651580, + "image":"projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20200810", + "legacyEndpointAccess":{"0.1":0,"v1beta1":0}, + "licenses":[{"id":"2211838267635035815"}], + "machineType":"projects/728618928092/machineTypes/n1-standard-2", + "maintenanceEvent":"NONE", + "name":"instance-3", + "networkInterfaces":[ + { + "accessConfigs":[{"externalIp":"34.89.230.102","type":"ONE_TO_ONE_NAT"}], + "dnsServers":["169.254.169.254"], + "forwardedIps":[], + "gateway":"10.156.0.1", + "ip":"10.156.0.4", + "ipAliases":[], + "mac":"42:01:0a:9c:00:04", + "mtu":1460, + "network":"projects/728618928092/networks/default", + "subnetmask":"255.255.240.0", + "targetInstanceIps":[]}], + "preempted":"FALSE", + "remainingCpuTime":-1, + "scheduling":{"automaticRestart":"TRUE","onHostMaintenance":"MIGRATE","preemptible":"FALSE"}, + "serviceAccounts": + { + "728618928092-compute@developer.gserviceaccount.com": + { + "aliases":["default"], + "email":"728618928092-compute@developer.gserviceaccount.com", + "scopes":["https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/trace.append"] + }, + "default": + { + "aliases":["default"], + "email":"728618928092-compute@developer.gserviceaccount.com", + "scopes":["https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/trace.append"]} + }, + "tags":[], + "virtualClock":{"driftToken":"0"}, + "zone":"projects/728618928092/zones/europe-west3-c" + }, + "oslogin":{"authenticate":{"sessions":{}}}, + "project": + { + "attributes": + { + "ssh-keys":"john_doe:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDA9D8Op48TtEiDmb+Gtna3Bs9B google-ssh {\"userName\":\"john.doe@puppet.com\",\"expireOn\":\"2020-08-13T12:17:19+0000\"}\n" + }, + "numericProjectId":728618928092, + "projectId":"facter-performance-history" + } +} \ No newline at end of file