From 47ca40c0109425aede756d84d7e9c65fab9fb858 Mon Sep 17 00:00:00 2001 From: Yim Lee Date: Mon, 29 Nov 2021 23:08:51 -0800 Subject: [PATCH] This is a continuation of /~https://github.com/apple/swift-package-manager/pull/3879. Wire up fingerprint storage such that it is used for integrity checks of package downloads. Fingerprint must match previously recorded value (if any) or else it would result in an error. --- Package.swift | 4 + Sources/Basics/FileSystem+Extensions.swift | 9 +- Sources/Commands/CMakeLists.txt | 1 + Sources/Commands/Options.swift | 12 +- Sources/Commands/SwiftTool.swift | 2 + .../FilePackageFingerprintStorage.swift | 2 +- Sources/PackageFingerprint/Model.swift | 15 +- .../PackageFingerprintStorage.swift | 18 + Sources/PackageRegistry/CMakeLists.txt | 1 + Sources/PackageRegistry/RegistryClient.swift | 110 ++-- .../InMemoryGitRepository.swift | 4 +- Sources/SPMTestSupport/MockPackage.swift | 10 +- .../MockPackageFingerprintStorage.swift | 78 +++ Sources/SPMTestSupport/MockRegistry.swift | 6 +- Sources/SPMTestSupport/MockWorkspace.swift | 24 +- Sources/Workspace/CMakeLists.txt | 1 + .../SourceControlPackageContainer.swift | 69 +++ Sources/Workspace/Workspace.swift | 34 +- .../Workspace/WorkspaceConfiguration.swift | 18 +- .../RegistryClientTests.swift | 482 +++++++++++++++--- Tests/WorkspaceTests/WorkspaceTests.swift | 18 +- 21 files changed, 791 insertions(+), 127 deletions(-) create mode 100644 Sources/SPMTestSupport/MockPackageFingerprintStorage.swift diff --git a/Package.swift b/Package.swift index 87dc6aaf70f..679efb7753c 100644 --- a/Package.swift +++ b/Package.swift @@ -155,6 +155,7 @@ let package = Package( name: "PackageRegistry", dependencies: [ "Basics", + "PackageFingerprint", "PackageLoading", "PackageModel" ], @@ -309,6 +310,7 @@ let package = Package( name: "Workspace", dependencies: [ "Basics", + "PackageFingerprint", "PackageGraph", "PackageModel", "SourceControl", @@ -328,6 +330,7 @@ let package = Package( "Basics", "Build", "PackageCollections", + "PackageFingerprint", "PackageGraph", "SourceControl", "Workspace", @@ -388,6 +391,7 @@ let package = Package( name: "SPMTestSupport", dependencies: [ "Basics", + "PackageFingerprint", "PackageGraph", "PackageLoading", "PackageRegistry", diff --git a/Sources/Basics/FileSystem+Extensions.swift b/Sources/Basics/FileSystem+Extensions.swift index d50dee5021f..0f529defc56 100644 --- a/Sources/Basics/FileSystem+Extensions.swift +++ b/Sources/Basics/FileSystem+Extensions.swift @@ -1,7 +1,7 @@ /* This source file is part of the Swift.org open source project - Copyright (c) 2020 Apple Inc. and the Swift project authors + Copyright (c) 2020-2021 Apple Inc. and the Swift project authors Licensed under Apache License v2.0 with Runtime Library Exception See http://swift.org/LICENSE.txt for license information @@ -18,7 +18,12 @@ import TSCBasic extension FileSystem { /// SwiftPM directory under user's home directory (~/.swiftpm) public var dotSwiftPM: AbsolutePath { - return self.homeDirectory.appending(component: ".swiftpm") + self.homeDirectory.appending(component: ".swiftpm") + } + + /// SwiftPM security directory + public var swiftPMSecurityDirectory: AbsolutePath { + self.dotSwiftPM.appending(component: "security") } } diff --git a/Sources/Commands/CMakeLists.txt b/Sources/Commands/CMakeLists.txt index f0e8ac60735..8b3d6f1e5f2 100644 --- a/Sources/Commands/CMakeLists.txt +++ b/Sources/Commands/CMakeLists.txt @@ -34,6 +34,7 @@ target_link_libraries(Commands PUBLIC Basics Build PackageCollections + PackageFingerprint PackageGraph SourceControl TSCBasic diff --git a/Sources/Commands/Options.swift b/Sources/Commands/Options.swift index 5a2a726678a..b98ee381bf0 100644 --- a/Sources/Commands/Options.swift +++ b/Sources/Commands/Options.swift @@ -1,7 +1,7 @@ /* This source file is part of the Swift.org open source project - Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors + Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors Licensed under Apache License v2.0 with Runtime Library Exception See http://swift.org/LICENSE.txt for license information @@ -11,6 +11,7 @@ import ArgumentParser import TSCBasic import TSCUtility +import PackageFingerprint import PackageModel import SPMBuildCore import Build @@ -90,6 +91,12 @@ enum BuildSystemKind: String, ExpressibleByArgument, CaseIterable { case xcode } +extension FingerprintCheckingMode: ExpressibleByArgument { + public init?(argument: String) { + self.init(rawValue: argument) + } +} + public extension Sanitizer { init(argument: String) throws { if let sanitizer = Sanitizer(rawValue: argument) { @@ -348,6 +355,9 @@ public struct SwiftToolOptions: ParsableArguments { help: .hidden) var keychain: Bool = false #endif + + @Option(name: .customLong("resolver-fingerprint-checking")) + var resolverFingerprintCheckingMode: FingerprintCheckingMode = .warn @Flag(name: .customLong("netrc"), help: .hidden) var _deprecated_netrc: Bool = false diff --git a/Sources/Commands/SwiftTool.swift b/Sources/Commands/SwiftTool.swift index d0445d3d866..1bc01ec454c 100644 --- a/Sources/Commands/SwiftTool.swift +++ b/Sources/Commands/SwiftTool.swift @@ -658,6 +658,7 @@ public class SwiftTool { workingDirectory: buildPath, editsDirectory: self.editsDirectory(), resolvedVersionsFile: self.resolvedVersionsFile(), + sharedSecurityDirectory: localFileSystem.swiftPMSecurityDirectory, sharedCacheDirectory: sharedCacheDirectory, sharedConfigurationDirectory: sharedConfigurationDirectory ), @@ -669,6 +670,7 @@ public class SwiftTool { additionalFileRules: isXcodeBuildSystemEnabled ? FileRuleDescription.xcbuildFileTypes : FileRuleDescription.swiftpmFileTypes, resolverUpdateEnabled: !options.skipDependencyUpdate, resolverPrefetchingEnabled: options.shouldEnableResolverPrefetching, + resolverFingerprintCheckingMode: self.options.resolverFingerprintCheckingMode, sharedRepositoriesCacheEnabled: self.options.useRepositoriesCache, delegate: delegate ) diff --git a/Sources/PackageFingerprint/FilePackageFingerprintStorage.swift b/Sources/PackageFingerprint/FilePackageFingerprintStorage.swift index f1a7e12c25d..7db93f1a055 100644 --- a/Sources/PackageFingerprint/FilePackageFingerprintStorage.swift +++ b/Sources/PackageFingerprint/FilePackageFingerprintStorage.swift @@ -22,7 +22,7 @@ public struct FilePackageFingerprintStorage: PackageFingerprintStorage { private let encoder: JSONEncoder private let decoder: JSONDecoder - init(fileSystem: FileSystem, directoryPath: AbsolutePath) { + public init(fileSystem: FileSystem, directoryPath: AbsolutePath) { self.fileSystem = fileSystem self.directoryPath = directoryPath diff --git a/Sources/PackageFingerprint/Model.swift b/Sources/PackageFingerprint/Model.swift index c6a5352210f..2d610784ff2 100644 --- a/Sources/PackageFingerprint/Model.swift +++ b/Sources/PackageFingerprint/Model.swift @@ -14,6 +14,11 @@ import TSCUtility public struct Fingerprint: Equatable { public let origin: Origin public let value: String + + public init(origin: Origin, value: String) { + self.origin = origin + self.value = value + } } public extension Fingerprint { @@ -26,7 +31,7 @@ public extension Fingerprint { case sourceControl(Foundation.URL) case registry(Foundation.URL) - var kind: Fingerprint.Kind { + public var kind: Fingerprint.Kind { switch self { case .sourceControl: return .sourceControl @@ -35,7 +40,7 @@ public extension Fingerprint { } } - var url: Foundation.URL? { + public var url: Foundation.URL? { switch self { case .sourceControl(let url): return url @@ -56,3 +61,9 @@ public extension Fingerprint { } public typealias PackageFingerprints = [Version: [Fingerprint.Kind: Fingerprint]] + +public enum FingerprintCheckingMode: String { + case strict + case warn + case none +} diff --git a/Sources/PackageFingerprint/PackageFingerprintStorage.swift b/Sources/PackageFingerprint/PackageFingerprintStorage.swift index bb43665cc08..d2223a8ad05 100644 --- a/Sources/PackageFingerprint/PackageFingerprintStorage.swift +++ b/Sources/PackageFingerprint/PackageFingerprintStorage.swift @@ -28,6 +28,24 @@ public protocol PackageFingerprintStorage { callback: @escaping (Result) -> Void) } +public extension PackageFingerprintStorage { + func get(package: PackageIdentity, + version: Version, + kind: Fingerprint.Kind, + observabilityScope: ObservabilityScope, + callbackQueue: DispatchQueue, + callback: @escaping (Result) -> Void) { + self.get(package: package, version: version, observabilityScope: observabilityScope, callbackQueue: callbackQueue) { result in + callback(result.tryMap { fingerprints in + guard let fingerprint = fingerprints[kind] else { + throw PackageFingerprintStorageError.notFound + } + return fingerprint + }) + } + } +} + public enum PackageFingerprintStorageError: Error, Equatable { case conflict(given: Fingerprint, existing: Fingerprint) case notFound diff --git a/Sources/PackageRegistry/CMakeLists.txt b/Sources/PackageRegistry/CMakeLists.txt index 9994e413ed6..1af50819327 100644 --- a/Sources/PackageRegistry/CMakeLists.txt +++ b/Sources/PackageRegistry/CMakeLists.txt @@ -12,6 +12,7 @@ add_library(PackageRegistry RegistryClient.swift) target_link_libraries(PackageRegistry PUBLIC Basics + PackageFingerprint PackageLoading PackageModel TSCBasic diff --git a/Sources/PackageRegistry/RegistryClient.swift b/Sources/PackageRegistry/RegistryClient.swift index e5186d9ff1c..68430e2117d 100644 --- a/Sources/PackageRegistry/RegistryClient.swift +++ b/Sources/PackageRegistry/RegistryClient.swift @@ -14,14 +14,13 @@ import class Foundation.JSONDecoder import struct Foundation.URL import struct Foundation.URLComponents import struct Foundation.URLQueryItem +import PackageFingerprint import PackageLoading import PackageModel import TSCBasic import protocol TSCUtility.Archiver import struct TSCUtility.ZipArchiver -/// Package registry client. -/// API specification: /~https://github.com/apple/swift-package-manager/blob/main/Documentation/Registry.md public enum RegistryError: Error, CustomStringConvertible { case registryNotConfigured(scope: PackageIdentity.Scope?) case invalidPackage(PackageIdentity) @@ -35,6 +34,7 @@ public enum RegistryError: Error, CustomStringConvertible { case unsupportedHashAlgorithm(String) case failedToDetermineExpectedChecksum(Error) case failedToComputeChecksum(Error) + case checksumChanged(latest: String, previous: String) case invalidChecksum(expected: String, actual: String) case pathAlreadyExists(AbsolutePath) case failedRetrievingReleases(Error) @@ -72,6 +72,8 @@ public enum RegistryError: Error, CustomStringConvertible { return "Failed determining registry source archive checksum: \(error)" case .failedToComputeChecksum(let error): return "Failed computing registry source archive checksum: \(error)" + case .checksumChanged(let latest, let previous): + return "The latest checksum '\(latest)' is different from the previously recorded value '\(previous)'" case .invalidChecksum(let expected, let actual): return "Invalid registry source archive checksum '\(actual)', expected '\(expected)'" case .pathAlreadyExists(let path): @@ -88,6 +90,8 @@ public enum RegistryError: Error, CustomStringConvertible { } } +/// Package registry client. +/// API specification: /~https://github.com/apple/swift-package-manager/blob/main/Documentation/Registry.md public final class RegistryClient { private let apiVersion: APIVersion = .v1 @@ -96,11 +100,15 @@ public final class RegistryClient { private let archiverProvider: (FileSystem) -> Archiver private let httpClient: HTTPClient private let authorizationProvider: HTTPClientAuthorizationProvider? + private let fingerprintStorage: PackageFingerprintStorage + private let fingerprintCheckingMode: FingerprintCheckingMode private let jsonDecoder: JSONDecoder public init( configuration: RegistryConfiguration, identityResolver: IdentityResolver, + fingerprintStorage: PackageFingerprintStorage, + fingerprintCheckingMode: FingerprintCheckingMode, authorizationProvider: HTTPClientAuthorizationProvider? = .none, customHTTPClient: HTTPClient? = .none, customArchiverProvider: ((FileSystem) -> Archiver)? = .none @@ -110,6 +118,8 @@ public final class RegistryClient { self.authorizationProvider = authorizationProvider self.httpClient = customHTTPClient ?? HTTPClient() self.archiverProvider = customArchiverProvider ?? { fileSystem in ZipArchiver(fileSystem: fileSystem) } + self.fingerprintStorage = fingerprintStorage + self.fingerprintCheckingMode = fingerprintCheckingMode self.jsonDecoder = JSONDecoder.makeWithDefaults() } @@ -162,7 +172,7 @@ public final class RegistryClient { .compactMap { Version($0.key) } .sorted(by: >) return versions - }.mapError{ + }.mapError { RegistryError.failedRetrievingReleases($0) } ) @@ -221,19 +231,19 @@ public final class RegistryClient { let toolsVersion = try ToolsVersionLoader().load(utf8String: manifestContent) result[Manifest.filename] = toolsVersion - let alternativeManifests = try response.headers.get("Link").map { try parseLinkHeader($0) }.flatMap{ $0 } + let alternativeManifests = try response.headers.get("Link").map { try parseLinkHeader($0) }.flatMap { $0 } for alternativeManifest in alternativeManifests { result[alternativeManifest.filename] = alternativeManifest.toolsVersion } return result - }.mapError{ + }.mapError { RegistryError.failedRetrievingManifest($0) } ) } func parseLinkHeader(_ value: String) throws -> [ManifestLink] { - let linkLines = value.split(separator: ",").map(String.init).map{ $0.spm_chuzzle() ?? $0 } + let linkLines = value.split(separator: ",").map(String.init).map { $0.spm_chuzzle() ?? $0 } return try linkLines.compactMap { linkLine in try parseLinkLine(linkLine) } @@ -243,7 +253,7 @@ public final class RegistryClient { func parseLinkLine(_ value: String) throws -> ManifestLink? { let fields = value.split(separator: ";") .map(String.init) - .map{ $0.spm_chuzzle() ?? $0 } + .map { $0.spm_chuzzle() ?? $0 } guard fields.count == 4 else { return nil @@ -279,7 +289,7 @@ public final class RegistryClient { func parseLinkFieldValue(_ field: String) -> String? { let parts = field.split(separator: "=") .map(String.init) - .map{ $0.spm_chuzzle() ?? $0 } + .map { $0.spm_chuzzle() ?? $0 } guard parts.count == 2 else { return nil @@ -351,7 +361,7 @@ public final class RegistryClient { } return manifestContent - }.mapError{ + }.mapError { RegistryError.failedRetrievingManifest($0) } ) @@ -395,8 +405,9 @@ public final class RegistryClient { ) self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) { result in - completion( - result.tryMap { response in + switch result { + case .success(let response): + do { try self.checkResponseStatusAndHeaders(response, expectedStatusCode: 200, expectedContentType: .json) guard let data = response.body else { @@ -412,11 +423,34 @@ public final class RegistryClient { throw RegistryError.invalidSourceArchive } - return checksum - }.mapError{ - RegistryError.failedRetrievingReleaseChecksum($0) + self.fingerprintStorage.put(package: package, + version: version, + fingerprint: .init(origin: .registry(registry.url), value: checksum), + observabilityScope: observabilityScope, + callbackQueue: callbackQueue) { storageResult in + switch storageResult { + case .success: + completion(.success(checksum)) + case .failure(PackageFingerprintStorageError.conflict(_, let existing)): + switch self.fingerprintCheckingMode { + case .strict: + completion(.failure(RegistryError.checksumChanged(latest: checksum, previous: existing.value))) + case .warn: + observabilityScope.emit(warning: "The checksum \(checksum) from \(registry.url.absoluteString) does not match previously recorded value \(existing.value) from \(String(describing: existing.origin.url?.absoluteString))") + completion(.success(checksum)) + case .none: + completion(.success(checksum)) + } + case .failure(let error): + completion(.failure(error)) + } + } + } catch { + completion(.failure(RegistryError.failedRetrievingReleaseChecksum(error))) } - ) + case .failure(let error): + completion(.failure(RegistryError.failedRetrievingReleaseChecksum(error))) + } } } @@ -425,7 +459,6 @@ public final class RegistryClient { version: Version, fileSystem: FileSystem, destinationPath: AbsolutePath, - expectedChecksum: String?, // previously recorded checksum, if any checksumAlgorithm: HashAlgorithm, // the same algorithm used by `package compute-checksum` tool progressHandler: ((_ bytesReceived: Int64, _ totalBytes: Int64?) -> Void)?, timeout: DispatchTimeInterval? = .none, @@ -495,8 +528,16 @@ public final class RegistryClient { do { let contents = try fileSystem.readFileContents(downloadPath) let actualChecksum = checksumAlgorithm.hash(contents).hexadecimalRepresentation - guard expectedChecksum == actualChecksum else { - return completion(.failure(RegistryError.invalidChecksum(expected: expectedChecksum, actual: actualChecksum))) + + if expectedChecksum != actualChecksum { + switch self.fingerprintCheckingMode { + case .strict: + return completion(.failure(RegistryError.invalidChecksum(expected: expectedChecksum, actual: actualChecksum))) + case .warn: + observabilityScope.emit(warning: "The checksum \(actualChecksum) does not match previously recorded value \(expectedChecksum)") + case .none: + break + } } let archiver = self.archiverProvider(fileSystem) @@ -526,16 +567,26 @@ public final class RegistryClient { // We either use a previously recorded checksum, or fetch it from the registry func withExpectedChecksum(body: @escaping (Result) -> Void) { - if let expectedChecksum = expectedChecksum { - return body(.success(expectedChecksum)) + self.fingerprintStorage.get(package: package, + version: version, + kind: .registry, + observabilityScope: observabilityScope, + callbackQueue: callbackQueue) { result in + switch result { + case .success(let fingerprint): + body(.success(fingerprint.value)) + case .failure(let error): + if error as? PackageFingerprintStorageError != .notFound { + observabilityScope.emit(error: "Failed to get registry fingerprint for \(package) \(version) from storage: \(error)") + } + // Try fetching checksum from registry again no matter which kind of error it is + self.fetchSourceArchiveChecksum(package: package, + version: version, + observabilityScope: observabilityScope, + callbackQueue: callbackQueue, + completion: body) + } } - self.fetchSourceArchiveChecksum( - package: package, - version: version, - observabilityScope: observabilityScope, - callbackQueue: callbackQueue, - completion: body - ) } } @@ -655,9 +706,8 @@ private extension RegistryClient { // MARK: - Serialization -extension RegistryClient { - public enum Serialization { - +public extension RegistryClient { + enum Serialization { public struct PackageMetadata: Codable { public var releases: [String: Release] diff --git a/Sources/SPMTestSupport/InMemoryGitRepository.swift b/Sources/SPMTestSupport/InMemoryGitRepository.swift index dc0c72a365f..cbb0fd501af 100644 --- a/Sources/SPMTestSupport/InMemoryGitRepository.swift +++ b/Sources/SPMTestSupport/InMemoryGitRepository.swift @@ -107,9 +107,9 @@ public final class InMemoryGitRepository { /// Commits the current state of the repository filesystem and returns the commit identifier. @discardableResult - public func commit() throws -> String { + public func commit(hash: String? = nil) throws -> String { // Create a fake hash for this commit. - let hash = String((UUID().uuidString + UUID().uuidString).prefix(40)) + let hash = hash ?? String((UUID().uuidString + UUID().uuidString).prefix(40)) self.lock.withLock { self.head.hash = hash // Store the commit in history. diff --git a/Sources/SPMTestSupport/MockPackage.swift b/Sources/SPMTestSupport/MockPackage.swift index ff59ad739aa..a1b26388647 100644 --- a/Sources/SPMTestSupport/MockPackage.swift +++ b/Sources/SPMTestSupport/MockPackage.swift @@ -1,7 +1,7 @@ /* This source file is part of the Swift.org open source project - Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors + Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors Licensed under Apache License v2.0 with Runtime Library Exception See http://swift.org/LICENSE.txt for license information @@ -20,6 +20,8 @@ public struct MockPackage { public let products: [MockProduct] public let dependencies: [MockDependency] public let versions: [String?] + /// Provides revision identifier for the given version. A random identifier might be assigned if this is nil. + public let revisionProvider: ((String) -> String)? // FIXME: This should be per-version. public let toolsVersion: ToolsVersion? @@ -31,6 +33,7 @@ public struct MockPackage { products: [MockProduct] = [], dependencies: [MockDependency] = [], versions: [String?] = [], + revisionProvider: ((String) -> String)? = nil, toolsVersion: ToolsVersion? = nil ) { self.name = name @@ -40,6 +43,7 @@ public struct MockPackage { self.products = products self.dependencies = dependencies self.versions = versions + self.revisionProvider = revisionProvider self.toolsVersion = toolsVersion } @@ -51,6 +55,7 @@ public struct MockPackage { products: [MockProduct], dependencies: [MockDependency] = [], versions: [String?] = [], + revisionProvider: ((String) -> String)? = nil, toolsVersion: ToolsVersion? = nil ) { self.name = name @@ -60,6 +65,7 @@ public struct MockPackage { self.products = products self.dependencies = dependencies self.versions = versions + self.revisionProvider = revisionProvider self.toolsVersion = toolsVersion } @@ -71,6 +77,7 @@ public struct MockPackage { products: [MockProduct], dependencies: [MockDependency] = [], versions: [String?] = [], + revisionProvider: ((String) -> String)? = nil, toolsVersion: ToolsVersion? = nil ) { self.name = name @@ -80,6 +87,7 @@ public struct MockPackage { self.products = products self.dependencies = dependencies self.versions = versions + self.revisionProvider = revisionProvider self.toolsVersion = toolsVersion } diff --git a/Sources/SPMTestSupport/MockPackageFingerprintStorage.swift b/Sources/SPMTestSupport/MockPackageFingerprintStorage.swift new file mode 100644 index 00000000000..8f0cce7c834 --- /dev/null +++ b/Sources/SPMTestSupport/MockPackageFingerprintStorage.swift @@ -0,0 +1,78 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2021 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See http://swift.org/LICENSE.txt for license information + See http://swift.org/CONTRIBUTORS.txt for Swift project authors + */ + +import Basics +import Dispatch +import PackageFingerprint +import PackageModel +import TSCBasic +import TSCUtility + +public class MockPackageFingerprintStorage: PackageFingerprintStorage { + private var packageFingerprints: [PackageIdentity: [Version: [Fingerprint.Kind: Fingerprint]]] + private let lock = Lock() + + public init(_ packageFingerprints: [PackageIdentity: [Version: [Fingerprint.Kind: Fingerprint]]] = [:]) { + self.packageFingerprints = packageFingerprints + } + + public func get(package: PackageIdentity, + version: Version, + observabilityScope: ObservabilityScope, + callbackQueue: DispatchQueue, + callback: @escaping (Result<[Fingerprint.Kind: Fingerprint], Error>) -> Void) + { + if let fingerprints = self.lock.withLock({ self.packageFingerprints[package]?[version] }) { + callbackQueue.async { + callback(.success(fingerprints)) + } + } else { + callbackQueue.async { + callback(.failure(PackageFingerprintStorageError.notFound)) + } + } + } + + public func put(package: PackageIdentity, + version: Version, + fingerprint: Fingerprint, + observabilityScope: ObservabilityScope, + callbackQueue: DispatchQueue, + callback: @escaping (Result) -> Void) + { + do { + try self.lock.withLock { + var versionFingerprints = self.packageFingerprints[package] ?? [:] + var fingerprints = versionFingerprints[version] ?? [:] + + if let existing = fingerprints[fingerprint.origin.kind] { + // Error if we try to write a different fingerprint + guard fingerprint == existing else { + throw PackageFingerprintStorageError.conflict(given: fingerprint, existing: existing) + } + // Don't need to do anything if fingerprints are the same + return + } + + fingerprints[fingerprint.origin.kind] = fingerprint + versionFingerprints[version] = fingerprints + self.packageFingerprints[package] = versionFingerprints + } + + callbackQueue.async { + callback(.success(())) + } + } catch { + callbackQueue.async { + callback(.failure(error)) + } + } + } +} diff --git a/Sources/SPMTestSupport/MockRegistry.swift b/Sources/SPMTestSupport/MockRegistry.swift index e687486fadb..02292817de4 100644 --- a/Sources/SPMTestSupport/MockRegistry.swift +++ b/Sources/SPMTestSupport/MockRegistry.swift @@ -10,6 +10,7 @@ import Basics import Foundation +import PackageFingerprint import PackageGraph import PackageLoading import PackageModel @@ -31,7 +32,8 @@ class MockRegistry { init( identityResolver: IdentityResolver, checksumAlgorithm: HashAlgorithm, - filesystem: FileSystem + filesystem: FileSystem, + fingerprintStorage: PackageFingerprintStorage ) { self.checksumAlgorithm = checksumAlgorithm self.fileSystem = filesystem @@ -43,6 +45,8 @@ class MockRegistry { self.registryClient = RegistryClient( configuration: configuration, identityResolver: identityResolver, + fingerprintStorage: fingerprintStorage, + fingerprintCheckingMode: .strict, authorizationProvider: .none, customHTTPClient: HTTPClient(handler: self.httpHandler), customArchiverProvider: { fileSystem in MockRegistryArchiver(fileSystem: fileSystem) } diff --git a/Sources/SPMTestSupport/MockWorkspace.swift b/Sources/SPMTestSupport/MockWorkspace.swift index c78c05719bc..6f16ba9190f 100644 --- a/Sources/SPMTestSupport/MockWorkspace.swift +++ b/Sources/SPMTestSupport/MockWorkspace.swift @@ -1,7 +1,7 @@ /* This source file is part of the Swift.org open source project - Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors + Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors Licensed under Apache License v2.0 with Runtime Library Exception See http://swift.org/LICENSE.txt for license information @@ -27,6 +27,7 @@ public final class MockWorkspace { public var registryClient: RegistryClient public let archiver: MockArchiver public let checksumAlgorithm: MockHashAlgorithm + public let fingerprintStorage: MockPackageFingerprintStorage let roots: [MockPackage] let packages: [MockPackage] public let mirrors: DependencyMirrors @@ -49,6 +50,7 @@ public final class MockWorkspace { customRegistryClient: RegistryClient? = .none, customBinaryArchiver: MockArchiver? = .none, customChecksumAlgorithm: MockHashAlgorithm? = .none, + customFingerprintStorage: MockPackageFingerprintStorage? = .none, resolverUpdateEnabled: Bool = true ) throws { let archiver = customBinaryArchiver ?? MockArchiver() @@ -59,6 +61,7 @@ public final class MockWorkspace { self.httpClient = httpClient self.archiver = archiver self.checksumAlgorithm = customChecksumAlgorithm ?? MockHashAlgorithm() + self.fingerprintStorage = customFingerprintStorage ?? MockPackageFingerprintStorage() self.mirrors = mirrors ?? DependencyMirrors() self.identityResolver = DefaultIdentityResolver(locationMapper: self.mirrors.effectiveURL(for:)) self.roots = roots @@ -69,7 +72,8 @@ public final class MockWorkspace { self.registry = MockRegistry( identityResolver: self.identityResolver, checksumAlgorithm: self.checksumAlgorithm, - filesystem: self.fileSystem + filesystem: self.fileSystem, + fingerprintStorage: self.fingerprintStorage ) self.registryClient = customRegistryClient ?? self.registry.registryClient self.toolsVersion = toolsVersion @@ -148,10 +152,17 @@ public final class MockWorkspace { if let specifier = sourceControlSpecifier { let repository = self.repositoryProvider.specifierMap[specifier] ?? .init(path: packagePath, fs: self.fileSystem) try writePackageContent(fileSystem: repository, root: .root, toolsVersion: toolsVersion) - try repository.commit() - for version in packageVersions.compactMap({ $0 }) { - try repository.tag(name: version) + + let versions = packageVersions.compactMap({ $0 }) + if versions.isEmpty { + try repository.commit() + } else { + for version in versions { + try repository.commit(hash: package.revisionProvider.map { $0(version) }) + try repository.tag(name: version) + } } + self.repositoryProvider.add(specifier: specifier, repository: repository) } else if let identity = registryIdentity { let source = InMemoryRegistrySource(path: packagePath, fileSystem: self.fileSystem) @@ -215,6 +226,7 @@ public final class MockWorkspace { workingDirectory: self.sandbox.appending(component: ".build"), editsDirectory: self.sandbox.appending(component: "edits"), resolvedVersionsFile: self.sandbox.appending(component: "Package.resolved"), + sharedSecurityDirectory: self.fileSystem.swiftPMSecurityDirectory, sharedCacheDirectory: self.fileSystem.swiftPMCacheDirectory, sharedConfigurationDirectory: self.fileSystem.swiftPMConfigDirectory ), @@ -227,8 +239,10 @@ public final class MockWorkspace { customHTTPClient: self.httpClient, customArchiver: self.archiver, customChecksumAlgorithm: self.checksumAlgorithm, + customFingerprintStorage: self.fingerprintStorage, resolverUpdateEnabled: self.resolverUpdateEnabled, resolverPrefetchingEnabled: true, + resolverFingerprintCheckingMode: .strict, delegate: self.delegate ) diff --git a/Sources/Workspace/CMakeLists.txt b/Sources/Workspace/CMakeLists.txt index 0762d7c1137..a1b97fec283 100644 --- a/Sources/Workspace/CMakeLists.txt +++ b/Sources/Workspace/CMakeLists.txt @@ -30,6 +30,7 @@ target_link_libraries(Workspace PUBLIC TSCUtility Basics SPMBuildCore + PackageFingerprint PackageGraph PackageLoading PackageModel diff --git a/Sources/Workspace/SourceControlPackageContainer.swift b/Sources/Workspace/SourceControlPackageContainer.swift index 5b99945f932..d7495055168 100644 --- a/Sources/Workspace/SourceControlPackageContainer.swift +++ b/Sources/Workspace/SourceControlPackageContainer.swift @@ -10,6 +10,7 @@ import Basics import Dispatch +import PackageFingerprint import PackageGraph import PackageLoading import PackageModel @@ -54,6 +55,8 @@ internal final class SourceControlPackageContainer: PackageContainer, CustomStri private let manifestLoader: ManifestLoaderProtocol private let toolsVersionLoader: ToolsVersionLoaderProtocol private let currentToolsVersion: ToolsVersion + private let fingerprintStorage: PackageFingerprintStorage + private let fingerprintCheckingMode: FingerprintCheckingMode private let observabilityScope: ObservabilityScope /// The cached dependency information. @@ -76,6 +79,8 @@ internal final class SourceControlPackageContainer: PackageContainer, CustomStri manifestLoader: ManifestLoaderProtocol, toolsVersionLoader: ToolsVersionLoaderProtocol, currentToolsVersion: ToolsVersion, + fingerprintStorage: PackageFingerprintStorage, + fingerprintCheckingMode: FingerprintCheckingMode, observabilityScope: ObservabilityScope ) throws { self.package = package @@ -85,6 +90,8 @@ internal final class SourceControlPackageContainer: PackageContainer, CustomStri self.manifestLoader = manifestLoader self.toolsVersionLoader = toolsVersionLoader self.currentToolsVersion = currentToolsVersion + self.fingerprintStorage = fingerprintStorage + self.fingerprintCheckingMode = fingerprintCheckingMode self.observabilityScope = observabilityScope } @@ -142,6 +149,68 @@ internal final class SourceControlPackageContainer: PackageContainer, CustomStri return try? self.knownVersions()[version] } + func checkIntegrity(version: Version, revision: Revision) throws { + guard case .remoteSourceControl(let sourceControlURL) = self.package.kind else { + return + } + + let packageIdentity = self.package.identity + let fingerprint: Fingerprint + do { + fingerprint = try temp_await { + self.fingerprintStorage.get( + package: packageIdentity, + version: version, + kind: .sourceControl, + observabilityScope: self.observabilityScope, + callbackQueue: .sharedConcurrent, + callback: $0 + ) + } + } catch PackageFingerprintStorageError.notFound { + fingerprint = Fingerprint(origin: .sourceControl(sourceControlURL), value: revision.identifier) + // Write to storage if fingerprint not yet recorded + do { + try temp_await { + self.fingerprintStorage.put( + package: packageIdentity, + version: version, + fingerprint: fingerprint, + observabilityScope: self.observabilityScope, + callbackQueue: .sharedConcurrent, + callback: $0 + ) + } + } catch PackageFingerprintStorageError.conflict(_, let existing) { + let message = "Revision \(revision.identifier) for \(self.package) version \(version) does not match previously recorded value \(existing.value) from \(String(describing: existing.origin.url?.absoluteString))" + switch self.fingerprintCheckingMode { + case .strict: + throw StringError(message) + case .warn: + observabilityScope.emit(warning: message) + case .none: + return + } + } + } catch { + self.observabilityScope.emit(error: "Failed to get source control fingerprint for \(self.package) version \(version) from storage: \(error)") + throw error + } + + // The revision (i.e., hash) must match that in fingerprint storage otherwise the integrity check fails + if revision.identifier != fingerprint.value { + let message = "Revision \(revision.identifier) for \(self.package) version \(version) does not match previously recorded value \(fingerprint.value)" + switch self.fingerprintCheckingMode { + case .strict: + throw StringError(message) + case .warn: + observabilityScope.emit(warning: message) + case .none: + return + } + } + } + /// Returns revision for the given tag. public func getRevision(forTag tag: String) throws -> Revision { return try repository.resolveRevision(tag: tag) diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index 6e209e475da..352e452371e 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -1,7 +1,7 @@ /* This source file is part of the Swift.org open source project - Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors + Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors Licensed under Apache License v2.0 with Runtime Library Exception See http://swift.org/LICENSE.txt for license information @@ -14,6 +14,7 @@ import TSCUtility import Foundation import PackageLoading import PackageModel +import PackageFingerprint import PackageGraph import PackageRegistry import SourceControl @@ -214,12 +215,18 @@ public class Workspace { /// The algorithm used for generating file checksums. fileprivate let checksumAlgorithm: HashAlgorithm + + /// The package fingerprint storage + fileprivate let fingerprintStorage: PackageFingerprintStorage /// Enable prefetching containers in resolver. fileprivate let resolverPrefetchingEnabled: Bool /// Update containers while fetching them. fileprivate let resolverUpdateEnabled: Bool + + /// Fingerprint checking mode. + fileprivate let resolverFingerprintCheckingMode: FingerprintCheckingMode fileprivate let additionalFileRules: [FileRuleDescription] @@ -254,8 +261,9 @@ public class Workspace { /// - customChecksumAlgorithm: A custom checksum algorithm. /// - additionalFileRules: File rules to determine resource handling behavior. /// - resolverUpdateEnabled: Enables the dependencies resolver automatic version update check. Enabled by default. When disabled the resolver relies only on the resolved version file - /// - resolverPrefetchingEnabled: Enables the dependencies resolver prefetching based on the resolved version file. Enabled by default.. - /// - sharedRepositoriesCacheEnabled: Enables the shared repository cache. Enabled by default.. + /// - resolverPrefetchingEnabled: Enables the dependencies resolver prefetching based on the resolved version file. Enabled by default. + /// - resolverFingerprintCheckingMode: Fingerprint checking mode. Defaults to `.warn`. + /// - sharedRepositoriesCacheEnabled: Enables the shared repository cache. Enabled by default. /// - delegate: Delegate for workspace events public init( fileSystem: FileSystem, @@ -272,9 +280,11 @@ public class Workspace { customHTTPClient: HTTPClient? = .none, customArchiver: Archiver? = .none, customChecksumAlgorithm: HashAlgorithm? = .none, + customFingerprintStorage: PackageFingerprintStorage? = .none, additionalFileRules: [FileRuleDescription]? = .none, resolverUpdateEnabled: Bool? = .none, resolverPrefetchingEnabled: Bool? = .none, + resolverFingerprintCheckingMode: FingerprintCheckingMode = .warn, sharedRepositoriesCacheEnabled: Bool? = .none, delegate: WorkspaceDelegate? = .none ) throws { @@ -295,11 +305,18 @@ public class Workspace { provider: repositoryProvider, delegate: delegate.map(WorkspaceRepositoryManagerDelegate.init(workspaceDelegate:)), cachePath: sharedRepositoriesCacheEnabled ? location.sharedRepositoriesCacheDirectory : .none + ) + let fingerprintStorage = customFingerprintStorage ?? FilePackageFingerprintStorage( + fileSystem: fileSystem, + directoryPath: location.sharedFingerprintsDirectory ) + let registryClient = customRegistryClient ?? registries.map { configuration in RegistryClient( configuration: configuration, identityResolver: identityResolver, + fingerprintStorage: fingerprintStorage, + fingerprintCheckingMode: resolverFingerprintCheckingMode, authorizationProvider: authorizationProvider?.httpAuthorizationHeader(for:) ) } @@ -334,6 +351,7 @@ public class Workspace { self.registryClient = registryClient self.identityResolver = identityResolver self.checksumAlgorithm = checksumAlgorithm + self.fingerprintStorage = fingerprintStorage self.pinsStore = LoadableResult { try PinsStore( @@ -347,6 +365,7 @@ public class Workspace { self.additionalFileRules = additionalFileRules self.resolverUpdateEnabled = resolverUpdateEnabled self.resolverPrefetchingEnabled = resolverPrefetchingEnabled + self.resolverFingerprintCheckingMode = resolverFingerprintCheckingMode self.state = WorkspaceState(dataPath: self.location.workingDirectory, fileSystem: fileSystem) } @@ -386,6 +405,7 @@ public class Workspace { workingDirectory: dataPath, editsDirectory: editablesPath, resolvedVersionsFile: pinsFile, + sharedSecurityDirectory: fileSystem.swiftPMSecurityDirectory, sharedCacheDirectory: cachePath, sharedConfigurationDirectory: nil // legacy ), @@ -455,8 +475,8 @@ public class Workspace { public convenience init( fileSystem: FileSystem? = .none, forRootPackage packagePath: AbsolutePath, - customManifestLoader: ManifestLoaderProtocol? = .none, - delegate: WorkspaceDelegate? = .none + customManifestLoader: ManifestLoaderProtocol? = .none, + delegate: WorkspaceDelegate? = .none ) throws { let fileSystem = fileSystem ?? localFileSystem let location = Location(forRootPackage: packagePath, fileSystem: fileSystem) @@ -2594,6 +2614,7 @@ extension Workspace { throw InternalError("unable to get tag for \(package) \(version); available versions \(try container.versionsDescending())") } let revision = try container.getRevision(forTag: tag) + try container.checkIntegrity(version: version, revision: revision) return try self.checkoutRepository(package: package, at: .version(version, revision: revision), observabilityScope: observabilityScope) } else if let _ = container as? RegistryPackageContainer { return try self.downloadRegistryArchive(package: package, at: version, observabilityScope: observabilityScope) @@ -3051,6 +3072,8 @@ extension Workspace: PackageContainerProvider { manifestLoader: self.manifestLoader, toolsVersionLoader: self.toolsVersionLoader, currentToolsVersion: self.currentToolsVersion, + fingerprintStorage: self.fingerprintStorage, + fingerprintCheckingMode: self.resolverFingerprintCheckingMode, observabilityScope: observabilityScope ) } @@ -3326,7 +3349,6 @@ extension Workspace { version: version, fileSystem: self.fileSystem, destinationPath: downloadPath, - expectedChecksum: nil, // we dont know at this point checksumAlgorithm: self.checksumAlgorithm, progressHandler: progressHandler, observabilityScope: observabilityScope, diff --git a/Sources/Workspace/WorkspaceConfiguration.swift b/Sources/Workspace/WorkspaceConfiguration.swift index db52c772295..6f575c7a3b0 100644 --- a/Sources/Workspace/WorkspaceConfiguration.swift +++ b/Sources/Workspace/WorkspaceConfiguration.swift @@ -30,6 +30,9 @@ extension Workspace { /// Path to the Package.resolved file. public var resolvedVersionsFile: AbsolutePath + + /// Path to the shared security directory + public var sharedSecurityDirectory: AbsolutePath /// Path to the shared cache directory public var sharedCacheDirectory: AbsolutePath? @@ -56,6 +59,11 @@ extension Workspace { public var artifactsDirectory: AbsolutePath { self.workingDirectory.appending(component: "artifacts") } + + /// Path to the shared fingerprints directory. + public var sharedFingerprintsDirectory: AbsolutePath { + self.sharedSecurityDirectory.appending(component: "fingerprints") + } /// Path to the shared repositories cache. public var sharedRepositoriesCacheDirectory: AbsolutePath? { @@ -67,7 +75,7 @@ extension Workspace { self.sharedCacheDirectory.map { DefaultLocations.manifestsDirectory(at: $0) } } - /// Path to the shared cache. + /// Path to the shared mirrors configuration. public var sharedMirrorsConfigurationFile: AbsolutePath? { self.sharedConfigurationDirectory.map { DefaultLocations.mirrorsConfigurationFile(at: $0) } } @@ -88,18 +96,21 @@ extension Workspace { /// - workingDirectory: Path to working directory for this workspace. /// - editsDirectory: Path to store the editable versions of dependencies. /// - resolvedVersionsFile: Path to the Package.resolved file. - /// - sharedCacheDirectory: Path to the shared cache directory - /// - sharedConfigurationDirectory: Path to the shared configuration directory + /// - sharedSecurityDirectory: Path to the shared security directory. + /// - sharedCacheDirectory: Path to the shared cache directory. + /// - sharedConfigurationDirectory: Path to the shared configuration directory. public init( workingDirectory: AbsolutePath, editsDirectory: AbsolutePath, resolvedVersionsFile: AbsolutePath, + sharedSecurityDirectory: AbsolutePath, sharedCacheDirectory: AbsolutePath?, sharedConfigurationDirectory: AbsolutePath? ) { self.workingDirectory = workingDirectory self.editsDirectory = editsDirectory self.resolvedVersionsFile = resolvedVersionsFile + self.sharedSecurityDirectory = sharedSecurityDirectory self.sharedCacheDirectory = sharedCacheDirectory self.sharedConfigurationDirectory = sharedConfigurationDirectory } @@ -113,6 +124,7 @@ extension Workspace { workingDirectory: DefaultLocations.workingDirectory(forRootPackage: rootPath), editsDirectory: DefaultLocations.editsDirectory(forRootPackage: rootPath), resolvedVersionsFile: DefaultLocations.resolvedVersionsFile(forRootPackage: rootPath), + sharedSecurityDirectory: fileSystem.swiftPMSecurityDirectory, sharedCacheDirectory: fileSystem.swiftPMCacheDirectory, sharedConfigurationDirectory: fileSystem.swiftPMConfigDirectory ) diff --git a/Tests/PackageRegistryTests/RegistryClientTests.swift b/Tests/PackageRegistryTests/RegistryClientTests.swift index 74a8b6d69d8..a0f7ebb658b 100644 --- a/Tests/PackageRegistryTests/RegistryClientTests.swift +++ b/Tests/PackageRegistryTests/RegistryClientTests.swift @@ -10,6 +10,7 @@ import Basics import Foundation +import PackageFingerprint import PackageLoading import PackageModel import PackageRegistry @@ -71,14 +72,8 @@ final class RegistryClientTests: XCTestCase { var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: URL(string: registryURL)!) - let registryManager = RegistryClient( - configuration: configuration, - identityResolver: DefaultIdentityResolver(), - customHTTPClient: httpClient, - customArchiverProvider: { _ in MockArchiver() } - ) - - let versions = try registryManager.fetchVersions(package: identity) + let registryClient = makeRegistryClient(configuration: configuration, httpClient: httpClient) + let versions = try registryClient.fetchVersions(package: identity) XCTAssertEqual(["1.1.1", "1.0.0"], versions) } @@ -139,25 +134,18 @@ final class RegistryClientTests: XCTestCase { var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: URL(string: registryURL)!) - let registryManager = RegistryClient( - configuration: configuration, - identityResolver: DefaultIdentityResolver(), - customHTTPClient: httpClient, - customArchiverProvider: { _ in MockArchiver() } - ) - - let availableManifests = try registryManager.getAvailableManifests( + let registryClient = makeRegistryClient(configuration: configuration, httpClient: httpClient) + let availableManifests = try registryClient.getAvailableManifests( package: identity, version: version ) XCTAssertEqual(availableManifests, - [ - "Package.swift": .v5_5, - "Package@swift-4.swift": .v4, - "Package@swift-4.2.swift": .v4_2, - "Package@swift-5.3.swift": .v5_3, - ] - ) + [ + "Package.swift": .v5_5, + "Package@swift-4.swift": .v4, + "Package@swift-4.2.swift": .v4_2, + "Package@swift-5.3.swift": .v5_3, + ]) } func testGetManifestContent() throws { @@ -169,7 +157,7 @@ final class RegistryClientTests: XCTestCase { let handler: HTTPClient.Handler = { request, _, completion in var components = URLComponents(url: request.url, resolvingAgainstBaseURL: false)! - let toolsVersion = components.queryItems?.first{ $0.name == "swift-version" }.flatMap{ ToolsVersion(string: $0.value!) } ?? ToolsVersion.currentToolsVersion + let toolsVersion = components.queryItems?.first { $0.name == "swift-version" }.flatMap { ToolsVersion(string: $0.value!) } ?? ToolsVersion.currentToolsVersion // remove query components.query = nil let urlWithoutQuery = components.url @@ -190,7 +178,7 @@ final class RegistryClientTests: XCTestCase { headers: .init([ .init(name: "Content-Length", value: "\(data.count)"), .init(name: "Content-Type", value: "text/x-swift"), - .init(name: "Content-Version", value: "1") + .init(name: "Content-Version", value: "1"), ]), body: data ))) @@ -206,15 +194,10 @@ final class RegistryClientTests: XCTestCase { var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: URL(string: registryURL)!) - let registryManager = RegistryClient( - configuration: configuration, - identityResolver: DefaultIdentityResolver(), - customHTTPClient: httpClient, - customArchiverProvider: { _ in MockArchiver() } - ) + let registryClient = makeRegistryClient(configuration: configuration, httpClient: httpClient) do { - let manifest = try registryManager.getManifestContent( + let manifest = try registryClient.getManifestContent( package: identity, version: version, customToolsVersion: nil @@ -225,7 +208,7 @@ final class RegistryClientTests: XCTestCase { } do { - let manifest = try registryManager.getManifestContent( + let manifest = try registryClient.getManifestContent( package: identity, version: version, customToolsVersion: .v5_3 @@ -236,7 +219,7 @@ final class RegistryClientTests: XCTestCase { } do { - let manifest = try registryManager.getManifestContent( + let manifest = try registryClient.getManifestContent( package: identity, version: version, customToolsVersion: .v4 @@ -253,13 +236,14 @@ final class RegistryClientTests: XCTestCase { let (scope, name) = identity.scopeAndName! let version = Version("1.1.1") let metadataURL = URL(string: "\(registryURL)/\(scope)/\(name)/\(version)")! + let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" let handler: HTTPClient.Handler = { request, _, completion in switch (request.method, request.url) { case (.get, metadataURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") - let data = #""" + let data = """ { "id": "mona.LinkedList", "version": "1.1.1", @@ -267,14 +251,14 @@ final class RegistryClientTests: XCTestCase { { "name": "source-archive", "type": "application/zip", - "checksum": "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" + "checksum": "\(checksum)" } ], "metadata": { "description": "One thing links to another." } } - """#.data(using: .utf8)! + """.data(using: .utf8)! completion(.success(.init( statusCode: 200, @@ -297,18 +281,255 @@ final class RegistryClientTests: XCTestCase { var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: URL(string: registryURL)!) - let registryManager = RegistryClient( + let fingerprintStorage = MockPackageFingerprintStorage() + let registryClient = makeRegistryClient(configuration: configuration, httpClient: httpClient, fingerprintStorage: fingerprintStorage) + + let checksumResponse = try registryClient.fetchSourceArchiveChecksum(package: identity, version: version) + XCTAssertEqual(checksum, checksumResponse) + + // Checksum should have been saved to storage + let fingerprint = try tsc_await { callback in fingerprintStorage.get(package: identity, version: version, kind: .registry, + observabilityScope: ObservabilitySystem.NOOP, callbackQueue: .sharedConcurrent, + callback: callback) } + XCTAssertEqual(registryURL, fingerprint.origin.url?.absoluteString) + XCTAssertEqual(checksum, fingerprint.value) + } + + func testFetchSourceArchiveChecksum_storageConflict() throws { + let registryURL = "https://packages.example.com" + let identity = PackageIdentity.plain("mona.LinkedList") + let (scope, name) = identity.scopeAndName! + let version = Version("1.1.1") + let metadataURL = URL(string: "\(registryURL)/\(scope)/\(name)/\(version)")! + let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" + + let handler: HTTPClient.Handler = { request, _, completion in + switch (request.method, request.url) { + case (.get, metadataURL): + XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") + + let data = """ + { + "id": "mona.LinkedList", + "version": "1.1.1", + "resources": [ + { + "name": "source-archive", + "type": "application/zip", + "checksum": "\(checksum)" + } + ], + "metadata": { + "description": "One thing links to another." + } + } + """.data(using: .utf8)! + + completion(.success(.init( + statusCode: 200, + headers: .init([ + .init(name: "Content-Length", value: "\(data.count)"), + .init(name: "Content-Type", value: "application/json"), + .init(name: "Content-Version", value: "1"), + ]), + body: data + ))) + default: + completion(.failure(StringError("method and url should match"))) + } + } + + var httpClient = HTTPClient(handler: handler) + httpClient.configuration.circuitBreakerStrategy = .none + httpClient.configuration.retryStrategy = .none + + var configuration = RegistryConfiguration() + configuration.defaultRegistry = Registry(url: URL(string: registryURL)!) + + let fingerprintStorage = MockPackageFingerprintStorage([ + identity: [ + version: [.registry: Fingerprint(origin: .registry(URL(string: registryURL)!), value: "non-matching checksum")], + ], + ]) + let registryClient = makeRegistryClient(configuration: configuration, + httpClient: httpClient, + fingerprintStorage: fingerprintStorage, + fingerprintCheckingMode: .strict) + + XCTAssertThrowsError(try registryClient.fetchSourceArchiveChecksum(package: identity, version: version)) { error in + guard case RegistryError.checksumChanged = error else { + return XCTFail("Expected RegistryError.checksumChanged, got \(error)") + } + } + } + + func testFetchSourceArchiveChecksum_storageConflict_fingerprintChecking_warn() throws { + let registryURL = "https://packages.example.com" + let identity = PackageIdentity.plain("mona.LinkedList") + let (scope, name) = identity.scopeAndName! + let version = Version("1.1.1") + let metadataURL = URL(string: "\(registryURL)/\(scope)/\(name)/\(version)")! + let checksum = "a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812" + + let handler: HTTPClient.Handler = { request, _, completion in + switch (request.method, request.url) { + case (.get, metadataURL): + XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") + + let data = """ + { + "id": "mona.LinkedList", + "version": "1.1.1", + "resources": [ + { + "name": "source-archive", + "type": "application/zip", + "checksum": "\(checksum)" + } + ], + "metadata": { + "description": "One thing links to another." + } + } + """.data(using: .utf8)! + + completion(.success(.init( + statusCode: 200, + headers: .init([ + .init(name: "Content-Length", value: "\(data.count)"), + .init(name: "Content-Type", value: "application/json"), + .init(name: "Content-Version", value: "1"), + ]), + body: data + ))) + default: + completion(.failure(StringError("method and url should match"))) + } + } + + var httpClient = HTTPClient(handler: handler) + httpClient.configuration.circuitBreakerStrategy = .none + httpClient.configuration.retryStrategy = .none + + var configuration = RegistryConfiguration() + configuration.defaultRegistry = Registry(url: URL(string: registryURL)!) + + let storedChecksum = "non-matching checksum" + let fingerprintStorage = MockPackageFingerprintStorage([ + identity: [ + version: [.registry: Fingerprint(origin: .registry(URL(string: registryURL)!), value: storedChecksum)], + ], + ]) + let registryClient = makeRegistryClient(configuration: configuration, + httpClient: httpClient, + fingerprintStorage: fingerprintStorage, + fingerprintCheckingMode: .warn) + + let observability = ObservabilitySystem.makeForTesting() + + // The checksum differs from that in storage, but error is not thrown + // because fingerprintCheckingMode=.warn + let checksumResponse = try registryClient.fetchSourceArchiveChecksum( + package: identity, + version: version, + observabilityScope: observability.topScope + ) + XCTAssertEqual(checksum, checksumResponse) + + // But there should be a warning + testDiagnostics(observability.diagnostics) { result in + result.check(diagnostic: .contains("does not match previously recorded value"), severity: .warning) + } + + // Storage should NOT be updated + let fingerprint = try tsc_await { callback in fingerprintStorage.get(package: identity, version: version, kind: .registry, + observabilityScope: ObservabilitySystem.NOOP, callbackQueue: .sharedConcurrent, + callback: callback) } + XCTAssertEqual(registryURL, fingerprint.origin.url?.absoluteString) + XCTAssertEqual(storedChecksum, fingerprint.value) + } + + func testDownloadSourceArchive_matchingChecksumInStorage() throws { + let registryURL = "https://packages.example.com" + let identity = PackageIdentity.plain("mona.LinkedList") + let (scope, name) = identity.scopeAndName! + let version = Version("1.1.1") + let downloadURL = URL(string: "\(registryURL)/\(scope)/\(name)/\(version).zip")! + + let checksumAlgorithm: HashAlgorithm = SHA256() + let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation + + let handler: HTTPClient.Handler = { request, _, completion in + switch (request.kind, request.method, request.url) { + case (.download(let fileSystem, let path), .get, downloadURL): + XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+zip") + + let data = Data(emptyZipFile.contents) + try! fileSystem.writeFileContents(path, data: data) + + completion(.success(.init( + statusCode: 200, + headers: .init([ + .init(name: "Content-Length", value: "\(data.count)"), + .init(name: "Content-Type", value: "application/zip"), + .init(name: "Content-Version", value: "1"), + .init(name: "Content-Disposition", value: #"attachment; filename="LinkedList-1.1.1.zip""#), + .init(name: "Digest", value: "sha-256=bc6c9a5d2f2226cfa1ef4fad8344b10e1cc2e82960f468f70d9ed696d26b3283"), + ]), + body: nil + ))) + default: + completion(.failure(StringError("method and url should match"))) + } + } + + var httpClient = HTTPClient(handler: handler) + httpClient.configuration.circuitBreakerStrategy = .none + httpClient.configuration.retryStrategy = .none + + var configuration = RegistryConfiguration() + configuration.defaultRegistry = Registry(url: URL(string: registryURL)!) + + let fingerprintStorage = MockPackageFingerprintStorage([ + identity: [ + version: [.registry: Fingerprint(origin: .registry(URL(string: registryURL)!), value: checksum)], + ], + ]) + let registryClient = RegistryClient( configuration: configuration, identityResolver: DefaultIdentityResolver(), + fingerprintStorage: fingerprintStorage, + fingerprintCheckingMode: .strict, customHTTPClient: httpClient, - customArchiverProvider: { _ in MockArchiver() } + customArchiverProvider: { fileSystem in + MockArchiver(handler: { _, from, to, callback in + let data = try fileSystem.readFileContents(from) + XCTAssertEqual(data, emptyZipFile) + + let packagePath = to.appending(component: "package") + try fileSystem.createDirectory(packagePath, recursive: true) + try fileSystem.writeFileContents(packagePath.appending(component: "Package.swift"), string: "") + callback(.success(())) + }) + } + ) + + let fileSystem = InMemoryFileSystem() + let path = AbsolutePath("/LinkedList-1.1.1") + + try registryClient.downloadSourceArchive( + package: identity, + version: version, + fileSystem: fileSystem, + destinationPath: path, + checksumAlgorithm: checksumAlgorithm ) - let checksum = try registryManager.fetchSourceArchiveChecksum(package: identity, version: version) - XCTAssertEqual("a2ac54cf25fbc1ad0028f03f0aa4b96833b83bb05a14e510892bb27dea4dc812", checksum) + let contents = try fileSystem.getDirectoryContents(path) + XCTAssertEqual(contents, ["Package.swift"]) } - func testDownloadSourceArchiveWithExpectedChecksumProvided() throws { + func testDownloadSourceArchive_nonMatchingChecksumInStorage() throws { let registryURL = "https://packages.example.com" let identity = PackageIdentity.plain("mona.LinkedList") let (scope, name) = identity.scopeAndName! @@ -316,10 +537,9 @@ final class RegistryClientTests: XCTestCase { let downloadURL = URL(string: "\(registryURL)/\(scope)/\(name)/\(version).zip")! let checksumAlgorithm: HashAlgorithm = SHA256() - let expectedChecksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation let handler: HTTPClient.Handler = { request, _, completion in - switch (request.kind, request.method, request.url) { + switch (request.kind, request.method, request.url) { case (.download(let fileSystem, let path), .get, downloadURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+zip") @@ -349,9 +569,16 @@ final class RegistryClientTests: XCTestCase { var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: URL(string: registryURL)!) - let registryManager = RegistryClient( + let fingerprintStorage = MockPackageFingerprintStorage([ + identity: [ + version: [.registry: Fingerprint(origin: .registry(URL(string: registryURL)!), value: "non-matching checksum")], + ], + ]) + let registryClient = RegistryClient( configuration: configuration, identityResolver: DefaultIdentityResolver(), + fingerprintStorage: fingerprintStorage, + fingerprintCheckingMode: .strict, customHTTPClient: httpClient, customArchiverProvider: { fileSystem in MockArchiver(handler: { _, from, to, callback in @@ -369,20 +596,113 @@ final class RegistryClientTests: XCTestCase { let fileSystem = InMemoryFileSystem() let path = AbsolutePath("/LinkedList-1.1.1") - try registryManager.downloadSourceArchive( + XCTAssertThrowsError( + try registryClient.downloadSourceArchive( + package: identity, + version: version, + fileSystem: fileSystem, + destinationPath: path, + checksumAlgorithm: checksumAlgorithm + )) { error in + guard case RegistryError.invalidChecksum = error else { + return XCTFail("Expected RegistryError.invalidChecksum, got \(error)") + } + } + + // Unzip didn't take place so directory is empty + let contents = try fileSystem.getDirectoryContents(path) + XCTAssertEqual(contents, []) + } + + func testDownloadSourceArchive_nonMatchingChecksumInStorage_fingerprintChecking_warn() throws { + let registryURL = "https://packages.example.com" + let identity = PackageIdentity.plain("mona.LinkedList") + let (scope, name) = identity.scopeAndName! + let version = Version("1.1.1") + let downloadURL = URL(string: "\(registryURL)/\(scope)/\(name)/\(version).zip")! + + let checksumAlgorithm: HashAlgorithm = SHA256() + + let handler: HTTPClient.Handler = { request, _, completion in + switch (request.kind, request.method, request.url) { + case (.download(let fileSystem, let path), .get, downloadURL): + XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+zip") + + let data = Data(emptyZipFile.contents) + try! fileSystem.writeFileContents(path, data: data) + + completion(.success(.init( + statusCode: 200, + headers: .init([ + .init(name: "Content-Length", value: "\(data.count)"), + .init(name: "Content-Type", value: "application/zip"), + .init(name: "Content-Version", value: "1"), + .init(name: "Content-Disposition", value: #"attachment; filename="LinkedList-1.1.1.zip""#), + .init(name: "Digest", value: "sha-256=bc6c9a5d2f2226cfa1ef4fad8344b10e1cc2e82960f468f70d9ed696d26b3283"), + ]), + body: nil + ))) + default: + completion(.failure(StringError("method and url should match"))) + } + } + + var httpClient = HTTPClient(handler: handler) + httpClient.configuration.circuitBreakerStrategy = .none + httpClient.configuration.retryStrategy = .none + + var configuration = RegistryConfiguration() + configuration.defaultRegistry = Registry(url: URL(string: registryURL)!) + + let fingerprintStorage = MockPackageFingerprintStorage([ + identity: [ + version: [.registry: Fingerprint(origin: .registry(URL(string: registryURL)!), value: "non-matching checksum")], + ], + ]) + let registryClient = RegistryClient( + configuration: configuration, + identityResolver: DefaultIdentityResolver(), + fingerprintStorage: fingerprintStorage, + fingerprintCheckingMode: .warn, + customHTTPClient: httpClient, + customArchiverProvider: { fileSystem in + MockArchiver(handler: { _, from, to, callback in + let data = try fileSystem.readFileContents(from) + XCTAssertEqual(data, emptyZipFile) + + let packagePath = to.appending(component: "package") + try fileSystem.createDirectory(packagePath, recursive: true) + try fileSystem.writeFileContents(packagePath.appending(component: "Package.swift"), string: "") + callback(.success(())) + }) + } + ) + + let fileSystem = InMemoryFileSystem() + let path = AbsolutePath("/LinkedList-1.1.1") + let observability = ObservabilitySystem.makeForTesting() + + // The checksum differs from that in storage, but error is not thrown + // because fingerprintCheckingMode=.warn + try registryClient.downloadSourceArchive( package: identity, version: version, fileSystem: fileSystem, destinationPath: path, - expectedChecksum: expectedChecksum, - checksumAlgorithm: checksumAlgorithm + checksumAlgorithm: checksumAlgorithm, + observabilityScope: observability.topScope ) + // But there should be a warning + testDiagnostics(observability.diagnostics) { result in + result.check(diagnostic: .contains("does not match previously recorded value"), severity: .warning) + } + let contents = try fileSystem.getDirectoryContents(path) XCTAssertEqual(contents, ["Package.swift"]) } - func testDownloadSourceArchiveWithoutExpectedChecksumProvided() throws { + func testDownloadSourceArchive_checksumNotInStorage() throws { let registryURL = "https://packages.example.com" let identity = PackageIdentity.plain("mona.LinkedList") let (scope, name) = identity.scopeAndName! @@ -391,6 +711,7 @@ final class RegistryClientTests: XCTestCase { let metadataURL = URL(string: "\(registryURL)/\(scope)/\(name)/\(version)")! let checksumAlgorithm: HashAlgorithm = SHA256() + let checksum = checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation let handler: HTTPClient.Handler = { request, _, completion in switch (request.kind, request.method, request.url) { @@ -411,7 +732,7 @@ final class RegistryClientTests: XCTestCase { ]), body: nil ))) - // `downloadSourceArchive` calls this API to fetch checksum + // `downloadSourceArchive` calls this API to fetch checksum case (.generic, .get, metadataURL): XCTAssertEqual(request.headers.get("Accept").first, "application/vnd.swift.registry.v1+json") @@ -423,7 +744,7 @@ final class RegistryClientTests: XCTestCase { { "name": "source-archive", "type": "application/zip", - "checksum": "\(checksumAlgorithm.hash(emptyZipFile).hexadecimalRepresentation)" + "checksum": "\(checksum)" } ], "metadata": { @@ -453,9 +774,12 @@ final class RegistryClientTests: XCTestCase { var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: URL(string: registryURL)!) - let registryManager = RegistryClient( + let fingerprintStorage = MockPackageFingerprintStorage() + let registryClient = RegistryClient( configuration: configuration, identityResolver: DefaultIdentityResolver(), + fingerprintStorage: fingerprintStorage, + fingerprintCheckingMode: .strict, customHTTPClient: httpClient, customArchiverProvider: { fileSystem in MockArchiver(handler: { _, from, to, callback in @@ -473,17 +797,23 @@ final class RegistryClientTests: XCTestCase { let fileSystem = InMemoryFileSystem() let path = AbsolutePath("/LinkedList-1.1.1") - try registryManager.downloadSourceArchive( + try registryClient.downloadSourceArchive( package: identity, version: version, fileSystem: fileSystem, destinationPath: path, - expectedChecksum: .none, checksumAlgorithm: checksumAlgorithm ) let contents = try fileSystem.getDirectoryContents(path) XCTAssertEqual(contents, ["Package.swift"]) + + // Expected checksum is not found in storage so the metadata API will be called + let fingerprint = try tsc_await { callback in fingerprintStorage.get(package: identity, version: version, kind: .registry, + observabilityScope: ObservabilitySystem.NOOP, callbackQueue: .sharedConcurrent, + callback: callback) } + XCTAssertEqual(registryURL, fingerprint.origin.url?.absoluteString) + XCTAssertEqual(checksum, fingerprint.value) } func testLookupIdentities() throws { @@ -525,14 +855,8 @@ final class RegistryClientTests: XCTestCase { var configuration = RegistryConfiguration() configuration.defaultRegistry = Registry(url: URL(string: registryURL)!) - let registryManager = RegistryClient( - configuration: configuration, - identityResolver: DefaultIdentityResolver(), - customHTTPClient: httpClient, - customArchiverProvider: { _ in MockArchiver() } - ) - - let identities = try registryManager.lookupIdentities(url: packageURL) + let registryClient = makeRegistryClient(configuration: configuration, httpClient: httpClient) + let identities = try registryClient.lookupIdentities(url: packageURL) XCTAssertEqual([PackageIdentity.plain("mona.LinkedList")], identities) } } @@ -554,7 +878,7 @@ private extension RegistryClient { func getAvailableManifests( package: PackageIdentity, version: Version - ) throws -> [String : ToolsVersion] { + ) throws -> [String: ToolsVersion] { return try tsc_await { self.getAvailableManifests( package: package, @@ -583,12 +907,16 @@ private extension RegistryClient { } } - func fetchSourceArchiveChecksum(package: PackageIdentity, version: Version) throws -> String { + func fetchSourceArchiveChecksum( + package: PackageIdentity, + version: Version, + observabilityScope: ObservabilityScope = ObservabilitySystem.NOOP + ) throws -> String { return try tsc_await { self.fetchSourceArchiveChecksum( package: package, version: version, - observabilityScope: ObservabilitySystem.NOOP, + observabilityScope: observabilityScope, callbackQueue: .sharedConcurrent, completion: $0 ) @@ -600,19 +928,18 @@ private extension RegistryClient { version: Version, fileSystem: FileSystem, destinationPath: AbsolutePath, - expectedChecksum: String?, - checksumAlgorithm: HashAlgorithm - ) throws -> Void { + checksumAlgorithm: HashAlgorithm, + observabilityScope: ObservabilityScope = ObservabilitySystem.NOOP + ) throws { return try tsc_await { self.downloadSourceArchive( package: package, version: version, fileSystem: fileSystem, destinationPath: destinationPath, - expectedChecksum: expectedChecksum, checksumAlgorithm: checksumAlgorithm, progressHandler: .none, - observabilityScope: ObservabilitySystem.NOOP, + observabilityScope: observabilityScope, callbackQueue: .sharedConcurrent, completion: $0 ) @@ -630,3 +957,18 @@ private extension RegistryClient { } } } + +private func makeRegistryClient(configuration: RegistryConfiguration, + httpClient: HTTPClient, + fingerprintStorage: PackageFingerprintStorage = MockPackageFingerprintStorage(), + fingerprintCheckingMode: FingerprintCheckingMode = .strict) -> RegistryClient +{ + RegistryClient( + configuration: configuration, + identityResolver: DefaultIdentityResolver(), + fingerprintStorage: fingerprintStorage, + fingerprintCheckingMode: fingerprintCheckingMode, + customHTTPClient: httpClient, + customArchiverProvider: { _ in MockArchiver() } + ) +} diff --git a/Tests/WorkspaceTests/WorkspaceTests.swift b/Tests/WorkspaceTests/WorkspaceTests.swift index 8f92d05f788..32bc117b1b7 100644 --- a/Tests/WorkspaceTests/WorkspaceTests.swift +++ b/Tests/WorkspaceTests/WorkspaceTests.swift @@ -1,7 +1,7 @@ /* This source file is part of the Swift.org open source project - Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors + Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors Licensed under Apache License v2.0 with Runtime Library Exception See http://swift.org/LICENSE.txt for license information @@ -9,6 +9,7 @@ */ import Basics +import PackageFingerprint import PackageGraph import PackageLoading import PackageModel @@ -3608,6 +3609,10 @@ final class WorkspaceTests: XCTestCase { let sandbox = AbsolutePath("/tmp/ws/") let fs = InMemoryFileSystem() + // Use the same revision (hash) for "foo" to indicate they are the same + // package despite having different URLs. + let fooRevision = String((UUID().uuidString + UUID().uuidString).prefix(40)) + let workspace = try MockWorkspace( sandbox: sandbox, fileSystem: fs, @@ -3674,7 +3679,8 @@ final class WorkspaceTests: XCTestCase { products: [ MockProduct(name: "Foo", targets: ["Foo"]), ], - versions: ["1.0.0"] + versions: ["1.0.0"], + revisionProvider: { _ in fooRevision } ), MockPackage( name: "Foo", @@ -3685,7 +3691,8 @@ final class WorkspaceTests: XCTestCase { products: [ MockProduct(name: "OtherFoo", targets: ["OtherFoo"]), ], - versions: ["1.0.0"] + versions: ["1.0.0"], + revisionProvider: { _ in fooRevision } ), ] ) @@ -9334,6 +9341,8 @@ final class WorkspaceTests: XCTestCase { fileSystem: FileSystem, configuration: PackageRegistry.RegistryConfiguration? = .none, identityResolver: IdentityResolver? = .none, + fingerprintStorage: PackageFingerprintStorage? = .none, + fingerprintCheckingMode: FingerprintCheckingMode = .strict, authorizationProvider: AuthorizationProvider? = .none, releasesRequestHandler: HTTPClient.Handler? = .none, versionMetadataRequestHandler: HTTPClient.Handler? = .none, @@ -9430,10 +9439,13 @@ final class WorkspaceTests: XCTestCase { } let archiver = archiver ?? MockArchiver() + let fingerprintStorage = fingerprintStorage ?? MockPackageFingerprintStorage() return RegistryClient( configuration: configuration!, identityResolver: identityResolver, + fingerprintStorage: fingerprintStorage, + fingerprintCheckingMode: fingerprintCheckingMode, authorizationProvider: authorizationProvider?.httpAuthorizationHeader(for:), customHTTPClient: HTTPClient(configuration: .init(), handler: { request, progress , completion in switch request.url {