From c01aea0cb45f9a8697408201e01eff6119464a87 Mon Sep 17 00:00:00 2001 From: daz Date: Wed, 31 Jul 2024 15:21:13 -0600 Subject: [PATCH 1/6] Introduce cheerio for fast HTML querying --- sources/package-lock.json | 305 ++++++++++++++++++++++++++++++++++++++ sources/package.json | 1 + 2 files changed, 306 insertions(+) diff --git a/sources/package-lock.json b/sources/package-lock.json index ffc416d6..7247c8c5 100644 --- a/sources/package-lock.json +++ b/sources/package-lock.json @@ -20,6 +20,7 @@ "@actions/tool-cache": "2.0.1", "@octokit/rest": "20.1.0", "@octokit/webhooks-types": "7.5.0", + "cheerio": "^1.0.0-rc.12", "semver": "7.6.0", "string-argv": "0.3.2", "typed-rest-client": "1.8.11", @@ -3309,6 +3310,11 @@ "readable-stream": "^3.4.0" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "node_modules/bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", @@ -3526,6 +3532,42 @@ "node": ">=10" } }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "/~https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "/~https://github.com/sponsors/fb55" + } + }, "node_modules/ci-info": { "version": "3.8.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz", @@ -3723,6 +3765,32 @@ "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==", "deprecated": "This package is no longer supported. It's now a built-in Node module. If you've depended on crypto, you should switch to the one that's built-in." }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "/~https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "/~https://github.com/sponsors/fb55" + } + }, "node_modules/damerau-levenshtein": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", @@ -3900,6 +3968,57 @@ "node": ">=6.0.0" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "/~https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "/~https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "/~https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "/~https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/dot-object": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/dot-object/-/dot-object-2.1.4.tgz", @@ -3944,6 +4063,17 @@ "once": "^1.4.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "/~https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -5187,6 +5317,24 @@ "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "/~https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "/~https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, "node_modules/human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", @@ -7060,6 +7208,17 @@ "node": ">=8" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "/~https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/object-inspect": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", @@ -7293,6 +7452,29 @@ "url": "/~https://github.com/sponsors/sindresorhus" } }, + "node_modules/parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dependencies": { + "entities": "^4.4.0" + }, + "funding": { + "url": "/~https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "dependencies": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + }, + "funding": { + "url": "/~https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/pascal-case": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.2.tgz", @@ -11649,6 +11831,11 @@ "readable-stream": "^3.4.0" } }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", @@ -11790,6 +11977,33 @@ "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", "dev": true }, + "cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "requires": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + } + }, + "cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "requires": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + } + }, "ci-info": { "version": "3.8.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz", @@ -11936,6 +12150,23 @@ "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz", "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==" }, + "css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "requires": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + } + }, + "css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==" + }, "damerau-levenshtein": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz", @@ -12062,6 +12293,39 @@ "esutils": "^2.0.2" } }, + "dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "requires": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + } + }, + "domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==" + }, + "domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "requires": { + "domelementtype": "^2.3.0" + } + }, + "domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "requires": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + } + }, "dot-object": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/dot-object/-/dot-object-2.1.4.tgz", @@ -12097,6 +12361,11 @@ "once": "^1.4.0" } }, + "entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==" + }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -13007,6 +13276,17 @@ "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true }, + "htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "requires": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, "human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", @@ -14426,6 +14706,14 @@ "path-key": "^3.0.0" } }, + "nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "requires": { + "boolbase": "^1.0.0" + } + }, "object-inspect": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", @@ -14584,6 +14872,23 @@ "lines-and-columns": "^1.1.6" } }, + "parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "requires": { + "entities": "^4.4.0" + } + }, + "parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "requires": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + } + }, "pascal-case": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.2.tgz", diff --git a/sources/package.json b/sources/package.json index 6f72bd8a..889a65cf 100644 --- a/sources/package.json +++ b/sources/package.json @@ -42,6 +42,7 @@ "@actions/tool-cache": "2.0.1", "@octokit/rest": "20.1.0", "@octokit/webhooks-types": "7.5.0", + "cheerio": "^1.0.0-rc.12", "semver": "7.6.0", "string-argv": "0.3.2", "typed-rest-client": "1.8.11", From 7179909719dd3bd6e6850349476f5d3c3c35a1b5 Mon Sep 17 00:00:00 2001 From: daz Date: Wed, 31 Jul 2024 15:21:41 -0600 Subject: [PATCH 2/6] Verify wrappers for distribution-snapshots By slurping the checksum URLs from https://services.gradle.org/distributions-snapshots/ we can include these unpublished wrapper checksums in validation. Fixes #281 --- sources/src/wrapper-validation/checksums.ts | 30 ++++++++++++++++++- .../jest/wrapper-validation/checksums.test.ts | 16 ++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/sources/src/wrapper-validation/checksums.ts b/sources/src/wrapper-validation/checksums.ts index a1835cb1..6a04af83 100644 --- a/sources/src/wrapper-validation/checksums.ts +++ b/sources/src/wrapper-validation/checksums.ts @@ -1,4 +1,5 @@ import * as httpm from 'typed-rest-client/HttpClient' +import * as cheerio from 'cheerio' import fileWrapperChecksums from './wrapper-checksums.json' @@ -54,7 +55,15 @@ export async function fetchUnknownChecksums( // eslint-disable-next-line @typescript-eslint/no-explicit-any (entry: any) => entry.wrapperChecksumUrl as string ) - const checksums = await Promise.all(checksumUrls.map(async (url: string) => httpGetText(url))) + console.log(`Fetching checksums for ${checksumUrls.length} versions`) + if (allowSnapshots) { + await addDistributionSnapshotChecksums(checksumUrls) + } + console.log(`Fetching checksums for ${checksumUrls.length} versions after snapshot check`) + const checksums = await Promise.all(checksumUrls.map(async (url: string) => { + // console.log(`Fetching checksum from ${url}`) + return httpGetText(url) + })) return new Set(checksums) } @@ -66,3 +75,22 @@ async function httpGetText(url: string): Promise { const response = await httpc.get(url) return await response.readBody() } + +// Public for testing +export async function addDistributionSnapshotChecksums(checksumUrls: string[]): Promise { + // Load the index page of the distribution snapshot repository + const indexPage = await httpGetText('https://services.gradle.org/distributions-snapshots/') + + // // Extract all wrapper checksum from the index page. These end in -wrapper.jar.sha256 + // // Load the HTML into cheerio + const $ = cheerio.load(indexPage); + + // // Find all links ending with '-wrapper.jar.sha256' + const wrapperChecksumLinks = $('a[href$="-wrapper.jar.sha256"]'); + + // build the absolute URL for each wrapper checksum + wrapperChecksumLinks.each((index, element) => { + const url = $(element).attr('href') + checksumUrls.push(`https://services.gradle.org${url}`) + }) +} diff --git a/sources/test/jest/wrapper-validation/checksums.test.ts b/sources/test/jest/wrapper-validation/checksums.test.ts index 91794c59..b9d87c93 100644 --- a/sources/test/jest/wrapper-validation/checksums.test.ts +++ b/sources/test/jest/wrapper-validation/checksums.test.ts @@ -32,6 +32,22 @@ test('fetches wrapper jars checksums', async () => { ).toBe(true) }) +test('fetches wrapper jar checksums for snapshots', async () => { + const nonSnapshotChecksums = await checksums.fetchUnknownChecksums(false, new checksums.WrapperChecksums) + const validChecksums = await checksums.fetchUnknownChecksums(true, new checksums.WrapperChecksums) + + // Expect that at least one snapshot checksum is different from the non-snapshot checksums + expect(validChecksums.size).toBeGreaterThan(nonSnapshotChecksums.size) +}) + +test('fetches all wrapper checksum URLS for snapshots', async () => { + const checksumUrls: string[] = [] + await checksums.addDistributionSnapshotChecksums(checksumUrls) + + expect(checksumUrls.length).toBeGreaterThan(100) // May only be a few unique checksums + console.log(checksumUrls) +}) + describe('retry', () => { afterEach(() => { nock.cleanAll() From b644be617f2fa8573a685d076118202b1c9ae209 Mon Sep 17 00:00:00 2001 From: daz Date: Wed, 31 Jul 2024 20:38:10 -0600 Subject: [PATCH 3/6] Enable wrapper validation by default - Add 'allow-snapshot-wrappers' input parameter - Default 'validate-wrappers' to 'true' Fixes #12 --- .github/actions/init-integ-test/action.yml | 5 +++++ .../integ-test-wrapper-validation.yml | 4 ++-- dependency-submission/action.yml | 15 ++++++++++++++ docs/setup-gradle.md | 20 +++++++++++++++++-- docs/wrapper-validation.md | 3 +++ setup-gradle/action.yml | 9 ++++++++- sources/src/actions/setup-gradle/main.ts | 7 ++----- sources/src/configuration.ts | 10 ++++++++-- sources/src/setup-gradle.ts | 19 +++++++++++++++--- sources/src/wrapper-validation/checksums.ts | 16 +++++++-------- 10 files changed, 85 insertions(+), 23 deletions(-) diff --git a/.github/actions/init-integ-test/action.yml b/.github/actions/init-integ-test/action.yml index d8dc5da4..b1838d92 100644 --- a/.github/actions/init-integ-test/action.yml +++ b/.github/actions/init-integ-test/action.yml @@ -9,6 +9,11 @@ runs: distribution: 'temurin' java-version: 11 + - name: Configure environment + shell: bash + run: | + echo "ALLOWED_GRADLE_WRAPPER_CHECKSUMS=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" >> "$GITHUB_ENV" + # Downloads a 'dist' directory artifact that was uploaded in an earlier 'build-dist' step - name: Download dist if: ${{ env.SKIP_DIST != 'true' && !env.ACT }} diff --git a/.github/workflows/integ-test-wrapper-validation.yml b/.github/workflows/integ-test-wrapper-validation.yml index 199f358e..25d29f5e 100644 --- a/.github/workflows/integ-test-wrapper-validation.yml +++ b/.github/workflows/integ-test-wrapper-validation.yml @@ -29,8 +29,8 @@ jobs: - name: Run wrapper-validation-action id: setup-gradle uses: ./setup-gradle - with: - validate-wrappers: true + env: + ALLOWED_GRADLE_WRAPPER_CHECKSUMS: '' continue-on-error: true - name: Check failure diff --git a/dependency-submission/action.yml b/dependency-submission/action.yml index f80489db..32b178c4 100644 --- a/dependency-submission/action.yml +++ b/dependency-submission/action.yml @@ -172,6 +172,21 @@ inputs: description: The Develocity short-lived access tokens expiry in hours. Default is 2 hours. required: false + # Wrapper validation configuration + validate-wrappers: + description: | + When 'true' the action will automatically validate all wrapper jars found in the repository. + If the wrapper checksums are not valid, the action will fail. + required: false + default: false + + allow-snapshot-wrappers: + description: | + When 'true', wrapper validation will include the checksums of snapshot wrapper jars. + Use this if you are running with nightly or snapshot versions of the Gradle wrapper. + required: false + default: false + # DEPRECATED ACTION INPUTS # EXPERIMENTAL ACTION INPUTS diff --git a/docs/setup-gradle.md b/docs/setup-gradle.md index 98d3ba67..b705ae11 100644 --- a/docs/setup-gradle.md +++ b/docs/setup-gradle.md @@ -515,14 +515,30 @@ Since Gradle applies init scripts in alphabetical order, one way to ensure this ## Gradle Wrapper validation -Instead of using the [wrapper-validation action](./wrapper-validation.md) separately, you can enable -wrapper validation directly in your Setup Gradle step. +By default, this action will perform the same wrapper validation as is performed by the dedicated +[wrapper-validation action](./wrapper-validation.md). +This means that invalid wrapper jars will be automatically detected when using `setup-gradle`. + +If you do not want wrapper-validation to occur automatically, you can disable it: + +```yaml + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + with: + validate-wrappers: false +``` + +If your repository uses snapshot versions of the Gradle wrapper, such as nightly builds, then you'll need to +explicitly allow snapshot wrappers in wrapper validation. +These are not allowed by default. + ```yaml - name: Setup Gradle uses: gradle/actions/setup-gradle@v3 with: validate-wrappers: true + allow-snapshot-wrappers: true ``` If you need more advanced configuration, then you're advised to continue using a separate workflow step diff --git a/docs/wrapper-validation.md b/docs/wrapper-validation.md index 21e6dfce..66f42a27 100644 --- a/docs/wrapper-validation.md +++ b/docs/wrapper-validation.md @@ -4,6 +4,9 @@ This action validates the checksums of _all_ [Gradle Wrapper](https://docs.gradl The action should be run in the root of the repository, as it will recursively search for any files named `gradle-wrapper.jar`. +The `setup-gradle` action will perform wrapper validation on each execution. If you are using `setup-gradle` in your +workflows, it is unlikely that you will need to use this action. + ## The Gradle Wrapper Problem in Open Source The `gradle-wrapper.jar` is a binary blob of executable code that is checked into nearly diff --git a/setup-gradle/action.yml b/setup-gradle/action.yml index 87943970..134eca74 100644 --- a/setup-gradle/action.yml +++ b/setup-gradle/action.yml @@ -190,9 +190,16 @@ inputs: # Wrapper validation configuration validate-wrappers: description: | - When 'true', the action will perform the 'wrapper-validation' action automatically. + When 'true' (the default) the action will automatically validate all wrapper jars found in the repository. If the wrapper checksums are not valid, the action will fail. required: false + default: true + + allow-snapshot-wrappers: + description: | + When 'true', wrapper validation will include the checksums of snapshot wrapper jars. + Use this if you are running with nightly or snapshot versions of the Gradle wrapper. + required: false default: false # DEPRECATED ACTION INPUTS diff --git a/sources/src/actions/setup-gradle/main.ts b/sources/src/actions/setup-gradle/main.ts index 75289a7b..b7a20f04 100644 --- a/sources/src/actions/setup-gradle/main.ts +++ b/sources/src/actions/setup-gradle/main.ts @@ -6,7 +6,7 @@ import { CacheConfig, DependencyGraphConfig, GradleExecutionConfig, - doValidateWrappers, + WrapperValidationConfig, getActionId, setActionId } from '../../configuration' @@ -26,10 +26,7 @@ export async function run(): Promise { setActionId('gradle/actions/setup-gradle') - // Check for invalid wrapper JARs if requested - if (doValidateWrappers()) { - await setupGradle.checkNoInvalidWrapperJars() - } + await setupGradle.validateWrappers(new WrapperValidationConfig()) // Configure Gradle environment (Gradle User Home) await setupGradle.setup(new CacheConfig(), new BuildScanConfig()) diff --git a/sources/src/configuration.ts b/sources/src/configuration.ts index 6f30dfd2..a354a2e4 100644 --- a/sources/src/configuration.ts +++ b/sources/src/configuration.ts @@ -357,8 +357,14 @@ export class GradleExecutionConfig { } } -export function doValidateWrappers(): boolean { - return getBooleanInput('validate-wrappers') +export class WrapperValidationConfig { + doValidateWrappers(): boolean { + return getBooleanInput('validate-wrappers') + } + + allowSnapshotWrappers(): boolean { + return getBooleanInput('allow-snapshot-wrappers') + } } // Internal parameters diff --git a/sources/src/setup-gradle.ts b/sources/src/setup-gradle.ts index 8b788e5d..792151ee 100644 --- a/sources/src/setup-gradle.ts +++ b/sources/src/setup-gradle.ts @@ -10,7 +10,13 @@ import * as buildScan from './develocity/build-scan' import {loadBuildResults, markBuildResultsProcessed} from './build-results' import {CacheListener, generateCachingReport} from './caching/cache-reporting' import {DaemonController} from './daemon-controller' -import {BuildScanConfig, CacheConfig, SummaryConfig, getWorkspaceDirectory} from './configuration' +import { + BuildScanConfig, + CacheConfig, + SummaryConfig, + WrapperValidationConfig, + getWorkspaceDirectory +} from './configuration' import {findInvalidWrapperJars} from './wrapper-validation/validate' import {JobFailure} from './errors' @@ -117,9 +123,16 @@ async function determineUserHome(): Promise { return userHome } -export async function checkNoInvalidWrapperJars(rootDir = getWorkspaceDirectory()): Promise { +export async function validateWrappers( + config: WrapperValidationConfig, + rootDir = getWorkspaceDirectory() +): Promise { + if (!config.doValidateWrappers()) { + return // Wrapper validation is disabled + } + const allowedChecksums = process.env['ALLOWED_GRADLE_WRAPPER_CHECKSUMS']?.split(',') || [] - const result = await findInvalidWrapperJars(rootDir, 1, false, allowedChecksums) + const result = await findInvalidWrapperJars(rootDir, 0, config.allowSnapshotWrappers(), allowedChecksums) if (result.isValid()) { core.info(result.toDisplayString()) } else { diff --git a/sources/src/wrapper-validation/checksums.ts b/sources/src/wrapper-validation/checksums.ts index 6a04af83..27485085 100644 --- a/sources/src/wrapper-validation/checksums.ts +++ b/sources/src/wrapper-validation/checksums.ts @@ -55,15 +55,15 @@ export async function fetchUnknownChecksums( // eslint-disable-next-line @typescript-eslint/no-explicit-any (entry: any) => entry.wrapperChecksumUrl as string ) - console.log(`Fetching checksums for ${checksumUrls.length} versions`) if (allowSnapshots) { await addDistributionSnapshotChecksums(checksumUrls) } - console.log(`Fetching checksums for ${checksumUrls.length} versions after snapshot check`) - const checksums = await Promise.all(checksumUrls.map(async (url: string) => { - // console.log(`Fetching checksum from ${url}`) - return httpGetText(url) - })) + const checksums = await Promise.all( + checksumUrls.map(async (url: string) => { + // console.log(`Fetching checksum from ${url}`) + return httpGetText(url) + }) + ) return new Set(checksums) } @@ -83,10 +83,10 @@ export async function addDistributionSnapshotChecksums(checksumUrls: string[]): // // Extract all wrapper checksum from the index page. These end in -wrapper.jar.sha256 // // Load the HTML into cheerio - const $ = cheerio.load(indexPage); + const $ = cheerio.load(indexPage) // // Find all links ending with '-wrapper.jar.sha256' - const wrapperChecksumLinks = $('a[href$="-wrapper.jar.sha256"]'); + const wrapperChecksumLinks = $('a[href$="-wrapper.jar.sha256"]') // build the absolute URL for each wrapper checksum wrapperChecksumLinks.each((index, element) => { From ce4c3a6c5e64220081ef4d05b6a4547fb0d4987b Mon Sep 17 00:00:00 2001 From: daz Date: Thu, 1 Aug 2024 08:53:37 -0600 Subject: [PATCH 4/6] Move wrapper-validation into common setup code --- .../src/actions/dependency-submission/main.ts | 5 +-- sources/src/actions/setup-gradle/main.ts | 4 +-- sources/src/setup-gradle.ts | 31 +++++-------------- .../wrapper-validation/wrapper-validator.ts | 23 ++++++++++++++ 4 files changed, 35 insertions(+), 28 deletions(-) create mode 100644 sources/src/wrapper-validation/wrapper-validator.ts diff --git a/sources/src/actions/dependency-submission/main.ts b/sources/src/actions/dependency-submission/main.ts index 72221847..8ff3fb1b 100644 --- a/sources/src/actions/dependency-submission/main.ts +++ b/sources/src/actions/dependency-submission/main.ts @@ -10,7 +10,8 @@ import { DependencyGraphConfig, DependencyGraphOption, GradleExecutionConfig, - setActionId + setActionId, + WrapperValidationConfig } from '../../configuration' import {saveDeprecationState} from '../../deprecation-collector' import {handleMainActionError} from '../../errors' @@ -23,7 +24,7 @@ export async function run(): Promise { setActionId('gradle/actions/dependency-submission') // Configure Gradle environment (Gradle User Home) - await setupGradle.setup(new CacheConfig(), new BuildScanConfig()) + await setupGradle.setup(new CacheConfig(), new BuildScanConfig(), new WrapperValidationConfig()) // Capture the enabled state of dependency-graph const originallyEnabled = process.env['GITHUB_DEPENDENCY_GRAPH_ENABLED'] diff --git a/sources/src/actions/setup-gradle/main.ts b/sources/src/actions/setup-gradle/main.ts index b7a20f04..65b05b82 100644 --- a/sources/src/actions/setup-gradle/main.ts +++ b/sources/src/actions/setup-gradle/main.ts @@ -26,10 +26,8 @@ export async function run(): Promise { setActionId('gradle/actions/setup-gradle') - await setupGradle.validateWrappers(new WrapperValidationConfig()) - // Configure Gradle environment (Gradle User Home) - await setupGradle.setup(new CacheConfig(), new BuildScanConfig()) + await setupGradle.setup(new CacheConfig(), new BuildScanConfig(), new WrapperValidationConfig()) // Configure the dependency graph submission await dependencyGraph.setup(new DependencyGraphConfig()) diff --git a/sources/src/setup-gradle.ts b/sources/src/setup-gradle.ts index 792151ee..023d6442 100644 --- a/sources/src/setup-gradle.ts +++ b/sources/src/setup-gradle.ts @@ -17,15 +17,18 @@ import { WrapperValidationConfig, getWorkspaceDirectory } from './configuration' -import {findInvalidWrapperJars} from './wrapper-validation/validate' -import {JobFailure} from './errors' +import * as wrapperValidator from './wrapper-validation/wrapper-validator' const GRADLE_SETUP_VAR = 'GRADLE_BUILD_ACTION_SETUP_COMPLETED' const USER_HOME = 'USER_HOME' const GRADLE_USER_HOME = 'GRADLE_USER_HOME' const CACHE_LISTENER = 'CACHE_LISTENER' -export async function setup(cacheConfig: CacheConfig, buildScanConfig: BuildScanConfig): Promise { +export async function setup( + cacheConfig: CacheConfig, + buildScanConfig: BuildScanConfig, + wrapperValidationConfig: WrapperValidationConfig +): Promise { const userHome = await determineUserHome() const gradleUserHome = await determineGradleUserHome() @@ -43,6 +46,8 @@ export async function setup(cacheConfig: CacheConfig, buildScanConfig: BuildScan core.saveState(USER_HOME, userHome) core.saveState(GRADLE_USER_HOME, gradleUserHome) + await wrapperValidator.validateWrappers(wrapperValidationConfig, getWorkspaceDirectory()) + const cacheListener = new CacheListener() await caches.restore(userHome, gradleUserHome, cacheListener, cacheConfig) @@ -122,23 +127,3 @@ async function determineUserHome(): Promise { core.debug(`Determined user.home from java -version output: '${userHome}'`) return userHome } - -export async function validateWrappers( - config: WrapperValidationConfig, - rootDir = getWorkspaceDirectory() -): Promise { - if (!config.doValidateWrappers()) { - return // Wrapper validation is disabled - } - - const allowedChecksums = process.env['ALLOWED_GRADLE_WRAPPER_CHECKSUMS']?.split(',') || [] - const result = await findInvalidWrapperJars(rootDir, 0, config.allowSnapshotWrappers(), allowedChecksums) - if (result.isValid()) { - core.info(result.toDisplayString()) - } else { - core.info(result.toDisplayString()) - throw new JobFailure( - `Gradle Wrapper Validation Failed!\n See /~https://github.com/gradle/actions/blob/main/docs/wrapper-validation.md#reporting-failures\n${result.toDisplayString()}` - ) - } -} diff --git a/sources/src/wrapper-validation/wrapper-validator.ts b/sources/src/wrapper-validation/wrapper-validator.ts new file mode 100644 index 00000000..440f7a19 --- /dev/null +++ b/sources/src/wrapper-validation/wrapper-validator.ts @@ -0,0 +1,23 @@ +import * as core from '@actions/core' +import {WrapperValidationConfig} from '../configuration' +import {findInvalidWrapperJars} from './validate' +import {JobFailure} from '../errors' + +export async function validateWrappers(config: WrapperValidationConfig, workspaceRoot: string): Promise { + if (!config.doValidateWrappers()) { + return // Wrapper validation is disabled + } + + const allowedChecksums = process.env['ALLOWED_GRADLE_WRAPPER_CHECKSUMS']?.split(',') || [] + const result = await findInvalidWrapperJars(workspaceRoot, 0, config.allowSnapshotWrappers(), allowedChecksums) + if (result.isValid()) { + await core.group('All Gradle Wrapper jars are valid', async () => { + core.info(result.toDisplayString()) + }) + } else { + core.info(result.toDisplayString()) + throw new JobFailure( + `Gradle Wrapper Validation Failed!\n See /~https://github.com/gradle/actions/blob/main/docs/wrapper-validation.md#reporting-failures\n${result.toDisplayString()}` + ) + } +} From b6395da67c9887f995671dbf1729f9f9a5c413db Mon Sep 17 00:00:00 2001 From: daz Date: Thu, 1 Aug 2024 09:39:30 -0600 Subject: [PATCH 5/6] Cache validated checksums for later executions The most common case for validation will be that the wrapper jars are unchanged from a previous workflow run. In this case, we cache the validated wrapper checksums to minimise the work required on a subsequent run. Fixes #172 --- .../caching/gradle-home-extry-extractor.ts | 5 +-- sources/src/caching/gradle-user-home-cache.ts | 8 ++-- sources/src/configuration.ts | 2 + sources/src/setup-gradle.ts | 4 +- sources/src/wrapper-validation/cache.ts | 26 +++++++++++++ sources/src/wrapper-validation/validate.ts | 7 +++- .../wrapper-validation/wrapper-validator.ts | 22 ++++++++++- .../jest/wrapper-validation/validate.test.ts | 37 +++++++++++++++++++ 8 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 sources/src/wrapper-validation/cache.ts diff --git a/sources/src/caching/gradle-home-extry-extractor.ts b/sources/src/caching/gradle-home-extry-extractor.ts index 2f4ecee4..35a1f482 100644 --- a/sources/src/caching/gradle-home-extry-extractor.ts +++ b/sources/src/caching/gradle-home-extry-extractor.ts @@ -4,12 +4,11 @@ import * as core from '@actions/core' import * as glob from '@actions/glob' import * as semver from 'semver' -import {META_FILE_DIR} from './gradle-user-home-cache' import {CacheEntryListener, CacheListener} from './cache-reporting' import {cacheDebug, hashFileNames, isCacheDebuggingEnabled, restoreCache, saveCache, tryDelete} from './cache-utils' import {BuildResult, loadBuildResults} from '../build-results' -import {CacheConfig} from '../configuration' +import {CacheConfig, ACTION_METADATA_DIR} from '../configuration' import {getCacheKeyBase} from './cache-key' const SKIP_RESTORE_VAR = 'GRADLE_BUILD_ACTION_SKIP_RESTORE' @@ -298,7 +297,7 @@ abstract class AbstractEntryExtractor { } private getCacheMetadataFile(): string { - const actionMetadataDirectory = path.resolve(this.gradleUserHome, META_FILE_DIR) + const actionMetadataDirectory = path.resolve(this.gradleUserHome, ACTION_METADATA_DIR) fs.mkdirSync(actionMetadataDirectory, {recursive: true}) return path.resolve(actionMetadataDirectory, `${this.extractorName}-entry-metadata.json`) diff --git a/sources/src/caching/gradle-user-home-cache.ts b/sources/src/caching/gradle-user-home-cache.ts index 04f4e08b..5d86d84f 100644 --- a/sources/src/caching/gradle-user-home-cache.ts +++ b/sources/src/caching/gradle-user-home-cache.ts @@ -7,14 +7,12 @@ import fs from 'fs' import {generateCacheKey} from './cache-key' import {CacheListener} from './cache-reporting' import {saveCache, restoreCache, cacheDebug, isCacheDebuggingEnabled, tryDelete} from './cache-utils' -import {CacheConfig} from '../configuration' +import {CacheConfig, ACTION_METADATA_DIR} from '../configuration' import {GradleHomeEntryExtractor, ConfigurationCacheEntryExtractor} from './gradle-home-extry-extractor' import {getPredefinedToolchains, mergeToolchainContent, readResourceFileAsString} from './gradle-user-home-utils' const RESTORED_CACHE_KEY_KEY = 'restored-cache-key' -export const META_FILE_DIR = '.setup-gradle' - export class GradleUserHomeCache { private readonly cacheName = 'home' private readonly cacheDescription = 'Gradle User Home' @@ -172,7 +170,7 @@ export class GradleUserHomeCache { */ protected getCachePath(): string[] { const rawPaths: string[] = this.cacheConfig.getCacheIncludes() - rawPaths.push(META_FILE_DIR) + rawPaths.push(ACTION_METADATA_DIR) const resolvedPaths = rawPaths.map(x => this.resolveCachePath(x)) cacheDebug(`Using cache paths: ${resolvedPaths}`) return resolvedPaths @@ -188,7 +186,7 @@ export class GradleUserHomeCache { private initializeGradleUserHome(): void { // Create a directory for storing action metadata - const actionCacheDir = path.resolve(this.gradleUserHome, META_FILE_DIR) + const actionCacheDir = path.resolve(this.gradleUserHome, ACTION_METADATA_DIR) fs.mkdirSync(actionCacheDir, {recursive: true}) this.copyInitScripts() diff --git a/sources/src/configuration.ts b/sources/src/configuration.ts index a354a2e4..9bbefe37 100644 --- a/sources/src/configuration.ts +++ b/sources/src/configuration.ts @@ -8,6 +8,8 @@ import path from 'path' const ACTION_ID_VAR = 'GRADLE_ACTION_ID' +export const ACTION_METADATA_DIR = '.setup-gradle' + export class DependencyGraphConfig { getDependencyGraphOption(): DependencyGraphOption { const val = core.getInput('dependency-graph') diff --git a/sources/src/setup-gradle.ts b/sources/src/setup-gradle.ts index 023d6442..8f8353f2 100644 --- a/sources/src/setup-gradle.ts +++ b/sources/src/setup-gradle.ts @@ -46,13 +46,13 @@ export async function setup( core.saveState(USER_HOME, userHome) core.saveState(GRADLE_USER_HOME, gradleUserHome) - await wrapperValidator.validateWrappers(wrapperValidationConfig, getWorkspaceDirectory()) - const cacheListener = new CacheListener() await caches.restore(userHome, gradleUserHome, cacheListener, cacheConfig) core.saveState(CACHE_LISTENER, cacheListener.stringify()) + await wrapperValidator.validateWrappers(wrapperValidationConfig, getWorkspaceDirectory(), gradleUserHome) + await buildScan.setup(buildScanConfig) return true diff --git a/sources/src/wrapper-validation/cache.ts b/sources/src/wrapper-validation/cache.ts new file mode 100644 index 00000000..90aba54d --- /dev/null +++ b/sources/src/wrapper-validation/cache.ts @@ -0,0 +1,26 @@ +import fs from 'fs' +import path from 'path' +import {ACTION_METADATA_DIR} from '../configuration' + +export class ChecksumCache { + private readonly cacheFile: string + + constructor(gradleUserHome: string) { + this.cacheFile = path.resolve(gradleUserHome, ACTION_METADATA_DIR, 'valid-wrappers.json') + } + + load(): string[] { + // Load previously validated checksums saved in Gradle User Home + if (fs.existsSync(this.cacheFile)) { + return JSON.parse(fs.readFileSync(this.cacheFile, 'utf-8')) + } + return [] + } + + save(checksums: string[]): void { + const uniqueChecksums = [...new Set(checksums)] + // Save validated checksums to Gradle User Home + fs.mkdirSync(path.dirname(this.cacheFile), {recursive: true}) + fs.writeFileSync(this.cacheFile, JSON.stringify(uniqueChecksums)) + } +} diff --git a/sources/src/wrapper-validation/validate.ts b/sources/src/wrapper-validation/validate.ts index 13be7f65..e8314620 100644 --- a/sources/src/wrapper-validation/validate.ts +++ b/sources/src/wrapper-validation/validate.ts @@ -8,6 +8,7 @@ export async function findInvalidWrapperJars( minWrapperCount: number, allowSnapshots: boolean, allowedChecksums: string[], + previouslyValidatedChecksums: string[] = [], knownValidChecksums: checksums.WrapperChecksums = checksums.KNOWN_CHECKSUMS ): Promise { const wrapperJars = await find.findWrapperJars(gitRepoRoot) @@ -21,7 +22,11 @@ export async function findInvalidWrapperJars( const notYetValidatedWrappers = [] for (const wrapperJar of wrapperJars) { const sha = await hash.sha256File(resolve(gitRepoRoot, wrapperJar)) - if (allowedChecksums.includes(sha) || knownValidChecksums.checksums.has(sha)) { + if ( + allowedChecksums.includes(sha) || + previouslyValidatedChecksums.includes(sha) || + knownValidChecksums.checksums.has(sha) + ) { result.valid.push(new WrapperJar(wrapperJar, sha)) } else { notYetValidatedWrappers.push(new WrapperJar(wrapperJar, sha)) diff --git a/sources/src/wrapper-validation/wrapper-validator.ts b/sources/src/wrapper-validation/wrapper-validator.ts index 440f7a19..3bc4992a 100644 --- a/sources/src/wrapper-validation/wrapper-validator.ts +++ b/sources/src/wrapper-validation/wrapper-validator.ts @@ -1,17 +1,33 @@ import * as core from '@actions/core' + import {WrapperValidationConfig} from '../configuration' +import {ChecksumCache} from './cache' import {findInvalidWrapperJars} from './validate' import {JobFailure} from '../errors' -export async function validateWrappers(config: WrapperValidationConfig, workspaceRoot: string): Promise { +export async function validateWrappers( + config: WrapperValidationConfig, + workspaceRoot: string, + gradleUserHome: string +): Promise { if (!config.doValidateWrappers()) { return // Wrapper validation is disabled } + const checksumCache = new ChecksumCache(gradleUserHome) const allowedChecksums = process.env['ALLOWED_GRADLE_WRAPPER_CHECKSUMS']?.split(',') || [] - const result = await findInvalidWrapperJars(workspaceRoot, 0, config.allowSnapshotWrappers(), allowedChecksums) + const previouslyValidatedChecksums = checksumCache.load() + + const result = await findInvalidWrapperJars( + workspaceRoot, + 0, + config.allowSnapshotWrappers(), + allowedChecksums, + previouslyValidatedChecksums + ) if (result.isValid()) { await core.group('All Gradle Wrapper jars are valid', async () => { + core.info(`Loaded previously validated checksums from cache: ${previouslyValidatedChecksums.join(', ')}`) core.info(result.toDisplayString()) }) } else { @@ -20,4 +36,6 @@ export async function validateWrappers(config: WrapperValidationConfig, workspac `Gradle Wrapper Validation Failed!\n See /~https://github.com/gradle/actions/blob/main/docs/wrapper-validation.md#reporting-failures\n${result.toDisplayString()}` ) } + + checksumCache.save(result.valid.map(wrapper => wrapper.checksum)) } diff --git a/sources/test/jest/wrapper-validation/validate.test.ts b/sources/test/jest/wrapper-validation/validate.test.ts index 42c0d87b..c4b86ce5 100644 --- a/sources/test/jest/wrapper-validation/validate.test.ts +++ b/sources/test/jest/wrapper-validation/validate.test.ts @@ -1,11 +1,15 @@ import * as path from 'path' +import * as fs from 'fs' import * as validate from '../../../src/wrapper-validation/validate' import {expect, test, jest} from '@jest/globals' import { WrapperChecksums } from '../../../src/wrapper-validation/checksums' +import { ChecksumCache } from '../../../src/wrapper-validation/cache' +import exp from 'constants' jest.setTimeout(30000) const baseDir = path.resolve('./test/jest/wrapper-validation') +const tmpDir = path.resolve('./test/jest/tmp') test('succeeds if all found wrapper jars are valid', async () => { const result = await validate.findInvalidWrapperJars(baseDir, 3, false, [ @@ -24,6 +28,24 @@ test('succeeds if all found wrapper jars are valid', async () => { ) }) +test('succeeds if all found wrapper jars are previously valid', async () => { + const result = await validate.findInvalidWrapperJars(baseDir, 3, false, [], [ + 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', + '3888c76faa032ea8394b8a54e04ce2227ab1f4be64f65d450f8509fe112d38ce' + ]) + + expect(result.isValid()).toBe(true) + // Only hardcoded and explicitly allowed checksums should have been used + expect(result.fetchedChecksums).toBe(false) + + expect(result.toDisplayString()).toBe( + '✓ Found known Gradle Wrapper JAR files:\n' + + ' e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 data/invalid/gradle-wrapper.jar\n' + + ' e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 data/invalid/gradlе-wrapper.jar\n' + // homoglyph + ' 3888c76faa032ea8394b8a54e04ce2227ab1f4be64f65d450f8509fe112d38ce data/valid/gradle-wrapper.jar' + ) +}) + test('succeeds if all found wrapper jars are valid (and checksums are fetched from Gradle API)', async () => { const knownValidChecksums = new WrapperChecksums() const result = await validate.findInvalidWrapperJars( @@ -31,6 +53,7 @@ test('succeeds if all found wrapper jars are valid (and checksums are fetched fr 1, false, ['e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'], + [], knownValidChecksums ) console.log(`fetchedChecksums = ${result.fetchedChecksums}`) @@ -98,3 +121,17 @@ test('fails if not enough wrapper jars are found', async () => { ' 3888c76faa032ea8394b8a54e04ce2227ab1f4be64f65d450f8509fe112d38ce data/valid/gradle-wrapper.jar' ) }) + +test('can save and load checksums', async () => { + const cacheDir = path.join(tmpDir, 'wrapper-validation-cache') + fs.rmSync(cacheDir, {recursive: true, force: true}) + + const checksumCache = new ChecksumCache(cacheDir) + + expect(checksumCache.load()).toEqual([]) + + checksumCache.save(['123', '456']) + + expect(checksumCache.load()).toEqual(['123', '456']) + expect(fs.existsSync(cacheDir)).toBe(true) +}) From 73f1290de76639a6128e0d60940038f3cb553077 Mon Sep 17 00:00:00 2001 From: daz Date: Thu, 1 Aug 2024 08:52:56 -0600 Subject: [PATCH 6/6] Improve docs linked for wrapper-validation failure --- docs/wrapper-validation.md | 20 +++++++++++-------- .../wrapper-validation/wrapper-validator.ts | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/wrapper-validation.md b/docs/wrapper-validation.md index 66f42a27..9fc4a6b8 100644 --- a/docs/wrapper-validation.md +++ b/docs/wrapper-validation.md @@ -93,18 +93,22 @@ We recommend the message commit contents of: From there, you can easily follow the rest of the prompts to create a Pull Request against the project. -## Reporting Failures +## Validation Failures -If this GitHub action fails because a `gradle-wrapper.jar` doesn't match one of our published SHA-256 checksums, -we highly recommend that you reach out to us at [security@gradle.com](mailto:security@gradle.com). +A wrapper jar can fail validation for a few reasons: +1. The wrapper is from a snapshot build of Gradle (nightly or release nightly) and you have not set `allow-snapshots` + or `allow-snapshot-wrappers` to `true`. +2. The wrapper jar is from a version of Gradle with an unverifiable wrapper jar (see below). +3. The wrapper jar was not published by Gradle, and could be compromised. -**Note:** `gradle-wrapper.jar` generated by Gradle 3.3 to 4.0 are not verifiable because those files were dynamically generated by Gradle in a non-reproducible way. It's not possible to verify the `gradle-wrapper.jar` for those versions are legitimate using a hash comparison. You should try to determine if the `gradle-wrapper.jar` was generated by one of these versions before running the build. +If this GitHub action fails because a `gradle-wrapper.jar` was not published by Gradle, +we highly recommend that you reach out to us at [security@gradle.com](mailto:security@gradle.com). -If the Gradle version in `gradle-wrapper.properties` is out of this range, you may need to regenerate the `gradle-wrapper.jar` by running `./gradlew wrapper`. If you need to use a version of Gradle between 3.3 and 4.0, you can use a newer version of Gradle to generate the `gradle-wrapper.jar`. +#### Unverifiable Wrapper Jars +Wrapper Jars generated by Gradle versions `3.3` to `4.0` are not verifiable because those files were dynamically generated by Gradle in a non-reproducible way. It's not possible to verify the `gradle-wrapper.jar` for those versions are legitimate using a hash comparison. If you have a validation failure, you should try to determine if the `gradle-wrapper.jar` was generated by one of these versions before running the build. -If you're curious and want to explore what the differences are between the `gradle-wrapper.jar` in your possession -and one of our valid release, you can compare them using this online utility: [diffoscope](https://try.diffoscope.org/). -Regardless of what you find, we still kindly request that you reach out to us and let us know. +- If the Gradle version in `gradle-wrapper.properties` is outside of this range, you can regenerate the `gradle-wrapper.jar` by running `./gradlew wrapper`. This will generate a new, verifiable wrapper jar. +- If you need to run your build with a version of Gradle between 3.3 and 4.0, you can use a newer version of Gradle to generate the `gradle-wrapper.jar`. ## Resources diff --git a/sources/src/wrapper-validation/wrapper-validator.ts b/sources/src/wrapper-validation/wrapper-validator.ts index 3bc4992a..8958fab3 100644 --- a/sources/src/wrapper-validation/wrapper-validator.ts +++ b/sources/src/wrapper-validation/wrapper-validator.ts @@ -33,7 +33,7 @@ export async function validateWrappers( } else { core.info(result.toDisplayString()) throw new JobFailure( - `Gradle Wrapper Validation Failed!\n See /~https://github.com/gradle/actions/blob/main/docs/wrapper-validation.md#reporting-failures\n${result.toDisplayString()}` + `Gradle Wrapper Validation Failed!\n See /~https://github.com/gradle/actions/blob/main/docs/wrapper-validation.md#validation-failures\n${result.toDisplayString()}` ) }