diff --git a/docs/usage/self-hosted-experimental.md b/docs/usage/self-hosted-experimental.md index bae03e095f81b4..04124a9ba7add2 100644 --- a/docs/usage/self-hosted-experimental.md +++ b/docs/usage/self-hosted-experimental.md @@ -141,6 +141,10 @@ This feature is in private beta. If set, Renovate will query the merge-confidence JSON API only for datasources that are part of this list. The expected value for this environment variable is a JSON array of strings. +## `RENOVATE_X_NUGET_DOWNLOAD_NUPKGS` + +If set to any value, Renovate will download `nupkg` files for determining package metadata. + ## `RENOVATE_X_PLATFORM_VERSION` Specify this string for Renovate to skip API checks and provide GitLab/Bitbucket server version directly. diff --git a/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3-no-repo.nupkg b/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3-no-repo.nupkg new file mode 100644 index 00000000000000..77ce3d810f1ade Binary files /dev/null and b/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3-no-repo.nupkg differ diff --git a/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3.nupkg b/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3.nupkg new file mode 100644 index 00000000000000..6ec85f1ec04651 Binary files /dev/null and b/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3.nupkg differ diff --git a/lib/modules/datasource/nuget/common.ts b/lib/modules/datasource/nuget/common.ts index e3c9afb28fd2d6..14bb38d6942111 100644 --- a/lib/modules/datasource/nuget/common.ts +++ b/lib/modules/datasource/nuget/common.ts @@ -12,10 +12,14 @@ export function removeBuildMeta(version: string): string { const urlWhitespaceRe = regEx(/\s/g); -export function massageUrl(url: string): string { +export function massageUrl(url: string | null | undefined): string | null { + if (url === null || url === undefined) { + return null; + } + let resultUrl = url; - // During `dotnet pack` certain URLs are being URL decoded which may introduce whitespaces + // During `dotnet pack` certain URLs are being URL decoded which may introduce whitespace // and causes Markdown link generation problems. resultUrl = resultUrl.replace(urlWhitespaceRe, '%20'); diff --git a/lib/modules/datasource/nuget/index.spec.ts b/lib/modules/datasource/nuget/index.spec.ts index db7bbaa7b52d61..ce25cf7722b9c0 100644 --- a/lib/modules/datasource/nuget/index.spec.ts +++ b/lib/modules/datasource/nuget/index.spec.ts @@ -1,8 +1,12 @@ +import { Readable } from 'stream'; import { mockDeep } from 'jest-mock-extended'; +import { join } from 'upath'; import { getPkgReleases } from '..'; import { Fixtures } from '../../../../test/fixtures'; import * as httpMock from '../../../../test/http-mock'; -import { logger } from '../../../../test/util'; +import { logger, mocked } from '../../../../test/util'; +import { GlobalConfig } from '../../../config/global'; +import * as _packageCache from '../../../util/cache/package'; import * as _hostRules from '../../../util/host-rules'; import { id as versioning } from '../../versioning/nuget'; import { parseRegistryUrl } from './common'; @@ -14,6 +18,9 @@ const hostRules: any = _hostRules; jest.mock('../../../util/host-rules', () => mockDeep()); +jest.mock('../../../util/cache/package', () => mockDeep()); +const packageCache = mocked(_packageCache); + const pkgInfoV3FromNuget = Fixtures.get('nunit/v3_nuget_org.xml'); const pkgListV3Registration = Fixtures.get('nunit/v3_registration.json'); @@ -105,6 +112,10 @@ const configV3AzureDevOps = { }; describe('modules/datasource/nuget/index', () => { + beforeEach(() => { + GlobalConfig.reset(); + }); + describe('parseRegistryUrl', () => { it('extracts feed version from registry URL hash (v3)', () => { const parsed = parseRegistryUrl('https://my-registry#protocolVersion=3'); @@ -302,6 +313,160 @@ describe('modules/datasource/nuget/index', () => { ); }); + describe('determine source URL from nupkg', () => { + beforeEach(() => { + GlobalConfig.set({ + cacheDir: join('/tmp/cache'), + }); + process.env.RENOVATE_X_NUGET_DOWNLOAD_NUPKGS = 'true'; + }); + + afterEach(() => { + delete process.env.RENOVATE_X_NUGET_DOWNLOAD_NUPKGS; + }); + + it('can determine source URL from nupkg when PackageBaseAddress is missing', async () => { + const nugetIndex = ` + { + "version": "3.0.0", + "resources": [ + { + "@id": "https://some-registry/v3/metadata", + "@type": "RegistrationsBaseUrl/3.0.0-beta", + "comment": "Get package metadata." + } + ] + } + `; + const nlogRegistration = ` + { + "count": 1, + "items": [ + { + "@id": "https://some-registry/v3/metadata/nlog/4.7.3.json", + "lower": "4.7.3", + "upper": "4.7.3", + "count": 1, + "items": [ + { + "@id": "foo", + "catalogEntry": { + "id": "NLog", + "version": "4.7.3", + "packageContent": "https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg" + } + } + ] + } + ] + } + `; + httpMock + .scope('https://some-registry') + .get('/v3/index.json') + .twice() + .reply(200, nugetIndex) + .get('/v3/metadata/nlog/index.json') + .reply(200, nlogRegistration) + .get('/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg') + .reply(200, () => { + const readableStream = new Readable(); + readableStream.push(Fixtures.getBinary('nlog/NLog.4.7.3.nupkg')); + readableStream.push(null); + return readableStream; + }); + const res = await getPkgReleases({ + datasource, + versioning, + packageName: 'NLog', + registryUrls: ['https://some-registry/v3/index.json'], + }); + expect(logger.logger.debug).toHaveBeenCalledWith( + 'Determined sourceUrl https://github.com/NLog/NLog.git from https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg', + ); + expect(packageCache.set).toHaveBeenCalledWith( + 'datasource-nuget', + 'cache-decorator:source-url:https://some-registry/v3/index.json:NLog', + { + cachedAt: expect.any(String), + value: 'https://github.com/NLog/NLog.git', + }, + 60 * 24 * 7, + ); + expect(res?.sourceUrl).toBeDefined(); + }); + + it('can handle nupkg without repository metadata', async () => { + const nugetIndex = ` + { + "version": "3.0.0", + "resources": [ + { + "@id": "https://some-registry/v3/metadata", + "@type": "RegistrationsBaseUrl/3.0.0-beta", + "comment": "Get package metadata." + } + ] + } + `; + const nlogRegistration = ` + { + "count": 1, + "items": [ + { + "@id": "https://some-registry/v3/metadata/nlog/4.7.3.json", + "lower": "4.7.3", + "upper": "4.7.3", + "count": 1, + "items": [ + { + "@id": "foo", + "catalogEntry": { + "id": "NLog", + "version": "4.7.3", + "packageContent": "https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg" + } + } + ] + } + ] + } + `; + httpMock + .scope('https://some-registry') + .get('/v3/index.json') + .twice() + .reply(200, nugetIndex) + .get('/v3/metadata/nlog/index.json') + .reply(200, nlogRegistration) + .get('/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg') + .reply(200, () => { + const readableStream = new Readable(); + readableStream.push( + Fixtures.getBinary('nlog/NLog.4.7.3-no-repo.nupkg'), + ); + readableStream.push(null); + return readableStream; + }); + const res = await getPkgReleases({ + datasource, + versioning, + packageName: 'NLog', + registryUrls: ['https://some-registry/v3/index.json'], + }); + expect(packageCache.set).toHaveBeenCalledWith( + 'datasource-nuget', + 'cache-decorator:source-url:https://some-registry/v3/index.json:NLog', + { + cachedAt: expect.any(String), + value: null, + }, + 60 * 24 * 7, + ); + expect(res?.sourceUrl).toBeUndefined(); + }); + }); + it('returns null for non 200 (v3v2)', async () => { httpMock.scope('https://api.nuget.org').get('/v3/index.json').reply(500); httpMock diff --git a/lib/modules/datasource/nuget/types.ts b/lib/modules/datasource/nuget/types.ts index 29aba2a5c6b4a5..36fa672e83b223 100644 --- a/lib/modules/datasource/nuget/types.ts +++ b/lib/modules/datasource/nuget/types.ts @@ -5,11 +5,13 @@ export interface ServicesIndexRaw { }[]; } +// See https://learn.microsoft.com/en-us/nuget/api/registration-base-url-resource#catalog-entry export interface CatalogEntry { version: string; published?: string; projectUrl?: string; listed?: boolean; + packageContent?: string; } export interface CatalogPage { diff --git a/lib/modules/datasource/nuget/v3.ts b/lib/modules/datasource/nuget/v3.ts index 2024c175cbee6a..1d0dc587c01d3a 100644 --- a/lib/modules/datasource/nuget/v3.ts +++ b/lib/modules/datasource/nuget/v3.ts @@ -1,9 +1,14 @@ import is from '@sindresorhus/is'; +import extract from 'extract-zip'; import semver from 'semver'; +import upath from 'upath'; import { XmlDocument } from 'xmldoc'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import * as packageCache from '../../../util/cache/package'; +import { cache } from '../../../util/cache/package/decorator'; +import * as fs from '../../../util/fs'; +import { ensureCacheDir } from '../../../util/fs'; import { Http, HttpError } from '../../../util/http'; import * as p from '../../../util/promises'; import { regEx } from '../../../util/regex'; @@ -151,8 +156,15 @@ export class NugetV3Api { let homepage: string | null = null; let latestStable: string | null = null; + let nupkgUrl: string | null = null; const releases = catalogEntries.map( - ({ version, published: releaseTimestamp, projectUrl, listed }) => { + ({ + version, + published: releaseTimestamp, + projectUrl, + listed, + packageContent, + }) => { const release: Release = { version: removeBuildMeta(version) }; if (releaseTimestamp) { release.releaseTimestamp = releaseTimestamp; @@ -160,6 +172,7 @@ export class NugetV3Api { if (versioning.isValid(version) && versioning.isStable(version)) { latestStable = removeBuildMeta(version); homepage = projectUrl ? massageUrl(projectUrl) : homepage; + nupkgUrl = massageUrl(packageContent); } if (listed === false) { release.isDeprecated = true; @@ -177,6 +190,7 @@ export class NugetV3Api { const last = catalogEntries.pop()!; latestStable = removeBuildMeta(last.version); homepage ??= last.projectUrl ?? null; + nupkgUrl ??= massageUrl(last.packageContent); } const dep: ReleaseResult = { @@ -189,7 +203,6 @@ export class NugetV3Api { registryUrl, 'PackageBaseAddress', ); - // istanbul ignore else: this is a required v3 api if (is.nonEmptyString(packageBaseAddress)) { const nuspecUrl = `${ensureTrailingSlash( packageBaseAddress, @@ -203,6 +216,18 @@ export class NugetV3Api { if (sourceUrl) { dep.sourceUrl = massageUrl(sourceUrl); } + } else if (nupkgUrl) { + const sourceUrl = await this.getSourceUrlFromNupkg( + http, + registryUrl, + pkgName, + latestStable, + nupkgUrl, + ); + if (sourceUrl) { + dep.sourceUrl = massageUrl(sourceUrl); + logger.debug(`Determined sourceUrl ${sourceUrl} from ${nupkgUrl}`); + } } } catch (err) { // istanbul ignore if: not easy testable with nock @@ -233,4 +258,52 @@ export class NugetV3Api { return dep; } + + @cache({ + namespace: NugetV3Api.cacheNamespace, + key: ( + _http: Http, + registryUrl: string, + packageName: string, + _packageVersion: string | null, + _nupkgUrl: string, + ) => `source-url:${registryUrl}:${packageName}`, + ttlMinutes: 10080, // 1 week + }) + async getSourceUrlFromNupkg( + http: Http, + _registryUrl: string, + packageName: string, + packageVersion: string | null, + nupkgUrl: string, + ): Promise { + // istanbul ignore if: experimental feature + if (!process.env.RENOVATE_X_NUGET_DOWNLOAD_NUPKGS) { + logger.once.debug('RENOVATE_X_NUGET_DOWNLOAD_NUPKGS is not set'); + return null; + } + const cacheDir = await ensureCacheDir('nuget'); + const nupkgFile = upath.join( + cacheDir, + `${packageName}.${packageVersion}.nupkg`, + ); + const nupkgContentsDir = upath.join( + cacheDir, + `${packageName}.${packageVersion}`, + ); + const readStream = http.stream(nupkgUrl); + try { + const writeStream = fs.createCacheWriteStream(nupkgFile); + await fs.pipeline(readStream, writeStream); + await extract(nupkgFile, { dir: nupkgContentsDir }); + const nuspecFile = upath.join(nupkgContentsDir, `${packageName}.nuspec`); + const nuspec = new XmlDocument( + await fs.readCacheFile(nuspecFile, 'utf8'), + ); + return nuspec.valueWithPath('metadata.repository@url') ?? null; + } finally { + await fs.rmCache(nupkgFile); + await fs.rmCache(nupkgContentsDir); + } + } }