feat(nuget): allow detecting source URLs via package contents (#28071)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

docs/usage/self-hosted-experimental.md

@@ -141,6 +141,10 @@ This feature is in private beta.
 If set, Renovate will query the merge-confidence JSON API only for datasources that are part of this list.
 The expected value for this environment variable is a JSON array of strings.
 
+## `RENOVATE_X_NUGET_DOWNLOAD_NUPKGS`
+
+If set to any value, Renovate will download `nupkg` files for determining package metadata.
+
 ## `RENOVATE_X_PLATFORM_VERSION`
 
 Specify this string for Renovate to skip API checks and provide GitLab/Bitbucket server version directly.

lib/modules/datasource/nuget/common.ts

@@ -12,10 +12,14 @@ export function removeBuildMeta(version: string): string {
 
 const urlWhitespaceRe = regEx(/\s/g);
 
-export function massageUrl(url: string): string {
+export function massageUrl(url: string | null | undefined): string | null {
+  if (url === null || url === undefined) {
+    return null;
+  }
+
   let resultUrl = url;
 
-  // During `dotnet pack` certain URLs are being URL decoded which may introduce whitespaces
+  // During `dotnet pack` certain URLs are being URL decoded which may introduce whitespace
   // and causes Markdown link generation problems.
   resultUrl = resultUrl.replace(urlWhitespaceRe, '%20');
 

lib/modules/datasource/nuget/index.spec.ts

@@ -1,8 +1,12 @@
+import { Readable } from 'stream';
 import { mockDeep } from 'jest-mock-extended';
+import { join } from 'upath';
 import { getPkgReleases } from '..';
 import { Fixtures } from '../../../../test/fixtures';
 import * as httpMock from '../../../../test/http-mock';
-import { logger } from '../../../../test/util';
+import { logger, mocked } from '../../../../test/util';
+import { GlobalConfig } from '../../../config/global';
+import * as _packageCache from '../../../util/cache/package';
 import * as _hostRules from '../../../util/host-rules';
 import { id as versioning } from '../../versioning/nuget';
 import { parseRegistryUrl } from './common';
@@ -14,6 +18,9 @@ const hostRules: any = _hostRules;
 
 jest.mock('../../../util/host-rules', () => mockDeep());
 
+jest.mock('../../../util/cache/package', () => mockDeep());
+const packageCache = mocked(_packageCache);
+
 const pkgInfoV3FromNuget = Fixtures.get('nunit/v3_nuget_org.xml');
 const pkgListV3Registration = Fixtures.get('nunit/v3_registration.json');
 
@@ -105,6 +112,10 @@ const configV3AzureDevOps = {
 };
 
 describe('modules/datasource/nuget/index', () => {
+  beforeEach(() => {
+    GlobalConfig.reset();
+  });
+
   describe('parseRegistryUrl', () => {
     it('extracts feed version from registry URL hash (v3)', () => {
       const parsed = parseRegistryUrl('https://my-registry#protocolVersion=3');
@@ -302,6 +313,160 @@ describe('modules/datasource/nuget/index', () => {
       );
     });
 
+    describe('determine source URL from nupkg', () => {
+      beforeEach(() => {
+        GlobalConfig.set({
+          cacheDir: join('/tmp/cache'),
+        });
+        process.env.RENOVATE_X_NUGET_DOWNLOAD_NUPKGS = 'true';
+      });
+
+      afterEach(() => {
+        delete process.env.RENOVATE_X_NUGET_DOWNLOAD_NUPKGS;
+      });
+
+      it('can determine source URL from nupkg when PackageBaseAddress is missing', async () => {
+        const nugetIndex = `
+          {
+            "version": "3.0.0",
+            "resources": [
+              {
+                "@id": "https://some-registry/v3/metadata",
+                "@type": "RegistrationsBaseUrl/3.0.0-beta",
+                "comment": "Get package metadata."
+              }
+            ]
+          }
+        `;
+        const nlogRegistration = `
+          {
+            "count": 1,
+            "items": [
+              {
+                "@id": "https://some-registry/v3/metadata/nlog/4.7.3.json",
+                "lower": "4.7.3",
+                "upper": "4.7.3",
+                "count": 1,
+                "items": [
+                  {
+                    "@id": "foo",
+                    "catalogEntry": {
+                      "id": "NLog",
+                      "version": "4.7.3",
+                      "packageContent": "https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg"
+                    }
+                  }
+                ]
+              }
+            ]
+          }
+        `;
+        httpMock
+          .scope('https://some-registry')
+          .get('/v3/index.json')
+          .twice()
+          .reply(200, nugetIndex)
+          .get('/v3/metadata/nlog/index.json')
+          .reply(200, nlogRegistration)
+          .get('/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg')
+          .reply(200, () => {
+            const readableStream = new Readable();
+            readableStream.push(Fixtures.getBinary('nlog/NLog.4.7.3.nupkg'));
+            readableStream.push(null);
+            return readableStream;
+          });
+        const res = await getPkgReleases({
+          datasource,
+          versioning,
+          packageName: 'NLog',
+          registryUrls: ['https://some-registry/v3/index.json'],
+        });
+        expect(logger.logger.debug).toHaveBeenCalledWith(
+          'Determined sourceUrl https://github.com/NLog/NLog.git from https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg',
+        );
+        expect(packageCache.set).toHaveBeenCalledWith(
+          'datasource-nuget',
+          'cache-decorator:source-url:https://some-registry/v3/index.json:NLog',
+          {
+            cachedAt: expect.any(String),
+            value: 'https://github.com/NLog/NLog.git',
+          },
+          60 * 24 * 7,
+        );
+        expect(res?.sourceUrl).toBeDefined();
+      });
+
+      it('can handle nupkg without repository metadata', async () => {
+        const nugetIndex = `
+          {
+            "version": "3.0.0",
+            "resources": [
+              {
+                "@id": "https://some-registry/v3/metadata",
+                "@type": "RegistrationsBaseUrl/3.0.0-beta",
+                "comment": "Get package metadata."
+              }
+            ]
+          }
+        `;
+        const nlogRegistration = `
+          {
+            "count": 1,
+            "items": [
+              {
+                "@id": "https://some-registry/v3/metadata/nlog/4.7.3.json",
+                "lower": "4.7.3",
+                "upper": "4.7.3",
+                "count": 1,
+                "items": [
+                  {
+                    "@id": "foo",
+                    "catalogEntry": {
+                      "id": "NLog",
+                      "version": "4.7.3",
+                      "packageContent": "https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg"
+                    }
+                  }
+                ]
+              }
+            ]
+          }
+        `;
+        httpMock
+          .scope('https://some-registry')
+          .get('/v3/index.json')
+          .twice()
+          .reply(200, nugetIndex)
+          .get('/v3/metadata/nlog/index.json')
+          .reply(200, nlogRegistration)
+          .get('/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg')
+          .reply(200, () => {
+            const readableStream = new Readable();
+            readableStream.push(
+              Fixtures.getBinary('nlog/NLog.4.7.3-no-repo.nupkg'),
+            );
+            readableStream.push(null);
+            return readableStream;
+          });
+        const res = await getPkgReleases({
+          datasource,
+          versioning,
+          packageName: 'NLog',
+          registryUrls: ['https://some-registry/v3/index.json'],
+        });
+        expect(packageCache.set).toHaveBeenCalledWith(
+          'datasource-nuget',
+          'cache-decorator:source-url:https://some-registry/v3/index.json:NLog',
+          {
+            cachedAt: expect.any(String),
+            value: null,
+          },
+          60 * 24 * 7,
+        );
+        expect(res?.sourceUrl).toBeUndefined();
+      });
+    });
+
     it('returns null for non 200 (v3v2)', async () => {
       httpMock.scope('https://api.nuget.org').get('/v3/index.json').reply(500);
       httpMock

lib/modules/datasource/nuget/types.ts

@@ -5,11 +5,13 @@ export interface ServicesIndexRaw {
   }[];
 }
 
+// See https://learn.microsoft.com/en-us/nuget/api/registration-base-url-resource#catalog-entry
 export interface CatalogEntry {
   version: string;
   published?: string;
   projectUrl?: string;
   listed?: boolean;
+  packageContent?: string;
 }
 
 export interface CatalogPage {

lib/modules/datasource/nuget/v3.ts

@@ -1,9 +1,14 @@
 import is from '@sindresorhus/is';
+import extract from 'extract-zip';
 import semver from 'semver';
+import upath from 'upath';
 import { XmlDocument } from 'xmldoc';
 import { logger } from '../../../logger';
 import { ExternalHostError } from '../../../types/errors/external-host-error';
 import * as packageCache from '../../../util/cache/package';
+import { cache } from '../../../util/cache/package/decorator';
+import * as fs from '../../../util/fs';
+import { ensureCacheDir } from '../../../util/fs';
 import { Http, HttpError } from '../../../util/http';
 import * as p from '../../../util/promises';
 import { regEx } from '../../../util/regex';
@@ -151,15 +156,23 @@ export class NugetV3Api {
 
     let homepage: string | null = null;
     let latestStable: string | null = null;
+    let nupkgUrl: string | null = null;
     const releases = catalogEntries.map(
-      ({ version, published: releaseTimestamp, projectUrl, listed }) => {
+      ({
+        version,
+        published: releaseTimestamp,
+        projectUrl,
+        listed,
+        packageContent,
+      }) => {
         const release: Release = { version: removeBuildMeta(version) };
         if (releaseTimestamp) {
           release.releaseTimestamp = releaseTimestamp;
         }
         if (versioning.isValid(version) && versioning.isStable(version)) {
           latestStable = removeBuildMeta(version);
           homepage = projectUrl ? massageUrl(projectUrl) : homepage;
+          nupkgUrl = massageUrl(packageContent);
         }
         if (listed === false) {
           release.isDeprecated = true;
@@ -177,6 +190,7 @@ export class NugetV3Api {
       const last = catalogEntries.pop()!;
       latestStable = removeBuildMeta(last.version);
       homepage ??= last.projectUrl ?? null;
+      nupkgUrl ??= massageUrl(last.packageContent);
     }
 
     const dep: ReleaseResult = {
@@ -189,7 +203,6 @@ export class NugetV3Api {
         registryUrl,
         'PackageBaseAddress',
       );
-      // istanbul ignore else: this is a required v3 api
       if (is.nonEmptyString(packageBaseAddress)) {
         const nuspecUrl = `${ensureTrailingSlash(
           packageBaseAddress,
@@ -203,6 +216,18 @@ export class NugetV3Api {
         if (sourceUrl) {
           dep.sourceUrl = massageUrl(sourceUrl);
         }
+      } else if (nupkgUrl) {
+        const sourceUrl = await this.getSourceUrlFromNupkg(
+          http,
+          registryUrl,
+          pkgName,
+          latestStable,
+          nupkgUrl,
+        );
+        if (sourceUrl) {
+          dep.sourceUrl = massageUrl(sourceUrl);
+          logger.debug(`Determined sourceUrl ${sourceUrl} from ${nupkgUrl}`);
+        }
       }
     } catch (err) {
       // istanbul ignore if: not easy testable with nock
@@ -233,4 +258,52 @@ export class NugetV3Api {
 
     return dep;
   }
+
+  @cache({
+    namespace: NugetV3Api.cacheNamespace,
+    key: (
+      _http: Http,
+      registryUrl: string,
+      packageName: string,
+      _packageVersion: string | null,
+      _nupkgUrl: string,
+    ) => `source-url:${registryUrl}:${packageName}`,
+    ttlMinutes: 10080, // 1 week
+  })
+  async getSourceUrlFromNupkg(
+    http: Http,
+    _registryUrl: string,
+    packageName: string,
+    packageVersion: string | null,
+    nupkgUrl: string,
+  ): Promise<string | null> {
+    // istanbul ignore if: experimental feature
+    if (!process.env.RENOVATE_X_NUGET_DOWNLOAD_NUPKGS) {
+      logger.once.debug('RENOVATE_X_NUGET_DOWNLOAD_NUPKGS is not set');
+      return null;
+    }
+    const cacheDir = await ensureCacheDir('nuget');
+    const nupkgFile = upath.join(
+      cacheDir,
+      `${packageName}.${packageVersion}.nupkg`,
+    );
+    const nupkgContentsDir = upath.join(
+      cacheDir,
+      `${packageName}.${packageVersion}`,
+    );
+    const readStream = http.stream(nupkgUrl);
+    try {
+      const writeStream = fs.createCacheWriteStream(nupkgFile);
+      await fs.pipeline(readStream, writeStream);
+      await extract(nupkgFile, { dir: nupkgContentsDir });
+      const nuspecFile = upath.join(nupkgContentsDir, `${packageName}.nuspec`);
+      const nuspec = new XmlDocument(
+        await fs.readCacheFile(nuspecFile, 'utf8'),
+      );
+      return nuspec.valueWithPath('metadata.repository@url') ?? null;
+    } finally {
+      await fs.rmCache(nupkgFile);
+      await fs.rmCache(nupkgContentsDir);
+    }
+  }
 }