Merge pull request package-url#19 from package-url/url2purl

pombredanne · web-flow · commit 7de9dea2f02c · 2019-01-02T17:16:53.000+01:00
Add support for nuget, pypi, and sourceforge in url2purl
diff --git a/src/packageurl/contrib/url2purl.py b/src/packageurl/contrib/url2purl.py
@@ -29,6 +29,7 @@
 from __future__ import unicode_literals
 
 import os
+import re
 
 try:
     from urlparse import urlparse  # Python 2
@@ -65,9 +66,22 @@ def get_purl(uri):
             return
 
 
+def purl_from_pattern(type_, pattern, uri):
+    uri = unquote_plus(uri)
+    compiled_pattern = re.compile(pattern, re.VERBOSE)
+    match = compiled_pattern.match(uri)
+
+    if match:
+        purl_data = {
+            field: value for field, value in match.groupdict().items()
+            if field in PackageURL._fields
+        }
+        return PackageURL(type_, **purl_data)
+
+
 @purl_router.route('https?://registry.npmjs.*/.*',
                    'https?://registry.yarnpkg.com/.*')
-def build_npm_url(uri):
+def build_npm_purl(uri):
     # npm URLs are difficult to disambiguate with regex
     if '/-/' in uri:
         return build_npm_download_purl(uri)
@@ -157,14 +171,95 @@ def build_maven_purl(uri):
     return PackageURL('maven', namespace, name, version, qualifiers)
 
 
-@purl_router.route('https?://rubygems.org/downloads/.*')
-def build_rubygems_url(uri):
-    if uri.endswith('/') or not uri.endswith('.gem'):
-        return
+# https://rubygems.org/downloads/jwt-0.1.8.gem
+rubygems_pattern = (
+    r"^https?://rubygems.org/downloads/"
+    r"(?P<name>.+)-(?P<version>.+)"
+    r"(\.gem)$"
+)
+
+
+@purl_router.route(rubygems_pattern)
+def build_rubygems_purl(uri):
+    return purl_from_pattern('rubygems', rubygems_pattern, uri)
 
+
+# https://pypi.python.org/packages/source/a/anyjson/anyjson-0.3.3.tar.gz
+pypi_pattern = (
+    r"(?P<name>.+)-(?P<version>.+)"
+    r"\.(zip|tar.gz|tar.bz2|.tgz)$"
+)
+
+# This pattern can be found in the following locations:
+# - wheel.wheelfile.WHEEL_INFO_RE
+# - distlib.wheel.FILENAME_RE
+# - setuptools.wheel.WHEEL_NAME
+# - pip._internal.wheel.Wheel.wheel_file_re
+wheel_file_re = re.compile(
+    r"^(?P<namever>(?P<name>.+?)-(?P<version>.*?))"
+    r"((-(?P<build>\d[^-]*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)"
+    r"\.whl)$",
+    re.VERBOSE
+)
+
+
+@purl_router.route('https?://.+python.+org/packages/.*')
+def build_pypi_purl(uri):
     path = unquote_plus(urlparse(uri).path)
     last_segment = path.split('/')[-1]
-    archive_basename = last_segment.rstrip('.gem')
-    name, _, version = archive_basename.rpartition('-')
 
-    return PackageURL('rubygems', name=name, version=version)
+    # /wheel-0.29.0-py2.py3-none-any.whl
+    if last_segment.endswith('.whl'):
+        match = wheel_file_re.match(last_segment)
+        if match:
+            return PackageURL(
+                'pypi',
+                name=match.group('name'),
+                version=match.group('version'),
+            )
+
+    return purl_from_pattern('pypi', pypi_pattern, last_segment)
+
+
+# http://nuget.org/packages/EntityFramework/4.2.0.0
+# https://www.nuget.org/api/v2/package/Newtonsoft.Json/11.0.1
+nuget_pattern1 = (
+    r"^https?://.*nuget.org/(api/v2/)?packages?/"
+    r"(?P<name>.+)/"
+    r"(?P<version>.+)$"
+)
+
+
+@purl_router.route(nuget_pattern1)
+def build_nuget_purl(uri):
+    return purl_from_pattern('nuget', nuget_pattern1, uri)
+
+
+# https://api.nuget.org/v3-flatcontainer/newtonsoft.json/10.0.1/newtonsoft.json.10.0.1.nupkg
+nuget_pattern2 = (
+    r"^https?://api.nuget.org/v3-flatcontainer/"
+    r"(?P<name>.+)/"
+    r"(?P<version>.+)/"
+    r".*(nupkg)$"  # ends with "nupkg"
+)
+
+
+@purl_router.route(nuget_pattern2)
+def build_nuget_purl(uri):
+    return purl_from_pattern('nuget', nuget_pattern2, uri)
+
+
+# http://master.dl.sourceforge.net/project/libpng/zlib/1.2.3/zlib-1.2.3.tar.bz2
+sourceforge_pattern = (
+    r"^https?://.*sourceforge.net/project/"
+    r"(?P<namespace>([^/]+))/"  # do not allow more "/" segments
+    r"(?P<name>.+)/"
+    r"(?P<version>[0-9\.]+)/"  # version restricted to digits and dots
+    r"(?P=name)-(?P=version).*"  # {name}-{version} repeated in the filename
+    r"[^/]$"  # not ending with "/"
+)
+
+
+@purl_router.route(sourceforge_pattern)
+def build_sourceforge_purl(uri):
+    return purl_from_pattern('sourceforge', sourceforge_pattern, uri)
diff --git a/tests/contrib/data/url2purl.json b/tests/contrib/data/url2purl.json
@@ -81,6 +81,8 @@
   "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz": "pkg:npm/xdg-basedir@3.0.0", 
   "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz": "pkg:npm/yallist@2.1.2", 
 
+  "http://rubygems.org/downloads/":  null,
+  "http://rubygems.org/downloads/macaddr-1.6.1":  null,
   "http://rubygems.org/downloads/macaddr-1.6.1.gem": "pkg:rubygems/macaddr@1.6.1",
   "http://rubygems.org/downloads/open4-1.3.0.gem": "pkg:rubygems/open4@1.3.0",
   "https://rubygems.org/downloads/actionmailer-4.0.3.gem": "pkg:rubygems/actionmailer@4.0.3",
@@ -92,5 +94,46 @@
   "https://rubygems.org/downloads/ref-1.0.5.gem": "pkg:rubygems/ref@1.0.5", 
   "https://rubygems.org/downloads/talentbox-delayed_job_sequel-4.0.0.gem": "pkg:rubygems/talentbox-delayed_job_sequel@4.0.0", 
   "https://rubygems.org/downloads/unf-0.1.3.gem": "pkg:rubygems/unf@0.1.3", 
-  "https://rubygems.org/downloads/yajl-ruby-1.2.0.gem": "pkg:rubygems/yajl-ruby@1.2.0"
+  "https://rubygems.org/downloads/yajl-ruby-1.2.0.gem": "pkg:rubygems/yajl-ruby@1.2.0",
+
+  "https://pypi.python.org/packages/source/z/zc.recipe.egg/zc.recipe.egg-2.0.0.tar.gz": "pkg:pypi/zc.recipe.egg@2.0.0",
+  "https://pypi.python.org/packages/source/p/python-openid/python-openid-2.2.5.zip": "pkg:pypi/python-openid@2.2.5",
+  "https://pypi.python.org/packages/38/e2/b23434f4030bbb1af3bcdbb2ecff6b11cf2e467622446ce66a08e99f2ea9/pluggy-0.4.0.zip#md5=447a92368175965d2fbacaef9f3df842": "pkg:pypi/pluggy@0.4.0",
+  "https://pypi.python.org/packages/py2.py3/w/wheel/bad-wheel-name-any.whl": null,
+  "https://pypi.python.org/packages/py2.py3/w/wheel/wheel-0.29.0-py2.py3-none-any.whl": "pkg:pypi/wheel@0.29.0",
+  "https://pypi.python.org/packages/py2.py3/w/wheel/wheel-0.29.0-py2.py3-none-any.whl#md5=d7db45db5c131af262b8ffccde46a88a": "pkg:pypi/wheel@0.29.0",
+  "https://files.pythonhosted.org/packages/87/44/0fa8e9d0cccb8eb86fc1b5170208229dc6d6e9fd6e57ea1fe19cbeea68f5/aboutcode_toolkit-3.4.0rc1-py2.py3-none-any.whl": "pkg:pypi/aboutcode-toolkit@3.4.0rc1",
+  "https://files.pythonhosted.org/packages/7f/cf/12d4611fc67babd4ae250c9e8249c5650ae1933395488e9e7e3562b4ff24/amqp-2.3.2-py2.py3-none-any.whl#sha256=eed41946890cd43e8dee44a316b85cf6fee5a1a34bb4a562b660a358eb529e1b": "pkg:pypi/amqp@2.3.2",
+  "https://pypi.python.org/packages/34/c1/8806f99713ddb993c5366c362b2f908f18269f8d792aff1abfd700775a77/click-6.7-py2.py3-none-any.whl#md5=5e7a4e296b3212da2ff11017675d7a4d": "pkg:pypi/click@6.7",
+  "https://pypi.python.org/packages/f6/ae/bbc6a204f33d9d57c798fb3857a072cd14b836792244eea4b446fdb674c6/pycryptodome-3.4.7-cp27-cp27m-win32.whl#md5=78b341de1cd686077745cd9e3a93d8d3": "pkg:pypi/pycryptodome@3.4.7",
+  "https://pypi.python.org/packages/bd/e8/ea44ba5357a0b4fd16e5fb60c355fc8722eae31b93d7597eec50f7c35a52/pycryptodome-3.4.7-cp27-cp27m-win_amd64.whl#md5=f20bb847322baf7ae24700e5cbb15e07": "pkg:pypi/pycryptodome@3.4.7",
+  "https://pypi.python.org/packages/1e/75/8005d086cac4cc41d3b320d338972c5e5c6a21f88472f21ac9d0e031d300/pyahocorasick-1.1.4.tar.bz2#md5=ad445b6648dc06e9040705ce1ccb4384": "pkg:pypi/pyahocorasick@1.1.4",
+
+  "http://nuget.org/packages/EntityFramework/4.2.0.0": "pkg:nuget/EntityFramework@4.2.0.0",
+  "http://www.nuget.org/packages/SharpGIS.GZipWebClient/1.2.0": "pkg:nuget/SharpGIS.GZipWebClient@1.2.0",
+  "https://www.nuget.org/api/v2/package/Newtonsoft.Json/11.0.1": "pkg:nuget/Newtonsoft.Json@11.0.1",
+  "http://www.nuget.org/api/v2/package/EntityFramework/6.1.0": "pkg:nuget/EntityFramework@6.1.0",
+  "https://www.nuget.org/api/v2/package/MvvmLightLibs/4.1.23": "pkg:nuget/MvvmLightLibs@4.1.23",
+  "https://www.nuget.org/api/v2/package/Twilio/3.4.1": "pkg:nuget/Twilio@3.4.1",
+  "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/10.0.1/newtonsoft.json.10.0.1.nupkg": "pkg:nuget/newtonsoft.json@10.0.1",
+
+  "http://master.dl.sourceforge.net/project/zznotes/zznotes/1.1.2/zznotes-1.1.2.tar.gz": "pkg:sourceforge/zznotes/zznotes@1.1.2",
+  "http://master.dl.sourceforge.net/project/zapping/zvbi/0.2.35/zvbi-0.2.35.tar.bz2": "pkg:sourceforge/zapping/zvbi@0.2.35",
+  "http://master.dl.sourceforge.net/project/libpng/zlib/1.2.3/zlib-1.2.3.tar.bz2": "pkg:sourceforge/libpng/zlib@1.2.3",
+  "http://master.dl.sourceforge.net/project/xmlstar/xmlstarlet/1.0.0/xmlstarlet-1.0.0-1.src.rpm": "pkg:sourceforge/xmlstar/xmlstarlet@1.0.0",
+  "http://master.dl.sourceforge.net/project/wxmozilla/wxMozilla/0.5.5/wxMozilla-0.5.5.exe": "pkg:sourceforge/wxmozilla/wxMozilla@0.5.5",
+  "http://iweb.dl.sourceforge.net/project/sblim/sblim-cim-client2/2.2.5/sblim-cim-client2-2.2.5-src.zip": "pkg:sourceforge/sblim/sblim-cim-client2@2.2.5",
+  "http://master.dl.sourceforge.net/project/zinnia/zinnia-win32/0.06/zinnia-win32-0.06.zip": "pkg:sourceforge/zinnia/zinnia-win32@0.06",
+  "http://iweb.dl.sourceforge.net/project/findbugs/findbugs/1.3.4/findbugs-1.3.4.tar.gz/": null,
+  "http://master.dl.sourceforge.net/project/arestc/net/sf/arestc/arestc/0.1.4/arestc-0.1.4-javadoc.jar": null,
+  "http://master.dl.sourceforge.net/project/intraperson/OldFiles/intraperson/0.28/intraperson-0.28.tar.gz":  null,
+  "http://master.dl.sourceforge.net/project/pwiki/pwiki/0.1.2/0.1.2.zip": null,
+  "http://master.dl.sourceforge.net/project/iswraid/iswraid/0.1.4.3/2.4.28-pre3-iswraid.patch.gz": null,
+  "http://master.dl.sourceforge.net/project/aloyscore/aloyscore/0.1a1%20stable/0.1a1_stable_AloysCore.zip": null,
+  "http://master.dl.sourceforge.net/project/myenterprise/OldFiles/1.0.0.2.MyEnterprise.Source.zip": null,
+  "http://master.dl.sourceforge.net/project/wxhaskell/wxhaskell/wxhaskell-0.9/wxhaskell-src-0.9.zip": null,
+  "http://master.dl.sourceforge.net/project/a2freedom/A2/1.2/a2freedom-1.2.zip": null,
+  "http://master.dl.sourceforge.net/project/tinyos/OldFiles/tinyos/1.1.0/tinyos-1.1.0.tar.gz": null,
+  "http://master.dl.sourceforge.net/project/urlchecker/lu/ng/urlchecker/urlchecker/1.7/urlchecker-1.7-javadoc.jar": null,
+  "http://master.dl.sourceforge.net/project/zclasspath/maven2/org/zclasspath/zclasspath/1.5/zclasspath-1.5.jar": null
 }
