Use PEP 440 style requirements

CHANGELOG.rst

@@ -79,6 +79,11 @@ Changed
 
   Contributed by @LiamRiddell
 
+* Use PEP 440 direct reference requirements instead of legacy PIP VCS requirements. Now, our ``*.requirements.txt`` files use
+  ``package-name@ git+https://url@version ; markers`` instead of ``git+https://url@version#egg=package-name ; markers``. #5673
+
+  Contributed by @cognifloyd
+
 Removed
 ~~~~~~~
 

contrib/linux/requirements.txt

@@ -1,3 +1,3 @@
 # used by file watcher sensor
-pyinotify>=0.9.5,<=0.10
--e git+/~https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper
+pyinotify>=0.9.5,<=0.10 ; platform_system=="Linux"
+logshipper@ git+/~https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux"

contrib/runners/action_chain_runner/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

contrib/runners/announcement_runner/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

contrib/runners/http_runner/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

contrib/runners/inquirer_runner/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

contrib/runners/local_runner/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

contrib/runners/noop_runner/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

contrib/runners/orquesta_runner/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

contrib/runners/orquesta_runner/in-requirements.txt

@@ -1 +1 @@
-git+/~https://github.com/StackStorm/orquesta.git@v1.5.0#egg=orquesta
+orquesta@ git+/~https://github.com/StackStorm/orquesta.git@v1.5.0

contrib/runners/orquesta_runner/requirements.txt

@@ -5,4 +5,4 @@
 # If you want to update depdencies for a single component, modify the
 # in-requirements.txt for that component and then run 'make requirements' to
 # update the component requirements.txt
-git+/~https://github.com/StackStorm/orquesta.git@v1.5.0#egg=orquesta
+orquesta@ git+/~https://github.com/StackStorm/orquesta.git@v1.5.0

contrib/runners/python_runner/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

contrib/runners/remote_runner/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

contrib/runners/winrm_runner/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

fixed-requirements.txt

@@ -39,7 +39,7 @@ oslo.utils<5.0,>=4.0.0
 paramiko==2.10.1
 passlib==1.7.4
 prompt-toolkit==1.0.15
-pyinotify==0.9.6; platform_system=="Linux"
+pyinotify==0.9.6 ; platform_system=="Linux"
 pymongo==3.11.3
 pyparsing<3
 zstandard==0.15.2

requirements.txt

@@ -18,11 +18,6 @@ decorator==4.4.2
 dnspython>=1.16.0,<2.0.0
 eventlet==0.30.2
 flex==6.14.1
-git+/~https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper
-git+/~https://github.com/StackStorm/orquesta.git@v1.5.0#egg=orquesta
-git+/~https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file
-git+/~https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap
-git+/~https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend
 gitdb==4.0.2
 gitpython==3.1.15
 greenlet==1.0.0
@@ -33,13 +28,15 @@ jsonpath-rw==1.4.0
 jsonschema==2.6.0
 kombu==5.0.2
 lockfile==0.12.2
+logshipper@ git+/~https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux"
 mock==4.0.3
 mongoengine==0.23.0
 networkx>=2.5.1,<2.6
 nose
 nose-parallel==0.4.0
 nose-timer==1.0.1
 orjson==3.5.2
+orquesta@ git+/~https://github.com/StackStorm/orquesta.git@v1.5.0
 oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
 paramiko==2.10.1
@@ -48,7 +45,7 @@ prettytable==2.1.0
 prompt-toolkit==1.0.15
 psutil==5.8.0
 pyOpenSSL<=21.0.0
-pyinotify==0.9.6; platform_system=="Linux"
+pyinotify==0.9.6 ; platform_system=="Linux"
 pymongo==3.11.3
 pyparsing<3
 pyrabbit
@@ -69,6 +66,9 @@ semver==2.13.0
 simplejson
 six==1.13.0
 sseclient-py==1.7
+st2-auth-backend-flat-file@ git+/~https://github.com/StackStorm/st2-auth-backend-flat-file.git@master
+st2-auth-ldap@ git+/~https://github.com/StackStorm/st2-auth-ldap.git@master
+st2-rbac-backend@ git+/~https://github.com/StackStorm/st2-rbac-backend.git@master
 stevedore==1.30.1
 tenacity>=3.2.1,<7.0.0
 tooz==2.8.0

scripts/dist_utils.py

@@ -122,6 +122,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

scripts/fixate-requirements.py

@@ -225,7 +225,14 @@ def write_requirements(
         # we don't have any idea how to process links, so just add them
         if linkreq.link and linkreq.link not in links:
             links.add(linkreq.link)
-            rline = str(linkreq.link)
+            if hasattr(req, "req") and req.req and str(req.req).count("@") == 2:
+                # PEP 440 direct reference
+                rline = str(linkreq.req)
+                # Also write out environment markers
+                if linkreq.markers:
+                    rline += " ; {}".format(str(linkreq.markers))
+            else:
+                rline = str(linkreq.link)
 
             if (hasattr(req, "is_editable") and req.is_editable) or (
                 hasattr(req, "editable") and req.editable

st2actions/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

st2actions/in-requirements.txt

@@ -18,7 +18,7 @@ gitpython
 lockfile
 # needed by core "linux" pack - TODO: create virtualenv for linux pack on postinst
 pyinotify
-git+/~https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper
+logshipper@ git+/~https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux"
 # required by pack_mgmt/setup_virtualenv.py#L135
 virtualenv
 # needed by requests

st2actions/requirements.txt

@@ -9,14 +9,14 @@ MarkupSafe<2.1.0,>=0.23
 apscheduler==3.7.0
 chardet<3.1.0
 eventlet==0.30.2
-git+/~https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper
 gitpython==3.1.15
 jinja2==2.11.3
 kombu==5.0.2
 lockfile==0.12.2
+logshipper@ git+/~https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux"
 oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
-pyinotify==0.9.6; platform_system=="Linux"
+pyinotify==0.9.6 ; platform_system=="Linux"
 pyparsing<3
 python-dateutil==2.8.1
 python-json-logger

st2api/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

st2auth/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

st2auth/in-requirements.txt

@@ -7,6 +7,6 @@ pymongo
 six
 stevedore
 # For backward compatibility reasons, flat file backend is installed by default
-git+/~https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file
-git+/~https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap
+st2-auth-backend-flat-file@ git+/~https://github.com/StackStorm/st2-auth-backend-flat-file.git@master
+st2-auth-ldap@ git+/~https://github.com/StackStorm/st2-auth-ldap.git@master
 gunicorn

st2auth/requirements.txt

@@ -7,11 +7,11 @@
 # update the component requirements.txt
 bcrypt==3.2.0
 eventlet==0.30.2
-git+/~https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file
-git+/~https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap
 gunicorn==20.1.0
 oslo.config>=1.12.1,<1.13
 passlib==1.7.4
 pymongo==3.11.3
 six==1.13.0
+st2-auth-backend-flat-file@ git+/~https://github.com/StackStorm/st2-auth-backend-flat-file.git@master
+st2-auth-ldap@ git+/~https://github.com/StackStorm/st2-auth-ldap.git@master
 stevedore==1.30.1

st2client/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

st2common/dist_utils.py

@@ -118,6 +118,12 @@ def _get_link(line):
 
                 link = line.replace("-e ", "").strip()
                 return link, req_name[0]
+            elif vcs_prefix in line and line.count("@") == 2:
+                # PEP 440 direct reference: <package name>@ <url>@version
+                req_name, link = line.split("@", 1)
+                req_name = req_name.strip()
+                link = f"{link.strip()}#egg={req_name}"
+                return link, req_name
 
         return None, None
 

st2common/in-requirements.txt

@@ -14,8 +14,8 @@ mongoengine
 networkx
 # used by networkx
 decorator
-git+/~https://github.com/StackStorm/orquesta.git@v1.5.0#egg=orquesta
-git+/~https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend
+orquesta@ git+/~https://github.com/StackStorm/orquesta.git@v1.5.0
+st2-rbac-backend@ git+/~https://github.com/StackStorm/st2-rbac-backend.git@master
 oslo.config
 paramiko
 pyyaml

st2common/requirements.txt

@@ -15,8 +15,6 @@ decorator==4.4.2
 dnspython>=1.16.0,<2.0.0
 eventlet==0.30.2
 flex==6.14.1
-git+/~https://github.com/StackStorm/orquesta.git@v1.5.0#egg=orquesta
-git+/~https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend
 gitdb==4.0.2
 gitpython==3.1.15
 greenlet==1.0.0
@@ -28,6 +26,7 @@ lockfile==0.12.2
 mongoengine==0.23.0
 networkx>=2.5.1,<2.6
 orjson==3.5.2
+orquesta@ git+/~https://github.com/StackStorm/orquesta.git@v1.5.0
 oslo.config>=1.12.1,<1.13
 paramiko==2.10.1
 pyOpenSSL<=21.0.0
@@ -41,6 +40,7 @@ retrying==1.3.3
 routes==2.4.1
 semver==2.13.0
 six==1.13.0
+st2-rbac-backend@ git+/~https://github.com/StackStorm/st2-rbac-backend.git@master
 tenacity>=3.2.1,<7.0.0
 tooz==2.8.0
 udatetime==0.0.16