Merge branch 'master' into try-fix/logging-new-services

CHANGELOG.rst

@@ -48,6 +48,8 @@ Changed
 
   If user wants to retrieve and view all the attributes, they can use ``--attr all`` CLI command
   argument (same as before). (improvement) #4396
+* Update various internal dependencies to latest stable versions (greenlet, pymongo, pytz,
+  stevedore, tooz). #4410
 
 * Improve ``st2.conf`` migration for the new services by using prod-friendly logging settings by default #4415
 

Makefile

@@ -383,7 +383,8 @@ requirements: virtualenv .sdist-requirements install-runners
 	$(VIRTUALENV_DIR)/bin/pip install "six==1.11.0"
 
 	# Fix for Travis CI caching issue
-	$(VIRTUALENV_DIR)/bin/pip uninstall -y "pytz"
+	$(VIRTUALENV_DIR)/bin/pip uninstall -y "pytz" || echo "not installed"
+	$(VIRTUALENV_DIR)/bin/pip uninstall -y "python-dateutil" || echo "not installed"
 
 	# Install requirements
 	#

contrib/runners/action_chain_runner/tests/unit/test_actionchain.py

@@ -702,8 +702,8 @@ def test_chain_task_passes_invalid_parameter_type_to_action(self, mock_request):
         chain_runner.pre_run()
 
         action_parameters = {}
-        expected_msg = ('Failed to cast value "stringnotanarray" \(type: str\) for parameter '
-                        '"arrtype" of type "array"')
+        expected_msg = (r'Failed to cast value "stringnotanarray" \(type: str\) for parameter '
+                        r'"arrtype" of type "array"')
         self.assertRaisesRegexp(ValueError, expected_msg, chain_runner.run,
                                 action_parameters=action_parameters)
 

contrib/runners/local_runner/local_runner/base.py

@@ -182,8 +182,8 @@ def _run(self, action):
 
         # Detect if user provided an invalid sudo password or sudo is not configured for that user
         if self._sudo_password:
-            if re.search('sudo: \d+ incorrect password attempts', stderr):
-                match = re.search('\[sudo\] password for (.+?)\:', stderr)
+            if re.search(r'sudo: \d+ incorrect password attempts', stderr):
+                match = re.search(r'\[sudo\] password for (.+?)\:', stderr)
 
                 if match:
                     username = match.groups()[0]

contrib/runners/orquesta_runner/tests/unit/test_error_handling.py

@@ -111,15 +111,15 @@ def test_fail_inspection(self):
             {
                 'type': 'content',
                 'message': 'The action "std.noop" is not registered in the database.',
-                'schema_path': 'properties.tasks.patternProperties.^\w+$.properties.action',
+                'schema_path': r'properties.tasks.patternProperties.^\w+$.properties.action',
                 'spec_path': 'tasks.task3.action'
             },
             {
                 'type': 'context',
                 'language': 'yaql',
                 'expression': '<% ctx().foobar %>',
                 'message': 'Variable "foobar" is referenced before assignment.',
-                'schema_path': 'properties.tasks.patternProperties.^\w+$.properties.input',
+                'schema_path': r'properties.tasks.patternProperties.^\w+$.properties.input',
                 'spec_path': 'tasks.task1.input',
             },
             {
@@ -131,7 +131,7 @@ def test_fail_inspection(self):
                     'position 0 of expression \'<% succeeded()\''
                 ),
                 'schema_path': (
-                    'properties.tasks.patternProperties.^\w+$.'
+                    r'properties.tasks.patternProperties.^\w+$.'
                     'properties.next.items.properties.when'
                 ),
                 'spec_path': 'tasks.task2.next[0].when'
@@ -142,7 +142,7 @@ def test_fail_inspection(self):
                     '[{\'cmd\': \'echo <% ctx().macro %>\'}] is '
                     'not valid under any of the given schemas'
                 ),
-                'schema_path': 'properties.tasks.patternProperties.^\w+$.properties.input.oneOf',
+                'schema_path': r'properties.tasks.patternProperties.^\w+$.properties.input.oneOf',
                 'spec_path': 'tasks.task2.input'
             }
         ]

contrib/runners/python_runner/tests/unit/test_pythonrunner.py

@@ -104,10 +104,10 @@ def test_action_returns_non_serializable_result(self):
 
         if six.PY2:
             expected_result_re = (r"\[{'a': '1'}, {'h': 3, 'c': 2}, {'e': "
-                                  "<non_simple_type.Test object at .*?>}\]")
+                                  r"<non_simple_type.Test object at .*?>}\]")
         else:
             expected_result_re = (r"\[{'a': '1'}, {'c': 2, 'h': 3}, {'e': "
-                                  "<non_simple_type.Test object at .*?>}\]")
+                                  r"<non_simple_type.Test object at .*?>}\]")
 
         match = re.match(expected_result_re, output['result'])
         self.assertTrue(match)
@@ -624,17 +624,17 @@ def test_python_action_wrapper_action_script_file_doesnt_exist_friendly_error(se
         wrapper = PythonActionWrapper(pack='dummy_pack_5', file_path=ACTION_1_PATH,
                                       user='joe')
 
-        expected_msg = ('Failed to load action class from file ".*?list_repos_doesnt_exist.py" '
-                       '\(action file most likely doesn\'t exist or contains invalid syntax\): '
-                       '\[Errno 2\] No such file or directory')
+        expected_msg = (r'Failed to load action class from file ".*?list_repos_doesnt_exist.py" '
+                       r'\(action file most likely doesn\'t exist or contains invalid syntax\): '
+                       r'\[Errno 2\] No such file or directory')
         self.assertRaisesRegexp(Exception, expected_msg, wrapper._get_action_instance)
 
     def test_python_action_wrapper_action_script_file_contains_invalid_syntax_friendly_error(self):
         wrapper = PythonActionWrapper(pack='dummy_pack_5', file_path=ACTION_2_PATH,
                                       user='joe')
-        expected_msg = ('Failed to load action class from file ".*?invalid_syntax.py" '
-                       '\(action file most likely doesn\'t exist or contains invalid syntax\): '
-                       'No module named \'?invalid\'?')
+        expected_msg = (r'Failed to load action class from file ".*?invalid_syntax.py" '
+                       r'\(action file most likely doesn\'t exist or contains invalid syntax\): '
+                       r'No module named \'?invalid\'?')
         self.assertRaisesRegexp(Exception, expected_msg, wrapper._get_action_instance)
 
     def test_simple_action_log_messages_and_log_level_runner_param(self):

contrib/runners/windows_runner/tests/unit/test_windows_runners.py

@@ -72,7 +72,7 @@ def test_get_winexe_command_args(self):
             [
                 'winexe',
                 '--interactive', '0',
-                '-U', 'MyDomain\Administrator3%bar3',
+                '-U', r'MyDomain\Administrator3%bar3',
                 '//localhost',
                 'dir'
             ]
@@ -123,7 +123,7 @@ def test_get_smbclient_command_args(self):
             ],
             [
                 'smbclient',
-                '-U', 'MyDomain\Administrator3%bar3',
+                '-U', r'MyDomain\Administrator3%bar3',
                 '//localhost/E$',
                 '-c', 'dir'
             ],

contrib/runners/windows_runner/windows_runner/base.py

@@ -62,7 +62,7 @@ def _get_winexe_command_args(self, host, username, password, command, domain=Non
         args += ['--interactive', '0']
 
         if domain:
-            args += ['-U', '%s\%s%%%s' % (domain, username, password)]
+            args += ['-U', r'%s\%s%%%s' % (domain, username, password)]
         else:
             args += ['-U', '%s%%%s' % (username, password)]
 
@@ -84,7 +84,7 @@ def _get_smbclient_command_args(self, host, username, password, command, share='
 
         values = {'domain': domain, 'username': username, 'password': password}
         if domain:
-            auth_string = '%(domain)s\%(username)s%%%(password)s' % values
+            auth_string = r'%(domain)s\%(username)s%%%(password)s' % values
         else:
             auth_string = '%(username)s%%%(password)s' % values
 

contrib/runners/windows_runner/windows_runner/windows_script_runner.py

@@ -314,7 +314,7 @@ def _parse_share_information(self, stdout):
 
         for line in lines:
             line = line.strip()
-            split = re.split('\s{3,}', line)
+            split = re.split(r'\s{3,}', line)
 
             if len(split) not in [1, 2]:
                 # Invalid line, skip it

fixed-requirements.txt

@@ -1,6 +1,6 @@
 # Packages versions fixed for the whole st2 stack
 # Note: greenlet is used by eventlet
-greenlet==0.4.14
+greenlet==0.4.15
 # Note: 0.20.0 removed select.poll() on which some of our code and libraries we
 # depend on rely
 eventlet==0.24.1
@@ -19,15 +19,15 @@ requests[security]<2.15,>=2.14.1
 apscheduler==3.5.3
 gitpython==2.1.11
 jsonschema==2.6.0
-pymongo==3.7.1
+pymongo==3.7.2
 passlib==1.7.1
 lockfile==0.12.2
 python-gnupg==0.4.3
 jsonpath-rw==1.4.0
 pyinotify==0.9.6
 semver==2.8.1
-pytz==2018.5
-stevedore==1.29.0
+pytz==2018.6
+stevedore==1.30.0
 paramiko==2.4.2
 networkx==1.11
 python-keyczar==0.716
@@ -38,7 +38,7 @@ virtualenv==15.1.0
 sseclient==0.0.19
 python-editor==1.0.3
 prompt-toolkit==1.0.15
-tooz==1.62.0
+tooz==1.63.1
 zake==0.2.2
 routes==2.4.1
 flex==6.13.2
@@ -53,3 +53,4 @@ python-statsd==2.1.0
 prometheus_client==0.1.1
 mock==2.0.0
 ujson==1.35
+python-dateutil==2.7.4

lint-configs/python/.flake8

@@ -1,4 +1,4 @@
 [flake8]
 max-line-length = 100
-ignore = E128,E402,E722
+ignore = E128,E402,E722,W504
 exclude=*.egg/*

requirements.txt

@@ -12,7 +12,7 @@ git+/~https://github.com/StackStorm/orquesta.git@142a19ddabbeea6a8d9d10341ba705851
 git+/~https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient
 git+/~https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file
 gitpython==2.1.11
-greenlet==0.4.14
+greenlet==0.4.15
 gunicorn==19.9.0
 ipaddr
 jinja2
@@ -34,14 +34,14 @@ prometheus_client==0.1.1
 prompt-toolkit==1.0.15
 psutil==5.4.7
 pyinotify==0.9.6
-pymongo==3.7.1
+pymongo==3.7.2
 pyrabbit
-python-dateutil
+python-dateutil==2.7.4
 python-editor==1.0.3
 python-gnupg==0.4.3
 python-json-logger
 python-statsd==2.1.0
-pytz==2018.5
+pytz==2018.6
 pywinrm==0.3.0
 pyyaml==3.13
 rednose
@@ -51,8 +51,8 @@ routes==2.4.1
 semver==2.8.1
 six==1.11.0
 sseclient==0.0.19
-stevedore==1.29.0
-tooz==1.62.0
+stevedore==1.30.0
+tooz==1.63.1
 ujson==1.35
 unittest2
 webob==1.8.2

scripts/travis/install-requirements.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash  
 
 if [ "${TASK}" = 'compilepy3 ci-py3-unit' ] || [ "${TASK}" = 'ci-py3-integration' ]; then
-    pip install "tox==3.1.3"
+    pip install "tox==3.5.2"
 
 	# Install runners
     . virtualenv/bin/activate

st2actions/tests/unit/policies/test_concurrency.py

@@ -92,6 +92,8 @@ def setUp(self):
         MockLiveActionPublisherNonBlocking.wait_all()
 
     def tearDown(self):
+        MockLiveActionPublisherNonBlocking.wait_all()
+
         for liveaction in LiveAction.get_all():
             action_service.update_status(
                 liveaction, action_constants.LIVEACTION_STATUS_CANCELED)

st2actions/tests/unit/policies/test_concurrency_by_attr.py

@@ -91,6 +91,8 @@ def setUp(self):
         MockLiveActionPublisherNonBlocking.wait_all()
 
     def tearDown(self):
+        MockLiveActionPublisherNonBlocking.wait_all()
+
         for liveaction in LiveAction.get_all():
             action_service.update_status(
                 liveaction, action_constants.LIVEACTION_STATUS_CANCELED)

st2actions/tests/unit/test_actions_registrar.py

@@ -66,8 +66,8 @@ def test_pack_name_missing(self):
             content = yaml.safe_load(fd)
             action_name = str(content['name'])
             action_db = Action.get_by_name(action_name)
-            self.assertEqual(action_db.pack, 'dummy', 'Content pack must be ' +
-                             'set to dummy')
+            expected_msg = 'Content pack must be set to dummy'
+            self.assertEqual(action_db.pack, 'dummy', expected_msg)
             Action.delete(action_db)
 
     @mock.patch.object(action_validator, '_is_valid_pack', mock.MagicMock(return_value=True))
@@ -138,6 +138,6 @@ def test_action_update(self):
             content = yaml.safe_load(fd)
             action_name = str(content['name'])
             action_db = Action.get_by_name(action_name)
-            self.assertEqual(action_db.pack, 'wolfpack', 'Content pack must be ' +
-                             'set to wolfpack')
+            expected_msg = 'Content pack must be set to wolfpack'
+            self.assertEqual(action_db.pack, 'wolfpack', expected_msg)
             Action.delete(action_db)

st2actions/tests/unit/test_notifier.py

@@ -110,8 +110,8 @@ def test_notify_triggers(self):
         liveaction_db.notify = NotificationSchema(on_success=on_success,
                                                   on_failure=on_failure)
         liveaction_db.start_timestamp = date_utils.get_datetime_utc_now()
-        liveaction_db.end_timestamp = (liveaction_db.start_timestamp +
-                                       datetime.timedelta(seconds=50))
+        liveaction_db.end_timestamp = \
+            (liveaction_db.start_timestamp + datetime.timedelta(seconds=50))
         LiveAction.add_or_update(liveaction_db)
 
         execution = MOCK_EXECUTION
@@ -181,8 +181,8 @@ def test_notify_triggers_jinja_patterns(self, dispatch):
                                            data={'stdout': '{{action_results.stdout}}'})
         liveaction_db.notify = NotificationSchema(on_success=on_success)
         liveaction_db.start_timestamp = date_utils.get_datetime_utc_now()
-        liveaction_db.end_timestamp = (liveaction_db.start_timestamp +
-                                       datetime.timedelta(seconds=50))
+        liveaction_db.end_timestamp = \
+            (liveaction_db.start_timestamp + datetime.timedelta(seconds=50))
 
         LiveAction.add_or_update(liveaction_db)
 

st2api/st2api/controllers/resource.py

@@ -66,7 +66,7 @@ def parameter_validation(validator, properties, instance, schema):
         "description": "Input parameters for the action.",
         "type": "object",
         "patternProperties": {
-            "^\w+$": util_schema.get_action_parameters_schema()
+            r"^\w+$": util_schema.get_action_parameters_schema()
         },
         'additionalProperties': False,
         "default": {}

st2api/tests/unit/controllers/v1/test_executions_auth.py

@@ -73,8 +73,12 @@
     }
 }
 
+# NOTE: We use a longer expiry time because this variable is initialized on module import (aka
+# when nosetests or similar imports this module before running the tests.
+# Depending on when the import happens and when the tests actually run, token could already expire
+# by that time and the tests would fail.
 NOW = date_utils.get_datetime_utc_now()
-EXPIRY = NOW + datetime.timedelta(seconds=300)
+EXPIRY = NOW + datetime.timedelta(seconds=1000)
 SYS_TOKEN = TokenDB(id=bson.ObjectId(), user='system', token=uuid.uuid4().hex, expiry=EXPIRY)
 USR_TOKEN = TokenDB(id=bson.ObjectId(), user='tokenuser', token=uuid.uuid4().hex, expiry=EXPIRY)
 

st2api/tests/unit/controllers/v1/test_workflow_inspection.py

@@ -73,15 +73,15 @@ def test_inspection_return_errors(self):
             {
                 'type': 'content',
                 'message': 'The action "std.noop" is not registered in the database.',
-                'schema_path': 'properties.tasks.patternProperties.^\w+$.properties.action',
+                'schema_path': r'properties.tasks.patternProperties.^\w+$.properties.action',
                 'spec_path': 'tasks.task3.action'
             },
             {
                 'type': 'context',
                 'language': 'yaql',
                 'expression': '<% ctx().foobar %>',
                 'message': 'Variable "foobar" is referenced before assignment.',
-                'schema_path': 'properties.tasks.patternProperties.^\w+$.properties.input',
+                'schema_path': r'properties.tasks.patternProperties.^\w+$.properties.input',
                 'spec_path': 'tasks.task1.input',
             },
             {
@@ -93,7 +93,7 @@ def test_inspection_return_errors(self):
                     'position 0 of expression \'<% succeeded()\''
                 ),
                 'schema_path': (
-                    'properties.tasks.patternProperties.^\w+$.'
+                    r'properties.tasks.patternProperties.^\w+$.'
                     'properties.next.items.properties.when'
                 ),
                 'spec_path': 'tasks.task2.next[0].when'
@@ -104,7 +104,7 @@ def test_inspection_return_errors(self):
                     '[{\'cmd\': \'echo <% ctx().macro %>\'}] is '
                     'not valid under any of the given schemas'
                 ),
-                'schema_path': 'properties.tasks.patternProperties.^\w+$.properties.input.oneOf',
+                'schema_path': r'properties.tasks.patternProperties.^\w+$.properties.input.oneOf',
                 'spec_path': 'tasks.task2.input'
             }
         ]

st2client/requirements.txt

@@ -5,9 +5,9 @@ jsonpath-rw==1.4.0
 jsonschema==2.6.0
 prettytable
 prompt-toolkit==1.0.15
-python-dateutil
+python-dateutil==2.7.4
 python-editor==1.0.3
-pytz==2018.5
+pytz==2018.6
 pyyaml==3.13
 requests[security]<2.15,>=2.14.1
 six==1.11.0

st2client/st2client/utils/jsutil.py

@@ -33,7 +33,7 @@
 #  a..b
 #  abc.
 #  a(*
-SIMPLE_EXPRESSION_REGEX = "^([a-zA-Z0-9\-_]+\.)*([a-zA-Z0-9\-_]+)$"
+SIMPLE_EXPRESSION_REGEX = r"^([a-zA-Z0-9\-_]+\.)*([a-zA-Z0-9\-_]+)$"
 SIMPLE_EXPRESSION_REGEX_CMPL = re.compile(SIMPLE_EXPRESSION_REGEX)