From a4a893052017b2da17f9f7090eb88055f701d200 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 8 Mar 2015 20:03:12 +0100 Subject: [PATCH 01/19] We want to use flake8 < 2.4 since newer version of flake8 pips pep8 (finally!) to a different / incompatible version we do. --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 1e3fe2c76e..6440577106 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,6 @@ coverage pep8>=1.6.0,<1.7 -flake8 +flake8>=2.3.0,<2.4 ipython mock>=1.0 nose From e5ea6ef33fdc85afc544575bdf0f6d9f85ddd417 Mon Sep 17 00:00:00 2001 From: Kirill Enykeev Date: Mon, 9 Mar 2015 12:13:58 +0600 Subject: [PATCH 02/19] Fix webui docs --- docs/source/conf.py | 5 +- docs/source/index.rst | 1 - docs/source/install/index.rst | 1 + docs/source/install/webui.rst | 80 +++++++++++++++++++++++ docs/source/webui.rst | 120 ---------------------------------- 5 files changed, 85 insertions(+), 122 deletions(-) create mode 100644 docs/source/install/webui.rst delete mode 100644 docs/source/webui.rst diff --git a/docs/source/conf.py b/docs/source/conf.py index 057370ee47..476b24e4da 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -84,7 +84,10 @@ def previous_version(ver): 'github_mistral': ('/~https://github.com/StackStorm/mistral/blob/master/%s', None), 'github_contrib': ('/~https://github.com/StackStorm/st2contrib/blob/master/%s', None), - 'github_devenv': ('/~https://github.com/StackStorm/devenv/blob/master/%s', None) + 'github_devenv': ('/~https://github.com/StackStorm/devenv/blob/master/%s', None), + 'github_st2web': ('/~https://github.com/StackStorm/st2web/blob/master/%s', None), + 'ops_latest': + ('https://ops.stackstorm.net/releases/st2/' + release + '/%s/', None) } # Inserted at the bottom of all rst files. diff --git a/docs/source/index.rst b/docs/source/index.rst index 051a185e04..c3663579af 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,7 +8,6 @@ Contents: overview install/index - webui video start packs diff --git a/docs/source/install/index.rst b/docs/source/install/index.rst index 1b4407ec84..e7b18e6f5b 100644 --- a/docs/source/install/index.rst +++ b/docs/source/install/index.rst @@ -28,6 +28,7 @@ Installation should take about 5 min. Grab a coffee and watch :doc:`/video` whil Docker deploy sources + webui .. note:: diff --git a/docs/source/install/webui.rst b/docs/source/install/webui.rst new file mode 100644 index 0000000000..236e7c6983 --- /dev/null +++ b/docs/source/install/webui.rst @@ -0,0 +1,80 @@ +Web UI +====== + +st2web is an Angular-based web application. It allows you to control the whole process of execution, from running an action to seeing the results of the execution. It also helps you to explore workflow executions up to the results of individual tasks. All in real time. + +Express installation +-------------------- + +Production version of st2web is pre-installed on st2express and any other st2_deploy installation. You can access the UI by accessing ``/webui`` endpoint of st2api server. For vagrant deployment of st2express, it would be http://172.168.90.50:9101/webui. + +It can also be installed by extracting the latest tar-ball from :ops_latest:`webui` into ``/opt/stackstorm/static/webui``. + +This type of installation has no additional requirements. + +Production +---------- + +st2web is a pure html5 application and consist only of js scripts, html templates, css styles and a number of static files including custom fonts and svg images. For application to work correctly, all this files should be served to the browser. In most cases, the particular web server doesn't matter. It might be apache or nginx, pecan or node.js, files could be served from local server or uploaded to S3, Heroku or even Github Pages. + +The application itself is running within the browser so API endpoint should be accessible to the user itself, not the static server. + +To generate the latest production version of st2web from the source code, run ``gulp production`` inside st2web directory. Then, copy ``build/`` directory to the server public folder or upload it to the cloud. For more information on working with gulp, please see :github_st2web:`README.md`. + +Development +----------- + +With production version it is almost impossible to make any changes in code, besides configuration. In case any changes are needed, development version has to be installed. For information on how to install development version, please refer to :github_st2web:`README.md`. + +Configuration +------------- + +For UI to work properly, both client and server side should be configured accordingly. + +On a UI side, there is a file ``config.js`` in a root of the project which contains the list of servers this UI can connect to. The file consists of an array of objects, each have a ``name``, ``url`` and ``auth`` properties. + +:: + + hosts: [{ + name: 'Express Deployment', + url: 'http://172.168.90.50:9101', + auth: true + },{ + name: 'Development Environment', + url: 'http://172.168.50.50:9101' + }] + + +Multiple servers could be configured for user to pick from. To disconnect from the current server and return to login screen, pick 'Disconnect' from the drop down at the top right corner of the UI. + +On an ST2 side, `CORS `__ should also be properly configured. In st2.conf, ``allow_origin`` property of the [api] section should contain the Origin header browser sends with every request. For example, if you have deployed UI on its own server and accessing it using url `http://webui.example.com:8000`, your config should look like that: + +:: + + [api] + # Host and port to bind the API server. + host = 0.0.0.0 + port = 9101 + logging = st2api/conf/logging.conf + # List of allowed origins for CORS, use when deploying st2web client + # The URL must match the one the browser uses to access the st2web + allow_origin = http://webui.example.com:8000 + +Origin consists of scheme, hostname and port (if it isn't 80). Path (including tailing slash) should be omitted. + +Please note that some of the origins is already included by default and do not require additional configuration: + +* http://localhost:3000 - development version of `gulp` running locally +* http://localhost:9101,http://127.0.0.1:9101 - st2api pecan deployment (st2_deploy default) +* `api_url` from [auth] section of st2.conf + +Also, please note that although this is not recommended and will undermine your security, you can allow every web UI deployment to connect to your server by setting ``allow_origin = *``. + +Authentication +-------------- + +To configure st2web to support authentication, edit ``config.js`` and add ``auth:true`` to every server that supports authentication. To enable authentication on a server side, please refer to :doc:`/install/deploy`. + +For now, UI assumes st2auth is running on the same server with st2api on the port 9100. This is known issue and will be fixed in the next release. + +It is highly recommended to only use authentication alongside with SSL encryption (for st2web, st2api and st2auth) to mitigate possible MITM attacks and avoid sending passwords and auth tokens in plain text. diff --git a/docs/source/webui.rst b/docs/source/webui.rst deleted file mode 100644 index ba53a80c94..0000000000 --- a/docs/source/webui.rst +++ /dev/null @@ -1,120 +0,0 @@ -Web UI -====== - -st2web is an Angular-based web application. It allows you to control the whole process of execution, from running an action to seeing the results of the execution. It also helps you to explore workflow executions up to the results of individual tasks. All in real time. - -Express installation --------------------- - -Production version of st2web is pre-installed on st2express and any other st2_deploy installation. You can access the UI by accessing **/webui** endpoint of st2api server. For vagrant deployment of st2express, it would be http://172.168.90.50:9101/webui. - -It can also be installed by extracting the latest tar-ball from `https://ops.stackstorm.net/releases/st2//webui/` into `/opt/stackstorm/static/webui`. - -This type of installation has no additional requirements. - -Development ------------ - -With production version it is almost impossible to make any changes in code, besides configuration. In case any changes are needed, development version has to be installed. - -First of all, you need to make sure you have latest stable node and npm packages installed. - -:: - - $ node -v - v0.10.32 - - $ npm -v - 1.4.9 - -then you need to globally install bower and gulp. - -:: - - $ npm install -g bower - $ npm install -g gulp - - -Clone the latest version of st2web repository - -:: - - $ git clone /~https://github.com/StackStorm/st2web.git - $ cd st2web - -then you need to install the requirements - -:: - - $ npm install - $ bower install - -and finally run build system to lint js files, compile css and watch for changes - -:: - - $ gulp - -At that point you should be able to point your browser to http://localhost:3000/ and see the the page. Every change in code would be automatically recompiled. - -Gulp has several additional options: - -* `gulp build` - just lint and compile everything -* `gulp test` - build the project and then run e2e tests -* `gulp serve` - build the project and start serving it at 3000 port -* `gulp production` - build production version of the project - -The production version of the code is located in the `build/` folder. It consists of compiled and minified versions of JS and CSS files, doesn't have any additional dependencies and can be deployed to any server that can serve static files. The version produced by `gulp production` is an exact copy of the data used in Express installation and can be deployed the same way. - -Configuration -------------- - -For UI to work properly, both client and server side should be configured properly. - -On a UI side, there is a file `config.js` in a root of the project which contains the list of servers this UI can connect to. The file consists of an array of objects, each have a `name`, `url` and `auth` properties. - -:: - - hosts: [{ - name: 'Express Deployment', - url: 'http://172.168.90.50:9101', - auth: true - },{ - name: 'Development Environment', - url: 'http://172.168.50.50:9101' - }] - - -Multiple servers could be configured for user to pick from. To disconnect from the current server and return to login screen, pick 'Disconnect' from the drop down at the top right corner of the UI. - -On an ST2 side, `CORS `__ should also be properly configured. In st2.conf, `allow_origin` property of the [api] section should contain the Origin header browser sends with every request. For example, if you have deployed UI on its own server and accessing it using url `http://webui.example.com:8000`, your config should look like that: - -:: - - [api] - # Host and port to bind the API server. - host = 0.0.0.0 - port = 9101 - logging = st2api/conf/logging.conf - # List of allowed origins for CORS, use when deploying st2web client - # The URL must match the one the browser uses to access the st2web - allow_origin = http://st2web.example.com:3000 - -Origin consists of scheme, hostname and port (if it isn't 80). Path (including tailing slash) should be omitted. - -Please note that some of the origins is already included by default and do not require additional configuration: - -* http://localhost:3000 - development version of `gulp` running locally -* http://localhost:9101,http://127.0.0.1:9101 - st2api pecan deployment (st2_deploy default) -* `api_url` from [auth] section of st2.conf - -Also, please note that although this is not recommended and will undermine your security, you can allow every web UI deployment to connect to your server by setting `allow_origin = *`. - -Authentication --------------- - -To configure st2web to support authentication, edit `config.js` and add `auth:true` to every server that supports authentication. To enable authentication on a server side, please refer to :doc:`/install/deploy`. - -For now, UI assumes st2auth is running on the same server with st2api on the port 9100. This is known issue and will be fixed in the next release. - -It is highly recommended to ony use authentication alongside with SSL encryption (for st2web, st2api and st2auth) to mitigate possible MITM attacks and avoid sending passwords and auth tokens in plain text. \ No newline at end of file From b21950df6a68b427e961b9bbf351525e2cf4b8be Mon Sep 17 00:00:00 2001 From: manasdk Date: Thu, 12 Mar 2015 05:24:11 +0000 Subject: [PATCH 03/19] max(...) throws an error for empty values so add handling. --- st2client/st2client/formatters/table.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/st2client/st2client/formatters/table.py b/st2client/st2client/formatters/table.py index 14462ae61c..b99a07c5d2 100644 --- a/st2client/st2client/formatters/table.py +++ b/st2client/st2client/formatters/table.py @@ -147,9 +147,8 @@ def _get_friendly_column_name(name): @staticmethod def _get_required_column_width(values, minimum_width=0): - width = minimum_width - max_width = len(max(values, key=len)) - return max_width if max_width > width else width + max_width = len(max(values, key=len)) if values else minimum_width + return max_width if max_width > minimum_width else minimum_width class PropertyValueTable(formatters.Formatter): From c289e2f15887bed44cb308e361afdc1888e76da3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 11 Mar 2015 23:19:01 +0100 Subject: [PATCH 04/19] Select a method based on the runner_type and not based on the presence and value of entry_point parameter. Bad code! Go away! --- st2actions/st2actions/runners/fabricrunner.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/st2actions/st2actions/runners/fabricrunner.py b/st2actions/st2actions/runners/fabricrunner.py index 56017fb552..e2f7692714 100755 --- a/st2actions/st2actions/runners/fabricrunner.py +++ b/st2actions/st2actions/runners/fabricrunner.py @@ -114,9 +114,16 @@ def pre_run(self): def run(self, action_parameters): LOG.debug(' action_parameters = %s', action_parameters) - remote_action = self._get_fabric_remote_action(action_parameters) \ - if self.entry_point is None or len(self.entry_point) < 1 \ - else self._get_fabric_remote_script_action(action_parameters) + + runner_type = self.action.runner_type['name'] + + if runner_type == 'run-remote-script': + remote_action = self._get_fabric_remote_script_action(action_parameters) + elif runner_type == 'run-remote': + remote_action = self._get_fabric_remote_action(action_parameters) + else: + raise Exception('Invalid runner: %s' % (runner_type)) + LOG.debug('Will execute remote_action : %s.', str(remote_action)) result = self._run(remote_action) LOG.debug('Executed remote_action : %s. Result is : %s.', remote_action, result) From 3c0c2969151406174343a32dbe422d92f6f1bb39 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 11 Mar 2015 23:20:30 +0100 Subject: [PATCH 05/19] Throw if a run-remote-script action missed an entry_point. Empty entry poing for remote script runner doesn't make sense. --- st2actions/st2actions/runners/fabricrunner.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/st2actions/st2actions/runners/fabricrunner.py b/st2actions/st2actions/runners/fabricrunner.py index e2f7692714..f29630cfbb 100755 --- a/st2actions/st2actions/runners/fabricrunner.py +++ b/st2actions/st2actions/runners/fabricrunner.py @@ -162,6 +162,13 @@ def _get_fabric_remote_action(self, action_paramaters): cwd=self._cwd) def _get_fabric_remote_script_action(self, action_parameters): + # remote script actions without entry_point don't make sense, user probably wanted to use + # "run-remote" action + if not self.entry_point: + msg = ('Action "%s" is missing entry_point attribute. Perhaps wanted to use ' + '"run-remote" runner?') + raise Exception(msg % (self.action_name)) + script_local_path_abs = self.entry_point pos_args, named_args = self._get_script_args(action_parameters) named_args = self._transform_named_args(named_args) From 75ef0f45c4926f41fb042d9a8348250a61267ee0 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 11 Mar 2015 23:31:19 +0100 Subject: [PATCH 06/19] Update changelog. Conflicts: CHANGELOG.rst --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1e7cff058d..c0de6fb57c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Changelog ========= +v0.8.3 - TBD +------------ + +* Don't allow ``run-remote-script`` actions without an ``entry_point`` attribute - throw an + exception when running an action. (improvement) + v0.8.2 - March 10, 2015 ----------------------- From 72a8f33445f5b4c081e53b6ec1a8df34b96fe36e Mon Sep 17 00:00:00 2001 From: James Fryman Date: Fri, 13 Mar 2015 14:21:07 -0500 Subject: [PATCH 07/19] params in actionchain, not parameters --- docs/source/actionchain.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/actionchain.rst b/docs/source/actionchain.rst index 97b98bdb30..de64301955 100644 --- a/docs/source/actionchain.rst +++ b/docs/source/actionchain.rst @@ -68,7 +68,7 @@ For a user to provide input to an ActionChain the input parameters must be defin --- # ... - parameters: + params: input1: type: "string" required: true @@ -84,7 +84,7 @@ The input parameter ``input1`` can now be referenced in the parameters field of - name: "action1" ref: "core.local" - parameters: + params: action1_input: "{{input1}}" # ... From 545b34fb40068a70a070fab8ed42abd44993c3ed Mon Sep 17 00:00:00 2001 From: James Fryman Date: Tue, 17 Mar 2015 10:22:29 -0500 Subject: [PATCH 08/19] adding kiril doc fix repairs --- docs/source/install/deb.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/source/install/deb.rst b/docs/source/install/deb.rst index a910f61521..9ce3b3ceff 100644 --- a/docs/source/install/deb.rst +++ b/docs/source/install/deb.rst @@ -22,8 +22,7 @@ Pip The easiest way to install these is to use the requirements.txt file from the |st2| downloads server. This is kept up to date for the version specified in the path. :: - - curl -q -k -O https://ops.stackstorm.net/releases/st2/0.8.2/requirements.txt + curl -q -k -O https://raw.githubusercontent.com/StackStorm/st2/0.8.3/requirements.txt pip install -r requirements.txt RabbitMQ @@ -33,7 +32,7 @@ In order to get the latest version of RabbitMQ, you will want to follow the dire :: - http://www.rabbitmq.com/install-rpm.html + http://www.rabbitmq.com/install-debian.html Once you have RabbitMQ installed, you will need to run the following commands to enable certain plugins. From 403e412e1b28ec6bf603c6050696f0b303d3731d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 19 Mar 2015 19:39:00 +0100 Subject: [PATCH 09/19] Fix packs.setup_virtualenv command so it correctly looks for a pack in all the pack search paths. --- .../actions/pack_mgmt/setup_virtualenv.py | 21 +++++++++------ st2common/st2common/content/utils.py | 27 +++++++++++++++++++ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py index 66bc9a62af..b43af68b03 100644 --- a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py +++ b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py @@ -9,7 +9,8 @@ from st2actions.runners.pythonrunner import Action from st2common.constants.pack import PACK_NAME_WHITELIST from st2common.constants.pack import BASE_PACK_REQUIREMENTS -from st2common.content.utils import get_system_packs_base_path +from st2common.content.utils import get_packs_base_paths +from st2common.content.utils import get_pack_directory class SetupVirtualEnvironmentAction(Action): @@ -27,7 +28,6 @@ def __init__(self, config=None): super(SetupVirtualEnvironmentAction, self).__init__(config=config) self.initialize() - self._base_packs_path = get_system_packs_base_path() self._base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/') @@ -51,6 +51,8 @@ def run(self, packs): def _setup_pack_virtualenv(self, pack_name): """ + Setup virtual environment for the provided pack. + :param pack_name: Pack name. :type pack_name: ``str`` """ @@ -61,13 +63,16 @@ def _setup_pack_virtualenv(self, pack_name): self.logger.debug('Setting up virtualenv for pack "%s"' % (pack_name)) - pack_name = pipes.quote(pack_name) - pack_path = os.path.join(self._base_packs_path, pack_name) - virtualenv_path = os.path.join(self._base_virtualenvs_path, pack_name) + virtualenv_path = os.path.join(self._base_virtualenvs_path, pipes.quote(pack_name)) + + # Ensure pack directory exists in one of the search paths + pack_path = get_pack_directory(pack_name=pack_name) - # Ensure virtualenvs directory exists - if not os.path.isdir(pack_path): - raise Exception('Pack "%s" is not installed' % (pack_name)) + if not pack_path: + packs_base_paths = get_packs_base_paths() + search_paths = ', '.join(packs_base_paths) + msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths) + raise Exception(msg) if not os.path.exists(self._base_virtualenvs_path): os.makedirs(self._base_virtualenvs_path) diff --git a/st2common/st2common/content/utils.py b/st2common/st2common/content/utils.py index 5f9233eaed..16f13c0c75 100644 --- a/st2common/st2common/content/utils.py +++ b/st2common/st2common/content/utils.py @@ -25,6 +25,7 @@ 'get_system_packs_base_path', 'get_packs_base_paths', 'get_pack_base_path', + 'get_pack_directory', 'check_pack_directory_exists', 'check_pack_content_directory_exists' ] @@ -138,6 +139,32 @@ def get_pack_base_path(pack_name): return pack_base_path +def get_pack_directory(pack_name): + """ + Retrieve a directory for the provided pack. + + If a directory for the provided pack doesn't exist in any of the search paths, None + is returned instead. + + Note: If same pack exists in multiple search path, path to the first one is returned. + + :param pack_name: Pack name. + :type pack_name: ``str`` + + :return: Pack to the pack directory. + :rtype: ``str`` or ``None`` + """ + packs_base_paths = get_packs_base_paths() + for packs_base_path in packs_base_paths: + pack_base_path = os.path.join(packs_base_path, pipes.quote(pack_name)) + pack_base_path = os.path.abspath(pack_base_path) + + if os.path.isdir(pack_base_path): + return pack_base_path + + return None + + def get_entry_point_abs_path(pack=None, entry_point=None): """ Return full absolute path of an action entry point in a pack. From f6c7d9e497a13e406cada1099f343528174d5cbb Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 19 Mar 2015 19:50:15 +0100 Subject: [PATCH 10/19] Fix a bug in the python runner - we also need to pass parent args to the python wrapper otherwise Python actions won't have access to the correct config values. --- .../st2actions/runners/python_action_wrapper.py | 16 +++++++++++++--- st2actions/st2actions/runners/pythonrunner.py | 4 +++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/st2actions/st2actions/runners/python_action_wrapper.py b/st2actions/st2actions/runners/python_action_wrapper.py index 419d6287c3..80e443b731 100644 --- a/st2actions/st2actions/runners/python_action_wrapper.py +++ b/st2actions/st2actions/runners/python_action_wrapper.py @@ -32,7 +32,7 @@ class PythonActionWrapper(object): - def __init__(self, pack, file_path, parameters=None): + def __init__(self, pack, file_path, parameters=None, parent_args=None): """ :param pack: Name of the pack this action belongs to. :type pack: ``str`` @@ -42,13 +42,17 @@ def __init__(self, pack, file_path, parameters=None): :param parameters: action parameters. :type parameters: ``dict`` or ``None`` + + :param parent_args: Command line arguments passed to the parent process. + :type parse_args: ``list`` """ self._pack = pack self._file_path = file_path self._parameters = parameters or {} + self._parent_args = parent_args or [] try: - config.parse_args(args=[]) + config.parse_args(args=self._parent_args) except Exception: pass @@ -95,12 +99,18 @@ def _get_action_instance(self): help='Path to the action module') parser.add_argument('--parameters', required=False, help='Serialized action parameters') + parser.add_argument('--parent-args', required=False, + help='Command line arguments passed to the parent process') args = parser.parse_args() parameters = args.parameters parameters = json.loads(parameters) if parameters else {} + parent_args = json.loads(args.parent_args) if args.parent_args else [] + assert isinstance(parent_args, list) obj = PythonActionWrapper(pack=args.pack, file_path=args.file_path, - parameters=parameters) + parameters=parameters, + parent_args=parent_args) + obj.run() diff --git a/st2actions/st2actions/runners/pythonrunner.py b/st2actions/st2actions/runners/pythonrunner.py index c9bac1b666..88cb50a633 100644 --- a/st2actions/st2actions/runners/pythonrunner.py +++ b/st2actions/st2actions/runners/pythonrunner.py @@ -14,6 +14,7 @@ # limitations under the License. import os +import sys import abc import json import uuid @@ -122,7 +123,8 @@ def run(self, action_parameters): WRAPPER_SCRIPT_PATH, '--pack=%s' % (pack), '--file-path=%s' % (self.entry_point), - '--parameters=%s' % (serialized_parameters) + '--parameters=%s' % (serialized_parameters), + '--parent-args=%s' % (json.dumps(sys.argv[1:])) ] # We need to ensure all the st2 dependencies are also available to the From 13172305320011134fb56d764817d660fdfe7429 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 19 Mar 2015 19:54:04 +0100 Subject: [PATCH 11/19] Remove now unnecessary initialize method - config parsing now correctly happens inside the python process wrapper. --- contrib/packs/actions/pack_mgmt/delete.py | 9 --------- contrib/packs/actions/pack_mgmt/setup_virtualenv.py | 9 --------- contrib/packs/actions/pack_mgmt/unload.py | 9 +-------- 3 files changed, 1 insertion(+), 26 deletions(-) diff --git a/contrib/packs/actions/pack_mgmt/delete.py b/contrib/packs/actions/pack_mgmt/delete.py index 7806c639c7..6d1caed4c1 100644 --- a/contrib/packs/actions/pack_mgmt/delete.py +++ b/contrib/packs/actions/pack_mgmt/delete.py @@ -4,7 +4,6 @@ from oslo.config import cfg -import st2common.config as config from st2actions.runners.pythonrunner import Action from st2common.constants.pack import SYSTEM_PACK_NAMES @@ -14,17 +13,9 @@ class UninstallPackAction(Action): def __init__(self, config=None): super(UninstallPackAction, self).__init__(config=config) - self.initialize() - self._base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/') - def initialize(self): - try: - config.parse_args() - except: - pass - def run(self, packs, abs_repo_base): intersection = BLOCKED_PACKS & frozenset(packs) if len(intersection) > 0: diff --git a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py index b43af68b03..cc559a440c 100644 --- a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py +++ b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py @@ -4,7 +4,6 @@ from oslo.config import cfg -import st2common.config as config from st2common.util.shell import run_command from st2actions.runners.pythonrunner import Action from st2common.constants.pack import PACK_NAME_WHITELIST @@ -26,17 +25,9 @@ class SetupVirtualEnvironmentAction(Action): def __init__(self, config=None): super(SetupVirtualEnvironmentAction, self).__init__(config=config) - self.initialize() - self._base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path, 'virtualenvs/') - def initialize(self): - try: - config.parse_args() - except: - pass - def run(self, packs): """ :param packs: A list of packs to create the environment for. diff --git a/contrib/packs/actions/pack_mgmt/unload.py b/contrib/packs/actions/pack_mgmt/unload.py index 69ab77832c..079b594bcc 100644 --- a/contrib/packs/actions/pack_mgmt/unload.py +++ b/contrib/packs/actions/pack_mgmt/unload.py @@ -1,6 +1,5 @@ from oslo.config import cfg -import st2common.config as config from st2common.models.db import db_setup from st2actions.runners.pythonrunner import Action from st2common.persistence import reactor @@ -16,13 +15,7 @@ def __init__(self, config=None): self.initialize() def initialize(self): - # 1. Parse config - try: - config.parse_args() - except: - pass - - # 2. Setup db connection + # 1. Setup db connection username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None db_setup(cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port, From 80de397cc61ccff3e57ffac99b6dcc18ab626cab Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 19 Mar 2015 20:14:01 +0100 Subject: [PATCH 12/19] Update changelog. --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c0de6fb57c..d50e4fcbfb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,8 @@ v0.8.3 - TBD * Don't allow ``run-remote-script`` actions without an ``entry_point`` attribute - throw an exception when running an action. (improvement) +* Fix ``packs.setup_virtualenv`` command so it works correctly if user specified multiple packs + search paths. (bug-fix) v0.8.2 - March 10, 2015 ----------------------- From e81e57140fd1010feaa2f2e05df77b370eb92087 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 19 Mar 2015 20:36:25 +0100 Subject: [PATCH 13/19] Update affected / failing test. --- st2actions/tests/unit/test_pythonrunner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/st2actions/tests/unit/test_pythonrunner.py b/st2actions/tests/unit/test_pythonrunner.py index e3b3b37c6a..e996830de3 100644 --- a/st2actions/tests/unit/test_pythonrunner.py +++ b/st2actions/tests/unit/test_pythonrunner.py @@ -30,7 +30,13 @@ PACAL_ROW_ACTION_PATH = os.path.join(tests_base.get_resources_path(), 'packs', 'pythonactions/actions/pascal_row.py') +# Note: runner inherits parent args which doesn't work with tests since test pass additional +# unrecognized args +mock_sys = mock.Mock() +mock_sys.argv = [] + +@mock.patch('st2actions.runners.pythonrunner.sys', mock_sys) class PythonRunnerTestCase(TestCase): @classmethod From b5d7d57b395987adeac74ff14d0220779cb3f42e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 20 Mar 2015 14:31:43 +0100 Subject: [PATCH 14/19] Update sensor to use "auth.api_url" setting when talking to the API. This way it also works correctly if sensor_container and sensors run on a different host. --- st2common/st2common/util/api.py | 27 ++++++++++++++++--- .../st2reactor/container/process_container.py | 4 +-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/st2common/st2common/util/api.py b/st2common/st2common/util/api.py index c433eae091..77117d1724 100644 --- a/st2common/st2common/util/api.py +++ b/st2common/st2common/util/api.py @@ -16,17 +16,36 @@ from oslo.config import cfg from st2common.constants.api import DEFAULT_API_VERSION +from st2common.util.url import get_url_without_trailing_slash __all__ = [ - 'get_full_api_url' + 'get_base_public_api_url', + 'get_full_public_api_url' ] -def get_full_api_url(api_version=DEFAULT_API_VERSION): +def get_base_public_api_url(): """ - Return full URL to the API endpoint. + Return full public URL to the API endpoint (excluding the API version). :rtype: ``str`` """ - api_url = 'http://%s:%s/%s' % (cfg.CONF.api.host, cfg.CONF.api.port, api_version) + # Note: This is here for backward compatibility reasons - if api_url is not set we fall back + # to the old approach (using api listen host and port) + if cfg.CONF.auth.api_url: + api_url = get_url_without_trailing_slash(cfg.CONF.auth.api_url) + else: + api_url = 'http://%s:%s' % (cfg.CONF.api.host, cfg.CONF.api.port) + + return api_url + + +def get_full_public_api_url(api_version=DEFAULT_API_VERSION): + """ + Return full public URL to the API endpoint (including the API version). + + :rtype: ``str`` + """ + api_url = get_base_public_api_url() + api_url = '%s/%s' % (api_url, api_version) return api_url diff --git a/st2reactor/st2reactor/container/process_container.py b/st2reactor/st2reactor/container/process_container.py index 8bd5ca74bc..93caa1e101 100644 --- a/st2reactor/st2reactor/container/process_container.py +++ b/st2reactor/st2reactor/container/process_container.py @@ -25,7 +25,7 @@ from st2common.constants.system import AUTH_TOKEN_ENV_VARIABLE_NAME from st2common.constants.error_messages import PACK_VIRTUALENV_DOESNT_EXIST from st2common.services.access import create_token -from st2common.util.api import get_full_api_url +from st2common.util.api import get_full_public_api_url from st2common.util.sandboxing import get_sandbox_python_path from st2common.util.sandboxing import get_sandbox_python_binary_path from st2common.util.sandboxing import get_sandbox_virtualenv_path @@ -200,7 +200,7 @@ def _spawn_sensor_process(self, sensor): ttl = (24 * 60 * 60) temporary_token = create_token(username='sensors_container', ttl=ttl) - env[API_URL_ENV_VARIABLE_NAME] = get_full_api_url() + env[API_URL_ENV_VARIABLE_NAME] = get_full_public_api_url() env[AUTH_TOKEN_ENV_VARIABLE_NAME] = temporary_token.token # TODO 1: Purge temporary token when service stops or sensor process dies From d02b6dc15e1d2c223b972e622ad36ea6aba4ec87 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 20 Mar 2015 14:50:36 +0100 Subject: [PATCH 15/19] Add tests for it. --- st2common/tests/unit/test_util_api.py | 68 +++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 st2common/tests/unit/test_util_api.py diff --git a/st2common/tests/unit/test_util_api.py b/st2common/tests/unit/test_util_api.py new file mode 100644 index 0000000000..d0ec7ccd26 --- /dev/null +++ b/st2common/tests/unit/test_util_api.py @@ -0,0 +1,68 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest2 + +from oslo.config import cfg + +from st2common.constants.api import DEFAULT_API_VERSION +from st2common.util.api import get_base_public_api_url +from st2common.util.api import get_full_public_api_url +from st2tests.config import parse_args +parse_args() + + +class APIUtilsTestCase(unittest2.TestCase): + def test_get_base_public_api_url(self): + values = [ + 'http://foo.bar.com', + 'http://foo.bar.com/', + 'http://foo.bar.com:8080', + 'http://foo.bar.com:8080/', + 'http://localhost:8080/', + ] + expected = [ + 'http://foo.bar.com', + 'http://foo.bar.com', + 'http://foo.bar.com:8080', + 'http://foo.bar.com:8080', + 'http://localhost:8080', + ] + + for mock_value, expected_result in zip(values, expected): + cfg.CONF.auth.api_url = mock_value + actual = get_base_public_api_url() + self.assertEqual(actual, expected_result) + + def test_get_full_public_api_url(self): + values = [ + 'http://foo.bar.com', + 'http://foo.bar.com/', + 'http://foo.bar.com:8080', + 'http://foo.bar.com:8080/', + 'http://localhost:8080/', + ] + expected = [ + 'http://foo.bar.com/' + DEFAULT_API_VERSION, + 'http://foo.bar.com/' + DEFAULT_API_VERSION, + 'http://foo.bar.com:8080/' + DEFAULT_API_VERSION, + 'http://foo.bar.com:8080/' + DEFAULT_API_VERSION, + 'http://localhost:8080/' + DEFAULT_API_VERSION, + ] + + for mock_value, expected_result in zip(values, expected): + cfg.CONF.auth.api_url = mock_value + actual = get_full_public_api_url() + self.assertEqual(actual, expected_result) From 983504e111c1477eecfa973faa733b71f9e9fdb8 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 20 Mar 2015 14:52:05 +0100 Subject: [PATCH 16/19] Update changelog. --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d50e4fcbfb..57b939714a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,9 @@ v0.8.3 - TBD exception when running an action. (improvement) * Fix ``packs.setup_virtualenv`` command so it works correctly if user specified multiple packs search paths. (bug-fix) +* Update sensor container to use ``auth.api_url`` setting when talking to the API (e.g. when + accessing a datastore, etc.). This way it also works correctly if sensor container is running + on a different host than the API. (bug-fix) v0.8.2 - March 10, 2015 ----------------------- From 46558d48db82abd42df79e041bcf6b38a8fc9eb3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 20 Mar 2015 15:00:51 +0100 Subject: [PATCH 17/19] Remove old and unused setting. --- conf/st2.conf | 1 - st2debug/tests/integration/fixtures/configs/st2.conf | 1 - st2tests/conf/st2.conf | 1 - 3 files changed, 3 deletions(-) diff --git a/conf/st2.conf b/conf/st2.conf index 37c611fa6e..8f94b760e9 100644 --- a/conf/st2.conf +++ b/conf/st2.conf @@ -10,7 +10,6 @@ serve_webui_files = True # allow_origin = http://myhost1.example.com:3000,http://myhost2.example.com:3000 [sensorcontainer] -actionexecution_base_url = http://0.0.0.0:9101/actionexecutions logging = st2reactor/conf/logging.sensorcontainer.conf [rulesengine] diff --git a/st2debug/tests/integration/fixtures/configs/st2.conf b/st2debug/tests/integration/fixtures/configs/st2.conf index 2f00b7ee40..0f64c1fc2d 100644 --- a/st2debug/tests/integration/fixtures/configs/st2.conf +++ b/st2debug/tests/integration/fixtures/configs/st2.conf @@ -17,7 +17,6 @@ password = ponies url = ponies [sensorcontainer] -actionexecution_base_url = http://0.0.0.0:9101/actionexecutions logging = st2reactor/conf/logging.sensorcontainer.conf [rulesengine] diff --git a/st2tests/conf/st2.conf b/st2tests/conf/st2.conf index 345b983a4b..223863e31a 100644 --- a/st2tests/conf/st2.conf +++ b/st2tests/conf/st2.conf @@ -9,7 +9,6 @@ logging = st2api/conf/logging.conf # allow_origin = http://myhost1.example.com:3000,http://myhost2.example.com:3000 [sensorcontainer] -actionexecution_base_url = http://0.0.0.0:9101/actionexecutions logging = st2reactor/conf/logging.sensorcontainer.conf [rulesengine] From fe09b8787b2290d0d55aa1915bb2ccee536b7110 Mon Sep 17 00:00:00 2001 From: manasdk Date: Mon, 23 Mar 2015 14:37:49 -0700 Subject: [PATCH 18/19] update version --- st2client/st2client/__init__.py | 2 +- st2common/st2common/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/st2client/st2client/__init__.py b/st2client/st2client/__init__.py index fc9476188b..700cb3da27 100644 --- a/st2client/st2client/__init__.py +++ b/st2client/st2client/__init__.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.8.2' +__version__ = '0.8.3' diff --git a/st2common/st2common/__init__.py b/st2common/st2common/__init__.py index fc9476188b..700cb3da27 100644 --- a/st2common/st2common/__init__.py +++ b/st2common/st2common/__init__.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.8.2' +__version__ = '0.8.3' From 73e3910ceb618b2615fe1d52f451d0880a534836 Mon Sep 17 00:00:00 2001 From: manasdk Date: Mon, 23 Mar 2015 14:47:40 -0700 Subject: [PATCH 19/19] Add date to CHANGELOG --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 57b939714a..d869301e9e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,8 +1,8 @@ Changelog ========= -v0.8.3 - TBD ------------- +v0.8.3 - March 23, 2015 +----------------------- * Don't allow ``run-remote-script`` actions without an ``entry_point`` attribute - throw an exception when running an action. (improvement)