diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 821dc1d..4416259 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -11,12 +11,20 @@ on: jobs: unit_tests_linux: - name: "Python ${{ matrix.python-version }}" + name: ${{ matrix.name }} Python ${{ matrix.python }} runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: - # test minimum and maximum supported python version - python-version: ["3.10", "3.13"] + include: + - os: ubuntu-latest + python-version: "3.10" + - os: ubuntu-latest + python-version: "3.13" + - os: ubuntu-latest + python-version: "3.13" + pip-flags: "--pre" + name: PRE-RELEASE DEPENDENCIES env: # GitHub currently has 4 cores available for Linux runners @@ -39,14 +47,7 @@ jobs: run: sudo apt-get install -y libgit2-dev pandoc - name: "Install DSO" - run: pip install '.[test]' + run: pip install ${{ matrix.pip-flags }} '.[test]' - name: "Run unit tests" run: pytest -vv tests -n ${{ env.worker_cores }} --cov=dso --cov-report=xml - - # - name: "Upload coverage to Codecov" - # uses: codecov/codecov-action@v4 - # with: - # files: ./coverage.xml - # env: - # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 406b6f9..c156edc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,12 +19,16 @@ and this project adheres to [Semantic Versioning][]. - Python API that mirrors `dso-r` functionality (e.g. to be used from Jupyter notebooks) ([#30](/~https://github.com/Boehringer-Ingelheim/dso/pull/30)) +## Documentation + +- Various documentation updates, working towards the first public version of the docs. + ### Chore - Refactor CLI into separate module ([#30](/~https://github.com/Boehringer-Ingelheim/dso/pull/30)) - Defer imports in CLI until they are actually needed to speed up CLI ([#30](/~https://github.com/Boehringer-Ingelheim/dso/pull/30)) - Make all modules explicitly private that are not part of the public API ([#30](/~https://github.com/Boehringer-Ingelheim/dso/pull/30)) -- Relicense the package as LGPL-3.0-or-later, with an exception for the templates ([#76](/~https://github.com/Boehringer-Ingelheim/dso/pull/76)) +- Relicense the package as LGPL-3.0-or-later, with a more permissive exception for the templates ([#76](/~https://github.com/Boehringer-Ingelheim/dso/pull/76)) ## v0.10.1 diff --git a/docs/command_reference.md b/docs/cli_command_reference.md similarity index 100% rename from docs/command_reference.md rename to docs/cli_command_reference.md diff --git a/docs/cli_configuration.md b/docs/cli_configuration.md new file mode 100644 index 0000000..f022077 --- /dev/null +++ b/docs/cli_configuration.md @@ -0,0 +1,15 @@ +# Configuration + +This section provides and overview of dso settings and how to apply them. + +## pyproject.toml + +Project-specific dso settings can be set in the `pyproject.toml` file at the root of each project in the +`[tool.dso]` section: + +```toml +[tool.dso] +# whether to compile relative paths declared with `!path` into absolute paths or relative paths (relative to each stage). +# default is `true` +use_relative_path = true +``` diff --git a/docs/cli_installation.md b/docs/cli_installation.md new file mode 100644 index 0000000..7cd4222 --- /dev/null +++ b/docs/cli_installation.md @@ -0,0 +1,3 @@ +# Installation + +TODO diff --git a/docs/contributing.md b/docs/contributing.md index e2fceb6..2309c25 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -1,7 +1,6 @@ # Contributing guide We assume that you are already familiar with git and with making pull requests on GitHub. -If not, please refer to the [scanpy developer guide][]. ## Installing dev dependencies @@ -114,7 +113,6 @@ Please adhere to [Semantic Versioning][semver], in brief > > Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format. - [semver]: https://semver.org/ [managing GitHub releases]: https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository [pypi]: https://pypi.org/ diff --git a/docs/dso_python.md b/docs/dso_python.md deleted file mode 100644 index 10cafdc..0000000 --- a/docs/dso_python.md +++ /dev/null @@ -1,11 +0,0 @@ -# Python API - -TODO - -```{eval-rst} -.. module:: dso -.. currentmodule:: dso - -.. autosummary:: - :toctree: generated -``` diff --git a/docs/index.md b/docs/index.md index 29ae1fc..2100e37 100644 --- a/docs/index.md +++ b/docs/index.md @@ -15,7 +15,9 @@ getting_started.md :maxdepth: 1 :caption: DSO CLI -command_reference.md +cli_installation.md +cli_command_reference.md +cli_configuration.md CHANGELOG.md contributing.md ``` @@ -25,7 +27,8 @@ contributing.md :maxdepth: 1 :caption: DSO Python API -dso_python.md +python_usage.md +python_api.md ``` ```{toctree} diff --git a/docs/python_api.md b/docs/python_api.md new file mode 100644 index 0000000..e9a92c7 --- /dev/null +++ b/docs/python_api.md @@ -0,0 +1,6 @@ +# API reference + +```{eval-rst} +.. automodule:: dso + :members: +``` diff --git a/docs/python_usage.md b/docs/python_usage.md new file mode 100644 index 0000000..1bcb4f6 --- /dev/null +++ b/docs/python_usage.md @@ -0,0 +1,34 @@ +# Usage + +## Installation + +The DSO Python API is part of the DSO CLI package. See [installation](cli_installation.md) for more details. + +## Typical usage + +The purpose of the Python API is to provide covenient access to stage parameters from Python scripts or notebooks. +Using {func}`~dso.read_params` the `params.yaml` file of the specified stage is compiled and loaded +into a dictionary. The path must be specified relative to the project root -- this ensures that the correct stage is +found irrespective of the current working directory, as long as it the project root or any subdirectory thereof. +Only parameters that are declared as `params`, `dep`, or `output` in dvc.yaml are loaded to +ensure that one does not forget to keep the `dvc.yaml` updated. + +```python +from dso import read_params + +params = read_params("subfolder/my_stage") +``` + +By default, DSO compiles paths in configuration files to paths relative to each stage (see [configuration](cli_configuration.md#pyprojecttoml)). +From Python, you can use {func}`~dso.stage_here` to resolve paths +relative to the current stage independent of your current working directory, e.g. + +```python +import pandas as pd +from dso import stage_here + +pd.read_csv(stage_here(params["samplesheet"])) +``` + +This works, because `read_params` has stored the path of the current stage in a configuration object that persists in +the current Python session. `stage_here` can use this information to resolve relative paths. diff --git a/src/dso/__init__.py b/src/dso/__init__.py index 5160e86..02461e4 100644 --- a/src/dso/__init__.py +++ b/src/dso/__init__.py @@ -1,4 +1,4 @@ from ._metadata import __version__ # noqa -from .api import here, read_params, set_stage, stage_here +from .api import here, read_params, set_stage, stage_here, CONFIG -__all__ = ["read_params", "here", "stage_here", "set_stage"] +__all__ = ["read_params", "here", "stage_here", "set_stage", "CONFIG"] diff --git a/src/dso/api.py b/src/dso/api.py index 9924f86..108fd09 100644 --- a/src/dso/api.py +++ b/src/dso/api.py @@ -45,13 +45,13 @@ def stage_here(rel_path: str | Path | None = None) -> Path: """ Get the absolute path to the current stage - The current stage is stored in `dso.CONFIG` and can be set using `dso.set_stage` or - `dso.read_params`. + The current stage is stored in `dso.CONFIG` and can be set using :func:`dso.set_stage` or + :func:`dso.read_params`. Parameters ---------- rel_path - Relative path to be appended to the project root + Relative path to be appended to the stage root """ if CONFIG.stage_here is None: raise RuntimeError("No stage has been set. Run `read_params` or `set_stage` first!") @@ -89,6 +89,23 @@ def set_stage(stage: str | Path) -> None: def read_params(stage: str | Path) -> dict: - """Set stage dir and load parameters from params.yaml""" + """ + Set stage dir and load parameters from the stage's params.yaml + + It is required to provide the path of the current stage relative to the project root to ensure that + the correct config is loaded, no matter of the current working directory (as long as the working directory + is any subdirectory of the project root). The function recompiles params.in.yaml to params.yaml on-the-fly + to ensure that up-to-date params are always loaded. + + Only parameters that are declared as `params`, `dep`, or `output` in dvc.yaml are loaded to + ensure that one does not forget to keep the `dvc.yaml` updated. + + Calls :func:`~dso.set_stage` internally. + + Parameters + ---------- + stage + Path to stage, relative to the project root + """ set_stage(stage) return get_config(str(stage), skip_compile=bool(int(os.environ.get("DSO_SKIP_COMPILE", 0))))