diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0730264..8ba8db4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,6 +20,9 @@ jobs: cache: "poetry" - name: Install dependencies run: poetry install --no-interaction --no-root --with dev - - uses: psf/black@stable - - name: Run pyright + - name: Check code formatting + run: poetry run black . + - name: Lint code run: poetry run pyright + - name: Run tests + run: poetry run pytest diff --git a/README.md b/README.md index ce72840..ed8a01c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ +![GitHub_headerImage](https://user-images.githubusercontent.com/3262610/221191767-73b8a8d9-9f8b-440e-8ab6-75cb3c82f2bc.png) + # autometrics-py -A Python decorator that makes it easy to understand the error rate, response time, and production usage of any function in your code. Jump straight from your IDE to live Prometheus charts for each HTTP/RPC handler, database method, or other piece of application logic. +A Python library that exports a decorator that makes it easy to understand the error rate, response time, and production usage of any function in your code. Jump straight from your IDE to live Prometheus charts for each HTTP/RPC handler, database method, or other piece of application logic. Autometrics for Python provides: @@ -18,8 +20,7 @@ See [Why Autometrics?](/~https://github.com/autometrics-dev#why-autometrics) for m - 🔗 Create links to live Prometheus charts directly into each functions docstrings (with tooltips coming soon!) - 📊 (Coming Soon!) Grafana dashboard showing the performance of all instrumented functions -- 🚨 (Coming Soon!) Generates Prometheus alerting rules using SLO best practices - from simple annotations in your code +- 🚨 Enable Prometheus alerts using SLO best practices from simple annotations in your code - ⚡ Minimal runtime overhead ## Using autometrics-py @@ -45,15 +46,96 @@ def sayHello: > Note that we cannot support tooltips without a VSCode extension due to behavior of the [static analyzer](/~https://github.com/davidhalter/jedi/issues/1921) used in VSCode. +## Alerts / SLOs + +Autometrics makes it easy to add Prometheus alerts using Service-Level Objectives (SLOs) to a function or group of functions. + +In order to receive alerts you need to add a set of rules to your Prometheus set up. You can find out more about those rules here: [Prometheus alerting rules](/~https://github.com/autometrics-dev/autometrics-shared#prometheus-recording--alerting-rules). Once added, most of the recording rules are dormant. They are enabled by specific metric labels that can be automatically attached by autometrics. + +To use autometrics SLOs and alerts, create one or multiple `Objective`s based on the function(s) success rate and/or latency, as shown below. The `Objective` can be passed as an argument to the `autometrics` macro to include the given function in that objective. + +```python +from autometrics import autometrics +from autometrics.objectives import Objective, ObjectiveLatency, ObjectivePercentile + +API_SLO = Objective( + "random", + success_rate=ObjectivePercentile.P99_9, + latency=(ObjectiveLatency.Ms250, ObjectivePercentile.P99), +) + +@autometrics(objective=API_SLO) +def api_handler(): + # ... +``` + +Autometrics by default will try to store information on which function calls a decorated function. As such you may want to place the autometrics in the top/first decorator, as otherwise you may get `inner` or `wrapper` as the caller function. + +So instead of writing: + +```py +from functools import wraps +from typing import Any, TypeVar, Callable + +R = TypeVar("R") + +def noop(func: Callable[..., R]) -> Callable[..., R]: + """A noop decorator that does nothing.""" + + @wraps(func) + def inner(*args: Any, **kwargs: Any) -> Any: + return func(*args, **kwargs) + + return inner + +@noop +@autometrics +def api_handler(): + # ... +``` + +You may want to switch the order of the decorator + +```py +@autometrics +@noop +def api_handler(): + # ... +``` + ## Development of the package This package uses [poetry](https://python-poetry.org) as a package manager, with all dependencies separated into three groups: - - root level dependencies, required - - `dev`, everything that is needed for development or in ci - - `examples`, dependencies of everything in `examples/` directory + +- root level dependencies, required +- `dev`, everything that is needed for development or in ci +- `examples`, dependencies of everything in `examples/` directory By default, poetry will only install required dependencies, if you want to run examples, install using this command: -`poetry install --with examples` +```sh +poetry install --with examples +``` + +Code in this repository is: + +- formatted using [black](https://black.readthedocs.io/en/stable/). +- contains type definitions (which are linted by [pyright](https://microsoft.github.io/pyright/)) +- tested using [pytest](https://docs.pytest.org/) + +In order to run these tools locally you have to install them, you can install them using poetry: -Code in this repository is formatted using [black](https://black.readthedocs.io/en/stable/) and contains type definitions (which are linted by [pyright](https://microsoft.github.io/pyright/)) +```sh +poetry install --with dev +``` + +After that you can run the tools individually + +```sh +# Formatting using black +poetry run black . +# Lint using pyright +poetry run pyright +# Run the tests using pytest +poetry run pytest +``` diff --git a/examples/example.py b/examples/example.py index ce2384f..f60cbea 100644 --- a/examples/example.py +++ b/examples/example.py @@ -1,7 +1,8 @@ -from prometheus_client import start_http_server -from autometrics import autometrics import time import random +from prometheus_client import start_http_server +from autometrics import autometrics +from autometrics.objectives import Objective, ObjectiveLatency, ObjectivePercentile # Defines a class called `Operations`` that has two methods: @@ -36,6 +37,24 @@ def div_unhandled(num1, num2): return result +RANDOM_SLO = Objective( + "random", + success_rate=ObjectivePercentile.P99_9, + latency=(ObjectiveLatency.Ms250, ObjectivePercentile.P99), +) + + +@autometrics(objective=RANDOM_SLO) +def random_error(): + """This function will randomly return an error or ok.""" + + result = random.choice(["ok", "error"]) + if result == "error": + time.sleep(1) + raise RuntimeError("random error") + return result + + ops = Operations() # Show the docstring (with links to prometheus metrics) for the `add` method @@ -59,3 +78,4 @@ def div_unhandled(num1, num2): time.sleep(2) # Call `div_unhandled` such that it raises an error div_unhandled(2, 0) + random_error() diff --git a/poetry.lock b/poetry.lock index 55f171d..c04093e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -21,6 +21,56 @@ doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"] trio = ["trio (>=0.16,<0.22)"] +[[package]] +name = "black" +version = "23.3.0" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"}, + {file = "black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"}, + {file = "black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"}, + {file = "black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"}, + {file = "black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"}, + {file = "black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"}, + {file = "black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"}, + {file = "black-23.3.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b"}, + {file = "black-23.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2"}, + {file = "black-23.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331"}, + {file = "black-23.3.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5"}, + {file = "black-23.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961"}, + {file = "black-23.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"}, + {file = "black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"}, + {file = "black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"}, + {file = "black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"}, + {file = "black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"}, + {file = "black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + [[package]] name = "bleach" version = "6.0.0" @@ -319,6 +369,21 @@ files = [ {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] +[[package]] +name = "exceptiongroup" +version = "1.1.1" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, +] + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "fastapi" version = "0.95.0" @@ -404,6 +469,18 @@ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "jaraco-classes" version = "3.2.3" @@ -513,11 +590,23 @@ files = [ {file = "more_itertools-9.1.0-py3-none-any.whl", hash = "sha256:d2bc7f02446e86a68911e58ded76d6561eea00cddfb2a91e7019bbb586c799f3"}, ] +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +category = "dev" +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + [[package]] name = "nodeenv" version = "1.7.0" description = "Node.js virtual environment builder" -category = "main" +category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -540,6 +629,18 @@ files = [ {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, ] +[[package]] +name = "pathspec" +version = "0.11.1" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, + {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, +] + [[package]] name = "pkginfo" version = "1.9.6" @@ -555,6 +656,38 @@ files = [ [package.extras] testing = ["pytest", "pytest-cov"] +[[package]] +name = "platformdirs" +version = "3.2.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.2.0-py3-none-any.whl", hash = "sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e"}, + {file = "platformdirs-3.2.0.tar.gz", hash = "sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08"}, +] + +[package.extras] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "prometheus-client" version = "0.16.0" @@ -669,7 +802,7 @@ tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} name = "pyright" version = "1.1.302" description = "Command line wrapper for pyright" -category = "main" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -684,6 +817,29 @@ nodeenv = ">=1.6.0" all = ["twine (>=3.4.1)"] dev = ["twine (>=3.4.1)"] +[[package]] +name = "pytest" +version = "7.3.0" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.3.0-py3-none-any.whl", hash = "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"}, + {file = "pytest-7.3.0.tar.gz", hash = "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + [[package]] name = "python-dotenv" version = "1.0.0" @@ -823,7 +979,7 @@ jeepney = ">=0.6" name = "setuptools" version = "67.6.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -918,7 +1074,7 @@ urllib3 = ">=1.26.0" name = "typing-extensions" version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -993,4 +1149,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "90aeb247a71d500a324409a2d7ae33e9d8150665f5851d8e0428f4905f146f20" +content-hash = "ac28fcba5ed105020dd055b90b69c98b3f0efa34f8f3039efbb3c0015e89c88d" diff --git a/pyproject.toml b/pyproject.toml index c06785b..5dbdeef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,12 +18,15 @@ packages = [{include = "autometrics", from = "src"}] python = "^3.8" prometheus-client = "0.16.0" python-dotenv = "1.0.0" +typing-extensions = "^4.5.0" [tool.poetry.group.dev] optional = true [tool.poetry.group.dev.dependencies] pyright = "^1.1.302" +pytest = "^7.3.0" +black = "^23.3.0" [tool.poetry.group.examples] optional = true @@ -59,7 +62,6 @@ six = "1.16.0" sniffio = "1.3.0" starlette = "0.26.1" twine = "4.0.2" -typing-extensions = "4.5.0" urllib3 = "1.26.15" uvicorn = "0.21.1" webencodings = "0.5.1" diff --git a/src/autometrics/constants.py b/src/autometrics/constants.py new file mode 100644 index 0000000..14c6fbd --- /dev/null +++ b/src/autometrics/constants.py @@ -0,0 +1,15 @@ +"""Constants used by autometrics""" + +COUNTER_DESCRIPTION = "Autometrics counter for tracking function calls" +HISTOGRAM_DESCRIPTION = "Autometrics histogram for tracking function call duration" + +# The following constants are used to create the labels +OBJECTIVE_NAME = "objective.name".replace(".", "_") +OBJECTIVE_PERCENTILE = "objective.percentile".replace(".", "_") +OBJECTIVE_LATENCY_THRESHOLD = "objective.latency_threshold".replace(".", "_") + +# The values are updated to use underscores instead of periods to avoid issues with prometheus. +# A similar thing is done in the rust library, which supports multiple exporters +OBJECTIVE_NAME_PROMETHEUS = OBJECTIVE_NAME.replace(".", "_") +OBJECTIVE_PERCENTILE_PROMETHEUS = OBJECTIVE_PERCENTILE.replace(".", "_") +OBJECTIVE_LATENCY_THRESHOLD_PROMETHEUS = OBJECTIVE_LATENCY_THRESHOLD.replace(".", "_") diff --git a/src/autometrics/decorator.py b/src/autometrics/decorator.py index be218d8..3fba103 100644 --- a/src/autometrics/decorator.py +++ b/src/autometrics/decorator.py @@ -1,72 +1,64 @@ -from collections.abc import Callable -from prometheus_client import Counter, Histogram, Gauge +"""Autometrics module.""" import time -import inspect -from .prometheus_url import Generator -import os from functools import wraps -from typing import Any, TypeVar +from typing import overload, TypeVar, Callable, Optional +from typing_extensions import ParamSpec +from .objectives import Objective +from .emit import count, histogram, Result +from .utils import get_module_name, get_caller_function, write_docs -prom_counter = Counter( - "function_calls_count", "query??", ["function", "module", "result", "caller"] -) -prom_histogram = Histogram("function_calls_duration", "query??", ["function", "module"]) -R = TypeVar("R") +P = ParamSpec("P") +T = TypeVar("T") -def autometrics(func: Callable) -> Callable: - func_name = func.__name__ - fullname = func.__qualname__ - filename = get_filename_as_module(func) - if fullname == func_name: - module_name = filename - else: - classname = func.__qualname__.rsplit(".", 1)[0] - module_name = f"{filename}.{classname}" +# Bare decorator usage +@overload +def autometrics(func: Callable[P, T]) -> Callable[P, T]: + ... - @wraps(func) - def wrapper(*args, **kwargs): - func_name = func.__name__ - start_time = time.time() - caller = get_caller_function() - try: - result = func(*args, **kwargs) - prom_counter.labels(func_name, module_name, "ok", caller).inc() - except Exception as e: - result = e.__class__.__name__ - prom_counter.labels(func_name, module_name, "error", caller).inc() - duration = time.time() - start_time - prom_histogram.labels(func_name, module_name).observe(duration) - return result - if func.__doc__ is not None: - wrapper.__doc__ = f"{func.__doc__}\n{write_docs(func_name, module_name)}" - else: - wrapper.__doc__ = write_docs(func_name, module_name) - return wrapper +# Decorator with arguments +@overload +def autometrics(*, objective: Optional[Objective] = None) -> Callable: + ... -def get_filename_as_module(func: Callable) -> str: - fullpath = inspect.getsourcefile(func) - if fullpath == None: - return "" - filename = os.path.basename(fullpath) - module_part = os.path.splitext(filename)[0] - return module_part +def autometrics( + func: Optional[Callable] = None, + *, + objective: Optional[Objective] = None, +): + """Decorator for tracking function calls and duration.""" + def decorator(func: Callable[P, T]) -> Callable[P, T]: + module_name = get_module_name(func) + func_name = func.__name__ + + @wraps(func) + def wrapper(*args: P.args, **kwds: P.kwargs) -> T: + start_time = time.time() + caller = get_caller_function() -def write_docs(func_name: str, module_name: str): - g = Generator(func_name, module_name) - urls = g.createURLs() - docs = f"Prometheus Query URLs for Function - {func_name} and Module - {module_name}: \n\n" - for key, value in urls.items(): - docs = f"{docs}{key} : {value} \n\n" - docs = f"{docs}-------------------------------------------\n" - return docs + try: + result = func(*args, **kwds) + count(func_name, module_name, caller, objective, result=Result.OK) + histogram(func_name, module_name, start_time, objective) + except Exception as exception: + result = exception.__class__.__name__ + count(func_name, module_name, caller, objective, result=Result.ERROR) + histogram(func_name, module_name, start_time, objective) + # Reraise exception + raise exception + return result + if func.__doc__ is None: + wrapper.__doc__ = write_docs(func_name, module_name) + else: + wrapper.__doc__ = f"{func.__doc__}\n{write_docs(func_name, module_name)}" + return wrapper -def get_caller_function(): - caller_frame = inspect.stack()[2] - caller_function_name = caller_frame[3] - return caller_function_name + if func is None: + return decorator + else: + return decorator(func) diff --git a/src/autometrics/emit.py b/src/autometrics/emit.py new file mode 100644 index 0000000..f16e240 --- /dev/null +++ b/src/autometrics/emit.py @@ -0,0 +1,96 @@ +import time +from enum import Enum +from typing import Optional +from prometheus_client import Counter, Histogram + +from .constants import ( + COUNTER_DESCRIPTION, + HISTOGRAM_DESCRIPTION, + OBJECTIVE_NAME_PROMETHEUS, + OBJECTIVE_PERCENTILE_PROMETHEUS, + OBJECTIVE_LATENCY_THRESHOLD_PROMETHEUS, +) +from .objectives import Objective + +prom_counter = Counter( + "function_calls_count", + COUNTER_DESCRIPTION, + [ + "function", + "module", + "result", + "caller", + OBJECTIVE_NAME_PROMETHEUS, + OBJECTIVE_PERCENTILE_PROMETHEUS, + ], +) +prom_histogram = Histogram( + "function_calls_duration", + HISTOGRAM_DESCRIPTION, + [ + "function", + "module", + OBJECTIVE_NAME_PROMETHEUS, + OBJECTIVE_PERCENTILE_PROMETHEUS, + OBJECTIVE_LATENCY_THRESHOLD_PROMETHEUS, + ], +) + + +class Result(Enum): + """Result of the function call.""" + + OK = "ok" + ERROR = "error" + + +def count( + func_name: str, + module_name: str, + caller: str, + objective: Optional[Objective] = None, + result: Result = Result.OK, +): + """Increment the counter for the function call.""" + + objective_name = "" if objective is None else objective.name + percentile = ( + "" + if objective is None or objective.success_rate is None + else objective.success_rate.value + ) + + prom_counter.labels( + func_name, + module_name, + result.value, + caller, + objective_name, + percentile, + ).inc() + + +def histogram( + func_name: str, + module_name: str, + start_time: float, + objective: Optional[Objective] = None, +): + """Observe the duration of the function call.""" + duration = time.time() - start_time + + objective_name = "" if objective is None else objective.name + latency = None if objective is None else objective.latency + percentile = "" + threshold = "" + if latency is not None: + threshold = latency[0].value + percentile = latency[1].value + + prom_histogram.labels( + func_name, + module_name, + objective_name, + percentile, + threshold, + ).observe(duration) diff --git a/src/autometrics/objectives.py b/src/autometrics/objectives.py new file mode 100644 index 0000000..73e89ca --- /dev/null +++ b/src/autometrics/objectives.py @@ -0,0 +1,91 @@ +from enum import Enum +from typing import Optional, Tuple + + +class ObjectivePercentile(Enum): + """The percentage of requests that must meet the given criteria (success rate or latency).""" + + P90 = "90" + P95 = "95" + P99 = "99" + P99_9 = "99.9" + + +class ObjectiveLatency(Enum): + """The latency threshold for the given percentile.""" + + Ms5 = "0.005" + Ms10 = "0.01" + Ms25 = "0.025" + Ms50 = "0.05" + Ms75 = "0.075" + Ms100 = "0.1" + Ms250 = "0.25" + Ms500 = "0.5" + Ms750 = "0.75" + Ms1000 = "1.0" + Ms2500 = "2.5" + Ms5000 = "5.0" + Ms7500 = "7.5" + Ms10000 = "10.0" + + +# This represents a Service-Level Objective (SLO) for a function or group of functions. +# The objective should be given a descriptive name and can represent +# a success rate and/or latency objective. +# +# For details on SLOs, see +# +# Example: +# ```python +# from autometrics import autometrics +# from autometrics.objectives import Objective, ObjectivePercentile, ObjectiveLatency +# API_SLO = Objective( +# "api", +# success_rate=ObjectivePercentile.P99_9, +# latency=(ObjectiveLatency.Ms250, ObjectivePercentile.P99), +# ) +# +# @autometrics(objective = API_SLO) +# def api_handler() : +# # ... +# ``` +# +# ## How this works +# +# When an objective is added to a function, the metrics for that function will +# have additional labels attached to specify the SLO details. +# +# Autometrics comes with a set of Prometheus [recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) +# and [alerting rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) +# that will fire alerts when the given objective is being violated. +# +# By default, these recording rules will effectively lay dormant. +# However, they are enabled when the special labels are present on certain metrics. +class Objective: + """A Service-Level Objective (SLO) for a function or group of functions.""" + + name: str + """name: The name of the objective. This should be something descriptive of the function or group of functions it covers.""" + success_rate: Optional[ObjectivePercentile] + """Specify the success rate for this objective. + + This means that the function or group of functions that are part of this objective + should return an `Ok` result at least this percentage of the time.""" + latency: Optional[Tuple[ObjectiveLatency, ObjectivePercentile]] + + def __init__( + self, + name: str, + success_rate: Optional[ObjectivePercentile] = None, + latency: Optional[Tuple[ObjectiveLatency, ObjectivePercentile]] = None, + ): + """Create a new objective with the given name. + + The name should be something descriptive of the function or group of functions it covers. + For example, if you have an objective covering all of the HTTP handlers in your API you might call it "api". + """ + + self.name = name + self.success_rate = success_rate + self.latency = latency diff --git a/src/autometrics/prometheus_url.py b/src/autometrics/prometheus_url.py index 96e4bbd..1f21f1a 100644 --- a/src/autometrics/prometheus_url.py +++ b/src/autometrics/prometheus_url.py @@ -1,38 +1,48 @@ import urllib.parse import os +from typing import Optional from dotenv import load_dotenv +def cleanup_url(url: str) -> str: + """Remove the trailing slash if there is one.""" + if url[-1] == "/": + url = url[:-1] + return url + + class Generator: - def __init__(self, functionName, moduleName, baseUrl=None): + """Generate prometheus query urls for a given function/module.""" + + def __init__( + self, function_name: str, module_name: str, base_url: Optional[str] = None + ): load_dotenv() - self.functionName = functionName - self.moduleName = moduleName - self.baseUrl = baseUrl or os.getenv("PROMETHEUS_URL") - if self.baseUrl is None: - self.baseUrl = "http://localhost:9090" - elif self.baseUrl[-1] == "/": - self.baseUrl = self.baseUrl[ - :-1 - ] # Remove the trailing slash if there is one - - def createURLs(self): - requestRateQuery = f'sum by (function, module) (rate (function_calls_count_total{{function="{self.functionName}",module="{self.moduleName}"}}[5m]))' - latencyQuery = f'sum by (le, function, module) (rate(function_calls_duration_bucket{{function="{self.functionName}",module="{self.moduleName}"}}[5m]))' - errorRatioQuery = f'sum by (function, module) (rate (function_calls_count_total{{function="{self.functionName}",module="{self.moduleName}", result="error"}}[5m])) / {requestRateQuery}' - - queries = [requestRateQuery, latencyQuery, errorRatioQuery] + self.function_name = function_name + self.module_name = module_name + + url = base_url or os.getenv("PROMETHEUS_URL") or "http://localhost:9090" + self.base_url = cleanup_url(url) + + def create_urls(self): + """Create the prometheus query urls for the function and module.""" + request_rate_query = f'sum by (function, module) (rate (function_calls_count_total{{function="{self.function_name}",module="{self.module_name}"}}[5m]))' + latency_query = f'sum by (le, function, module) (rate(function_calls_duration_bucket{{function="{self.function_name}",module="{self.module_name}"}}[5m]))' + error_ratio_query = f'sum by (function, module) (rate (function_calls_count_total{{function="{self.function_name}",module="{self.module_name}", result="error"}}[5m])) / {request_rate_query}' + + queries = [request_rate_query, latency_query, error_ratio_query] names = ["Request rate URL", "Latency URL", "Error Ratio URL"] urls = {} - for n in names: + for name in names: for query in queries: - generateUrl = self.createPrometheusUrl(query) - urls[n] = generateUrl + generated_url = self.create_prometheus_url(query) + urls[name] = generated_url queries.remove(query) break return urls - def createPrometheusUrl(self, query): - urlEncode = urllib.parse.quote(query) - url = f"{self.baseUrl}/graph?g0.expr={urlEncode}&g0.tab=0" + def create_prometheus_url(self, query: str): + """Create a the full query url for a given query.""" + encoded_query = urllib.parse.quote(query) + url = f"{self.base_url}/graph?g0.expr={encoded_query}&g0.tab=0" return url diff --git a/src/autometrics/test_decorator.py b/src/autometrics/test_decorator.py new file mode 100644 index 0000000..c4c767f --- /dev/null +++ b/src/autometrics/test_decorator.py @@ -0,0 +1,140 @@ +"""Test the autometrics decorator.""" +from prometheus_client.exposition import generate_latest +from prometheus_client import registry, Metric +from pytest import raises + +from .decorator import autometrics +from .objectives import ObjectiveLatency, Objective, ObjectivePercentile +from .utils import get_caller_function + + +def basic_function(): + """This is a basic function.""" + return True + + +def find_metric_with_name(metrics: "list[Metric]", name: str): + """Find a metric with a given name.""" + for metric in metrics: + if metric.name == name: + return metric + + return None + + +def test_basic(): + """This is a basic test.""" + + # set up the function + basic variables + caller = get_caller_function(depth=1) + assert caller is not None + assert caller != "" + function_name = basic_function.__name__ + wrapped_function = autometrics(basic_function) + wrapped_function() + + # get the metrics + blob = generate_latest(registry.REGISTRY) + assert blob is not None + data = blob.decode("utf-8") + + total_count = f"""function_calls_count_total{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="ok"}} 1.0""" + assert total_count in data + count_created = f"""function_calls_count_created{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="ok"}}""" + assert count_created in data + + for latency in ObjectiveLatency: + query = f"""function_calls_duration_bucket{{function="{function_name}",le="{latency.value}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert query in data + + duration_count = f"""function_calls_duration_count{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_count in data + + duration_sum = f"""function_calls_duration_sum{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_sum in data + + duration_created = f"""function_calls_duration_created{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_created in data + + +def test_objectives(): + """This is a test that covers objectives.""" + + # set up the function + objective variables + caller = get_caller_function(depth=1) + assert caller is not None + assert caller != "" + objective_name = "test_objective" + success_rate = ObjectivePercentile.P90 + latency = (ObjectiveLatency.Ms100, ObjectivePercentile.P99) + objective = Objective( + name=objective_name, success_rate=success_rate, latency=latency + ) + function_name = basic_function.__name__ + wrapped_function = autometrics(objective=objective)(basic_function) + + # call the function + wrapped_function() + + # get the metrics + blob = generate_latest(registry.REGISTRY) + assert blob is not None + data = blob.decode("utf-8") + + total_count = f"""function_calls_count_total{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="{objective_name}",objective_percentile="{success_rate.value}",result="ok"}} 1.0""" + assert total_count in data + count_created = f"""function_calls_count_created{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="{objective_name}",objective_percentile="{success_rate.value}",result="ok"}}""" + assert count_created in data + + for objective in ObjectiveLatency: + query = f"""function_calls_duration_bucket{{function="{function_name}",le="{objective.value}",module="test_decorator",objective_latency_threshold="{latency[0].value}",objective_name="{objective_name}",objective_percentile="{latency[1].value}"}}""" + assert query in data + + duration_count = f"""function_calls_duration_count{{function="{function_name}",module="test_decorator",objective_latency_threshold="{latency[0].value}",objective_name="{objective_name}",objective_percentile="{latency[1].value}"}}""" + assert duration_count in data + + duration_sum = f"""function_calls_duration_sum{{function="{function_name}",module="test_decorator",objective_latency_threshold="{latency[0].value}",objective_name="{objective_name}",objective_percentile="{latency[1].value}"}}""" + assert duration_sum in data + + duration_created = f"""function_calls_duration_created{{function="{function_name}",module="test_decorator",objective_latency_threshold="{latency[0].value}",objective_name="{objective_name}",objective_percentile="{latency[1].value}"}}""" + assert duration_created in data + + +def error_function(): + """This is a function that raises an error.""" + raise RuntimeError("This is a test error") + + +def test_exception(): + caller = get_caller_function(depth=1) + assert caller is not None + assert caller != "" + function_name = error_function.__name__ + wrapped_function = autometrics(error_function) + with raises(RuntimeError) as exception: + wrapped_function() + assert "This is a test error" in str(exception.value) + + # get the metrics + blob = generate_latest(registry.REGISTRY) + assert blob is not None + data = blob.decode("utf-8") + print("data", data) + + total_count = f"""function_calls_count_total{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="error"}} 1.0""" + assert total_count in data + count_created = f"""function_calls_count_created{{caller="{caller}",function="{function_name}",module="test_decorator",objective_name="",objective_percentile="",result="error"}}""" + assert count_created in data + + for latency in ObjectiveLatency: + query = f"""function_calls_duration_bucket{{function="{function_name}",le="{latency.value}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert query in data + + duration_count = f"""function_calls_duration_count{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_count in data + + duration_sum = f"""function_calls_duration_sum{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_sum in data + + duration_created = f"""function_calls_duration_created{{function="{function_name}",module="test_decorator",objective_latency_threshold="",objective_name="",objective_percentile=""}}""" + assert duration_created in data diff --git a/src/test_prometheus_url.py b/src/autometrics/test_prometheus_url.py similarity index 58% rename from src/test_prometheus_url.py rename to src/autometrics/test_prometheus_url.py index d9b7f65..a8c0c1d 100644 --- a/src/test_prometheus_url.py +++ b/src/autometrics/test_prometheus_url.py @@ -1,14 +1,17 @@ import unittest -from autometrics.prometheus_url import Generator +from .prometheus_url import Generator # Defaults to localhost:9090 class TestPrometheusUrlGeneratorDefault(unittest.TestCase): + """Test the prometheus url generator with default values.""" + def setUp(self): self.generator = Generator("myFunction", "myModule") - def test_createPrometheusUrl(self): - url = self.generator.createPrometheusUrl("myQuery") + def test_create_prometheus_url(self): + """Test that the prometheus url is created correctly.""" + url = self.generator.create_prometheus_url("myQuery") self.assertTrue( url.startswith("http://localhost:9090/graph?g0.expr=") ) # Make sure the base URL is correct @@ -17,18 +20,17 @@ def test_createPrometheusUrl(self): # Creates proper urls when given a custom base URL class TestPrometheusUrlGeneratorCustomUrl(unittest.TestCase): + """Test the prometheus url generator with a custom base URL.""" + def setUp(self): self.generator = Generator( - "myFunction", "myModule", baseUrl="http://localhost:9091" + "myFunction", "myModule", base_url="http://localhost:9091" ) - def test_createPrometheusUrl(self): - url = self.generator.createPrometheusUrl("myQuery") + def test_create_prometheus_url(self): + """Test that the prometheus url is created correctly.""" + url = self.generator.create_prometheus_url("myQuery") self.assertTrue( url.startswith("http://localhost:9091/graph?g0.expr=") ) # Make sure the base URL is correct self.assertIn("myQuery", url) # Make sure the query is included in the URL - - -if __name__ == "__main__": - unittest.main() diff --git a/src/autometrics/utils.py b/src/autometrics/utils.py new file mode 100644 index 0000000..1dd9cf4 --- /dev/null +++ b/src/autometrics/utils.py @@ -0,0 +1,47 @@ +import inspect +import os +from collections.abc import Callable +from .prometheus_url import Generator + + +def get_module_name(func: Callable) -> str: + """Get the name of the module that contains the function.""" + func_name = func.__name__ + fullname = func.__qualname__ + filename = get_filename_as_module(func) + if fullname == func_name: + return filename + + classname = func.__qualname__.rsplit(".", 1)[0] + return f"{filename}.{classname}" + + +def get_filename_as_module(func: Callable) -> str: + """Get the filename of the module that contains the function.""" + fullpath = inspect.getsourcefile(func) + if fullpath is None: + return "" + + filename = os.path.basename(fullpath) + module_part = os.path.splitext(filename)[0] + return module_part + + +def write_docs(func_name: str, module_name: str): + """Write the prometheus query urls to the function docstring.""" + generator = Generator(func_name, module_name) + docs = f"Prometheus Query URLs for Function - {func_name} and Module - {module_name}: \n\n" + + urls = generator.create_urls() + for key, value in urls.items(): + docs = f"{docs}{key} : {value} \n\n" + + docs = f"{docs}-------------------------------------------\n" + return docs + + +def get_caller_function(depth: int = 2): + """Get the name of the function. Default depth is 2 to get the caller of the caller of the function being decorated.""" + caller_frame = inspect.stack()[depth] + caller_function_name = caller_frame[3] + return caller_function_name