From 7b6dbac1fa1e2c8abba1db6593f36c99fe077585 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 9 Aug 2025 13:11:26 -0500 Subject: [PATCH 1/5] fix: tweak log verbiage --- src/wuttatell/cli/tell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wuttatell/cli/tell.py b/src/wuttatell/cli/tell.py index 7a52dd4..1cbe1f6 100644 --- a/src/wuttatell/cli/tell.py +++ b/src/wuttatell/cli/tell.py @@ -58,7 +58,7 @@ def tell( telemetry = app.get_telemetry_handler() data = telemetry.collect_all_data(profile=profile) - log.info("data collected okay: %s", ', '.join(sorted(data))) + log.info("data collected for: %s", ', '.join(sorted(data))) log.debug("%s", data) if dry_run: From e0dd704247f8bd0e55de28c62a1950045bcd384b Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 10 Aug 2025 11:57:28 -0500 Subject: [PATCH 2/5] fix: remove `submit_uuid` from telemetry config profile the `submit_url` should be all we expect by default, since even that has no built-in behavior yet --- docs/narr/install.rst | 6 ++---- src/wuttatell/telemetry.py | 8 -------- tests/test_telemetry.py | 3 --- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/docs/narr/install.rst b/docs/narr/install.rst index f05fee2..aa41f30 100644 --- a/docs/narr/install.rst +++ b/docs/narr/install.rst @@ -12,15 +12,13 @@ Install the WuttaTell package to your virtual environment: pip install WuttaTell Edit your :term:`config file` to add telemetry submission info, and -related settings. Please note, the following example is just that - -and will not work as-is: +related settings: .. code-block:: ini [wutta.telemetry] default.collect_keys = os, python - default.submit_url = /nodes/telemetry - default.submit_uuid = 06897767-eb70-7790-8000-13f368a40ea3 + default.submit_url = https://example.com/api/my-node/telemetry .. note:: diff --git a/src/wuttatell/telemetry.py b/src/wuttatell/telemetry.py index 86d7de6..2194652 100644 --- a/src/wuttatell/telemetry.py +++ b/src/wuttatell/telemetry.py @@ -251,13 +251,6 @@ class TelemetryProfile(WuttaConfigProfile): .. attribute:: submit_url URL to which collected telemetry data should be submitted. - - .. attribute:: submit_uuid - - UUID identifying the record to update when submitting telemetry - data. This value will only make sense in the context of the - collection service responsible for receiving telemetry - submissions. """ @property @@ -270,4 +263,3 @@ class TelemetryProfile(WuttaConfigProfile): keys = self.get_str('collect.keys', default='os,python') self.collect_keys = self.config.parse_list(keys) self.submit_url = self.get_str('submit.url') - self.submit_uuid = self.get_str('submit.uuid') diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index c2880e0..ae90b97 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -187,13 +187,10 @@ class TestTelemetryProfile(ConfigTestCase): profile = self.make_profile() self.assertEqual(profile.collect_keys, ['os', 'python']) self.assertIsNone(profile.submit_url) - self.assertIsNone(profile.submit_uuid) # configured self.config.setdefault('wutta.telemetry.default.collect.keys', 'os,network,python') self.config.setdefault('wutta.telemetry.default.submit.url', '/nodes/telemetry') - self.config.setdefault('wutta.telemetry.default.submit.uuid', '06897669-272b-7c74-8000-4d311924d24f') profile = self.make_profile() self.assertEqual(profile.collect_keys, ['os', 'network', 'python']) self.assertEqual(profile.submit_url, '/nodes/telemetry') - self.assertEqual(profile.submit_uuid, '06897669-272b-7c74-8000-4d311924d24f') From 0f42626cb17e6553896c020a773c0a878c657532 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 10 Aug 2025 15:13:20 -0500 Subject: [PATCH 3/5] feat: add native API client for common submission use case this should work for simple API endpoint which requires an auth token. if more is needed, you'll have to override telemetry handler --- docs/api/wuttatell.client.rst | 6 + docs/conf.py | 1 + docs/index.rst | 1 + src/wuttatell/client.py | 197 +++++++++++++++++++++++++++++++++ src/wuttatell/telemetry.py | 13 ++- tests/test_client.py | 203 ++++++++++++++++++++++++++++++++++ tests/test_telemetry.py | 21 +++- 7 files changed, 437 insertions(+), 5 deletions(-) create mode 100644 docs/api/wuttatell.client.rst create mode 100644 src/wuttatell/client.py create mode 100644 tests/test_client.py diff --git a/docs/api/wuttatell.client.rst b/docs/api/wuttatell.client.rst new file mode 100644 index 0000000..7a3ef40 --- /dev/null +++ b/docs/api/wuttatell.client.rst @@ -0,0 +1,6 @@ + +``wuttatell.client`` +==================== + +.. automodule:: wuttatell.client + :members: diff --git a/docs/conf.py b/docs/conf.py index 54e91d8..d65a6ba 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,6 +28,7 @@ templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] intersphinx_mapping = { + 'requests': ('https://requests.readthedocs.io/en/latest/', None), 'wuttjamaican': ('https://docs.wuttaproject.org/wuttjamaican/', None), } diff --git a/docs/index.rst b/docs/index.rst index 0fa4d2a..ccecc89 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,4 +25,5 @@ project. api/wuttatell.app api/wuttatell.cli api/wuttatell.cli.tell + api/wuttatell.client api/wuttatell.telemetry diff --git a/src/wuttatell/client.py b/src/wuttatell/client.py new file mode 100644 index 0000000..1a77496 --- /dev/null +++ b/src/wuttatell/client.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttaTell -- Telemetry submission for Wutta Framework +# Copyright © 2025 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +Simple API Client +""" + +import json +from urllib.parse import urlparse + +import requests + + +class SimpleAPIClient: + """ + Simple client for "typical" API service. + + This basically assumes telemetry can be submitted to a single API + endpoint, and the request should contain an auth token. + + :param config: App :term:`config object`. + + :param base_url: Base URL of the API. + + :param token: Auth token for the API. + + :param ssl_verify: Whether the SSL cert presented by the server + should be verified. This is effectively true by default, but + may be disabled for testing with self-signed certs etc. + + :param max_retries: Maximum number of retries each connection + should attempt. This value is ultimately given to the + :class:`~requests:requests.adapters.HTTPAdapter` instance. + + Most params may be omitted, if config specifies instead: + + .. code-block:: ini + + [wutta.api] + base_url = https://my.example.com/api + token = XYZPDQ12345 + ssl_verify = false + max_retries = 5 + + Upon instantiation, :attr:`session` will be ``None`` until the + first request is made. (Technically when :meth:`init_session()` + first happens.) + + .. attribute:: session + + :class:`requests:requests.Session` instance being used to make + API requests. + """ + + def __init__(self, config, base_url=None, token=None, ssl_verify=None, max_retries=None): + self.config = config + + self.base_url = base_url or self.config.require(f'{self.config.appname}.api.base_url') + self.base_url = self.base_url.rstrip('/') + self.token = token or self.config.require(f'{self.config.appname}.api.token') + + if max_retries is not None: + self.max_retries = max_retries + else: + self.max_retries = self.config.get_int(f'{self.config.appname}.api.max_retries') + + if ssl_verify is not None: + self.ssl_verify = ssl_verify + else: + self.ssl_verify = self.config.get_bool(f'{self.config.appname}.api.ssl_verify', + default=True) + + self.session = None + + def init_session(self): + """ + Initialize the HTTP session with the API. + + This method is invoked as part of :meth:`make_request()`. + + It first checks :attr:`session` and will skip if already initialized. + + For initialization, it establishes a new + :class:`requests:requests.Session` instance, and modifies it + as needed per config. + """ + if self.session: + return + + self.session = requests.Session() + + # maybe *disable* SSL cert verification + # (should only be used for testing e.g. w/ self-signed certs) + if not self.ssl_verify: + self.session.verify = False + + # maybe set max retries, e.g. for flaky connections + if self.max_retries is not None: + adapter = requests.adapters.HTTPAdapter(max_retries=self.max_retries) + self.session.mount(self.base_url, adapter) + + # TODO: is this a good idea, or hacky security risk..? + # without it, can get error response: + # 400 Client Error: Bad CSRF Origin for url + parts = urlparse(self.base_url) + self.session.headers.update({ + 'Origin': f'{parts.scheme}://{parts.netloc}', + }) + + # authenticate via token only (for now?) + self.session.headers.update({ + 'Authorization': f'Bearer {self.token}', + }) + + def make_request(self, request_method, api_method, params=None, data=None): + """ + Make a request to the API, and return the response. + + This first calls :meth:`init_session()` to establish the + session if needed. + + :param request_method: HTTP request method; for now only + ``'GET'`` and ``'POST'`` are supported. + + :param api_method: API method endpoint to use, + e.g. ``'/my/telemetry'`` + + :param params: Dict of query string params for the request, if + applicable. + + :param data: Payload data for the request, if applicable. + Should be JSON-serializable, e.g. a list or dict. + + :rtype: :class:`requests:requests.Response` instance. + """ + self.init_session() + api_method = api_method.lstrip('/') + url = f'{self.base_url}/{api_method}' + if request_method == 'GET': + response = self.session.get(url, params=params) + elif request_method == 'POST': + response = self.session.post(url, params=params, + data=json.dumps(data)) + else: + raise NotImplementedError(f"unsupported request method: {request_method}") + response.raise_for_status() + return response + + def get(self, api_method, params=None): + """ + Perform a GET request for the given API method, and return the + response. + + This calls :meth:`make_request()` for the heavy lifting. + + :param api_method: API method endpoint to use, + e.g. ``'/my/telemetry'`` + + :param params: Dict of query string params for the request, if + applicable. + + :rtype: :class:`requests:requests.Response` instance. + """ + return self.make_request('GET', api_method, params=params) + + def post(self, api_method, **kwargs): + """ + Perform a POST request for the given API method, and return + the response. + + This calls :meth:`make_request()` for the heavy lifting. + + :param api_method: API method endpoint to use, + e.g. ``'/my/telemetry'`` + + :rtype: :class:`requests:requests.Response` instance. + """ + return self.make_request('POST', api_method, **kwargs) diff --git a/src/wuttatell/telemetry.py b/src/wuttatell/telemetry.py index 2194652..28b6b82 100644 --- a/src/wuttatell/telemetry.py +++ b/src/wuttatell/telemetry.py @@ -31,6 +31,8 @@ import subprocess from wuttjamaican.app import GenericHandler from wuttjamaican.conf import WuttaConfigProfile +from wuttatell.client import SimpleAPIClient + class TelemetryHandler(GenericHandler): """ @@ -220,14 +222,21 @@ class TelemetryHandler(GenericHandler): """ Submit telemetry data to the configured collection service. - Default logic is not implemented; subclass must override. + Default logic will use + :class:`~wuttatell.client.SimpleAPIClient` and submit all + collected data to the configured API endpoint. :param profile: :class:`TelemetryProfile` instance. :param data: Data dict as obtained by :meth:`collect_all_data()`. """ - raise NotImplementedError + profile = self.get_profile(profile) + if data is None: + data = self.collect_all_data(profile) + + client = SimpleAPIClient(self.config) + client.post(profile.submit_url, data=data) class TelemetryProfile(WuttaConfigProfile): diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..95691ee --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8; -*- + +import json +import threading +import time +from http import HTTPStatus +from http.server import HTTPServer, BaseHTTPRequestHandler + +import requests +from urllib3.util.retry import Retry + +from wuttjamaican.testing import ConfigTestCase +from wuttatell import client as mod + + +class TestSimpleAPIClient(ConfigTestCase): + + def make_client(self, **kw): + return mod.SimpleAPIClient(self.config, **kw) + + def test_constructor(self): + + # caller specifies params + client = self.make_client(base_url='https://example.com/api/', + token='XYZPDQ12345', + ssl_verify=False, + max_retries=5) + self.assertEqual(client.base_url, 'https://example.com/api') # no trailing slash + self.assertEqual(client.token, 'XYZPDQ12345') + self.assertFalse(client.ssl_verify) + self.assertEqual(client.max_retries, 5) + self.assertIsNone(client.session) + + # now with some defaults + client = self.make_client(base_url='https://example.com/api/', + token='XYZPDQ12345') + self.assertEqual(client.base_url, 'https://example.com/api') # no trailing slash + self.assertEqual(client.token, 'XYZPDQ12345') + self.assertTrue(client.ssl_verify) + self.assertIsNone(client.max_retries) + self.assertIsNone(client.session) + + # now from config + self.config.setdefault('wutta.api.base_url', 'https://another.com/api/') + self.config.setdefault('wutta.api.token', '9843243q4') + self.config.setdefault('wutta.api.ssl_verify', 'false') + self.config.setdefault('wutta.api.max_retries', '4') + client = self.make_client() + self.assertEqual(client.base_url, 'https://another.com/api') # no trailing slash + self.assertEqual(client.token, '9843243q4') + self.assertFalse(client.ssl_verify) + self.assertEqual(client.max_retries, 4) + self.assertIsNone(client.session) + + def test_init_session(self): + + # client begins with no session + client = self.make_client(base_url='https://example.com/api', token='1234') + self.assertIsNone(client.session) + + # session is created here + client.init_session() + self.assertIsInstance(client.session, requests.Session) + self.assertTrue(client.session.verify) + self.assertTrue(all([a.max_retries.total == 0 for a in client.session.adapters.values()])) + self.assertIn('Authorization', client.session.headers) + self.assertEqual(client.session.headers['Authorization'], 'Bearer 1234') + + # session is never re-created + orig_session = client.session + client.init_session() + self.assertIs(client.session, orig_session) + + # new client/session with no ssl_verify + client = self.make_client(base_url='https://example.com/api', token='1234', ssl_verify=False) + client.init_session() + self.assertFalse(client.session.verify) + + # new client/session with max_retries + client = self.make_client(base_url='https://example.com/api', token='1234', max_retries=5) + client.init_session() + self.assertEqual(client.session.adapters['https://example.com/api'].max_retries.total, 5) + + def test_make_request_get(self): + + # start server + threading.Thread(target=start_server).start() + while not SERVER['running']: + time.sleep(0.02) + + # server returns our headers + client = self.make_client(base_url=f'http://127.0.0.1:{SERVER["port"]}', token='1234', ssl_verify=False) + response = client.make_request('GET', '/telemetry') + result = response.json() + self.assertIn('headers', result) + self.assertIn('Authorization', result['headers']) + self.assertEqual(result['headers']['Authorization'], 'Bearer 1234') + self.assertNotIn('payload', result) + + def test_make_request_post(self): + + # start server + threading.Thread(target=start_server).start() + while not SERVER['running']: + time.sleep(0.02) + + # server returns our headers + payload + client = self.make_client(base_url=f'http://127.0.0.1:{SERVER["port"]}', token='1234', ssl_verify=False) + response = client.make_request('POST', '/telemetry', data={'os': {'name': 'debian'}}) + result = response.json() + self.assertIn('headers', result) + self.assertIn('Authorization', result['headers']) + self.assertEqual(result['headers']['Authorization'], 'Bearer 1234') + self.assertIn('payload', result) + self.assertEqual(json.loads(result['payload']), {'os': {'name': 'debian'}}) + + def test_make_request_unsupported(self): + + # start server + threading.Thread(target=start_server).start() + while not SERVER['running']: + time.sleep(0.02) + + # e.g. DELETE is not implemented + client = self.make_client(base_url=f'http://127.0.0.1:{SERVER["port"]}', token='1234', ssl_verify=False) + self.assertRaises(NotImplementedError, client.make_request, 'DELETE', '/telemetry') + + # nb. issue valid request to stop the server + client.make_request('GET', '/telemetry') + + def test_get(self): + + # start server + threading.Thread(target=start_server).start() + while not SERVER['running']: + time.sleep(0.02) + + # server returns our headers + client = self.make_client(base_url=f'http://127.0.0.1:{SERVER["port"]}', token='1234', ssl_verify=False) + response = client.get('/telemetry') + result = response.json() + self.assertIn('headers', result) + self.assertIn('Authorization', result['headers']) + self.assertEqual(result['headers']['Authorization'], 'Bearer 1234') + self.assertNotIn('payload', result) + + def test_post(self): + + # start server + threading.Thread(target=start_server).start() + while not SERVER['running']: + time.sleep(0.02) + + # server returns our headers + payload + client = self.make_client(base_url=f'http://127.0.0.1:{SERVER["port"]}', token='1234', ssl_verify=False) + response = client.post('/telemetry', data={'os': {'name': 'debian'}}) + result = response.json() + self.assertIn('headers', result) + self.assertIn('Authorization', result['headers']) + self.assertEqual(result['headers']['Authorization'], 'Bearer 1234') + self.assertIn('payload', result) + self.assertEqual(json.loads(result['payload']), {'os': {'name': 'debian'}}) + + +class FakeRequestHandler(BaseHTTPRequestHandler): + """ """ + + def do_GET(self): + headers = dict([(k, v) for k, v in self.headers.items()]) + result = {'headers': headers} + result = json.dumps(result).encode('utf_8') + + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", 'text/json') + self.send_header("Content-Length", str(len(result))) + self.end_headers() + self.wfile.write(result) + + def do_POST(self): + headers = dict([(k, v) for k, v in self.headers.items()]) + length = int(self.headers.get('Content-Length')) + payload = self.rfile.read(length).decode('utf_8') + result = {'headers': headers, 'payload': payload} + result = json.dumps(result).encode('utf_8') + + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", 'text/json') + self.send_header("Content-Length", str(len(result))) + self.end_headers() + self.wfile.write(result) + + +SERVER = {'running': False, 'port': 7314} + +def start_server(): + if SERVER['running']: + raise RuntimeError("http server is already running") + + with HTTPServer(('127.0.0.1', SERVER['port']), FakeRequestHandler) as httpd: + SERVER['running'] = True + httpd.handle_request() + + SERVER['running'] = False diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index ae90b97..fb40b23 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,6 +1,6 @@ # -*- coding: utf-8; -*- -from unittest.mock import patch +from unittest.mock import patch, MagicMock from wuttjamaican.testing import ConfigTestCase @@ -160,9 +160,24 @@ class TestTelemetryHandler(ConfigTestCase): self.assertNotIn('errors', data) def test_submit_all_data(self): + profile = self.handler.get_profile('default') + profile.submit_url = '/testing' - # not (yet?) implemented - self.assertRaises(NotImplementedError, self.handler.submit_all_data) + with patch.object(mod, 'SimpleAPIClient') as SimpleAPIClient: + client = MagicMock() + SimpleAPIClient.return_value = client + + # collecting all data + with patch.object(self.handler, 'collect_all_data') as collect_all_data: + collect_all_data.return_value = [] + self.handler.submit_all_data(profile) + collect_all_data.assert_called_once_with(profile) + client.post.assert_called_once_with('/testing', data=[]) + + # use data from caller + client.post.reset_mock() + self.handler.submit_all_data(profile, data=['foo']) + client.post.assert_called_once_with('/testing', data=['foo']) class TestTelemetryProfile(ConfigTestCase): From ec8fe97cffe1a65ad1ec15b554cd0c60f26dfe36 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 10 Aug 2025 15:20:00 -0500 Subject: [PATCH 4/5] fix: add dependency for `requests` --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 42cb544..e359127 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ ] requires-python = ">= 3.8" dependencies = [ + "requests", "WuttJamaican", ] From 18f7fa6c51d5e7108c9c333b66cacfe144aa7b87 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 10 Aug 2025 16:01:45 -0500 Subject: [PATCH 5/5] =?UTF-8?q?bump:=20version=200.1.0=20=E2=86=92=200.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 12 ++++++++++++ pyproject.toml | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2ad346..d534f6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to WuttaTell will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## v0.2.0 (2025-08-10) + +### Feat + +- add native API client for common submission use case + +### Fix + +- add dependency for `requests` +- remove `submit_uuid` from telemetry config profile +- tweak log verbiage + ## v0.1.0 (2025-08-09) ### Feat diff --git a/pyproject.toml b/pyproject.toml index e359127..6edf0eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "WuttaTell" -version = "0.1.0" +version = "0.2.0" description = "Telemetry submission for Wutta Framework" readme = "README.md" authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}] @@ -27,7 +27,7 @@ classifiers = [ requires-python = ">= 3.8" dependencies = [ "requests", - "WuttJamaican", + "WuttJamaican>=0.23.0", ]