diff --git a/CHANGELOG.md b/CHANGELOG.md index d534f6a..f2ad346 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,18 +5,6 @@ All notable changes to WuttaTell will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## v0.2.0 (2025-08-10) - -### Feat - -- add native API client for common submission use case - -### Fix - -- add dependency for `requests` -- remove `submit_uuid` from telemetry config profile -- tweak log verbiage - ## v0.1.0 (2025-08-09) ### Feat diff --git a/docs/api/wuttatell.client.rst b/docs/api/wuttatell.client.rst deleted file mode 100644 index 7a3ef40..0000000 --- a/docs/api/wuttatell.client.rst +++ /dev/null @@ -1,6 +0,0 @@ - -``wuttatell.client`` -==================== - -.. automodule:: wuttatell.client - :members: diff --git a/docs/conf.py b/docs/conf.py index d65a6ba..54e91d8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,6 @@ templates_path = ['_templates'] exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] intersphinx_mapping = { - 'requests': ('https://requests.readthedocs.io/en/latest/', None), 'wuttjamaican': ('https://docs.wuttaproject.org/wuttjamaican/', None), } diff --git a/docs/index.rst b/docs/index.rst index ccecc89..0fa4d2a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,5 +25,4 @@ project. api/wuttatell.app api/wuttatell.cli api/wuttatell.cli.tell - api/wuttatell.client api/wuttatell.telemetry diff --git a/docs/narr/install.rst b/docs/narr/install.rst index aa41f30..f05fee2 100644 --- a/docs/narr/install.rst +++ b/docs/narr/install.rst @@ -12,13 +12,15 @@ Install the WuttaTell package to your virtual environment: pip install WuttaTell Edit your :term:`config file` to add telemetry submission info, and -related settings: +related settings. Please note, the following example is just that - +and will not work as-is: .. code-block:: ini [wutta.telemetry] default.collect_keys = os, python - default.submit_url = https://example.com/api/my-node/telemetry + default.submit_url = /nodes/telemetry + default.submit_uuid = 06897767-eb70-7790-8000-13f368a40ea3 .. note:: diff --git a/pyproject.toml b/pyproject.toml index 6edf0eb..42cb544 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "WuttaTell" -version = "0.2.0" +version = "0.1.0" description = "Telemetry submission for Wutta Framework" readme = "README.md" authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}] @@ -26,8 +26,7 @@ classifiers = [ ] requires-python = ">= 3.8" dependencies = [ - "requests", - "WuttJamaican>=0.23.0", + "WuttJamaican", ] diff --git a/src/wuttatell/cli/tell.py b/src/wuttatell/cli/tell.py index 1cbe1f6..7a52dd4 100644 --- a/src/wuttatell/cli/tell.py +++ b/src/wuttatell/cli/tell.py @@ -58,7 +58,7 @@ def tell( telemetry = app.get_telemetry_handler() data = telemetry.collect_all_data(profile=profile) - log.info("data collected for: %s", ', '.join(sorted(data))) + log.info("data collected okay: %s", ', '.join(sorted(data))) log.debug("%s", data) if dry_run: diff --git a/src/wuttatell/client.py b/src/wuttatell/client.py deleted file mode 100644 index 1a77496..0000000 --- a/src/wuttatell/client.py +++ /dev/null @@ -1,197 +0,0 @@ -# -*- coding: utf-8; -*- -################################################################################ -# -# WuttaTell -- Telemetry submission for Wutta Framework -# Copyright © 2025 Lance Edgar -# -# This file is part of Wutta Framework. -# -# Wutta Framework is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# Wutta Framework is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# Wutta Framework. If not, see . -# -################################################################################ -""" -Simple API Client -""" - -import json -from urllib.parse import urlparse - -import requests - - -class SimpleAPIClient: - """ - Simple client for "typical" API service. - - This basically assumes telemetry can be submitted to a single API - endpoint, and the request should contain an auth token. - - :param config: App :term:`config object`. - - :param base_url: Base URL of the API. - - :param token: Auth token for the API. - - :param ssl_verify: Whether the SSL cert presented by the server - should be verified. This is effectively true by default, but - may be disabled for testing with self-signed certs etc. - - :param max_retries: Maximum number of retries each connection - should attempt. This value is ultimately given to the - :class:`~requests:requests.adapters.HTTPAdapter` instance. - - Most params may be omitted, if config specifies instead: - - .. code-block:: ini - - [wutta.api] - base_url = https://my.example.com/api - token = XYZPDQ12345 - ssl_verify = false - max_retries = 5 - - Upon instantiation, :attr:`session` will be ``None`` until the - first request is made. (Technically when :meth:`init_session()` - first happens.) - - .. attribute:: session - - :class:`requests:requests.Session` instance being used to make - API requests. - """ - - def __init__(self, config, base_url=None, token=None, ssl_verify=None, max_retries=None): - self.config = config - - self.base_url = base_url or self.config.require(f'{self.config.appname}.api.base_url') - self.base_url = self.base_url.rstrip('/') - self.token = token or self.config.require(f'{self.config.appname}.api.token') - - if max_retries is not None: - self.max_retries = max_retries - else: - self.max_retries = self.config.get_int(f'{self.config.appname}.api.max_retries') - - if ssl_verify is not None: - self.ssl_verify = ssl_verify - else: - self.ssl_verify = self.config.get_bool(f'{self.config.appname}.api.ssl_verify', - default=True) - - self.session = None - - def init_session(self): - """ - Initialize the HTTP session with the API. - - This method is invoked as part of :meth:`make_request()`. - - It first checks :attr:`session` and will skip if already initialized. - - For initialization, it establishes a new - :class:`requests:requests.Session` instance, and modifies it - as needed per config. - """ - if self.session: - return - - self.session = requests.Session() - - # maybe *disable* SSL cert verification - # (should only be used for testing e.g. w/ self-signed certs) - if not self.ssl_verify: - self.session.verify = False - - # maybe set max retries, e.g. for flaky connections - if self.max_retries is not None: - adapter = requests.adapters.HTTPAdapter(max_retries=self.max_retries) - self.session.mount(self.base_url, adapter) - - # TODO: is this a good idea, or hacky security risk..? - # without it, can get error response: - # 400 Client Error: Bad CSRF Origin for url - parts = urlparse(self.base_url) - self.session.headers.update({ - 'Origin': f'{parts.scheme}://{parts.netloc}', - }) - - # authenticate via token only (for now?) - self.session.headers.update({ - 'Authorization': f'Bearer {self.token}', - }) - - def make_request(self, request_method, api_method, params=None, data=None): - """ - Make a request to the API, and return the response. - - This first calls :meth:`init_session()` to establish the - session if needed. - - :param request_method: HTTP request method; for now only - ``'GET'`` and ``'POST'`` are supported. - - :param api_method: API method endpoint to use, - e.g. ``'/my/telemetry'`` - - :param params: Dict of query string params for the request, if - applicable. - - :param data: Payload data for the request, if applicable. - Should be JSON-serializable, e.g. a list or dict. - - :rtype: :class:`requests:requests.Response` instance. - """ - self.init_session() - api_method = api_method.lstrip('/') - url = f'{self.base_url}/{api_method}' - if request_method == 'GET': - response = self.session.get(url, params=params) - elif request_method == 'POST': - response = self.session.post(url, params=params, - data=json.dumps(data)) - else: - raise NotImplementedError(f"unsupported request method: {request_method}") - response.raise_for_status() - return response - - def get(self, api_method, params=None): - """ - Perform a GET request for the given API method, and return the - response. - - This calls :meth:`make_request()` for the heavy lifting. - - :param api_method: API method endpoint to use, - e.g. ``'/my/telemetry'`` - - :param params: Dict of query string params for the request, if - applicable. - - :rtype: :class:`requests:requests.Response` instance. - """ - return self.make_request('GET', api_method, params=params) - - def post(self, api_method, **kwargs): - """ - Perform a POST request for the given API method, and return - the response. - - This calls :meth:`make_request()` for the heavy lifting. - - :param api_method: API method endpoint to use, - e.g. ``'/my/telemetry'`` - - :rtype: :class:`requests:requests.Response` instance. - """ - return self.make_request('POST', api_method, **kwargs) diff --git a/src/wuttatell/telemetry.py b/src/wuttatell/telemetry.py index 28b6b82..86d7de6 100644 --- a/src/wuttatell/telemetry.py +++ b/src/wuttatell/telemetry.py @@ -31,8 +31,6 @@ import subprocess from wuttjamaican.app import GenericHandler from wuttjamaican.conf import WuttaConfigProfile -from wuttatell.client import SimpleAPIClient - class TelemetryHandler(GenericHandler): """ @@ -222,21 +220,14 @@ class TelemetryHandler(GenericHandler): """ Submit telemetry data to the configured collection service. - Default logic will use - :class:`~wuttatell.client.SimpleAPIClient` and submit all - collected data to the configured API endpoint. + Default logic is not implemented; subclass must override. :param profile: :class:`TelemetryProfile` instance. :param data: Data dict as obtained by :meth:`collect_all_data()`. """ - profile = self.get_profile(profile) - if data is None: - data = self.collect_all_data(profile) - - client = SimpleAPIClient(self.config) - client.post(profile.submit_url, data=data) + raise NotImplementedError class TelemetryProfile(WuttaConfigProfile): @@ -260,6 +251,13 @@ class TelemetryProfile(WuttaConfigProfile): .. attribute:: submit_url URL to which collected telemetry data should be submitted. + + .. attribute:: submit_uuid + + UUID identifying the record to update when submitting telemetry + data. This value will only make sense in the context of the + collection service responsible for receiving telemetry + submissions. """ @property @@ -272,3 +270,4 @@ class TelemetryProfile(WuttaConfigProfile): keys = self.get_str('collect.keys', default='os,python') self.collect_keys = self.config.parse_list(keys) self.submit_url = self.get_str('submit.url') + self.submit_uuid = self.get_str('submit.uuid') diff --git a/tests/test_client.py b/tests/test_client.py deleted file mode 100644 index 95691ee..0000000 --- a/tests/test_client.py +++ /dev/null @@ -1,203 +0,0 @@ -# -*- coding: utf-8; -*- - -import json -import threading -import time -from http import HTTPStatus -from http.server import HTTPServer, BaseHTTPRequestHandler - -import requests -from urllib3.util.retry import Retry - -from wuttjamaican.testing import ConfigTestCase -from wuttatell import client as mod - - -class TestSimpleAPIClient(ConfigTestCase): - - def make_client(self, **kw): - return mod.SimpleAPIClient(self.config, **kw) - - def test_constructor(self): - - # caller specifies params - client = self.make_client(base_url='https://example.com/api/', - token='XYZPDQ12345', - ssl_verify=False, - max_retries=5) - self.assertEqual(client.base_url, 'https://example.com/api') # no trailing slash - self.assertEqual(client.token, 'XYZPDQ12345') - self.assertFalse(client.ssl_verify) - self.assertEqual(client.max_retries, 5) - self.assertIsNone(client.session) - - # now with some defaults - client = self.make_client(base_url='https://example.com/api/', - token='XYZPDQ12345') - self.assertEqual(client.base_url, 'https://example.com/api') # no trailing slash - self.assertEqual(client.token, 'XYZPDQ12345') - self.assertTrue(client.ssl_verify) - self.assertIsNone(client.max_retries) - self.assertIsNone(client.session) - - # now from config - self.config.setdefault('wutta.api.base_url', 'https://another.com/api/') - self.config.setdefault('wutta.api.token', '9843243q4') - self.config.setdefault('wutta.api.ssl_verify', 'false') - self.config.setdefault('wutta.api.max_retries', '4') - client = self.make_client() - self.assertEqual(client.base_url, 'https://another.com/api') # no trailing slash - self.assertEqual(client.token, '9843243q4') - self.assertFalse(client.ssl_verify) - self.assertEqual(client.max_retries, 4) - self.assertIsNone(client.session) - - def test_init_session(self): - - # client begins with no session - client = self.make_client(base_url='https://example.com/api', token='1234') - self.assertIsNone(client.session) - - # session is created here - client.init_session() - self.assertIsInstance(client.session, requests.Session) - self.assertTrue(client.session.verify) - self.assertTrue(all([a.max_retries.total == 0 for a in client.session.adapters.values()])) - self.assertIn('Authorization', client.session.headers) - self.assertEqual(client.session.headers['Authorization'], 'Bearer 1234') - - # session is never re-created - orig_session = client.session - client.init_session() - self.assertIs(client.session, orig_session) - - # new client/session with no ssl_verify - client = self.make_client(base_url='https://example.com/api', token='1234', ssl_verify=False) - client.init_session() - self.assertFalse(client.session.verify) - - # new client/session with max_retries - client = self.make_client(base_url='https://example.com/api', token='1234', max_retries=5) - client.init_session() - self.assertEqual(client.session.adapters['https://example.com/api'].max_retries.total, 5) - - def test_make_request_get(self): - - # start server - threading.Thread(target=start_server).start() - while not SERVER['running']: - time.sleep(0.02) - - # server returns our headers - client = self.make_client(base_url=f'http://127.0.0.1:{SERVER["port"]}', token='1234', ssl_verify=False) - response = client.make_request('GET', '/telemetry') - result = response.json() - self.assertIn('headers', result) - self.assertIn('Authorization', result['headers']) - self.assertEqual(result['headers']['Authorization'], 'Bearer 1234') - self.assertNotIn('payload', result) - - def test_make_request_post(self): - - # start server - threading.Thread(target=start_server).start() - while not SERVER['running']: - time.sleep(0.02) - - # server returns our headers + payload - client = self.make_client(base_url=f'http://127.0.0.1:{SERVER["port"]}', token='1234', ssl_verify=False) - response = client.make_request('POST', '/telemetry', data={'os': {'name': 'debian'}}) - result = response.json() - self.assertIn('headers', result) - self.assertIn('Authorization', result['headers']) - self.assertEqual(result['headers']['Authorization'], 'Bearer 1234') - self.assertIn('payload', result) - self.assertEqual(json.loads(result['payload']), {'os': {'name': 'debian'}}) - - def test_make_request_unsupported(self): - - # start server - threading.Thread(target=start_server).start() - while not SERVER['running']: - time.sleep(0.02) - - # e.g. DELETE is not implemented - client = self.make_client(base_url=f'http://127.0.0.1:{SERVER["port"]}', token='1234', ssl_verify=False) - self.assertRaises(NotImplementedError, client.make_request, 'DELETE', '/telemetry') - - # nb. issue valid request to stop the server - client.make_request('GET', '/telemetry') - - def test_get(self): - - # start server - threading.Thread(target=start_server).start() - while not SERVER['running']: - time.sleep(0.02) - - # server returns our headers - client = self.make_client(base_url=f'http://127.0.0.1:{SERVER["port"]}', token='1234', ssl_verify=False) - response = client.get('/telemetry') - result = response.json() - self.assertIn('headers', result) - self.assertIn('Authorization', result['headers']) - self.assertEqual(result['headers']['Authorization'], 'Bearer 1234') - self.assertNotIn('payload', result) - - def test_post(self): - - # start server - threading.Thread(target=start_server).start() - while not SERVER['running']: - time.sleep(0.02) - - # server returns our headers + payload - client = self.make_client(base_url=f'http://127.0.0.1:{SERVER["port"]}', token='1234', ssl_verify=False) - response = client.post('/telemetry', data={'os': {'name': 'debian'}}) - result = response.json() - self.assertIn('headers', result) - self.assertIn('Authorization', result['headers']) - self.assertEqual(result['headers']['Authorization'], 'Bearer 1234') - self.assertIn('payload', result) - self.assertEqual(json.loads(result['payload']), {'os': {'name': 'debian'}}) - - -class FakeRequestHandler(BaseHTTPRequestHandler): - """ """ - - def do_GET(self): - headers = dict([(k, v) for k, v in self.headers.items()]) - result = {'headers': headers} - result = json.dumps(result).encode('utf_8') - - self.send_response(HTTPStatus.OK) - self.send_header("Content-Type", 'text/json') - self.send_header("Content-Length", str(len(result))) - self.end_headers() - self.wfile.write(result) - - def do_POST(self): - headers = dict([(k, v) for k, v in self.headers.items()]) - length = int(self.headers.get('Content-Length')) - payload = self.rfile.read(length).decode('utf_8') - result = {'headers': headers, 'payload': payload} - result = json.dumps(result).encode('utf_8') - - self.send_response(HTTPStatus.OK) - self.send_header("Content-Type", 'text/json') - self.send_header("Content-Length", str(len(result))) - self.end_headers() - self.wfile.write(result) - - -SERVER = {'running': False, 'port': 7314} - -def start_server(): - if SERVER['running']: - raise RuntimeError("http server is already running") - - with HTTPServer(('127.0.0.1', SERVER['port']), FakeRequestHandler) as httpd: - SERVER['running'] = True - httpd.handle_request() - - SERVER['running'] = False diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index fb40b23..c2880e0 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,6 +1,6 @@ # -*- coding: utf-8; -*- -from unittest.mock import patch, MagicMock +from unittest.mock import patch from wuttjamaican.testing import ConfigTestCase @@ -160,24 +160,9 @@ class TestTelemetryHandler(ConfigTestCase): self.assertNotIn('errors', data) def test_submit_all_data(self): - profile = self.handler.get_profile('default') - profile.submit_url = '/testing' - with patch.object(mod, 'SimpleAPIClient') as SimpleAPIClient: - client = MagicMock() - SimpleAPIClient.return_value = client - - # collecting all data - with patch.object(self.handler, 'collect_all_data') as collect_all_data: - collect_all_data.return_value = [] - self.handler.submit_all_data(profile) - collect_all_data.assert_called_once_with(profile) - client.post.assert_called_once_with('/testing', data=[]) - - # use data from caller - client.post.reset_mock() - self.handler.submit_all_data(profile, data=['foo']) - client.post.assert_called_once_with('/testing', data=['foo']) + # not (yet?) implemented + self.assertRaises(NotImplementedError, self.handler.submit_all_data) class TestTelemetryProfile(ConfigTestCase): @@ -202,10 +187,13 @@ class TestTelemetryProfile(ConfigTestCase): profile = self.make_profile() self.assertEqual(profile.collect_keys, ['os', 'python']) self.assertIsNone(profile.submit_url) + self.assertIsNone(profile.submit_uuid) # configured self.config.setdefault('wutta.telemetry.default.collect.keys', 'os,network,python') self.config.setdefault('wutta.telemetry.default.submit.url', '/nodes/telemetry') + self.config.setdefault('wutta.telemetry.default.submit.uuid', '06897669-272b-7c74-8000-4d311924d24f') profile = self.make_profile() self.assertEqual(profile.collect_keys, ['os', 'network', 'python']) self.assertEqual(profile.submit_url, '/nodes/telemetry') + self.assertEqual(profile.submit_uuid, '06897669-272b-7c74-8000-4d311924d24f')