feat: add Users view; improve CRUD master for SQLAlchemy models

docs/api/wuttaweb/forms.schema.rst

@@ -0,0 +1,6 @@
+
+``wuttaweb.forms.schema``
+=========================
+
+.. automodule:: wuttaweb.forms.schema
+   :members:

docs/api/wuttaweb/forms.widgets.rst

@@ -0,0 +1,6 @@
+
+``wuttaweb.forms.widgets``
+==========================
+
+.. automodule:: wuttaweb.forms.widgets
+   :members:

docs/api/wuttaweb/index.rst

@@ -12,6 +12,8 @@
    db
    forms
    forms.base
+   forms.schema
+   forms.widgets
    grids
    grids.base
    handler
@@ -28,3 +30,4 @@
    views.master
    views.people
    views.settings
+   views.users

docs/api/wuttaweb/views.users.rst

@@ -0,0 +1,6 @@
+
+``wuttaweb.views.users``
+========================
+
+.. automodule:: wuttaweb.views.users
+   :members:

pyproject.toml

@@ -30,6 +30,7 @@ classifiers = [
 ]
 requires-python = ">= 3.8"
 dependencies = [
+        "ColanderAlchemy",
         "pyramid>=2",
         "pyramid_beaker",
         "pyramid_deform",

src/wuttaweb/forms/base.py

@@ -24,65 +24,20 @@
 Base form classes
 """
 
-import json
 import logging
 
 import colander
 import deform
+from colanderalchemy import SQLAlchemySchemaNode
 from pyramid.renderers import render
 from webhelpers2.html import HTML
 
-from wuttaweb.util import get_form_data, get_model_fields
+from wuttaweb.util import FieldList, get_form_data, get_model_fields, make_json_safe
 
 
 log = logging.getLogger(__name__)
 
 
-class FieldList(list):
-    """
-    Convenience wrapper for a form's field list.  This is a subclass
-    of :class:`python:list`.
-
-    You normally would not need to instantiate this yourself, but it
-    is used under the hood for :attr:`Form.fields` as well as
-    :attr:`~wuttaweb.grids.base.Grid.columns`.
-    """
-
-    def insert_before(self, field, newfield):
-        """
-        Insert a new field, before an existing field.
-
-        :param field: String name for the existing field.
-
-        :param newfield: String name for the new field, to be inserted
-           just before the existing ``field``.
-        """
-        if field in self:
-            i = self.index(field)
-            self.insert(i, newfield)
-        else:
-            log.warning("field '%s' not found, will append new field: %s",
-                        field, newfield)
-            self.append(newfield)
-
-    def insert_after(self, field, newfield):
-        """
-        Insert a new field, after an existing field.
-
-        :param field: String name for the existing field.
-
-        :param newfield: String name for the new field, to be inserted
-           just after the existing ``field``.
-        """
-        if field in self:
-            i = self.index(field)
-            self.insert(i + 1, newfield)
-        else:
-            log.warning("field '%s' not found, will append new field: %s",
-                        field, newfield)
-            self.append(newfield)
-
-
 class Form:
     """
     Base class for all forms.
@@ -112,9 +67,11 @@ class Form:
 
     .. attribute:: fields
 
-       :class:`FieldList` instance containing string field names for
+       :class:`~wuttaweb.util.FieldList` instance containing string
-       the form.  By default, fields will appear in the same order as
+       field names for the form.  By default, fields will appear in
-       they are in this list.
+       the same order as they are in this list.
+
+       See also :meth:`set_fields()`.
 
     .. attribute:: schema
 
@@ -142,6 +99,27 @@ class Form:
        SQLAlchemy-mapped.  (In that case :attr:`model_class` can be
        determined automatically.)
 
+    .. attribute:: nodes
+
+       Dict of node overrides, used to construct the form in
+       :meth:`get_schema()`.
+
+       See also :meth:`set_node()`.
+
+    .. attribute:: widgets
+
+       Dict of widget overrides, used to construct the form in
+       :meth:`get_schema()`.
+
+       See also :meth:`set_widget()`.
+
+    .. attribute:: validators
+
+       Dict of node validators, used to construct the form in
+       :meth:`get_schema()`.
+
+       See also :meth:`set_validator()`.
+
     .. attribute:: readonly
 
        Boolean indicating the form does not allow submit.  In practice
@@ -267,6 +245,9 @@ class Form:
             schema=None,
             model_class=None,
             model_instance=None,
+            nodes={},
+            widgets={},
+            validators={},
             readonly=False,
             readonly_fields=[],
             required_fields={},
@@ -287,6 +268,9 @@ class Form:
     ):
         self.request = request
         self.schema = schema
+        self.nodes = nodes or {}
+        self.widgets = widgets or {}
+        self.validators = validators or {}
         self.readonly = readonly
         self.readonly_fields = set(readonly_fields or [])
         self.required_fields = required_fields or {}
@@ -311,13 +295,10 @@ class Form:
         self.model_class = model_class
         self.model_instance = model_instance
         if self.model_instance and not self.model_class:
-            self.model_class = type(self.model_instance)
+            if type(self.model_instance) is not dict:
+                self.model_class = type(self.model_instance)
 
-        fields = fields or self.get_fields()
+        self.set_fields(fields or self.get_fields())
-        if fields:
-            self.set_fields(fields)
-        else:
-            self.fields = None
 
     def __contains__(self, name):
         """
@@ -388,12 +369,108 @@ class Form:
         Explicitly set the list of form fields.
 
         This will overwrite :attr:`fields` with a new
-        :class:`FieldList` instance.
+        :class:`~wuttaweb.util.FieldList` instance.
 
         :param fields: List of string field names.
         """
         self.fields = FieldList(fields)
 
+    def remove(self, *keys):
+        """
+        Remove some fields(s) from the form.
+
+        This is a convenience to allow removal of multiple fields at
+        once::
+
+           form.remove('first_field',
+                       'second_field',
+                       'third_field')
+
+        It will remove each field from :attr:`fields`.
+        """
+        for key in keys:
+            if key in self.fields:
+                self.fields.remove(key)
+
+    def set_node(self, key, nodeinfo, **kwargs):
+        """
+        Set/override the node for a field.
+
+        :param key: Name of field.
+
+        :param nodeinfo: Should be either a
+           :class:`colander:colander.SchemaNode` instance, or else a
+           :class:`colander:colander.SchemaType` instance.
+
+        If ``nodeinfo`` is a proper node instance, it will be used
+        as-is.  Otherwise an
+        :class:`~wuttaweb.forms.schema.ObjectNode` instance will be
+        constructed using ``nodeinfo`` as the type (``typ``).
+
+        Node overrides are tracked via :attr:`nodes`.
+        """
+        if isinstance(nodeinfo, colander.SchemaNode):
+            # assume nodeinfo is a complete node
+            node = nodeinfo
+
+        else: # assume nodeinfo is a schema type
+            kwargs.setdefault('name', key)
+
+            from wuttaweb.forms.schema import ObjectNode
+
+            # node = colander.SchemaNode(nodeinfo, **kwargs)
+            node = ObjectNode(nodeinfo, **kwargs)
+
+        self.nodes[key] = node
+
+        # must explicitly replace node, if we already have a schema
+        if self.schema:
+            self.schema[key] = node
+
+    def set_widget(self, key, widget):
+        """
+        Set/override the widget for a field.
+
+        :param key: Name of field.
+
+        :param widget: Instance of
+           :class:`deform:deform.widget.Widget`.
+
+        Widget overrides are tracked via :attr:`widgets`.
+        """
+        self.widgets[key] = widget
+
+        # update schema if necessary
+        if self.schema and key in self.schema:
+            self.schema[key].widget = widget
+
+    def set_validator(self, key, validator):
+        """
+        Set/override the validator for a field, or the form.
+
+        :param key: Name of field.  This may also be ``None`` in which
+           case the validator will apply to the whole form instead of
+           a field.
+
+        :param validator: Callable which accepts ``(node, value)``
+           args.  For instance::
+
+              def validate_foo(node, value):
+                  if value == 42:
+                      node.raise_invalid("42 is not allowed!")
+
+              form = Form(fields=['foo', 'bar'])
+
+              form.set_validator('foo', validate_foo)
+
+        Validator overrides are tracked via :attr:`validators`.
+        """
+        self.validators[key] = validator
+
+        # nb. must apply to existing schema if present
+        if self.schema and key in self.schema:
+            self.schema[key].validator = validator
+
     def set_readonly(self, key, readonly=True):
         """
         Enable or disable the "readonly" flag for a given field.
@@ -512,6 +589,8 @@ class Form:
         if fields:
             return fields
 
+        return []
+
     def get_model_fields(self, model_class=None):
         """
         This method is a shortcut which calls
@@ -534,25 +613,83 @@ class Form:
         """
         if not self.schema:
 
+            ##############################
+            # create schema
+            ##############################
+
             # get fields
             fields = self.get_fields()
             if not fields:
                 raise NotImplementedError
 
-            # make basic schema
+            if self.model_class:
-            schema = colander.Schema()
+
-            for name in fields:
+                # first define full list of 'includes' - final schema
-                schema.add(colander.SchemaNode(
+                # should contain all of these fields
-                    colander.String(),
+                includes = list(fields)
-                    name=name))
+
+                # determine which we want ColanderAlchemy to handle
+                auto_includes = []
+                for key in includes:
+
+                    # skip if we already have a node defined
+                    if key in self.nodes:
+                        continue
+
+                    # we want the magic for this field
+                    auto_includes.append(key)
+
+                # make initial schema with ColanderAlchemy magic
+                schema = SQLAlchemySchemaNode(self.model_class,
+                                              includes=auto_includes)
+
+                # now fill in the blanks for non-magic fields
+                for key in includes:
+                    if key not in auto_includes:
+                        node = self.nodes[key]
+                        schema.add(node)
+
+            else:
+
+                # make basic schema
+                schema = colander.Schema()
+                for key in fields:
+                    node = None
+
+                    # use node override if present
+                    if key in self.nodes:
+                        node = self.nodes[key]
+                    if not node:
+
+                        # otherwise make simple string node
+                        node = colander.SchemaNode(
+                            colander.String(),
+                            name=key)
+
+                    schema.add(node)
+
+            ##############################
+            # customize schema
+            ##############################
+
+            # apply widget overrides
+            for key, widget in self.widgets.items():
+                if key in schema:
+                    schema[key].widget = widget
+
+            # apply validator overrides
+            for key, validator in self.validators.items():
+                if key is None:
+                    # nb. this one is form-wide
+                    schema.validator = validator
+                elif key in schema: # field-level
+                    schema[key].validator = validator
 
             # apply required flags
             for key, required in self.required_fields.items():
                 if key in schema:
                     if required is False:
-                        # TODO: (why) should we not use colander.null here?
+                        schema[key].missing = colander.null
-                        #schema[key].missing = colander.null
-                        schema[key].missing = None
 
             self.schema = schema
 
@@ -569,16 +706,13 @@ class Form:
             kwargs = {}
 
             if self.model_instance:
+                # TODO: would it be smarter to test with hasattr() ?
+                # if hasattr(schema, 'dictify'):
                 if isinstance(self.model_instance, model.Base):
-                    kwargs['appstruct'] = dict(self.model_instance)
+                    kwargs['appstruct'] = schema.dictify(self.model_instance)
                 else:
                     kwargs['appstruct'] = self.model_instance
 
-                # TODO: ugh why is this necessary?
-                for key, value in list(kwargs['appstruct'].items()):
-                    if value is None:
-                        kwargs['appstruct'][key] = colander.null
-
             form = deform.Form(schema, **kwargs)
             self.deform_form = form
 
@@ -632,6 +766,7 @@ class Form:
         context['dform'] = self.get_deform()
         context.setdefault('form_attrs', {})
         context.setdefault('request', self.request)
+        context['model_data'] = self.get_vue_model_data()
 
         # auto disable button on submit
         if self.auto_disable_submit:
@@ -729,50 +864,44 @@ class Form:
 
         return HTML.tag('b-field', c=[html], **attrs)
 
-    def get_field_errors(self, field):
+    def get_vue_model_data(self):
         """
-        Return a list of error messages for the given field.
+        Returns a dict with form model data.  Values may be nested
+        depending on the types of fields contained in the form.
 
-        Not useful unless a call to :meth:`validate()` failed.
+        Note that the values need not be "converted" (to be
+        JSON-compatible) at this stage, for instance ``colander.null``
+        is not a problem here.  The point is to collect the raw data.
+
+        The dict should have a key/value for each field in the form.
+
+        This method is called by :meth:`render_vue_model_data()` which
+        is responsible for ensuring JSON compatibility.
         """
         dform = self.get_deform()
-        if field in dform:
+        model_data = {}
-            error = dform[field].errormsg
-            if error:
-                return [error]
-        return []
 
-    def get_vue_field_value(self, field):
+        def assign(field):
-        """
+            model_data[field.oid] = make_json_safe(field.cstruct)
-        This method returns a JSON string which will be assigned as
-        the initial model value for the given field.  This JSON will
-        be written as part of the overall response, to be interpreted
-        on the client side.
 
-        Again, this must return a *string* such as:
+        for key in self.fields:
 
-        * ``'null'``
+            # TODO: i thought commented code was useful, but no longer sure?
-        * ``'{"foo": "bar"}'``
 
-        In practice this calls :meth:`jsonify_value()` to convert the
+            # TODO: need to describe the scenario when this is true
-        ``field.cstruct`` value to string.
+            if key not in dform:
-        """
+                # log.warning("field '%s' is missing from deform", key)
-        if isinstance(field, str):
+                continue
-            dform = self.get_deform()
-            field = dform[field]
 
-        return self.jsonify_value(field.cstruct)
+            field = dform[key]
 
-    def jsonify_value(self, value):
+            # if hasattr(field, 'children'):
-        """
+            #     for subfield in field.children:
-        Convert a Python value to JSON string.
+            #         assign(subfield)
 
-        See also :meth:`get_vue_field_value()`.
+            assign(field)
-        """
-        if value is colander.null:
-            return 'null'
 
-        return json.dumps(value)
+        return model_data
 
     def validate(self):
         """
@@ -824,6 +953,20 @@ class Form:
         try:
             self.validated = dform.validate(controls)
         except deform.ValidationFailure:
+            log.debug("form not valid: %s", dform.error)
             return False
 
         return self.validated
+
+    def get_field_errors(self, field):
+        """
+        Return a list of error messages for the given field.
+
+        Not useful unless a call to :meth:`validate()` failed.
+        """
+        dform = self.get_deform()
+        if field in dform:
+            error = dform[field].errormsg
+            if error:
+                return [error]
+        return []

src/wuttaweb/forms/schema.py

@@ -0,0 +1,259 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  wuttaweb -- Web App for Wutta Framework
+#  Copyright © 2024 Lance Edgar
+#
+#  This file is part of Wutta Framework.
+#
+#  Wutta Framework is free software: you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option) any
+#  later version.
+#
+#  Wutta Framework is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Wutta Framework.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Form schema types
+"""
+
+import colander
+
+from wuttaweb.db import Session
+from wuttaweb.forms import widgets
+from wuttjamaican.db.model import Person
+
+
+class ObjectNode(colander.SchemaNode):
+    """
+    Custom schema node class which adds methods for compatibility with
+    ColanderAlchemy.  This is a direct subclass of
+    :class:`colander:colander.SchemaNode`.
+
+    ColanderAlchemy will call certain methods on any node found in the
+    schema.  However these methods are not "standard" and only exist
+    for ColanderAlchemy nodes.
+
+    So we must add nodes using this class, to ensure the node has all
+    methods needed by ColanderAlchemy.
+    """
+
+    def dictify(self, obj):
+        """
+        This method is called by ColanderAlchemy when translating the
+        in-app Python object to a value suitable for use in the form
+        data dict.
+
+        The logic here will look for a ``dictify()`` method on the
+        node's "type" instance (``self.typ``; see also
+        :class:`colander:colander.SchemaNode`) and invoke it if found.
+
+        For an example type which is supported in this way, see
+        :class:`ObjectRef`.
+
+        If the node's type does not have a ``dictify()`` method, this
+        will raise ``NotImplementeError``.
+        """
+        if hasattr(self.typ, 'dictify'):
+            return self.typ.dictify(obj)
+
+        class_name = self.typ.__class__.__name__
+        raise NotImplementedError(f"you must define {class_name}.dictify()")
+
+    def objectify(self, value):
+        """
+        This method is called by ColanderAlchemy when translating form
+        data to the final Python representation.
+
+        The logic here will look for an ``objectify()`` method on the
+        node's "type" instance (``self.typ``; see also
+        :class:`colander:colander.SchemaNode`) and invoke it if found.
+
+        For an example type which is supported in this way, see
+        :class:`ObjectRef`.
+
+        If the node's type does not have an ``objectify()`` method,
+        this will raise ``NotImplementeError``.
+        """
+        if hasattr(self.typ, 'objectify'):
+            return self.typ.objectify(value)
+
+        class_name = self.typ.__class__.__name__
+        raise NotImplementedError(f"you must define {class_name}.objectify()")
+
+
+class ObjectRef(colander.SchemaType):
+    """
+    Custom schema type for a model class reference field.
+
+    This expects the incoming ``appstruct`` to be either a model
+    record instance, or ``None``.
+
+    Serializes to the instance UUID as string, or ``colander.null``;
+    form data should be of the same nature.
+
+    This schema type is not useful directly, but various other types
+    will subclass it.  Each should define (at least) the
+    :attr:`model_class` attribute or property.
+
+    :param request: Current :term:`request` object.
+
+    :param empty_option: If a select widget is used, this determines
+       whether an empty option is included for the dropdown.  Set
+       this to one of the following to add an empty option:
+
+       * ``True`` to add the default empty option
+       * label text for the empty option
+       * tuple of ``(value, label)`` for the empty option
+
+       Note that in the latter, ``value`` must be a string.
+    """
+
+    default_empty_option = ('', "(none)")
+
+    def __init__(
+            self,
+            request,
+            empty_option=None,
+            session=None,
+            *args,
+            **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.request = request
+        self.config = self.request.wutta_config
+        self.app = self.config.get_app()
+        self.model_instance = None
+        self.session = session or Session()
+
+        if empty_option:
+            if empty_option is True:
+                self.empty_option = self.default_empty_option
+            elif isinstance(empty_option, tuple) and len(empty_option) == 2:
+                self.empty_option = empty_option
+            else:
+                self.empty_option = ('', str(empty_option))
+        else:
+            self.empty_option = None
+
+    @property
+    def model_class(self):
+        """
+        Should be a reference to the model class to which this schema
+        type applies
+        (e.g. :class:`~wuttjamaican:wuttjamaican.db.model.base.Person`).
+        """
+        class_name = self.__class__.__name__
+        raise NotImplementedError(f"you must define {class_name}.model_class")
+
+    def serialize(self, node, appstruct):
+        """ """
+        if appstruct is colander.null:
+            return colander.null
+
+        # nb. keep a ref to this for later use
+        node.model_instance = appstruct
+
+        # serialize to uuid
+        return appstruct.uuid
+
+    def deserialize(self, node, cstruct):
+        """ """
+        if not cstruct:
+            return colander.null
+
+        # nb. use shortcut to fetch model instance from DB
+        return self.objectify(cstruct)
+
+    def dictify(self, obj):
+        """ """
+
+        # TODO: would we ever need to do something else?
+        return obj
+
+    def objectify(self, value):
+        """
+        For the given UUID value, returns the object it represents
+        (based on :attr:`model_class`).
+
+        If the value is empty, returns ``None``.
+
+        If the value is not empty but object cannot be found, raises
+        ``colander.Invalid``.
+        """
+        if not value:
+            return
+
+        if isinstance(value, self.model_class):
+            return value
+
+        # fetch object from DB
+        model = self.app.model
+        obj = self.session.query(self.model_class).get(value)
+
+        # raise error if not found
+        if not obj:
+            class_name = self.model_class.__name__
+            raise ValueError(f"{class_name} not found: {value}")
+
+        return obj
+
+    def get_query(self):
+        """
+        Returns the main SQLAlchemy query responsible for locating the
+        dropdown choices for the select widget.
+
+        This is called by :meth:`widget_maker()`.
+        """
+        query = self.session.query(self.model_class)
+        query = self.sort_query(query)
+        return query
+
+    def sort_query(self, query):
+        """
+        TODO
+        """
+        return query
+
+    def widget_maker(self, **kwargs):
+        """
+        This method is responsible for producing the default widget
+        for the schema node.
+
+        Deform calls this method automatically when constructing the
+        default widget for a field.
+
+        :returns: Instance of
+           :class:`~wuttaweb.forms.widgets.ObjectRefWidget`.
+        """
+
+        if 'values' not in kwargs:
+            query = self.get_query()
+            objects = query.all()
+            values = [(obj.uuid, str(obj))
+                      for obj in objects]
+            if self.empty_option:
+                values.insert(0, self.empty_option)
+            kwargs['values'] = values
+
+        return widgets.ObjectRefWidget(self.request, **kwargs)
+
+
+class PersonRef(ObjectRef):
+    """
+    Custom schema type for a ``Person`` reference field.
+
+    This is a subclass of :class:`ObjectRef`.
+    """
+    model_class = Person
+
+    def sort_query(self, query):
+        """ """
+        return query.order_by(self.model_class.full_name)

src/wuttaweb/forms/widgets.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  wuttaweb -- Web App for Wutta Framework
+#  Copyright © 2024 Lance Edgar
+#
+#  This file is part of Wutta Framework.
+#
+#  Wutta Framework is free software: you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option) any
+#  later version.
+#
+#  Wutta Framework is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Wutta Framework.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Form widgets
+
+This module defines some custom widgets for use with WuttaWeb.
+
+However for convenience it also makes other Deform widgets available
+in the namespace:
+
+* :class:`deform:deform.widget.Widget` (base class)
+* :class:`deform:deform.widget.TextInputWidget`
+* :class:`deform:deform.widget.SelectWidget`
+"""
+
+from deform.widget import Widget, TextInputWidget, SelectWidget
+from webhelpers2.html import HTML
+
+
+class ObjectRefWidget(SelectWidget):
+    """
+    Widget for use with model "object reference" fields, e.g.  foreign
+    key UUID => TargetModel instance.
+
+    While you may create instances of this widget directly, it
+    normally happens automatically when schema nodes of the
+    :class:`~wuttaweb.forms.schema.ObjectRef` (sub)type are part of
+    the form schema; via
+    :meth:`~wuttaweb.forms.schema.ObjectRef.widget_maker()`.
+
+    .. attribute:: model_instance
+
+       Reference to the model record instance, i.e. the "far side" of
+       the foreign key relationship.
+
+       .. note::
+
+          You do not need to provide the ``model_instance`` when
+          constructing the widget.  Rather, it is set automatically
+          when the :class:`~wuttaweb.forms.schema.ObjectRef` type
+          instance (associated with the node) is serialized.
+    """
+
+    def __init__(self, request, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.request = request
+
+    def serialize(self, field, cstruct, **kw):
+        """
+        Serialize the widget.
+
+        In readonly mode, returns a ``<span>`` tag around the
+        :attr:`model_instance` rendered as string.
+
+        Otherwise renders via the ``deform/select`` template.
+        """
+        readonly = kw.get('readonly', self.readonly)
+        if readonly:
+            obj = field.schema.model_instance
+            return HTML.tag('span', c=str(obj or ''))
+
+        return super().serialize(field, cstruct, **kw)

src/wuttaweb/grids/base.py

@@ -24,11 +24,18 @@
 Base grid classes
 """
 
+import json
+import logging
+
+import sqlalchemy as sa
+
 from pyramid.renderers import render
 from webhelpers2.html import HTML
 
-from wuttaweb.forms import FieldList
+from wuttaweb.util import FieldList, get_model_fields, make_json_safe
-from wuttaweb.util import get_model_fields
+
+
+log = logging.getLogger(__name__)
 
 
 class Grid:
@@ -114,11 +121,7 @@ class Grid:
         self.config = self.request.wutta_config
         self.app = self.config.get_app()
 
-        columns = columns or self.get_columns()
+        self.set_columns(columns or self.get_columns())
-        if columns:
-            self.set_columns(columns)
-        else:
-            self.columns = None
 
     def get_columns(self):
         """
@@ -139,6 +142,8 @@ class Grid:
         if columns:
             return columns
 
+        return []
+
     def get_model_columns(self, model_class=None):
         """
         This method is a shortcut which calls
@@ -172,6 +177,23 @@ class Grid:
         """
         self.columns = FieldList(columns)
 
+    def remove(self, *keys):
+        """
+        Remove some column(s) from the grid.
+
+        This is a convenience to allow removal of multiple columns at
+        once::
+
+           grid.remove('first_field',
+                       'second_field',
+                       'third_field')
+
+        It will remove each column from :attr:`columns`.
+        """
+        for key in keys:
+            if key in self.columns:
+                self.columns.remove(key)
+
     def set_link(self, key, link=True):
         """
         Explicitly enable or disable auto-link behavior for a given
@@ -296,22 +318,52 @@ class Grid:
         Returns a list of Vue-compatible data records.
 
         This uses :attr:`data` as the basis, but may add some extra
-        values to each record for sake of action URLs etc.
+        values to each record, e.g. URLs for :attr:`actions` etc.
 
-        See also :meth:`get_vue_columns()`.
+        Importantly, this also ensures each value in the dict is
+        JSON-serializable, using
+        :func:`~wuttaweb.util.make_json_safe()`.
+
+        :returns: List of data record dicts for use with Vue table
+           component.
         """
-        # use data as-is unless we have actions
+        original_data = self.data or []
-        if not self.actions:
+
-            return self.data
+        # TODO: at some point i thought it was useful to wrangle the
+        # columns here, but now i can't seem to figure out why..?
+
+        # # determine which columns are relevant for data set
+        # columns = None
+        # if not columns:
+        #     columns = self.get_columns()
+        #     if not columns:
+        #         raise ValueError("cannot determine columns for the grid")
+        # columns = set(columns)
+        # if self.model_class:
+        #     mapper = sa.inspect(self.model_class)
+        #     for column in mapper.primary_key:
+        #         columns.add(column.key)
+
+        # # prune data fields for which no column is defined
+        # for i, record in enumerate(original_data):
+        #     original_data[i]= dict([(key, record[key])
+        #                             for key in columns])
 
         # we have action(s), so add URL(s) for each record in data
         data = []
-        for i, record in enumerate(self.data):
+        for i, record in enumerate(original_data):
-            record = dict(record)
+
+            # convert data if needed, for json compat
+            record = make_json_safe(record,
+                                    # TODO: is this a good idea?
+                                    warn=False)
+
+            # add action urls to each record
             for action in self.actions:
                 url = action.get_url(record, i)
                 key = f'_action_url_{action.key}'
                 record[key] = url
+
             data.append(record)
 
         return data

src/wuttaweb/menus.py

@@ -147,6 +147,12 @@ class MenuHandler(GenericHandler):
             'title': "Admin",
             'type': 'menu',
             'items': [
+                {
+                    'title': "Users",
+                    'route': 'users',
+                    'perm': 'users.list',
+                },
+                {'type': 'sep'},
                 {
                     'title': "App Info",
                     'route': 'appinfo',

src/wuttaweb/templates/appinfo/index.mako

@@ -3,7 +3,7 @@
 
 <%def name="page_content()">
 
-  <nav class="panel item-panel">
+  <nav class="panel">
     <p class="panel-heading">Application</p>
     <div class="panel-block">
       <div style="width: 100%;">
@@ -20,7 +20,7 @@
     </div>
   </nav>
 
-  <nav class="panel item-panel">
+  <nav class="panel">
     <p class="panel-heading">Configuration Files</p>
     <div class="panel-block">
       <div style="width: 100%;">

src/wuttaweb/templates/deform/checkbox.pt

@@ -0,0 +1,11 @@
+<div tal:omit-tag=""
+     tal:define="name name|field.name;
+                 oid oid|field.oid;
+                 vmodel vmodel|'modelData.'+oid;">
+  <b-checkbox name="${name}"
+              v-model="${vmodel}"
+              native-value="true"
+              tal:attributes="attributes|field.widget.attributes|{};">
+    {{ ${vmodel} }}
+  </b-checkbox>
+</div>

src/wuttaweb/templates/deform/checked_password.pt

@@ -1,5 +1,6 @@
 <div tal:define="name name|field.name;
-                 vmodel vmodel|'model_'+name;">
+                 oid oid|field.oid;
+                 vmodel vmodel|'modelData.'+oid;">
   ${field.start_mapping()}
   <b-input name="${name}"
            value="${field.widget.redisplay and cstruct or ''}"

src/wuttaweb/templates/deform/password.pt

@@ -1,6 +1,7 @@
 <div tal:omit-tag=""
      tal:define="name name|field.name;
-                 vmodel vmodel|'model_'+name;">
+                 oid oid|field.oid;
+                 vmodel vmodel|'modelData.'+oid;">
   <b-input name="${name}"
            v-model="${vmodel}"
            type="password"

src/wuttaweb/templates/deform/select.pt

@@ -0,0 +1,50 @@
+<div tal:define="
+     name name|field.name;
+     oid oid|field.oid;
+     style style|field.widget.style;
+     size size|field.widget.size;
+     css_class css_class|field.widget.css_class;
+     unicode unicode|str;
+     optgroup_class optgroup_class|field.widget.optgroup_class;
+     multiple multiple|field.widget.multiple;
+     autofocus autofocus|field.autofocus;
+     vmodel vmodel|'modelData.'+oid;"
+     tal:omit-tag="">
+
+  <input type="hidden" name="__start__" value="${name}:sequence"
+         tal:condition="multiple" />
+  <b-select tal:attributes="
+          name name;
+          v-model vmodel;
+          id oid;
+          class string: ${css_class or ''};
+          multiple multiple;
+          size size;
+          style style;
+          autofocus autofocus;
+          attributes|field.widget.attributes|{};">
+    <tal:loop tal:repeat="item values">
+      <optgroup tal:condition="isinstance(item, optgroup_class)"
+                tal:attributes="label item.label">
+        <option tal:repeat="(value, description) item.options"
+                tal:attributes="
+                selected python:field.widget.get_select_value(cstruct, value);
+                readonly 'readonly' in getattr(field.widget, 'attributes', {}) and field.widget.get_select_value(cstruct, item[0]);
+                disabled 'readonly' in getattr(field.widget, 'attributes', {}) and not field.widget.get_select_value(cstruct, item[0]);
+                class css_class;
+                label field.widget.long_label_generator and description;
+                value value"
+                tal:content="field.widget.long_label_generator and field.widget.long_label_generator(item.label, description) or description"/>
+      </optgroup>
+      <option tal:condition="not isinstance(item, optgroup_class)"
+              tal:attributes="
+              selected python:field.widget.get_select_value(cstruct, item[0]);
+              readonly 'readonly' in getattr(field.widget, 'attributes', {}) and field.widget.get_select_value(cstruct, item[0]);
+              disabled 'readonly' in getattr(field.widget, 'attributes', {}) and not field.widget.get_select_value(cstruct, item[0]);
+              class css_class;
+              value item[0];">${item[1]}</option>
+    </tal:loop>
+  </b-select>
+  <input type="hidden" name="__end__" value="${name}:sequence"
+         tal:condition="multiple" />
+</div>

src/wuttaweb/templates/deform/textinput.pt

@@ -1,6 +1,7 @@
 <div tal:omit-tag=""
      tal:define="name name|field.name;
-                 vmodel vmodel|'model_'+name;">
+                 oid oid|field.oid;
+                 vmodel vmodel|'modelData.'+oid;">
   <b-input name="${name}"
            v-model="${vmodel}"
            tal:attributes="attributes|field.widget.attributes|{};" />

src/wuttaweb/templates/forms/vue_template.mako

@@ -56,12 +56,7 @@
 
       % if not form.readonly:
 
-          ## field model values
+          modelData: ${json.dumps(model_data)|n},
-          % for key in form:
-              % if key in dform:
-                  model_${key}: ${form.get_vue_field_value(key)|n},
-              % endif
-          % endfor
 
           % if form.auto_disable_submit:
               formSubmitting: false,

src/wuttaweb/util.py

@@ -25,10 +25,62 @@ Web Utilities
 """
 
 import importlib
+import json
+import logging
 
+import colander
 from webhelpers2.html import HTML, tags
 
 
+log = logging.getLogger(__name__)
+
+
+class FieldList(list):
+    """
+    Convenience wrapper for a form's field list.  This is a subclass
+    of :class:`python:list`.
+
+    You normally would not need to instantiate this yourself, but it
+    is used under the hood for
+    :attr:`~wuttaweb.forms.base.Form.fields` as well as
+    :attr:`~wuttaweb.grids.base.Grid.columns`.
+    """
+
+    def insert_before(self, field, newfield):
+        """
+        Insert a new field, before an existing field.
+
+        :param field: String name for the existing field.
+
+        :param newfield: String name for the new field, to be inserted
+           just before the existing ``field``.
+        """
+        if field in self:
+            i = self.index(field)
+            self.insert(i, newfield)
+        else:
+            log.warning("field '%s' not found, will append new field: %s",
+                        field, newfield)
+            self.append(newfield)
+
+    def insert_after(self, field, newfield):
+        """
+        Insert a new field, after an existing field.
+
+        :param field: String name for the existing field.
+
+        :param newfield: String name for the new field, to be inserted
+           just after the existing ``field``.
+        """
+        if field in self:
+            i = self.index(field)
+            self.insert(i + 1, newfield)
+        else:
+            log.warning("field '%s' not found, will append new field: %s",
+                        field, newfield)
+            self.append(newfield)
+
+
 def get_form_data(request):
     """
     Returns the effective form data for the given request.
@@ -376,3 +428,47 @@ def get_model_fields(config, model_class=None):
             mapper = sa.inspect(model_class)
             fields = list([prop.key for prop in mapper.iterate_properties])
             return fields
+
+
+def make_json_safe(value, key=None, warn=True):
+    """
+    Convert a Python value as needed, to ensure it is compatible with
+    :func:`python:json.dumps()`.
+
+    :param value: Python value.
+
+    :param key: Optional key for the value, if known.  This is used
+       when logging warnings, if applicable.
+
+    :param warn: Whether warnings should be logged if the value is not
+       already JSON-compatible.
+
+    :returns: A (possibly new) Python value which is guaranteed to be
+       JSON-serializable.
+    """
+
+    # convert null => None
+    if value is colander.null:
+        return None
+
+    # recursively convert dict
+    if isinstance(value, dict):
+        parent = dict(value)
+        for key, value in parent.items():
+            parent[key] = make_json_safe(value, key=key, warn=warn)
+        value = parent
+
+    # ensure JSON-compatibility, warn if problems
+    try:
+        json.dumps(value)
+    except TypeError as error:
+        if warn:
+            prefix = "value"
+            if key:
+                prefix += f" for '{key}'"
+            log.warning("%s is not json-friendly: %s", prefix, repr(value))
+        value = str(value)
+        if warn:
+            log.warning("forced value to: %s", value)
+
+    return value

src/wuttaweb/views/essential.py

@@ -33,6 +33,7 @@ That will in turn include the following modules:
 * :mod:`wuttaweb.views.common`
 * :mod:`wuttaweb.views.settings`
 * :mod:`wuttaweb.views.people`
+* :mod:`wuttaweb.views.users`
 """
 
 
@@ -43,6 +44,7 @@ def defaults(config, **kwargs):
     config.include(mod('wuttaweb.views.common'))
     config.include(mod('wuttaweb.views.settings'))
     config.include(mod('wuttaweb.views.people'))
+    config.include(mod('wuttaweb.views.users'))
 
 
 def includeme(config):

src/wuttaweb/views/master.py

@@ -1026,9 +1026,11 @@ class MasterView(View):
         ready to use as-is, but this method can further modify it
         based on request details etc.
         """
+        if 'uuid' in grid.columns:
+            grid.columns.remove('uuid')
+
         for key in self.get_model_key():
             grid.set_link(key)
-            # print("set link:", key)
 
     def get_instance(self, session=None):
         """
@@ -1201,13 +1203,10 @@ class MasterView(View):
         already be "complete" and ready to use as-is, but this method
         can further modify it based on request details etc.
         """
-        model_keys = self.get_model_key()
+        form.remove('uuid')
-
-        if 'uuid' in form:
-            form.fields.remove('uuid')
 
         if self.editing:
-            for key in model_keys:
+            for key in self.get_model_key():
                 form.set_readonly(key)
 
     def objectify(self, form):
@@ -1228,22 +1227,15 @@ class MasterView(View):
 
         See also :meth:`edit_save_form()` which calls this method.
         """
-        model = self.app.model
 
-        model_class = self.get_model_class()
+        # use ColanderAlchemy magic if possible
-        if model_class and issubclass(model_class, model.Base):
+        schema = form.get_schema()
-
+        if hasattr(schema, 'objectify'):
-            # update instance attrs for sqlalchemy model
+            # this returns a model instance
-            if self.creating:
+            return schema.objectify(form.validated,
-                obj = model_class()
+                                    context=form.model_instance)
-            else:
-                obj = form.model_instance
-            data = form.validated
-            for key in form.fields:
-                if key in data:
-                    setattr(obj, key, data[key])
-            return obj
 
+        # otherwise return data dict as-is
         return form.validated
 
     def persist(self, obj, session=None):

src/wuttaweb/views/users.py

@@ -0,0 +1,117 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  wuttaweb -- Web App for Wutta Framework
+#  Copyright © 2024 Lance Edgar
+#
+#  This file is part of Wutta Framework.
+#
+#  Wutta Framework is free software: you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option) any
+#  later version.
+#
+#  Wutta Framework is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Wutta Framework.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Views for users
+"""
+
+import colander
+
+from wuttjamaican.db.model import User
+from wuttaweb.views import MasterView
+from wuttaweb.forms.schema import PersonRef
+from wuttaweb.db import Session
+
+
+class UserView(MasterView):
+    """
+    Master view for users.
+
+    Notable URLs provided by this class:
+
+    * ``/users/``
+    * ``/users/new``
+    * ``/users/XXX``
+    * ``/users/XXX/edit``
+    * ``/users/XXX/delete``
+    """
+    model_class = User
+
+    grid_columns = [
+        'username',
+        'person',
+        'active',
+    ]
+
+    # TODO: master should handle this, possibly via configure_form()
+    def get_query(self, session=None):
+        """ """
+        model = self.app.model
+        query = super().get_query(session=session)
+        return query.order_by(model.User.username)
+
+    def configure_grid(self, g):
+        """ """
+        super().configure_grid(g)
+
+        # never show these
+        g.remove('person_uuid',
+                 'role_refs',
+                 'password')
+
+        # username
+        g.set_link('username')
+
+        # person
+        g.set_link('person')
+
+    def configure_form(self, f):
+        """ """
+        super().configure_form(f)
+
+        # never show these
+        f.remove('person_uuid',
+                 'password',
+                 'role_refs')
+
+        # person
+        f.set_node('person', PersonRef(self.request, empty_option=True))
+        f.set_required('person', False)
+
+        # username
+        f.set_validator('username', self.unique_username)
+
+    def unique_username(self, node, value):
+        """ """
+        model = self.app.model
+        session = Session()
+
+        query = session.query(model.User)\
+                       .filter(model.User.username == value)
+
+        if self.editing:
+            uuid = self.request.matchdict['uuid']
+            query = query.filter(model.User.uuid != uuid)
+
+        if query.count():
+            node.raise_invalid("Username must be unique")
+
+
+def defaults(config, **kwargs):
+    base = globals()
+
+    UserView = kwargs.get('UserView', base['UserView'])
+    UserView.defaults(config)
+
+
+def includeme(config):
+    defaults(config)

tests/forms/test_base.py

@@ -8,46 +8,15 @@ import deform
 from pyramid import testing
 
 from wuttjamaican.conf import WuttaConfig
-from wuttaweb.forms import base
+from wuttaweb.forms import base, widgets
 from wuttaweb import helpers
 
 
-class TestFieldList(TestCase):
-
-    def test_insert_before(self):
-        fields = base.FieldList(['f1', 'f2'])
-        self.assertEqual(fields, ['f1', 'f2'])
-
-        # typical
-        fields.insert_before('f1', 'XXX')
-        self.assertEqual(fields, ['XXX', 'f1', 'f2'])
-        fields.insert_before('f2', 'YYY')
-        self.assertEqual(fields, ['XXX', 'f1', 'YYY', 'f2'])
-
-        # appends new field if reference field is invalid
-        fields.insert_before('f3', 'ZZZ')
-        self.assertEqual(fields, ['XXX', 'f1', 'YYY', 'f2', 'ZZZ'])
-
-    def test_insert_after(self):
-        fields = base.FieldList(['f1', 'f2'])
-        self.assertEqual(fields, ['f1', 'f2'])
-
-        # typical
-        fields.insert_after('f1', 'XXX')
-        self.assertEqual(fields, ['f1', 'XXX', 'f2'])
-        fields.insert_after('XXX', 'YYY')
-        self.assertEqual(fields, ['f1', 'XXX', 'YYY', 'f2'])
-
-        # appends new field if reference field is invalid
-        fields.insert_after('f3', 'ZZZ')
-        self.assertEqual(fields, ['f1', 'XXX', 'YYY', 'f2', 'ZZZ'])
-
-
 class TestForm(TestCase):
 
     def setUp(self):
         self.config = WuttaConfig(defaults={
-            'wutta.web.menus.handler_spec': 'tests.utils:NullMenuHandler',
+            'wutta.web.menus.handler_spec': 'tests.util:NullMenuHandler',
         })
         self.app = self.config.get_app()
         self.request = testing.DummyRequest(wutta_config=self.config, use_oruga=False)
@@ -74,7 +43,7 @@ class TestForm(TestCase):
 
     def test_init_with_none(self):
         form = self.make_form()
-        self.assertIsNone(form.fields)
+        self.assertEqual(form.fields, [])
 
     def test_init_with_fields(self):
         form = self.make_form(fields=['foo', 'bar'])
@@ -115,6 +84,79 @@ class TestForm(TestCase):
         form.set_fields(['baz'])
         self.assertEqual(form.fields, ['baz'])
 
+    def test_remove(self):
+        form = self.make_form(fields=['one', 'two', 'three', 'four'])
+        self.assertEqual(form.fields, ['one', 'two', 'three', 'four'])
+        form.remove('two', 'three')
+        self.assertEqual(form.fields, ['one', 'four'])
+
+    def test_set_node(self):
+        form = self.make_form(fields=['foo', 'bar'])
+        self.assertEqual(form.nodes, {})
+
+        # complete node
+        node = colander.SchemaNode(colander.Bool(), name='foo')
+        form.set_node('foo', node)
+        self.assertIs(form.nodes['foo'], node)
+
+        # type only
+        typ = colander.Bool()
+        form.set_node('foo', typ)
+        node = form.nodes['foo']
+        self.assertIsInstance(node, colander.SchemaNode)
+        self.assertIsInstance(node.typ, colander.Bool)
+        self.assertEqual(node.name, 'foo')
+
+        # schema is updated if already present
+        schema = form.get_schema()
+        self.assertIsNotNone(schema)
+        typ = colander.Date()
+        form.set_node('foo', typ)
+        node = form.nodes['foo']
+        self.assertIsInstance(node, colander.SchemaNode)
+        self.assertIsInstance(node.typ, colander.Date)
+        self.assertEqual(node.name, 'foo')
+
+    def test_set_widget(self):
+        form = self.make_form(fields=['foo', 'bar'])
+        self.assertEqual(form.widgets, {})
+
+        # basic
+        widget = widgets.SelectWidget()
+        form.set_widget('foo', widget)
+        self.assertIs(form.widgets['foo'], widget)
+
+        # schema is updated if already present
+        schema = form.get_schema()
+        self.assertIsNotNone(schema)
+        self.assertIs(schema['foo'].widget, widget)
+        new_widget = widgets.TextInputWidget()
+        form.set_widget('foo', new_widget)
+        self.assertIs(form.widgets['foo'], new_widget)
+        self.assertIs(schema['foo'].widget, new_widget)
+
+    def test_set_validator(self):
+        form = self.make_form(fields=['foo', 'bar'])
+        self.assertEqual(form.validators, {})
+
+        def validate1(node, value):
+            pass
+
+        # basic
+        form.set_validator('foo', validate1)
+        self.assertIs(form.validators['foo'], validate1)
+
+        def validate2(node, value):
+            pass
+
+        # schema is updated if already present
+        schema = form.get_schema()
+        self.assertIsNotNone(schema)
+        self.assertIs(schema['foo'].validator, validate1)
+        form.set_validator('foo', validate2)
+        self.assertIs(form.validators['foo'], validate2)
+        self.assertIs(schema['foo'].validator, validate2)
+
     def test_get_schema(self):
         model = self.app.model
         form = self.make_form()
@@ -144,6 +186,13 @@ class TestForm(TestCase):
         self.assertIn('name', schema)
         self.assertIn('value', schema)
 
+        # but node overrides are honored when auto-generating
+        form = self.make_form(model_class=model.Setting)
+        value_node = colander.SchemaNode(colander.Bool(), name='value')
+        form.set_node('value', value_node)
+        schema = form.get_schema()
+        self.assertIs(schema['value'], value_node)
+
         # schema is auto-generated if model_instance provided
         form = self.make_form(model_instance=model.Setting(name='uhoh'))
         self.assertEqual(form.fields, ['name', 'value'])
@@ -166,7 +215,23 @@ class TestForm(TestCase):
         form.set_required('bar', False)
         schema = form.get_schema()
         self.assertIs(schema['foo'].missing, colander.required)
-        self.assertIsNone(schema['bar'].missing)
+        self.assertIs(schema['bar'].missing, colander.null)
+
+        # validator overrides are honored
+        def validate(node, value): pass
+        form = self.make_form(model_class=model.Setting)
+        form.set_validator('name', validate)
+        schema = form.get_schema()
+        self.assertIs(schema['name'].validator, validate)
+
+        # validator can be set for whole form
+        form = self.make_form(model_class=model.Setting)
+        schema = form.get_schema()
+        self.assertIsNone(schema.validator)
+        form = self.make_form(model_class=model.Setting)
+        form.set_validator(None, validate)
+        schema = form.get_schema()
+        self.assertIs(schema.validator, validate)
 
     def test_get_deform(self):
         model = self.app.model
@@ -372,34 +437,6 @@ class TestForm(TestCase):
         self.assertEqual(len(errors), 1)
         self.assertEqual(errors[0], "something is wrong")
 
-    def test_get_vue_field_value(self):
-        schema = self.make_schema()
-        form = self.make_form(schema=schema)
-
-        # null field value
-        value = form.get_vue_field_value('foo')
-        self.assertEqual(value, 'null')
-
-        # non-default / explicit value
-        # TODO: surely need a different approach to set value
-        dform = form.get_deform()
-        dform['foo'].cstruct = 'blarg'
-        value = form.get_vue_field_value('foo')
-        self.assertEqual(value, '"blarg"')
-
-    def test_jsonify_value(self):
-        form = self.make_form()
-
-        # null field value
-        value = form.jsonify_value(colander.null)
-        self.assertEqual(value, 'null')
-        value = form.jsonify_value(None)
-        self.assertEqual(value, 'null')
-
-        # string value
-        value = form.jsonify_value('blarg')
-        self.assertEqual(value, '"blarg"')
-
     def test_validate(self):
         schema = self.make_schema()
         form = self.make_form(schema=schema)

tests/forms/test_schema.py

@@ -0,0 +1,199 @@
+# -*- coding: utf-8; -*-
+
+from unittest.mock import patch
+
+import colander
+from pyramid import testing
+
+from sqlalchemy import orm
+
+from wuttjamaican.conf import WuttaConfig
+from wuttaweb.forms import schema as mod
+from tests.util import DataTestCase
+
+
+class TestObjectNode(DataTestCase):
+
+    def setUp(self):
+        self.setup_db()
+        self.request = testing.DummyRequest(wutta_config=self.config)
+
+    def test_dictify(self):
+        model = self.app.model
+        person = model.Person(full_name="Betty Boop")
+
+        # unsupported type raises error
+        node = mod.ObjectNode(colander.String())
+        self.assertRaises(NotImplementedError, node.dictify, person)
+
+        # but supported type can dictify
+        node = mod.ObjectNode(mod.PersonRef(self.request))
+        value = node.dictify(person)
+        self.assertIs(value, person)
+
+    def test_objectify(self):
+        model = self.app.model
+        person = model.Person(full_name="Betty Boop")
+
+        # unsupported type raises error
+        node = mod.ObjectNode(colander.String())
+        self.assertRaises(NotImplementedError, node.objectify, person)
+
+        # but supported type can objectify
+        node = mod.ObjectNode(mod.PersonRef(self.request))
+        value = node.objectify(person)
+        self.assertIs(value, person)
+
+
+class TestObjectRef(DataTestCase):
+
+    def setUp(self):
+        self.setup_db()
+        self.request = testing.DummyRequest(wutta_config=self.config)
+
+    def test_empty_option(self):
+
+        # null by default
+        typ = mod.ObjectRef(self.request)
+        self.assertIsNone(typ.empty_option)
+
+        # passing true yields default empty option
+        typ = mod.ObjectRef(self.request, empty_option=True)
+        self.assertEqual(typ.empty_option, ('', "(none)"))
+
+        # can set explicitly
+        typ = mod.ObjectRef(self.request, empty_option=('foo', 'bar'))
+        self.assertEqual(typ.empty_option, ('foo', 'bar'))
+
+        # can set just a label
+        typ = mod.ObjectRef(self.request, empty_option="(empty)")
+        self.assertEqual(typ.empty_option, ('', "(empty)"))
+
+    def test_model_class(self):
+        typ = mod.ObjectRef(self.request)
+        self.assertRaises(NotImplementedError, getattr, typ, 'model_class')
+
+    def test_serialize(self):
+        model = self.app.model
+        node = colander.SchemaNode(colander.String())
+
+        # null
+        typ = mod.ObjectRef(self.request)
+        value = typ.serialize(node, colander.null)
+        self.assertIs(value, colander.null)
+
+        # model instance
+        person = model.Person(full_name="Betty Boop")
+        self.session.add(person)
+        self.session.commit()
+        self.assertIsNotNone(person.uuid)
+        typ = mod.ObjectRef(self.request)
+        value = typ.serialize(node, person)
+        self.assertEqual(value, person.uuid)
+
+    def test_deserialize(self):
+        model = self.app.model
+        node = colander.SchemaNode(colander.String())
+
+        # null
+        typ = mod.ObjectRef(self.request)
+        value = typ.deserialize(node, colander.null)
+        self.assertIs(value, colander.null)
+
+        # model instance
+        person = model.Person(full_name="Betty Boop")
+        self.session.add(person)
+        self.session.commit()
+        self.assertIsNotNone(person.uuid)
+        with patch.object(mod.ObjectRef, 'model_class', new=model.Person):
+            typ = mod.ObjectRef(self.request, session=self.session)
+            value = typ.deserialize(node, person.uuid)
+            self.assertIs(value, person)
+
+    def test_dictify(self):
+        model = self.app.model
+        node = colander.SchemaNode(colander.String())
+
+        # model instance
+        person = model.Person(full_name="Betty Boop")
+        self.session.add(person)
+        self.session.commit()
+        self.assertIsNotNone(person.uuid)
+        typ = mod.ObjectRef(self.request)
+        value = typ.dictify(person)
+        self.assertIs(value, person)
+
+    def test_objectify(self):
+        model = self.app.model
+        node = colander.SchemaNode(colander.String())
+
+        # null
+        typ = mod.ObjectRef(self.request)
+        value = typ.objectify(None)
+        self.assertIsNone(value)
+
+        # model instance
+        person = model.Person(full_name="Betty Boop")
+        self.session.add(person)
+        self.session.commit()
+        self.assertIsNotNone(person.uuid)
+        with patch.object(mod.ObjectRef, 'model_class', new=model.Person):
+            typ = mod.ObjectRef(self.request, session=self.session)
+            value = typ.objectify(person.uuid)
+            self.assertIs(value, person)
+
+        # error if not found
+        with patch.object(mod.ObjectRef, 'model_class', new=model.Person):
+            typ = mod.ObjectRef(self.request, session=self.session)
+            self.assertRaises(ValueError, typ.objectify, 'WRONG-UUID')
+
+    def test_get_query(self):
+        model = self.app.model
+        with patch.object(mod.ObjectRef, 'model_class', new=model.Person):
+            typ = mod.ObjectRef(self.request, session=self.session)
+            query = typ.get_query()
+            self.assertIsInstance(query, orm.Query)
+
+    def test_sort_query(self):
+        model = self.app.model
+        with patch.object(mod.ObjectRef, 'model_class', new=model.Person):
+            typ = mod.ObjectRef(self.request, session=self.session)
+            query = typ.get_query()
+            sorted_query = typ.sort_query(query)
+            self.assertIs(sorted_query, query)
+
+    def test_widget_maker(self):
+        model = self.app.model
+        person = model.Person(full_name="Betty Boop")
+        self.session.add(person)
+        self.session.commit()
+
+        # basic
+        with patch.object(mod.ObjectRef, 'model_class', new=model.Person):
+            typ = mod.ObjectRef(self.request, session=self.session)
+            widget = typ.widget_maker()
+            self.assertEqual(len(widget.values), 1)
+            self.assertEqual(widget.values[0][1], "Betty Boop")
+
+        # empty option
+        with patch.object(mod.ObjectRef, 'model_class', new=model.Person):
+            typ = mod.ObjectRef(self.request, session=self.session, empty_option=True)
+            widget = typ.widget_maker()
+            self.assertEqual(len(widget.values), 2)
+            self.assertEqual(widget.values[0][1], "(none)")
+            self.assertEqual(widget.values[1][1], "Betty Boop")
+
+
+class TestPersonRef(DataTestCase):
+
+    def setUp(self):
+        self.setup_db()
+        self.request = testing.DummyRequest(wutta_config=self.config)
+
+    def test_sort_query(self):
+        typ = mod.PersonRef(self.request, session=self.session)
+        query = typ.get_query()
+        self.assertIsInstance(query, orm.Query)
+        sorted_query = typ.sort_query(query)
+        self.assertIsInstance(sorted_query, orm.Query)
+        self.assertIsNot(sorted_query, query)

tests/forms/test_widgets.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8; -*-
+
+import colander
+import deform
+from pyramid import testing
+
+from wuttaweb.forms import widgets
+from wuttaweb.forms.schema import PersonRef
+from tests.util import WebTestCase
+
+class TestObjectRefWidget(WebTestCase):
+
+    def test_serialize(self):
+        model = self.app.model
+        person = model.Person(full_name="Betty Boop")
+        self.session.add(person)
+        self.session.commit()
+
+        # standard (editable)
+        node = colander.SchemaNode(PersonRef(self.request, session=self.session))
+        widget = widgets.ObjectRefWidget(self.request)
+        field = deform.Field(node)
+        html = widget.serialize(field, person.uuid)
+        self.assertIn('<select ', html)
+
+        # readonly
+        node = colander.SchemaNode(PersonRef(self.request, session=self.session))
+        node.model_instance = person
+        widget = widgets.ObjectRefWidget(self.request)
+        field = deform.Field(node)
+        html = widget.serialize(field, person.uuid, readonly=True)
+        self.assertEqual(html, '<span>Betty Boop</span>')

tests/grids/test_base.py

@@ -14,7 +14,7 @@ class TestGrid(TestCase):
 
     def setUp(self):
         self.config = WuttaConfig(defaults={
-            'wutta.web.menus.handler_spec': 'tests.utils:NullMenuHandler',
+            'wutta.web.menus.handler_spec': 'tests.util:NullMenuHandler',
         })
         self.app = self.config.get_app()
 
@@ -35,7 +35,7 @@ class TestGrid(TestCase):
         # empty
         grid = self.make_grid()
         self.assertIsNone(grid.key)
-        self.assertIsNone(grid.columns)
+        self.assertEqual(grid.columns, [])
         self.assertIsNone(grid.data)
 
         # now with columns
@@ -56,8 +56,8 @@ class TestGrid(TestCase):
 
         # empty
         grid = self.make_grid()
-        self.assertIsNone(grid.columns)
+        self.assertEqual(grid.columns, [])
-        self.assertIsNone(grid.get_columns())
+        self.assertEqual(grid.get_columns(), [])
 
         # explicit
         grid = self.make_grid(columns=['foo', 'bar'])
@@ -69,6 +69,12 @@ class TestGrid(TestCase):
         self.assertEqual(grid.columns, ['name', 'value'])
         self.assertEqual(grid.get_columns(), ['name', 'value'])
 
+    def test_remove(self):
+        grid = self.make_grid(columns=['one', 'two', 'three', 'four'])
+        self.assertEqual(grid.columns, ['one', 'two', 'three', 'four'])
+        grid.remove('two', 'three')
+        self.assertEqual(grid.columns, ['one', 'four'])
+
     def test_linked_columns(self):
         grid = self.make_grid(columns=['foo', 'bar'])
         self.assertEqual(grid.linked_columns, [])
@@ -118,18 +124,17 @@ class TestGrid(TestCase):
 
     def test_get_vue_data(self):
 
-        # null by default
+        # empty if no columns defined
         grid = self.make_grid()
         data = grid.get_vue_data()
-        self.assertIsNone(data)
+        self.assertEqual(data, [])
 
-        # is usually a list
+        # typical data is a list
         mydata = [
             {'foo': 'bar'},
         ]
-        grid = self.make_grid(data=mydata)
+        grid = self.make_grid(columns=['foo'], data=mydata)
         data = grid.get_vue_data()
-        self.assertIs(data, mydata)
         self.assertEqual(data, [{'foo': 'bar'}])
 
         # if grid has actions, that list may be supplemented

tests/test_subscribers.py

@@ -215,7 +215,7 @@ class TestBeforeRender(TestCase):
 
     def setUp(self):
         self.config = WuttaConfig(defaults={
-            'wutta.web.menus.handler_spec': 'tests.utils:NullMenuHandler',
+            'wutta.web.menus.handler_spec': 'tests.util:NullMenuHandler',
         })
 
     def make_request(self):

tests/test_util.py

@@ -1,8 +1,10 @@
 # -*- coding: utf-8; -*-
 
+import json
 from unittest import TestCase
 from unittest.mock import patch, MagicMock
 
+import colander
 from fanstatic import Library, Resource
 from pyramid import testing
 
@@ -10,6 +12,37 @@ from wuttjamaican.conf import WuttaConfig
 from wuttaweb import util
 
 
+class TestFieldList(TestCase):
+
+    def test_insert_before(self):
+        fields = util.FieldList(['f1', 'f2'])
+        self.assertEqual(fields, ['f1', 'f2'])
+
+        # typical
+        fields.insert_before('f1', 'XXX')
+        self.assertEqual(fields, ['XXX', 'f1', 'f2'])
+        fields.insert_before('f2', 'YYY')
+        self.assertEqual(fields, ['XXX', 'f1', 'YYY', 'f2'])
+
+        # appends new field if reference field is invalid
+        fields.insert_before('f3', 'ZZZ')
+        self.assertEqual(fields, ['XXX', 'f1', 'YYY', 'f2', 'ZZZ'])
+
+    def test_insert_after(self):
+        fields = util.FieldList(['f1', 'f2'])
+        self.assertEqual(fields, ['f1', 'f2'])
+
+        # typical
+        fields.insert_after('f1', 'XXX')
+        self.assertEqual(fields, ['f1', 'XXX', 'f2'])
+        fields.insert_after('XXX', 'YYY')
+        self.assertEqual(fields, ['f1', 'XXX', 'YYY', 'f2'])
+
+        # appends new field if reference field is invalid
+        fields.insert_after('f3', 'ZZZ')
+        self.assertEqual(fields, ['f1', 'XXX', 'YYY', 'f2', 'ZZZ'])
+
+
 class TestGetLibVer(TestCase):
 
     def setUp(self):
@@ -455,3 +488,40 @@ class TestRenderCsrfToken(TestCase):
         self.assertIn('name="_csrf"', html)
         token = util.get_csrf_token(self.request)
         self.assertIn(f'value="{token}"', html)
+
+
+class TestMakeJsonSafe(TestCase):
+
+    def setUp(self):
+        self.config = WuttaConfig()
+        self.app = self.config.get_app()
+
+    def test_null(self):
+        value = util.make_json_safe(colander.null)
+        self.assertIsNone(value)
+
+        value = util.make_json_safe(None)
+        self.assertIsNone(value)
+
+    def test_invalid(self):
+        model = self.app.model
+        person = model.Person(full_name="Betty Boop")
+        self.assertRaises(TypeError, json.dumps, person)
+        value = util.make_json_safe(person, key='person')
+        self.assertEqual(value, "Betty Boop")
+
+    def test_dict(self):
+        model = self.app.model
+        person = model.Person(full_name="Betty Boop")
+
+        data = {
+            'foo': 'bar',
+            'person': person,
+        }
+
+        self.assertRaises(TypeError, json.dumps, data)
+        value = util.make_json_safe(data)
+        self.assertEqual(value, {
+            'foo': 'bar',
+            'person': "Betty Boop",
+        })

tests/util.py

@@ -7,9 +7,36 @@ from pyramid import testing
 
 from wuttjamaican.conf import WuttaConfig
 from wuttaweb import subscribers
+from wuttaweb.menus import MenuHandler
 
 
-class WebTestCase(TestCase):
+class DataTestCase(TestCase):
+    """
+    Base class for test suites requiring a full (typical) database.
+    """
+
+    def setUp(self):
+        self.setup_db()
+
+    def setup_db(self):
+        self.config = WuttaConfig(defaults={
+            'wutta.db.default.url': 'sqlite://',
+        })
+        self.app = self.config.get_app()
+
+        # init db
+        model = self.app.model
+        model.Base.metadata.create_all(bind=self.config.appdb_engine)
+        self.session = self.app.make_session()
+
+    def tearDown(self):
+        self.teardown_db()
+
+    def teardown_db(self):
+        pass
+
+
+class WebTestCase(DataTestCase):
     """
     Base class for test suites requiring a full (typical) web app.
     """
@@ -18,24 +45,15 @@ class WebTestCase(TestCase):
         self.setup_web()
 
     def setup_web(self):
-        self.config = WuttaConfig(defaults={
+        self.setup_db()
-            'wutta.db.default.url': 'sqlite://',
-            'wutta.web.menus.handler_spec': 'tests.utils:NullMenuHandler',
-        })
-
         self.request = testing.DummyRequest()
-
         self.pyramid_config = testing.setUp(request=self.request, settings={
             'wutta_config': self.config,
             'mako.directories': ['wuttaweb:templates'],
+            # TODO: have not need this yet, but will?
+            # 'pyramid_deform.template_search_path': 'wuttaweb:templates/deform',
         })
 
-        # init db
-        self.app = self.config.get_app()
-        model = self.app.model
-        model.Base.metadata.create_all(bind=self.config.appdb_engine)
-        self.session = self.app.make_session()
-
         # init web
         self.pyramid_config.include('pyramid_mako')
         self.pyramid_config.include('wuttaweb.static')
@@ -55,3 +73,12 @@ class WebTestCase(TestCase):
 
     def teardown_web(self):
         testing.tearDown()
+        self.teardown_db()
+
+
+class NullMenuHandler(MenuHandler):
+    """
+    Dummy menu handler for testing.
+    """
+    def make_menus(self, request, **kwargs):
+        return []

tests/utils.py

@@ -1,11 +0,0 @@
-# -*- coding: utf-8; -*-
-
-from wuttaweb.menus import MenuHandler
-
-
-class NullMenuHandler(MenuHandler):
-    """
-    Dummy menu handler for testing.
-    """
-    def make_menus(self, request, **kwargs):
-        return []

tests/views/test_master.py

@@ -11,8 +11,7 @@ from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 from wuttjamaican.conf import WuttaConfig
 from wuttaweb.views import master
 from wuttaweb.subscribers import new_request_set_user
-
+from tests.util import WebTestCase
-from tests.views.utils import WebTestCase
 
 
 class TestMasterView(WebTestCase):
@@ -387,6 +386,19 @@ class TestMasterView(WebTestCase):
         with patch.object(view, 'get_query', new=get_query):
             self.assertRaises(ValueError, view.get_grid_data, session=self.session)
 
+    def test_configure_grid(self):
+        model = self.app.model
+
+        # uuid field is pruned
+        with patch.multiple(master.MasterView, create=True,
+                            model_class=model.Setting):
+            view = master.MasterView(self.request)
+            grid = view.make_grid(model_class=model.Setting,
+                                  columns=['uuid', 'name', 'value'])
+            self.assertIn('uuid', grid.columns)
+            view.configure_grid(grid)
+            self.assertNotIn('uuid', grid.columns)
+
     def test_get_instance(self):
         model = self.app.model
         self.app.save_setting(self.session, 'foo', 'bar')
@@ -454,7 +466,7 @@ class TestMasterView(WebTestCase):
                             model_name='Widget',
                             model_key='uuid'):
             view = master.MasterView(self.request)
-            form = view.make_model_form()
+            form = view.make_model_form(fields=['name', 'description'])
             form.validated = {'name': 'first'}
             obj = view.objectify(form)
             self.assertIs(obj, form.validated)
@@ -666,7 +678,6 @@ class TestMasterView(WebTestCase):
                 self.assertIn('frazzle', response.text)
 
                 def delete_instance(setting):
-                    print(setting) # TODO
                     self.app.delete_setting(self.session, setting['name'])
 
                 # post request to save settings

tests/views/test_people.py

@@ -7,7 +7,7 @@ from sqlalchemy import orm
 from pyramid.httpexceptions import HTTPNotFound
 
 from wuttaweb.views import people
-from tests.views.utils import WebTestCase
+from tests.util import WebTestCase
 
 
 class TestPersonView(WebTestCase):

tests/views/test_settings.py

@@ -2,11 +2,10 @@
 
 from unittest.mock import patch
 
-from tests.views.utils import WebTestCase
-
 from pyramid.httpexceptions import HTTPNotFound
 
 from wuttaweb.views import settings
+from tests.util import WebTestCase
 
 
 class TestAppInfoView(WebTestCase):

tests/views/test_users.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8; -*-
+
+from unittest.mock import patch
+
+from sqlalchemy import orm
+
+import colander
+from pyramid.httpexceptions import HTTPNotFound
+
+from wuttaweb.views import users
+from tests.util import WebTestCase
+
+
+class TestPersonView(WebTestCase):
+
+    def make_view(self):
+        return users.UserView(self.request)
+
+    def test_get_query(self):
+        view = self.make_view()
+        query = view.get_query(session=self.session)
+        self.assertIsInstance(query, orm.Query)
+
+    def test_configure_grid(self):
+        model = self.app.model
+        view = self.make_view()
+        grid = view.make_grid(model_class=model.User)
+        self.assertFalse(grid.is_linked('person'))
+        view.configure_grid(grid)
+        self.assertTrue(grid.is_linked('person'))
+
+    def test_configure_form(self):
+        model = self.app.model
+        view = self.make_view()
+        form = view.make_form(model_class=model.Person)
+        self.assertIsNone(form.is_required('person'))
+        view.configure_form(form)
+        self.assertFalse(form.is_required('person'))
+
+    def test_unique_username(self):
+        model = self.app.model
+        view = self.make_view()
+
+        user = model.User(username='foo')
+        self.session.add(user)
+        self.session.commit()
+
+        with patch.object(users, 'Session', return_value=self.session):
+
+            # invalid if same username in data
+            node = colander.SchemaNode(colander.String(), name='username')
+            self.assertRaises(colander.Invalid, view.unique_username, node, 'foo')
+
+            # but not if username belongs to current user
+            view.editing = True
+            self.request.matchdict = {'uuid': user.uuid}
+            node = colander.SchemaNode(colander.String(), name='username')
+            self.assertIsNone(view.unique_username(node, 'foo'))