feat: add backend pagination support for grids

src/wuttaweb/grids/base.py

@@ -30,9 +30,11 @@ import logging
 
 import sqlalchemy as sa
 
+import paginate
 from pyramid.renderers import render
 from webhelpers2.html import HTML
 
+from wuttaweb.db import Session
 from wuttaweb.util import FieldList, get_model_fields, make_json_safe
 
 
@@ -87,6 +89,9 @@ class Grid:
        model records) or else an object capable of producing such a
        list, e.g. SQLAlchemy query.
 
+       This is the "full" data set; see also
+       :meth:`get_visible_data()`.
+
     .. attribute:: labels
 
        Dict of column label overrides.
@@ -114,9 +119,23 @@ class Grid:
     .. attribute:: paginated
 
        Boolean indicating whether the grid data should be paginated
-       vs. all data is shown at once.  Default is ``False``.
+       vs. all data shown at once.  Default is ``False`` which means
+       the full set of grid data is sent for each request.
 
-       See also :attr:`pagesize` and :attr:`page`.
+       See also :attr:`pagesize` and :attr:`page`, and
+       :attr:`paginate_on_backend`.
+
+    .. attribute:: paginate_on_backend
+
+       Boolean indicating whether the grid data should be paginated on
+       the backend.  Default is ``True`` which means only one "page"
+       of data is sent to the client-side component.
+
+       If this is ``False``, the full set of grid data is sent for
+       each request, and the client-side Vue component will handle the
+       pagination.
+
+       Only relevant if :attr:`paginated` is also true.
 
     .. attribute:: pagesize_options
 
@@ -157,6 +176,7 @@ class Grid:
             actions=[],
             linked_columns=[],
             paginated=False,
+            paginate_on_backend=True,
             pagesize_options=None,
             pagesize=None,
             page=1,
@@ -177,6 +197,7 @@ class Grid:
         self.set_columns(columns or self.get_columns())
 
         self.paginated = paginated
+        self.paginate_on_backend = paginate_on_backend
         self.pagesize_options = pagesize_options or self.get_pagesize_options()
         self.pagesize = pagesize or self.get_pagesize()
         self.page = page
@@ -435,6 +456,149 @@ class Grid:
 
         return self.pagesize_options[0]
 
+    ##############################
+    # configuration methods
+    ##############################
+
+    def load_settings(self, store=True):
+        """
+        Load all effective settings for the grid, from the following
+        places:
+
+        * request params
+        * user session
+
+        The first value found for a given setting will be applied to
+        the grid.
+
+        .. note::
+
+           As of now, "pagination" settings are the only type
+           supported by this logic.  Filter/sort coming soon...
+
+        The overall logic for this method is as follows:
+
+        * collect settings
+        * apply settings to current grid
+        * optionally save settings to user session
+
+        Saving the settings to user session will allow the grid to
+        "remember" its current settings when user refreshes the page.
+
+        :param store: Flag indicating whether the collected settings
+           should then be saved to the user session.
+        """
+
+        # initial default settings
+        settings = {}
+        if self.paginated and self.paginate_on_backend:
+            settings['pagesize'] = self.pagesize
+            settings['page'] = self.page
+
+        # grab settings from request and/or user session
+        if self.paginated and self.paginate_on_backend:
+            self.update_page_settings(settings)
+
+        else:
+            # no settings were found in request or user session, so
+            # nothing needs to be saved
+            store = False
+
+        # maybe store settings in user session, for next time
+        if store:
+            self.persist_settings(settings)
+
+        # update ourself to reflect settings
+        if self.paginated and self.paginate_on_backend:
+            self.pagesize = settings['pagesize']
+            self.page = settings['page']
+
+    def request_has_settings(self):
+        """ """
+        for key in ['pagesize', 'page']:
+            if key in self.request.GET:
+                return True
+        return False
+
+    def update_page_settings(self, settings):
+        """ """
+        # update the settings dict from request and/or user session
+
+        # pagesize
+        pagesize = self.request.GET.get('pagesize')
+        if pagesize is not None:
+            if pagesize.isdigit():
+                settings['pagesize'] = int(pagesize)
+        else:
+            pagesize = self.request.session.get(f'grid.{self.key}.pagesize')
+            if pagesize is not None:
+                settings['pagesize'] = pagesize
+
+        # page
+        page = self.request.GET.get('page')
+        if page is not None:
+            if page.isdigit():
+                settings['page'] = int(page)
+        else:
+            page = self.request.session.get(f'grid.{self.key}.page')
+            if page is not None:
+                settings['page'] = int(page)
+
+    def persist_settings(self, settings):
+        """ """
+        model = self.app.model
+        session = Session()
+
+        # func to save a setting value to user session
+        def persist(key, value=lambda k: settings.get(k)):
+            skey = f'grid.{self.key}.{key}'
+            self.request.session[skey] = value(key)
+
+        if self.paginated and self.paginate_on_backend:
+            persist('pagesize')
+            persist('page')
+
+    ##############################
+    # data methods
+    ##############################
+
+    def get_visible_data(self):
+        """
+        Returns the "effective" visible data for the grid.
+
+        This uses :attr:`data` as the starting point but may morph it
+        for pagination etc. per the grid settings.
+
+        Code can either access :attr:`data` directly, or call this
+        method to get only the data for current view (e.g. assuming
+        pagination is used), depending on the need.
+
+        See also these methods which may be called by this one:
+
+        * :meth:`paginate_data()`
+        """
+        data = self.data or []
+
+        if self.paginated and self.paginate_on_backend:
+            self.pager = self.paginate_data(data)
+            data = self.pager
+
+        return data
+
+    def paginate_data(self, data):
+        """
+        Apply pagination to the given data set, based on grid settings.
+
+        This returns a "pager" object which can then be used as a
+        "data replacement" in subsequent logic.
+
+        This method is called by :meth:`get_visible_data()`.
+        """
+        pager = paginate.Page(data,
+                              items_per_page=self.pagesize,
+                              page=self.page)
+        return pager
+
     ##############################
     # rendering methods
     ##############################
@@ -517,17 +681,18 @@ class Grid:
         """
         Returns a list of Vue-compatible data records.
 
-        This uses :attr:`data` as the basis, but may add some extra
+        This calls :meth:`get_visible_data()` but then may modify the
-        values to each record, e.g. URLs for :attr:`actions` etc.
+        result, e.g. to add URLs for :attr:`actions` etc.
 
         Importantly, this also ensures each value in the dict is
         JSON-serializable, using
         :func:`~wuttaweb.util.make_json_safe()`.
 
         :returns: List of data record dicts for use with Vue table
-           component.
+           component.  May be the full set of data, or just the
+           current page, per :attr:`paginate_on_backend`.
         """
-        original_data = self.data or []
+        original_data = self.get_visible_data()
 
         # TODO: at some point i thought it was useful to wrangle the
         # columns here, but now i can't seem to figure out why..?
@@ -578,6 +743,22 @@ class Grid:
 
         return data
 
+    def get_vue_pager_stats(self):
+        """
+        Returns a simple dict with current grid pager stats.
+
+        This is used when :attr:`paginate_on_backend` is in effect.
+        """
+        pager = self.pager
+        return {
+            'item_count': pager.item_count,
+            'items_per_page': pager.items_per_page,
+            'page': pager.page,
+            'page_count': pager.page_count,
+            'first_item': pager.first_item,
+            'last_item': pager.last_item,
+        }
+
 
 class GridAction:
     """

src/wuttaweb/templates/grids/vue_template.mako

@@ -14,6 +14,11 @@
                   pagination-size="is-small"
                   :per-page="perPage"
                   :current-page="currentPage"
+                  @page-change="onPageChange"
+                  % if grid.paginate_on_backend:
+                      backend-pagination
+                      :total="pagerStats.item_count"
+                  % endif
               % endif
               >
 
@@ -53,8 +58,14 @@
             <div style="display: flex; gap: 0.5rem; align-items: center;">
               <span>
                 showing
+                {{ renderNumber(pagerStats.first_item) }}
+                - {{ renderNumber(pagerStats.last_item) }}
+                of {{ renderNumber(pagerStats.item_count) }} results;
               </span>
               <b-select v-model="perPage"
+                        % if grid.paginate_on_backend:
+                            @input="onPageSizeChange"
+                        % endif
                         size="is-small">
                 <option v-for="size in pageSizeOptions"
                         :value="size">
@@ -74,7 +85,7 @@
 
 <script>
 
-  const ${grid.vue_component}CurrentData = ${json.dumps(grid.get_vue_data())|n}
+  let ${grid.vue_component}CurrentData = ${json.dumps(grid.get_vue_data())|n}
 
   const ${grid.vue_component}Data = {
       data: ${grid.vue_component}CurrentData,
@@ -85,12 +96,120 @@
           pageSizeOptions: ${json.dumps(grid.pagesize_options)|n},
           perPage: ${json.dumps(grid.pagesize)|n},
           currentPage: ${json.dumps(grid.page)|n},
+          % if grid.paginate_on_backend:
+              pagerStats: ${json.dumps(grid.get_vue_pager_stats())|n},
+          % endif
       % endif
   }
 
   const ${grid.vue_component} = {
       template: '#${grid.vue_tagname}-template',
-      methods: {},
+      computed: {
+
+          % if not grid.paginate_on_backend:
+
+              pagerStats() {
+                  let last = this.currentPage * this.perPage
+                  let first = last - this.perPage + 1
+                  if (last > this.data.length) {
+                      last = this.data.length
+                  }
+                  return {
+                      'item_count': this.data.length,
+                      'items_per_page': this.perPage,
+                      'page': this.currentPage,
+                      'first_item': first,
+                      'last_item': last,
+                  }
+              },
+
+          % endif
+      },
+      methods: {
+
+          renderNumber(value) {
+              if (value != undefined) {
+                  return value.toLocaleString('en')
+              }
+          },
+
+          getBasicParams() {
+              return {
+                  % if grid.paginated and grid.paginate_on_backend:
+                      pagesize: this.perPage,
+                      page: this.currentPage,
+                  % endif
+              }
+          },
+
+          async fetchData(params, success, failure) {
+
+              if (params === undefined || params === null) {
+                  params = new URLSearchParams(this.getBasicParams())
+              } else {
+                  params = new URLSearchParams(params)
+              }
+              if (!params.has('partial')) {
+                  params.append('partial', true)
+              }
+              params = params.toString()
+
+              this.loading = true
+              this.$http.get(`${request.path_url}?${'$'}{params}`).then(response => {
+                  console.log(response)
+                  console.log(response.data)
+                  if (!response.data.error) {
+                      ${grid.vue_component}CurrentData = response.data.data
+                      this.data = ${grid.vue_component}CurrentData
+                      % if grid.paginated and grid.paginate_on_backend:
+                          this.pagerStats = response.data.pager_stats
+                      % endif
+                      this.loading = false
+                      if (success) {
+                          success()
+                      }
+                  } else {
+                      this.$buefy.toast.open({
+                          message: data.error,
+                          type: 'is-danger',
+                          duration: 2000, // 4 seconds
+                      })
+                      this.loading = false
+                      if (failure) {
+                          failure()
+                      }
+                  }
+              })
+              .catch((error) => {
+                  this.data = []
+                  % if grid.paginated and grid.paginate_on_backend:
+                      this.pagerStats = {}
+                  % endif
+                  this.loading = false
+                  if (failure) {
+                      failure()
+                  }
+                  throw error
+              })
+          },
+
+          % if grid.paginated:
+
+              % if grid.paginate_on_backend:
+                  onPageSizeChange(size) {
+                      this.fetchData()
+                  },
+              % endif
+
+              onPageChange(page) {
+                  this.currentPage = page
+                  % if grid.paginate_on_backend:
+                      this.fetchData()
+                  % endif
+              },
+
+          % endif
+      },
   }
 
 </script>

src/wuttaweb/views/base.py

@@ -25,6 +25,7 @@ Base Logic for Views
 """
 
 from pyramid import httpexceptions
+from pyramid.renderers import render_to_response
 
 from wuttaweb import forms, grids
 
@@ -117,3 +118,13 @@ class View:
         correctly no matter what.
         """
         return httpexceptions.HTTPFound(location=url, **kwargs)
+
+    def json_response(self, context):
+        """
+        Convenience method to return a JSON response.
+
+        :param context: Context data to be rendered as JSON.
+
+        :returns: A :term:`response` with JSON content type.
+        """
+        return render_to_response('json', context, request=self.request)

src/wuttaweb/views/master.py

@@ -189,6 +189,15 @@ class MasterView(View):
        This is used by :meth:`make_model_grid()` to set the grid's
        :attr:`~wuttaweb.grids.base.Grid.paginated` flag.
 
+    .. attribute:: paginate_on_backend
+
+       Boolean indicating whether the grid data for the
+       :meth:`index()` view should be paginated on the backend.
+       Default is ``True``.
+
+       This is used by :meth:`make_model_grid()` to set the grid's
+       :attr:`~wuttaweb.grids.base.Grid.paginate_on_backend` flag.
+
     .. attribute:: creatable
 
        Boolean indicating whether the view model supports "creating" -
@@ -238,6 +247,7 @@ class MasterView(View):
     listable = True
     has_grid = True
     paginated = True
+    paginate_on_backend = True
     creatable = True
     viewable = True
     editable = True
@@ -284,7 +294,16 @@ class MasterView(View):
         }
 
         if self.has_grid:
-            context['grid'] = self.make_model_grid()
+            grid = self.make_model_grid()
+
+            # so-called 'partial' requests get just data, no html
+            if self.request.GET.get('partial'):
+                context = {'data': grid.get_vue_data()}
+                if grid.paginated and grid.paginate_on_backend:
+                    context['pager_stats'] = grid.get_vue_pager_stats()
+                return self.json_response(context)
+
+            context['grid'] = grid
 
         return self.render_to_response('index', context)
 
@@ -1071,9 +1090,11 @@ class MasterView(View):
             kwargs['actions'] = actions
 
         kwargs.setdefault('paginated', self.paginated)
+        kwargs.setdefault('paginate_on_backend', self.paginate_on_backend)
 
         grid = self.make_grid(**kwargs)
         self.configure_grid(grid)
+        grid.load_settings()
         return grid
 
     def get_grid_columns(self):

tests/grids/test_base.py

@@ -3,6 +3,7 @@
 from unittest import TestCase
 from unittest.mock import patch
 
+from paginate import Page
 from pyramid import testing
 
 from wuttjamaican.conf import WuttaConfig
@@ -168,6 +169,105 @@ class TestGrid(WebTestCase):
         size = grid.get_pagesize()
         self.assertEqual(size, 15)
 
+    ##############################
+    # configuration methods
+    ##############################
+
+    def test_load_settings(self):
+        grid = self.make_grid(key='foo', paginated=True, paginate_on_backend=True,
+                              pagesize=20, page=1)
+
+        # settings are loaded, applied, saved
+        self.assertEqual(grid.page, 1)
+        self.assertNotIn('grid.foo.page', self.request.session)
+        self.request.GET = {'pagesize': '10', 'page': '2'}
+        grid.load_settings()
+        self.assertEqual(grid.page, 2)
+        self.assertEqual(self.request.session['grid.foo.page'], 2)
+
+        # can skip the saving step
+        self.request.GET = {'pagesize': '10', 'page': '3'}
+        grid.load_settings(store=False)
+        self.assertEqual(grid.page, 3)
+        self.assertEqual(self.request.session['grid.foo.page'], 2)
+
+        # no error for non-paginated grid
+        grid = self.make_grid(key='foo', paginated=False)
+        grid.load_settings()
+        self.assertFalse(grid.paginated)
+
+    def test_request_has_settings(self):
+        grid = self.make_grid(key='foo')
+
+        self.assertFalse(grid.request_has_settings())
+
+        with patch.object(self.request, 'GET', new={'pagesize': '20'}):
+            self.assertTrue(grid.request_has_settings())
+
+        with patch.object(self.request, 'GET', new={'page': '1'}):
+            self.assertTrue(grid.request_has_settings())
+
+    def test_update_page_settings(self):
+        grid = self.make_grid(key='foo')
+
+        # settings are updated from session
+        settings = {'pagesize': 20, 'page': 1}
+        self.request.session['grid.foo.pagesize'] = 10
+        self.request.session['grid.foo.page'] = 2
+        grid.update_page_settings(settings)
+        self.assertEqual(settings['pagesize'], 10)
+        self.assertEqual(settings['page'], 2)
+
+        # settings are updated from request
+        self.request.GET = {'pagesize': '15', 'page': '4'}
+        grid.update_page_settings(settings)
+        self.assertEqual(settings['pagesize'], 15)
+        self.assertEqual(settings['page'], 4)
+
+    def test_persist_settings(self):
+        grid = self.make_grid(key='foo', paginated=True, paginate_on_backend=True)
+
+        # nb. no error if empty settings, but it saves null values
+        grid.persist_settings({})
+        self.assertIsNone(self.request.session['grid.foo.page'])
+
+        # provided values are saved
+        grid.persist_settings({'pagesize': 15, 'page': 3})
+        self.assertEqual(self.request.session['grid.foo.page'], 3)
+
+    ##############################
+    # data methods
+    ##############################
+
+    def test_get_visible_data(self):
+        data = [
+            {'foo': 1, 'bar': 1},
+            {'foo': 2, 'bar': 2},
+            {'foo': 3, 'bar': 3},
+            {'foo': 4, 'bar': 4},
+            {'foo': 5, 'bar': 5},
+            {'foo': 6, 'bar': 6},
+            {'foo': 7, 'bar': 7},
+            {'foo': 8, 'bar': 8},
+            {'foo': 9, 'bar': 9},
+        ]
+        grid = self.make_grid(data=data,
+                              columns=['foo', 'bar'],
+                              paginated=True, paginate_on_backend=True,
+                              pagesize=4, page=2)
+        visible = grid.get_visible_data()
+        self.assertEqual(len(visible), 4)
+        self.assertEqual(visible[0], {'foo': 5, 'bar': 5})
+
+    def test_paginate_data(self):
+        grid = self.make_grid()
+        pager = grid.paginate_data([])
+        self.assertIsInstance(pager, Page)
+
+    ##############################
+    # rendering methods
+    ##############################
+
     def test_render_vue_tag(self):
         grid = self.make_grid(columns=['foo', 'bar'])
         html = grid.render_vue_tag()
@@ -221,6 +321,28 @@ class TestGrid(WebTestCase):
         data = grid.get_vue_data()
         self.assertEqual(data, [{'foo': 'blah blah', '_action_url_view': '/blarg'}])
 
+    def test_get_vue_pager_stats(self):
+        data = [
+            {'foo': 1, 'bar': 1},
+            {'foo': 2, 'bar': 2},
+            {'foo': 3, 'bar': 3},
+            {'foo': 4, 'bar': 4},
+            {'foo': 5, 'bar': 5},
+            {'foo': 6, 'bar': 6},
+            {'foo': 7, 'bar': 7},
+            {'foo': 8, 'bar': 8},
+            {'foo': 9, 'bar': 9},
+        ]
+
+        grid = self.make_grid(columns=['foo', 'bar'], pagesize=4, page=2)
+        grid.pager = grid.paginate_data(data)
+        stats = grid.get_vue_pager_stats()
+        self.assertEqual(stats['item_count'], 9)
+        self.assertEqual(stats['items_per_page'], 4)
+        self.assertEqual(stats['page'], 2)
+        self.assertEqual(stats['first_item'], 5)
+        self.assertEqual(stats['last_item'], 8)
+
 
 class TestGridAction(TestCase):
 

tests/views/test_base.py

@@ -1,46 +1,56 @@
 # -*- coding: utf-8; -*-
 
-from unittest import TestCase
-
-from pyramid import testing
 from pyramid.httpexceptions import HTTPFound, HTTPForbidden, HTTPNotFound
 
-from wuttjamaican.conf import WuttaConfig
+from wuttaweb.views import base as mod
-from wuttaweb.views import base
 from wuttaweb.forms import Form
-from wuttaweb.grids import Grid
+from wuttaweb.grids import Grid, GridAction
+from tests.util import WebTestCase
 
 
-class TestView(TestCase):
+class TestView(WebTestCase):
 
-    def setUp(self):
+    def make_view(self):
-        self.config = WuttaConfig()
+        return mod.View(self.request)
-        self.app = self.config.get_app()
-        self.request = testing.DummyRequest(wutta_config=self.config)
-        self.view = base.View(self.request)
 
     def test_basic(self):
-        self.assertIs(self.view.request, self.request)
+        view = self.make_view()
-        self.assertIs(self.view.config, self.config)
+        self.assertIs(view.request, self.request)
-        self.assertIs(self.view.app, self.app)
+        self.assertIs(view.config, self.config)
+        self.assertIs(view.app, self.app)
 
     def test_forbidden(self):
-        error = self.view.forbidden()
+        view = self.make_view()
+        error = view.forbidden()
         self.assertIsInstance(error, HTTPForbidden)
 
     def test_make_form(self):
-        form = self.view.make_form()
+        view = self.make_view()
+        form = view.make_form()
         self.assertIsInstance(form, Form)
 
     def test_make_grid(self):
-        grid = self.view.make_grid()
+        view = self.make_view()
+        grid = view.make_grid()
         self.assertIsInstance(grid, Grid)
 
+    def test_make_grid_action(self):
+        view = self.make_view()
+        action = view.make_grid_action('view')
+        self.assertIsInstance(action, GridAction)
+
     def test_notfound(self):
-        error = self.view.notfound()
+        view = self.make_view()
+        error = view.notfound()
         self.assertIsInstance(error, HTTPNotFound)
 
     def test_redirect(self):
-        error = self.view.redirect('/')
+        view = self.make_view()
+        error = view.redirect('/')
         self.assertIsInstance(error, HTTPFound)
         self.assertEqual(error.location, '/')
+
+    def test_json_response(self):
+        view = self.make_view()
+        response = view.json_response({'foo': 'bar'})
+        self.assertEqual(response.status_code, 200)

tests/views/test_master.py

@@ -747,6 +747,14 @@ class TestMasterView(WebTestCase):
             data = [{'name': 'foo', 'value': 'bar'}]
             with patch.object(view, 'get_grid_data', return_value=data):
                 response = view.index()
+                self.assertEqual(response.status_code, 200)
+                self.assertEqual(response.content_type, 'text/html')
+
+                # then once more as 'partial' - aka. data only
+                self.request.GET = {'partial': '1'}
+                response = view.index()
+                self.assertEqual(response.status_code, 200)
+                self.assertEqual(response.content_type, 'application/json')
 
     def test_create(self):
         self.pyramid_config.include('wuttaweb.views.common')