max_results in BigQuery API had a different meaning from HTTPIterator.

In BigQuery it means the page size, but the HTTPIterator it meant "don't
fetch any more pages once you have these many rows."

bigquery/google/cloud/bigquery/query.py

@@ -440,6 +440,9 @@ def fetch_data(self, max_results=None, page_token=None, start_index=None,
         if timeout_ms is not None:
             params['timeoutMs'] = timeout_ms
 
+        if max_results is not None:
+            params['maxResults'] = max_results
+
         path = '/projects/%s/queries/%s' % (self.project, self.name)
         iterator = page_iterator.HTTPIterator(
             client=client,
@@ -448,12 +451,10 @@ def fetch_data(self, max_results=None, page_token=None, start_index=None,
             item_to_value=_item_to_row,
             items_key='rows',
             page_token=page_token,
-            max_results=max_results,
             page_start=_rows_page_start_query,
+            next_token='pageToken',
             extra_params=params)
         iterator.query_result = self
-        # Over-ride the key used to retrieve the next page token.
-        iterator._NEXT_TOKEN = 'pageToken'
         return iterator
 
 

bigquery/google/cloud/bigquery/table.py

@@ -722,6 +722,11 @@ def fetch_data(self, max_results=None, page_token=None, client=None):
         if len(self._schema) == 0:
             raise ValueError(_TABLE_HAS_NO_SCHEMA)
 
+        params = {}
+
+        if max_results is not None:
+            params['maxResults'] = max_results
+
         client = self._require_client(client)
         path = '%s/data' % (self.path,)
         iterator = page_iterator.HTTPIterator(
@@ -731,11 +736,10 @@ def fetch_data(self, max_results=None, page_token=None, client=None):
             item_to_value=_item_to_row,
             items_key='rows',
             page_token=page_token,
-            max_results=max_results,
-            page_start=_rows_page_start)
+            page_start=_rows_page_start,
+            next_token='pageToken',
+            extra_params=params)
         iterator.schema = self._schema
-        # Over-ride the key used to retrieve the next page token.
-        iterator._NEXT_TOKEN = 'pageToken'
         return iterator
 
     def row_from_mapping(self, mapping):

core/google/api/core/page_iterator.py

@@ -275,6 +275,8 @@ class HTTPIterator(Iterator):
             signature takes the :class:`Iterator` that started the page,
             the :class:`Page` that was started and the dictionary containing
             the page response.
+        next_token (str): The name of the field used in the response for page
+            tokens.
 
     .. autoattribute:: pages
     """
@@ -283,13 +285,13 @@ class HTTPIterator(Iterator):
     _PAGE_TOKEN = 'pageToken'
     _MAX_RESULTS = 'maxResults'
     _NEXT_TOKEN = 'nextPageToken'
-    _RESERVED_PARAMS = frozenset([_PAGE_TOKEN, _MAX_RESULTS])
+    _RESERVED_PARAMS = frozenset([_PAGE_TOKEN])
     _HTTP_METHOD = 'GET'
 
     def __init__(self, client, api_request, path, item_to_value,
                  items_key=_DEFAULT_ITEMS_KEY,
                  page_token=None, max_results=None, extra_params=None,
-                 page_start=_do_nothing_page_start):
+                 page_start=_do_nothing_page_start, next_token=_NEXT_TOKEN):
         super(HTTPIterator, self).__init__(
             client, item_to_value, page_token=page_token,
             max_results=max_results)
@@ -298,6 +300,7 @@ def __init__(self, client, api_request, path, item_to_value,
         self._items_key = items_key
         self.extra_params = extra_params
         self._page_start = page_start
+        self._next_token = next_token
         # Verify inputs / provide defaults.
         if self.extra_params is None:
             self.extra_params = {}
@@ -327,7 +330,7 @@ def _next_page(self):
             items = response.get(self._items_key, ())
             page = Page(self, items, self._item_to_value)
             self._page_start(self, page, response)
-            self.next_page_token = response.get(self._NEXT_TOKEN)
+            self.next_page_token = response.get(self._next_token)
             return page
         else:
             return None