#36338: Grouping on JSONField subkey throws tuple index out of range error
-------------------------------------+-------------------------------------
Reporter: Marc DEBUREAUX | Owner: (none)
Type: Bug | Status: new
Component: Database layer | Version: 5.2
(models, ORM) |
Severity: Normal | Resolution:
Keywords: jsonfield | Triage Stage:
| Unreviewed
Has patch: 0 | Needs documentation: 0
Needs tests: 0 | Patch needs improvement: 0
Easy pickings: 0 | UI/UX: 0
-------------------------------------+-------------------------------------
Description changed by Marc DEBUREAUX:
Old description:
> Have this simple model:
>
> {{{
> class Link(models.Model):
> item = models.ForeignKey("Item", on_delete=models.CASCADE)
> extra = models.JSONField(blank=True, null=True)
> }}}
>
> And have this simple query:
>
> {{{
> Link.objects.filter(item_id=1).values("extra__key").annotate(count=Count("id"))
> }}}
>
> This throws the following stacktrace error since 5.2 release:
>
> {{{
> ---------------------------------------------------------------------------
> IndexError Traceback (most recent call
> last)
> File ~/python3.12/site-packages/IPython/core/formatters.py:770, in
> PlainTextFormatter.__call__(self, obj)
> 763 stream = StringIO()
> 764 printer = pretty.RepresentationPrinter(stream, self.verbose,
> 765 self.max_width, self.newline,
> 766 max_seq_length=self.max_seq_length,
> 767 singleton_pprinters=self.singleton_printers,
> 768 type_pprinters=self.type_printers,
> 769 deferred_pprinters=self.deferred_printers)
> --> 770 printer.pretty(obj)
> 771 printer.flush()
> 772 return stream.getvalue()
>
> File ~/python3.12/site-packages/IPython/lib/pretty.py:411, in
> RepresentationPrinter.pretty(self, obj)
> 400 return meth(obj, self, cycle)
> 401 if (
> 402 cls is not object
> 403 # check if cls defines __repr__
> (...) 409 and callable(_safe_getattr(cls,
> "__repr__", None))
> 410 ):
> --> 411 return _repr_pprint(obj, self, cycle)
> 413 return _default_pprint(obj, self, cycle)
> 414 finally:
>
> File ~/python3.12/site-packages/IPython/lib/pretty.py:786, in
> _repr_pprint(obj, p, cycle)
> 784 """A pprint that just redirects to the normal repr function."""
> 785 # Find newlines and replace them with p.break_()
> --> 786 output = repr(obj)
> 787 lines = output.splitlines()
> 788 with p.group():
>
> File ~/python3.12/site-packages/django/db/models/query.py:360, in
> QuerySet.__repr__(self)
> 359 def __repr__(self):
> --> 360 data = list(self[: REPR_OUTPUT_SIZE + 1])
> 361 if len(data) > REPR_OUTPUT_SIZE:
> 362 data[-1] = "...(remaining elements truncated)..."
>
> File ~/python3.12/site-packages/django/db/models/query.py:384, in
> QuerySet.__iter__(self)
> 369 def __iter__(self):
> 370 """
> 371 The queryset iterator protocol uses three nested iterators in
> the
> 372 default case:
> (...) 382 - Responsible for turning the rows into model
> objects.
> 383 """
> --> 384 self._fetch_all()
> 385 return iter(self._result_cache)
>
> File ~/python3.12/site-packages/django/db/models/query.py:1935, in
> QuerySet._fetch_all(self)
> 1933 def _fetch_all(self):
> 1934 if self._result_cache is None:
> -> 1935 self._result_cache = list(self._iterable_class(self))
> 1936 if self._prefetch_related_lookups and not
> self._prefetch_done:
> 1937 self._prefetch_related_objects()
>
> File ~/python3.12/site-packages/django/db/models/query.py:216, in
> ValuesIterable.__iter__(self)
> 210 names = [
> 211 *query.extra_select,
> 212 *query.values_select,
> 213 *query.annotation_select,
> 214 ]
> 215 indexes = range(len(names))
> --> 216 for row in compiler.results_iter(
> 217 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
> 218 ):
> 219 yield {names[i]: row[i] for i in indexes}
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1571, in
> SQLCompiler.results_iter(self, results, tuple_expected, chunked_fetch,
> chunk_size)
> 1569 """Return an iterator over the results from executing this
> query."""
> 1570 if results is None:
> -> 1571 results = self.execute_sql(
> 1572 MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size
> 1573 )
> 1574 fields = [s[0] for s in self.select[0 : self.col_count]]
> 1575 converters = self.get_converters(fields)
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1609, in
> SQLCompiler.execute_sql(self, result_type, chunked_fetch, chunk_size)
> 1607 result_type = result_type or NO_RESULTS
> 1608 try:
> -> 1609 sql, params = self.as_sql()
> 1610 if not sql:
> 1611 raise EmptyResultSet
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:765, in
> SQLCompiler.as_sql(self, with_limits, with_col_aliases)
> 763 try:
> 764 combinator = self.query.combinator
> --> 765 extra_select, order_by, group_by = self.pre_sql_setup(
> 766 with_col_aliases=with_col_aliases or bool(combinator),
> 767 )
> 768 for_update_part = None
> 769 # Is a LIMIT/OFFSET clause needed?
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:85, in
> SQLCompiler.pre_sql_setup(self, with_col_aliases)
> 79 def pre_sql_setup(self, with_col_aliases=False):
> 80 """
> 81 Do any necessary class setup immediately prior to producing
> SQL. This
> 82 is for things that can't necessarily be done in __init__
> because we
> 83 might not have all the pieces in place at that time.
> 84 """
> ---> 85 self.setup_query(with_col_aliases=with_col_aliases)
> 86 order_by = self.get_order_by()
> 87 self.where, self.having, self.qualify =
> self.query.where.split_having_qualify(
> 88 must_group_by=self.query.group_by is not None
> 89 )
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:74, in
> SQLCompiler.setup_query(self, with_col_aliases)
> 72 if all(self.query.alias_refcount[a] == 0 for a in
> self.query.alias_map):
> 73 self.query.get_initial_alias()
> ---> 74 self.select, self.klass_info, self.annotation_col_map =
> self.get_select(
> 75 with_col_aliases=with_col_aliases,
> 76 )
> 77 self.col_count = len(self.select)
>
> File ~/python3.12/site-packages/django/db/models/sql/compiler.py:286, in
> SQLCompiler.get_select(self, with_col_aliases)
> 284 # Reference to a column.
> 285 elif isinstance(expression, int):
> --> 286 expression = cols[expression]
> 287 # ColPairs cannot be aliased.
> 288 if isinstance(expression, ColPairs):
> }}}
New description:
Have this simple model:
{{{
class Link(models.Model):
item = models.ForeignKey("Item", on_delete=models.CASCADE)
extra = models.JSONField(blank=True, null=True)
}}}
And have this simple query:
{{{
Link.objects.all().values("extra__key",
"item_id").annotate(count=Count("id"))
}}}
This throws the following stacktrace error since 5.2 release:
{{{
---------------------------------------------------------------------------
IndexError Traceback (most recent call
last)
File ~/python3.12/site-packages/IPython/core/formatters.py:770, in
PlainTextFormatter.__call__(self, obj)
763 stream = StringIO()
764 printer = pretty.RepresentationPrinter(stream, self.verbose,
765 self.max_width, self.newline,
766 max_seq_length=self.max_seq_length,
767 singleton_pprinters=self.singleton_printers,
768 type_pprinters=self.type_printers,
769 deferred_pprinters=self.deferred_printers)
--> 770 printer.pretty(obj)
771 printer.flush()
772 return stream.getvalue()
File ~/python3.12/site-packages/IPython/lib/pretty.py:411, in
RepresentationPrinter.pretty(self, obj)
400 return meth(obj, self, cycle)
401 if (
402 cls is not object
403 # check if cls defines __repr__
(...) 409 and callable(_safe_getattr(cls,
"__repr__", None))
410 ):
--> 411 return _repr_pprint(obj, self, cycle)
413 return _default_pprint(obj, self, cycle)
414 finally:
File ~/python3.12/site-packages/IPython/lib/pretty.py:786, in
_repr_pprint(obj, p, cycle)
784 """A pprint that just redirects to the normal repr function."""
785 # Find newlines and replace them with p.break_()
--> 786 output = repr(obj)
787 lines = output.splitlines()
788 with p.group():
File ~/python3.12/site-packages/django/db/models/query.py:360, in
QuerySet.__repr__(self)
359 def __repr__(self):
--> 360 data = list(self[: REPR_OUTPUT_SIZE + 1])
361 if len(data) > REPR_OUTPUT_SIZE:
362 data[-1] = "...(remaining elements truncated)..."
File ~/python3.12/site-packages/django/db/models/query.py:384, in
QuerySet.__iter__(self)
369 def __iter__(self):
370 """
371 The queryset iterator protocol uses three nested iterators in
the
372 default case:
(...) 382 - Responsible for turning the rows into model
objects.
383 """
--> 384 self._fetch_all()
385 return iter(self._result_cache)
File ~/python3.12/site-packages/django/db/models/query.py:1935, in
QuerySet._fetch_all(self)
1933 def _fetch_all(self):
1934 if self._result_cache is None:
-> 1935 self._result_cache = list(self._iterable_class(self))
1936 if self._prefetch_related_lookups and not self._prefetch_done:
1937 self._prefetch_related_objects()
File ~/python3.12/site-packages/django/db/models/query.py:216, in
ValuesIterable.__iter__(self)
210 names = [
211 *query.extra_select,
212 *query.values_select,
213 *query.annotation_select,
214 ]
215 indexes = range(len(names))
--> 216 for row in compiler.results_iter(
217 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
218 ):
219 yield {names[i]: row[i] for i in indexes}
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1571, in
SQLCompiler.results_iter(self, results, tuple_expected, chunked_fetch,
chunk_size)
1569 """Return an iterator over the results from executing this
query."""
1570 if results is None:
-> 1571 results = self.execute_sql(
1572 MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size
1573 )
1574 fields = [s[0] for s in self.select[0 : self.col_count]]
1575 converters = self.get_converters(fields)
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1609, in
SQLCompiler.execute_sql(self, result_type, chunked_fetch, chunk_size)
1607 result_type = result_type or NO_RESULTS
1608 try:
-> 1609 sql, params = self.as_sql()
1610 if not sql:
1611 raise EmptyResultSet
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:765, in
SQLCompiler.as_sql(self, with_limits, with_col_aliases)
763 try:
764 combinator = self.query.combinator
--> 765 extra_select, order_by, group_by = self.pre_sql_setup(
766 with_col_aliases=with_col_aliases or bool(combinator),
767 )
768 for_update_part = None
769 # Is a LIMIT/OFFSET clause needed?
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:85, in
SQLCompiler.pre_sql_setup(self, with_col_aliases)
79 def pre_sql_setup(self, with_col_aliases=False):
80 """
81 Do any necessary class setup immediately prior to producing
SQL. This
82 is for things that can't necessarily be done in __init__
because we
83 might not have all the pieces in place at that time.
84 """
---> 85 self.setup_query(with_col_aliases=with_col_aliases)
86 order_by = self.get_order_by()
87 self.where, self.having, self.qualify =
self.query.where.split_having_qualify(
88 must_group_by=self.query.group_by is not None
89 )
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:74, in
SQLCompiler.setup_query(self, with_col_aliases)
72 if all(self.query.alias_refcount[a] == 0 for a in
self.query.alias_map):
73 self.query.get_initial_alias()
---> 74 self.select, self.klass_info, self.annotation_col_map =
self.get_select(
75 with_col_aliases=with_col_aliases,
76 )
77 self.col_count = len(self.select)
File ~/python3.12/site-packages/django/db/models/sql/compiler.py:286, in
SQLCompiler.get_select(self, with_col_aliases)
284 # Reference to a column.
285 elif isinstance(expression, int):
--> 286 expression = cols[expression]
287 # ColPairs cannot be aliased.
288 if isinstance(expression, ColPairs):
}}}
--
--
Ticket URL: <https://code.djangoproject.com/ticket/36338#comment:1>
Django <https://code.djangoproject.com/>
The Web framework for perfectionists with deadlines.
--
You received this message because you are subscribed to the Google Groups
"Django updates" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To view this discussion visit
https://groups.google.com/d/msgid/django-updates/0107019650220607-cfb02dae-cfff-433f-9b3c-a90449b78ae5-000000%40eu-central-1.amazonses.com.
