Weird database errors
Since we're logging application errors, I've seen some weird database errors. One example:
Traceback (most recent call last):
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/engine/cursor.py", line 858, in _indexes_for_keys
return [self._keymap[key][0] for key in keys]
~~~~~~~~~~~~^^^^^
KeyError: Column('id', Integer(), table=<app>, primary_key=True, nullable=False)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.13/site-packages/flask/app.py", line 880, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.13/site-packages/flask/app.py", line 865, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args) # type: ignore[no-any-return]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
File "/usr/local/lib/python3.13/site-packages/flask_jwt_extended/view_decorators.py", line 170, in decorator
return current_app.ensure_sync(fn)
(*args, **kwargs)
~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.13/site-packages/flask_cors/decorator.py", lin$ 130, in wrapped_function
resp = make_response(f(*args, **kwargs))
~^^^^^^^^^^^^^^^^^
File "/app/areas/apps/apps.py", line 26, in get_apps
apps = AppsService.get_accessible_apps()
File "/app/areas/apps/apps_service.py", line 41, in get_accessible_apps
return [app.to_dict() for app in apps if user_has_access(user, app)]
~~~~~~~~~~~~~~~^^^^^^^^^^^
File "/app/helpers/access_control.py", line 11, in user_has_access
App.slug == \'dashboard\').first()
~~~~~^^
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/orm/query.py", line 2728, in first
return self.limit(1)._iter().first() # type: ignore
~~~~~~~~~~~~~~~~~~~^
^
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/orm/query.py", line 2827, in _iter
result: Union[ScalarResult[_T], Result[_T]] = self.session.execute(
~~~~~~~~~~~~~~~~~~~~^
statement,
^^^^^^^^^^
params,
^^^^^^^
execution_options={"_sa_orm_load_options": self.load_options},
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/orm/session.py", line 2362, in execute
return self._execute_internal(
~~~~~~~~~~~~~~~~~~~~~~^
statement,
^^^^^^^^^^
...<4 lines>...
_add_event=_add_event,
^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/orm/session.py", line 2247, in _execute_internal
result: Result[Any] = compile_state_cls.orm_execute_statement(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
self,
^^^^^
...<4 lines>...
conn,
^^^^^
)
^
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/orm/context.py", line 308, in orm_execute_statement
return cls.orm_setup_cursor_result(
~~~~~~~~~~~~~~~~~~~~~~~~~~~^
session,
^^^^^^^^
...<4 lines>...
result,
^^^^^^^
)
^
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/orm/context.py", line 600, in orm_setup_cursor_result
return loading.instances(result, querycontext)
~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/orm/loading.py", line 132, in instances
with util.safe_reraise():
~~~~~~~~~~~~~~~~~^^
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/util/langhelpers.py", line 146, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/orm/loading.py", line 114, in instances
query_entity.row_processor(context, cursor)
~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/orm/context.py", line 2723, in row_processor
_instance = loading._instance_processor(
self,
...<7 lines>...
polymorphic_discriminator=self._polymorphic_discriminator,
)
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/orm/loading.py",
line 877, in _instance_processor
primary_key_getter = result._tuple_getter(pk_cols)
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/engine/result.py", line 1184, in _tuple_getter
return self._metadata._row_as_tuple_getter(keys)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/engine/result.py", line 179, in _row_as_tuple_getter
indexes = self._indexes_for_keys(keys)
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/engine/cursor.py", line 861, in _indexes_for_keys
CursorResultMetaData._key_fallback(self, ke.args[0], ke)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.13/site-packages/sqlalchemy/engine/cursor.py", line 824, in _key_fallback
raise exc.NoSuchColumnError(
...<2 lines>...
) from err
sqlalchemy.exc.NoSuchColumnError: Could not locate column in row for column 'app.id'
There could be a bug in our database code. However, I'm looking at the possibility that this happens because the database connection pool is shared among gunicorn worker processes. That is known to lead to problems, see this article. I'll attempt to fix this by disposing of database connections in app.py
, which is loaded by gunicorn before forking.