Merge branch '130-lock-file-keeps-in-place-after-crash-abort' into 'main'

Resolve "Lock file keeps in place after crash/abort"

Closes #130

See merge request stackspin/dashboard!123

backend/Dockerfile

@@ -20,8 +20,7 @@ RUN pip install --no-cache-dir -r requirements.txt
 # now copy all the files in this directory to /app
 COPY . .
 
-# Listen to port 80 at runtime
 EXPOSE 5000
 
 # Define our command to be run when launching the container
-CMD ["gunicorn", "app:app", "-b", "0.0.0.0:5000", "--workers", "8", "--reload", "--capture-output", "--enable-stdio-inheritance", "--log-level", "DEBUG"]
+CMD ["gunicorn", "app:app", "-b", "0.0.0.0:5000", "--workers", "8", "--preload", "--capture-output", "--enable-stdio-inheritance", "--log-level", "DEBUG"]

backend/app.py

@@ -37,6 +37,7 @@ import cluster_config
 from config import *
 import logging
 import migration_reset
+import os
 import sys
 
 # Configure logging.
@@ -69,35 +70,47 @@ cors = CORS(app)
 
 db.init_app(app)
 
-# We'll now perform some initialization routines. Because these have to be done
-# once at startup, not for every gunicorn worker, we take a machine-wide lock
-# for this.
-init_lock = NamedAtomicLock('dashboard_init')
-if init_lock.acquire():
+def init_routines():
+    with app.app_context():
+        # We have reset the alembic migration history at Stackspin version 2.2.
+        # This checks whether we need to prepare the database to follow that
+        # change.
+        migration_reset.reset()
+    flask_migrate.Migrate(app, db)
     try:
         with app.app_context():
-            # We have reset the alembic migration history at Stackspin version 2.2.
-            # This checks whether we need to prepare the database to follow that
-            # change.
-            migration_reset.reset()
-        flask_migrate.Migrate(app, db)
-        try:
-            with app.app_context():
-                flask_migrate.upgrade()
-        except Exception as e:
-            app.logger.info(f"upgrade failed: {type(e)}: {e}")
-            sys.exit(2)
-
-        # We need this app context in order to talk the database, which is managed by
-        # flask-sqlalchemy, which assumes a flask app context.
-        with app.app_context():
-            # Load the list of apps from a configmap and store any missing ones in the
-            # database.
-            cluster_config.populate_apps()
-            # Same for the list of oauthclients.
-            cluster_config.populate_oauthclients()
-    finally:
-        init_lock.release()
+            flask_migrate.upgrade()
+    # TODO: actually flask_migrate.upgrade will catch any errors and
+    # exit the program :/
+    except Exception as e:
+        app.logger.info(f"upgrade failed: {type(e)}: {e}")
+        sys.exit(2)
+
+    # We need this app context in order to talk the database, which is managed by
+    # flask-sqlalchemy, which assumes a flask app context.
+    with app.app_context():
+        # Load the list of apps from a configmap and store any missing ones in the
+        # database.
+        cluster_config.populate_apps()
+        # Same for the list of oauthclients.
+        cluster_config.populate_oauthclients()
+
+# `init_routines`should only run once per dashboard instance. To enforce this we
+# have different behaviour for production and development mode:
+# * we have "preload" on for gunicorn, so this file is loaded only once, before
+#   workers are forked (production);
+# * we make sure that in development mode we run this only once, even though
+#   this file is loaded twice by flask for some reason.
+if RUN_BY_GUNICORN:
+    logging.info("Running initialization code (production mode).")
+    init_routines()
+else:
+    logging.info("WERKZEUG_RUN_MAIN: {}".format(os.environ.get("WERKZEUG_RUN_MAIN", "unset")))
+    if os.environ.get("WERKZEUG_RUN_MAIN") == "true":
+        logging.info("Running initialization code (dev mode).")
+        init_routines()
+    else:
+        logging.info("Not running initialization code (dev mode).")
 
 app.register_blueprint(api_v1)
 app.register_blueprint(web)

backend/config.py

@@ -21,5 +21,6 @@ SQLALCHEMY_TRACK_MODIFICATIONS = False
 # running in a Kubernetes pod. Set it to "false" to load the config from the
 # `KUBECONFIG` environment variable.
 LOAD_INCLUSTER_CONFIG = os.environ.get("LOAD_INCLUSTER_CONFIG").lower() == "true"
+RUN_BY_GUNICORN = "gunicorn" in os.environ.get("SERVER_SOFTWARE", "")
 
 DEMO_INSTANCE = os.environ.get("DASHBOARD_DEMO_INSTANCE", "False").lower() in ('true', '1')