feat(database): database connecting backoff (#2668)

packages/core/database/package.json

@@ -19,7 +19,8 @@
     "mathjs": "^10.6.1",
     "semver": "^7.3.7",
     "sequelize": "^6.26.0",
-    "umzug": "^3.1.1"
+    "umzug": "^3.1.1",
+    "exponential-backoff": "^3.1.1"
   },
   "devDependencies": {
     "@types/glob": "^7.2.0"

packages/core/database/src/database.ts

@@ -69,6 +69,7 @@ import {
 import { patchSequelizeQueryInterface, snakeCase } from './utils';
 import { BaseValueParser, registerFieldValueParsers } from './value-parsers';
 import { ViewCollection } from './view-collection';
+import { backOff } from 'exponential-backoff';
 
 export type MergeOptions = merge.Options;
 
@@ -700,25 +701,33 @@ export class Database extends EventEmitter implements AsyncEmitter {
 
   async auth(options: Omit<QueryOptions, 'retry'> & { retry?: number | Pick<QueryOptions, 'retry'> } = {}) {
     const { retry = 10, ...others } = options;
-    const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
-    let count = 1;
+    const startingDelay = 50;
+    const timeMultiple = 2;
+
+    let attemptNumber = 1; // To track the current attempt number
+
     const authenticate = async () => {
       try {
         await this.sequelize.authenticate(others);
         console.log('Connection has been established successfully.');
-        return true;
       } catch (error) {
-        console.log(`Unable to connect to the database: ${error.message}`);
-        if (count >= (retry as number)) {
-          throw new Error('Connection failed, please check your database connection credentials and try again.');
-        }
-        console.log('reconnecting...', count);
-        ++count;
-        await delay(500);
-        return await authenticate();
+        console.log(`Attempt ${attemptNumber}/${retry}: Unable to connect to the database: ${error.message}`);
+        const nextDelay = startingDelay * Math.pow(timeMultiple, attemptNumber - 1);
+        console.log(`Will retry in ${nextDelay}ms...`);
+        attemptNumber++;
+        throw error; // Re-throw the error so that backoff can catch and handle it
       }
     };
-    return await authenticate();
+
+    try {
+      await backOff(authenticate, {
+        numOfAttempts: retry as number,
+        startingDelay: startingDelay,
+        timeMultiple: timeMultiple,
+      });
+    } catch (error) {
+      throw new Error('Connection failed, please check your database connection credentials and try again.');
+    }
   }
 
   async prepare() {

packages/core/server/src/application.ts

@@ -353,7 +353,7 @@ export class Application<StateT = DefaultState, ContextT = DefaultContext> exten
       return;
     }
     this._authenticated = true;
-    await this.db.auth({ retry: 30 });
+    await this.db.auth();
     await this.dbVersionCheck({ exit: true });
     await this.db.prepare();
   }

yarn.lock

@@ -11585,6 +11585,11 @@ expect@^29.6.2:
     jest-message-util "^29.6.2"
     jest-util "^29.6.2"
 
+exponential-backoff@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz#64ac7526fe341ab18a39016cd22c787d01e00bf6"
+  integrity sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==
+
 ext@^1.1.2:
   version "1.7.0"
   resolved "https://registry.npmmirror.com/ext/-/ext-1.7.0.tgz#0ea4383c0103d60e70be99e9a7f11027a33c4f5f"