v4.7.3-beta.1

This reverts commit 9ee60d1d49.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -29,5 +29,8 @@ If applicable, add screenshots to help explain your problem.
  - Type - Web/Application
  - Database engine: [e.g. MySQL/PostgreSQL/SQL Server]
 
-**Additional context**
-Anything else you think might be helpful
+**Additional info**
+You could provide additional context, to better understand your case.
+ - How looks your typical DbGate usage?
+ - What other DB software do you use?
+ - Anything else you think might be helpful

.github/ISSUE_TEMPLATE/feature_request.md

@@ -16,5 +16,8 @@ A clear and concise description of what you want to happen.
 **Describe alternatives you've considered**
 A clear and concise description of any alternative solutions or features you've considered.
 
-**Additional context**
-Add any other context or screenshots about the feature request here.
+**Additional info**
+You could provide additional context, to better understand your case.
+ - How looks your typical DbGate usage?
+ - What other DB software do you use?
+ - Anything else you think might be helpful

.github/ISSUE_TEMPLATE/question.md

@@ -16,3 +16,9 @@ App Version [help -> about]:
 
 **Screenshot [if appropriate]**:
 A screenshot of the app if that helps
+
+**Additional info**
+You could provide additional context, to better understand your case.
+ - How looks your typical DbGate usage?
+ - What other DB software do you use?
+ - Anything else you think might be helpful

.github/workflows/build-app-beta.yaml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.15, windows-2016, ubuntu-18.04]
+        os: [macOS-10.15, windows-2019, ubuntu-18.04]
 
     steps:
       - name: Context
@@ -42,7 +42,6 @@ jobs:
         if: matrix.os == 'ubuntu-18.04'
         uses: samuelmeuli/action-snapcraft@v1
       - name: Publish
-        if: matrix.os != 'macOS-10.15'
         run: |
           yarn run build:app
         env:
@@ -50,13 +49,6 @@ jobs:
           WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
           WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
 
-      - name: Publish Mac
-        if: matrix.os == 'macOS-10.15'
-        run: |
-          yarn run build:app:mac
-        env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
-
       - name: Save snap login
         if: matrix.os == 'ubuntu-18.04'
         run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
@@ -81,7 +73,7 @@ jobs:
           cp app/dist/*win*.exe artifacts/dbgate-beta.exe || true
           cp app/dist/*win_x64.zip artifacts/dbgate-windows-beta.zip || true
           cp app/dist/*win_arm64.zip artifacts/dbgate-windows-beta-arm64.zip || true
-          cp app/dist/*-mac.dmg artifacts/dbgate-beta.dmg || true
+          cp app/dist/*-mac_x64.dmg artifacts/dbgate-beta.dmg || true
           cp app/dist/*-mac_arm64.dmg artifacts/dbgate-beta-arm64.dmg || true
 
           mv app/dist/*.exe artifacts/ || true

.github/workflows/build-app.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.15, windows-2016, ubuntu-18.04]
+        os: [macOS-10.15, windows-2019, ubuntu-18.04]
 
     steps:
       - name: Context
@@ -32,6 +32,8 @@ jobs:
           node-version: 14.x
       - name: yarn install
         run: |
+          # yarn --version
+          # yarn config set network-timeout 300000
           yarn install
       - name: setCurrentVersion
         run: |
@@ -81,7 +83,7 @@ jobs:
           cp app/dist/*.exe artifacts/dbgate-latest.exe || true
           cp app/dist/*win_x64.zip artifacts/dbgate-windows-latest.zip || true
           cp app/dist/*win_arm64.zip artifacts/dbgate-windows-latest-arm64.zip || true
-          cp app/dist/*-mac.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*-mac_x64.dmg artifacts/dbgate-latest.dmg || true
           cp app/dist/*-mac_arm64.dmg artifacts/dbgate-latest-arm64.dmg || true
 
           mv app/dist/*.exe artifacts/ || true
@@ -116,7 +118,7 @@ jobs:
       #     mv app/dist/latest.yml artifacts/latest.yml || true
 
       - name: Copy latest.yml (windows)
-        if: matrix.os == 'windows-2016'
+        if: matrix.os == 'windows-2019'
         run: |
           mv app/dist/latest.yml artifacts/latest.yml || true
           mv app/dist/dbgate-pad.xml artifacts/ || true

.github/workflows/build-docker-beta.yaml

@@ -6,6 +6,7 @@ on:
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-docker.[0-9]+'
 
 jobs:
   build:
@@ -30,6 +31,8 @@ jobs:
           node-version: 14.x
       - name: yarn install
         run: |
+          # yarn --version
+          # yarn config set network-timeout 300000
           yarn install
       - name: setCurrentVersion
         run: |
@@ -45,3 +48,11 @@ jobs:
           docker tag dbgate dbgate/dbgate:beta
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push dbgate/dbgate:beta
+      - name: Build alpine docker image
+        run: |
+          docker build ./docker -t dbgate -f docker/Dockerfile-alpine
+      - name: Push alpine docker image
+        run: |
+          docker tag dbgate dbgate/dbgate:beta-alpine
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push dbgate/dbgate:beta-alpine

.github/workflows/build-docker.yaml

@@ -51,3 +51,11 @@ jobs:
           docker tag dbgate dbgate/dbgate
           docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
           docker push dbgate/dbgate
+      - name: Build alpine docker image
+        run: |
+          docker build ./docker -t dbgate -f docker/Dockerfile-alpine
+      - name: Push alpine docker image
+        run: |
+          docker tag dbgate dbgate/dbgate:alpine
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push dbgate/dbgate:alpine

.github/workflows/build-npm.yaml

@@ -99,6 +99,11 @@ jobs:
         run: |
           npm publish
 
+      - name: Publish dbgate-plugin-xml
+        working-directory: plugins/dbgate-plugin-xml
+        run: |
+          npm publish
+
       - name: Publish dbgate-plugin-excel
         working-directory: plugins/dbgate-plugin-excel
         run: |

CHANGELOG.md

@@ -1,5 +1,104 @@
 # ChangeLog
 
+### 4.7.2
+- CHANGED: documentation URL - https://dbgate.org/docs/
+- CHANGED: Close button available for all tab groups - #238
+- ADDED: Search function for the Keyboard Shortcuts overview - #239
+- ADDED: Editor font size settings - #229
+- ADDED: Rename MongoDB collection - #223
+- FIXED: bug in cache subsystem
+
+### 4.7.1
+- FIXED: Fixed connecting to MS SQL server running in docker container from DbGate running in docker container #236 
+- FIXED: Fixed export MongoDB collections into Excel and CSV #240
+- ADDED: Added support for docker volumes to persiste connections, when not using configuration via env variables #232
+- ADDED: DbGate in Docker can run in subdirectory #228
+- FIXED: DbGate in Docker can be proxied with nginx #228
+- FIDED: Theme persists when opening multiple windows #207
+- ADDED: Remember fullscreen state #230
+- ADDED: Improved fullscreen state, title bar with menu is hidden, menu is in hamburger menu, like in web version
+- ADDED: Theme choose dialog (added as tab in settings)
+- FIXED: Fixed crash when clicking on application layers #231
+### 4.7.0
+- CHANGED: Changed main menu style, menu and title bar is in one line (+ability to switch to system menu)
+- REMOVED: Removed main toolbar, use main menu or tab related bottom tool instead
+- ADDED: Added tab related context bottom toolbar
+- ADDED: Main menu is available also in web application, by clicking on hamburger menu
+- ADDED: Added support of SQLite to docker container #219
+- ADDED: Added Debian and Alpine docker distributions (default is Debian)
+- FIXED: Fixed performance problem of data grid, especially when there are cells with large data (eg. JSONs), now it is much faster
+- ADDED: Open JSON and array cell buttons
+- ADDED: Handle JSON in varchar cells
+- ADDED: Scroll tabs on mouse wheel
+- ADDED: Show edit edit MySQL column comments #218 #81
+- ADDED: Handle sparse (mssql), unsigned (mysql), zerofill (mysql) column flags
+- FIXED: Fixed same caching problems (eg. leading to indefinitely loading DB structure sometimes)
+- ADDED: Show estimated table row count for MySQL and MS SQL
+- FIXED: Fixed deleting rows from added rows in table data editor
+- ADDED: Better work with JSON lines file, added JSONL editor with preview
+
+### 4.6.3
+- FIXED: Fixed Windows build
+- FIXED: Fixed crash, when there is invalid value in browser local storage
+- FIXED: Fixed plugin description display, where author name or description is not correctly filled
+
+### 4.6.2
+- FIXED: Fixed issues of XML import plugin
+- ADDED: Split columns macro (available in data sheet editor)
+- CHANGED: Accepting non standard plugins names (which doesn't start with dbgate-plugin-)
+- ADDED: Support BLOB values #211
+- ADDED: Picture cell view
+- ADDED: HTML cell view
+- CHANGED: Code completion supports non-default schema names
+- FIXED: More robust MySQL analyser, when connecting to non-standard servers #214
+- FIXED: Fixed configuring connection to SQLite with environment variables #215
+
+### 4.6.1
+- ADDED: Ability to configure SSH tunnel over environment variables #210 (for docker container)
+- ADDED: XML export and import
+- ADDED: Archive file - show and edit source text file
+- ADDED: Window title shows current tab and database
+- ADDED: DbGate documentation
+- ADDED: Introduced application layers
+- ADDED: Virtual foreign key editor
+- ADDED: Application commands (SQL scripts related to database)
+- ADDED: Theme can be implemented in plugin
+- CHANGED: Dictionary description is stored in app
+- FIXED: Unique and index editor
+- FIXED: Posibility to edit UNIQUE index flag
+- CHANGED: UX improvements of table editor
+
+### 4.6.0
+- ADDED: ER diagrams #118
+    - Generate diagram from table or for database
+    - Automatic layout
+    - Diagram styles - colors, select columns to display, optional displaying data type or nullability
+    - Export diagram to HTML file
+- FIXED: Mac latest build link #204
+
+### 4.5.1
+- FIXED: MongoId detection
+- FIXED: #203 disabled spellchecker
+- FIXED: Prevented display filters in form view twice
+- FIXED: Query designer fixes
+
+### 4.5.0
+- ADDED: #220 functions, materialized views and stored procedures in code completion
+- ADDED: Query result in statusbar
+- ADDED: Highlight and execute current query
+- CHANGED: Code completion offers objects only from current query
+- CHANGED: Big optimalizations of electron app - removed embedded web server, removed remote module, updated electron to version 13
+- CHANGED: Removed dependency to electron-store module
+- FIXED: #201 fixed database URL definition, when running from Docvker container
+- FIXED: #192 Docker container stops in 1 second, ability to stop container with Ctrl+C
+- CHANGED: Web app - websocket replaced with SSE technology
+- CHANGED: Changed tab order, tabs are ordered by creation time
+- ADDED: Reorder tabs with drag & drop
+- CHANGED: Collapse left column in datagrid - removed from settings, remember last used state
+- ADDED: Ability to select multiple columns in column manager in datagrid + copy column names
+- ADDED: Show used filters in left datagrid column
+- FIXED: Fixed delete dependency cycle detection (delete didn't work for some tables)
+
 ### 4.4.4
 - FIXED: Database colors
 - CHANGED: Precise work with MongoDB ObjectId

README.md

@@ -50,6 +50,7 @@ DbGate is licensed under MIT license and is completely free.
 * Table data editing, with SQL change script preview
 * Edit table schema, indexes, primary and foreign keys
 * Compare and synchronize database structure
+* ER diagram
 * Light and dark theme
 * Master/detail views, foreign key lookups
 * Query designer
@@ -63,7 +64,7 @@ DbGate is licensed under MIT license and is completely free.
   * Add SQL LEFT/INNER/RIGHT join utility
 * Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
-* Import, export from/to CSV, Excel, JSON
+* Import, export from/to CSV, Excel, JSON, XML
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
 * Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
 * Charts, export chart to HTML page
@@ -77,6 +78,7 @@ Any contributions are welcome. If you want to contribute without coding, conside
 * Write review on [Slant.co](https://www.slant.co/improve/options/41086/~dbgate-review) or [G2](https://www.g2.com/products/dbgate/reviews) 
 * Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
 * Become a backer on [Open collective](https://opencollective.com/dbgate)
+* Where a small coding is acceptable for you, you could [create plugin](https://dbgate.org/docs/plugin-development.html). Plugins for new themes can be created actually without JS coding.
 
 Thank you!
 
@@ -108,27 +110,42 @@ Basic set of plugins is part of DbGate git repository and is installed with app.
 
 ## How to run development environment
 
+Simple variant - runs WEB application:
 ```sh
 yarn
 yarn start
 ```
 
-If you want to make modifications in libraries or plugins, run library compiler in watch mode in the second terminal:
+If you want more control, run WEB application:
 ```sh
-yarn lib
+yarn # install NPM packages
 ```
 
+And than run following 3 commands concurrently in 3 terminals:
+```
+yarn start:api # run API on port 3000
+yarn start:web # run web on port 5000
+yarn lib # watch typescript libraries and plugins modifications
+```
+This runs API on port 3000 and web application on port 5000  
 Open http://localhost:5000 in your browser
 
-You could run electron app (requires running localhost:5000):
+If you want to run electron app:
 ```sh
+yarn # install NPM packages
 cd app
-yarn
-yarn start
+yarn # install NPM packages for electron
+```
+
+And than run following 3 commands concurrently in 3 terminals:
+```
+yarn start:web # run web on port 5000 (only static JS and HTML files)
+yarn lib # watch typescript libraries and plugins modifications
+yarn start:app # run electron app
 ```
 
 ## How to run built electron app locally
-This mode is very similar to production run of electron app. Electron app forks process with API on dynamically allocated port, works with compiled javascript files and uses compiled version of plugins (doesn't use localhost:5000)
+This mode is very similar to production run of electron app. Electron doesn't use localhost:5000.
 
 ```sh
 cd app
@@ -141,3 +158,16 @@ yarn build:app:local
 yarn start:app:local
 ```
 
+## How to create plugin
+Creating plugin is described in [documentation](https://github.com/dbgate/dbgate/wiki/Plugin-development)
+
+But it is very simple:
+
+```sh
+npm install -g yo # install yeoman
+npm install -g generator-dbgate # install dbgate generator
+cd dbgate-plugin-my-new-plugin # this directory is created by wizard, edit, what you need to change
+yarn plugin # this compiles plugin and copies it into existing DbGate installation
+```
+
+After restarting DbGate, you could use your new plugin from DbGate.

app/package.json

@@ -6,8 +6,8 @@
   "description": "Opensource database administration tool",
   "dependencies": {
     "electron-log": "^4.4.1",
-    "electron-store": "^8.0.1",
     "electron-updater": "^4.6.1",
+    "lodash.clonedeepwith": "^4.5.0",
     "patch-package": "^6.4.7"
   },
   "repository": {
@@ -93,16 +93,16 @@
     "start:local": "cross-env electron .",
     "dist": "electron-builder",
     "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist",
-    "build:mac": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && node setMacPlatform x64 && yarn dist && node setMacPlatform arm64 && yarn dist",
     "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn predist",
-    "postinstall": "electron-builder install-app-deps && patch-package",
+    "postinstall": "yarn rebuild && patch-package",
+    "rebuild": "electron-builder install-app-deps",
     "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/public/*\" packages && copyfiles \"../packages/web/public/**/*\" packages && copyfiles --up 3 \"../plugins/dist/**/*\" packages/plugins"
   },
   "main": "src/electron.js",
   "devDependencies": {
     "copyfiles": "^2.2.0",
     "cross-env": "^6.0.3",
-    "electron": "12.1.0",
+    "electron": "13.6.3",
     "electron-builder": "22.14.5"
   },
   "optionalDependencies": {

app/setMacPlatform.js

@@ -1,8 +0,0 @@
-const fs = require('fs');
-
-const text = fs.readFileSync('package.json', { encoding: 'utf-8' });
-const json = JSON.parse(text);
-
-json.build.mac.target.arch = process.argv[2];
-
-fs.writeFileSync('package.json', JSON.stringify(json, null, 2), { encoding: 'utf-8' });

app/src/electron.js

@@ -1,10 +1,10 @@
 const electron = require('electron');
 const os = require('os');
+const fs = require('fs');
 const { Menu, ipcMain } = require('electron');
-const { fork } = require('child_process');
 const { autoUpdater } = require('electron-updater');
-const Store = require('electron-store');
 const log = require('electron-log');
+const _cloneDeepWith = require('lodash.clonedeepwith');
 
 // Module to control application life.
 const app = electron.app;
@@ -13,10 +13,29 @@ const BrowserWindow = electron.BrowserWindow;
 
 const path = require('path');
 const url = require('url');
+const mainMenuDefinition = require('./mainMenuDefinition');
+const { settings } = require('cluster');
 
 // require('@electron/remote/main').initialize();
 
-const store = new Store();
+const configRootPath = path.join(app.getPath('userData'), 'config-root.json');
+let initialConfig = {};
+
+try {
+  initialConfig = JSON.parse(fs.readFileSync(configRootPath, { encoding: 'utf-8' }));
+} catch (err) {
+  console.log('Error loading config-root:', err.message);
+  initialConfig = {};
+}
+
+// let settingsJson = {};
+// try {
+//   const datadir = path.join(os.homedir(), 'dbgate-data');
+//   settingsJson = JSON.parse(fs.readFileSync(path.join(datadir, 'settings.json'), { encoding: 'utf-8' }));
+// } catch (err) {
+//   console.log('Error loading settings.json:', err.message);
+//   settingsJson = {};
+// }
 
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
@@ -44,89 +63,15 @@ function commandItem(id) {
 }
 
 function buildMenu() {
-  const template = [
-    {
-      label: 'File',
-      submenu: [
-        commandItem('new.connection'),
-        commandItem('new.sqliteDatabase'),
-        commandItem('new.modelCompare'),
-        commandItem('new.freetable'),
-        { type: 'separator' },
-        commandItem('file.open'),
-        commandItem('file.openArchive'),
-        { type: 'separator' },
-        commandItem('group.save'),
-        commandItem('group.saveAs'),
-        commandItem('database.search'),
-        { type: 'separator' },
-        commandItem('tabs.closeTab'),
-        commandItem('file.exit'),
-      ],
-    },
-    {
-      label: 'Window',
-      submenu: [commandItem('new.query'), { type: 'separator' }, commandItem('tabs.closeAll'), { role: 'minimize' }],
-    },
+  const template = _cloneDeepWith(mainMenuDefinition, item => {
+    if (item.divider) {
+      return { type: 'separator' };
+    }
 
-    // {
-    //   label: 'Edit',
-    //   submenu: [
-    //     { role: 'undo' },
-    //     { role: 'redo' },
-    //     { type: 'separator' },
-    //     { role: 'cut' },
-    //     { role: 'copy' },
-    //     { role: 'paste' },
-    //   ],
-    // },
-    {
-      label: 'View',
-      submenu: [
-        { role: 'reload' },
-        { role: 'forcereload' },
-        { role: 'toggledevtools' },
-        { type: 'separator' },
-        { role: 'resetzoom' },
-        { role: 'zoomin' },
-        { role: 'zoomout' },
-        { type: 'separator' },
-        { role: 'togglefullscreen' },
-        commandItem('theme.changeTheme'),
-      ],
-    },
-    {
-      role: 'help',
-      submenu: [
-        {
-          label: 'dbgate.org',
-          click() {
-            electron.shell.openExternal('https://dbgate.org');
-          },
-        },
-        {
-          label: 'DbGate on GitHub',
-          click() {
-            electron.shell.openExternal('https://github.com/dbgate/dbgate');
-          },
-        },
-        {
-          label: 'DbGate on docker hub',
-          click() {
-            electron.shell.openExternal('https://hub.docker.com/r/dbgate/dbgate');
-          },
-        },
-        {
-          label: 'Report problem or feature request',
-          click() {
-            electron.shell.openExternal('https://github.com/dbgate/dbgate/issues/new');
-          },
-        },
-        commandItem('tabs.changelog'),
-        commandItem('about.show'),
-      ],
-    },
-  ];
+    if (item.command) {
+      return commandItem(item.command);
+    }
+  });
 
   return Menu.buildFromTemplate(template);
 }
@@ -151,6 +96,50 @@ ipcMain.on('update-commands', async (event, arg) => {
 ipcMain.on('close-window', async (event, arg) => {
   mainWindow.close();
 });
+ipcMain.on('set-title', async (event, arg) => {
+  mainWindow.setTitle(arg);
+});
+ipcMain.on('open-link', async (event, arg) => {
+  electron.shell.openExternal(arg);
+});
+ipcMain.on('window-action', async (event, arg) => {
+  switch (arg) {
+    case 'minimize':
+      mainWindow.minimize();
+      break;
+    case 'maximize':
+      if (mainWindow.isMaximized()) {
+        mainWindow.unmaximize();
+      } else {
+        mainWindow.maximize();
+      }
+      break;
+    case 'close':
+      mainWindow.close();
+      break;
+    case 'fullscreen-on':
+      mainWindow.setFullScreen(true);
+      break;
+    case 'fullscreen-off':
+      mainWindow.setFullScreen(false);
+      break;
+    case 'devtools':
+      mainWindow.webContents.toggleDevTools();
+      break;
+    case 'reload':
+      mainWindow.webContents.reloadIgnoringCache();
+      break;
+    case 'zoomin':
+      mainWindow.webContents.zoomLevel += 0.5;
+      break;
+    case 'zoomout':
+      mainWindow.webContents.zoomLevel -= 0.5;
+      break;
+    case 'zoomreset':
+      mainWindow.webContents.zoomLevel = 0;
+      break;
+  }
+});
 
 ipcMain.handle('showOpenDialog', async (event, options) => {
   const res = electron.dialog.showOpenDialogSync(mainWindow, options);
@@ -167,24 +156,53 @@ ipcMain.handle('openExternal', async (event, url) => {
   electron.shell.openExternal(url);
 });
 
+function fillMissingSettings(value) {
+  const res = {
+    ...value,
+  };
+  if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+    res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
+  }
+  return res;
+}
+
 function createWindow() {
-  const bounds = store.get('winBounds');
+  let settingsJson = {};
+  try {
+    const datadir = path.join(os.homedir(), 'dbgate-data');
+    settingsJson = fillMissingSettings(
+      JSON.parse(fs.readFileSync(path.join(datadir, 'settings.json'), { encoding: 'utf-8' }))
+    );
+  } catch (err) {
+    console.log('Error loading settings.json:', err.message);
+    settingsJson = fillMissingSettings({});
+  }
+
+  const bounds = initialConfig['winBounds'];
+  useNativeMenu = settingsJson['app.useNativeMenu'];
 
   mainWindow = new BrowserWindow({
     width: 1200,
     height: 800,
     title: 'DbGate',
+    frame: useNativeMenu,
+    titleBarStyle: useNativeMenu ? undefined : 'hidden',
     ...bounds,
     icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
+    partition: 'persist:dbgate',
     webPreferences: {
       nodeIntegration: true,
       contextIsolation: false,
+      spellcheck: false,
     },
   });
 
-  if (store.get('winIsMaximized')) {
+  if (initialConfig['winIsMaximized']) {
     mainWindow.maximize();
   }
+  if (settingsJson['app.fullscreen']) {
+    mainWindow.setFullScreen(true);
+  }
 
   mainMenu = buildMenu();
   mainWindow.setMenu(mainMenu);
@@ -198,20 +216,42 @@ function createWindow() {
         slashes: true,
       });
     mainWindow.on('close', () => {
-      store.set('winBounds', mainWindow.getBounds());
-      store.set('winIsMaximized', mainWindow.isMaximized());
+      try {
+        fs.writeFileSync(
+          configRootPath,
+          JSON.stringify({
+            winBounds: mainWindow.getBounds(),
+            winIsMaximized: mainWindow.isMaximized(),
+          }),
+          'utf-8'
+        );
+      } catch (err) {
+        console.log('Error saving config-root:', err.message);
+      }
     });
     mainWindow.loadURL(startUrl);
     if (os.platform() == 'linux') {
       mainWindow.setIcon(path.resolve(__dirname, '../icon.png'));
     }
+    // mainWindow.webContents.toggleDevTools();
   }
 
-  global.API_PACKAGE = process.env.DEVMODE ? '../packages/api/src/index' : '../packages/api/dist/bundle.js';
-  const api = require(path.join(
+  const apiPackage = path.join(
     __dirname,
     process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js'
-  ));
+  );
+
+  global.API_PACKAGE = apiPackage;
+  global.NATIVE_MODULES = path.join(__dirname, 'nativeModules');
+
+  // console.log('global.API_PACKAGE', global.API_PACKAGE);
+  const api = require(apiPackage);
+  // console.log(
+  //   'REQUIRED',
+  //   path.resolve(
+  //     path.join(__dirname, process.env.DEVMODE ? '../../packages/api/src/index' : '../packages/api/dist/bundle.js')
+  //   )
+  // );
   const main = api.getMainModule();
   main.initializeElectronSender(mainWindow.webContents);
   main.useAllControllers(null, electron);

app/src/mainMenuDefinition.js

@@ -0,0 +1,82 @@
+module.exports = [
+  {
+    label: 'File',
+    submenu: [
+      { command: 'new.connection', hideDisabled: true },
+      { command: 'new.sqliteDatabase', hideDisabled: true },
+      { divider: true },
+      { command: 'new.query', hideDisabled: true },
+      { command: 'new.freetable', hideDisabled: true },
+      { command: 'new.shell', hideDisabled: true },
+      { command: 'new.jsonl', hideDisabled: true },
+      { divider: true },
+      { command: 'file.open', hideDisabled: true },
+      { command: 'file.openArchive', hideDisabled: true },
+      { divider: true },
+      { command: 'group.save', hideDisabled: true },
+      { command: 'group.saveAs', hideDisabled: true },
+      { divider: true },
+      { command: 'file.exit', hideDisabled: true },
+    ],
+  },
+  {
+    label: 'Window',
+    submenu: [
+      { command: 'tabs.closeTab', hideDisabled: false },
+      { command: 'tabs.closeAll', hideDisabled: false },
+      { command: 'tabs.closeTabsWithCurrentDb', hideDisabled: false },
+      { command: 'tabs.closeTabsButCurrentDb', hideDisabled: false },
+      { divider: true },
+      { command: 'app.zoomIn', hideDisabled: true },
+      { command: 'app.zoomOut', hideDisabled: true },
+      { command: 'app.zoomReset', hideDisabled: true },
+    ],
+  },
+
+  // {
+  //   label: 'Edit',
+  //   submenu: [
+  //     { role: 'undo' },
+  //     { role: 'redo' },
+  //     { type: 'separator' },
+  //     { role: 'cut' },
+  //     { role: 'copy' },
+  //     { role: 'paste' },
+  //   ],
+  // },
+  {
+    label: 'View',
+    submenu: [
+      { command: 'app.reload', hideDisabled: true },
+      { command: 'app.toggleDevTools', hideDisabled: true },
+      { command: 'app.toggleFullScreen', hideDisabled: true },
+      { command: 'app.minimize', hideDisabled: true },
+      { divider: true },
+      { command: 'theme.changeTheme', hideDisabled: true },
+      { command: 'settings.show' },
+    ],
+  },
+  {
+    label: 'Tools',
+    submenu: [
+      { command: 'database.search', hideDisabled: true },
+      { command: 'commandPalette.show', hideDisabled: true },
+      { divider: true },
+      { command: 'sql.generator', hideDisabled: true },
+      { command: 'file.import', hideDisabled: true },
+      { command: 'new.modelCompare', hideDisabled: true },
+    ],
+  },
+  {
+    label: 'Help',
+    submenu: [
+      { command: 'app.openDocs', hideDisabled: true },
+      { command: 'app.openWeb', hideDisabled: true },
+      { command: 'app.openIssue', hideDisabled: true },
+      { command: 'app.openSponsoring', hideDisabled: true },
+      { divider: true },
+      { command: 'tabs.changelog', hideDisabled: true },
+      { command: 'about.show', hideDisabled: true },
+    ],
+  },
+];

app/yarn.lock

@@ -148,13 +148,6 @@
   resolved "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz#e77a97fbd345b76d83245edcd17d393b1b41fb31"
   integrity sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==
 
-ajv-formats@^2.1.1:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520"
-  integrity sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==
-  dependencies:
-    ajv "^8.0.0"
-
 ajv-keywords@^3.4.1:
   version "3.5.2"
   resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.5.2.tgz#31f29da5ab6e00d1c2d329acf7b5929614d5014d"
@@ -170,16 +163,6 @@ ajv@^6.10.0, ajv@^6.12.0:
     json-schema-traverse "^0.4.1"
     uri-js "^4.2.2"
 
-ajv@^8.0.0, ajv@^8.6.3:
-  version "8.8.2"
-  resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.8.2.tgz#01b4fef2007a28bf75f0b7fc009f62679de4abbb"
-  integrity sha512-x9VuX+R/jcFj1DHo/fCp99esgGDWiHENrKxaCENuCxpoMCmAt/COCGVDwA7kleEpEzJjDnvh3yGoOuLu0Dtllw==
-  dependencies:
-    fast-deep-equal "^3.1.1"
-    json-schema-traverse "^1.0.0"
-    require-from-string "^2.0.2"
-    uri-js "^4.2.2"
-
 ansi-align@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/ansi-align/-/ansi-align-3.0.1.tgz#0cdf12e111ace773a86e9a1fad1225c43cb19a59"
@@ -307,11 +290,6 @@ at-least-node@^1.0.0:
   resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2"
   integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==
 
-atomically@^1.7.0:
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/atomically/-/atomically-1.7.0.tgz#c07a0458432ea6dbc9a3506fffa424b48bccaafe"
-  integrity sha512-Xcz9l0z7y9yQ9rdDaxlmaI4uJHf/T8g9hOEzJcsEqX2SjCj4J20uK7+ldkDHMbpJDK76wF7xEIgxc/vSlsfw5w==
-
 balanced-match@^1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
@@ -625,22 +603,6 @@ concat-stream@^1.6.2:
     readable-stream "^2.2.2"
     typedarray "^0.0.6"
 
-conf@^10.0.3:
-  version "10.1.1"
-  resolved "https://registry.yarnpkg.com/conf/-/conf-10.1.1.tgz#ff08046d5aeeee0eaff55d57f5b4319193c3dfda"
-  integrity sha512-z2civwq/k8TMYtcn3SVP0Peso4otIWnHtcTuHhQ0zDZDdP4NTxqEc8owfkz4zBsdMYdn/LFcE+ZhbCeqkhtq3Q==
-  dependencies:
-    ajv "^8.6.3"
-    ajv-formats "^2.1.1"
-    atomically "^1.7.0"
-    debounce-fn "^4.0.0"
-    dot-prop "^6.0.1"
-    env-paths "^2.2.1"
-    json-schema-typed "^7.0.3"
-    onetime "^5.1.2"
-    pkg-up "^3.1.0"
-    semver "^7.3.5"
-
 config-chain@^1.1.11:
   version "1.1.13"
   resolved "https://registry.yarnpkg.com/config-chain/-/config-chain-1.1.13.tgz#fad0795aa6a6cdaff9ed1b68e9dff94372c232f4"
@@ -728,13 +690,6 @@ crypto-random-string@^2.0.0:
   resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5"
   integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==
 
-debounce-fn@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/debounce-fn/-/debounce-fn-4.0.0.tgz#ed76d206d8a50e60de0dd66d494d82835ffe61c7"
-  integrity sha512-8pYCQiL9Xdcg0UPSD3d+0KMlOjp+KGU5EPwYddgzQ7DATsg4fuUDjQtsYLmWjnk2obnNHgV3vE2Y4jejSOJVBQ==
-  dependencies:
-    mimic-fn "^3.0.0"
-
 debug@^2.6.8, debug@^2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -852,13 +807,6 @@ dot-prop@^5.2.0:
   dependencies:
     is-obj "^2.0.0"
 
-dot-prop@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-6.0.1.tgz#fc26b3cf142b9e59b74dbd39ed66ce620c681083"
-  integrity sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==
-  dependencies:
-    is-obj "^2.0.0"
-
 dotenv-expand@^5.1.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-5.1.0.tgz#3fbaf020bfd794884072ea26b1e9791d45a629f0"
@@ -929,14 +877,6 @@ electron-publish@22.14.5:
     lazy-val "^1.0.5"
     mime "^2.5.2"
 
-electron-store@^8.0.1:
-  version "8.0.1"
-  resolved "https://registry.yarnpkg.com/electron-store/-/electron-store-8.0.1.tgz#9b598c1d2edeffebee9d8c1cd957ad368c528925"
-  integrity sha512-ZyLvNywiqSpbwC/pp89O/AycVWY/UJIkmtyzF2Bd0Nm/rLmcFc0NTGuLdg6+LE8mS8qsiK5JMoe4PnrecLHH5w==
-  dependencies:
-    conf "^10.0.3"
-    type-fest "^1.0.2"
-
 electron-updater@^4.6.1:
   version "4.6.1"
   resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-4.6.1.tgz#80ca805c4f51b2e682aac29d18fed75d6a533d32"
@@ -951,10 +891,10 @@ electron-updater@^4.6.1:
     lodash.isequal "^4.5.0"
     semver "^7.3.5"
 
-electron@12.1.0:
-  version "12.1.0"
-  resolved "https://registry.yarnpkg.com/electron/-/electron-12.1.0.tgz#615a7f9dbb2fc79cc72361fba9f39d005c697bca"
-  integrity sha512-joQlYI/nTIrTUldO3GENZ2j225eKar9nTQBSEwSUSWN4h65QGDmXNQ7dbWPmLlkUQWtHhz8lXhFk30OLG9ZjLw==
+electron@13.6.3:
+  version "13.6.3"
+  resolved "https://registry.yarnpkg.com/electron/-/electron-13.6.3.tgz#c0217178807d3e0b2175c49dbe33ea8dac447e73"
+  integrity sha512-kevgR6/RuEhchJQbgCKhHle9HvJhi2dOJlicFZJqbbqa9BVpZARqqFDlwTSatYxmUPUJwu09FvyMwJG2DMQIng==
   dependencies:
     "@electron/get" "^1.0.1"
     "@types/node" "^14.6.2"
@@ -977,7 +917,7 @@ end-of-stream@^1.1.0, end-of-stream@^1.4.1:
   dependencies:
     once "^1.4.0"
 
-env-paths@^2.2.0, env-paths@^2.2.1:
+env-paths@^2.2.0:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.1.tgz#420399d416ce1fbe9bc0a07c62fa68d67fd0f8f2"
   integrity sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==
@@ -1063,13 +1003,6 @@ fill-range@^7.0.1:
   dependencies:
     to-regex-range "^5.0.1"
 
-find-up@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/find-up/-/find-up-3.0.0.tgz#49169f1d7993430646da61ecc5ae355c21c97b73"
-  integrity sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==
-  dependencies:
-    locate-path "^3.0.0"
-
 find-yarn-workspace-root@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz#f47fb8d239c900eb78179aa81b66673eac88f7bd"
@@ -1468,16 +1401,6 @@ json-schema-traverse@^0.4.1:
   resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660"
   integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==
 
-json-schema-traverse@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2"
-  integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==
-
-json-schema-typed@^7.0.3:
-  version "7.0.3"
-  resolved "https://registry.yarnpkg.com/json-schema-typed/-/json-schema-typed-7.0.3.tgz#23ff481b8b4eebcd2ca123b4fa0409e66469a2d9"
-  integrity sha512-7DE8mpG+/fVw+dTpjbxnx47TaMnDfOI1jwft9g1VybltZCduyRQPJPvc+zzKY9WPHxhPWczyFuYa6I8Mw4iU5A==
-
 json-stringify-safe@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
@@ -1532,13 +1455,10 @@ lazy-val@^1.0.4, lazy-val@^1.0.5:
   resolved "https://registry.yarnpkg.com/lazy-val/-/lazy-val-1.0.5.tgz#6cf3b9f5bc31cee7ee3e369c0832b7583dcd923d"
   integrity sha512-0/BnGCCfyUMkBpeDgWihanIAF9JmZhHBgUhEqzvf+adhNGLoP6TaiI5oF8oyb3I45P+PcnrqihSf01M0l0G5+Q==
 
-locate-path@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-3.0.0.tgz#dbec3b3ab759758071b58fe59fc41871af21400e"
-  integrity sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==
-  dependencies:
-    p-locate "^3.0.0"
-    path-exists "^3.0.0"
+lodash.clonedeepwith@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.clonedeepwith/-/lodash.clonedeepwith-4.5.0.tgz#6ee30573a03a1a60d670a62ef33c10cf1afdbdd4"
+  integrity sha1-buMFc6A6GmDWcKYu8zwQzxr9vdQ=
 
 lodash.escaperegexp@^4.1.2:
   version "4.1.2"
@@ -1611,16 +1531,6 @@ mime@^2.5.2:
   resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367"
   integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==
 
-mimic-fn@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
-  integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==
-
-mimic-fn@^3.0.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-3.1.0.tgz#65755145bbf3e36954b949c16450427451d5ca74"
-  integrity sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ==
-
 mimic-response@^1.0.0, mimic-response@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b"
@@ -1785,13 +1695,6 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0:
   dependencies:
     wrappy "1"
 
-onetime@^5.1.2:
-  version "5.1.2"
-  resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e"
-  integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==
-  dependencies:
-    mimic-fn "^2.1.0"
-
 open@^7.4.2:
   version "7.4.2"
   resolved "https://registry.yarnpkg.com/open/-/open-7.4.2.tgz#b8147e26dcf3e426316c730089fd71edd29c2321"
@@ -1810,25 +1713,6 @@ p-cancelable@^1.0.0:
   resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc"
   integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==
 
-p-limit@^2.0.0:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1"
-  integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==
-  dependencies:
-    p-try "^2.0.0"
-
-p-locate@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4"
-  integrity sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==
-  dependencies:
-    p-limit "^2.0.0"
-
-p-try@^2.0.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"
-  integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==
-
 package-json@^6.3.0:
   version "6.5.0"
   resolved "https://registry.yarnpkg.com/package-json/-/package-json-6.5.0.tgz#6feedaca35e75725876d0b0e64974697fed145b0"
@@ -1858,11 +1742,6 @@ patch-package@^6.4.7:
     slash "^2.0.0"
     tmp "^0.0.33"
 
-path-exists@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515"
-  integrity sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=
-
 path-is-absolute@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
@@ -1893,13 +1772,6 @@ pify@^3.0.0:
   resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176"
   integrity sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=
 
-pkg-up@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/pkg-up/-/pkg-up-3.1.0.tgz#100ec235cc150e4fd42519412596a28512a0def5"
-  integrity sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==
-  dependencies:
-    find-up "^3.0.0"
-
 plist@^3.0.1, plist@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/plist/-/plist-3.0.4.tgz#a62df837e3aed2bb3b735899d510c4f186019cbe"
@@ -2058,11 +1930,6 @@ require-directory@^2.1.1:
   resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
   integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I=
 
-require-from-string@^2.0.2:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909"
-  integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==
-
 responselike@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/responselike/-/responselike-1.0.2.tgz#918720ef3b631c5642be068f15ade5a46f4ba1e7"
@@ -2446,11 +2313,6 @@ type-fest@^0.20.2:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4"
   integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
 
-type-fest@^1.0.2:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-1.4.0.tgz#e9fb813fe3bf1744ec359d55d1affefa76f14be1"
-  integrity sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==
-
 typedarray-to-buffer@^3.1.5:
   version "3.1.5"
   resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080"

docker-compose.yaml

@@ -0,0 +1,53 @@
+# this compose file is for testing purposes only
+# use it for testing docker containsers built on local machine
+version: "3"
+services:
+  dbgate:
+    build: docker
+    # image: dbgate/dbgate:beta-alpine
+    # image: dbgate/dbgate:beta
+    restart: always
+    ports:
+      - 3100:3000
+    # volumes:
+    #   - /home/jena/dbgate-data:/root/dbgate-data
+
+    volumes:
+      - dbgate-data:/root/dbgate-data
+    
+    environment:
+      WEB_ROOT: /dbgate
+
+      # CONNECTIONS: mssql
+      # LABEL_mssql: MS Sql
+      # SERVER_mssql: mssql
+      # USER_mssql: sa
+      # PORT_mssql: 1433
+      # PASSWORD_mssql: Pwd2020Db
+      # ENGINE_mssql: mssql@dbgate-plugin-mssql
+  proxy:
+    # image: nginx
+    build: test/nginx
+    ports:
+      - 8082:80
+
+    # volumes:
+    #   - /home/jena/test/chinook:/mnt/sqt
+    # environment:
+    #   CONNECTIONS: sqlite
+
+    #   LABEL_sqlite: sqt
+    #   FILE_sqlite: /mnt/sqt/Chinook.db
+    #   ENGINE_sqlite: sqlite@dbgate-plugin-sqlite
+
+  # mssql:
+  #   image: mcr.microsoft.com/mssql/server
+  #   restart: always
+  #   environment:
+  #     - ACCEPT_EULA=Y
+  #     - SA_PASSWORD=Pwd2020Db
+  #     - MSSQL_PID=Express
+
+volumes:
+  dbgate-data:
+    driver: local

docker/.gitignore

@@ -0,0 +1,2 @@
+package.json
+yarn.lock

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-alpine
+FROM node:14
 
 WORKDIR /home/dbgate-docker
 
@@ -6,4 +6,5 @@ COPY . .
 
 WORKDIR /home/dbgate-docker
 EXPOSE 3000
+VOLUME /root/dbgate-data
 CMD node bundle.js

docker/Dockerfile-alpine

@@ -0,0 +1,10 @@
+FROM node:14-alpine
+
+WORKDIR /home/dbgate-docker
+
+COPY . .
+
+WORKDIR /home/dbgate-docker
+EXPOSE 3000
+VOLUME /root/dbgate-data
+CMD node bundle.js

package.json

@@ -1,6 +1,6 @@
 {
   "private": true,
-  "version": "4.4.5-beta.12",
+  "version": "4.7.3-beta.1",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
@@ -25,7 +25,6 @@
     "build:tools": "yarn workspace dbgate-tools build",
     "build:lib": "yarn build:querysplitter && yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
     "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
-    "build:app:mac": "yarn plugins:copydist && cd app && yarn install && yarn build:mac",
     "build:api": "yarn workspace dbgate-api build",
     "build:web:docker": "yarn workspace dbgate-web build",
     "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
@@ -41,7 +40,8 @@
     "resetPackagedPlugins": "node resetPackagedPlugins",
     "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
     "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
-    "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build",
+    "install:sqlite:docker": "cd docker && yarn init --yes && yarn add better-sqlite3 && cd ..",
+    "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build && yarn install:sqlite:docker",
     "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
     "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn start:querysplitter\" \"yarn build:plugins:frontend:watch\"",
     "ts:api": "yarn workspace dbgate-api ts",

packages/api/.env-portal

@@ -1,6 +1,6 @@
 DEVMODE=1
 
-CONNECTIONS=mysql,postgres
+CONNECTIONS=mysql,postgres,mongo,mongo2,mysqlssh,sqlite
 
 LABEL_mysql=MySql localhost
 SERVER_mysql=localhost
@@ -15,3 +15,30 @@ USER_postgres=postgres
 PASSWORD_postgres=test
 PORT_postgres=5433
 ENGINE_postgres=postgres@dbgate-plugin-postgres
+
+LABEL_mongo=Mongo URL
+URL_mongo=mongodb://localhost:27017
+ENGINE_mongo=mongo@dbgate-plugin-mongo
+
+LABEL_mongo2=Mongo Server
+SERVER_mongo2=localhost
+ENGINE_mongo2=mongo@dbgate-plugin-mongo
+
+LABEL_mysqlssh=MySql SSH
+SERVER_mysqlssh=localhost
+USER_mysqlssh=root
+PASSWORD_mysqlssh=xxx
+PORT_mysqlssh=3316
+ENGINE_mysqlssh=mysql@dbgate-plugin-mysql
+USE_SSH_mysqlssh=1
+SSH_HOST_mysqlssh=demo.dbgate.org
+SSH_PORT_mysqlssh=22
+SSH_MODE_mysqlssh=userPassword
+SSH_LOGIN_mysqlssh=root
+SSH_PASSWORD_mysqlssh=xxx
+
+LABEL_sqlite=sqlite
+FILE_sqlite=/home/jena/dbgate-data/files/sqlite/feeds.sqlite
+ENGINE_sqlite=sqlite@dbgate-plugin-sqlite
+
+# docker run -p 3000:3000 -e CONNECTIONS=mongo -e URL_mongo=mongodb://localhost:27017 -e ENGINE_mongo=mongo@dbgate-plugin-mongo -e LABEL_mongo=mongo dbgate/dbgate:beta

packages/api/package.json

@@ -44,7 +44,6 @@
     "line-reader": "^0.4.0",
     "lodash": "^4.17.21",
     "ncp": "^2.0.0",
-    "nedb-promises": "^4.0.1",
     "node-cron": "^2.0.3",
     "node-ssh-forward": "^0.7.2",
     "portfinder": "^1.0.28",

packages/api/src/controllers/apps.js

@@ -0,0 +1,268 @@
+const fs = require('fs-extra');
+const _ = require('lodash');
+const path = require('path');
+const { appdir } = require('../utility/directories');
+const socket = require('../utility/socket');
+const connections = require('./connections');
+
+module.exports = {
+  folders_meta: true,
+  async folders() {
+    const folders = await fs.readdir(appdir());
+    return [
+      ...folders.map(name => ({
+        name,
+      })),
+    ];
+  },
+
+  createFolder_meta: true,
+  async createFolder({ folder }) {
+    const name = await this.getNewAppFolder({ name: folder });
+    await fs.mkdir(path.join(appdir(), name));
+    socket.emitChanged('app-folders-changed');
+    this.emitChangedDbApp(folder);
+    return name;
+  },
+
+  files_meta: true,
+  async files({ folder }) {
+    if (!folder) return [];
+    const dir = path.join(appdir(), folder);
+    if (!(await fs.exists(dir))) return [];
+    const files = await fs.readdir(dir);
+
+    function fileType(ext, type) {
+      return files
+        .filter(name => name.endsWith(ext))
+        .map(name => ({
+          name: name.slice(0, -ext.length),
+          label: path.parse(name.slice(0, -ext.length)).base,
+          type,
+        }));
+    }
+
+    return [
+      ...fileType('.command.sql', 'command.sql'),
+      ...fileType('.query.sql', 'query.sql'),
+      ...fileType('.config.json', 'config.json'),
+    ];
+  },
+
+  async emitChangedDbApp(folder) {
+    const used = await this.getUsedAppFolders();
+    if (used.includes(folder)) {
+      socket.emitChanged('used-apps-changed');
+    }
+  },
+
+  refreshFiles_meta: true,
+  async refreshFiles({ folder }) {
+    socket.emitChanged(`app-files-changed-${folder}`);
+  },
+
+  refreshFolders_meta: true,
+  async refreshFolders() {
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFile_meta: true,
+  async deleteFile({ folder, file, fileType }) {
+    await fs.unlink(path.join(appdir(), folder, `${file}.${fileType}`));
+    socket.emitChanged(`app-files-changed-${folder}`);
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFile_meta: true,
+  async renameFile({ folder, file, newFile, fileType }) {
+    await fs.rename(
+      path.join(path.join(appdir(), folder), `${file}.${fileType}`),
+      path.join(path.join(appdir(), folder), `${newFile}.${fileType}`)
+    );
+    socket.emitChanged(`app-files-changed-${folder}`);
+    this.emitChangedDbApp(folder);
+  },
+
+  renameFolder_meta: true,
+  async renameFolder({ folder, newFolder }) {
+    const uniqueName = await this.getNewAppFolder({ name: newFolder });
+    await fs.rename(path.join(appdir(), folder), path.join(appdir(), uniqueName));
+    socket.emitChanged(`app-folders-changed`);
+  },
+
+  deleteFolder_meta: true,
+  async deleteFolder({ folder }) {
+    if (!folder) throw new Error('Missing folder parameter');
+    await fs.rmdir(path.join(appdir(), folder), { recursive: true });
+    socket.emitChanged(`app-folders-changed`);
+    socket.emitChanged(`app-files-changed-${folder}`);
+    socket.emitChanged('used-apps-changed');
+  },
+
+  async getNewAppFolder({ name }) {
+    if (!(await fs.exists(path.join(appdir(), name)))) return name;
+    let index = 2;
+    while (await fs.exists(path.join(appdir(), `${name}${index}`))) {
+      index += 1;
+    }
+    return `${name}${index}`;
+  },
+
+  getUsedAppFolders_meta: true,
+  async getUsedAppFolders() {
+    const list = await connections.list();
+    const apps = [];
+
+    for (const connection of list) {
+      for (const db of connection.databases || []) {
+        for (const key of _.keys(db || {})) {
+          if (key.startsWith('useApp:') && db[key]) {
+            apps.push(key.substring('useApp:'.length));
+          }
+        }
+      }
+    }
+
+    return _.intersection(_.uniq(apps), await fs.readdir(appdir()));
+  },
+
+  getUsedApps_meta: true,
+  async getUsedApps() {
+    const apps = await this.getUsedAppFolders();
+    const res = [];
+
+    for (const folder of apps) {
+      res.push(await this.loadApp({ folder }));
+    }
+    return res;
+  },
+
+  // getAppsForDb_meta: true,
+  // async getAppsForDb({ conid, database }) {
+  //   const connection = await connections.get({ conid });
+  //   if (!connection) return [];
+  //   const db = (connection.databases || []).find(x => x.name == database);
+  //   const apps = [];
+  //   const res = [];
+  //   if (db) {
+  //     for (const key of _.keys(db || {})) {
+  //       if (key.startsWith('useApp:') && db[key]) {
+  //         apps.push(key.substring('useApp:'.length));
+  //       }
+  //     }
+  //   }
+  //   for (const folder of apps) {
+  //     res.push(await this.loadApp({ folder }));
+  //   }
+  //   return res;
+  // },
+
+  loadApp_meta: true,
+  async loadApp({ folder }) {
+    const res = {
+      queries: [],
+      commands: [],
+      name: folder,
+    };
+    const dir = path.join(appdir(), folder);
+    if (await fs.exists(dir)) {
+      const files = await fs.readdir(dir);
+
+      async function processType(ext, field) {
+        for (const file of files) {
+          if (file.endsWith(ext)) {
+            res[field].push({
+              name: file.slice(0, -ext.length),
+              sql: await fs.readFile(path.join(dir, file), { encoding: 'utf-8' }),
+            });
+          }
+        }
+      }
+
+      await processType('.command.sql', 'commands');
+      await processType('.query.sql', 'queries');
+    }
+
+    try {
+      res.virtualReferences = JSON.parse(
+        await fs.readFile(path.join(dir, 'virtual-references.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.virtualReferences = [];
+    }
+    try {
+      res.dictionaryDescriptions = JSON.parse(
+        await fs.readFile(path.join(dir, 'dictionary-descriptions.config.json'), { encoding: 'utf-8' })
+      );
+    } catch (err) {
+      res.dictionaryDescriptions = [];
+    }
+
+    return res;
+  },
+
+  async saveConfigFile(appFolder, filename, filterFunc, newItem) {
+    const file = path.join(appdir(), appFolder, filename);
+
+    let json;
+    try {
+      json = JSON.parse(await fs.readFile(file, { encoding: 'utf-8' }));
+    } catch (err) {
+      json = [];
+    }
+
+    if (filterFunc) {
+      json = json.filter(filterFunc);
+    }
+
+    json = [...json, newItem];
+
+    await fs.writeFile(file, JSON.stringify(json, undefined, 2));
+
+    socket.emitChanged(`app-files-changed-${appFolder}`);
+    socket.emitChanged('used-apps-changed');
+  },
+
+  saveVirtualReference_meta: true,
+  async saveVirtualReference({ appFolder, schemaName, pureName, refSchemaName, refTableName, columns }) {
+    await this.saveConfigFile(
+      appFolder,
+      'virtual-references.config.json',
+      columns.length == 1
+        ? x =>
+            !(
+              x.schemaName == schemaName &&
+              x.pureName == pureName &&
+              x.columns.length == 1 &&
+              x.columns[0].columnName == columns[0].columnName
+            )
+        : null,
+      {
+        schemaName,
+        pureName,
+        refSchemaName,
+        refTableName,
+        columns,
+      }
+    );
+    return true;
+  },
+
+  saveDictionaryDescription_meta: true,
+  async saveDictionaryDescription({ appFolder, pureName, schemaName, expression, columns, delimiter }) {
+    await this.saveConfigFile(
+      appFolder,
+      'dictionary-descriptions.config.json',
+      x => !(x.schemaName == schemaName && x.pureName == pureName),
+      {
+        schemaName,
+        pureName,
+        expression,
+        columns,
+        delimiter,
+      }
+    );
+
+    return true;
+  },
+};

packages/api/src/controllers/archive.js

@@ -141,6 +141,13 @@ module.exports = {
     });
   },
 
+  saveText_meta: true,
+  async saveText({ folder, file, text }) {
+    await fs.writeFile(path.join(resolveArchiveFolder(folder), `${file}.jsonl`), text);
+    socket.emitChanged(`archive-files-changed-${folder}`);
+    return true;
+  },
+
   async getNewArchiveFolder({ database }) {
     const isLink = database.endsWith(database);
     const name = isLink ? database.slice(0, -5) : database;

packages/api/src/controllers/config.js

@@ -1,25 +1,29 @@
 const fs = require('fs-extra');
+const os = require('os');
 const path = require('path');
 const axios = require('axios');
 const { datadir } = require('../utility/directories');
 const hasPermission = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const _ = require('lodash');
+const AsyncLock = require('async-lock');
 
 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
 const connections = require('../controllers/connections');
 
-module.exports = {
-  settingsValue: {},
+const lock = new AsyncLock();
 
-  async _init() {
-    try {
-      this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
-    } catch (err) {
-      this.settingsValue = {};
-    }
-  },
+module.exports = {
+  // settingsValue: {},
+
+  // async _init() {
+  //   try {
+  //     this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
+  //   } catch (err) {
+  //     this.settingsValue = {};
+  //   }
+  // },
 
   get_meta: true,
   async get() {
@@ -40,24 +44,45 @@ module.exports = {
 
   getSettings_meta: true,
   async getSettings() {
-    return this.settingsValue;
+    try {
+      return this.fillMissingSettings(
+        JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }))
+      );
+    } catch (err) {
+      return this.fillMissingSettings({});
+    }
+  },
+
+  fillMissingSettings(value) {
+    const res = {
+      ...value,
+    };
+    if (value['app.useNativeMenu'] !== true && value['app.useNativeMenu'] !== false) {
+      res['app.useNativeMenu'] = os.platform() == 'darwin' ? true : false;
+    }
+    return res;
   },
 
   updateSettings_meta: true,
   async updateSettings(values) {
     if (!hasPermission(`settings/change`)) return false;
-    try {
-      const updated = {
-        ...this.settingsValue,
-        ...values,
-      };
-      await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
-      this.settingsValue = updated;
-      socket.emitChanged(`settings-changed`);
-      return updated;
-    } catch (err) {
-      return false;
-    }
+
+    const res = await lock.acquire('update', async () => {
+      const currentValue = await this.getSettings();
+      try {
+        const updated = {
+          ...currentValue,
+          ...values,
+        };
+        await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
+        // this.settingsValue = updated;
+        socket.emitChanged(`settings-changed`);
+        return updated;
+      } catch (err) {
+        return false;
+      }
+    });
+    return res;
   },
 
   changelog_meta: true,

packages/api/src/controllers/connections.js

@@ -1,13 +1,16 @@
 const path = require('path');
 const { fork } = require('child_process');
 const _ = require('lodash');
-const nedb = require('nedb-promises');
 const fs = require('fs-extra');
 
 const { datadir, filesdir } = require('../utility/directories');
 const socket = require('../utility/socket');
 const { encryptConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { pickSafeConnectionInfo } = require('../utility/crypting');
+const JsonLinesDatabase = require('../utility/JsonLinesDatabase');
+
+const processArgs = require('../utility/processArgs');
 
 function getNamedArgs() {
   const res = {};
@@ -37,7 +40,7 @@ function getDatabaseFileLabel(databaseFile) {
 
 function getPortalCollections() {
   if (process.env.CONNECTIONS) {
-    return _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
+    const connections = _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
       _id: id,
       engine: process.env[`ENGINE_${id}`],
       server: process.env[`SERVER_${id}`],
@@ -45,11 +48,42 @@ function getPortalCollections() {
       password: process.env[`PASSWORD_${id}`],
       port: process.env[`PORT_${id}`],
       databaseUrl: process.env[`URL_${id}`],
+      useDatabaseUrl: !!process.env[`URL_${id}`],
       databaseFile: process.env[`FILE_${id}`],
-      defaultDatabase: process.env[`DATABASE_${id}`],
-      singleDatabase: !!process.env[`DATABASE_${id}`],
+      defaultDatabase:
+        process.env[`DATABASE_${id}`] ||
+        (process.env[`FILE_${id}`] ? getDatabaseFileLabel(process.env[`FILE_${id}`]) : null),
+      singleDatabase: !!process.env[`DATABASE_${id}`] || !!process.env[`FILE_${id}`],
       displayName: process.env[`LABEL_${id}`],
+
+      // SSH tunnel
+      useSshTunnel: process.env[`USE_SSH_${id}`],
+      sshHost: process.env[`SSH_HOST_${id}`],
+      sshPort: process.env[`SSH_PORT_${id}`],
+      sshMode: process.env[`SSH_MODE_${id}`],
+      sshLogin: process.env[`SSH_LOGIN_${id}`],
+      sshPassword: process.env[`SSH_PASSWORD_${id}`],
+      sshKeyfile: process.env[`SSH_KEY_FILE_${id}`],
+      sshKeyfilePassword: process.env[`SSH_KEY_FILE_PASSWORD_${id}`],
+
+      // SSL
+      useSsl: process.env[`USE_SSL_${id}`],
+      sslCaFile: process.env[`SSL_CA_FILE_${id}`],
+      sslCertFile: process.env[`SSL_CERT_FILE_${id}`],
+      sslCertFilePassword: process.env[`SSL_CERT_FILE_PASSWORD_${id}`],
+      sslKeyFile: process.env[`SSL_KEY_FILE_${id}`],
+      sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`],
     }));
+    console.log('Using connections from ENV variables:');
+    console.log(JSON.stringify(connections.map(pickSafeConnectionInfo), undefined, 2));
+    const noengine = connections.filter(x => !x.engine);
+    if (noengine.length > 0) {
+      console.log(
+        'Warning: Invalid CONNECTIONS configutation, missing ENGINE for connection ID:',
+        noengine.map(x => x._id)
+      );
+    }
+    return connections;
   }
 
   const args = getNamedArgs();
@@ -122,7 +156,7 @@ module.exports = {
     const dir = datadir();
     if (!portalConnections) {
       // @ts-ignore
-      this.datastore = nedb.create(path.join(dir, 'connections.jsonl'));
+      this.datastore = new JsonLinesDatabase(path.join(dir, 'connections.jsonl'));
     }
   },
 
@@ -131,25 +165,26 @@ module.exports = {
     return portalConnections || this.datastore.find();
   },
 
-  test_meta: {
-    method: 'post',
-    raw: true,
-  },
-  test(req, res) {
+  test_meta: true,
+  test(connection) {
     const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
       '--start-process',
       'connectProcess',
-      ...process.argv.slice(3),
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
     ]);
-    subprocess.on('message', resp => {
-      if (handleProcessCommunication(resp, subprocess)) return;
-      // @ts-ignore
-      const { msgtype } = resp;
-      if (msgtype == 'connected' || msgtype == 'error') {
-        res.json(resp);
-      }
+    subprocess.send(connection);
+    return new Promise(resolve => {
+      subprocess.on('message', resp => {
+        if (handleProcessCommunication(resp, subprocess)) return;
+        // @ts-ignore
+        const { msgtype } = resp;
+        if (msgtype == 'connected' || msgtype == 'error') {
+          resolve(resp);
+        }
+      });
     });
-    subprocess.send(req.body);
   },
 
   save_meta: true,
@@ -158,18 +193,22 @@ module.exports = {
     let res;
     const encrypted = encryptConnection(connection);
     if (connection._id) {
-      res = await this.datastore.update(_.pick(connection, '_id'), encrypted);
+      res = await this.datastore.update(encrypted);
     } else {
       res = await this.datastore.insert(encrypted);
     }
     socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    // for (const db of connection.databases || []) {
+    //   socket.emitChanged(`db-apps-changed-${connection._id}-${db.name}`);
+    // }
     return res;
   },
 
   update_meta: true,
   async update({ _id, values }) {
     if (portalConnections) return;
-    const res = await this.datastore.update({ _id }, { $set: values });
+    const res = await this.datastore.patch(_id, values);
     socket.emitChanged('connection-list-changed');
     return res;
   },
@@ -177,31 +216,33 @@ module.exports = {
   updateDatabase_meta: true,
   async updateDatabase({ conid, database, values }) {
     if (portalConnections) return;
-    const conn = await this.datastore.find({ _id: conid });
-    let databases = conn[0].databases || [];
+    const conn = await this.datastore.get(conid);
+    let databases = (conn && conn.databases) || [];
     if (databases.find(x => x.name == database)) {
       databases = databases.map(x => (x.name == database ? { ...x, ...values } : x));
     } else {
       databases = [...databases, { name: database, ...values }];
     }
-    const res = await this.datastore.update({ _id: conid }, { $set: { databases } });
+    const res = await this.datastore.patch(conid, { databases });
     socket.emitChanged('connection-list-changed');
+    socket.emitChanged('used-apps-changed');
+    // socket.emitChanged(`db-apps-changed-${conid}-${database}`);
     return res;
   },
 
   delete_meta: true,
   async delete(connection) {
     if (portalConnections) return;
-    const res = await this.datastore.remove(_.pick(connection, '_id'));
+    const res = await this.datastore.remove(connection._id);
     socket.emitChanged('connection-list-changed');
     return res;
   },
 
   get_meta: true,
   async get({ conid }) {
-    if (portalConnections) return portalConnections.find(x => x._id == conid);
-    const res = await this.datastore.find({ _id: conid });
-    return res[0] || null;
+    if (portalConnections) return portalConnections.find(x => x._id == conid) || null;
+    const res = await this.datastore.get(conid);
+    return res || null;
   },
 
   newSqliteDatabase_meta: true,

packages/api/src/controllers/databaseConnections.js

@@ -25,6 +25,7 @@ const requireEngineDriver = require('../utility/requireEngineDriver');
 const generateDeploySql = require('../shell/generateDeploySql');
 const { createTwoFilesPatch } = require('diff');
 const diff2htmlPage = require('../utility/diff2htmlPage');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -61,9 +62,10 @@ module.exports = {
     delete this.requests[msgid];
   },
   handle_status(conid, database, { status }) {
+    // console.log('HANDLE SET STATUS', status);
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
-    if (existing.status == status) return;
+    if (existing.status && status && existing.status.counter > status.counter) return;
     existing.status = status;
     socket.emitChanged(`database-status-changed-${conid}-${database}`);
   },
@@ -75,9 +77,11 @@ module.exports = {
     if (existing) return existing;
     const connection = await connections.get({ conid });
     const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
       '--start-process',
       'databaseConnectionProcess',
-      ...process.argv.slice(3),
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
     ]);
     const lastClosed = this.closed[`${conid}/${database}`];
     const newOpened = {
@@ -106,7 +110,7 @@ module.exports = {
       msgtype: 'connect',
       connection: { ...connection, database },
       structure: lastClosed ? lastClosed.structure : null,
-      globalSettings: config.settingsValue,
+      globalSettings: await config.getSettings(),
     });
     return newOpened;
   },
@@ -249,6 +253,7 @@ module.exports = {
 
   serverVersion_meta: true,
   async serverVersion({ conid, database }) {
+    if (!conid) return null;
     const opened = await this.ensureOpened(conid, database);
     return opened.serverVersion || null;
   },

packages/api/src/controllers/files.js

@@ -1,11 +1,13 @@
 const uuidv1 = require('uuid/v1');
 const fs = require('fs-extra');
 const path = require('path');
-const { filesdir, archivedir, resolveArchiveFolder, uploadsdir } = require('../utility/directories');
+const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir } = require('../utility/directories');
 const getChartExport = require('../utility/getChartExport');
 const hasPermission = require('../utility/hasPermission');
 const socket = require('../utility/socket');
 const scheduler = require('./scheduler');
+const getDiagramExport = require('../utility/getDiagramExport');
+const apps = require('./apps');
 
 function serialize(format, data) {
   if (format == 'text') return data;
@@ -73,6 +75,11 @@ module.exports = {
         encoding: 'utf-8',
       });
       return deserialize(format, text);
+    } else if (folder.startsWith('app:')) {
+      const text = await fs.readFile(path.join(appdir(), folder.substring('app:'.length), file), {
+        encoding: 'utf-8',
+      });
+      return deserialize(format, text);
     } else {
       if (!hasPermission(`files/${folder}/read`)) return null;
       const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' });
@@ -86,8 +93,15 @@ module.exports = {
       const dir = resolveArchiveFolder(folder.substring('archive:'.length));
       await fs.writeFile(path.join(dir, file), serialize(format, data));
       socket.emitChanged(`archive-files-changed-${folder.substring('archive:'.length)}`);
+      return true;
+    } else if (folder.startsWith('app:')) {
+      const app = folder.substring('app:'.length);
+      await fs.writeFile(path.join(appdir(), app, file), serialize(format, data));
+      socket.emitChanged(`app-files-changed-${app}`);
+      apps.emitChangedDbApp(folder);
+      return true;
     } else {
-      if (!hasPermission(`files/${folder}/write`)) return;
+      if (!hasPermission(`files/${folder}/write`)) return false;
       const dir = path.join(filesdir(), folder);
       if (!(await fs.exists(dir))) {
         await fs.mkdir(dir);
@@ -98,6 +112,7 @@ module.exports = {
       if (folder == 'shell') {
         scheduler.reload();
       }
+      return true;
     }
   },
 
@@ -126,8 +141,8 @@ module.exports = {
   },
 
   generateUploadsFile_meta: true,
-  async generateUploadsFile() {
-    const fileName = `${uuidv1()}.html`;
+  async generateUploadsFile({ extension }) {
+    const fileName = `${uuidv1()}.${extension || 'html'}`;
     return {
       fileName,
       filePath: path.join(uploadsdir(), fileName),
@@ -150,4 +165,10 @@ module.exports = {
     }
     return true;
   },
+
+  exportDiagram_meta: true,
+  async exportDiagram({ filePath, html, css, themeType, themeClassName }) {
+    await fs.writeFile(filePath, getDiagramExport(html, css, themeType, themeClassName));
+    return true;
+  },
 };

packages/api/src/controllers/jsldata.js

@@ -1,6 +1,7 @@
 const fs = require('fs');
 const lineReader = require('line-reader');
 const _ = require('lodash');
+const { __ } = require('lodash/fp');
 const DatastoreProxy = require('../utility/DatastoreProxy');
 const { saveFreeTableData } = require('../utility/freeTableStorage');
 const getJslFileName = require('../utility/getJslFileName');
@@ -10,7 +11,10 @@ const socket = require('../utility/socket');
 function readFirstLine(file) {
   return new Promise((resolve, reject) => {
     lineReader.open(file, (err, reader) => {
-      if (err) reject(err);
+      if (err) {
+        reject(err);
+        return;
+      }
       if (reader.hasNextLine()) {
         reader.nextLine((err, line) => {
           if (err) reject(err);
@@ -108,7 +112,16 @@ module.exports = {
   async getInfo({ jslid }) {
     const file = getJslFileName(jslid);
     const firstLine = await readFirstLine(file);
-    if (firstLine) return JSON.parse(firstLine);
+    if (firstLine) {
+      const parsed = JSON.parse(firstLine);
+      if (parsed.__isStreamHeader) {
+        return parsed;
+      }
+      return {
+        __isStreamHeader: true,
+        __isDynamicStructure: true,
+      };
+    }
     return null;
   },
 
@@ -132,7 +145,7 @@ module.exports = {
   },
 
   async notifyChangedStats(stats) {
-    console.log('SENDING STATS', JSON.stringify(stats));
+    // console.log('SENDING STATS', JSON.stringify(stats));
     const datastore = this.datastores[stats.jslid];
     if (datastore) await datastore.notifyChanged();
     socket.emit(`jsldata-stats-${stats.jslid}`, stats);
@@ -151,4 +164,10 @@ module.exports = {
     saveFreeTableData(getJslFileName(jslid), data);
     return true;
   },
+
+  saveText_meta: true,
+  async saveText({ jslid, text }) {
+    await fs.promises.writeFile(getJslFileName(jslid), text);
+    return true;
+  },
 };

packages/api/src/controllers/plugins.js

@@ -72,7 +72,8 @@ module.exports = {
 
     const res = [];
     for (const packageName of _.union(files1, files2)) {
-      if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
+      if (packageName == 'dist') continue;
+      // if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
       try {
         if (packagedContent && packagedContent[packageName]) {
           const manifest = {

packages/api/src/controllers/queryHistory.js

@@ -48,7 +48,7 @@ module.exports = {
   async write({ data }) {
     const fileName = path.join(datadir(), 'query-history.jsonl');
     await fs.appendFile(fileName, JSON.stringify(data) + '\n');
-    socket.emitChanged('query-history-changed');
+    socket.emit('query-history-changed');
     return 'OK';
   },
 };

packages/api/src/controllers/runners.js

@@ -8,6 +8,7 @@ const { fork } = require('child_process');
 const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
 const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -98,15 +99,22 @@ module.exports = {
     const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
     console.log(`RUNNING SCRIPT ${scriptFile}`);
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
-    const subprocess = fork(scriptFile, ['--checkParent', ...process.argv.slice(3)], {
-      cwd: directory,
-      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
-      env: {
-        ...process.env,
-        DBGATE_API: global['API_PACKAGE'] || global['dbgateApiModulePath'] || process.argv[1],
-        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
-      },
-    });
+    const subprocess = fork(
+      scriptFile,
+      [
+        '--checkParent', // ...process.argv.slice(3)
+        ...processArgs.getPassArgs(),
+      ],
+      {
+        cwd: directory,
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        env: {
+          ...process.env,
+          DBGATE_API: global['API_PACKAGE'] || global['dbgateApiModulePath'] || process.argv[1],
+          ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
+        },
+      }
+    );
     const pipeDispatcher = severity => data =>
       this.dispatchMessage(runid, { severity, message: data.toString().trim() });
 
@@ -136,7 +144,7 @@ module.exports = {
       if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](runid, message);
     });
-    return newOpened;
+    return _.pick(newOpened, ['runid']);
   },
 
   start_meta: true,

packages/api/src/controllers/serverConnections.js

@@ -6,6 +6,7 @@ const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
 const lock = new AsyncLock();
 const config = require('./config');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   opened: [],
@@ -38,9 +39,11 @@ module.exports = {
       if (existing) return existing;
       const connection = await connections.get({ conid });
       const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+        '--is-forked-api',
         '--start-process',
         'serverConnectionProcess',
-        ...process.argv.slice(3),
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
       ]);
       const newOpened = {
         conid,
@@ -66,7 +69,7 @@ module.exports = {
         if (newOpened.disconnected) return;
         this.close(conid, false);
       });
-      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: config.settingsValue });
+      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: await config.getSettings() });
       return newOpened;
     });
     return res;

packages/api/src/controllers/sessions.js

@@ -5,6 +5,7 @@ const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
 const { handleProcessCommunication } = require('../utility/processComm');
+const processArgs = require('../utility/processArgs');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -66,9 +67,11 @@ module.exports = {
     const sesid = uuidv1();
     const connection = await connections.get({ conid });
     const subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+      '--is-forked-api',
       '--start-process',
       'sessionProcess',
-      ...process.argv.slice(3),
+      ...processArgs.getPassArgs(),
+      // ...process.argv.slice(3),
     ]);
     const newOpened = {
       conid,
@@ -85,7 +88,7 @@ module.exports = {
       this[`handle_${msgtype}`](sesid, message);
     });
     subprocess.send({ msgtype: 'connect', ...connection, database });
-    return newOpened;
+    return _.pick(newOpened, ['conid', 'database', 'sesid']);
   },
 
   executeQuery_meta: true,

packages/api/src/index.js

@@ -8,7 +8,7 @@ if (processArgs.startProcess) {
   const proc = require('./proc');
   const module = proc[processArgs.startProcess];
   module.start();
-} else if (!module['parent'] && !processArgs.checkParent) {
+} else if (!processArgs.checkParent && !global['API_PACKAGE'] && !global['dbgateApiModulePath']) {
   const main = require('./main');
 
   main.start();

packages/api/src/main.js

@@ -4,11 +4,8 @@ const bodyParser = require('body-parser');
 const fileUpload = require('express-fileupload');
 const http = require('http');
 const cors = require('cors');
-const fs = require('fs');
 const getPort = require('get-port');
-const childProcessChecker = require('./utility/childProcessChecker');
 const path = require('path');
-const crypto = require('crypto');
 
 const useController = require('./utility/useController');
 const socket = require('./utility/socket');
@@ -22,6 +19,7 @@ const runners = require('./controllers/runners');
 const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
 const archive = require('./controllers/archive');
+const apps = require('./controllers/apps');
 const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
@@ -30,11 +28,7 @@ const queryHistory = require('./controllers/queryHistory');
 
 const { rundir } = require('./utility/directories');
 const platformInfo = require('./utility/platformInfo');
-const processArgs = require('./utility/processArgs');
-const timingSafeCheckToken = require('./utility/timingSafeCheckToken');
-
-let authorization = null;
-let checkLocalhostOrigin = null;
+const getExpressPath = require('./utility/getExpressPath');
 
 function start() {
   // console.log('process.argv', process.argv);
@@ -55,35 +49,13 @@ function start() {
     );
   }
 
-  app.use(function (req, res, next) {
-    if (authorization && !timingSafeCheckToken(req.headers.authorization, authorization)) {
-      return res.status(403).json({ error: 'Not authorized!' });
-    }
-    if (checkLocalhostOrigin) {
-      if (
-        req.headers.origin &&
-        req.headers.origin != checkLocalhostOrigin &&
-        req.headers.origin != `http://${checkLocalhostOrigin}`
-      ) {
-        console.log('API origin check FAILED');
-        console.log('HEADERS', { ...req.headers, authorization: '***' });
-        return res.status(403).json({ error: 'Not authorized!' });
-      }
-      if (!req.headers.origin && req.headers.host != checkLocalhostOrigin) {
-        console.log('API host check FAILED');
-        console.log('HEADERS', { ...req.headers, authorization: '***' });
-        return res.status(403).json({ error: 'Not authorized!' });
-      }
-    }
-    next();
-  });
-
   app.use(cors());
 
-  app.get('/stream', async function (req, res) {
+  app.get(getExpressPath('/stream'), async function (req, res) {
     res.set({
       'Cache-Control': 'no-cache',
       'Content-Type': 'text/event-stream',
+      'X-Accel-Buffering': 'no',
       Connection: 'keep-alive',
     });
     res.flushHeaders();
@@ -96,7 +68,7 @@ function start() {
   app.use(bodyParser.json({ limit: '50mb' }));
 
   app.use(
-    '/uploads',
+    getExpressPath('/uploads'),
     fileUpload({
       limits: { fileSize: 4 * 1024 * 1024 },
     })
@@ -108,29 +80,47 @@ function start() {
   //   app.use('/pages', express.static(process.env.PAGES_DIRECTORY));
   // }
 
-  app.use('/runners/data', express.static(rundir()));
+  app.use(getExpressPath('/runners/data'), express.static(rundir()));
 
   if (platformInfo.isDocker) {
     // server static files inside docker container
-    app.use(express.static('/home/dbgate-docker/public'));
+    app.use(getExpressPath('/'), express.static('/home/dbgate-docker/public'));
   } else {
     if (!platformInfo.isNpmDist) {
-      app.get('/', (req, res) => {
+      app.get(getExpressPath('/'), (req, res) => {
         res.send('DbGate API');
       });
     }
   }
 
-if (platformInfo.isNpmDist) {
-    app.use(express.static(path.join(__dirname, '../../dbgate-web/public')));
+  if (platformInfo.isNpmDist) {
+    app.use(getExpressPath('/'), express.static(path.join(__dirname, '../../dbgate-web/public')));
     getPort({ port: 5000 }).then(port => {
       server.listen(port, () => {
         console.log(`DbGate API listening on port ${port}`);
       });
     });
   } else {
-    server.listen(process.env.PORT || 3000);
+    const port = process.env.PORT || 3000;
+    console.log('DbGate API listening on port', port);
+    server.listen(port);
   }
+
+  function shutdown() {
+    console.log('\nShutting down DbGate API server');
+    server.close(() => {
+      console.log('Server shut down, terminating');
+      process.exit(0);
+    });
+    setTimeout(() => {
+      console.log('Server close timeout, terminating');
+      process.exit(0);
+    }, 1000);
+  }
+
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+  process.on('SIGBREAK', shutdown);
 }
 
 function useAllControllers(app, electron) {
@@ -148,10 +138,11 @@ function useAllControllers(app, electron) {
   useController(app, electron, '/files', files);
   useController(app, electron, '/scheduler', scheduler);
   useController(app, electron, '/query-history', queryHistory);
+  useController(app, electron, '/apps', apps);
 }
 
 function initializeElectronSender(electronSender) {
   socket.setElectronSender(electronSender);
 }
 
-module.exports = { start, useAllControllers, initializeElectronSender };
+module.exports = { start, useAllControllers, initializeElectronSender, configController: config };

packages/api/src/nativeModules.js

@@ -1,5 +1,13 @@
 const argIndex = process.argv.indexOf('--native-modules');
-const redirectFile = argIndex > 0 ? process.argv[argIndex + 1] : null;
+const redirectFile = global['NATIVE_MODULES'] || (argIndex > 0 ? process.argv[argIndex + 1] : null);
 
-// @ts-ignore
-module.exports = redirectFile ? __non_webpack_require__(redirectFile) : require('./nativeModulesContent');
+function requireDynamic(file) {
+  try {
+    // @ts-ignore
+    return __non_webpack_require__(redirectFile);
+  } catch (err) {
+    return require(redirectFile);
+  }
+}
+
+module.exports = redirectFile ? requireDynamic(redirectFile) : require('./nativeModulesContent');

packages/api/src/proc/connectProcess.js

@@ -2,17 +2,9 @@ const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const { pickSafeConnectionInfo } = require('../utility/crypting');
 const _ = require('lodash');
 
-function pickSafeConnectionInfo(connection) {
-  return _.mapValues(connection, (v, k) => {
-    if (k == 'engine' || k == 'port' || k == 'authType' || k == 'sshMode' || k == 'passwordMode') return v;
-    if (v === null || v === true || v === false) return v;
-    if (v) return '***';
-    return undefined;
-  });
-}
-
 const formatErrorDetail = (e, connection) => `${e.stack}
 
 Error JSON: ${JSON.stringify(e, undefined, 2)}

packages/api/src/proc/databaseConnectionProcess.js

@@ -18,6 +18,12 @@ let lastStatus = null;
 let analysedTime = 0;
 let serverVersion;
 
+let statusCounter = 0;
+function getStatusCounter() {
+  statusCounter += 1;
+  return statusCounter;
+}
+
 async function checkedAsyncCall(promise) {
   try {
     const res = await promise;
@@ -79,7 +85,7 @@ function handleSyncModel() {
 function setStatus(status) {
   const statusString = stableStringify(status);
   if (lastStatus != statusString) {
-    process.send({ msgtype: 'status', status });
+    process.send({ msgtype: 'status', status: { ...status, counter: getStatusCounter() } });
     lastStatus = statusString;
   }
 }
@@ -270,6 +276,7 @@ function start() {
     try {
       await handleMessage(message);
     } catch (e) {
+      console.error('Error in DB connection', e);
       process.send({ msgtype: 'error', error: e.message });
     }
   });

packages/api/src/shell/copyStream.js

@@ -2,12 +2,17 @@ const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
 
 function copyStream(input, output) {
   return new Promise((resolve, reject) => {
-    const ensureHeader = new EnsureStreamHeaderStream();
     const finisher = output['finisher'] || output;
     finisher.on('finish', resolve);
     finisher.on('error', reject);
-    input.pipe(ensureHeader);
-    ensureHeader.pipe(output);
+
+    if (output.requireFixedStructure) {
+      const ensureHeader = new EnsureStreamHeaderStream();
+      input.pipe(ensureHeader);
+      ensureHeader.pipe(output);
+    } else {
+      input.pipe(output);
+    }
   });
 }
 

packages/api/src/shell/jsonArrayWriter.js

@@ -11,10 +11,7 @@ class StringifyStream extends stream.Transform {
     let skip = false;
 
     if (!this.wasHeader) {
-      skip =
-        chunk.__isStreamHeader ||
-        // TODO remove isArray test
-        Array.isArray(chunk.columns);
+      skip = chunk.__isStreamHeader;
       this.wasHeader = true;
     }
     if (!skip) {

packages/api/src/shell/jsonLinesReader.js

@@ -12,14 +12,14 @@ class ParseStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     const obj = JSON.parse(chunk);
     if (!this.wasHeader) {
-      if (
-        !obj.__isStreamHeader &&
-        // TODO remove isArray test
-        !Array.isArray(obj.columns)
-      ) {
-        this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
+      if (!obj.__isStreamHeader) {
+        this.push({
+          __isStreamHeader: true,
+          __isDynamicStructure: true,
+          // columns: Object.keys(obj).map(columnName => ({ columnName })),
+        });
       }
-      
+
       this.wasHeader = true;
     }
     if (!this.limitRows || this.rowsWritten < this.limitRows) {

packages/api/src/shell/jsonLinesWriter.js

@@ -10,11 +10,7 @@ class StringifyStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     let skip = false;
     if (!this.wasHeader) {
-      skip =
-        (chunk.__isStreamHeader ||
-          // TODO remove isArray test
-          Array.isArray(chunk.columns)) &&
-        !this.header;
+      skip = (chunk.__isStreamHeader && !this.header) || (chunk.__isStreamHeader && chunk.__isDynamicStructure);
       this.wasHeader = true;
     }
     if (!skip) {

packages/api/src/shell/sqlDataWriter.js

@@ -14,11 +14,7 @@ class SqlizeStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     let skip = false;
     if (!this.wasHeader) {
-      if (
-        chunk.__isStreamHeader ||
-        // TODO remove isArray test
-        Array.isArray(chunk.columns)
-      ) {
+      if (chunk.__isStreamHeader) {
         skip = true;
         this.tableName = chunk.pureName;
         if (chunk.engine) {

packages/api/src/utility/DatastoreProxy.js

@@ -1,6 +1,7 @@
 const { fork } = require('child_process');
 const uuidv1 = require('uuid/v1');
 const { handleProcessCommunication } = require('./processComm');
+const processArgs = require('../utility/processArgs');
 
 class DatastoreProxy {
   constructor(file) {
@@ -30,9 +31,11 @@ class DatastoreProxy {
   async ensureSubprocess() {
     if (!this.subprocess) {
       this.subprocess = fork(global['API_PACKAGE'] || process.argv[1], [
+        '--is-forked-api',
         '--start-process',
         'jslDatastoreProcess',
-        ...process.argv.slice(3),
+        ...processArgs.getPassArgs(),
+        // ...process.argv.slice(3),
       ]);
 
       this.subprocess.on('message', message => {

packages/api/src/utility/EnsureStreamHeaderStream.js

@@ -13,11 +13,7 @@ class EnsureStreamHeaderStream extends stream.Transform {
         return;
       }
 
-      if (
-        !chunk.__isStreamHeader &&
-        // TODO remove isArray test
-        !Array.isArray(chunk.columns)
-      ) {
+      if (!chunk.__isStreamHeader) {
         this.push({
           __isStreamHeader: true,
           __isComputedStructure: true,

packages/api/src/utility/JsonLinesDatabase.js

@@ -0,0 +1,142 @@
+const AsyncLock = require('async-lock');
+const fs = require('fs-extra');
+const uuidv1 = require('uuid/v1');
+
+const lock = new AsyncLock();
+
+// const lineReader = require('line-reader');
+// const { fetchNextLineFromReader } = require('./JsonLinesDatastore');
+
+class JsonLinesDatabase {
+  constructor(filename) {
+    this.filename = filename;
+    this.data = [];
+    this.loadedOk = false;
+    this.loadPerformed = false;
+  }
+
+  async _save() {
+    if (!this.loadedOk) {
+      // don't override data
+      return;
+    }
+    await fs.writeFile(this.filename, this.data.map(x => JSON.stringify(x)).join('\n'));
+  }
+
+  async _ensureLoaded() {
+    if (!this.loadPerformed) {
+      await lock.acquire('reader', async () => {
+        if (!this.loadPerformed) {
+          if (!(await fs.exists(this.filename))) {
+            this.loadedOk = true;
+            this.loadPerformed = true;
+            return;
+          }
+          try {
+            const text = await fs.readFile(this.filename, { encoding: 'utf-8' });
+            this.data = text
+              .split('\n')
+              .filter(x => x.trim())
+              .map(x => JSON.parse(x));
+            this.loadedOk = true;
+          } catch (err) {
+            console.error(`Error loading file ${this.filename}`, err);
+          }
+          this.loadPerformed = true;
+        }
+      });
+    }
+  }
+
+  async insert(obj) {
+    await this._ensureLoaded();
+    if (obj._id && (await this.get(obj._id))) {
+      throw new Error(`Cannot insert duplicate ID ${obj._id} into ${this.filename}`);
+    }
+    const elem = obj._id
+      ? obj
+      : {
+          ...obj,
+          _id: uuidv1(),
+        };
+    this.data.push(elem);
+    await this._save();
+    return elem;
+  }
+
+  async get(id) {
+    await this._ensureLoaded();
+    return this.data.find(x => x._id == id);
+  }
+
+  async find(cond) {
+    await this._ensureLoaded();
+    if (cond) {
+      return this.data.filter(x => {
+        for (const key of Object.keys(cond)) {
+          if (x[key] != cond[key]) return false;
+        }
+        return true;
+      });
+    } else {
+      return this.data;
+    }
+  }
+
+  async update(obj) {
+    await this._ensureLoaded();
+    this.data = this.data.map(x => (x._id == obj._id ? obj : x));
+    await this._save();
+    return obj;
+  }
+
+  async patch(id, values) {
+    await this._ensureLoaded();
+    this.data = this.data.map(x => (x._id == id ? { ...x, ...values } : x));
+    await this._save();
+    return this.data.find(x => x._id == id);
+  }
+
+  async remove(id) {
+    await this._ensureLoaded();
+    const removed = this.data.find(x => x._id == id);
+    this.data = this.data.filter(x => x._id != id);
+    await this._save();
+    return removed;
+  }
+
+  //   async _openReader() {
+  //     return new Promise((resolve, reject) =>
+  //       lineReader.open(this.filename, (err, reader) => {
+  //         if (err) reject(err);
+  //         resolve(reader);
+  //       })
+  //     );
+  //   }
+
+  //   async _read() {
+  //     this.data = [];
+  //     if (!(await fs.exists(this.filename))) return;
+  //     try {
+  //       const reader = await this._openReader();
+  //       for (;;) {
+  //         const line = await fetchNextLineFromReader(reader);
+  //         if (!line) break;
+  //         this.data.push(JSON.parse(line));
+  //       }
+  //     } catch (err) {
+  //       console.error(`Error loading file ${this.filename}`, err);
+  //     }
+  //   }
+
+  //   async _write() {
+  //     const fw = fs.createWriteStream(this.filename);
+  //     for (const obj of this.data) {
+  //       await fw.write(JSON.stringify(obj));
+  //       await fw.write('\n');
+  //     }
+  //     await fw.end();
+  //   }
+}
+
+module.exports = JsonLinesDatabase;

packages/api/src/utility/JsonLinesDatastore.js

@@ -4,7 +4,7 @@ const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
 
-async function fetchNextLine(reader) {
+function fetchNextLineFromReader(reader) {
   return new Promise((resolve, reject) => {
     if (!reader.hasNextLine()) {
       resolve(null);
@@ -27,6 +27,7 @@ class JsonLinesDatastore {
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
     this.notifyChangedCallback = null;
     this.currentFilter = null;
   }
@@ -37,6 +38,7 @@ class JsonLinesDatastore {
     this.reader = null;
     this.readedDataRowCount = 0;
     this.readedSchemaRow = false;
+    // this.firstRowToBeReturned = null;
     this.currentFilter = null;
     reader.close(() => {});
   }
@@ -61,8 +63,13 @@ class JsonLinesDatastore {
   }
 
   async _readLine(parse) {
+    // if (this.firstRowToBeReturned) {
+    //   const res = this.firstRowToBeReturned;
+    //   this.firstRowToBeReturned = null;
+    //   return res;
+    // }
     for (;;) {
-      const line = await fetchNextLine(this.reader);
+      const line = await fetchNextLineFromReader(this.reader);
       if (!line) {
         // EOF
         return null;
@@ -70,7 +77,11 @@ class JsonLinesDatastore {
 
       if (!this.readedSchemaRow) {
         this.readedSchemaRow = true;
-        return true;
+        const parsedLine = JSON.parse(line);
+        if (parsedLine.__isStreamHeader) {
+          // skip to next line
+          continue;
+        }
       }
       if (this.currentFilter) {
         const parsedLine = JSON.parse(line);
@@ -130,11 +141,21 @@ class JsonLinesDatastore {
       this.reader = reader;
       this.currentFilter = filter;
     }
-    if (!this.readedSchemaRow) {
-      await this._readLine(false); // skip structure
-    }
+    // if (!this.readedSchemaRow) {
+    //   const line = await this._readLine(true); // skip structure
+    //   if (!line.__isStreamHeader) {
+    //     // line contains data
+    //     this.firstRowToBeReturned = line;
+    //   }
+    // }
     while (this.readedDataRowCount < offset) {
-      await this._readLine(false);
+      const line = await this._readLine(false);
+      if (line == null) break;
+      // if (this.firstRowToBeReturned) {
+      //   this.firstRowToBeReturned = null;
+      // } else {
+      //   await this._readLine(false);
+      // }
     }
   }
 
@@ -148,7 +169,6 @@ class JsonLinesDatastore {
         res.push(line);
       }
     });
-    // console.log('RETURN', res.length);
     return res;
   }
 }

packages/api/src/utility/crypting.js

@@ -2,6 +2,7 @@ const crypto = require('crypto');
 const simpleEncryptor = require('simple-encryptor');
 const fs = require('fs');
 const path = require('path');
+const _ = require('lodash');
 
 const { datadir } = require('./directories');
 
@@ -81,8 +82,18 @@ function decryptConnection(connection) {
   return connection;
 }
 
+function pickSafeConnectionInfo(connection) {
+  return _.mapValues(connection, (v, k) => {
+    if (k == 'engine' || k == 'port' || k == 'authType' || k == 'sshMode' || k == 'passwordMode') return v;
+    if (v === null || v === true || v === false) return v;
+    if (v) return '***';
+    return undefined;
+  });
+}
+
 module.exports = {
   loadEncryptionKey,
   encryptConnection,
   decryptConnection,
+  pickSafeConnectionInfo,
 };

packages/api/src/utility/directories.js

@@ -38,9 +38,14 @@ const rundir = dirFunc('run', true);
 const uploadsdir = dirFunc('uploads', true);
 const pluginsdir = dirFunc('plugins');
 const archivedir = dirFunc('archive');
+const appdir = dirFunc('apps');
 const filesdir = dirFunc('files');
 
 function packagedPluginsDir() {
+  // console.log('CALL DIR FROM', new Error('xxx').stack);
+  // console.log('__dirname', __dirname);
+  // console.log('platformInfo.isElectronBundle', platformInfo.isElectronBundle);
+  // console.log('platformInfo.isForkedApi', platformInfo.isForkedApi);
   if (platformInfo.isDevMode) {
     return path.resolve(__dirname, '../../../../plugins');
   }
@@ -53,6 +58,12 @@ function packagedPluginsDir() {
   }
   if (platformInfo.isElectronBundle) {
     return path.resolve(__dirname, '../../plugins');
+
+    // if (platformInfo.isForkedApi) {
+    //   return path.resolve(__dirname, '../plugins');
+    // } else {
+    //   return path.resolve(__dirname, '../../plugins');
+    // }
   }
   return null;
 }
@@ -93,6 +104,7 @@ module.exports = {
   rundir,
   uploadsdir,
   archivedir,
+  appdir,
   ensureDirectory,
   pluginsdir,
   filesdir,

packages/api/src/utility/getChartExport.js

@@ -7,6 +7,9 @@ const getChartExport = (title, config, imageFile) => {
     <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/3.6.0/chart.min.js" integrity="sha512-GMGzUEevhWh8Tc/njS0bDpwgxdCJLQBWG3Z2Ct+JGOpVnEmjvNx6ts4v6A2XJf1HOrtOsfhv3hBKpK9kE5z8AQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.1/moment.min.js" integrity="sha512-qTXRIMyZIFb8iQcfjXWCO8+M5Tbc38Qi5WzdPOYZHIlZpzBHG3L3by84BBBOiRGiEb7KKtAOAs5qYdUiZiQNNQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-adapter-moment/1.0.0/chartjs-adapter-moment.min.js" integrity="sha512-oh5t+CdSBsaVVAvxcZKy3XJdP7ZbYUBSRCXDTVn0ODewMDDNnELsrG9eDm8rVZAQg7RsDD/8K3MjPAFB13o6eA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/hammer.js/2.0.8/hammer.min.js" integrity="sha512-UXumZrZNiOwnTcZSHLOfcTs0aos2MzBWHXOHOuB0J/R44QB0dwY5JgfbvljXcklVf65Gc4El6RjZ+lnwd2az2g==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-plugin-zoom/1.2.0/chartjs-plugin-zoom.min.js" integrity="sha512-TT0wAMqqtjXVzpc48sI0G84rBP+oTkBZPgeRYIOVRGUdwJsyS3WPipsNh///ay2LJ+onCM23tipnz6EvEy2/UA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
+
     <style>
         a { text-decoration: none }
 

packages/api/src/utility/getDiagramExport.js

@@ -0,0 +1,25 @@
+const getDiagramExport = (html, css, themeType, themeClassName) => {
+  return `<html>
+  <meta charset='utf-8'>
+  
+  <head>
+      <style>
+        ${css}
+
+        body {
+          background: var(--theme-bg-1);
+          color: var(--theme-font-1);
+        }
+      </style>
+
+      <link rel="stylesheet" href='https://cdn.jsdelivr.net/npm/@mdi/font@6.5.95/css/materialdesignicons.css' />
+  </head>
+  
+  <body class='${themeType == 'dark' ? 'theme-type-dark' : 'theme-type-light'} ${themeClassName}'>
+      ${html}
+  </body>
+  
+  </html>`;
+};
+
+module.exports = getDiagramExport;

packages/api/src/utility/getExpressPath.js

@@ -0,0 +1,10 @@
+function getExpressPath(path) {
+  path = path.replace(/\/*$/, '').replace(/^\/*/, '');
+  const root = (process.env.WEB_ROOT || '').replace(/^\/*/, '').replace(/\/*$/, '');
+  if (root) {
+    return `/${root}/${path}`;
+  }
+  return `/${path}`;
+}
+
+module.exports = getExpressPath;

packages/api/src/utility/platformInfo.js

@@ -11,6 +11,7 @@ const isLinux = platform === 'linux';
 const isDocker = fs.existsSync('/home/dbgate-docker/public');
 const isDevMode = process.env.DEVMODE == '1';
 const isNpmDist = !!global['dbgateApiModulePath'];
+const isForkedApi = processArgs.isForkedApi;
 
 // function moduleAvailable(name) {
 //   try {
@@ -21,14 +22,13 @@ const isNpmDist = !!global['dbgateApiModulePath'];
 //   }
 // }
 
-const isElectronBundle = processArgs.isElectronBundle;
-
 const platformInfo = {
   isWindows,
   isMac,
   isLinux,
   isDocker,
-  isElectronBundle,
+  isElectronBundle: isElectron() && !isDevMode,
+  isForkedApi,
   isElectron: isElectron(),
   isDevMode,
   isNpmDist,

packages/api/src/utility/processArgs.js

@@ -7,15 +7,17 @@ function getNamedArg(name) {
 }
 
 const checkParent = process.argv.includes('--checkParent');
-const dynport = process.argv.includes('--dynport');
-const nativeModules = getNamedArg('--native-modules');
 const startProcess = getNamedArg('--start-process');
-const isElectronBundle = process.argv.includes('--is-electron-bundle');
+const isForkedApi = process.argv.includes('--is-forked-api');
+
+function getPassArgs() {
+  if (global['NATIVE_MODULES']) return ['--native-modules', global['NATIVE_MODULES']];
+  return [];
+}
 
 module.exports = {
   checkParent,
-  nativeModules,
   startProcess,
-  dynport,
-  isElectronBundle,
+  isForkedApi,
+  getPassArgs,
 };

packages/api/src/utility/requireEngineDriver.js

@@ -1,5 +1,6 @@
 const _ = require('lodash');
 const requirePlugin = require('../shell/requirePlugin');
+const { pickSafeConnectionInfo } = require('./crypting');
 
 /** @returns {import('dbgate-types').EngineDriver} */
 function requireEngineDriver(connection) {
@@ -10,14 +11,14 @@ function requireEngineDriver(connection) {
     engine = connection.engine;
   }
   if (!engine) {
-    throw new Error('Could not get driver from connection');
+    throw new Error(`Could not get driver from connection ${JSON.stringify(pickSafeConnectionInfo(connection))}`);
   }
   if (engine.includes('@')) {
     const [shortName, packageName] = engine.split('@');
     const plugin = requirePlugin(packageName);
     return plugin.drivers.find(x => x.engine == engine);
   }
-  throw new Error(`Could not found engine driver ${engine}`);
+  throw new Error(`Could not find engine driver ${engine}`);
 }
 
 module.exports = requireEngineDriver;

packages/api/src/utility/socket.js

@@ -5,6 +5,7 @@ let init = '';
 module.exports = {
   setSseResponse(value) {
     sseResponse = value;
+    setInterval(() => this.emit('ping'), 29 * 1000);
   },
   setElectronSender(value) {
     electronSender = value;
@@ -23,7 +24,8 @@ module.exports = {
     }
   },
   emitChanged(key) {
-    this.emit('clean-cache', key);
-    this.emit(key);
+    // console.log('EMIT CHANGED', key);
+    this.emit('changed-cache', key);
+    // this.emit(key);
   },
 };

packages/api/src/utility/useController.js

@@ -1,5 +1,6 @@
 const _ = require('lodash');
 const express = require('express');
+const getExpressPath = require('./getExpressPath');
 
 /**
  * @param {string} route
@@ -31,6 +32,7 @@ module.exports = function useController(app, electron, route, controller) {
         // console.log('REGISTERING HANDLER', handler);
         electron.ipcMain.handle(handler, async (event, args) => {
           const data = await controller[key](args);
+          // console.log('HANDLED API', handler, data);
           return data;
         });
       }
@@ -73,6 +75,6 @@ module.exports = function useController(app, electron, route, controller) {
   }
 
   if (app) {
-    app.use(route, router);
+    app.use(getExpressPath(route), router);
   }
 };

packages/api/webpack.config.js

@@ -10,16 +10,16 @@ var config = {
   target: 'node',
   node: {
     __dirname: false,
-  },  
+  },
   output: {
     path: path.resolve(__dirname, 'dist'),
     filename: 'bundle.js',
     libraryTarget: 'commonjs2',
   },
 
-//   optimization: {
-//     minimize: false,
-//   },
+  // optimization: {
+  //   minimize: false,
+  // },
 
   module: {
     rules: [
@@ -45,6 +45,9 @@ var config = {
       },
     }),
   ],
+  externals: {
+    'better-sqlite3': 'commonjs better-sqlite3',
+  },
 };
 
 module.exports = config;

packages/datalib/src/ChangeSet.ts

@@ -337,6 +337,25 @@ export function revertChangeSetRowChanges(changeSet: ChangeSet, definition: Chan
   return changeSet;
 }
 
+function consolidateInsertIndexes(changeSet: ChangeSet, name: NamedObjectInfo): ChangeSet {
+  const indexes = changeSet.inserts
+    .filter(x => x.pureName == name.pureName && x.schemaName == name.schemaName)
+    .map(x => x.insertedRowIndex);
+
+  indexes.sort((a, b) => a - b);
+  if (indexes[indexes.length - 1] != indexes.length - 1) {
+    return {
+      ...changeSet,
+      inserts: changeSet.inserts.map(x => ({
+        ...x,
+        insertedRowIndex: indexes.indexOf(x.insertedRowIndex),
+      })),
+    };
+  }
+
+  return changeSet;
+}
+
 export function deleteChangeSetRows(changeSet: ChangeSet, definition: ChangeSetRowDefinition): ChangeSet {
   let [fieldName, existingItem] = findExistingChangeSetItem(changeSet, definition);
   if (fieldName == 'updates') {
@@ -344,7 +363,7 @@ export function deleteChangeSetRows(changeSet: ChangeSet, definition: ChangeSetR
     [fieldName, existingItem] = findExistingChangeSetItem(changeSet, definition);
   }
   if (fieldName == 'inserts') {
-    return revertChangeSetRowChanges(changeSet, definition);
+    return consolidateInsertIndexes(revertChangeSetRowChanges(changeSet, definition), definition);
   } else {
     if (existingItem && fieldName == 'deletes') return changeSet;
     return {
@@ -390,11 +409,7 @@ export function changeSetInsertNewRow(changeSet: ChangeSet, name?: NamedObjectIn
   };
 }
 
-export function changeSetInsertDocuments(
-  changeSet: ChangeSet,
-  documents: any[],
-  name?: NamedObjectInfo
-): ChangeSet {
+export function changeSetInsertDocuments(changeSet: ChangeSet, documents: any[], name?: NamedObjectInfo): ChangeSet {
   const insertedRows = getChangeSetInsertedRows(changeSet, name);
   return {
     ...changeSet,

packages/datalib/src/GridConfig.ts

@@ -20,7 +20,7 @@ export type GroupFunc = 'GROUP' | 'MAX' | 'MIN' | 'SUM' | 'AVG' | 'COUNT' | 'COU
 
 export interface GridConfig extends GridConfigColumns {
   filters: { [uniqueName: string]: string };
-  focusedColumn?: string;
+  focusedColumns?: string[];
   columnWidths: { [uniqueName: string]: number };
   sort: {
     uniqueName: string;
@@ -48,7 +48,7 @@ export function createGridConfig(): GridConfig {
     filters: {},
     columnWidths: {},
     sort: [],
-    focusedColumn: null,
+    focusedColumns: null,
     grouping: {},
     formFilterColumns: [],
   };

packages/datalib/src/GridDisplay.ts

@@ -28,6 +28,7 @@ export interface DisplayColumn {
   autoIncrement?: boolean;
   isPrimaryKey?: boolean;
   foreignKey?: ForeignKeyInfo;
+  isForeignKeyUnique?: boolean;
   isExpandable?: boolean;
   isChecked?: boolean;
   hintColumnNames?: string[];
@@ -89,6 +90,7 @@ export abstract class GridDisplay {
   isLoadedCorrectly = true;
   supportsReload = false;
   isDynamicStructure = false;
+  filterTypeOverride = null;
 
   setColumnVisibility(uniquePath: string[], isVisible: boolean) {
     const uniqueName = uniquePath.join('.');
@@ -104,10 +106,10 @@ export abstract class GridDisplay {
     this.includeInColumnSet('addedColumns', name, true);
   }
 
-  focusColumn(uniqueName: string) {
+  focusColumns(uniqueNames: string[]) {
     this.setConfig(cfg => ({
       ...cfg,
-      focusedColumn: uniqueName,
+      focusedColumns: uniqueNames,
     }));
   }
 
@@ -115,8 +117,8 @@ export abstract class GridDisplay {
     return false;
   }
 
-  get focusedColumn() {
-    return this.config.focusedColumn;
+  get focusedColumns() {
+    return this.config.focusedColumns;
   }
 
   get engine() {
@@ -127,6 +129,10 @@ export abstract class GridDisplay {
     return this.getColumns(null).filter(col => col.isChecked || col.uniquePath.length == 1);
   }
 
+  findColumn(uniqueName: string) {
+    return this.getColumns(null).find(x => x.uniqueName == uniqueName);
+  }
+
   getFkTarget(column: DisplayColumn): TableInfo {
     return null;
   }
@@ -617,16 +623,16 @@ export abstract class GridDisplay {
     if (!filters) return null;
     const conditions = [];
     for (const name in filters) {
-      const column = this.columns.find(x => (x.columnName = name));
-      if (!column) continue;
-      const filterType = getFilterType(column.dataType);
+      const column = this.isDynamicStructure ? null : this.columns.find(x => x.columnName == name);
+      if (!this.isDynamicStructure && !column) continue;
+      const filterType = this.isDynamicStructure ? this.filterTypeOverride ?? 'mongo' : getFilterType(column.dataType);
       try {
         const condition = parseFilter(filters[name], filterType);
         const replaced = _.cloneDeepWith(condition, (expr: Expression) => {
           if (expr.exprType == 'placeholder')
             return {
               exprType: 'column',
-              columnName: column.columnName,
+              columnName: this.isDynamicStructure ? name : column.columnName,
             };
         });
         conditions.push(replaced);

packages/datalib/src/JslGridDisplay.ts

@@ -12,13 +12,18 @@ export class JslGridDisplay extends GridDisplay {
     setConfig: ChangeConfigFunc,
     cache: GridCache,
     setCache: ChangeCacheFunc,
-    rows: any
+    rows: any,
+    isDynamicStructure: boolean,
+    supportsReload: boolean
   ) {
     super(config, setConfig, cache, setCache, null);
 
     this.filterable = true;
+    this.supportsReload = supportsReload;
+    this.isDynamicStructure = isDynamicStructure;
+    if (isDynamicStructure) this.filterTypeOverride = 'string';
 
-    if (structure.columns) {
+    if (structure?.columns) {
       this.columns = _.uniqBy(
         structure.columns
           .map(col => ({
@@ -39,7 +44,7 @@ export class JslGridDisplay extends GridDisplay {
       );
     }
 
-    if (structure.__isDynamicStructure) {
+    if (structure?.__isDynamicStructure) {
       this.columns = analyseCollectionDisplayColumns(rows, this);
     }
 

packages/datalib/src/TableGridDisplay.ts

@@ -1,5 +1,5 @@
 import _ from 'lodash';
-import { filterName } from 'dbgate-tools';
+import { filterName, isTableColumnUnique } from 'dbgate-tools';
 import { GridDisplay, ChangeCacheFunc, DisplayColumn, DisplayedColumnInfo, ChangeConfigFunc } from './GridDisplay';
 import {
   TableInfo,
@@ -79,10 +79,11 @@ export class TableGridDisplay extends GridDisplay {
           ...col,
           isChecked: this.isColumnChecked(col),
           hintColumnNames:
-            this.getFkDictionaryDescription(col.foreignKey)?.columns?.map(
+            this.getFkDictionaryDescription(col.isForeignKeyUnique ? col.foreignKey : null)?.columns?.map(
               columnName => `hint_${col.uniqueName}_${columnName}`
             ) || null,
-          hintColumnDelimiter: this.getFkDictionaryDescription(col.foreignKey)?.delimiter,
+          hintColumnDelimiter: this.getFkDictionaryDescription(col.isForeignKeyUnique ? col.foreignKey : null)
+            ?.delimiter,
           isExpandable: !!col.foreignKey,
         })) || []
     );
@@ -203,7 +204,8 @@ export class TableGridDisplay extends GridDisplay {
   }
 
   getFkTarget(column: DisplayColumn) {
-    const { uniqueName, foreignKey } = column;
+    const { uniqueName, foreignKey, isForeignKeyUnique } = column;
+    if (!isForeignKeyUnique) return null;
     const pureName = foreignKey.refTableName;
     const schemaName = foreignKey.refSchemaName;
     return this.findTable({ schemaName, pureName });
@@ -230,7 +232,7 @@ export class TableGridDisplay extends GridDisplay {
     const uniquePath = [...parentPath, col.columnName];
     const uniqueName = uniquePath.join('.');
     // console.log('this.config.addedColumns', this.config.addedColumns, uniquePath);
-    return {
+    const res = {
       ...col,
       pureName: table.pureName,
       schemaName: table.schemaName,
@@ -241,7 +243,19 @@ export class TableGridDisplay extends GridDisplay {
       foreignKey:
         table.foreignKeys &&
         table.foreignKeys.find(fk => fk.columns.length == 1 && fk.columns[0].columnName == col.columnName),
+      isForeignKeyUnique: false,
     };
+
+    if (res.foreignKey) {
+      const refTableInfo = this.dbinfo.tables.find(
+        x => x.schemaName == res.foreignKey.refSchemaName && x.pureName == res.foreignKey.refTableName
+      );
+      if (refTableInfo && isTableColumnUnique(refTableInfo, res.foreignKey.columns[0].refColumnName)) {
+        res.isForeignKeyUnique = true;
+      }
+    }
+
+    return res;
   }
 
   addAddedColumnsToSelect(

packages/datalib/src/deleteCascade.ts

@@ -17,7 +17,8 @@ function processDependencies(
   fkPath: ForeignKeyInfo[],
   table: TableInfo,
   baseCmd: ChangeSetItem,
-  dbinfo: DatabaseInfo
+  dbinfo: DatabaseInfo,
+  usedTables: string[]
 ) {
   if (result.find(x => x.title == table.pureName)) return;
 
@@ -28,8 +29,11 @@ function processDependencies(
   for (const fk of dependencies) {
     const depTable = dbinfo.tables.find(x => x.pureName == fk.pureName && x.schemaName == fk.schemaName);
     const subFkPath = [...fkPath, fk];
-    if (depTable && depTable.pureName != baseCmd.pureName) {
-      processDependencies(changeSet, result, allForeignKeys, subFkPath, depTable, baseCmd, dbinfo);
+    if (depTable && !usedTables.includes(depTable.pureName)) {
+      processDependencies(changeSet, result, allForeignKeys, subFkPath, depTable, baseCmd, dbinfo, [
+        ...usedTables,
+        depTable.pureName,
+      ]);
     }
 
     const refCmd: Delete = {
@@ -119,7 +123,7 @@ export function getDeleteCascades(changeSet: ChangeSet, dbinfo: DatabaseInfo): C
     const table = dbinfo.tables.find(x => x.pureName == baseCmd.pureName && x.schemaName == baseCmd.schemaName);
     if (!table.primaryKey) continue;
 
-    processDependencies(changeSet, result, allForeignKeys, [], table, baseCmd, dbinfo);
+    processDependencies(changeSet, result, allForeignKeys, [], table, baseCmd, dbinfo, [table.pureName]);
 
     // let resItem = result.find(x => x.title == baseCmd.pureName);
     // if (!resItem) {

packages/filterparser/src/parseFilter.ts

@@ -223,8 +223,8 @@ const createParser = (filterType: FilterType) => {
     null: () => word('NULL').map(unaryCondition('isNull')),
     empty: () => word('EMPTY').map(unaryCondition('isEmpty')),
     notEmpty: r => r.not.then(r.empty).map(unaryCondition('isNotEmpty')),
-    true: () => word('TRUE').map(binaryFixedValueCondition(1)),
-    false: () => word('FALSE').map(binaryFixedValueCondition(0)),
+    true: () => P.regexp(/true/i).map(binaryFixedValueCondition(1)),
+    false: () => P.regexp(/false/i).map(binaryFixedValueCondition(0)),
     trueNum: () => word('1').map(binaryFixedValueCondition(1)),
     falseNum: () => word('0').map(binaryFixedValueCondition(0)),
 

packages/sqltree/src/evaluateCondition.ts

@@ -28,6 +28,7 @@ export function evaluateCondition(condition: Condition, values) {
         case '=':
           return left == right;
         case '!=':
+        case '<>':
           return left != right;
         case '<=':
           return left <= right;

packages/sqltree/src/types.ts

@@ -58,7 +58,7 @@ export interface UnaryCondition {
 
 export interface BinaryCondition {
   conditionType: 'binary';
-  operator: '=' | '!=' | '<' | '>' | '>=' | '<=';
+  operator: '=' | '!=' | '<>' | '<' | '>' | '>=' | '<=';
   left: Expression;
   right: Expression;
 }

packages/tools/src/DatabaseAnalyser.ts

@@ -8,6 +8,26 @@ const STRUCTURE_FIELDS = ['tables', 'collections', 'views', 'matviews', 'functio
 
 const fp_pick = arg => array => _pick(array, arg);
 
+function mergeTableRowCounts(info: DatabaseInfo, rowCounts): DatabaseInfo {
+  return {
+    ...info,
+    tables: (info.tables || []).map(table => ({
+      ...table,
+      tableRowCount: rowCounts.find(x => x.objectId == table.objectId)?.tableRowCount ?? table.tableRowCount,
+    })),
+  };
+}
+
+function areDifferentRowCounts(db1: DatabaseInfo, db2: DatabaseInfo) {
+  for (const t1 of db1.tables || []) {
+    const t2 = (db2.tables || []).find(x => x.objectId == t1.objectId);
+    if (t1?.tableRowCount !== t2?.tableRowCount) {
+      return true;
+    }
+  }
+  return false;
+}
+
 export class DatabaseAnalyser {
   structure: DatabaseInfo;
   modifications: DatabaseModification[];
@@ -64,13 +84,29 @@ export class DatabaseAnalyser {
   async incrementalAnalysis(structure) {
     this.structure = structure;
 
-    this.modifications = await this.getModifications();
-    if (this.modifications == null) {
+    const modifications = await this.getModifications();
+    if (modifications == null) {
       // modifications not implemented, perform full analysis
       this.structure = null;
       return this.addEngineField(await this._runAnalysis());
     }
-    if (this.modifications.length == 0) return null;
+    const structureModifications = modifications.filter(x => x.action != 'setTableRowCounts');
+    const setTableRowCounts = modifications.find(x => x.action == 'setTableRowCounts');
+
+    let structureWithRowCounts = null;
+    if (setTableRowCounts) {
+      const newStructure = mergeTableRowCounts(structure, setTableRowCounts.rowCounts);
+      if (areDifferentRowCounts(structure, newStructure)) {
+        structureWithRowCounts = newStructure;
+      }
+    }
+
+    if (structureModifications.length == 0) {
+      return structureWithRowCounts ? this.addEngineField(structureWithRowCounts) : null;
+    }
+
+    this.modifications = structureModifications;
+    if (structureWithRowCounts) this.structure = structureWithRowCounts;
     console.log('DB modifications detected:', this.modifications);
     return this.addEngineField(this.mergeAnalyseResult(await this._runAnalysis()));
   }
@@ -226,9 +262,34 @@ export class DatabaseAnalyser {
       }
     }
 
+    const rowCounts = (snapshot.tables || [])
+      .filter(x => x.tableRowCount != null)
+      .map(x => ({
+        objectId: x.objectId,
+        tableRowCount: x.tableRowCount,
+      }));
+
+    if (rowCounts.length > 0) {
+      res.push({
+        action: 'setTableRowCounts',
+        rowCounts,
+      });
+    }
+
     return [..._compact(res), ...this.getDeletedObjects(snapshot)];
   }
 
+  async safeQuery(sql) {
+    try {
+      return await this.driver.query(this.pool, sql);
+    } catch (err) {
+      console.log('Error running analyser query', err.message);
+      return {
+        rows: [],
+      };
+    }
+  }
+
   static createEmptyStructure(): DatabaseInfo {
     return {
       tables: [],

packages/tools/src/SqlDumper.ts

@@ -22,6 +22,8 @@ import {
 import _isString from 'lodash/isString';
 import _isNumber from 'lodash/isNumber';
 import _isDate from 'lodash/isDate';
+import _isArray from 'lodash/isArray';
+import _isPlainObject from 'lodash/isPlainObject';
 import uuidv1 from 'uuid/v1';
 
 export class SqlDumper implements AlterProcessor {
@@ -57,6 +59,9 @@ export class SqlDumper implements AlterProcessor {
     this.putRaw(this.escapeString(value));
     this.putRaw("'");
   }
+  putByteArrayValue(value) {
+    this.putRaw('NULL');
+  }
   putValue(value) {
     if (value === null) this.putRaw('NULL');
     else if (value === true) this.putRaw('1');
@@ -64,6 +69,8 @@ export class SqlDumper implements AlterProcessor {
     else if (_isString(value)) this.putStringValue(value);
     else if (_isNumber(value)) this.putRaw(value.toString());
     else if (_isDate(value)) this.putStringValue(new Date(value).toISOString());
+    else if (value?.type == 'Buffer' && _isArray(value?.data)) this.putByteArrayValue(value?.data);
+    else if (_isPlainObject(value) || _isArray(value)) this.putStringValue(JSON.stringify(value));
     else this.putRaw('NULL');
   }
   putCmd(format, ...args) {
@@ -174,6 +181,8 @@ export class SqlDumper implements AlterProcessor {
     this.put(' ^auto_increment');
   }
 
+  specialColumnOptions(column) {}
+
   columnDefinition(column: ColumnInfo, { includeDefault = true, includeNullable = true, includeCollate = true } = {}) {
     if (column.computedExpression) {
       this.put('^as %s', column.computedExpression);
@@ -186,9 +195,7 @@ export class SqlDumper implements AlterProcessor {
     }
 
     this.putRaw(' ');
-    if (column.isSparse) {
-      this.put(' ^sparse ');
-    }
+    this.specialColumnOptions(column);
     if (includeNullable) {
       this.put(column.notNull ? '^not ^null' : '^null');
     }

packages/tools/src/createBulkInsertStreamBase.ts

@@ -14,6 +14,7 @@ export function createBulkInsertStreamBase(driver, stream, pool, name, options):
   writable.buffer = [];
   writable.structure = null;
   writable.columnNames = null;
+  writable.requireFixedStructure = !driver.dialect.nosql;
 
   writable.addRow = async row => {
     if (writable.structure) {
@@ -29,18 +30,18 @@ export function createBulkInsertStreamBase(driver, stream, pool, name, options):
     // console.log('ANALYSING', name, structure);
     if (structure && options.dropIfExists) {
       console.log(`Dropping table ${fullNameQuoted}`);
-      await driver.query(pool, `DROP TABLE ${fullNameQuoted}`);
+      await driver.script(pool, `DROP TABLE ${fullNameQuoted}`);
     }
     if (options.createIfNotExists && (!structure || options.dropIfExists)) {
       console.log(`Creating table ${fullNameQuoted}`);
       const dmp = driver.createDumper();
       dmp.createTable(prepareTableForImport({ ...writable.structure, ...name }));
       console.log(dmp.s);
-      await driver.query(pool, dmp.s);
+      await driver.script(pool, dmp.s);
       structure = await driver.analyseSingleTable(pool, name);
     }
     if (options.truncate) {
-      await driver.query(pool, `TRUNCATE TABLE ${fullNameQuoted}`);
+      await driver.script(pool, `TRUNCATE TABLE ${fullNameQuoted}`);
     }
 
     writable.columnNames = _intersection(

packages/tools/src/diffTools.ts

@@ -217,6 +217,24 @@ export function testEqualColumns(
     // opts.DiffLogger.Trace('Column {0}, {1}: different is_sparse: {2}; {3}', a, b, a.IsSparse, b.IsSparse);
     return false;
   }
+  if ((a.isUnsigned || false) != (b.isUnsigned || false)) {
+    console.debug(
+      `Column ${a.pureName}.${a.columnName}, ${b.pureName}.${b.columnName}: different unsigned: ${a.isUnsigned}, ${b.isUnsigned}`
+    );
+    return false;
+  }
+  if ((a.isZerofill || false) != (b.isZerofill || false)) {
+    console.debug(
+      `Column ${a.pureName}.${a.columnName}, ${b.pureName}.${b.columnName}: different zerofill: ${a.isZerofill}, ${b.isZerofill}`
+    );
+    return false;
+  }
+  if ((a.columnComment || '') != (b.columnComment || '')) {
+    console.debug(
+      `Column ${a.pureName}.${a.columnName}, ${b.pureName}.${b.columnName}: different comment: ${a.columnComment}, ${b.columnComment}`
+    );
+    return false;
+  }
 
   if (!testEqualTypes(a, b, opts)) {
     return false;

packages/tools/src/driverBase.ts

@@ -10,6 +10,10 @@ const dialect = {
   quoteIdentifier(s) {
     return s;
   },
+  columnProperties: {
+    isSparse: false,
+    isPersisted: false,
+  },
 };
 
 export const driverBase = {

packages/tools/src/stringTools.ts

@@ -1,5 +1,9 @@
+import _isString from 'lodash/isString';
+import _isArray from 'lodash/isArray';
+import _isPlainObject from 'lodash/isPlainObject';
+
 export function arrayToHexString(byteArray) {
-  return byteArray.reduce((output, elem) => output + ('0' + elem.toString(16)).slice(-2), '');
+  return byteArray.reduce((output, elem) => output + ('0' + elem.toString(16)).slice(-2), '').toUpperCase();
 }
 
 export function hexStringToArray(inputString) {
@@ -10,3 +14,48 @@ export function hexStringToArray(inputString) {
   }
   return res;
 }
+
+export function parseCellValue(value) {
+  if (!_isString(value)) return value;
+
+  if (value == '(NULL)') return null;
+
+  const mHex = value.match(/^0x([0-9a-fA-F][0-9a-fA-F])+$/);
+  if (mHex) {
+    return {
+      type: 'Buffer',
+      data: hexStringToArray(value.substring(2)),
+    };
+  }
+
+  const mOid = value.match(/^ObjectId\("([0-9a-f]{24})"\)$/);
+  if (mOid) {
+    return { $oid: mOid[1] };
+  }
+
+  return value;
+}
+
+export function stringifyCellValue(value) {
+  if (value === null) return '(NULL)';
+  if (value === undefined) return '(NoField)';
+  if (value?.type == 'Buffer' && _isArray(value.data)) return '0x' + arrayToHexString(value.data);
+  if (value?.$oid) return `ObjectId("${value?.$oid}")`;
+  if (_isPlainObject(value) || _isArray(value)) return JSON.stringify(value);
+  return value;
+}
+
+export function safeJsonParse(json, defaultValue?, logError = false) {
+  try {
+    return JSON.parse(json);
+  } catch (err) {
+    if (logError) {
+      console.error(`Error parsing JSON value "${json}"`, err);
+    }
+    return defaultValue;
+  }
+}
+
+export function isJsonLikeLongString(value) {
+  return _isString(value) && value.length > 100 && value.match(/^\s*\{.*\}\s*$|^\s*\[.*\]\s*$/);
+}

packages/tools/src/structureTools.ts

@@ -1,4 +1,4 @@
-import { DatabaseInfo, TableInfo } from 'dbgate-types';
+import { DatabaseInfo, TableInfo, ApplicationDefinition } from 'dbgate-types';
 import _flatten from 'lodash/flatten';
 
 export function addTableDependencies(db: DatabaseInfo): DatabaseInfo {
@@ -90,3 +90,31 @@ function fillDatabaseExtendedInfo(db: DatabaseInfo): DatabaseInfo {
 export function extendDatabaseInfo(db: DatabaseInfo): DatabaseInfo {
   return fillDatabaseExtendedInfo(addTableDependencies(db));
 }
+
+export function extendDatabaseInfoFromApps(db: DatabaseInfo, apps: ApplicationDefinition[]): DatabaseInfo {
+  if (!db || !apps) return db;
+  const dbExt = {
+    ...db,
+    tables: db.tables.map(table => ({
+      ...table,
+      foreignKeys: [
+        ...(table.foreignKeys || []),
+        ..._flatten(apps.map(app => app.virtualReferences || []))
+          .filter(fk => fk.pureName == table.pureName && fk.schemaName == table.schemaName)
+          .map(fk => ({ ...fk, constraintType: 'foreignKey', isVirtual: true })),
+      ],
+    })),
+  } as DatabaseInfo;
+  return addTableDependencies(dbExt);
+}
+
+export function isTableColumnUnique(table: TableInfo, column: string) {
+  if (table.primaryKey && table.primaryKey.columns.length == 1 && table.primaryKey.columns[0].columnName == column) {
+    return true;
+  }
+  const uqs = [...(table.uniques || []), ...(table.indexes || []).filter(x => x.isUnique)];
+  if (uqs.find(uq => uq.columns.length == 1 && uq.columns[0].columnName == column)) {
+    return true;
+  }
+  return false;
+}

packages/tools/src/tableTransforms.ts

@@ -4,6 +4,9 @@ import _cloneDeep from 'lodash/cloneDeep';
 export function prepareTableForImport(table: TableInfo): TableInfo {
   const res = _cloneDeep(table);
   res.foreignKeys = [];
+  res.indexes = [];
+  res.uniques = [];
+  res.checks = [];
   if (res.primaryKey) res.primaryKey.constraintName = null;
   return res;
 }

packages/types/appdefs.d.ts

@@ -0,0 +1,37 @@
+interface ApplicationCommand {
+  name: string;
+  sql: string;
+}
+
+interface ApplicationQuery {
+  name: string;
+  sql: string;
+}
+
+interface VirtualReferenceDefinition {
+  pureName: string;
+  schemaName?: string;
+  refSchemaName?: string;
+  refTableName: string;
+  columns: {
+    columnName: string;
+    refColumnName: string;
+  }[];
+}
+
+interface DictionaryDescriptionDefinition {
+  pureName: string;
+  schemaName?: string;
+  expression: string;
+  columns: string[];
+  delimiter: string;
+}
+
+export interface ApplicationDefinition {
+  name: string;
+
+  queries: ApplicationQuery[];
+  commands: ApplicationCommand[];
+  virtualReferences: VirtualReferenceDefinition[];
+  dictionaryDescriptions: DictionaryDescriptionDefinition[];
+}

packages/types/dbinfo.d.ts

@@ -54,6 +54,9 @@ export interface ColumnInfo extends NamedObjectInfo {
   isSparse?: boolean;
   defaultValue?: string;
   defaultConstraint?: string;
+  columnComment?: string;
+  isUnsigned?: boolean;
+  isZerofill?: boolean;
 }
 
 export interface DatabaseObjectInfo extends NamedObjectInfo {
@@ -63,6 +66,7 @@ export interface DatabaseObjectInfo extends NamedObjectInfo {
   modifyDate?: string;
   hashCode?: string;
   objectTypeField?: string;
+  obejctComment?: string;
 }
 
 export interface SqlObjectInfo extends DatabaseObjectInfo {
@@ -81,6 +85,7 @@ export interface TableInfo extends DatabaseObjectInfo {
   preloadedRows?: any[];
   preloadedRowsKey?: string[];
   preloadedRowsInsertOnly?: string[];
+  tableRowCount?: number | string;
   __isDynamicStructure?: boolean;
 }
 
@@ -92,7 +97,9 @@ export interface ViewInfo extends SqlObjectInfo {
 
 export interface ProcedureInfo extends SqlObjectInfo {}
 
-export interface FunctionInfo extends SqlObjectInfo {}
+export interface FunctionInfo extends SqlObjectInfo {
+  // returnDataType?: string;
+}
 
 export interface TriggerInfo extends SqlObjectInfo {}
 
@@ -114,4 +121,5 @@ export interface DatabaseInfoObjects {
 export interface DatabaseInfo extends DatabaseInfoObjects {
   schemas: SchemaInfo[];
   engine?: string;
+  defaultSchema?: string;
 }

packages/types/extensions.d.ts

@@ -22,9 +22,10 @@ export interface FileFormatDefinition {
 }
 
 export interface ThemeDefinition {
-  className: string;
+  themeClassName: string;
   themeName: string;
   themeType: 'light' | 'dark';
+  themeCss?: string;
 }
 
 export interface PluginDefinition {

packages/types/index.d.ts

@@ -11,6 +11,7 @@ export interface OpenedDatabaseConnection {
   status?: {
     name: string;
     message?: string;
+    counter: number;
   };
 }
 
@@ -43,3 +44,4 @@ export * from './dumper';
 export * from './dbtypes';
 export * from './extensions';
 export * from './alter-processor';
+export * from './appdefs';

packages/web/package.json

@@ -45,13 +45,17 @@
     "rollup-plugin-terser": "^7.0.0",
     "sirv-cli": "^1.0.0",
     "sql-formatter": "^2.3.3",
-    "svelte": "^3.43.0",
+    "svelte": "^3.46.4",
     "svelte-check": "^1.0.0",
     "svelte-markdown": "^0.1.4",
     "svelte-preprocess": "^4.9.5",
-    "svelte-select": "^3.17.0",
+    "svelte-select": "^4.4.7",
     "tslib": "^2.3.1",
     "typescript": "^4.4.3",
     "uuid": "^3.4.0"
+  },
+  "dependencies": {
+    "chartjs-plugin-zoom": "^1.2.0",
+    "interval-operations": "^1.0.7"
   }
 }

packages/web/public/dimensions.css

@@ -12,10 +12,16 @@
       (var(--dim-left-panel-width) + var(--dim-splitter-thickness))
   );
 
-  --dim-visible-toolbar: 1; /* set from JS */
+  --dim-visible-toolbar: 0; /* set from JS */
+  --dim-visible-titlebar: 0; /* set from JS */
 
   --dim-toolbar-height: 30px;
-  --dim-header-top: calc(var(--dim-toolbar-height) * var(--dim-visible-toolbar));
+  --dim-toolstrip-height: 28px;
+  --dim-titlebar-height: 30px;
+  --dim-toolbar-top: calc(var(--dim-titlebar-height) * var(--dim-visible-titlebar));
+  --dim-header-top: calc(
+    var(--dim-titlebar-height) * var(--dim-visible-titlebar) + var(--dim-toolbar-height) * var(--dim-visible-toolbar)
+  );
   --dim-content-top: calc(var(--dim-header-top) + var(--dim-tabs-panel-height));
 
   --dim-large-form-margin: 20px;

packages/web/public/global.css

@@ -45,6 +45,9 @@ body {
 .nowrap {
   white-space: nowrap;
 }
+.noselect {
+  user-select: none;
+}
 .bold {
   font-weight: bold;
 }

packages/web/src/App.svelte

@@ -15,6 +15,8 @@
   import { subscribeConnectionPingers } from './utility/connectionsPinger';
   import { subscribePermissionCompiler } from './utility/hasPermission';
   import { apiCall } from './utility/api';
+  import { getConfig, getSettings, getUsedApps } from './utility/metadataLoaders';
+  import AppTitleProvider from './utility/AppTitleProvider.svelte';
 
   let loadedApi = false;
 
@@ -27,10 +29,11 @@
     try {
       // console.log('************** LOADING API');
 
-      const settings = await apiCall('config/get-settings');
       const connections = await apiCall('connections/list');
-      const config = await apiCall('config/get');
-      loadedApi = settings && connections && config;
+      const settings = await getSettings();
+      const config = await getConfig();
+      const apps = await getUsedApps();
+      loadedApi = settings && connections && config && apps;
 
       if (loadedApi) {
         subscribeApiDependendStores();
@@ -68,6 +71,7 @@
   <DataGridRowHeightMeter />
   <CommandListener />
   <PluginsProvider />
+  <AppTitleProvider />
   {#if $loadingPluginStore?.loaded}
     <OpenTabsOnStartup />
     <Screen />

packages/web/src/Screen.svelte

@@ -9,6 +9,7 @@
     openedSnackbars,
     selectedWidget,
     visibleCommandPalette,
+    visibleTitleBar,
     visibleToolbar,
   } from './stores';
   import TabsPanel from './widgets/TabsPanel.svelte';
@@ -22,15 +23,40 @@
   import ModalLayer from './modals/ModalLayer.svelte';
   import DragAndDropFileTarget from './DragAndDropFileTarget.svelte';
   import dragDropFileTarget from './utility/dragDropFileTarget';
+  import TitleBar from './widgets/TitleBar.svelte';
+  import FontIcon from './icons/FontIcon.svelte';
+  import getElectron from './utility/getElectron';
 
   $: currentThemeType = $currentThemeDefinition?.themeType == 'dark' ? 'theme-type-dark' : 'theme-type-light';
+
+  $: themeStyle = `<style id="themePlugin">${$currentThemeDefinition?.themeCss}</style>`;
+
+  const isElectron = !!getElectron();
 </script>
 
+<svelte:head>
+  {#if $currentThemeDefinition?.themeCss}
+    {@html themeStyle}
+  {/if}
+</svelte:head>
+
+<div class="not-supported" class:isElectron>
+  <div class="m-5 big-icon"><FontIcon icon="img warn" /></div>
+  <div class="m-3">Sorry, DbGate is not supported on mobile devices.</div>
+  <div class="m-3">Please visit <a href="https://dbgate.org">DbGate web</a> for more info.</div>
+</div>
+
 <div
-  class={`${$currentTheme} ${currentThemeType} root`}
+  class={`${$currentTheme} ${currentThemeType} root dbgate-screen`}
+  class:isElectron
   use:dragDropFileTarget
   on:contextmenu={e => e.preventDefault()}
 >
+  {#if $visibleTitleBar}
+    <div class="titlebar">
+      <TitleBar />
+    </div>
+  {/if}
   <div class="iconbar">
     <WidgetIconPanel />
   </div>
@@ -109,19 +135,13 @@
     display: flex;
   }
   .tabs {
-    display: flex;
     position: fixed;
     top: var(--dim-header-top);
     left: var(--dim-content-left);
     height: var(--dim-tabs-panel-height);
     right: 0;
-    background-color: var(--theme-bg-2);
+    background-color: var(--theme-bg-1);
     border-top: 1px solid var(--theme-border);
-
-    overflow-x: auto;
-  }
-  .tabs::-webkit-scrollbar {
-    height: 7px;
   }
   .content {
     position: fixed;
@@ -138,7 +158,7 @@
   }
   .toolbar {
     position: fixed;
-    top: 0;
+    top: var(--dim-toolbar-top);
     height: var(--dim-toolbar-height);
     left: 0;
     right: 0;
@@ -157,4 +177,33 @@
     right: 0;
     bottom: var(--dim-statusbar-height);
   }
+
+  .titlebar {
+    position: fixed;
+    top: 0;
+    left: 0;
+    right: 0;
+    height: var(--dim-titlebar-height);
+  }
+
+  .not-supported {
+    display: none;
+  }
+
+  @media only screen and (max-width: 600px) {
+    .dbgate-screen:not(.isElectron) {
+      display: none;
+    }
+
+    .not-supported:not(.isElectron) {
+      display: block;
+    }
+  }
+
+  .not-supported {
+    text-align: center;
+  }
+  .big-icon {
+    font-size: 20pt;
+  }
 </style>

packages/web/src/appobj/AppFileAppObject.svelte

@@ -0,0 +1,119 @@
+<script lang="ts" context="module">
+  async function openTextFile(fileName, fileType, folderName, tabComponent, icon) {
+    const connProps: any = {};
+    let tooltip = undefined;
+
+    const resp = await apiCall('files/load', {
+      folder: 'app:' + folderName,
+      file: fileName + '.' + fileType,
+      format: 'text',
+    });
+
+    openNewTab(
+      {
+        title: fileName,
+        icon,
+        tabComponent,
+        tooltip,
+        props: {
+          savedFile:fileName + '.' + fileType,
+          savedFolder: 'app:' + folderName,
+          savedFormat: 'text',
+          appFolder: folderName,
+          ...connProps,
+        },
+      },
+      { editor: resp }
+    );
+  }
+
+  export const extractKey = data => data.fileName;
+  export const createMatcher = ({ fileName }) => filter => filterName(filter, fileName);
+  const APP_ICONS = {
+    'config.json': 'img json',
+    'command.sql': 'img app-command',
+    'query.sql': 'img app-query',
+  };
+
+  function getAppIcon(data) {
+    return APP_ICONS[data.fileType];
+  }
+</script>
+
+<script lang="ts">
+  import _ from 'lodash';
+  import { filterName } from 'dbgate-tools';
+  import { showModal } from '../modals/modalTools';
+
+  import openNewTab from '../utility/openNewTab';
+  import AppObjectCore from './AppObjectCore.svelte';
+  import InputTextModal from '../modals/InputTextModal.svelte';
+  import ConfirmModal from '../modals/ConfirmModal.svelte';
+  import { apiCall } from '../utility/api';
+  import { currentDatabase, currentDatabase } from '../stores';
+
+  export let data;
+
+  const handleRename = () => {
+    showModal(InputTextModal, {
+      value: data.fileName,
+      label: 'New file name',
+      header: 'Rename file',
+      onConfirm: newFile => {
+        apiCall('apps/rename-file', {
+          file: data.fileName,
+          folder: data.folderName,
+          fileType: data.fileType,
+          newFile,
+        });
+      },
+    });
+  };
+
+  const handleDelete = () => {
+    showModal(ConfirmModal, {
+      message: `Really delete file ${data.fileName}?`,
+      onConfirm: () => {
+        apiCall('apps/delete-file', {
+          file: data.fileName,
+          folder: data.folderName,
+          fileType: data.fileType,
+        });
+      },
+    });
+  };
+  const handleClick = () => {
+    if (data.fileType.endsWith('.sql')) {
+      handleOpenSqlFile();
+    }
+    if (data.fileType.endsWith('.json')) {
+      handleOpenJsonFile();
+    }
+  };
+  const handleOpenSqlFile = () => {
+    openTextFile(data.fileName, data.fileType, data.folderName, 'QueryTab', 'img sql-file');
+  };
+  const handleOpenJsonFile = () => {
+    openTextFile(data.fileName, data.fileType, data.folderName, 'JsonEditorTab', 'img json');
+  };
+
+  function createMenu() {
+    return [
+      { text: 'Delete', onClick: handleDelete },
+      { text: 'Rename', onClick: handleRename },
+      data.fileType.endsWith('.sql') && { text: 'Open SQL', onClick: handleOpenSqlFile },
+      data.fileType.endsWith('.json') && { text: 'Open JSON', onClick: handleOpenJsonFile },
+
+      // data.fileType.endsWith('.yaml') && { text: 'Open YAML', onClick: handleOpenYamlFile },
+    ];
+  }
+</script>
+
+<AppObjectCore
+  {...$$restProps}
+  {data}
+  title={data.fileLabel}
+  icon={getAppIcon(data)}
+  menu={createMenu}
+  on:click={handleClick}
+/>

packages/web/src/appobj/AppFolderAppObject.svelte

@@ -0,0 +1,97 @@
+<script lang="ts" context="module">
+  export const extractKey = data => data.name;
+  export const createMatcher = data => filter => filterName(filter, data.name);
+</script>
+
+<script lang="ts">
+  import _, { find } from 'lodash';
+  import { filterName } from 'dbgate-tools';
+
+  import { currentApplication, currentDatabase } from '../stores';
+
+  import AppObjectCore from './AppObjectCore.svelte';
+  import { showModal } from '../modals/modalTools';
+  import ConfirmModal from '../modals/ConfirmModal.svelte';
+  import InputTextModal from '../modals/InputTextModal.svelte';
+  import { apiCall } from '../utility/api';
+  import { useConnectionList } from '../utility/metadataLoaders';
+
+  export let data;
+
+  $: connections = useConnectionList();
+
+  const handleDelete = () => {
+    showModal(ConfirmModal, {
+      message: `Really delete application ${data.name}?`,
+      onConfirm: () => {
+        apiCall('apps/delete-folder', { folder: data.name });
+      },
+    });
+  };
+
+  const handleRename = () => {
+    const { name } = data;
+
+    showModal(InputTextModal, {
+      value: name,
+      label: 'New application name',
+      header: 'Rename application',
+      onConfirm: async newFolder => {
+        await apiCall('apps/rename-folder', {
+          folder: data.name,
+          newFolder: newFolder,
+        });
+        if ($currentApplication == data.name) {
+          $currentApplication = newFolder;
+        }
+      },
+    });
+  };
+
+  function setOnCurrentDb(value) {
+    apiCall('connections/update-database', {
+      conid: $currentDatabase?.connection?._id,
+      database: $currentDatabase?.name,
+      values: {
+        [`useApp:${data.name}`]: value,
+      },
+    });
+  }
+
+  function createMenu() {
+    return [
+      { text: 'Delete', onClick: handleDelete },
+      { text: 'Rename', onClick: handleRename },
+
+      $currentDatabase && [
+        !isOnCurrentDb($currentDatabase, $connections) && {
+          text: 'Enable on current database',
+          onClick: () => setOnCurrentDb(true),
+        },
+        isOnCurrentDb($currentDatabase, $connections) && {
+          text: 'Disable on current database',
+          onClick: () => setOnCurrentDb(false),
+        },
+      ],
+    ];
+  }
+
+  function isOnCurrentDb(currentDb, connections) {
+    if (!currentDb || !connections) return false;
+    const conn = connections.find(x => x._id == currentDb?.connection?._id);
+    const db = conn?.databases?.find(x => x.name == currentDb?.name);
+    return db && db[`useApp:${data.name}`];
+  }
+</script>
+
+<AppObjectCore
+  {...$$restProps}
+  {data}
+  title={data.name}
+  icon={'img app'}
+  statusIcon={isOnCurrentDb($currentDatabase, $connections) ? 'icon check' : null}
+  statusTitle={`Application ${data.name} is used for database ${$currentDatabase?.name}`}
+  isBold={data.name == $currentApplication}
+  on:click={() => ($currentApplication = data.name)}
+  menu={createMenu}
+/>

packages/web/src/appobj/ArchiveFileAppObject.svelte

@@ -70,7 +70,7 @@
   import { archiveFilesAsDataSheets, currentArchive, extensions, getCurrentDatabase } from '../stores';
 
   import createQuickExportMenu from '../utility/createQuickExportMenu';
-  import { exportElectronFile } from '../utility/exportElectronFile';
+  import { exportQuickExportFile } from '../utility/exportFileTools';
   import openNewTab from '../utility/openNewTab';
   import AppObjectCore from './AppObjectCore.svelte';
   import getConnectionLabel from '../utility/getConnectionLabel';
@@ -81,7 +81,7 @@
     markArchiveFileAsDataSheet,
     markArchiveFileAsReadonly,
   } from '../utility/archiveTools';
-import { apiCall } from '../utility/api';
+  import { apiCall } from '../utility/api';
 
   export let data;
 
@@ -157,39 +157,45 @@ import { apiCall } from '../utility/api';
   const handleOpenYamlFile = () => {
     openTextFile(data.fileName, data.fileType, data.folderName, 'YamlEditorTab', 'img yaml');
   };
+  const handleOpenJsonLinesText = () => {
+    openTextFile(data.fileName, data.fileType, data.folderName, 'JsonLinesEditorTab', 'img json');
+  };
 
   function createMenu() {
     return [
       data.fileType == 'jsonl' && { text: 'Open (readonly)', onClick: handleOpenRead },
       data.fileType == 'jsonl' && { text: 'Open as data sheet', onClick: handleOpenWrite },
+      data.fileType == 'jsonl' && { text: 'Open in text editor', onClick: handleOpenJsonLinesText },
       { text: 'Delete', onClick: handleDelete },
       { text: 'Rename', onClick: handleRename },
       data.fileType == 'jsonl' &&
-        createQuickExportMenu($extensions, fmt => async () => {
-          exportElectronFile(
-            data.fileName,
-            {
-              functionName: 'archiveReader',
-              props: {
-                fileName: data.fileName,
-                folderName: data.folderName,
+        createQuickExportMenu(
+          fmt => async () => {
+            exportQuickExportFile(
+              data.fileName,
+              {
+                functionName: 'archiveReader',
+                props: {
+                  fileName: data.fileName,
+                  folderName: data.folderName,
+                },
               },
+              fmt
+            );
+          },
+          {
+            text: 'Export',
+            onClick: () => {
+              showModal(ImportExportModal, {
+                initialValues: {
+                  sourceStorageType: 'archive',
+                  sourceArchiveFolder: data.folderName,
+                  sourceList: [data.fileName],
+                },
+              });
             },
-            fmt
-          );
-        }),
-      data.fileType == 'jsonl' && {
-        text: 'Export',
-        onClick: () => {
-          showModal(ImportExportModal, {
-            initialValues: {
-              sourceStorageType: 'archive',
-              sourceArchiveFolder: data.folderName,
-              sourceList: [data.fileName],
-            },
-          });
-        },
-      },
+          }
+        ),
       data.fileType.endsWith('.sql') && { text: 'Open SQL', onClick: handleOpenSqlFile },
       data.fileType.endsWith('.yaml') && { text: 'Open YAML', onClick: handleOpenYamlFile },
     ];

packages/web/src/appobj/ConnectionAppObject.svelte

@@ -26,7 +26,7 @@
   import { getDatabaseMenuItems } from './DatabaseAppObject.svelte';
   import getElectron from '../utility/getElectron';
   import getConnectionLabel from '../utility/getConnectionLabel';
-  import { getDatabaseList } from '../utility/metadataLoaders';
+  import { getDatabaseList, useUsedApps } from '../utility/metadataLoaders';
   import { getLocalStorage } from '../utility/storageCache';
   import { apiCall } from '../utility/api';
 
@@ -97,7 +97,7 @@
         value: 'newdb',
         label: 'Database name',
         onConfirm: name =>
-        apiCall('server-connections/create-database', {
+          apiCall('server-connections/create-database', {
             conid: data._id,
             name,
           }),
@@ -153,7 +153,7 @@
       ],
       data.singleDatabase && [
         { divider: true },
-        getDatabaseMenuItems(data, data.defaultDatabase, $extensions, $currentDatabase),
+        getDatabaseMenuItems(data, data.defaultDatabase, $extensions, $currentDatabase, $apps),
       ],
     ];
   };
@@ -186,6 +186,8 @@
       statusTitle = null;
     }
   }
+
+  $: apps = useUsedApps();
 </script>
 
 <AppObjectCore

packages/web/src/appobj/DatabaseAppObject.svelte

@@ -1,7 +1,8 @@
 <script lang="ts" context="module">
   export const extractKey = props => props.name;
 
-  export function getDatabaseMenuItems(connection, name, $extensions, $currentDatabase) {
+  export function getDatabaseMenuItems(connection, name, $extensions, $currentDatabase, $apps) {
+    const apps = filterAppsForDatabase(connection, name, $apps);
     const handleNewQuery = () => {
       const tooltip = `${getConnectionLabel(connection)}\n${name}`;
       openNewTab({
@@ -85,6 +86,34 @@
       });
     };
 
+    const handleShowDiagram = async () => {
+      const db = await getDatabaseInfo({
+        conid: connection._id,
+        database: name,
+      });
+      openNewTab(
+        {
+          title: 'Diagram #',
+          icon: 'img diagram',
+          tabComponent: 'DiagramTab',
+          props: {
+            conid: connection._id,
+            database: name,
+          },
+        },
+        {
+          editor: {
+            tables: db.tables.map(table => ({
+              ...table,
+              designerId: `${table.pureName}-${uuidv1()}`,
+            })),
+            references: [],
+            autoLayout: true,
+          },
+        }
+      );
+    };
+
     const handleDisconnect = () => {
       const electron = getElectron();
       if (electron) {
@@ -129,8 +158,20 @@
       openJsonDocument(db, name);
     };
 
+    async function handleConfirmSql(sql) {
+      const resp = await apiCall('database-connections/run-script', { conid: connection._id, database: name, sql });
+      const { errorMessage } = resp || {};
+      if (errorMessage) {
+        showModal(ErrorMessageModal, { title: 'Error when executing script', message: errorMessage });
+      } else {
+        showSnackbarSuccess('Saved to database');
+      }
+    }
+
     const driver = findEngineDriver(connection, getExtensions());
 
+    const commands = _.flatten((apps || []).map(x => x.commands || []));
+
     return [
       { onClick: handleNewQuery, text: 'New query', isNewQuery: true },
       !driver?.dialect?.nosql && { onClick: handleNewTable, text: 'New table' },
@@ -138,6 +179,7 @@
       { divider: true },
       { onClick: handleImport, text: 'Import' },
       { onClick: handleExport, text: 'Export' },
+      { onClick: handleShowDiagram, text: 'Show diagram' },
       { onClick: handleSqlGenerator, text: 'SQL Generator' },
       { onClick: handleOpenJsonModel, text: 'Open model as JSON' },
       { onClick: handleExportModel, text: 'Export DB model - experimental' },
@@ -151,12 +193,27 @@
 
       _.get($currentDatabase, 'connection._id') == _.get(connection, '_id') &&
         _.get($currentDatabase, 'name') == name && { onClick: handleDisconnect, text: 'Disconnect' },
+
+      commands.length > 0 && [
+        { divider: true },
+        commands.map((cmd: any) => ({
+          text: cmd.name,
+          onClick: () => {
+            showModal(ConfirmSqlModal, {
+              sql: cmd.sql,
+              onConfirm: () => handleConfirmSql(cmd.sql),
+              engine: driver.engine,
+            });
+          },
+        })),
+      ],
     ];
   }
 </script>
 
 <script lang="ts">
   import getConnectionLabel from '../utility/getConnectionLabel';
+  import uuidv1 from 'uuid/v1';
 
   import _, { find } from 'lodash';
   import ImportExportModal from '../modals/ImportExportModal.svelte';
@@ -177,18 +234,22 @@
   import { showSnackbarSuccess } from '../utility/snackbar';
   import { findEngineDriver } from 'dbgate-tools';
   import InputTextModal from '../modals/InputTextModal.svelte';
-  import { getDatabaseInfo } from '../utility/metadataLoaders';
+  import { getDatabaseInfo, useUsedApps } from '../utility/metadataLoaders';
   import { openJsonDocument } from '../tabs/JsonTab.svelte';
   import { apiCall } from '../utility/api';
+  import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
+  import ConfirmSqlModal from '../modals/ConfirmSqlModal.svelte';
+  import { filterAppsForDatabase } from '../utility/appTools';
 
   export let data;
   export let passProps;
 
   function createMenu() {
-    return getDatabaseMenuItems(data.connection, data.name, $extensions, $currentDatabase);
+    return getDatabaseMenuItems(data.connection, data.name, $extensions, $currentDatabase, $apps);
   }
 
   $: isPinned = !!$pinnedDatabases.find(x => x.name == data.name && x.connection?._id == data.connection?._id);
+  $: apps = useUsedApps();
 </script>
 
 <AppObjectCore

packages/web/src/appobj/DatabaseObjectAppObject.svelte

@@ -1,6 +1,6 @@
 <script lang="ts" context="module">
   export const extractKey = ({ schemaName, pureName }) => (schemaName ? `${schemaName}.${pureName}` : pureName);
-  export const createMatcher = ({ pureName }) => filter => filterName(filter, pureName);
+  export const createMatcher = ({ schemaName, pureName }) => filter => filterName(filter, pureName, schemaName);
   const electron = getElectron();
 
   export const databaseObjectIcons = {
@@ -53,15 +53,16 @@
         label: 'Query designer',
         isQueryDesigner: true,
       },
+      {
+        label: 'Show diagram',
+        isDiagram: true,
+      },
       {
         divider: true,
       },
-      {
-        isQuickExport: true,
-        functionName: 'tableReader',
-      },
       {
         label: 'Export',
+        functionName: 'tableReader',
         isExport: true,
       },
       {
@@ -127,13 +128,10 @@
       {
         divider: true,
       },
-      {
-        isQuickExport: true,
-        functionName: 'tableReader',
-      },
       {
         label: 'Export',
         isExport: true,
+        functionName: 'tableReader',
       },
       {
         label: 'Open as data sheet',
@@ -192,13 +190,10 @@
       {
         divider: true,
       },
-      {
-        isQuickExport: true,
-        functionName: 'tableReader',
-      },
       {
         label: 'Export',
         isExport: true,
+        functionName: 'tableReader',
       },
       {
         label: 'Open as data sheet',
@@ -300,18 +295,19 @@
           },
         },
       },
-      {
-        isQuickExport: true,
-        functionName: 'tableReader',
-      },
       {
         label: 'Export',
         isExport: true,
+        functionName: 'tableReader',
       },
       {
         label: 'Drop collection',
         isDropCollection: true,
       },
+      {
+        label: 'Rename collection',
+        isRenameCollection: true,
+      },
       {
         divider: true,
       },
@@ -388,85 +384,60 @@
       a.schemaName == b.schemaName
     );
   }
-</script>
 
-<script lang="ts">
-  import _ from 'lodash';
-  import AppObjectCore from './AppObjectCore.svelte';
-  import { currentDatabase, extensions, openedConnections, pinnedTables } from '../stores';
-  import openNewTab from '../utility/openNewTab';
-  import { filterName, generateDbPairingId, getAlterDatabaseScript } from 'dbgate-tools';
-  import { getConnectionInfo, getDatabaseInfo } from '../utility/metadataLoaders';
-  import fullDisplayName from '../utility/fullDisplayName';
-  import ImportExportModal from '../modals/ImportExportModal.svelte';
-  import { showModal } from '../modals/modalTools';
-  import { findEngineDriver } from 'dbgate-tools';
-  import uuidv1 from 'uuid/v1';
-  import SqlGeneratorModal from '../modals/SqlGeneratorModal.svelte';
-  import getConnectionLabel from '../utility/getConnectionLabel';
-  import getElectron from '../utility/getElectron';
-  import { exportElectronFile } from '../utility/exportElectronFile';
-  import createQuickExportMenu from '../utility/createQuickExportMenu';
-  import ConfirmSqlModal from '../modals/ConfirmSqlModal.svelte';
-  import { alterDatabaseDialog, renameDatabaseObjectDialog } from '../utility/alterDatabaseTools';
-  import ConfirmModal from '../modals/ConfirmModal.svelte';
-  import { apiCall } from '../utility/api';
+  export function createDatabaseObjectMenu(data) {
+    const getDriver = async () => {
+      const conn = await getConnectionInfo(data);
+      if (!conn) return;
+      const driver = findEngineDriver(conn, getExtensions());
+      return driver;
+    };
 
-  export let data;
-  export let passProps;
-
-  function handleClick(forceNewTab = false) {
-    handleDatabaseObjectClick(data, forceNewTab);
-  }
-
-  const getDriver = async () => {
-    const conn = await getConnectionInfo(data);
-    if (!conn) return;
-    const driver = findEngineDriver(conn, $extensions);
-    return driver;
-  };
-
-  function createMenu() {
     const { objectTypeField } = data;
     return menus[objectTypeField]
       .filter(x => x)
       .map(menu => {
         if (menu.divider) return menu;
 
-        if (menu.isQuickExport) {
-          return createQuickExportMenu($extensions, fmt => async () => {
-            const coninfo = await getConnectionInfo(data);
-            exportElectronFile(
-              data.pureName,
-              {
-                functionName: menu.functionName,
-                props: {
-                  connection: {
-                    ..._.omit(coninfo, ['_id', 'displayName']),
-                    ..._.pick(data, ['database']),
+        if (menu.isExport) {
+          return createQuickExportMenu(
+            fmt => async () => {
+              const coninfo = await getConnectionInfo(data);
+              exportQuickExportFile(
+                data.pureName,
+                {
+                  functionName: menu.functionName,
+                  props: {
+                    connection: {
+                      ..._.omit(coninfo, ['_id', 'displayName']),
+                      ..._.pick(data, ['database']),
+                    },
+                    ..._.pick(data, ['pureName', 'schemaName']),
                   },
-                  ..._.pick(data, ['pureName', 'schemaName']),
                 },
+                fmt
+              );
+            },
+            {
+              onClick: () => {
+                showModal(ImportExportModal, {
+                  initialValues: {
+                    sourceStorageType: 'database',
+                    sourceConnectionId: data.conid,
+                    sourceDatabaseName: data.database,
+                    sourceSchemaName: data.schemaName,
+                    sourceList: [data.pureName],
+                  },
+                });
               },
-              fmt
-            );
-          });
+            }
+          );
         }
 
         return {
           text: menu.label,
           onClick: async () => {
-            if (menu.isExport) {
-              showModal(ImportExportModal, {
-                initialValues: {
-                  sourceStorageType: 'database',
-                  sourceConnectionId: data.conid,
-                  sourceDatabaseName: data.database,
-                  sourceSchemaName: data.schemaName,
-                  sourceList: [data.pureName],
-                },
-              });
-            } else if (menu.isOpenFreeTable) {
+            if (menu.isOpenFreeTable) {
               const coninfo = await getConnectionInfo(data);
               openNewTab({
                 title: data.pureName,
@@ -531,6 +502,31 @@
                   },
                 }
               );
+            } else if (menu.isDiagram) {
+              openNewTab(
+                {
+                  title: 'Diagram #',
+                  icon: 'img diagram',
+                  tabComponent: 'DiagramTab',
+                  props: {
+                    conid: data.conid,
+                    database: data.database,
+                  },
+                },
+                {
+                  editor: {
+                    tables: [
+                      {
+                        ...data,
+                        designerId: `${data.pureName}-${uuidv1()}`,
+                        autoAddReferences: true,
+                      },
+                    ],
+                    references: [],
+                    autoLayout: true,
+                  },
+                }
+              );
             } else if (menu.sqlGeneratorProps) {
               showModal(SqlGeneratorModal, {
                 initialObjects: [data],
@@ -566,6 +562,20 @@
                   apiCall('database-connections/sync-model', dbid);
                 },
               });
+            } else if (menu.isRenameCollection) {
+              showModal(InputTextModal, {
+                label: 'New collection name',
+                header: 'Rename collection',
+                value: data.pureName,
+                onConfirm: async newName => {
+                  const dbid = _.pick(data, ['conid', 'database']);
+                  await apiCall('database-connections/run-script', {
+                    ...dbid,
+                    sql: `db.renameCollection('${data.pureName}', '${newName}')`,
+                  });
+                  apiCall('database-connections/sync-model', dbid);
+                },
+              });
             } else {
               openDatabaseObjectDetail(
                 menu.tab,
@@ -581,6 +591,47 @@
       });
   }
 
+  function formatRowCount(value) {
+    const num = parseInt(value);
+    if (_.isNaN(num)) return value;
+    return num.toLocaleString();
+  }
+</script>
+
+<script lang="ts">
+  import _ from 'lodash';
+  import AppObjectCore from './AppObjectCore.svelte';
+  import { currentDatabase, extensions, getExtensions, openedConnections, pinnedTables } from '../stores';
+  import openNewTab from '../utility/openNewTab';
+  import { filterName, generateDbPairingId, getAlterDatabaseScript } from 'dbgate-tools';
+  import { getConnectionInfo, getDatabaseInfo } from '../utility/metadataLoaders';
+  import fullDisplayName from '../utility/fullDisplayName';
+  import ImportExportModal from '../modals/ImportExportModal.svelte';
+  import { showModal } from '../modals/modalTools';
+  import { findEngineDriver } from 'dbgate-tools';
+  import uuidv1 from 'uuid/v1';
+  import SqlGeneratorModal from '../modals/SqlGeneratorModal.svelte';
+  import getConnectionLabel from '../utility/getConnectionLabel';
+  import getElectron from '../utility/getElectron';
+  import { exportQuickExportFile } from '../utility/exportFileTools';
+  import createQuickExportMenu from '../utility/createQuickExportMenu';
+  import ConfirmSqlModal from '../modals/ConfirmSqlModal.svelte';
+  import { alterDatabaseDialog, renameDatabaseObjectDialog } from '../utility/alterDatabaseTools';
+  import ConfirmModal from '../modals/ConfirmModal.svelte';
+  import { apiCall } from '../utility/api';
+  import InputTextModal from '../modals/InputTextModal.svelte';
+
+  export let data;
+  export let passProps;
+
+  function handleClick(forceNewTab = false) {
+    handleDatabaseObjectClick(data, forceNewTab);
+  }
+
+  function createMenu() {
+    return createDatabaseObjectMenu(data);
+  }
+
   $: isPinned = !!$pinnedTables.find(x => testEqual(data, x));
 </script>
 
@@ -594,6 +645,7 @@
   showPinnedInsteadOfUnpin={passProps?.showPinnedInsteadOfUnpin}
   onPin={isPinned ? null : () => pinnedTables.update(list => [...list, data])}
   onUnpin={isPinned ? () => pinnedTables.update(list => list.filter(x => !testEqual(x, data))) : null}
+  extInfo={data.tableRowCount != null ? `${formatRowCount(data.tableRowCount)} rows` : null}
   on:click={() => handleClick()}
   on:middleclick={() => handleClick(true)}
   on:expand

packages/web/src/appobj/SavedFileAppObject.svelte

@@ -55,6 +55,14 @@
     currentConnection: true,
   };
 
+  const diagrams: FileTypeHandler = {
+    icon: 'img diagram',
+    format: 'json',
+    tabComponent: 'DiagramTab',
+    folder: 'diagrams',
+    currentConnection: true,
+  };
+
   const HANDLERS = {
     sql,
     shell,
@@ -62,6 +70,7 @@
     charts,
     query,
     sqlite,
+    diagrams,
   };
 
   export const extractKey = data => data.file;
@@ -74,7 +83,7 @@
   import { showModal } from '../modals/modalTools';
 
   import { currentDatabase } from '../stores';
-import { apiCall } from '../utility/api';
+  import { apiCall } from '../utility/api';
 
   import getConnectionLabel from '../utility/getConnectionLabel';
   import hasPermission from '../utility/hasPermission';

packages/web/src/buttons/CloseSearchButton.svelte

@@ -4,15 +4,17 @@
   import InlineButton from './InlineButton.svelte';
 
   export let filter;
+  export let showDisabled = false;
 </script>
 
-{#if filter}
+{#if filter || showDisabled}
   <InlineButton
     on:click
     on:click={() => {
       filter = '';
     }}
     title="Clear filter"
+    disabled={!filter}
   >
     <FontIcon icon="icon close" />
   </InlineButton>