v4.2.4-beta.2

Remove "Plans for SQLite" now that it's there

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: dbgate
+patreon: # Replace with a single Patreon username
+open_collective: dbgate
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/build-app-beta.yaml

@@ -1,4 +1,4 @@
-name: Electron app
+name: Electron app BETA
 
 on:
   push:
@@ -11,8 +11,8 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-18.04, windows-2016]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        # os: [ubuntu-18.04, windows-2016]
+        os: [macOS-10.15, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context
@@ -35,10 +35,14 @@ jobs:
       - name: fillNativeModulesElectron
         run: |
           yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
       - name: Install Snapcraft
         if: matrix.os == 'ubuntu-18.04'
         uses: samuelmeuli/action-snapcraft@v1
       - name: Publish
+        if: matrix.os != 'macOS-10.15'
         run: |
           yarn run build:app
         env:
@@ -46,6 +50,13 @@ jobs:
           WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
           WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
 
+      - name: Publish Mac
+        if: matrix.os == 'macOS-10.15'
+        run: |
+          yarn run build:app:mac
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+
       - name: Save snap login
         if: matrix.os == 'ubuntu-18.04'
         run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
@@ -63,11 +74,21 @@ jobs:
         run: |
           mkdir artifacts          
 
+          cp app/dist/*.deb artifacts/dbgate-beta.deb || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-beta.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-beta-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-beta-armv7l.AppImage || true
+          cp app/dist/*.exe artifacts/dbgate-beta.exe || true
+          cp app/dist/*win*.zip artifacts/dbgate-windows-beta.zip || true
+          cp app/dist/*-mac.dmg artifacts/dbgate-beta.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-beta-arm64.dmg || true
+
           mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
           mv app/dist/*.AppImage artifacts/ || true
           mv app/dist/*.deb artifacts/ || true
           mv app/dist/*.snap artifacts/ || true
-#          mv app/dist/*.dmg artifacts/ || true
+          mv app/dist/*.dmg artifacts/ || true
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v1

.github/workflows/build-app.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        os: [macOS-10.15, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context
@@ -39,6 +39,9 @@ jobs:
       - name: fillNativeModulesElectron
         run: |
           yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
       - name: Install Snapcraft
         if: matrix.os == 'ubuntu-18.04'
         uses: samuelmeuli/action-snapcraft@v1
@@ -72,11 +75,16 @@ jobs:
           mkdir artifacts          
 
           cp app/dist/*.deb artifacts/dbgate-latest.deb || true
-          cp app/dist/*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-latest-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-latest-armv7l.AppImage || true
           cp app/dist/*.exe artifacts/dbgate-latest.exe || true
-          cp app/dist/*.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*win*.zip artifacts/dbgate-windows-latest.zip || true
+          cp app/dist/*-mac.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-latest-arm64.dmg || true
 
           mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
           mv app/dist/*.AppImage artifacts/ || true
           mv app/dist/*.deb artifacts/ || true
           mv app/dist/*.dmg artifacts/ || true
@@ -118,7 +126,7 @@ jobs:
           mv app/dist/latest-linux.yml artifacts/latest-linux.yml || true
 
       - name: Copy latest-mac.yml
-        if: matrix.os == 'macOS-10.14'
+        if: matrix.os == 'macOS-10.15'
         run: |
           mv app/dist/latest-mac.yml artifacts/latest-mac.yml || true
 

.github/workflows/build-docker-beta.yaml

@@ -0,0 +1,47 @@
+name: Docker image BETA
+
+# on: [push]
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-18.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+      - name: Use Node.js 10.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 10.x
+      - name: yarn install
+        run: |
+          yarn install
+      - name: setCurrentVersion
+        run: |
+          yarn setCurrentVersion
+      - name: Prepare docker image
+        run: |
+          yarn run prepare:docker
+      - name: Build docker image
+        run: |
+          docker build ./docker -t dbgate
+      - name: Push docker image
+        run: |
+          docker tag dbgate dbgate/dbgate:beta
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push dbgate/dbgate:beta

.github/workflows/build-docker.yaml

@@ -21,7 +21,6 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-18.04]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context

.github/workflows/build-npm.yaml

@@ -21,7 +21,6 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-18.04]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context
@@ -119,3 +118,8 @@ jobs:
         working-directory: plugins/dbgate-plugin-postgres
         run: |
           npm publish
+
+      - name: Publish dbgate-plugin-sqlite
+        working-directory: plugins/dbgate-plugin-sqlite
+        run: |
+          npm publish

.github/workflows/integration-tests.yaml

@@ -0,0 +1,58 @@
+name: Integration tests
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    container: node:10.18-jessie
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+      - name: yarn install
+        run: |
+          yarn install
+      - name: Run tests
+        run: |
+          cd integration-tests
+          yarn test:ci
+#          yarn wait:ci
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: integration-tests/result.json
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: Pwd2020Db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+      mysql:
+        image: mysql:8.0.18
+        env:
+          MYSQL_ROOT_PASSWORD: Pwd2020Db
+
+      mssql:
+        image: mcr.microsoft.com/mssql/server
+        env:
+          ACCEPT_EULA: Y
+          SA_PASSWORD: Pwd2020Db
+          MSSQL_PID: Express
+          
+      # cockroachdb:
+      #   image: cockroachdb/cockroach

.gitignore

@@ -30,4 +30,5 @@ yarn-debug.log*
 yarn-error.log*
 app/src/nativeModulesContent.js
 packages/api/src/nativeModulesContent.js
+packages/api/src/packagedPluginsContent.js
 .VSCodeCounter

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "jestrunner.jestCommand": "node_modules/.bin/cross-env DEVMODE=1 LOCALTEST=1 node_modules/.bin/jest"
+}

CHANGELOG.md

@@ -1,5 +1,55 @@
 # ChangeLog
 
+### 4.2.2
+- CHANGED: Further startup optimalization (approx. 2 times quicker start of electron app)
+
+### 4.2.1
+- FIXED: Fixed+optimalized app startup (esp. on Windows)
+
+### 4.2.0
+- ADDED: Support of SQLite database
+- ADDED: Support of Amazon Redshift database
+- ADDED: Support of CockcroachDB
+- CHANGED: DB Model is not auto-refreshed by default, refresh could be invoked from statusbar
+- FIXED: Fixed race conditions on startup
+- FIXED: Fixed broken style in data grid under strange circumstances
+- ADDED: Configure connections with commandline arguments  #108
+- CHANGED: Optimalized algorithm of incremental DB model updates
+- CHANGED: Loading queries from PostgreSQL doesn't need cursors, using streamed query instead
+- ADDED: Disconnect command
+- ADDED: Query executed on server has tab marker (formerly it had only "No DB" marker)
+- ADDED: Horizontal scroll using shift+mouse wheel #113
+- ADDED: Cosmetic improvements of MariaDB support
+
+### 4.1.11
+- FIX: Fixed crash of API process when using SSH tunnel connection (race condition)
+
+### 4.1.11
+- FIX: fixed processing postgre query containing $$
+- FIX: fixed postgre analysing procedures & functions
+- FIX: patched svelte crash #105
+- ADDED: ability to disbale background DB model updates
+- ADDED: Duplicate connection
+- ADDED: Duplicate tab
+- FIX: SSH tunnel connection using keyfile auth #106
+- FIX: All tables button fix in export #109
+- CHANGED: Add to favorites moved from toolbar to tab context menu
+- CHANGED: Toolbar design - current tab related commands are delimited
+
+### 4.1.10
+- ADDED: Default database option in connectin settings #96 #92
+- FIX: Bundle size optimalization for Windows #97
+- FIX: Popup menu placement on smaller displays #94
+- ADDED: Browse table data with SQL Server 2008 #93
+- FIX: Prevented malicious origins / DNS rebinding #91
+- ADDED: Handle JSON fields in data editor (eg. jsonb field in Postgres) #90
+- FIX: Fixed crash on Windows with Hyper-V #86
+- ADDED: Show database server version in status bar
+- ADDED: Show detailed info about error, when connect to database fails
+- ADDED: Portable ZIP distribution for Windows #84
+### 4.1.9
+- FIX: Incorrect row count info in query result #83
+
 ### 4.1.1
 - CHANGED: Default plugins are now part of installation
 ### 4.1.0

README.md

@@ -1,8 +1,8 @@
 [![NPM version](https://img.shields.io/npm/v/dbgate.svg)](https://www.npmjs.com/package/dbgate)
+![GitHub All Releases](https://img.shields.io/github/downloads/dbgate/dbgate/total) 
 [![dbgate](https://snapcraft.io/dbgate/badge.svg)](https://snapcraft.io/dbgate)
 [![dbgate](https://snapcraft.io/dbgate/trending.svg?name=0)](https://snapcraft.io/dbgate)
 [![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
-[![Donate](https://img.shields.io/badge/donate-paypal-blue.svg)](https://paypal.me/JanProchazkaCz/30eur)
 
 # DbGate - database administration tool
 
@@ -17,28 +17,40 @@ Supported databases:
 * PostgreSQL
 * SQL Server
 * MongoDB
+* SQLite
+* Amazon Redshift
+* CockroachDB
+* MariaDB
 
 ![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png)
 
 ## Features
-* Connect to Microsoft SQL Server, Postgre SQL, MySQL, MongoDB
 * Table data editing, with SQL change script preview
+* Light and dark theme
 * Master/detail views
 * Query designer
 * Form view for comfortable work with tables with many columns
-* JSON view on MognoDB collections
+* JSON view on MongoDB collections
 * Explore tables, views, procedures, functions, MongoDB collections
-* SQL editor, execute SQL script, SQL code formatter, SQL code completion, SQL join wizard
+* SQL editor
+  * execute SQL script
+  * SQL code formatter
+  * SQL code completion
+  * Add SQL LEFT/INNER/RIGHT join utility
 * Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
 * Import, export from/to CSV, Excel, JSON
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
 * Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
-* Light and dark theme
 * Charts
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
 
+## How to contribute
+Any contributions are welcome. If you want to contribute without coding, consider following:
+* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
+* Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
+
 ## Why is DbGate different
 There are many database managers now, so why DbGate? 
 * Works everywhere - Windows, Linux, Mac, Web browser (+mobile web is planned), without compromises in features
@@ -52,7 +64,6 @@ There are many database managers now, so why DbGate?
     * Backend - NodeJs, ExpressJs, socket.io, database connection drivers
     * JavaScript + TypeScript
     * App - electron
-    * There is plan to incorporate SQLite to support work with local datasets
 * Platform independed - will run as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
 
 ## Plugins
@@ -101,15 +112,3 @@ yarn build:app:local
 yarn start:app:local
 ```
 
-## Packages
-Some dbgate packages can be used also without DbGate. You can find them on [NPM repository](https://www.npmjs.com/search?q=keywords:dbgate)
-
-* [api](https://github.com/dbgate/dbgate/tree/master/packages/api) - backend, Javascript, ExpressJS [![NPM version](https://img.shields.io/npm/v/dbgate-api.svg)](https://www.npmjs.com/package/dbgate-api)
-* [datalib](https://github.com/dbgate/dbgate/tree/master/packages/datalib) - TypeScript library for utility classes [![NPM version](https://img.shields.io/npm/v/dbgate-datalib.svg)](https://www.npmjs.com/package/dbgate-datalib)
-* [app](https://github.com/dbgate/dbgate/tree/master/app) - application (JavaScript) structure, creating specific queries (JavaScript)
-* [filterparser](https://github.com/dbgate/dbgate/tree/master/packages/filterparser) - TypeScript library for parsing data filter expressions using parsimmon [![NPM version](https://img.shields.io/npm/v/dbgate-filterparser.svg)](https://www.npmjs.com/package/dbgate-filterparser)
-* [sqltree](https://github.com/dbgate/dbgate/tree/master/packages/sqltree) - JSON representation of SQL query, functions converting to SQL (TypeScript) [![NPM version](https://img.shields.io/npm/v/dbgate-sqltree.svg)](https://www.npmjs.com/package/dbgate-sqltree)
-* [types](https://github.com/dbgate/dbgate/tree/master/packages/types) - common TypeScript definitions [![NPM version](https://img.shields.io/npm/v/dbgate-types.svg)](https://www.npmjs.com/package/dbgate-types)
-* [web](https://github.com/dbgate/dbgate/tree/master/packages/web) - frontend in Svelte (JavaScript) [![NPM version](https://img.shields.io/npm/v/dbgate-web.svg)](https://www.npmjs.com/package/dbgate-web)
-* [tools](https://github.com/dbgate/dbgate/tree/master/packages/tools) - various tools [![NPM version](https://img.shields.io/npm/v/dbgate-tools.svg)](https://www.npmjs.com/package/dbgate-tools)
-

app/package.json

@@ -5,37 +5,56 @@
   "author": "Jan Prochazka <jenasoft.database@gmail.com>",
   "description": "Opensource database administration tool",
   "dependencies": {
+    "better-sqlite3-with-prebuilds": "^7.1.8",
     "electron-log": "^4.3.1",
     "electron-store": "^5.1.1",
-    "electron-updater": "^4.3.5"
+    "electron-updater": "^4.3.5",
+    "patch-package": "^6.4.7"
   },
   "repository": {
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
   },
   "build": {
+    "artifactName": "${productName}-${version}-${os}_${arch}.${ext}",
     "appId": "org.dbgate",
     "mac": {
       "category": "database",
       "icon": "icon512.png",
-      "artifactName": "dbgate-mac-${version}.${ext}",
+      "publish": [
+        "github"
+      ],
+      "target": {
+        "target": "default",
+        "arch": [
+          "arm64",
+          "x64"
+        ]
+      }
+    },
+    "linux": {
+      "target": [
+        "deb",
+        "snap",
+        {
+          "target": "AppImage",
+          "arch": [
+            "x64",
+            "armv7l",
+            "arm64"
+          ]
+        }
+      ],
+      "icon": "icon.png",
+      "category": "Development",
+      "synopsis": "Database manager for SQL Server, MySQL, PostgreSQL, MongoDB and SQLite",
       "publish": [
         "github"
       ]
     },
-    "linux": {
-      "target": [
-        "AppImage",
-        "deb",
-        "snap"
-      ],
-      "icon": "icon.png",
-      "artifactName": "dbgate-linux-${version}.${ext}",
-      "category": "Development",
-      "synopsis": "Database administration tool for MS SQL, MySQL and PostgreSQL",
-      "publish": [
-        "github"
-      ]
+    "appImage": {
+      "license": "./LICENSE",
+      "category": "Development"
     },
     "snap": {
       "publish": [
@@ -48,9 +67,9 @@
     },
     "win": {
       "target": [
-        "nsis"
+        "nsis",
+        "zip"
       ],
-      "artifactName": "dbgate-windows-${version}.${ext}",
       "icon": "icon.ico",
       "publish": [
         "github"
@@ -69,8 +88,9 @@
     "start:local": "cross-env electron .",
     "dist": "electron-builder",
     "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist",
+    "build:mac": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && node setMacPlatform x64 && yarn dist && node setMacPlatform arm64 && yarn dist",
     "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn predist",
-    "postinstall": "electron-builder install-app-deps",
+    "postinstall": "electron-builder install-app-deps && patch-package",
     "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/public/*\" packages && copyfiles \"../packages/web/public/**/*\" packages && copyfiles --up 3 \"../plugins/dist/**/*\" packages/plugins"
   },
   "main": "src/electron.js",
@@ -78,7 +98,7 @@
     "copyfiles": "^2.2.0",
     "cross-env": "^6.0.3",
     "electron": "11.2.3",
-    "electron-builder": "22.9.1"
+    "electron-builder": "22.10.5"
   },
   "optionalDependencies": {
     "msnodesqlv8": "^2.0.10"

app/setMacPlatform.js

@@ -0,0 +1,8 @@
+const fs = require('fs');
+
+const text = fs.readFileSync('package.json', { encoding: 'utf-8' });
+const json = JSON.parse(text);
+
+json.build.mac.target.arch = process.argv[2];
+
+fs.writeFileSync('package.json', JSON.stringify(json, null, 2), { encoding: 'utf-8' });

app/src/electron.js

@@ -19,7 +19,6 @@ const store = new Store();
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
 let mainWindow;
-let splashWindow;
 let mainMenu;
 
 log.transports.file.level = 'debug';
@@ -29,14 +28,6 @@ autoUpdater.logger = log;
 
 let commands = {};
 
-function hideSplash() {
-  if (splashWindow) {
-    splashWindow.destroy();
-    splashWindow = null;
-  }
-  mainWindow.show();
-}
-
 function commandItem(id) {
   const command = commands[id];
   return {
@@ -156,7 +147,6 @@ function createWindow() {
     title: 'DbGate',
     ...bounds,
     icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
-    show: false,
     webPreferences: {
       nodeIntegration: true,
       enableRemoteModule: true,
@@ -175,7 +165,7 @@ function createWindow() {
         slashes: true,
       });
     mainWindow.webContents.on('did-finish-load', function () {
-      hideSplash();
+      // hideSplash();
     });
     mainWindow.on('close', () => {
       store.set('winBounds', mainWindow.getBounds());
@@ -186,33 +176,21 @@ function createWindow() {
     }
   }
 
-  splashWindow = new BrowserWindow({
-    width: 300,
-    height: 120,
-    transparent: true,
-    frame: false,
-  });
-  splashWindow.loadURL(
-    url.format({
-      pathname: path.join(__dirname, '../packages/web/build/splash.html'),
-      protocol: 'file:',
-      slashes: true,
-    })
-  );
-
   if (process.env.ELECTRON_START_URL) {
     loadMainWindow();
   } else {
     const apiProcess = fork(path.join(__dirname, '../packages/api/dist/bundle.js'), [
       '--dynport',
+      '--is-electron-bundle',
       '--native-modules',
       path.join(__dirname, 'nativeModules'),
       // '../../../src/nativeModules'
     ]);
     apiProcess.on('message', msg => {
       if (msg.msgtype == 'listening') {
-        const { port } = msg;
+        const { port, authorization } = msg;
         global['port'] = port;
+        global['authorization'] = authorization;
         loadMainWindow();
       }
     });

fillNativeModules.js

@@ -2,10 +2,10 @@ const fs = require('fs');
 
 let fillContent = '';
 
-// if (!process.argv.includes('--electron')) {
 if (process.platform == 'win32') {
   fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');`;
 }
+fillContent += `content['better-sqlite3-with-prebuilds'] = () => require('better-sqlite3-with-prebuilds');`;
 
 const getContent = (empty) => `
 // this file is generated automatically by script fillNativeModules.js, do not edit it manually

fillPackagedPlugins.js

@@ -0,0 +1,23 @@
+const fs = require('fs');
+const path = require('path');
+
+function load() {
+  const plugins = {};
+
+  for (const packageName of fs.readdirSync('plugins')) {
+    if (!packageName.startsWith('dbgate-plugin-')) continue;
+    const dir = path.join('plugins', packageName);
+    const frontend = fs.readFileSync(path.join(dir, 'dist', 'frontend.js'), 'utf-8');
+    const readme = fs.readFileSync(path.join(dir, 'README.md'), 'utf-8');
+    const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf-8'));
+    plugins[packageName] = {
+      manifest,
+      frontend,
+      readme,
+    };
+  }
+
+  return plugins;
+}
+
+fs.writeFileSync('packages/api/src/packagedPluginsContent.js', `module.exports = () => (${JSON.stringify(load())});`);

integration-tests/.gitignore

@@ -0,0 +1 @@
+dbtemp

integration-tests/__tests__/object-analyse.spec.js

@@ -0,0 +1,89 @@
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const _ = require('lodash');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'CREATE TABLE t2 (id int)'];
+
+function flatSource() {
+  return _.flatten(
+    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+  );
+}
+
+const obj1Match = expect.objectContaining({
+  pureName: 'obj1',
+});
+const view1Match = expect.objectContaining({
+  pureName: 'obj1',
+  columns: expect.arrayContaining([
+    expect.objectContaining({
+      columnName: 'id',
+    }),
+  ]),
+});
+
+describe('Object analyse', () => {
+  test.each(flatSource())(
+    'Full analysis - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure[type].length).toEqual(1);
+      expect(structure[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.create1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(2);
+      expect(structure2[type].find(x => x.pureName == 'obj1')).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - drop - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop2);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(1);
+      expect(structure2[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Create SQL - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      expect(structure2[type].length).toEqual(0);
+
+      await driver.query(conn, structure1[type][0].createSql);
+
+      const structure3 = await driver.analyseIncremental(conn, structure2);
+
+      expect(structure3[type].length).toEqual(1);
+      expect(structure3[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+});

integration-tests/__tests__/query.spec.js

@@ -0,0 +1,141 @@
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'INSERT INTO t1 (id) VALUES (1)', 'INSERT INTO t1 (id) VALUES (2)'];
+
+expect.extend({
+  dataRow(row, expected) {
+    for (const key in expected) {
+      if (row[key] != expected[key]) {
+        return {
+          pass: false,
+          message: () => `Different key: ${key}`,
+        };
+      }
+    }
+    return {
+      pass: true,
+      message: () => '',
+    };
+  },
+});
+
+class StreamHandler {
+  constructor(resolve) {
+    this.results = [];
+    this.resolve = resolve;
+    this.infoRows = [];
+  }
+  row(row) {
+    this.results[this.results.length - 1].rows.push(row);
+  }
+  recordset(columns) {
+    this.results.push({
+      columns,
+      rows: [],
+    });
+  }
+  done(result) {
+    this.resolve(this.results);
+  }
+  info(msg) {
+    this.infoRows.push(msg);
+  }
+}
+
+function executeStream(driver, conn, sql) {
+  return new Promise(resolve => {
+    const handler = new StreamHandler(resolve);
+    driver.stream(conn, sql, handler);
+  });
+}
+
+describe('Query', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      const res = await driver.query(conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(res.columns).toEqual([
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ]);
+
+      expect(res.rows).toEqual([
+        expect.dataRow({
+          id: 1,
+        }),
+        expect.dataRow({
+          id: 2,
+        }),
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple stream query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(driver, conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(results.length).toEqual(1);
+      const res = results[0];
+
+      expect(res.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'More queries - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(
+        driver,
+        conn,
+        'SELECT id FROM t1 ORDER BY id; SELECT id FROM t1 ORDER BY id DESC'
+      );
+      expect(results.length).toEqual(2);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+
+      const res2 = results[1];
+      expect(res2.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res2.rows).toEqual([expect.dataRow({ id: 2 }), expect.dataRow({ id: 1 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script - %s',
+    testWrapper(async (conn, driver, engine) => {
+      const results = await executeStream(
+        driver,
+        conn,
+        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
+      );
+      expect(results.length).toEqual(1);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Save data query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(
+        conn,
+        'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;'
+      );
+      const res = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
+      console.log(res);
+      expect(res.rows[0].cnt == 3).toBeTruthy();
+    })
+  );
+});

integration-tests/__tests__/table-analyse.spec.js

@@ -0,0 +1,112 @@
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+
+const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50) null)';
+const t2Sql = 'CREATE TABLE t2 (id int not null primary key, val2 varchar(50) null)';
+
+const txMatch = (tname, vcolname, nextcol) =>
+  expect.objectContaining({
+    pureName: tname,
+    columns: [
+      expect.objectContaining({
+        columnName: 'id',
+        notNull: true,
+        dataType: expect.stringContaining('int'),
+      }),
+      expect.objectContaining({
+        columnName: vcolname,
+        notNull: false,
+        dataType: expect.stringMatching(/.*char.*\(50\)/),
+      }),
+      ...(nextcol
+        ? [
+            expect.objectContaining({
+              columnName: 'nextcol',
+              notNull: false,
+              dataType: expect.stringMatching(/.*char.*\(50\)/),
+            }),
+          ]
+        : []),
+    ],
+    primaryKey: expect.objectContaining({
+      columns: [
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ],
+    }),
+  });
+
+const t1Match = txMatch('t1', 'val1');
+const t2Match = txMatch('t2', 'val2');
+const t2NextColMatch = txMatch('t2', 'val2', true);
+
+describe('Table analyse', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table structure - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure.tables.length).toEqual(1);
+      expect(structure.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table add - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(1);
+      expect(structure1.tables[0]).toEqual(t2Match);
+
+      await driver.query(conn, t1Sql);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table remove - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(2);
+      expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+
+      await driver.query(conn, 'DROP TABLE t2');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(1);
+      expect(structure2.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table change - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+
+      if (engine.dbSnapshotBySeconds) await new Promise(resolve => setTimeout(resolve, 1100));
+      
+      await driver.query(conn, 'ALTER TABLE t2 ADD nextcol varchar(50)');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2).toBeTruthy(); // if falsy, no modification is detected
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch);
+    })
+  );
+});

integration-tests/docker-compose.yaml

@@ -0,0 +1,55 @@
+version: '3'
+services:
+  postgres:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: Pwd2020Db
+    ports: 
+      - 15000:5432
+
+  mysql:
+    image: mysql
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    ports: 
+      - 15001:3306
+    environment:
+      - MYSQL_ROOT_PASSWORD=Pwd2020Db
+
+  mssql:
+    image: mcr.microsoft.com/mssql/server
+    restart: always
+    ports: 
+      - 15002:1433
+    environment:
+      - ACCEPT_EULA=Y
+      - SA_PASSWORD=Pwd2020Db
+      - MSSQL_PID=Express
+      
+  cockroachdb:
+    image: cockroachdb/cockroach
+    ports:
+      - 15003:26257
+    command: start-single-node --insecure
+
+#   mongodb:
+#     image: mongo:4.0.12
+#     restart: always
+#     volumes:
+#       - mongo-data:/data/db
+#       - mongo-config:/data/configdb
+#     ports:
+#       - 27017:27017
+
+
+#   cockroachdb-init:
+#     image: cockroachdb/cockroach
+#     # build: cockroach
+#     # entrypoint: /cockroach/init.sh 
+#     entrypoint: ./cockroach sql --insecure --host="cockroachdb" --execute="CREATE DATABASE IF NOT EXISTS test;"
+
+#     depends_on:
+#       - cockroachdb
+#     restart: on-failure
+

integration-tests/engines.js

@@ -0,0 +1,108 @@
+const views = {
+  type: 'views',
+  create1: 'CREATE VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP VIEW obj1',
+  drop2: 'DROP VIEW obj2',
+};
+const matviews = {
+  type: 'matviews',
+  create1: 'CREATE MATERIALIZED VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE MATERIALIZED VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP MATERIALIZED VIEW obj1',
+  drop2: 'DROP MATERIALIZED VIEW obj2',
+};
+
+const engines = [
+  {
+    label: 'MySQL',
+    connection: {
+      engine: 'mysql@dbgate-plugin-mysql',
+      password: 'Pwd2020Db',
+      user: 'root',
+      server: 'mysql',
+      port: 3306,
+    },
+    local: {
+      server: 'localhost',
+      port: 15001,
+    },
+    // skipOnCI: true,
+    objects: [views],
+    dbSnapshotBySeconds: true,
+  },
+  {
+    label: 'PostgreSQL',
+    connection: {
+      engine: 'postgres@dbgate-plugin-postgres',
+      password: 'Pwd2020Db',
+      user: 'postgres',
+      server: 'postgres',
+      port: 5432,
+    },
+    local: {
+      server: 'localhost',
+      port: 15000,
+    },
+    objects: [
+      views,
+      matviews,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1() LANGUAGE SQL AS $$  select * from t1 $$',
+        create2: 'CREATE PROCEDURE obj2() LANGUAGE SQL AS $$  select * from t2 $$',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQL Server',
+    connection: {
+      engine: 'mssql@dbgate-plugin-mssql',
+      password: 'Pwd2020Db',
+      user: 'sa',
+      server: 'mssql',
+      port: 1433,
+    },
+    local: {
+      server: 'localhost',
+      port: 15002,
+    },
+    objects: [
+      views,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1 AS SELECT id FROM t1',
+        create2: 'CREATE PROCEDURE obj2 AS SELECT id FROM t2',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQLite',
+    generateDbFile: true,
+    connection: {
+      engine: 'sqlite@dbgate-plugin-sqlite',
+    },
+    objects: [views],
+  },
+  {
+    label: 'CockroachDB',
+    connection: {
+      engine: 'cockroach@dbgate-plugin-postgres',
+      user: 'root',
+      server: 'cockroachdb',
+      port: 26257,
+    },
+    local: {
+      server: 'localhost',
+      port: 15003,
+    },
+    skipOnCI: true,
+    objects: [views, matviews],
+  },
+];
+
+module.exports = process.env.CITEST ? engines.filter(x => !x.skipOnCI) : engines;

integration-tests/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "dbgate-integration-tests",
+  "version": "4.1.1",
+  "homepage": "https://dbgate.org/",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate.git"
+  },
+  "author": "Jan Prochazka",
+  "license": "MIT",
+  "scripts": {
+    "wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js",
+    "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
+
+    "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
+    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
+    
+    "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
+  },
+  "devDependencies": {
+    "cross-env": "^7.0.3",
+    "jest": "^27.0.1"
+  },
+  "dependencies": {}
+}

integration-tests/tools.js

@@ -0,0 +1,63 @@
+global.DBGATE_TOOLS = require('dbgate-tools');
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const crypto = require('crypto');
+
+function randomDbName() {
+  const generatedKey = crypto.randomBytes(6);
+  const newKey = generatedKey.toString('hex');
+  return `db${newKey}`;
+}
+
+function extractConnection(engine) {
+  const { connection } = engine;
+
+  if (process.env.LOCALTEST && engine.local) {
+    return {
+      ...connection,
+      ...engine.local,
+    };
+  }
+
+  return connection;
+}
+
+async function connect(engine, database) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+
+  if (engine.generateDbFile) {
+    const conn = await driver.connect({
+      ...connection,
+      databaseFile: `dbtemp/${database}`,
+    });
+    return conn;
+  } else {
+    const conn = await driver.connect(connection);
+    await driver.query(conn, `CREATE DATABASE ${database}`);
+    await driver.close(conn);
+
+    const res = await driver.connect({
+      ...connection,
+      database,
+    });
+    return res;
+  }
+}
+
+const testWrapper = body => async (label, ...other) => {
+  const engine = other[other.length - 1];
+  const driver = requireEngineDriver(engine.connection);
+  const conn = await connect(engine, randomDbName());
+  try {
+    await body(conn, driver, ...other);
+  } finally {
+    await driver.close(conn);
+  }
+};
+
+module.exports = {
+  randomDbName,
+  connect,
+  extractConnection,
+  testWrapper,
+};

integration-tests/wait.js

@@ -0,0 +1,29 @@
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const engines = require('./engines');
+const { extractConnection } = require('./tools');
+global.DBGATE_TOOLS = require('dbgate-tools');
+
+async function connectEngine(engine) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+  for (;;) {
+    try {
+      const conn = await driver.connect(connection);
+      await driver.getVersion(conn);
+      console.log(`Connect to ${engine.label} - OK`);
+      await driver.close(conn);
+      return;
+    } catch (err) {
+      console.log(`Waiting for ${engine.label}, error: ${err.message}`);
+      await new Promise(resolve => setTimeout(resolve, 2500));
+      continue;
+    }
+  }
+}
+
+async function run() {
+  await new Promise(resolve => setTimeout(resolve, 10000));
+  await Promise.all(engines.map(engine => connectEngine(engine)));
+}
+
+run();

package.json

@@ -1,10 +1,11 @@
 {
   "private": true,
-  "version": "4.1.2",
+  "version": "4.2.4-beta.2",
   "name": "dbgate-all",
   "workspaces": [
     "packages/*",
-    "plugins/*"
+    "plugins/*",
+    "integration-tests"
   ],
   "scripts": {
     "start:api": "yarn workspace dbgate-api start",
@@ -21,6 +22,7 @@
     "build:tools": "yarn workspace dbgate-tools build",
     "build:lib": "yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
     "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
+    "build:app:mac": "yarn plugins:copydist && cd app && yarn install && yarn build:mac",
     "build:api": "yarn workspace dbgate-api build",
     "build:web:docker": "yarn workspace dbgate-web build",
     "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
@@ -31,17 +33,18 @@
     "setCurrentVersion": "node setCurrentVersion",
     "generatePadFile": "node generatePadFile",
     "fillNativeModules": "node fillNativeModules",
-    "fillNativeModulesElectron": "node fillNativeModules --eletron",
+    "fillNativeModulesElectron": "node fillNativeModules --electron",
+    "fillPackagedPlugins": "node fillPackagedPlugins",
+    "resetPackagedPlugins": "node resetPackagedPlugins",
     "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
     "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
     "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build",
-    "prepare": "yarn build:lib",
     "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
     "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn build:plugins:frontend:watch\"",
     "ts:api": "yarn workspace dbgate-api ts",
     "ts:web": "yarn workspace dbgate-web ts",
     "ts": "yarn ts:api && yarn ts:web",
-    "postinstall": "patch-package && yarn fillNativeModules && yarn build:plugins:frontend"
+    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend"
   },
   "dependencies": {
     "concurrently": "^5.1.0",

packages/api/.covid-env

@@ -1,12 +0,0 @@
-CONNECTIONS=mysql
-
-LABEL_mysql=MySql
-SERVER_mysql=dbgate.org
-USER_mysql=reader
-PASSWORD_mysql=CovidReader2020
-PORT_mysql=3326
-ENGINE_mysql=mysql@dbgate-plugin-mysql
-SINGLE_CONNECTION=mysql
-SINGLE_DATABASE=covid
-
-PERMISSIONS=files/charts/read

packages/api/.env

@@ -1,15 +1 @@
-CONNECTIONS=mysql,postgres
-
-LABEL_mysql=MySql localhost
-SERVER_mysql=localhost
-USER_mysql=root
-PASSWORD_mysql=test
-PORT_mysql=3307
-ENGINE_mysql=mysql@dbgate-plugin-mysql
-
-LABEL_postgres=Postgres localhost
-SERVER_postgres=localhost
-USER_postgres=postgres
-PASSWORD_postgres=test
-PORT_postgres=5433
-ENGINE_postgres=postgres@dbgate-plugin-postgres
+DEVMODE=1

packages/api/.env-portal

@@ -0,0 +1,17 @@
+DEVMODE=1
+
+CONNECTIONS=mysql,postgres
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+LABEL_postgres=Postgres localhost
+SERVER_postgres=localhost
+USER_postgres=postgres
+PASSWORD_postgres=test
+PORT_postgres=5433
+ENGINE_postgres=postgres@dbgate-plugin-postgres

packages/api/.env-singledb

@@ -0,0 +1,15 @@
+DEVMODE=1
+
+CONNECTIONS=mysql
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+SINGLE_CONNECTION=mysql
+SINGLE_DATABASE=Chinook
+
+PERMISSIONS=files/charts/read

packages/api/package.json

@@ -7,7 +7,6 @@
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
   },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
   "author": "Jan Prochazka",
   "license": "MIT",
   "keywords": [
@@ -19,7 +18,8 @@
   ],
   "dependencies": {
     "async-lock": "^1.2.4",
-    "axios": "^0.19.0",
+    "axios": "^0.21.1",
+    "better-sqlite3-with-prebuilds": "^7.1.8",
     "body-parser": "^1.19.0",
     "bufferutil": "^4.0.1",
     "byline": "^5.0.0",
@@ -32,12 +32,12 @@
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
     "express-fileupload": "^1.2.0",
-    "find-free-port": "^2.0.0",
     "fs-extra": "^9.1.0",
+    "get-port": "^5.1.1",
     "http": "^0.0.0",
     "json-stable-stringify": "^1.0.1",
     "line-reader": "^0.4.0",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.21",
     "ncp": "^2.0.0",
     "nedb-promises": "^4.0.1",
     "node-cron": "^2.0.3",
@@ -49,9 +49,11 @@
     "uuid": "^3.4.0"
   },
   "scripts": {
-    "start": "node src/index.js",
-    "start:portal": "env-cmd nodemon src/index.js",
-    "start:covid": "env-cmd -f .covid-env nodemon src/index.js",
+    "start": "env-cmd node src/index.js",
+    "start:portal": "env-cmd -f .env-portal node src/index.js",
+    "start:singledb": "env-cmd -f .env-singledb node src/index.js",
+    "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db",
+    "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test",
     "ts": "tsc",
     "build": "webpack"
   },

packages/api/src/controllers/config.js

@@ -1,33 +1,32 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { datadir } = require('../utility/directories');
+const hasPermission = require('../utility/hasPermission');
+const socket = require('../utility/socket');
+const _ = require('lodash');
+
 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
+const connections = require('../controllers/connections');
 
 module.exports = {
+  settingsValue: {},
+
+  async _init() {
+    try {
+      this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
+    } catch (err) {
+      this.settingsValue = {};
+    }
+  },
+
   get_meta: 'get',
   async get() {
-    // const toolbarButtons = process.env.TOOLBAR;
-    // const toolbar = toolbarButtons
-    //   ? toolbarButtons.split(',').map((name) => ({
-    //       name,
-    //       icon: process.env[`ICON_${name}`],
-    //       title: process.env[`TITLE_${name}`],
-    //       page: process.env[`PAGE_${name}`],
-    //     }))
-    //   : null;
-    // const startupPages = process.env.STARTUP_PAGES ? process.env.STARTUP_PAGES.split(',') : [];
     const permissions = process.env.PERMISSIONS ? process.env.PERMISSIONS.split(',') : null;
-    const singleDatabase =
-      process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE
-        ? {
-            conid: process.env.SINGLE_CONNECTION,
-            database: process.env.SINGLE_DATABASE,
-          }
-        : null;
 
     return {
-      runAsPortal: !!process.env.CONNECTIONS,
-      // toolbar,
-      // startupPages,
-      singleDatabase,
+      runAsPortal: !!connections.portalConnections,
+      singleDatabase: connections.singleDatabase,
       permissions,
       ...currentVersion,
     };
@@ -37,5 +36,26 @@ module.exports = {
   async platformInfo() {
     return platformInfo;
   },
-  
+
+  getSettings_meta: 'get',
+  async getSettings() {
+    return this.settingsValue;
+  },
+
+  updateSettings_meta: 'post',
+  async updateSettings(values) {
+    if (!hasPermission(`settings/change`)) return false;
+    try {
+      const updated = {
+        ...this.settingsValue,
+        ...values,
+      };
+      await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
+      this.settingsValue = updated;
+      socket.emitChanged(`settings-changed`);
+      return updated;
+    } catch (err) {
+      return false;
+    }
+  },
 };

packages/api/src/controllers/connections.js

@@ -8,6 +8,32 @@ const socket = require('../utility/socket');
 const { encryptConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');
 
+function getNamedArgs() {
+  const res = {};
+  for (let i = 0; i < process.argv.length; i++) {
+    const name = process.argv[i];
+    if (name.startsWith('--')) {
+      let value = process.argv[i + 1];
+      if (value && value.startsWith('--')) value = null;
+      res[name.substring(2)] = value == null ? true : value;
+      i++;
+    } else {
+      if (name.endsWith('.db') || name.endsWith('.sqlite') || name.endsWith('.sqlite3')) {
+        res.databaseFile = name;
+        res.engine = 'sqlite@dbgate-plugin-sqlite';
+      }
+    }
+  }
+  return res;
+}
+
+function getDatabaseFileLabel(databaseFile) {
+  if (!databaseFile) return databaseFile;
+  const m = databaseFile.match(/[\/]([^\/]+)$/);
+  if (m) return m[1];
+  return databaseFile;
+}
+
 function getPortalCollections() {
   if (process.env.CONNECTIONS) {
     return _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
@@ -17,16 +43,79 @@ function getPortalCollections() {
       user: process.env[`USER_${id}`],
       password: process.env[`PASSWORD_${id}`],
       port: process.env[`PORT_${id}`],
+      databaseUrl: process.env[`URL_${id}`],
+      databaseFile: process.env[`FILE_${id}`],
+      defaultDatabase: process.env[`DATABASE_${id}`],
+      singleDatabase: !!process.env[`DATABASE_${id}`],
       displayName: process.env[`LABEL_${id}`],
     }));
   }
+
+  const args = getNamedArgs();
+  if (args.databaseFile) {
+    return [
+      {
+        _id: 'argv',
+        databaseFile: args.databaseFile,
+        singleDatabase: true,
+        defaultDatabase: getDatabaseFileLabel(args.databaseFile),
+        engine: args.engine,
+      },
+    ];
+  }
+  if (args.databaseUrl) {
+    return [
+      {
+        _id: 'argv',
+        useDatabaseUrl: true,
+        ...args,
+      },
+    ];
+  }
+  if (args.server) {
+    return [
+      {
+        _id: 'argv',
+        ...args,
+      },
+    ];
+  }
+
   return null;
 }
 const portalConnections = getPortalCollections();
 
+function getSingleDatabase() {
+  if (process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE) {
+    // @ts-ignore
+    const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
+    return {
+      connection,
+      name: process.env.SINGLE_DATABASE,
+    };
+  }
+  // @ts-ignore
+  const arg0 = (portalConnections || []).find(x => x._id == 'argv');
+  if (arg0) {
+    // @ts-ignore
+    if (arg0.singleDatabase) {
+      return {
+        connection: arg0,
+        // @ts-ignore
+        name: arg0.defaultDatabase,
+      };
+    }
+  }
+  return null;
+}
+
+const singleDatabase = getSingleDatabase();
+
 module.exports = {
   datastore: null,
   opened: [],
+  singleDatabase,
+  portalConnections,
 
   async _init() {
     const dir = datadir();
@@ -46,7 +135,7 @@ module.exports = {
     raw: true,
   },
   test(req, res) {
-    const subprocess = fork(process.argv[1], ['connectProcess', ...process.argv.slice(3)]);
+    const subprocess = fork(process.argv[1], ['--start-process', 'connectProcess', ...process.argv.slice(3)]);
     subprocess.on('message', resp => {
       if (handleProcessCommunication(resp, subprocess)) return;
       // @ts-ignore

packages/api/src/controllers/databaseConnections.js

@@ -4,6 +4,7 @@ const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const { DatabaseAnalyser } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
+const config = require('./config');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -17,6 +18,19 @@ module.exports = {
     existing.structure = structure;
     socket.emitChanged(`database-structure-changed-${conid}-${database}`);
   },
+  handle_structureTime(conid, database, { analysedTime }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.analysedTime = analysedTime;
+    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+  },
+  handle_version(conid, database, { version }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.serverVersion = version;
+    socket.emitChanged(`database-server-version-changed-${conid}-${database}`);
+  },
+
   handle_error(conid, database, props) {
     const { error } = props;
     console.log(`Error in database connection ${conid}, database ${database}: ${error}`);
@@ -40,13 +54,18 @@ module.exports = {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
     const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['databaseConnectionProcess', ...process.argv.slice(3)]);
+    const subprocess = fork(process.argv[1], [
+      '--start-process',
+      'databaseConnectionProcess',
+      ...process.argv.slice(3),
+    ]);
     const lastClosed = this.closed[`${conid}/${database}`];
     const newOpened = {
       conid,
       database,
       subprocess,
       structure: lastClosed ? lastClosed.structure : DatabaseAnalyser.createEmptyStructure(),
+      serverVersion: lastClosed ? lastClosed.serverVersion : null,
       connection,
       status: { name: 'pending' },
     };
@@ -67,6 +86,7 @@ module.exports = {
       msgtype: 'connect',
       connection: { ...connection, database },
       structure: lastClosed ? lastClosed.structure : null,
+      globalSettings: config.settingsValue,
     });
     return newOpened;
   },
@@ -109,9 +129,19 @@ module.exports = {
   status_meta: 'get',
   async status({ conid, database }) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
-    if (existing) return existing.status;
+    if (existing) {
+      return {
+        ...existing.status,
+        analysedTime: existing.analysedTime,
+      };
+    }
     const lastClosed = this.closed[`${conid}/${database}`];
-    if (lastClosed) return lastClosed.status;
+    if (lastClosed) {
+      return {
+        ...lastClosed.status,
+        analysedTime: lastClosed.analysedTime,
+      };
+    }
     return {
       name: 'error',
       message: 'Not connected',
@@ -127,7 +157,7 @@ module.exports = {
     } else {
       existing = await this.ensureOpened(conid, database);
     }
-    
+
     return {
       status: 'ok',
       connectionStatus: existing ? existing.status : null,
@@ -135,13 +165,20 @@ module.exports = {
   },
 
   refresh_meta: 'post',
-  async refresh({ conid, database }) {
-    this.close(conid, database);
+  async refresh({ conid, database, keepOpen }) {
+    if (!keepOpen) this.close(conid, database);
 
     await this.ensureOpened(conid, database);
     return { status: 'ok' };
   },
 
+  syncModel_meta: 'post',
+  async syncModel({ conid, database }) {
+    const conn = await this.ensureOpened(conid, database);
+    conn.subprocess.send({ msgtype: 'syncModel' });
+    return { status: 'ok' };
+  },
+
   close(conid, database, kill = true) {
     const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {
@@ -159,6 +196,12 @@ module.exports = {
     }
   },
 
+  disconnect_meta: 'post',
+  async disconnect({ conid, database }) {
+    await this.close(conid, database, true);
+    return { status: 'ok' };
+  },
+
   structure_meta: 'get',
   async structure({ conid, database }) {
     const opened = await this.ensureOpened(conid, database);
@@ -171,6 +214,12 @@ module.exports = {
     // };
   },
 
+  serverVersion_meta: 'get',
+  async serverVersion({ conid, database }) {
+    const opened = await this.ensureOpened(conid, database);
+    return opened.serverVersion;
+  },
+
   sqlPreview_meta: 'post',
   async sqlPreview({ conid, database, objects, options }) {
     // wait for structure

packages/api/src/controllers/plugins.js

@@ -9,10 +9,17 @@ const requirePlugin = require('../shell/requirePlugin');
 const downloadPackage = require('../utility/downloadPackage');
 const hasPermission = require('../utility/hasPermission');
 const _ = require('lodash');
+const packagedPluginsContent = require('../packagedPluginsContent');
 
 module.exports = {
   script_meta: 'get',
   async script({ packageName }) {
+    const packagedContent = packagedPluginsContent();
+
+    if (packagedContent && packagedContent[packageName]) {
+      return packagedContent[packageName].frontend;
+    }
+
     const file1 = path.join(packagedPluginsDir(), packageName, 'dist', 'frontend.js');
     const file2 = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
     // @ts-ignore
@@ -58,25 +65,40 @@ module.exports = {
 
   installed_meta: 'get',
   async installed() {
-    const files1 = await fs.readdir(packagedPluginsDir());
+    const packagedContent = packagedPluginsContent();
+
+    const files1 = packagedContent ? _.keys(packagedContent) : await fs.readdir(packagedPluginsDir());
     const files2 = await fs.readdir(pluginsdir());
 
     const res = [];
     for (const packageName of _.union(files1, files2)) {
       if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
-      const isPackaged = files1.includes(packageName);
-      const manifest = await fs
-        .readFile(path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'package.json'), {
-          encoding: 'utf-8',
-        })
-        .then(x => JSON.parse(x));
-      const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
-      // @ts-ignore
-      if (await fs.exists(readmeFile)) {
-        manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+      try {
+        if (packagedContent && packagedContent[packageName]) {
+          const manifest = {
+            ...packagedContent[packageName].manifest,
+          };
+          manifest.isPackaged = true;
+          manifest.readme = packagedContent[packageName].readme;
+          res.push(manifest);
+        } else {
+          const isPackaged = files1.includes(packageName);
+          const manifest = await fs
+            .readFile(path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'package.json'), {
+              encoding: 'utf-8',
+            })
+            .then(x => JSON.parse(x));
+          const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
+          // @ts-ignore
+          if (await fs.exists(readmeFile)) {
+            manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+          }
+          manifest.isPackaged = isPackaged;
+          res.push(manifest);
+        }
+      } catch (err) {
+        console.log(`Skipped plugin ${packageName}, error:`, err.message);
       }
-      manifest.isPackaged = isPackaged;
-      res.push(manifest);
     }
     return res;
   },
@@ -131,8 +153,9 @@ module.exports = {
   async authTypes({ engine }) {
     const packageName = extractPackageName(engine);
     const content = requirePlugin(packageName);
-    if (!content.driver || content.driver.engine != engine || !content.driver.getAuthTypes) return null;
-    return content.driver.getAuthTypes() || null;
+    const driver = content.drivers.find(x => x.engine == engine);
+    if (!driver || !driver.getAuthTypes) return null;
+    return driver.getAuthTypes() || null;
   },
 
   // async _init() {

packages/api/src/controllers/runners.js

@@ -5,7 +5,7 @@ const uuidv1 = require('uuid/v1');
 const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
-const { rundir, uploadsdir, pluginsdir } = require('../utility/directories');
+const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
 const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
 
@@ -92,7 +92,7 @@ module.exports = {
     const scriptFile = path.join(uploadsdir(), runid + '.js');
     fs.writeFileSync(`${scriptFile}`, scriptText);
     fs.mkdirSync(directory);
-    const pluginNames = fs.readdirSync(pluginsdir());
+    const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
     console.log(`RUNNING SCRIPT ${scriptFile}`);
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
     const subprocess = fork(scriptFile, ['--checkParent', ...process.argv.slice(3)], {
@@ -101,7 +101,7 @@ module.exports = {
       env: {
         ...process.env,
         DBGATE_API: global['dbgateApiModulePath'] || process.argv[1],
-        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, path.join(pluginsdir(), name)])),
+        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
       },
     });
     const pipeDispatcher = severity => data =>

packages/api/src/controllers/serverConnections.js

@@ -5,6 +5,7 @@ const _ = require('lodash');
 const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
 const lock = new AsyncLock();
+const config = require('./config');
 
 module.exports = {
   opened: [],
@@ -17,6 +18,12 @@ module.exports = {
     existing.databases = databases;
     socket.emitChanged(`database-list-changed-${conid}`);
   },
+  handle_version(conid, { version }) {
+    const existing = this.opened.find(x => x.conid == conid);
+    if (!existing) return;
+    existing.version = version;
+    socket.emitChanged(`server-version-changed-${conid}`);
+  },
   handle_status(conid, { status }) {
     const existing = this.opened.find(x => x.conid == conid);
     if (!existing) return;
@@ -30,7 +37,11 @@ module.exports = {
       const existing = this.opened.find(x => x.conid == conid);
       if (existing) return existing;
       const connection = await connections.get({ conid });
-      const subprocess = fork(process.argv[1], ['serverConnectionProcess', ...process.argv.slice(3)]);
+      const subprocess = fork(process.argv[1], [
+        '--start-process',
+        'serverConnectionProcess',
+        ...process.argv.slice(3),
+      ]);
       const newOpened = {
         conid,
         subprocess,
@@ -55,7 +66,7 @@ module.exports = {
         if (newOpened.disconnected) return;
         this.close(conid, false);
       });
-      subprocess.send({ msgtype: 'connect', ...connection });
+      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: config.settingsValue });
       return newOpened;
     });
     return res;
@@ -75,12 +86,24 @@ module.exports = {
     }
   },
 
+  disconnect_meta: 'post',
+  async disconnect({ conid }) {
+    await this.close(conid, true);
+    return { status: 'ok' };
+  },
+
   listDatabases_meta: 'get',
   async listDatabases({ conid }) {
     const opened = await this.ensureOpened(conid);
     return opened.databases;
   },
 
+  version_meta: 'get',
+  async version({ conid }) {
+    const opened = await this.ensureOpened(conid);
+    return opened.version;
+  },
+
   serverStatus_meta: 'get',
   async serverStatus() {
     return {
@@ -106,8 +129,8 @@ module.exports = {
   },
 
   refresh_meta: 'post',
-  async refresh({ conid }) {
-    this.close(conid);
+  async refresh({ conid, keepOpen }) {
+    if (!keepOpen) this.close(conid);
 
     await this.ensureOpened(conid);
     return { status: 'ok' };

packages/api/src/controllers/sessions.js

@@ -65,7 +65,7 @@ module.exports = {
   async create({ conid, database }) {
     const sesid = uuidv1();
     const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['sessionProcess', ...process.argv.slice(3)]);
+    const subprocess = fork(process.argv[1], ['--start-process', 'sessionProcess', ...process.argv.slice(3)]);
     const newOpened = {
       conid,
       database,

packages/api/src/index.js

@@ -1,15 +1,17 @@
 const shell = require('./shell');
+const processArgs = require('./utility/processArgs');
+const dbgateTools = require('dbgate-tools');
 
-const argument = process.argv[2];
-if (argument && argument.endsWith('Process')) {
+global['DBGATE_TOOLS'] = dbgateTools;
+
+if (processArgs.startProcess) {
   const proc = require('./proc');
-
-  const module = proc[argument];
+  const module = proc[processArgs.startProcess];
   module.start();
-} else if (!module['parent'] && !process.argv.includes('--checkParent')) {
+} else if (!module['parent'] && !processArgs.checkParent) {
   const main = require('./main');
 
-  main.start(argument);
+  main.start();
 }
 
 module.exports = {

packages/api/src/main.js

@@ -6,9 +6,10 @@ const http = require('http');
 const cors = require('cors');
 const io = require('socket.io');
 const fs = require('fs');
-const findFreePort = require('find-free-port');
+const getPort = require('get-port');
 const childProcessChecker = require('./utility/childProcessChecker');
 const path = require('path');
+const crypto = require('crypto');
 
 const useController = require('./utility/useController');
 const socket = require('./utility/socket');
@@ -28,8 +29,14 @@ const files = require('./controllers/files');
 const scheduler = require('./controllers/scheduler');
 
 const { rundir } = require('./utility/directories');
+const platformInfo = require('./utility/platformInfo');
+const processArgs = require('./utility/processArgs');
+const timingSafeCheckToken = require('./utility/timingSafeCheckToken');
 
-function start(argument = null) {
+let authorization = null;
+let checkLocalhostOrigin = null;
+
+function start() {
   // console.log('process.argv', process.argv);
 
   const app = express();
@@ -49,6 +56,29 @@ function start(argument = null) {
     );
   }
 
+  app.use(function (req, res, next) {
+    if (authorization && !timingSafeCheckToken(req.headers.authorization, authorization)) {
+      return res.status(403).json({ error: 'Not authorized!' });
+    }
+    if (checkLocalhostOrigin) {
+      if (
+        req.headers.origin &&
+        req.headers.origin != checkLocalhostOrigin &&
+        req.headers.origin != `http://${checkLocalhostOrigin}`
+      ) {
+        console.log('API origin check FAILED');
+        console.log('HEADERS', { ...req.headers, authorization: '***' });
+        return res.status(403).json({ error: 'Not authorized!' });
+      }
+      if (!req.headers.origin && req.headers.host != checkLocalhostOrigin) {
+        console.log('API host check FAILED');
+        console.log('HEADERS', { ...req.headers, authorization: '***' });
+        return res.status(403).json({ error: 'Not authorized!' });
+      }
+    }
+    next();
+  });
+
   app.use(cors());
   app.use(bodyParser.json({ limit: '50mb' }));
 
@@ -79,29 +109,32 @@ function start(argument = null) {
 
   app.use('/runners/data', express.static(rundir()));
 
-  if (fs.existsSync('/home/dbgate-docker/public')) {
+  if (platformInfo.isDocker) {
     // server static files inside docker container
     app.use(express.static('/home/dbgate-docker/public'));
   } else {
-    if (argument != 'startNodeWeb') {
+    if (!platformInfo.isNpmDist) {
       app.get('/', (req, res) => {
         res.send('DbGate API');
       });
     }
   }
 
-  if (argument == '--dynport') {
+  if (processArgs.dynport) {
     childProcessChecker();
 
-    findFreePort(53911, function (err, port) {
+    authorization = crypto.randomBytes(32).toString('hex');
+
+    getPort().then(port => {
+      checkLocalhostOrigin = `localhost:${port}`;
       server.listen(port, () => {
         console.log(`DbGate API listening on port ${port}`);
-        process.send({ msgtype: 'listening', port });
+        process.send({ msgtype: 'listening', port, authorization });
       });
     });
-  } else if (argument == 'startNodeWeb') {
+  } else if (platformInfo.isNpmDist) {
     app.use(express.static(path.join(__dirname, '../../dbgate-web/public')));
-    findFreePort(5000, function (err, port) {
+    getPort({ port: 5000 }).then(port => {
       server.listen(port, () => {
         console.log(`DbGate API listening on port ${port}`);
       });

packages/api/src/proc/connectProcess.js

@@ -2,6 +2,25 @@ const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const _ = require('lodash');
+
+function pickSafeConnectionInfo(connection) {
+  return _.mapValues(connection, (v, k) => {
+    if (k == 'engine' || k == 'port' || k == 'authType' || k == 'sshMode' || k == 'passwordMode') return v;
+    if (v === null || v === true || v === false) return v;
+    if (v) return '***';
+    return undefined;
+  });
+}
+
+const formatErrorDetail = (e, connection) => `${e.stack}
+
+Error JSON: ${JSON.stringify(e, undefined, 2)}
+
+Connection: ${JSON.stringify(pickSafeConnectionInfo(connection), undefined, 2)}
+
+Platform: ${process.platform}
+`;
 
 function start() {
   childProcessChecker();
@@ -14,7 +33,11 @@ function start() {
       process.send({ msgtype: 'connected', ...res });
     } catch (e) {
       console.error(e);
-      process.send({ msgtype: 'error', error: e.message });
+      process.send({
+        msgtype: 'error',
+        error: e.message,
+        detail: formatErrorDetail(e, connection),
+      });
     }
   });
 }

packages/api/src/proc/databaseConnectionProcess.js

@@ -1,5 +1,6 @@
 const stableStringify = require('json-stable-stringify');
 const childProcessChecker = require('../utility/childProcessChecker');
+const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -11,6 +12,7 @@ let afterConnectCallbacks = [];
 let analysedStructure = null;
 let lastPing = null;
 let lastStatus = null;
+let analysedTime = 0;
 
 async function checkedAsyncCall(promise) {
   try {
@@ -27,21 +29,42 @@ async function checkedAsyncCall(promise) {
   }
 }
 
+let loadingModel = false;
+
 async function handleFullRefresh() {
+  loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
+  setStatusName('loadStructure');
   analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection));
+  analysedTime = new Date().getTime();
   process.send({ msgtype: 'structure', structure: analysedStructure });
+  process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
+  loadingModel = false;
 }
 
-async function handleIncrementalRefresh() {
+async function handleIncrementalRefresh(forceSend) {
+  loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
+  setStatusName('checkStructure');
   const newStructure = await checkedAsyncCall(driver.analyseIncremental(systemConnection, analysedStructure));
+  analysedTime = new Date().getTime();
   if (newStructure != null) {
     analysedStructure = newStructure;
+  }
+  
+  if (forceSend || newStructure != null) {
     process.send({ msgtype: 'structure', structure: analysedStructure });
   }
+
+  process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
+  loadingModel = false;
+}
+
+function handleSyncModel() {
+  if (loadingModel) return;
+  handleIncrementalRefresh();
 }
 
 function setStatus(status) {
@@ -56,20 +79,34 @@ function setStatusName(name) {
   setStatus({ name });
 }
 
-async function handleConnect({ connection, structure }) {
+async function readVersion() {
+  const driver = requireEngineDriver(storedConnection);
+  const version = await driver.getVersion(systemConnection);
+  process.send({ msgtype: 'version', version });
+}
+
+async function handleConnect({ connection, structure, globalSettings }) {
   storedConnection = connection;
   lastPing = new Date().getTime();
 
   if (!structure) setStatusName('pending');
   const driver = requireEngineDriver(storedConnection);
   systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection));
+  readVersion();
   if (structure) {
     analysedStructure = structure;
-    handleIncrementalRefresh();
+    handleIncrementalRefresh(true);
   } else {
     handleFullRefresh();
   }
-  setInterval(handleIncrementalRefresh, 30 * 1000);
+
+  if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
+    setInterval(
+      handleIncrementalRefresh,
+      extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 3, 3600) * 1000
+    );
+  }
+
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
   }
@@ -155,6 +192,7 @@ const messageHandlers = {
   collectionData: handleCollectionData,
   sqlPreview: handleSqlPreview,
   ping: handlePing,
+  syncModel: handleSyncModel,
   // runCommand: handleRunCommand,
 };
 

packages/api/src/proc/serverConnectionProcess.js

@@ -1,4 +1,5 @@
 const stableStringify = require('json-stable-stringify');
+const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
@@ -31,6 +32,12 @@ async function handleRefresh() {
   }
 }
 
+async function readVersion() {
+  const driver = requireEngineDriver(storedConnection);
+  const version = await driver.getVersion(systemConnection);
+  process.send({ msgtype: 'version', version });
+}
+
 function setStatus(status) {
   const statusString = stableStringify(status);
   if (lastStatus != statusString) {
@@ -45,14 +52,18 @@ function setStatusName(name) {
 
 async function handleConnect(connection) {
   storedConnection = connection;
+  const { globalSettings } = storedConnection;
   setStatusName('pending');
   lastPing = new Date().getTime();
 
   const driver = requireEngineDriver(storedConnection);
   try {
     systemConnection = await connectUtility(driver, storedConnection);
+    readVersion();
     handleRefresh();
-    setInterval(handleRefresh, 30 * 1000);
+    if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
+      setInterval(handleRefresh, extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000);
+    }
   } catch (err) {
     setStatus({
       name: 'error',

packages/api/src/shell/requirePlugin.js

@@ -1,8 +1,8 @@
 const path = require('path');
 const fs = require('fs');
-const { pluginsdir, packagedPluginsDir } = require('../utility/directories');
+const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories');
 const nativeModules = require('../nativeModules');
-const _isRunOnSource = require('../utility/_isRunOnSource');
+const platformInfo = require('../utility/platformInfo');
 
 const loadedPlugins = {};
 
@@ -10,32 +10,19 @@ const dbgateEnv = {
   dbgateApi: null,
   nativeModules,
 };
-
-function getModulePath(packageName) {
-  const packagedModulePath = _isRunOnSource()
-    ? path.join(packagedPluginsDir(), packageName, 'src', 'backend', 'index.js')
-    : path.join(packagedPluginsDir(), packageName, 'dist', 'backend.js');
-
-  if (fs.existsSync(packagedModulePath)) {
-    return packagedModulePath;
-  }
-
-  return path.join(pluginsdir(), packageName, 'dist', 'backend.js');
-}
-
 function requirePlugin(packageName, requiredPlugin = null) {
   if (!packageName) throw new Error('Missing packageName in plugin');
   if (loadedPlugins[packageName]) return loadedPlugins[packageName];
 
   if (requiredPlugin == null) {
     let module;
-    const modulePath = getModulePath(packageName);
+    const modulePath = getPluginBackendPath(packageName);
     console.log(`Loading module ${packageName} from ${modulePath}`);
     try {
       // @ts-ignore
       module = __non_webpack_require__(modulePath);
     } catch (err) {
-      console.log('Failed load webpacked module', err.message);
+      // console.log('Failed load webpacked module', err.message);
       module = require(modulePath);
     }
     requiredPlugin = module.__esModule ? module.default : module;

packages/api/src/shell/runScript.js

@@ -1,7 +1,8 @@
 const childProcessChecker = require('../utility/childProcessChecker');
+const processArgs = require('../utility/processArgs');
 
 async function runScript(func) {
-  if (process.argv.includes('--checkParent')) {
+  if (processArgs.checkParent) {
     childProcessChecker();
   }
   try {

packages/api/src/utility/DatastoreProxy.js

@@ -29,7 +29,7 @@ class DatastoreProxy {
 
   async ensureSubprocess() {
     if (!this.subprocess) {
-      this.subprocess = fork(process.argv[1], ['jslDatastoreProcess', ...process.argv.slice(3)]);
+      this.subprocess = fork(process.argv[1], ['--start-process', 'jslDatastoreProcess', ...process.argv.slice(3)]);
 
       this.subprocess.on('message', message => {
         // @ts-ignore

packages/api/src/utility/_isRunOnSource.js

@@ -1,5 +0,0 @@
-function _isRunOnSource() {
-  return __filename.endsWith('_isRunOnSource.js');
-}
-
-module.exports = _isRunOnSource;

packages/api/src/utility/connectUtility.js

@@ -7,6 +7,7 @@ const { getSshTunnelProxy } = require('./sshTunnelProxy');
 
 async function connectUtility(driver, storedConnection) {
   const connection = {
+    database: storedConnection.defaultDatabase,
     ...decryptConnection(storedConnection),
   };
 

packages/api/src/utility/directories.js

@@ -2,7 +2,6 @@ const os = require('os');
 const path = require('path');
 const fs = require('fs');
 const cleanDirectory = require('./cleanDirectory');
-const _isRunOnSource = require('./_isRunOnSource');
 const platformInfo = require('./platformInfo');
 
 const createDirectories = {};
@@ -42,17 +41,34 @@ const archivedir = dirFunc('archive');
 const filesdir = dirFunc('files');
 
 function packagedPluginsDir() {
-  if (_isRunOnSource()) {
+  if (platformInfo.isDevMode) {
     return path.resolve(__dirname, '../../../../plugins');
   }
   if (platformInfo.isDocker) {
     return '/home/dbgate-docker/plugins';
   }
-  if (process.argv[2] == 'startNodeWeb') {
+  if (platformInfo.isNpmDist) {
     // node_modules
     return global['dbgateApiPackagedPluginsPath'];
   }
-  return path.resolve(__dirname, '../../plugins');
+  if (platformInfo.isElectronBundle) {
+    return path.resolve(__dirname, '../../plugins');
+  }
+  return null;
+}
+
+const packagedPluginList =
+  packagedPluginsDir() != null ? fs.readdirSync(packagedPluginsDir()).filter(x => x.startsWith('dbgate-plugin-')) : [];
+
+function getPluginBackendPath(packageName) {
+  if (packagedPluginList.includes(packageName)) {
+    if (platformInfo.isDevMode) {
+      return path.join(packagedPluginsDir(), packageName, 'src', 'backend', 'index.js');
+    }
+    return path.join(packagedPluginsDir(), packageName, 'dist', 'backend.js');
+  }
+
+  return path.join(pluginsdir(), packageName, 'dist', 'backend.js');
 }
 
 module.exports = {
@@ -65,4 +81,6 @@ module.exports = {
   pluginsdir,
   filesdir,
   packagedPluginsDir,
+  packagedPluginList,
+  getPluginBackendPath,
 };

packages/api/src/utility/platformInfo.js

@@ -1,26 +1,42 @@
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
+const processArgs = require('./processArgs');
 
-const p = process;
-const platform = p.env.OS_OVERRIDE ? p.env.OS_OVERRIDE : p.platform;
+const platform = process.env.OS_OVERRIDE ? process.env.OS_OVERRIDE : process.platform;
 const isWindows = platform === 'win32';
 const isMac = platform === 'darwin';
 const isLinux = platform === 'linux';
-const isDocker = fs.existsSync('/home/dbgate-docker/build');
+const isDocker = fs.existsSync('/home/dbgate-docker/public');
+const isDevMode = process.env.DEVMODE == '1';
+const isNpmDist = !!global['dbgateApiModulePath'];
+
+// function moduleAvailable(name) {
+//   try {
+//     require.resolve(name);
+//     return true;
+//   } catch (e) {
+//     return false;
+//   }
+// }
+
+const isElectronBundle = processArgs.isElectronBundle;
 
 const platformInfo = {
   isWindows,
   isMac,
   isLinux,
   isDocker,
-  isSnap: p.env.ELECTRON_SNAP == 'true',
-  isPortable: isWindows && p.env.PORTABLE_EXECUTABLE_DIR,
-  isAppImage: p.env.DESKTOPINTEGRATION === 'AppImageLauncher',
-  sshAuthSock: p.env.SSH_AUTH_SOCK,
+  isElectronBundle,
+  isDevMode,
+  isNpmDist,
+  isSnap: process.env.ELECTRON_SNAP == 'true',
+  isPortable: isWindows && process.env.PORTABLE_EXECUTABLE_DIR,
+  isAppImage: process.env.DESKTOPINTEGRATION === 'AppImageLauncher',
+  sshAuthSock: process.env.SSH_AUTH_SOCK,
   environment: process.env.NODE_ENV,
   platform,
-  runningInWebpack: !!p.env.WEBPACK_DEV_SERVER_URL,
+  runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
   defaultKeyFile: path.join(os.homedir(), '.ssh/id_rsa'),
 };
 

packages/api/src/utility/processArgs.js

@@ -0,0 +1,21 @@
+function getNamedArg(name) {
+  const argIndex = process.argv.indexOf(name);
+  if (argIndex > 0) {
+    return process.argv[argIndex + 1];
+  }
+  return null;
+}
+
+const checkParent = process.argv.includes('--checkParent');
+const dynport = process.argv.includes('--dynport');
+const nativeModules = getNamedArg('--native-modules');
+const startProcess = getNamedArg('--start-process');
+const isElectronBundle = process.argv.includes('--is-electron-bundle');
+
+module.exports = {
+  checkParent,
+  nativeModules,
+  startProcess,
+  dynport,
+  isElectronBundle,
+};

packages/api/src/utility/requireEngineDriver.js

@@ -15,7 +15,7 @@ function requireEngineDriver(connection) {
   if (engine.includes('@')) {
     const [shortName, packageName] = engine.split('@');
     const plugin = requirePlugin(packageName);
-    return plugin.driver;
+    return plugin.drivers.find(x => x.engine == engine);
   }
   throw new Error(`Could not found engine driver ${engine}`);
 }

packages/api/src/utility/sshTunnel.js

@@ -4,6 +4,8 @@ const portfinder = require('portfinder');
 const stableStringify = require('json-stable-stringify');
 const _ = require('lodash');
 const platformInfo = require('./platformInfo');
+const AsyncLock = require('async-lock');
+const lock = new AsyncLock();
 
 const sshConnectionCache = {};
 const sshTunnelCache = {};
@@ -34,7 +36,7 @@ async function getSshConnection(connection) {
     password: connection.sshMode == 'userPassword' ? connection.sshPassword : undefined,
     agentSocket: connection.sshMode == 'agent' ? platformInfo.sshAuthSock : undefined,
     privateKey:
-      connection.sshMode == 'keyFile' && connection.sshKeyFile ? await fs.readFile(connection.sshKeyFile) : undefined,
+      connection.sshMode == 'keyFile' && connection.sshKeyfile ? await fs.readFile(connection.sshKeyfile) : undefined,
     skipAutoPrivateKey: true,
     noReadline: true,
   };
@@ -45,36 +47,43 @@ async function getSshConnection(connection) {
 }
 
 async function getSshTunnel(connection) {
-  const sshConn = await getSshConnection(connection);
   const tunnelCacheKey = stableStringify(_.pick(connection, TUNNEL_FIELDS));
-  if (sshTunnelCache[tunnelCacheKey]) return sshTunnelCache[tunnelCacheKey];
 
-  const localPort = await portfinder.getPortPromise({ port: 10000, stopPort: 60000 });
-  // workaround for `getPortPromise` not releasing the port quickly enough
-  await new Promise(resolve => setTimeout(resolve, 500));
-  const tunnelConfig = {
-    fromPort: localPort,
-    toPort: connection.port,
-    toHost: connection.server,
-  };
-  try {
-    const tunnel = await sshConn.forward(tunnelConfig);
-    console.log(
-      `Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
-    );
+  return await lock.acquire(tunnelCacheKey, async () => {
+    const sshConn = await getSshConnection(connection);
+    if (sshTunnelCache[tunnelCacheKey]) return sshTunnelCache[tunnelCacheKey];
 
-    sshTunnelCache[tunnelCacheKey] = {
-      state: 'ok',
-      localPort,
+    const localPort = await portfinder.getPortPromise({ port: 10000, stopPort: 60000 });
+    // workaround for `getPortPromise` not releasing the port quickly enough
+    await new Promise(resolve => setTimeout(resolve, 500));
+    const tunnelConfig = {
+      fromPort: localPort,
+      toPort: connection.port,
+      toHost: connection.server,
     };
-    return sshTunnelCache[tunnelCacheKey];
-  } catch (err) {
-    // error is not cached
-    return {
-      state: 'error',
-      message: err.message,
-    };
-  }
+    try {
+      console.log(
+        `Creating SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
+      );
+
+      const tunnel = await sshConn.forward(tunnelConfig);
+      console.log(
+        `Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
+      );
+
+      sshTunnelCache[tunnelCacheKey] = {
+        state: 'ok',
+        localPort,
+      };
+      return sshTunnelCache[tunnelCacheKey];
+    } catch (err) {
+      // error is not cached
+      return {
+        state: 'error',
+        message: err.message,
+      };
+    }
+  });
 }
 
 module.exports = {

packages/api/src/utility/timingSafeCheckToken.js

@@ -0,0 +1,9 @@
+const crypto = require('crypto');
+
+function timingSafeCheckToken(a, b) {
+  if (!a || !b) return false;
+  if (a.length != b.length) return false;
+  return crypto.timingSafeEqual(Buffer.from(a), Buffer.from(b));
+}
+
+module.exports = timingSafeCheckToken;

packages/datalib/package.json

@@ -4,7 +4,6 @@
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
   "scripts": {
-    "prepare": "yarn build",
     "build": "tsc",
     "start": "tsc --watch"
   },

packages/datalib/src/FormViewDisplay.ts

@@ -1,6 +1,14 @@
 import _ from 'lodash';
 import { GridConfig, GridCache, GridConfigColumns, createGridCache, GroupFunc } from './GridConfig';
-import { ForeignKeyInfo, TableInfo, ColumnInfo, EngineDriver, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
+import {
+  ForeignKeyInfo,
+  TableInfo,
+  ColumnInfo,
+  EngineDriver,
+  NamedObjectInfo,
+  DatabaseInfo,
+  SqlDialect,
+} from 'dbgate-types';
 import { parseFilter, getFilterType, getFilterValueExpression } from 'dbgate-filterparser';
 import { filterName } from './filterName';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
@@ -12,6 +20,7 @@ export class FormViewDisplay {
   isLoadedCorrectly = true;
   columns: DisplayColumn[];
   public baseTable: TableInfo;
+  dialect: SqlDialect;
 
   constructor(
     public config: GridConfig,
@@ -19,8 +28,11 @@ export class FormViewDisplay {
     public cache: GridCache,
     protected setCache: ChangeCacheFunc,
     public driver?: EngineDriver,
-    public dbinfo: DatabaseInfo = null
-  ) {}
+    public dbinfo: DatabaseInfo = null,
+    public serverVersion = null
+  ) {
+    this.dialect = (driver?.dialectByVersion && driver?.dialectByVersion(serverVersion)) || driver?.dialect;
+  }
 
   addFilterColumn(column) {
     if (!column) return;

packages/datalib/src/GridConfig.ts

@@ -33,6 +33,7 @@ export interface GridConfig extends GridConfigColumns {
   formViewKey?: { [uniqueName: string]: string };
   formViewKeyRequested?: { [uniqueName: string]: string };
   formFilterColumns: string[];
+  formColumnFilterText?: string;
 }
 
 export interface GridCache {

packages/datalib/src/GridDisplay.ts

@@ -8,6 +8,7 @@ import {
   NamedObjectInfo,
   DatabaseInfo,
   CollectionInfo,
+  SqlDialect,
 } from 'dbgate-types';
 import { parseFilter, getFilterType } from 'dbgate-filterparser';
 import { filterName } from './filterName';
@@ -60,8 +61,12 @@ export abstract class GridDisplay {
     public cache: GridCache,
     protected setCache: ChangeCacheFunc,
     public driver?: EngineDriver,
-    public dbinfo: DatabaseInfo = null
-  ) {}
+    public dbinfo: DatabaseInfo = null,
+    public serverVersion = null
+  ) {
+    this.dialect = (driver?.dialectByVersion && driver?.dialectByVersion(serverVersion)) || driver?.dialect;
+  }
+  dialect: SqlDialect;
   columns: DisplayColumn[];
   baseTable?: TableInfo;
   baseCollection?: CollectionInfo;
@@ -283,8 +288,8 @@ export abstract class GridDisplay {
     return this.config.expandedColumns.includes(uniqueName);
   }
 
-  toggleExpandedColumn(uniqueName: string) {
-    this.includeInColumnSet('expandedColumns', uniqueName, !this.isExpandedColumn(uniqueName));
+  toggleExpandedColumn(uniqueName: string, value?: boolean) {
+    this.includeInColumnSet('expandedColumns', uniqueName, value == null ? !this.isExpandedColumn(uniqueName) : value);
   }
 
   getFilter(uniqueName: string) {
@@ -460,12 +465,75 @@ export abstract class GridDisplay {
     return select;
   }
 
+  getRowNumberOverSelect(select: Select, offset: number, count: number): Select {
+    const innerSelect: Select = {
+      commandType: 'select',
+      from: select.from,
+      where: select.where,
+      columns: [
+        ...select.columns,
+        {
+          alias: '_rowNumber',
+          exprType: 'rowNumber',
+          orderBy: select.orderBy
+            ? select.orderBy.map(x =>
+                x.exprType != 'column'
+                  ? x
+                  : x.source
+                  ? x
+                  : {
+                      ...x,
+                      source: { alias: 'basetbl' },
+                    }
+              )
+            : [
+                {
+                  ...select.columns[0],
+                  direction: 'ASC',
+                },
+              ],
+        },
+      ],
+    };
+
+    const res: Select = {
+      commandType: 'select',
+      selectAll: true,
+      from: {
+        subQuery: innerSelect,
+        alias: '_RowNumberResult',
+      },
+      where: {
+        conditionType: 'between',
+        expr: {
+          exprType: 'column',
+          columnName: '_RowNumber',
+          source: {
+            alias: '_RowNumberResult',
+          },
+        },
+        left: {
+          exprType: 'value',
+          value: offset + 1,
+        },
+        right: {
+          exprType: 'value',
+          value: offset + count,
+        },
+      },
+    };
+
+    return res;
+  }
+
   getPageQuery(offset: number, count: number) {
     if (!this.driver) return null;
-    const select = this.createSelect();
+    let select = this.createSelect();
     if (!select) return null;
-    if (this.driver.dialect.rangeSelect) select.range = { offset: offset, limit: count };
-    else if (this.driver.dialect.limitSelect) select.topRecords = count;
+    if (this.dialect.rangeSelect) select.range = { offset: offset, limit: count };
+    else if (this.dialect.rowNumberOverPaging && offset > 0)
+      select = this.getRowNumberOverSelect(select, offset, count);
+    else if (this.dialect.limitSelect) select.topRecords = count;
     const sql = treeToSql(this.driver, select, dumpSqlSelect);
     return sql;
   }

packages/datalib/src/TableFormViewDisplay.ts

@@ -28,10 +28,22 @@ export class TableFormViewDisplay extends FormViewDisplay {
     setConfig: ChangeConfigFunc,
     cache: GridCache,
     setCache: ChangeCacheFunc,
-    dbinfo: DatabaseInfo
+    dbinfo: DatabaseInfo,
+    displayOptions,
+    serverVersion
   ) {
-    super(config, setConfig, cache, setCache, driver, dbinfo);
-    this.gridDisplay = new TableGridDisplay(tableName, driver, config, setConfig, cache, setCache, dbinfo);
+    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
+    this.gridDisplay = new TableGridDisplay(
+      tableName,
+      driver,
+      config,
+      setConfig,
+      cache,
+      setCache,
+      dbinfo,
+      displayOptions,
+      serverVersion
+    );
     this.gridDisplay.addAllExpandedColumnsToSelected = true;
 
     this.isLoadedCorrectly = this.gridDisplay.isLoadedCorrectly && !!this.driver;
@@ -253,8 +265,8 @@ export class TableFormViewDisplay extends FormViewDisplay {
     };
   }
 
-  toggleExpandedColumn(uniqueName: string) {
-    this.gridDisplay.toggleExpandedColumn(uniqueName);
+  toggleExpandedColumn(uniqueName: string, value?: boolean) {
+    this.gridDisplay.toggleExpandedColumn(uniqueName, value);
     this.gridDisplay.reload();
   }
 

packages/datalib/src/TableGridDisplay.ts

@@ -17,9 +17,11 @@ export class TableGridDisplay extends GridDisplay {
     setConfig: ChangeConfigFunc,
     cache: GridCache,
     setCache: ChangeCacheFunc,
-    dbinfo: DatabaseInfo
+    dbinfo: DatabaseInfo,
+    public displayOptions: any,
+    serverVersion
   ) {
-    super(config, setConfig, cache, setCache, driver, dbinfo);
+    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
 
     this.table = this.findTable(tableName);
     if (!this.table) {
@@ -168,7 +170,7 @@ export class TableGridDisplay extends GridDisplay {
 
   processReferences(select: Select, displayedColumnInfo: DisplayedColumnInfo, options) {
     this.addJoinsFromExpandedColumns(select, this.columns, 'basetbl', displayedColumnInfo);
-    if (!options.isExport) {
+    if (!options.isExport && this.displayOptions.showHintColumns) {
       this.addHintsToSelect(select);
     }
   }

packages/datalib/src/ViewGridDisplay.ts

@@ -10,9 +10,10 @@ export class ViewGridDisplay extends GridDisplay {
     config: GridConfig,
     setConfig: ChangeConfigFunc,
     cache: GridCache,
-    setCache: ChangeCacheFunc
+    setCache: ChangeCacheFunc,
+    serverVersion
   ) {
-    super(config, setConfig, cache, setCache, driver);
+    super(config, setConfig, cache, setCache, driver, serverVersion);
     this.columns = this.getDisplayColumns(view);
     this.filterable = true;
     this.sortable = true;

packages/dbgate/README.md

@@ -1,5 +1,4 @@
 [![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
-[![Donate](https://img.shields.io/badge/donate-paypal-blue.svg)](https://paypal.me/JanProchazkaCz/30eur)
 [![NPM version](https://img.shields.io/npm/v/dbgate.svg)](https://www.npmjs.com/package/dbgate)
 
 # DbGate - database administration tool

packages/dbgate/bin/dbgate.js

@@ -1,9 +1,10 @@
 #!/usr/bin/env node
 
 const path = require('path');
-const dbgateApi = require('dbgate-api');
 
-global.dbgateApiModulePath = require.resolve('dbgate-api');
+global.dbgateApiModulePath = path.dirname(path.dirname(require.resolve('dbgate-api')));
 global.dbgateApiPackagedPluginsPath = path.dirname(global.dbgateApiModulePath);
 
-dbgateApi.getMainModule().start('startNodeWeb');
+const dbgateApi = require('dbgate-api');
+
+dbgateApi.getMainModule().start();

packages/dbgate/package.json

@@ -6,7 +6,6 @@
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
   },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
   "description": "Opensource database administration tool - web interface",
   "author": "Jan Prochazka",
   "license": "MIT",

packages/filterparser/package.json

@@ -4,7 +4,6 @@
   "main": "lib/index.js",
   "typings": "lib/index.d.ts",
   "scripts": {
-    "prepare": "yarn build",
     "build": "tsc",
     "start": "tsc --watch",
     "test": "jest"
@@ -23,7 +22,7 @@
   "dependencies": {
     "@types/parsimmon": "^1.10.1",
     "dbgate-tools": "^4.1.1",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.21",
     "moment": "^2.24.0",
     "parsimmon": "^1.13.0"
   }

packages/sqltree/package.json

@@ -8,7 +8,6 @@
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
   },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
   "author": "Jan Prochazka",
   "license": "MIT",
   "keywords": [
@@ -20,7 +19,6 @@
     "dbgate"
   ],
   "scripts": {
-    "prepare": "yarn build",
     "build": "tsc",
     "start": "tsc --watch"
   },
@@ -33,6 +31,6 @@
     "typescript": "^3.7.5"
   },
   "dependencies": {
-    "lodash": "^4.17.15"
+    "lodash": "^4.17.21"
   }
 }

packages/sqltree/src/dumpSqlCondition.ts

@@ -62,5 +62,12 @@ export function dumpSqlCondition(dmp: SqlDumper, condition: Condition) {
       dumpSqlSelect(dmp, condition.subQuery);
       dmp.put(')');
       break;
+    case 'between':
+      dumpSqlExpression(dmp, condition.expr);
+      dmp.put(' ^between ');
+      dumpSqlExpression(dmp, condition.left);
+      dmp.put(' ^and ');
+      dumpSqlExpression(dmp, condition.right);
+      break;
   }
 }

packages/sqltree/src/dumpSqlExpression.ts

@@ -38,5 +38,14 @@ export function dumpSqlExpression(dmp: SqlDumper, expr: Expression) {
     case 'transform':
       dmp.transform(expr.transform, () => dumpSqlExpression(dmp, expr.expr));
       break;
+
+    case 'rowNumber':
+      dmp.put(" ^row_number() ^over (^order ^by ");
+      dmp.putCollection(', ', expr.orderBy, x => {
+        dumpSqlExpression(dmp, x);
+        dmp.put(' %k', x.direction);
+      });
+      dmp.put(")");
+      break;
   }
 }

packages/sqltree/src/types.ts

@@ -92,6 +92,13 @@ export interface NotExistsCondition {
   subQuery: Select;
 }
 
+export interface BetweenCondition {
+  conditionType: 'between';
+  expr: Expression;
+  left: Expression;
+  right: Expression;
+}
+
 export type Condition =
   | BinaryCondition
   | NotCondition
@@ -99,7 +106,8 @@ export type Condition =
   | CompoudCondition
   | LikeCondition
   | ExistsCondition
-  | NotExistsCondition;
+  | NotExistsCondition
+  | BetweenCondition;
 
 export interface Source {
   name?: NamedObjectInfo;
@@ -153,13 +161,19 @@ export interface TranformExpression {
   transform: TransformType;
 }
 
+export interface RowNumberExpression {
+  exprType: 'rowNumber';
+  orderBy: OrderByExpression[];
+}
+
 export type Expression =
   | ColumnRefExpression
   | ValueExpression
   | PlaceholderExpression
   | RawExpression
   | CallExpression
-  | TranformExpression;
+  | TranformExpression
+  | RowNumberExpression;
 export type OrderByExpression = Expression & { direction: 'ASC' | 'DESC' };
 
 export type ResultField = Expression & { alias?: string };

packages/tools/package.json

@@ -8,7 +8,6 @@
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
   },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
   "author": "Jan Prochazka",
   "license": "MIT",
   "keywords": [
@@ -16,7 +15,6 @@
     "dbgate"
   ],
   "scripts": {
-    "prepare": "yarn build",
     "build": "tsc",
     "start": "tsc --watch",
     "test": "jest",
@@ -33,6 +31,6 @@
     "typescript": "^3.7.5"
   },
   "dependencies": {
-    "lodash": "^4.17.15"
+    "lodash": "^4.17.21"
   }
 }

packages/tools/src/DatabaseAnalyser.ts

@@ -2,12 +2,14 @@ import { DatabaseInfo, DatabaseModification, EngineDriver } from 'dbgate-types';
 import _sortBy from 'lodash/sortBy';
 import _groupBy from 'lodash/groupBy';
 import _pick from 'lodash/pick';
+import _compact from 'lodash/compact';
 
 const fp_pick = arg => array => _pick(array, arg);
 export class DatabaseAnalyser {
   structure: DatabaseInfo;
   modifications: DatabaseModification[];
   singleObjectFilter: any;
+  singleObjectId: string = null;
 
   constructor(public pool, public driver: EngineDriver) {}
 
@@ -15,15 +17,30 @@ export class DatabaseAnalyser {
     return DatabaseAnalyser.createEmptyStructure();
   }
 
-  /** @returns {Promise<import('dbgate-types').DatabaseModification[]>} */
-  async getModifications() {
-    if (this.structure == null) throw new Error('DatabaseAnalyse.getModifications - structure must be filled');
-
+  async _getFastSnapshot(): Promise<DatabaseInfo> {
     return null;
   }
 
+  async _computeSingleObjectId() {}
+
   async fullAnalysis() {
-    return this._runAnalysis();
+    const res = await this._runAnalysis();
+    // console.log('FULL ANALYSIS', res);
+    return res;
+  }
+
+  async singleObjectAnalysis(name, typeField) {
+    // console.log('Analysing SINGLE OBJECT', name, typeField);
+    this.singleObjectFilter = { ...name, typeField };
+    await this._computeSingleObjectId();
+    const res = await this._runAnalysis();
+    // console.log('SINGLE OBJECT RES', res);
+    const obj =
+      res[typeField]?.length == 1
+        ? res[typeField][0]
+        : res[typeField]?.find(x => x.pureName == name.pureName && x.schemaName == name.schemaName);
+    // console.log('SINGLE OBJECT', obj);
+    return obj;
   }
 
   async incrementalAnalysis(structure) {
@@ -37,7 +54,7 @@ export class DatabaseAnalyser {
     }
     if (this.modifications.length == 0) return null;
     console.log('DB modifications detected:', this.modifications);
-    return this._runAnalysis();
+    return this.mergeAnalyseResult(await this._runAnalysis());
   }
 
   mergeAnalyseResult(newlyAnalysed) {
@@ -49,7 +66,7 @@ export class DatabaseAnalyser {
     }
 
     const res = {};
-    for (const field of ['tables', 'collections', 'views', 'functions', 'procedures', 'triggers']) {
+    for (const field of ['tables', 'collections', 'views', 'matviews', 'functions', 'procedures', 'triggers']) {
       const removedIds = this.modifications
         .filter(x => x.action == 'remove' && x.objectTypeField == field)
         .map(x => x.objectId);
@@ -57,9 +74,19 @@ export class DatabaseAnalyser {
       const addedChangedIds = newArray.map(x => x.objectId);
       const removeAllIds = [...removedIds, ...addedChangedIds];
       res[field] = _sortBy(
-        [...this.structure[field].filter(x => !removeAllIds.includes(x.objectId)), ...newArray],
+        [...(this.structure[field] || []).filter(x => !removeAllIds.includes(x.objectId)), ...newArray],
         x => x.pureName
       );
+
+      // merge missing data from old structure
+      for (const item of res[field]) {
+        const original = (this.structure[field] || []).find(x => x.objectId == item.objectId);
+        if (original) {
+          for (const key in original) {
+            if (!item[key]) item[key] = original[key];
+          }
+        }
+      }
     }
 
     return res;
@@ -71,15 +98,125 @@ export class DatabaseAnalyser {
     // }
   }
 
+  getRequestedObjectPureNames(objectTypeField, allPureNames) {
+    if (this.singleObjectFilter) {
+      const { typeField, pureName } = this.singleObjectFilter;
+      if (typeField == objectTypeField) return [pureName];
+    }
+    if (this.modifications) {
+      return this.modifications
+        .filter(x => x.objectTypeField == objectTypeField)
+        .filter(x => x.newName)
+        .map(x => x.newName.pureName);
+    }
+    return allPureNames;
+  }
+
   // findObjectById(id) {
   //   return this.structure.tables.find((x) => x.objectId == id);
   // }
 
+  createQuery(template, typeFields) {
+    // let res = template;
+    if (this.singleObjectFilter) {
+      const { typeField } = this.singleObjectFilter;
+      if (!this.singleObjectId) return null;
+      if (!typeFields || !typeFields.includes(typeField)) return null;
+      return template.replace(/=OBJECT_ID_CONDITION/g, ` = '${this.singleObjectId}'`);
+    }
+    if (!this.modifications || !typeFields || this.modifications.length == 0) {
+      return template.replace(/=OBJECT_ID_CONDITION/g, ' is not null');
+    }
+    if (this.modifications.some(x => typeFields.includes(x.objectTypeField) && x.action == 'all')) {
+      // do not filter objects
+      return template.replace(/=OBJECT_ID_CONDITION/g, ' is not null');
+    }
+    const filterIds = this.modifications
+      .filter(x => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change'))
+      .map(x => x.objectId);
+    if (filterIds.length == 0) {
+      return template.replace(/=OBJECT_ID_CONDITION/g, " = '0'");
+    }
+    return template.replace(/=OBJECT_ID_CONDITION/g, ` in (${filterIds.map(x => `'${x}'`).join(',')})`);
+  }
+
+  getDeletedObjectsForField(snapshot, objectTypeField) {
+    const items = snapshot[objectTypeField];
+    if (!items) return [];
+    if (!this.structure[objectTypeField]) return [];
+    return this.structure[objectTypeField]
+      .filter(x => !items.find(y => x.objectId == y.objectId))
+      .map(x => ({
+        oldName: _pick(x, ['schemaName', 'pureName']),
+        objectId: x.objectId,
+        action: 'remove',
+        objectTypeField,
+      }));
+  }
+
+  getDeletedObjects(snapshot) {
+    return [
+      ...this.getDeletedObjectsForField(snapshot, 'tables'),
+      ...this.getDeletedObjectsForField(snapshot, 'collections'),
+      ...this.getDeletedObjectsForField(snapshot, 'views'),
+      ...this.getDeletedObjectsForField(snapshot, 'matviews'),
+      ...this.getDeletedObjectsForField(snapshot, 'procedures'),
+      ...this.getDeletedObjectsForField(snapshot, 'functions'),
+      ...this.getDeletedObjectsForField(snapshot, 'triggers'),
+    ];
+  }
+
+  async getModifications() {
+    const snapshot = await this._getFastSnapshot();
+    if (!snapshot) return null;
+
+    // console.log('STRUCTURE', this.structure);
+    // console.log('SNAPSHOT', snapshot);
+
+    const res = [];
+    for (const field in snapshot) {
+      const items = snapshot[field];
+      if (items === null) {
+        res.push({ objectTypeField: field, action: 'all' });
+        continue;
+      }
+      if (items === undefined) {
+        // skip - undefined meens, that field is not supported
+        continue;
+      }
+      for (const item of items) {
+        const { objectId, schemaName, pureName, contentHash } = item;
+        const obj = this.structure[field].find(x => x.objectId == objectId);
+
+        if (obj && contentHash && obj.contentHash == contentHash) continue;
+
+        const action = obj
+          ? {
+              newName: { schemaName, pureName },
+              oldName: _pick(obj, ['schemaName', 'pureName']),
+              action: 'change',
+              objectTypeField: field,
+              objectId,
+            }
+          : {
+              newName: { schemaName, pureName },
+              action: 'add',
+              objectTypeField: field,
+              objectId,
+            };
+        res.push(action);
+      }
+    }
+
+    return [..._compact(res), ...this.getDeletedObjects(snapshot)];
+  }
+
   static createEmptyStructure(): DatabaseInfo {
     return {
       tables: [],
       collections: [],
       views: [],
+      matviews: [],
       functions: [],
       procedures: [],
       triggers: [],

packages/tools/src/SqlDumper.ts

@@ -293,6 +293,20 @@ export class SqlDumper {
   changeViewSchema(obj: ViewInfo, newSchema: string) {}
   renameView(obj: ViewInfo, newSchema: string) {}
 
+  createMatview(obj: ViewInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropMatview(obj: ViewInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^materialized ^view  %f', obj);
+  }
+  alterMatview(obj: ViewInfo) {
+    this.putRaw(obj.createSql.replace(/create\s+view/i, 'ALTER VIEW'));
+    this.endCommand();
+  }
+  changeMatviewSchema(obj: ViewInfo, newSchema: string) {}
+  renameMatview(obj: ViewInfo, newSchema: string) {}
+
   createProcedure(obj: ProcedureInfo) {
     this.putRaw(obj.createSql);
     this.endCommand();

packages/tools/src/SqlGenerator.ts

@@ -7,7 +7,8 @@ import {
   TriggerInfo,
   ViewInfo,
 } from 'dbgate-types';
-import _ from 'lodash';
+import _flatten from 'lodash/flatten';
+import _uniqBy from 'lodash/uniqBy'
 import { SqlDumper } from './SqlDumper';
 import { extendDatabaseInfo } from './structureTools';
 
@@ -29,6 +30,10 @@ interface SqlGeneratorOptions {
   checkIfViewExists: boolean;
   createViews: boolean;
 
+  dropMatviews: boolean;
+  checkIfMatviewExists: boolean;
+  createMatviews: boolean;
+
   dropProcedures: boolean;
   checkIfProcedureExists: boolean;
   createProcedures: boolean;
@@ -51,6 +56,7 @@ interface SqlGeneratorObject {
 export class SqlGenerator {
   private tables: TableInfo[];
   private views: ViewInfo[];
+  private matviews: ViewInfo[];
   private procedures: ProcedureInfo[];
   private functions: FunctionInfo[];
   private triggers: TriggerInfo[];
@@ -69,6 +75,7 @@ export class SqlGenerator {
     this.dbinfo = extendDatabaseInfo(dbinfo);
     this.tables = this.extract('tables');
     this.views = this.extract('views');
+    this.matviews = this.extract('matviews');
     this.procedures = this.extract('procedures');
     this.functions = this.extract('functions');
     this.triggers = this.extract('triggers');
@@ -89,6 +96,8 @@ export class SqlGenerator {
       if (this.checkDumper()) return;
       this.dropObjects(this.views, 'View');
       if (this.checkDumper()) return;
+      this.dropObjects(this.matviews, 'Matview');
+      if (this.checkDumper()) return;
       this.dropObjects(this.triggers, 'Trigger');
       if (this.checkDumper()) return;
 
@@ -113,6 +122,8 @@ export class SqlGenerator {
       if (this.checkDumper()) return;
       this.createObjects(this.views, 'View');
       if (this.checkDumper()) return;
+      this.createObjects(this.matviews, 'Matview');
+      if (this.checkDumper()) return;
       this.createObjects(this.triggers, 'Trigger');
       if (this.checkDumper()) return;
     } finally {
@@ -122,9 +133,9 @@ export class SqlGenerator {
 
   createForeignKeys() {
     const fks = [];
-    if (this.options.createForeignKeys) fks.push(..._.flatten(this.tables.map(x => x.foreignKeys || [])));
-    if (this.options.createReferences) fks.push(..._.flatten(this.tables.map(x => x.dependencies || [])));
-    for (const fk of _.uniqBy(fks, 'constraintName')) {
+    if (this.options.createForeignKeys) fks.push(..._flatten(this.tables.map(x => x.foreignKeys || [])));
+    if (this.options.createReferences) fks.push(..._flatten(this.tables.map(x => x.dependencies || [])));
+    for (const fk of _uniqBy(fks, 'constraintName')) {
       this.dmp.createForeignKey(fk);
       if (this.checkDumper()) return;
     }
@@ -152,7 +163,7 @@ export class SqlGenerator {
       }
     }
     if (this.options.createIndexes) {
-      for (const index of _.flatten(this.tables.map(x => x.indexes || []))) {
+      for (const index of _flatten(this.tables.map(x => x.indexes || []))) {
         this.dmp.createIndex(index);
       }
     }
@@ -204,7 +215,7 @@ export class SqlGenerator {
 
   dropTables() {
     if (this.options.dropReferences) {
-      for (const fk of _.flatten(this.tables.map(x => x.dependencies || []))) {
+      for (const fk of _flatten(this.tables.map(x => x.dependencies || []))) {
         this.dmp.dropForeignKey(fk);
       }
     }

packages/tools/src/driverBase.ts

@@ -22,9 +22,7 @@ export const driverBase = {
   },
   async analyseSingleObject(pool, name, typeField = 'tables') {
     const analyser = new this.analyserClass(pool, this);
-    analyser.singleObjectFilter = { ...name, typeField };
-    const res = await analyser.fullAnalysis();
-    return res.tables[0];
+    return analyser.singleObjectAnalysis(name, typeField);
   },
   analyseSingleTable(pool, name) {
     return this.analyseSingleObject(pool, name, 'tables');

packages/tools/src/index.ts

@@ -7,6 +7,6 @@ export * from './DatabaseAnalyser';
 export * from './driverBase';
 export * from './SqlDumper';
 export * from './testPermission';
-export * from './splitPostgresQuery';
 export * from './SqlGenerator';
 export * from './structureTools';
+export * from './settingsExtractors';

packages/tools/src/settingsExtractors.ts

@@ -0,0 +1,21 @@
+import _isNaN from 'lodash/isNaN';
+import _isNumber from 'lodash/isNumber';
+
+export function extractIntSettingsValue(settings, name, defaultValue, min = null, max = null) {
+  const parsed = parseInt(settings[name]);
+  if (_isNaN(parsed)) {
+    return defaultValue;
+  }
+  if (_isNumber(parsed)) {
+    if (min != null && parsed < min) return min;
+    if (max != null && parsed > max) return max;
+    return parsed;
+  }
+  return defaultValue;
+}
+
+export function extractBoolSettingsValue(settings, name, defaultValue) {
+  const res = settings[name];
+  if (res == null) return defaultValue;
+  return !!res;
+}

packages/tools/src/splitPostgresQuery.ts

@@ -1,292 +0,0 @@
-const SINGLE_QUOTE = "'";
-const DOUBLE_QUOTE = '"';
-// const BACKTICK = '`';
-const DOUBLE_DASH_COMMENT_START = '--';
-const HASH_COMMENT_START = '#';
-const C_STYLE_COMMENT_START = '/*';
-const SEMICOLON = ';';
-const LINE_FEED = '\n';
-const DELIMITER_KEYWORD = 'DELIMITER';
-
-export interface SplitOptions {
-  multipleStatements?: boolean;
-  retainComments?: boolean;
-}
-
-interface SqlStatement {
-  value: string;
-  supportMulti: boolean;
-}
-
-interface SplitExecutionContext extends Required<SplitOptions> {
-  unread: string;
-  currentDelimiter: string;
-  currentStatement: SqlStatement;
-  output: SqlStatement[];
-}
-
-interface FindExpResult {
-  expIndex: number;
-  exp: string | null;
-  nextIndex: number;
-}
-
-const regexEscapeSetRegex = /[-/\\^$*+?.()|[\]{}]/g;
-const singleQuoteStringEndRegex = /(?<!\\)'/;
-const doubleQuoteStringEndRegex = /(?<!\\)"/;
-// const backtickQuoteEndRegex = /(?<!`)`(?!`)/;
-const doubleDashCommentStartRegex = /--[ \f\n\r\t\v]/;
-const cStyleCommentStartRegex = /\/\*/;
-const cStyleCommentEndRegex = /(?<!\/)\*\//;
-const newLineRegex = /(?:[\r\n]+|$)/;
-const delimiterStartRegex = /(?:^|[\n\r]+)[ \f\t\v]*DELIMITER[ \t]+/i;
-// Best effort only, unable to find a syntax specification on delimiter
-const delimiterTokenRegex = /^(?:'(.+)'|"(.+)"|`(.+)`|([^\s]+))/;
-const semicolonKeyTokenRegex = buildKeyTokenRegex(SEMICOLON);
-const quoteEndRegexDict: Record<string, RegExp> = {
-  [SINGLE_QUOTE]: singleQuoteStringEndRegex,
-  [DOUBLE_QUOTE]: doubleQuoteStringEndRegex,
-  // [BACKTICK]: backtickQuoteEndRegex,
-};
-
-function escapeRegex(value: string): string {
-  return value.replace(regexEscapeSetRegex, '\\$&');
-}
-
-function buildKeyTokenRegex(delimiter: string): RegExp {
-  return new RegExp(
-    '(?:' +
-      [
-        escapeRegex(delimiter),
-        SINGLE_QUOTE,
-        DOUBLE_QUOTE,
-        // BACKTICK,
-        doubleDashCommentStartRegex.source,
-        HASH_COMMENT_START,
-        cStyleCommentStartRegex.source,
-        delimiterStartRegex.source,
-      ].join('|') +
-      ')',
-    'i'
-  );
-}
-
-function findExp(content: string, regex: RegExp): FindExpResult {
-  const match = content.match(regex);
-  let result: FindExpResult;
-  if (match?.index !== undefined) {
-    result = {
-      expIndex: match.index,
-      exp: match[0],
-      nextIndex: match.index + match[0].length,
-    };
-  } else {
-    result = {
-      expIndex: -1,
-      exp: null,
-      nextIndex: content.length,
-    };
-  }
-  return result;
-}
-
-function findKeyToken(content: string, currentDelimiter: string): FindExpResult {
-  let regex;
-  if (currentDelimiter === SEMICOLON) {
-    regex = semicolonKeyTokenRegex;
-  } else {
-    regex = buildKeyTokenRegex(currentDelimiter);
-  }
-  return findExp(content, regex);
-}
-
-function findEndQuote(content: string, quote: string): FindExpResult {
-  if (!(quote in quoteEndRegexDict)) {
-    throw new TypeError(`Incorrect quote ${quote} supplied`);
-  }
-  return findExp(content, quoteEndRegexDict[quote]);
-}
-
-function read(
-  context: SplitExecutionContext,
-  readToIndex: number,
-  nextUnreadIndex?: number,
-  checkSemicolon?: boolean
-): void {
-  if (checkSemicolon === undefined) {
-    checkSemicolon = true;
-  }
-  const readContent = context.unread.slice(0, readToIndex);
-  if (checkSemicolon && readContent.includes(SEMICOLON)) {
-    context.currentStatement.supportMulti = false;
-  }
-  context.currentStatement.value += readContent;
-  if (nextUnreadIndex !== undefined && nextUnreadIndex > 0) {
-    context.unread = context.unread.slice(nextUnreadIndex);
-  } else {
-    context.unread = context.unread.slice(readToIndex);
-  }
-}
-
-function readTillNewLine(context: SplitExecutionContext, checkSemicolon?: boolean): void {
-  const findResult = findExp(context.unread, newLineRegex);
-  read(context, findResult.expIndex, findResult.expIndex, checkSemicolon);
-}
-
-function discard(context: SplitExecutionContext, nextUnreadIndex: number): void {
-  if (nextUnreadIndex > 0) {
-    context.unread = context.unread.slice(nextUnreadIndex);
-  }
-}
-
-function discardTillNewLine(context: SplitExecutionContext): void {
-  const findResult = findExp(context.unread, newLineRegex);
-  discard(context, findResult.expIndex);
-}
-
-function publishStatementInMultiMode(splitOutput: SqlStatement[], currentStatement: SqlStatement): void {
-  if (splitOutput.length === 0) {
-    splitOutput.push({
-      value: '',
-      supportMulti: true,
-    });
-  }
-  const lastSplitResult = splitOutput[splitOutput.length - 1];
-  if (currentStatement.supportMulti) {
-    if (lastSplitResult.supportMulti) {
-      if (lastSplitResult.value !== '' && !lastSplitResult.value.endsWith(LINE_FEED)) {
-        lastSplitResult.value += LINE_FEED;
-      }
-      lastSplitResult.value += currentStatement.value + SEMICOLON;
-    } else {
-      splitOutput.push({
-        value: currentStatement.value + SEMICOLON,
-        supportMulti: true,
-      });
-    }
-  } else {
-    splitOutput.push({
-      value: currentStatement.value,
-      supportMulti: false,
-    });
-  }
-}
-
-function publishStatement(context: SplitExecutionContext): void {
-  const trimmed = context.currentStatement.value.trim();
-  if (trimmed !== '') {
-    if (!context.multipleStatements) {
-      context.output.push({
-        value: trimmed,
-        supportMulti: context.currentStatement.supportMulti,
-      });
-    } else {
-      context.currentStatement.value = trimmed;
-      publishStatementInMultiMode(context.output, context.currentStatement);
-    }
-  }
-  context.currentStatement.value = '';
-  context.currentStatement.supportMulti = true;
-}
-
-function handleKeyTokenFindResult(context: SplitExecutionContext, findResult: FindExpResult): void {
-  switch (findResult.exp?.trim()) {
-    case context.currentDelimiter:
-      read(context, findResult.expIndex, findResult.nextIndex);
-      publishStatement(context);
-      break;
-    // case BACKTICK:
-    case SINGLE_QUOTE:
-    case DOUBLE_QUOTE: {
-      read(context, findResult.nextIndex);
-      const findQuoteResult = findEndQuote(context.unread, findResult.exp);
-      read(context, findQuoteResult.nextIndex, undefined, false);
-      break;
-    }
-    case DOUBLE_DASH_COMMENT_START: {
-      if (context.retainComments) {
-        read(context, findResult.nextIndex);
-        readTillNewLine(context, false);
-      } else {
-        read(context, findResult.expIndex, findResult.expIndex + DOUBLE_DASH_COMMENT_START.length);
-        discardTillNewLine(context);
-      }
-      break;
-    }
-    case HASH_COMMENT_START: {
-      if (context.retainComments) {
-        read(context, findResult.nextIndex);
-        readTillNewLine(context, false);
-      } else {
-        read(context, findResult.expIndex, findResult.nextIndex);
-        discardTillNewLine(context);
-      }
-      break;
-    }
-    case C_STYLE_COMMENT_START: {
-      if (['!', '+'].includes(context.unread[findResult.nextIndex]) || context.retainComments) {
-        // Should not be skipped, see https://dev.mysql.com/doc/refman/5.7/en/comments.html
-        read(context, findResult.nextIndex);
-        const findCommentResult = findExp(context.unread, cStyleCommentEndRegex);
-        read(context, findCommentResult.nextIndex);
-      } else {
-        read(context, findResult.expIndex, findResult.nextIndex);
-        const findCommentResult = findExp(context.unread, cStyleCommentEndRegex);
-        discard(context, findCommentResult.nextIndex);
-      }
-      break;
-    }
-    case DELIMITER_KEYWORD: {
-      read(context, findResult.expIndex, findResult.nextIndex);
-      // MySQL client will return `DELIMITER cannot contain a backslash character` if backslash is used
-      // Shall we reject backslash as well?
-      const matched = context.unread.match(delimiterTokenRegex);
-      if (matched?.index !== undefined) {
-        context.currentDelimiter = matched[0].trim();
-        discard(context, matched[0].length);
-      }
-      discardTillNewLine(context);
-      break;
-    }
-    case undefined:
-    case null:
-      read(context, findResult.nextIndex);
-      publishStatement(context);
-      break;
-    default:
-      // This should never happen
-      throw new Error(`Unknown token '${findResult.exp ?? '(null)'}'`);
-  }
-}
-
-export function splitPostgresQuery(sql: string, options?: SplitOptions): string[] {
-  options = options ?? {};
-  const context: SplitExecutionContext = {
-    multipleStatements: options.multipleStatements ?? false,
-    retainComments: options.retainComments ?? false,
-    unread: sql,
-    currentDelimiter: SEMICOLON,
-    currentStatement: {
-      value: '',
-      supportMulti: true,
-    },
-    output: [],
-  };
-  let findResult: FindExpResult = {
-    expIndex: -1,
-    exp: null,
-    nextIndex: 0,
-  };
-  let lastUnreadLength;
-  do {
-    lastUnreadLength = context.unread.length;
-    findResult = findKeyToken(context.unread, context.currentDelimiter);
-    handleKeyTokenFindResult(context, findResult);
-    // Prevent infinite loop by returning incorrect result
-    if (lastUnreadLength === context.unread.length) {
-      read(context, context.unread.length);
-    }
-  } while (context.unread !== '');
-  publishStatement(context);
-  return context.output.map(v => v.value);
-}

packages/tools/src/structureTools.ts

@@ -1,8 +1,8 @@
 import { DatabaseInfo } from 'dbgate-types';
-import _ from 'lodash';
+import _flatten from 'lodash/flatten';
 
 export function addTableDependencies(db: DatabaseInfo): DatabaseInfo {
-  const allForeignKeys = _.flatten(db.tables.map(x => x.foreignKeys || []));
+  const allForeignKeys = _flatten(db.tables.map(x => x.foreignKeys || []));
   return {
     ...db,
     tables: db.tables.map(table => ({
@@ -64,6 +64,10 @@ function fillTableExtendedInfo(db: DatabaseInfo): DatabaseInfo {
       ...obj,
       objectTypeField: 'views',
     })),
+    matviews: (db.matviews || []).map(obj => ({
+      ...obj,
+      objectTypeField: 'matviews',
+    })),
     procedures: (db.procedures || []).map(obj => ({
       ...obj,
       objectTypeField: 'procedures',

packages/types/dbinfo.d.ts

@@ -95,6 +95,7 @@ export interface DatabaseInfoObjects {
   tables: TableInfo[];
   collections: CollectionInfo[];
   views: ViewInfo[];
+  matviews: ViewInfo[];
   procedures: ProcedureInfo[];
   functions: FunctionInfo[];
   triggers: TriggerInfo[];

packages/types/dialect.d.ts

@@ -1,6 +1,7 @@
 export interface SqlDialect {
   rangeSelect?: boolean;
   limitSelect?: boolean;
+  rowNumberOverPaging?: boolean;
   stringEscapeChar: string;
   offsetFetchRangeSyntax?: boolean;
   quoteIdentifier(s: string): string;

packages/types/engines.d.ts

@@ -38,8 +38,13 @@ export interface EngineDriver {
   title: string;
   defaultPort?: number;
   supportsDatabaseUrl?: boolean;
+  isElectronOnly?: boolean;
+  showConnectionField?: (field: string, values: any) => boolean;
+  showConnectionTab?: (tab: 'ssl' | 'sshTunnel', values: any) => boolean;
+  beforeConnectionSave?: (values: any) => any;
   databaseUrlPlaceholder?: string;
-  connect({ server, port, user, password, database }): any;
+  connect({ server, port, user, password, database }): Promise<any>;
+  close(pool): Promise<any>;
   query(pool: any, sql: string): Promise<QueryResult>;
   stream(pool: any, sql: string, options: StreamOptions);
   readQuery(pool: any, sql: string, structure?: TableInfo): Promise<stream.Readable>;
@@ -61,6 +66,7 @@ export interface EngineDriver {
   analyseFull(pool: any): Promise<DatabaseInfo>;
   analyseIncremental(pool: any, structure: DatabaseInfo): Promise<DatabaseInfo>;
   dialect: SqlDialect;
+  dialectByVersion(version): SqlDialect;
   createDumper(): SqlDumper;
   getAuthTypes(): EngineAuthType[];
   readCollection(pool: any, options: ReadCollectionOptions): Promise<any>;
@@ -76,6 +82,6 @@ export interface DatabaseModification {
   oldName?: NamedObjectInfo;
   newName?: NamedObjectInfo;
   objectId?: string;
-  action: 'add' | 'remove' | 'change';
+  action: 'add' | 'remove' | 'change' | 'all';
   objectTypeField: keyof DatabaseInfo;
 }

packages/types/index.d.ts

@@ -4,6 +4,8 @@ export interface OpenedDatabaseConnection {
   conid: string;
   database: string;
   structure: DatabaseInfo;
+  analysedTime?: number;
+  serverVersion?: any;
   subprocess: ChildProcess;
   disconnected?: boolean;
   status?: {

packages/types/package.json

@@ -6,7 +6,6 @@
     "type": "git",
     "url": "https://github.com/dbgate/dbgate.git"
   },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
   "author": "Jan Prochazka",
   "license": "MIT",
   "keywords": [

packages/web/package.json

@@ -29,7 +29,7 @@
     "file-selector": "^0.2.4",
     "json-stable-stringify": "^1.0.1",
     "localforage": "^1.9.0",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.21",
     "randomcolor": "^0.6.2",
     "resize-observer-polyfill": "^1.5.1",
     "rollup": "^2.3.4",

packages/web/public/index.html

@@ -21,9 +21,25 @@
 	<link rel='stylesheet' href='build/fonts/materialdesignicons.css'>
 
 	<script defer src='build/bundle.js'></script>
+
+	<style>
+		#starting_dbgate_zero {
+			position: absolute;
+			left: 0;
+			top: 0;
+			right: 0;
+			bottom: 0;
+			display: flex;
+			align-items: center;
+			justify-content: space-around;
+		}
+	</style>
 </head>
 
 <body>
+	<div id='starting_dbgate_zero'>
+		Loading DbGate App ...
+	</div>
 </body>
 
 </html>

packages/web/src/App.svelte

@@ -1,16 +1,68 @@
 <script lang="ts">
+  import { onMount } from 'svelte';
+
   import CommandListener from './commands/CommandListener.svelte';
   import DataGridRowHeightMeter from './datagrid/DataGridRowHeightMeter.svelte';
+  import LoadingInfo from './elements/LoadingInfo.svelte';
 
   import PluginsProvider from './plugins/PluginsProvider.svelte';
   import Screen from './Screen.svelte';
+  import { loadingPluginStore } from './stores';
+  import { setAppLoaded } from './utility/appLoadManager';
+  import axiosInstance from './utility/axiosInstance';
   import ErrorHandler from './utility/ErrorHandler.svelte';
   import OpenTabsOnStartup from './utility/OpenTabsOnStartup.svelte';
+
+  let loadedApi = false;
+
+  async function loadApi() {
+    try {
+      const settings = await axiosInstance.get('config/get-settings');
+      const connections = await axiosInstance.get('connections/list');
+      const config = await axiosInstance.get('config/get');
+      loadedApi = settings?.data && connections?.data && config?.data;
+      if (!loadedApi) {
+        console.log('API not initialized correctly, trying again in 1s');
+        setTimeout(loadApi, 1000);
+      }
+    } catch (err) {
+      console.log('Error calling API, trying again in 1s');
+      setTimeout(loadApi, 1000);
+    }
+  }
+
+  onMount(loadApi);
+
+  onMount(() => {
+    const removed = document.getElementById('starting_dbgate_zero');
+    if (removed) removed.remove();
+  });
+
+  $: {
+    if (loadedApi && $loadingPluginStore?.loaded) {
+      setAppLoaded();
+    }
+  }
+
 </script>
 
 <DataGridRowHeightMeter />
 <ErrorHandler />
-<PluginsProvider />
 <CommandListener />
-<OpenTabsOnStartup />
-<Screen />
+
+{#if loadedApi}
+  <PluginsProvider />
+  {#if $loadingPluginStore?.loaded}
+    <OpenTabsOnStartup />
+    <Screen />
+  {:else}
+    <LoadingInfo
+      message={$loadingPluginStore.loadingPackageName
+        ? `Loading plugin ${$loadingPluginStore.loadingPackageName} ...`
+        : 'Preparing plugins ...'}
+      wrapper
+    />
+  {/if}
+{:else}
+  <LoadingInfo message="Starting DbGate ..." wrapper />
+{/if}

packages/web/src/DragAndDropFileTarget.svelte

@@ -10,6 +10,7 @@
   $: fileTypeNames = _.compact([
     ...$extensions.fileFormats.filter(x => x.readerFunc).map(x => x.name),
     electron ? 'SQL' : null,
+    electron ? 'SQLite database' : null,
   ]);
 </script>
 

packages/web/src/appobj/ConnectionAppObject.svelte

@@ -1,68 +1,4 @@
 <script context="module">
-  const getContextMenu = (data, $openedConnections) => () => {
-    const config = getCurrentConfig();
-    const handleRefresh = () => {
-      axiosInstance.post('server-connections/refresh', { conid: data._id });
-    };
-    const handleDisconnect = () => {
-      openedConnections.update(list => list.filter(x => x != data._id));
-    };
-    const handleConnect = () => {
-      openedConnections.update(list => _.uniq([...list, data._id]));
-    };
-    const handleEdit = () => {
-      showModal(ConnectionModal, { connection: data });
-    };
-    const handleDelete = () => {
-      showModal(ConfirmModal, {
-        message: `Really delete connection ${data.displayName || data.server}?`,
-        onConfirm: () => axiosInstance.post('connections/delete', data),
-      });
-    };
-    const handleCreateDatabase = () => {
-      showModal(InputTextModal, {
-        header: 'Create database',
-        value: 'newdb',
-        label: 'Database name',
-        onConfirm: name =>
-          axiosInstance.post('server-connections/create-database', {
-            conid: data._id,
-            name,
-          }),
-      });
-    };
-
-    return [
-      config.runAsPortal == false && [
-        {
-          text: 'Edit',
-          onClick: handleEdit,
-        },
-        {
-          text: 'Delete',
-          onClick: handleDelete,
-        },
-      ],
-      !$openedConnections.includes(data._id) && {
-        text: 'Connect',
-        onClick: handleConnect,
-      },
-      $openedConnections.includes(data._id) &&
-        data.status && {
-          text: 'Refresh',
-          onClick: handleRefresh,
-        },
-      $openedConnections.includes(data._id) && {
-        text: 'Disconnect',
-        onClick: handleDisconnect,
-      },
-      $openedConnections.includes(data._id) && {
-        text: 'Create database',
-        onClick: handleCreateDatabase,
-      },
-    ];
-  };
-
   export const extractKey = data => data._id;
   export const createMatcher = ({ displayName, server }) => filter => filterName(filter, displayName, server);
 </script>
@@ -77,6 +13,10 @@
   import ConnectionModal from '../modals/ConnectionModal.svelte';
   import ConfirmModal from '../modals/ConfirmModal.svelte';
   import InputTextModal from '../modals/InputTextModal.svelte';
+  import openNewTab from '../utility/openNewTab';
+  import { getDatabaseMenuItems } from './DatabaseAppObject.svelte';
+  import getElectron from '../utility/getElectron';
+  import getConnectionLabel from '../utility/getConnectionLabel';
 
   export let data;
 
@@ -86,6 +26,125 @@
   let engineStatusIcon = null;
   let engineStatusTitle = null;
 
+  const electron = getElectron();
+
+  const handleConnect = () => {
+    if (data.singleDatabase) {
+      $currentDatabase = { connection: data, name: data.defaultDatabase };
+      axiosInstance.post('database-connections/refresh', {
+        conid: data._id,
+        database: data.defaultDatabase,
+        keepOpen: true,
+      });
+    } else {
+      $openedConnections = _.uniq([...$openedConnections, data._id]);
+      axiosInstance.post('server-connections/refresh', {
+        conid: data._id,
+        keepOpen: true,
+      });
+    }
+  };
+
+  const getContextMenu = () => {
+    const config = getCurrentConfig();
+    const handleRefresh = () => {
+      axiosInstance.post('server-connections/refresh', { conid: data._id });
+    };
+    const handleDisconnect = () => {
+      openedConnections.update(list => list.filter(x => x != data._id));
+      if (electron) {
+        axiosInstance.post('server-connections/disconnect', { conid: data._id });
+      }
+      if (_.get($currentDatabase, 'connection._id') == data._id) {
+        if (electron) {
+          axiosInstance.post('database-connections/disconnect', { conid: data._id, database: $currentDatabase.name });
+        }
+        currentDatabase.set(null);
+      }
+    };
+    const handleEdit = () => {
+      showModal(ConnectionModal, { connection: data });
+    };
+    const handleDelete = () => {
+      showModal(ConfirmModal, {
+        message: `Really delete connection ${getConnectionLabel(data)}?`,
+        onConfirm: () => axiosInstance.post('connections/delete', data),
+      });
+    };
+    const handleDuplicate = () => {
+      axiosInstance.post('connections/save', {
+        ...data,
+        _id: undefined,
+        displayName: `${getConnectionLabel(data)} - copy`,
+      });
+    };
+    const handleCreateDatabase = () => {
+      showModal(InputTextModal, {
+        header: 'Create database',
+        value: 'newdb',
+        label: 'Database name',
+        onConfirm: name =>
+          axiosInstance.post('server-connections/create-database', {
+            conid: data._id,
+            name,
+          }),
+      });
+    };
+    const handleNewQuery = () => {
+      const tooltip = `${getConnectionLabel(data)}`;
+      openNewTab({
+        title: 'Query #',
+        icon: 'img sql-file',
+        tooltip,
+        tabComponent: 'QueryTab',
+        props: {
+          conid: data._id,
+        },
+      });
+    };
+
+    return [
+      config.runAsPortal == false && [
+        {
+          text: 'Edit',
+          onClick: handleEdit,
+        },
+        {
+          text: 'Delete',
+          onClick: handleDelete,
+        },
+        {
+          text: 'Duplicate',
+          onClick: handleDuplicate,
+        },
+      ],
+      !data.singleDatabase && [
+        !$openedConnections.includes(data._id) && {
+          text: 'Connect',
+          onClick: handleConnect,
+        },
+        { onClick: handleNewQuery, text: 'New query' },
+        $openedConnections.includes(data._id) &&
+          data.status && {
+            text: 'Refresh',
+            onClick: handleRefresh,
+          },
+        $openedConnections.includes(data._id) && {
+          text: 'Disconnect',
+          onClick: handleDisconnect,
+        },
+        $openedConnections.includes(data._id) && {
+          text: 'Create database',
+          onClick: handleCreateDatabase,
+        },
+      ],
+      data.singleDatabase && [
+        { divider: true },
+        getDatabaseMenuItems(data, data.defaultDatabase, $extensions, $currentDatabase),
+      ],
+    ];
+  };
+
   $: {
     if ($extensions.drivers.find(x => x.engine == data.engine)) {
       const match = (data.engine || '').match(/^([^@]*)@/);
@@ -114,34 +173,21 @@
       statusTitle = null;
     }
   }
-
-  // const handleEdit = () => {
-  //   showModal(modalState => <ConnectionModal modalState={modalState} connection={data} />);
-  // };
-  // const handleDelete = () => {
-  //   showModal(modalState => (
-  //     <ConfirmModal
-  //       modalState={modalState}
-  //       message={`Really delete connection ${data.displayName || data.server}?`}
-  //       onConfirm={() => axios.post('connections/delete', data)}
-  //     />
-  //   ));
-  // };
-  // const handleCreateDatabase = () => {
-  //   showModal(modalState => <CreateDatabaseModal modalState={modalState} conid={data._id} />);
-  // };
 </script>
 
 <AppObjectCore
   {...$$restProps}
   {data}
-  title={data.displayName || data.server}
-  icon="img server"
-  isBold={_.get($currentDatabase, 'connection._id') == data._id}
+  title={getConnectionLabel(data)}
+  icon={data.singleDatabase ? 'img database' : 'img server'}
+  isBold={data.singleDatabase
+    ? _.get($currentDatabase, 'connection._id') == data._id && _.get($currentDatabase, 'name') == data.defaultDatabase
+    : _.get($currentDatabase, 'connection._id') == data._id}
   statusIcon={statusIcon || engineStatusIcon}
   statusTitle={statusTitle || engineStatusTitle}
   {extInfo}
-  menu={getContextMenu(data, $openedConnections)}
-  on:click={() => ($openedConnections = _.uniq([...$openedConnections, data._id]))}
+  menu={getContextMenu}
+  on:click={handleConnect}
   on:click
+  on:expand
 />

packages/web/src/appobj/DatabaseAppObject.svelte

@@ -1,67 +1,90 @@
 <script lang="ts" context="module">
   export const extractKey = props => props.name;
-</script>
+  const electron = getElectron();
 
-<script lang="ts">
-  import _ from 'lodash';
-  import ImportExportModal from '../modals/ImportExportModal.svelte';
-  import { showModal } from '../modals/modalTools';
-  import { getDefaultFileFormat } from '../plugins/fileformats';
-  import { currentDatabase, extensions } from '../stores';
-  import openNewTab from '../utility/openNewTab';
-  import AppObjectCore from './AppObjectCore.svelte';
-  export let data;
+  export function getDatabaseMenuItems(connection, name, $extensions, $currentDatabase) {
+    const handleNewQuery = () => {
+      const tooltip = `${getConnectionLabel(connection)}\n${name}`;
+      openNewTab({
+        title: 'Query #',
+        icon: 'img sql-file',
+        tooltip,
+        tabComponent: 'QueryTab',
+        props: {
+          conid: connection._id,
+          database: name,
+        },
+      });
+    };
 
-  const handleNewQuery = () => {
-    const { connection, name } = data;
-    const tooltip = `${connection.displayName || connection.server}\n${name}`;
-    openNewTab({
-      title: 'Query #',
-      icon: 'img sql-file',
-      tooltip,
-      tabComponent: 'QueryTab',
-      props: {
+    const handleImport = () => {
+      showModal(ImportExportModal, {
+        initialValues: {
+          sourceStorageType: getDefaultFileFormat($extensions).storageType,
+          targetStorageType: 'database',
+          targetConnectionId: connection._id,
+          targetDatabaseName: name,
+        },
+      });
+    };
+
+    const handleExport = () => {
+      showModal(ImportExportModal, {
+        initialValues: {
+          targetStorageType: getDefaultFileFormat($extensions).storageType,
+          sourceStorageType: 'database',
+          sourceConnectionId: connection._id,
+          sourceDatabaseName: name,
+        },
+      });
+    };
+
+    const handleSqlGenerator = () => {
+      showModal(SqlGeneratorModal, {
         conid: connection._id,
         database: name,
-      },
-    });
-  };
+      });
+    };
 
-  const handleImport = () => {
-    const { connection, name } = data;
+    const handleDisconnect = () => {
+      if (electron) {
+        axiosInstance.post('database-connections/disconnect', { conid: connection._id, database: name });
+      }
+      currentDatabase.set(null);
+    };
 
-    showModal(ImportExportModal, {
-      initialValues: {
-        sourceStorageType: getDefaultFileFormat($extensions).storageType,
-        targetStorageType: 'database',
-        targetConnectionId: connection._id,
-        targetDatabaseName: name,
-      },
-    });
-  };
-
-  const handleExport = () => {
-    const { connection, name } = data;
-
-    showModal(ImportExportModal, {
-      initialValues: {
-        targetStorageType: getDefaultFileFormat($extensions).storageType,
-        sourceStorageType: 'database',
-        sourceConnectionId: connection._id,
-        sourceDatabaseName: name,
-      },
-    });
-  };
-
-  function createMenu() {
     return [
       { onClick: handleNewQuery, text: 'New query' },
       { onClick: handleImport, text: 'Import' },
       { onClick: handleExport, text: 'Export' },
+      { onClick: handleSqlGenerator, text: 'SQL Generator' },
+
+      _.get($currentDatabase, 'connection._id') == _.get(connection, '_id') &&
+        _.get($currentDatabase, 'name') == name && { onClick: handleDisconnect, text: 'Disconnect' },
     ];
   }
 </script>
 
+<script lang="ts">
+  import getConnectionLabel from '../utility/getConnectionLabel';
+
+  import _ from 'lodash';
+  import ImportExportModal from '../modals/ImportExportModal.svelte';
+  import { showModal } from '../modals/modalTools';
+  import SqlGeneratorModal from '../modals/SqlGeneratorModal.svelte';
+  import { getDefaultFileFormat } from '../plugins/fileformats';
+  import { currentDatabase, extensions } from '../stores';
+  import axiosInstance from '../utility/axiosInstance';
+  import getElectron from '../utility/getElectron';
+  import openNewTab from '../utility/openNewTab';
+  import AppObjectCore from './AppObjectCore.svelte';
+  export let data;
+
+  function createMenu() {
+    return getDatabaseMenuItems(data.connection, data.name, $extensions, $currentDatabase);
+  }
+</script>
+
 <AppObjectCore
   {...$$restProps}
   {data}

packages/web/src/appobj/DatabaseObjectAppObject.svelte

@@ -6,6 +6,7 @@
     tables: 'img table',
     collections: 'img collection',
     views: 'img view',
+    matviews: 'img view',
     procedures: 'img procedure',
     functions: 'img function',
   };
@@ -14,6 +15,7 @@
     tables: 'TableDataTab',
     collections: 'CollectionDataTab',
     views: 'ViewDataTab',
+    matviews: 'ViewDataTab',
   };
 
   const menus = {
@@ -146,6 +148,63 @@
         },
       },
     ],
+    matviews: [
+      {
+        label: 'Open data',
+        tab: 'ViewDataTab',
+        forceNewTab: true,
+      },
+      {
+        label: 'Open structure',
+        tab: 'TableStructureTab',
+      },
+      {
+        label: 'Query designer',
+        isQueryDesigner: true,
+      },
+      {
+        divider: true,
+      },
+      {
+        label: 'Export',
+        isExport: true,
+      },
+      {
+        label: 'Open in free table editor',
+        isOpenFreeTable: true,
+      },
+      {
+        label: 'Open active chart',
+        isActiveChart: true,
+      },
+      {
+        divider: true,
+      },
+      {
+        label: 'SQL: CREATE MATERIALIZED VIEW',
+        scriptTemplate: 'CREATE OBJECT',
+      },
+      {
+        label: 'SQL: CREATE TABLE',
+        scriptTemplate: 'CREATE TABLE',
+      },
+      {
+        label: 'SQL: SELECT',
+        scriptTemplate: 'SELECT',
+      },
+      {
+        label: 'SQL Generator: CREATE MATERIALIZED VIEW',
+        sqlGeneratorProps: {
+          createMatviews: true,
+        },
+      },
+      {
+        label: 'SQL Generator: DROP MATERIALIZED VIEW',
+        sqlGeneratorProps: {
+          dropMatviews: true,
+        },
+      },
+    ],
     procedures: [
       {
         label: 'SQL: CREATE PROCEDURE',
@@ -228,7 +287,7 @@
     initialData
   ) {
     const connection = await getConnectionInfo({ conid });
-    const tooltip = `${connection.displayName || connection.server}\n${database}\n${fullDisplayName({
+    const tooltip = `${getConnectionLabel(connection)}\n${database}\n${fullDisplayName({
       schemaName,
       pureName,
     })}`;
@@ -267,6 +326,7 @@
   import { findEngineDriver } from 'dbgate-tools';
   import uuidv1 from 'uuid/v1';
   import SqlGeneratorModal from '../modals/SqlGeneratorModal.svelte';
+import getConnectionLabel from '../utility/getConnectionLabel';
 
   export let data;
 

packages/web/src/appobj/SavedFileAppObject.svelte

@@ -67,6 +67,7 @@
   import { currentDatabase } from '../stores';
 
   import axiosInstance from '../utility/axiosInstance';
+  import getConnectionLabel from '../utility/getConnectionLabel';
   import hasPermission from '../utility/hasPermission';
   import openNewTab from '../utility/openNewTab';
 
@@ -130,7 +131,7 @@
       const database = _.get($currentDatabase, 'name');
       connProps.conid = connection._id;
       connProps.database = database;
-      tooltip = `${connection.displayName || connection.server}\n${database}`;
+      tooltip = `${getConnectionLabel(connection)}\n${database}`;
     }
 
     openNewTab(

packages/web/src/celldata/TextCellView.svelte

@@ -1,6 +1,19 @@
 <script lang="ts">
+  import _ from 'lodash';
+
   export let selection;
   export let wrap;
 </script>
 
-<textarea class="flex1" {wrap} readonly value={selection.map(cell => cell.value).join('\n')} />
+<textarea
+  class="flex1"
+  {wrap}
+  readonly
+  value={selection
+    .map(cell => {
+      const { value } = cell;
+      if (_.isPlainObject(value) || _.isArray(value)) return JSON.stringify(value, undefined, 2);
+      return cell.value;
+    })
+    .join('\n')}
+/>

packages/web/src/commands/recentDatabaseSwitch.ts

@@ -1,5 +1,6 @@
 import _ from 'lodash';
 import { recentDatabases, currentDatabase, getRecentDatabases } from '../stores';
+import getConnectionLabel from '../utility/getConnectionLabel';
 import registerCommand from './registerCommand';
 
 currentDatabase.subscribe(value => {
@@ -15,7 +16,7 @@ currentDatabase.subscribe(value => {
 
 function switchDatabaseCommand(db) {
   return {
-    text: `${db.name} on ${db?.connection?.displayName || db?.connection?.server}`,
+    text: `${db.name} on ${getConnectionLabel(db?.connection, { allowExplicitDatabase: false })}`,
     onClick: () => currentDatabase.set(db),
   };
 }

packages/web/src/commands/registerCommand.ts

@@ -27,6 +27,7 @@ export interface GlobalCommand {
   menuName?: string;
   toolbarOrder?: number;
   disableHandleKeyText?: string;
+  isRelatedToTab?: boolean,
 }
 
 export default function registerCommand(command: GlobalCommand) {

packages/web/src/commands/stdCommands.ts

@@ -4,6 +4,7 @@ import { derived, get } from 'svelte/store';
 import { ThemeDefinition } from 'dbgate-types';
 import ConnectionModal from '../modals/ConnectionModal.svelte';
 import AboutModal from '../modals/AboutModal.svelte';
+import SettingsModal from '../settings/SettingsModal.svelte';
 import ImportExportModal from '../modals/ImportExportModal.svelte';
 import SqlGeneratorModal from '../modals/SqlGeneratorModal.svelte';
 import { showModal } from '../modals/modalTools';
@@ -15,6 +16,7 @@ import { openElectronFile } from '../utility/openElectronFile';
 import { getDefaultFileFormat } from '../plugins/fileformats';
 import { getCurrentConfig, getCurrentDatabase } from '../stores';
 import './recentDatabaseSwitch';
+import hasPermission from '../utility/hasPermission';
 
 const electron = getElectron();
 
@@ -44,7 +46,7 @@ registerCommand({
   id: 'toolbar.show',
   category: 'Toolbar',
   name: 'Show',
-  onClick: () => visibleToolbar.set(1),
+  onClick: () => visibleToolbar.set(true),
   testEnabled: () => !getVisibleToolbar(),
 });
 
@@ -52,7 +54,7 @@ registerCommand({
   id: 'toolbar.hide',
   category: 'Toolbar',
   name: 'Hide',
-  onClick: () => visibleToolbar.set(0),
+  onClick: () => visibleToolbar.set(false),
   testEnabled: () => getVisibleToolbar(),
 });
 
@@ -203,19 +205,29 @@ registerCommand({
     }),
 });
 
-registerCommand({
-  id: 'settings.commands',
-  category: 'Settings',
-  name: 'Keyboard shortcuts',
-  onClick: () => {
-    openNewTab({
-      title: 'Keyboard Shortcuts',
-      icon: 'icon keyboard',
-      tabComponent: 'CommandListTab',
-      props: {},
-    });
-  },
-});
+if (hasPermission('settings/change')) {
+  registerCommand({
+    id: 'settings.commands',
+    category: 'Settings',
+    name: 'Keyboard shortcuts',
+    onClick: () => {
+      openNewTab({
+        title: 'Keyboard Shortcuts',
+        icon: 'icon keyboard',
+        tabComponent: 'CommandListTab',
+        props: {},
+      });
+    },
+  });
+
+  registerCommand({
+    id: 'settings.show',
+    category: 'Settings',
+    name: 'Change',
+    toolbarName: 'Settings',
+    onClick: () => showModal(SettingsModal),
+  });
+}
 
 export function registerFileCommands({
   idPrefix,
@@ -229,6 +241,7 @@ export function registerFileCommands({
   toggleComment = false,
   findReplace = false,
   undoRedo = false,
+  executeAdditionalCondition = null,
 }) {
   if (save) {
     registerCommand({
@@ -239,6 +252,7 @@ export function registerFileCommands({
       // keyText: 'Ctrl+S',
       icon: 'icon save',
       toolbar: true,
+      isRelatedToTab: true,
       testEnabled: () => getCurrentEditor() != null,
       onClick: () => saveTabFile(getCurrentEditor(), false, folder, format, fileExtension),
     });
@@ -259,8 +273,12 @@ export function registerFileCommands({
       name: 'Execute',
       icon: 'icon run',
       toolbar: true,
+      isRelatedToTab: true,
       keyText: 'F5 | Ctrl+Enter',
-      testEnabled: () => getCurrentEditor() != null && !getCurrentEditor()?.isBusy(),
+      testEnabled: () =>
+        getCurrentEditor() != null &&
+        !getCurrentEditor()?.isBusy() &&
+        (executeAdditionalCondition == null || executeAdditionalCondition()),
       onClick: () => getCurrentEditor().execute(),
     });
     registerCommand({
@@ -269,6 +287,7 @@ export function registerFileCommands({
       name: 'Kill',
       icon: 'icon close',
       toolbar: true,
+      isRelatedToTab: true,
       testEnabled: () => getCurrentEditor()?.canKill && getCurrentEditor().canKill(),
       onClick: () => getCurrentEditor().kill(),
     });

packages/web/src/datagrid/CollapseButton.svelte

@@ -0,0 +1,26 @@
+<script lang="ts">
+  import FontIcon from '../icons/FontIcon.svelte';
+
+  export let collapsed;
+</script>
+
+<div on:click|stopPropagation class='collapseButtonMarker'>
+  <FontIcon icon={collapsed ? 'icon triple-right' : 'icon triple-left'} />
+</div>
+
+<style>
+  div {
+    position: absolute;
+    left: 0px;
+    top: 4px;
+    color: var(--theme-font-3);
+    background-color: var(--theme-bg-1);
+    border: 1px solid var(--theme-bg-1);
+  }
+
+  div:hover {
+    color: var(--theme-font-hover);
+    border: var(--theme-border);
+    top: 4px;
+  }
+</style>

packages/web/src/datagrid/DataFilterControl.svelte

@@ -226,6 +226,7 @@
     on:paste={handlePaste}
     class:isError
     class:isOk
+    placeholder='Filter'
   />
   <DropDownButton icon="icon filter" menu={createMenu} />
   {#if showResizeSplitter}

packages/web/src/datagrid/DataGrid.svelte

@@ -28,6 +28,15 @@
     onClick: () => getCurrentEditor().switchToView('table'),
   });
 
+  registerCommand({
+    id: 'dataGrid.toggleLeftPanel',
+    category: 'Data grid',
+    name: 'Toggle left panel',
+    keyText: 'Ctrl+L',
+    testEnabled: () => getCurrentEditor() != null,
+    onClick: () => getCurrentEditor().toggleLeftPanel(),
+  });
+
   function extractMacroValuesForMacro(macroValues, macro) {
     // return {};
     if (!macro) return {};
@@ -57,6 +66,9 @@
   import _ from 'lodash';
   import registerCommand from '../commands/registerCommand';
   import { registerMenu } from '../utility/contextMenu';
+  import { useSettings } from '../utility/metadataLoaders';
+  import { getCurrentSettings } from '../stores';
+  import { getBoolSettingsValue } from '../settings/settingsTools';
 
   export let config;
   export let setConfig;
@@ -88,6 +100,7 @@
   setContext('macroValues', macroValues);
 
   let managerSize;
+  const collapsedLeftColumnStore = writable(getBoolSettingsValue('dataGrid.hideLeftColumn', false));
 
   $: isFormView = !!(formDisplay && formDisplay.config && formDisplay.config.isFormView);
   $: isJsonView = !!config?.isJsonView;
@@ -120,22 +133,22 @@
     }
   }
 
+  export function toggleLeftPanel() {
+    collapsedLeftColumnStore.update(x => !x);
+  }
+
   registerMenu(
     { command: 'dataGrid.switchToForm', tag: 'switch', hideDisabled: true },
     { command: 'dataGrid.switchToTable', tag: 'switch', hideDisabled: true },
-    { command: 'dataGrid.switchToJson', tag: 'switch', hideDisabled: true }
+    { command: 'dataGrid.switchToJson', tag: 'switch', hideDisabled: true },
+    { command: 'dataGrid.toggleLeftPanel', tag: 'switch' }
   );
 </script>
 
-<HorizontalSplitter initialValue="300px" bind:size={managerSize}>
+<HorizontalSplitter initialValue="300px" bind:size={managerSize} hideFirst={$collapsedLeftColumnStore}>
   <div class="left" slot="1">
     <WidgetColumnBar>
-      <WidgetColumnBarItem
-        title="Columns"
-        name="columns"
-        height={showReferences ? '40%' : '60%'}
-        skip={freeTableColumn || isFormView}
-      >
+      <WidgetColumnBarItem title="Columns" name="columns" height="45%" skip={freeTableColumn || isFormView}>
         <ColumnManager {...$$props} {managerSize} {isJsonView} />
       </WidgetColumnBarItem>
 
@@ -161,7 +174,7 @@
         <ReferenceManager {...$$props} {managerSize} />
       </WidgetColumnBarItem>
 
-      <WidgetColumnBarItem title="Macros" name="macros" skip={!showMacros} collapsed={isDetailView}>
+      <WidgetColumnBarItem title="Macros" name="macros" skip={!showMacros} collapsed>
         <MacroManager {...$$props} {managerSize} />
       </WidgetColumnBarItem>
     </WidgetColumnBar>
@@ -177,6 +190,7 @@
           <svelte:component
             this={gridCoreComponent}
             {...$$props}
+            {collapsedLeftColumnStore}
             formViewAvailable={!!formViewComponent && !!formDisplay}
             macroValues={extractMacroValuesForMacro($macroValues, $selectedMacro)}
             macroPreview={$selectedMacro}