v4.2.4

commented testing code
v4.2.4-beta.4
2021-06-07 18:07:43 +02:00 · 2021-06-07 18:06:49 +02:00 · 2021-06-06 18:41:11 +02:00 · 2021-06-06 18:40:44 +02:00 · 2021-06-06 18:40:05 +02:00 · 2021-06-06 18:23:14 +02:00
400 changed files with 17051 additions and 2503 deletions
@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: dbgate
+patreon: # Replace with a single Patreon username
+open_collective: dbgate
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
@@ -1,4 +1,4 @@
-name: Electron app
+name: Electron app BETA

 on:
  push:
@@ -11,8 +11,8 @@ jobs:

    strategy:
      matrix:
-        os: [ubuntu-18.04, windows-2016]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        # os: [ubuntu-18.04, windows-2016]
+        os: [macOS-10.15, windows-2016, ubuntu-18.04]

    steps:
      - name: Context
@@ -35,10 +35,14 @@ jobs:
      - name: fillNativeModulesElectron
        run: |
          yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
      - name: Install Snapcraft
        if: matrix.os == 'ubuntu-18.04'
        uses: samuelmeuli/action-snapcraft@v1
      - name: Publish
+        if: matrix.os != 'macOS-10.15'
        run: |
          yarn run build:app
        env:
@@ -46,6 +50,13 @@ jobs:
          WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}

+      - name: Publish Mac
+        if: matrix.os == 'macOS-10.15'
+        run: |
+          yarn run build:app:mac
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+
      - name: Save snap login
        if: matrix.os == 'ubuntu-18.04'
        run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
@@ -63,11 +74,21 @@ jobs:
        run: |
          mkdir artifacts          

+          cp app/dist/*.deb artifacts/dbgate-beta.deb || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-beta.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-beta-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-beta-armv7l.AppImage || true
+          cp app/dist/*.exe artifacts/dbgate-beta.exe || true
+          cp app/dist/*win*.zip artifacts/dbgate-windows-beta.zip || true
+          cp app/dist/*-mac.dmg artifacts/dbgate-beta.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-beta-arm64.dmg || true
+
          mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
          mv app/dist/*.AppImage artifacts/ || true
          mv app/dist/*.deb artifacts/ || true
          mv app/dist/*.snap artifacts/ || true
-#          mv app/dist/*.dmg artifacts/ || true
+          mv app/dist/*.dmg artifacts/ || true

      - name: Upload artifacts
        uses: actions/upload-artifact@v1
@@ -16,7 +16,7 @@ jobs:
    strategy:
      matrix:
        # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        os: [macOS-10.15, windows-2016, ubuntu-18.04]

    steps:
      - name: Context
@@ -39,6 +39,9 @@ jobs:
      - name: fillNativeModulesElectron
        run: |
          yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
      - name: Install Snapcraft
        if: matrix.os == 'ubuntu-18.04'
        uses: samuelmeuli/action-snapcraft@v1
@@ -72,11 +75,16 @@ jobs:
          mkdir artifacts          

          cp app/dist/*.deb artifacts/dbgate-latest.deb || true
-          cp app/dist/*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-latest-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-latest-armv7l.AppImage || true
          cp app/dist/*.exe artifacts/dbgate-latest.exe || true
-          cp app/dist/*.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*win*.zip artifacts/dbgate-windows-latest.zip || true
+          cp app/dist/*-mac.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-latest-arm64.dmg || true

          mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
          mv app/dist/*.AppImage artifacts/ || true
          mv app/dist/*.deb artifacts/ || true
          mv app/dist/*.dmg artifacts/ || true
@@ -118,7 +126,7 @@ jobs:
          mv app/dist/latest-linux.yml artifacts/latest-linux.yml || true

      - name: Copy latest-mac.yml
-        if: matrix.os == 'macOS-10.14'
+        if: matrix.os == 'macOS-10.15'
        run: |
          mv app/dist/latest-mac.yml artifacts/latest-mac.yml || true

@@ -0,0 +1,47 @@
+name: Docker image BETA
+
+# on: [push]
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-18.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+      - name: Use Node.js 10.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 10.x
+      - name: yarn install
+        run: |
+          yarn install
+      - name: setCurrentVersion
+        run: |
+          yarn setCurrentVersion
+      - name: Prepare docker image
+        run: |
+          yarn run prepare:docker
+      - name: Build docker image
+        run: |
+          docker build ./docker -t dbgate
+      - name: Push docker image
+        run: |
+          docker tag dbgate dbgate/dbgate:beta
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push dbgate/dbgate:beta
@@ -21,7 +21,6 @@ jobs:
    strategy:
      matrix:
        os: [ubuntu-18.04]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]

    steps:
      - name: Context
@@ -21,7 +21,6 @@ jobs:
    strategy:
      matrix:
        os: [ubuntu-18.04]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]

    steps:
      - name: Context
@@ -80,6 +79,11 @@ jobs:
        run: |
          npm publish

+      - name: Publish query-splitter
+        working-directory: packages/query-splitter
+        run: |
+          npm publish
+          
      - name: Publish web
        working-directory: packages/web
        run: |
@@ -89,3 +93,38 @@ jobs:
        working-directory: packages/dbgate
        run: |
          npm publish
+
+      - name: Publish dbgate-plugin-csv
+        working-directory: plugins/dbgate-plugin-csv
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-excel
+        working-directory: plugins/dbgate-plugin-excel
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mssql
+        working-directory: plugins/dbgate-plugin-mssql
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mysql
+        working-directory: plugins/dbgate-plugin-mysql
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mongo
+        working-directory: plugins/dbgate-plugin-mongo
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-postgres
+        working-directory: plugins/dbgate-plugin-postgres
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-sqlite
+        working-directory: plugins/dbgate-plugin-sqlite
+        run: |
+          npm publish
@@ -0,0 +1,81 @@
+name: Run tests
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  test-runner:
+    runs-on: ubuntu-latest
+    container: node:10.18-jessie
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+      - name: yarn install
+        run: |
+          yarn install
+      - name: Integration tests
+        run: |
+          cd integration-tests
+          yarn test:ci
+#          yarn wait:ci
+      - name: Filter parser tests
+        if: always()
+        run: |
+          cd packages/filterparser
+          yarn test:ci
+      - name: Query spliiter tests
+        if: always()
+        run: |
+          cd packages/query-splitter
+          yarn test:ci
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: integration-tests/result.json
+          action-name: Integration tests
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/filterparser/result.json
+          action-name: Filter parser test results
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/query-splitter/result.json
+          action-name: Query splitter test results
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: Pwd2020Db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+      mysql:
+        image: mysql:8.0.18
+        env:
+          MYSQL_ROOT_PASSWORD: Pwd2020Db
+
+      mssql:
+        image: mcr.microsoft.com/mssql/server
+        env:
+          ACCEPT_EULA: Y
+          SA_PASSWORD: Pwd2020Db
+          MSSQL_PID: Express
+          
+      # cockroachdb:
+      #   image: cockroachdb/cockroach
@@ -13,8 +13,10 @@ build
 dist

 app/packages/web/public
+app/packages/plugins
 docker/public
 docker/bundle.js
+docker/plugins

 # misc
 .DS_Store
@@ -28,4 +30,5 @@ yarn-debug.log*
 yarn-error.log*
 app/src/nativeModulesContent.js
 packages/api/src/nativeModulesContent.js
+packages/api/src/packagedPluginsContent.js
 .VSCodeCounter
@@ -0,0 +1,3 @@
+{
+    "jestrunner.jestCommand": "node_modules/.bin/cross-env DEVMODE=1 LOCALTEST=1 node_modules/.bin/jest"
+}
@@ -1,5 +1,75 @@
 # ChangeLog

+### 4.2.4 - to be released
+- FIXED: Procedures in PostgreSQL #122
+- ADDED: Support of materialized views for PostgreSQL #123
+- ADDED: Integration tests
+- FIXED: Fixes in DB structure analysis in PostgreSQL, SQLite, MySQL
+- FIXED: Save data in SQLite, PostgreSQL
+- CHANGED: Introduced package dbgate-query-splitter, instead of sql-query-identifier and @verycrazydog/mysql-parse
+
+### 4.2.3
+- ADDED: ARM builds for MacOS and Linux
+- ADDED: Filter by columns in form view
+
+### 4.2.2
+- CHANGED: Further startup optimalization (approx. 2 times quicker start of electron app)
+
+### 4.2.1
+- FIXED: Fixed+optimalized app startup (esp. on Windows)
+
+### 4.2.0
+- ADDED: Support of SQLite database
+- ADDED: Support of Amazon Redshift database
+- ADDED: Support of CockcroachDB
+- CHANGED: DB Model is not auto-refreshed by default, refresh could be invoked from statusbar
+- FIXED: Fixed race conditions on startup
+- FIXED: Fixed broken style in data grid under strange circumstances
+- ADDED: Configure connections with commandline arguments  #108
+- CHANGED: Optimalized algorithm of incremental DB model updates
+- CHANGED: Loading queries from PostgreSQL doesn't need cursors, using streamed query instead
+- ADDED: Disconnect command
+- ADDED: Query executed on server has tab marker (formerly it had only "No DB" marker)
+- ADDED: Horizontal scroll using shift+mouse wheel #113
+- ADDED: Cosmetic improvements of MariaDB support
+
+### 4.1.11
+- FIX: Fixed crash of API process when using SSH tunnel connection (race condition)
+
+### 4.1.11
+- FIX: fixed processing postgre query containing $$
+- FIX: fixed postgre analysing procedures & functions
+- FIX: patched svelte crash #105
+- ADDED: ability to disbale background DB model updates
+- ADDED: Duplicate connection
+- ADDED: Duplicate tab
+- FIX: SSH tunnel connection using keyfile auth #106
+- FIX: All tables button fix in export #109
+- CHANGED: Add to favorites moved from toolbar to tab context menu
+- CHANGED: Toolbar design - current tab related commands are delimited
+
+### 4.1.10
+- ADDED: Default database option in connectin settings #96 #92
+- FIX: Bundle size optimalization for Windows #97
+- FIX: Popup menu placement on smaller displays #94
+- ADDED: Browse table data with SQL Server 2008 #93
+- FIX: Prevented malicious origins / DNS rebinding #91
+- ADDED: Handle JSON fields in data editor (eg. jsonb field in Postgres) #90
+- FIX: Fixed crash on Windows with Hyper-V #86
+- ADDED: Show database server version in status bar
+- ADDED: Show detailed info about error, when connect to database fails
+- ADDED: Portable ZIP distribution for Windows #84
+### 4.1.9
+- FIX: Incorrect row count info in query result #83
+
+### 4.1.1
+- CHANGED: Default plugins are now part of installation
+### 4.1.0
+- ADDED: MongoDB support
+- ADDED: Configurable keyboard shortcuts
+- ADDED: JSON row cell data view
+- FIX: Fixed some problems from migration to Svelte
+
 ### 4.0.3
 - FIX: fixes for FireFox (mainly incorrent handle of bind:clientHeight, replaces with resizeobserver)
 ### 4.0.2
@@ -1,36 +1,56 @@
 [![NPM version](https://img.shields.io/npm/v/dbgate.svg)](https://www.npmjs.com/package/dbgate)
+![GitHub All Releases](https://img.shields.io/github/downloads/dbgate/dbgate/total) 
 [![dbgate](https://snapcraft.io/dbgate/badge.svg)](https://snapcraft.io/dbgate)
 [![dbgate](https://snapcraft.io/dbgate/trending.svg?name=0)](https://snapcraft.io/dbgate)
 [![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
-[![Donate](https://img.shields.io/badge/donate-paypal-blue.svg)](https://paypal.me/JanProchazkaCz/30eur)

 # DbGate - database administration tool

-DbGate is fast and easy to use database manager. Works with MySQL, PostgreSQL and SQL Server.
+DbGate modern, fast and easy to use database manager

 * Try it online - [demo.dbgate.org](https://demo.dbgate.org) - online demo application
 * Download application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
 * Run web version as [NPM package](https://www.npmjs.com/package/dbgate) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)

+Supported databases:
+* MySQL
+* PostgreSQL
+* SQL Server
+* MongoDB
+* SQLite
+* Amazon Redshift
+* CockroachDB
+* MariaDB
+
 ![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png)

 ## Features
-* Connect to Microsoft SQL Server, Postgre SQL, MySQL
 * Table data editing, with SQL change script preview
+* Light and dark theme
 * Master/detail views
 * Query designer
 * Form view for comfortable work with tables with many columns
-* Explore tables, views, procedures, functions
-* SQL editor, execute SQL script, SQL code formatter, SQL code completion, SQL join wizard
+* JSON view on MongoDB collections
+* Explore tables, views, procedures, functions, MongoDB collections
+* SQL editor
+  * execute SQL script
+  * SQL code formatter
+  * SQL code completion
+  * Add SQL LEFT/INNER/RIGHT join utility
+* Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
 * Import, export from/to CSV, Excel, JSON
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
 * Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
-* Light and dark theme
 * Charts
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture

+## How to contribute
+Any contributions are welcome. If you want to contribute without coding, consider following:
+* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
+* Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
+
 ## Why is DbGate different
 There are many database managers now, so why DbGate? 
 * Works everywhere - Windows, Linux, Mac, Web browser (+mobile web is planned), without compromises in features
@@ -44,7 +64,6 @@ There are many database managers now, so why DbGate?
    * Backend - NodeJs, ExpressJs, socket.io, database connection drivers
    * JavaScript + TypeScript
    * App - electron
-    * There is plan to incorporate SQLite to support work with local datasets
 * Platform independed - will run as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac

 ## Plugins
@@ -56,6 +75,8 @@ Currently following extensions can be implemented using plugins:
 - File format parsers/writers
 - Database engine connectors

+Basic set of plugins is part of DbGate git repository and is installed with app. Additional plugins pust be downloaded from NPM (this task is handled by DbGate)
+
 ## How to run development environment

 ```sh
@@ -63,7 +84,7 @@ yarn
 yarn start
 ```

-If you want to make modifications in TypeScript packages, run TypeScript compiler in watch mode in seconds terminal:
+If you want to make modifications in libraries or plugins, run library compiler in watch mode in the second terminal:
 ```sh
 yarn lib
 ```
@@ -78,7 +99,7 @@ yarn start
 ```

 ## How to run built electron app locally
-This mode is very similar to production run of electron app. Electron app forks process with API on dynamically allocated port, works with compiled javascript files (doesn't use localhost:5000)
+This mode is very similar to production run of electron app. Electron app forks process with API on dynamically allocated port, works with compiled javascript files and uses compiled version of plugins (doesn't use localhost:5000)

 ```sh
 cd app
@@ -91,15 +112,3 @@ yarn build:app:local
 yarn start:app:local
 ```

-## Packages
-Some dbgate packages can be used also without DbGate. You can find them on [NPM repository](https://www.npmjs.com/search?q=keywords:dbgate)
-
-* [api](https://github.com/dbgate/dbgate/tree/master/packages/api) - backend, Javascript, ExpressJS [![NPM version](https://img.shields.io/npm/v/dbgate-api.svg)](https://www.npmjs.com/package/dbgate-api)
-* [datalib](https://github.com/dbgate/dbgate/tree/master/packages/datalib) - TypeScript library for utility classes [![NPM version](https://img.shields.io/npm/v/dbgate-datalib.svg)](https://www.npmjs.com/package/dbgate-datalib)
-* [app](https://github.com/dbgate/dbgate/tree/master/app) - application (JavaScript) structure, creating specific queries (JavaScript)
-* [filterparser](https://github.com/dbgate/dbgate/tree/master/packages/filterparser) - TypeScript library for parsing data filter expressions using parsimmon [![NPM version](https://img.shields.io/npm/v/dbgate-filterparser.svg)](https://www.npmjs.com/package/dbgate-filterparser)
-* [sqltree](https://github.com/dbgate/dbgate/tree/master/packages/sqltree) - JSON representation of SQL query, functions converting to SQL (TypeScript) [![NPM version](https://img.shields.io/npm/v/dbgate-sqltree.svg)](https://www.npmjs.com/package/dbgate-sqltree)
-* [types](https://github.com/dbgate/dbgate/tree/master/packages/types) - common TypeScript definitions [![NPM version](https://img.shields.io/npm/v/dbgate-types.svg)](https://www.npmjs.com/package/dbgate-types)
-* [web](https://github.com/dbgate/dbgate/tree/master/packages/web) - frontend in Svelte (JavaScript) [![NPM version](https://img.shields.io/npm/v/dbgate-web.svg)](https://www.npmjs.com/package/dbgate-web)
-* [tools](https://github.com/dbgate/dbgate/tree/master/packages/tools) - various tools [![NPM version](https://img.shields.io/npm/v/dbgate-tools.svg)](https://www.npmjs.com/package/dbgate-tools)
-
@@ -1,41 +1,60 @@
 {
  "name": "dbgate",
-  "version": "4.0.0",
+  "version": "4.1.1",
  "private": true,
  "author": "Jan Prochazka <jenasoft.database@gmail.com>",
  "description": "Opensource database administration tool",
  "dependencies": {
+    "better-sqlite3-with-prebuilds": "^7.1.8",
    "electron-log": "^4.3.1",
    "electron-store": "^5.1.1",
-    "electron-updater": "^4.3.5"
+    "electron-updater": "^4.3.5",
+    "patch-package": "^6.4.7"
  },
  "repository": {
    "type": "git",
    "url": "https://github.com/dbgate/dbgate.git"
  },
  "build": {
+    "artifactName": "${productName}-${version}-${os}_${arch}.${ext}",
    "appId": "org.dbgate",
    "mac": {
      "category": "database",
      "icon": "icon512.png",
-      "artifactName": "dbgate-mac-${version}.${ext}",
+      "publish": [
+        "github"
+      ],
+      "target": {
+        "target": "default",
+        "arch": [
+          "arm64",
+          "x64"
+        ]
+      }
+    },
+    "linux": {
+      "target": [
+        "deb",
+        "snap",
+        {
+          "target": "AppImage",
+          "arch": [
+            "x64",
+            "armv7l",
+            "arm64"
+          ]
+        }
+      ],
+      "icon": "icon.png",
+      "category": "Development",
+      "synopsis": "Database manager for SQL Server, MySQL, PostgreSQL, MongoDB and SQLite",
      "publish": [
        "github"
      ]
    },
-    "linux": {
-      "target": [
-        "AppImage",
-        "deb",
-        "snap"
-      ],
-      "icon": "icon.png",
-      "artifactName": "dbgate-linux-${version}.${ext}",
-      "category": "Development",
-      "synopsis": "Database administration tool for MS SQL, MySQL and PostgreSQL",
-      "publish": [
-        "github"
-      ]
+    "appImage": {
+      "license": "./LICENSE",
+      "category": "Development"
    },
    "snap": {
      "publish": [
@@ -48,9 +67,9 @@
    },
    "win": {
      "target": [
-        "nsis"
+        "nsis",
+        "zip"
      ],
-      "artifactName": "dbgate-windows-${version}.${ext}",
      "icon": "icon.ico",
      "publish": [
        "github"
@@ -69,16 +88,17 @@
    "start:local": "cross-env electron .",
    "dist": "electron-builder",
    "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist",
+    "build:mac": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && node setMacPlatform x64 && yarn dist && node setMacPlatform arm64 && yarn dist",
    "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn predist",
-    "postinstall": "electron-builder install-app-deps",
-    "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/public/*\" packages && copyfiles \"../packages/web/public/**/*\" packages"
+    "postinstall": "electron-builder install-app-deps && patch-package",
+    "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/public/*\" packages && copyfiles \"../packages/web/public/**/*\" packages && copyfiles --up 3 \"../plugins/dist/**/*\" packages/plugins"
  },
  "main": "src/electron.js",
  "devDependencies": {
    "copyfiles": "^2.2.0",
    "cross-env": "^6.0.3",
    "electron": "11.2.3",
-    "electron-builder": "22.9.1"
+    "electron-builder": "22.10.5"
  },
  "optionalDependencies": {
    "msnodesqlv8": "^2.0.10"
@@ -0,0 +1,8 @@
+const fs = require('fs');
+
+const text = fs.readFileSync('package.json', { encoding: 'utf-8' });
+const json = JSON.parse(text);
+
+json.build.mac.target.arch = process.argv[2];
+
+fs.writeFileSync('package.json', JSON.stringify(json, null, 2), { encoding: 'utf-8' });
@@ -19,7 +19,6 @@ const store = new Store();
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
 let mainWindow;
-let splashWindow;
 let mainMenu;

 log.transports.file.level = 'debug';
@@ -29,14 +28,6 @@ autoUpdater.logger = log;

 let commands = {};

-function hideSplash() {
-  if (splashWindow) {
-    splashWindow.destroy();
-    splashWindow = null;
-  }
-  mainWindow.show();
-}
-
 function commandItem(id) {
  const command = commands[id];
  return {
@@ -156,7 +147,6 @@ function createWindow() {
    title: 'DbGate',
    ...bounds,
    icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
-    show: false,
    webPreferences: {
      nodeIntegration: true,
      enableRemoteModule: true,
@@ -175,7 +165,7 @@ function createWindow() {
        slashes: true,
      });
    mainWindow.webContents.on('did-finish-load', function () {
-      hideSplash();
+      // hideSplash();
    });
    mainWindow.on('close', () => {
      store.set('winBounds', mainWindow.getBounds());
@@ -186,33 +176,21 @@ function createWindow() {
    }
  }

-  splashWindow = new BrowserWindow({
-    width: 300,
-    height: 120,
-    transparent: true,
-    frame: false,
-  });
-  splashWindow.loadURL(
-    url.format({
-      pathname: path.join(__dirname, '../packages/web/build/splash.html'),
-      protocol: 'file:',
-      slashes: true,
-    })
-  );
-
  if (process.env.ELECTRON_START_URL) {
    loadMainWindow();
  } else {
    const apiProcess = fork(path.join(__dirname, '../packages/api/dist/bundle.js'), [
      '--dynport',
+      '--is-electron-bundle',
      '--native-modules',
      path.join(__dirname, 'nativeModules'),
      // '../../../src/nativeModules'
    ]);
    apiProcess.on('message', msg => {
      if (msg.msgtype == 'listening') {
-        const { port } = msg;
+        const { port, authorization } = msg;
        global['port'] = port;
+        global['authorization'] = authorization;
        loadMainWindow();
      }
    });
@@ -2,10 +2,10 @@ const fs = require('fs');

 let fillContent = '';

-// if (!process.argv.includes('--electron')) {
 if (process.platform == 'win32') {
  fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');`;
 }
+fillContent += `content['better-sqlite3-with-prebuilds'] = () => require('better-sqlite3-with-prebuilds');`;

 const getContent = (empty) => `
 // this file is generated automatically by script fillNativeModules.js, do not edit it manually
@@ -0,0 +1,23 @@
+const fs = require('fs');
+const path = require('path');
+
+function load() {
+  const plugins = {};
+
+  for (const packageName of fs.readdirSync('plugins')) {
+    if (!packageName.startsWith('dbgate-plugin-')) continue;
+    const dir = path.join('plugins', packageName);
+    const frontend = fs.readFileSync(path.join(dir, 'dist', 'frontend.js'), 'utf-8');
+    const readme = fs.readFileSync(path.join(dir, 'README.md'), 'utf-8');
+    const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf-8'));
+    plugins[packageName] = {
+      manifest,
+      frontend,
+      readme,
+    };
+  }
+
+  return plugins;
+}
+
+fs.writeFileSync('packages/api/src/packagedPluginsContent.js', `module.exports = () => (${JSON.stringify(load())});`);
@@ -0,0 +1 @@
+dbtemp
@@ -0,0 +1,89 @@
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const _ = require('lodash');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'CREATE TABLE t2 (id int)'];
+
+function flatSource() {
+  return _.flatten(
+    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+  );
+}
+
+const obj1Match = expect.objectContaining({
+  pureName: 'obj1',
+});
+const view1Match = expect.objectContaining({
+  pureName: 'obj1',
+  columns: expect.arrayContaining([
+    expect.objectContaining({
+      columnName: 'id',
+    }),
+  ]),
+});
+
+describe('Object analyse', () => {
+  test.each(flatSource())(
+    'Full analysis - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure[type].length).toEqual(1);
+      expect(structure[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.create1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(2);
+      expect(structure2[type].find(x => x.pureName == 'obj1')).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - drop - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop2);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(1);
+      expect(structure2[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Create SQL - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      expect(structure2[type].length).toEqual(0);
+
+      await driver.query(conn, structure1[type][0].createSql);
+
+      const structure3 = await driver.analyseIncremental(conn, structure2);
+
+      expect(structure3[type].length).toEqual(1);
+      expect(structure3[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+});
@@ -0,0 +1,163 @@
+const engines = require('../engines');
+const { splitQuery } = require('dbgate-query-splitter');
+const { testWrapper } = require('../tools');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'INSERT INTO t1 (id) VALUES (1)', 'INSERT INTO t1 (id) VALUES (2)'];
+
+expect.extend({
+  dataRow(row, expected) {
+    for (const key in expected) {
+      if (row[key] != expected[key]) {
+        return {
+          pass: false,
+          message: () => `Different key: ${key}`,
+        };
+      }
+    }
+    return {
+      pass: true,
+      message: () => '',
+    };
+  },
+});
+
+class StreamHandler {
+  constructor(resolve) {
+    this.results = [];
+    this.resolve = resolve;
+    this.infoRows = [];
+  }
+  row(row) {
+    this.results[this.results.length - 1].rows.push(row);
+  }
+  recordset(columns) {
+    this.results.push({
+      columns,
+      rows: [],
+    });
+  }
+  done(result) {
+    this.resolve(this.results);
+  }
+  info(msg) {
+    this.infoRows.push(msg);
+  }
+}
+
+function executeStreamItem(driver, conn, sql) {
+  return new Promise(resolve => {
+    const handler = new StreamHandler(resolve);
+    driver.stream(conn, sql, handler);
+  });
+}
+
+async function executeStream(driver, conn, sql) {
+  const results = [];
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('stream'))) {
+    const item = await executeStreamItem(driver, conn, sqlItem);
+    results.push(...item);
+  }
+  return results;
+}
+
+describe('Query', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      const res = await driver.query(conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(res.columns).toEqual([
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ]);
+
+      expect(res.rows).toEqual([
+        expect.dataRow({
+          id: 1,
+        }),
+        expect.dataRow({
+          id: 2,
+        }),
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple stream query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(driver, conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(results.length).toEqual(1);
+      const res = results[0];
+
+      expect(res.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'More queries - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(
+        driver,
+        conn,
+        'SELECT id FROM t1 ORDER BY id; SELECT id FROM t1 ORDER BY id DESC'
+      );
+      expect(results.length).toEqual(2);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+
+      const res2 = results[1];
+      expect(res2.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res2.rows).toEqual([expect.dataRow({ id: 2 }), expect.dataRow({ id: 1 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script - return data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      const results = await executeStream(
+        driver,
+        conn,
+        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
+      );
+      expect(results.length).toEqual(1);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script - no data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      const results = await executeStream(
+        driver,
+        conn,
+        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2) '
+      );
+      expect(results.length).toEqual(0);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Save data query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.script(
+        conn,
+        'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;'
+      );
+      const res = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
+      // console.log(res);
+      expect(res.rows[0].cnt == 3).toBeTruthy();
+    })
+  );
+});
@@ -0,0 +1,112 @@
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+
+const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50) null)';
+const t2Sql = 'CREATE TABLE t2 (id int not null primary key, val2 varchar(50) null)';
+
+const txMatch = (tname, vcolname, nextcol) =>
+  expect.objectContaining({
+    pureName: tname,
+    columns: [
+      expect.objectContaining({
+        columnName: 'id',
+        notNull: true,
+        dataType: expect.stringContaining('int'),
+      }),
+      expect.objectContaining({
+        columnName: vcolname,
+        notNull: false,
+        dataType: expect.stringMatching(/.*char.*\(50\)/),
+      }),
+      ...(nextcol
+        ? [
+            expect.objectContaining({
+              columnName: 'nextcol',
+              notNull: false,
+              dataType: expect.stringMatching(/.*char.*\(50\)/),
+            }),
+          ]
+        : []),
+    ],
+    primaryKey: expect.objectContaining({
+      columns: [
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ],
+    }),
+  });
+
+const t1Match = txMatch('t1', 'val1');
+const t2Match = txMatch('t2', 'val2');
+const t2NextColMatch = txMatch('t2', 'val2', true);
+
+describe('Table analyse', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table structure - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure.tables.length).toEqual(1);
+      expect(structure.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table add - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(1);
+      expect(structure1.tables[0]).toEqual(t2Match);
+
+      await driver.query(conn, t1Sql);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table remove - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(2);
+      expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+
+      await driver.query(conn, 'DROP TABLE t2');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(1);
+      expect(structure2.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table change - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+
+      if (engine.dbSnapshotBySeconds) await new Promise(resolve => setTimeout(resolve, 1100));
+      
+      await driver.query(conn, 'ALTER TABLE t2 ADD nextcol varchar(50)');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2).toBeTruthy(); // if falsy, no modification is detected
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch);
+    })
+  );
+});
@@ -0,0 +1,55 @@
+version: '3'
+services:
+  postgres:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: Pwd2020Db
+    ports: 
+      - 15000:5432
+
+  mysql:
+    image: mysql:8.0.18
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    ports: 
+      - 15001:3306
+    environment:
+      - MYSQL_ROOT_PASSWORD=Pwd2020Db
+
+  mssql:
+    image: mcr.microsoft.com/mssql/server
+    restart: always
+    ports: 
+      - 15002:1433
+    environment:
+      - ACCEPT_EULA=Y
+      - SA_PASSWORD=Pwd2020Db
+      - MSSQL_PID=Express
+      
+  cockroachdb:
+    image: cockroachdb/cockroach
+    ports:
+      - 15003:26257
+    command: start-single-node --insecure
+
+#   mongodb:
+#     image: mongo:4.0.12
+#     restart: always
+#     volumes:
+#       - mongo-data:/data/db
+#       - mongo-config:/data/configdb
+#     ports:
+#       - 27017:27017
+
+
+#   cockroachdb-init:
+#     image: cockroachdb/cockroach
+#     # build: cockroach
+#     # entrypoint: /cockroach/init.sh 
+#     entrypoint: ./cockroach sql --insecure --host="cockroachdb" --execute="CREATE DATABASE IF NOT EXISTS test;"
+
+#     depends_on:
+#       - cockroachdb
+#     restart: on-failure
+
@@ -0,0 +1,108 @@
+const views = {
+  type: 'views',
+  create1: 'CREATE VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP VIEW obj1',
+  drop2: 'DROP VIEW obj2',
+};
+const matviews = {
+  type: 'matviews',
+  create1: 'CREATE MATERIALIZED VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE MATERIALIZED VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP MATERIALIZED VIEW obj1',
+  drop2: 'DROP MATERIALIZED VIEW obj2',
+};
+
+const engines = [
+  {
+    label: 'MySQL',
+    connection: {
+      engine: 'mysql@dbgate-plugin-mysql',
+      password: 'Pwd2020Db',
+      user: 'root',
+      server: 'mysql',
+      port: 3306,
+    },
+    local: {
+      server: 'localhost',
+      port: 15001,
+    },
+    // skipOnCI: true,
+    objects: [views],
+    dbSnapshotBySeconds: true,
+  },
+  {
+    label: 'PostgreSQL',
+    connection: {
+      engine: 'postgres@dbgate-plugin-postgres',
+      password: 'Pwd2020Db',
+      user: 'postgres',
+      server: 'postgres',
+      port: 5432,
+    },
+    local: {
+      server: 'localhost',
+      port: 15000,
+    },
+    objects: [
+      views,
+      matviews,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1() LANGUAGE SQL AS $$  select * from t1 $$',
+        create2: 'CREATE PROCEDURE obj2() LANGUAGE SQL AS $$  select * from t2 $$',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQL Server',
+    connection: {
+      engine: 'mssql@dbgate-plugin-mssql',
+      password: 'Pwd2020Db',
+      user: 'sa',
+      server: 'mssql',
+      port: 1433,
+    },
+    local: {
+      server: 'localhost',
+      port: 15002,
+    },
+    objects: [
+      views,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1 AS SELECT id FROM t1',
+        create2: 'CREATE PROCEDURE obj2 AS SELECT id FROM t2',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQLite',
+    generateDbFile: true,
+    connection: {
+      engine: 'sqlite@dbgate-plugin-sqlite',
+    },
+    objects: [views],
+  },
+  {
+    label: 'CockroachDB',
+    connection: {
+      engine: 'cockroach@dbgate-plugin-postgres',
+      user: 'root',
+      server: 'cockroachdb',
+      port: 26257,
+    },
+    local: {
+      server: 'localhost',
+      port: 15003,
+    },
+    skipOnCI: true,
+    objects: [views, matviews],
+  },
+];
+
+module.exports = process.env.CITEST ? engines.filter(x => !x.skipOnCI) : engines;
@@ -0,0 +1,25 @@
+{
+  "name": "dbgate-integration-tests",
+  "version": "4.1.1",
+  "homepage": "https://dbgate.org/",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate.git"
+  },
+  "author": "Jan Prochazka",
+  "license": "MIT",
+  "scripts": {
+    "wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js",
+    "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
+
+    "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
+    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
+    
+    "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
+  },
+  "devDependencies": {
+    "cross-env": "^7.0.3",
+    "jest": "^27.0.1"
+  },
+  "dependencies": {}
+}
@@ -0,0 +1,63 @@
+global.DBGATE_TOOLS = require('dbgate-tools');
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const crypto = require('crypto');
+
+function randomDbName() {
+  const generatedKey = crypto.randomBytes(6);
+  const newKey = generatedKey.toString('hex');
+  return `db${newKey}`;
+}
+
+function extractConnection(engine) {
+  const { connection } = engine;
+
+  if (process.env.LOCALTEST && engine.local) {
+    return {
+      ...connection,
+      ...engine.local,
+    };
+  }
+
+  return connection;
+}
+
+async function connect(engine, database) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+
+  if (engine.generateDbFile) {
+    const conn = await driver.connect({
+      ...connection,
+      databaseFile: `dbtemp/${database}`,
+    });
+    return conn;
+  } else {
+    const conn = await driver.connect(connection);
+    await driver.query(conn, `CREATE DATABASE ${database}`);
+    await driver.close(conn);
+
+    const res = await driver.connect({
+      ...connection,
+      database,
+    });
+    return res;
+  }
+}
+
+const testWrapper = body => async (label, ...other) => {
+  const engine = other[other.length - 1];
+  const driver = requireEngineDriver(engine.connection);
+  const conn = await connect(engine, randomDbName());
+  try {
+    await body(conn, driver, ...other);
+  } finally {
+    await driver.close(conn);
+  }
+};
+
+module.exports = {
+  randomDbName,
+  connect,
+  extractConnection,
+  testWrapper,
+};
@@ -0,0 +1,29 @@
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const engines = require('./engines');
+const { extractConnection } = require('./tools');
+global.DBGATE_TOOLS = require('dbgate-tools');
+
+async function connectEngine(engine) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+  for (;;) {
+    try {
+      const conn = await driver.connect(connection);
+      await driver.getVersion(conn);
+      console.log(`Connect to ${engine.label} - OK`);
+      await driver.close(conn);
+      return;
+    } catch (err) {
+      console.log(`Waiting for ${engine.label}, error: ${err.message}`);
+      await new Promise(resolve => setTimeout(resolve, 2500));
+      continue;
+    }
+  }
+}
+
+async function run() {
+  await new Promise(resolve => setTimeout(resolve, 10000));
+  await Promise.all(engines.map(engine => connectEngine(engine)));
+}
+
+run();
@@ -1,9 +1,11 @@
 {
  "private": true,
-  "version": "4.0.3-beta.1",
+  "version": "4.2.4",
  "name": "dbgate-all",
  "workspaces": [
-    "packages/*"
+    "packages/*",
+    "plugins/*",
+    "integration-tests"
  ],
  "scripts": {
    "start:api": "yarn workspace dbgate-api start",
@@ -14,30 +16,37 @@
    "start:tools": "yarn workspace dbgate-tools start",
    "start:datalib": "yarn workspace dbgate-datalib start",
    "start:filterparser": "yarn workspace dbgate-filterparser start",
+    "start:querysplitter": "yarn workspace dbgate-query-splitter start",
    "build:sqltree": "yarn workspace dbgate-sqltree build",
    "build:datalib": "yarn workspace dbgate-datalib build",
    "build:filterparser": "yarn workspace dbgate-filterparser build",
+    "build:querysplitter": "yarn workspace dbgate-query-splitter build",
    "build:tools": "yarn workspace dbgate-tools build",
-    "build:lib": "yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
-    "build:app": "cd app && yarn install && yarn build",
+    "build:lib": "yarn build:querysplitter && yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
+    "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
+    "build:app:mac": "yarn plugins:copydist && cd app && yarn install && yarn build:mac",
    "build:api": "yarn workspace dbgate-api build",
    "build:web:docker": "yarn workspace dbgate-web build",
-    "build:app:local": "cd app && yarn build:local",
+    "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
+    "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch",
+    "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist",
+    "build:app:local": "yarn plugins:copydist && cd app && yarn build:local",
    "start:app:local": "cd app && yarn start:local",
    "setCurrentVersion": "node setCurrentVersion",
    "generatePadFile": "node generatePadFile",
    "fillNativeModules": "node fillNativeModules",
-    "fillNativeModulesElectron": "node fillNativeModules --eletron",
+    "fillNativeModulesElectron": "node fillNativeModules --electron",
+    "fillPackagedPlugins": "node fillPackagedPlugins",
+    "resetPackagedPlugins": "node resetPackagedPlugins",
    "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
-    "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2",
-    "prepare:docker": "yarn build:web:docker && yarn build:api && yarn copy:docker:build",
-    "prepare": "yarn build:lib",
+    "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
+    "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build",
    "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
-    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\"",
+    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn start:querysplitter\" \"yarn build:plugins:frontend:watch\"",
    "ts:api": "yarn workspace dbgate-api ts",
    "ts:web": "yarn workspace dbgate-web ts",
    "ts": "yarn ts:api && yarn ts:web",
-    "postinstall": "patch-package && yarn fillNativeModules"
+    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend"
  },
  "dependencies": {
    "concurrently": "^5.1.0",
@@ -46,6 +55,7 @@
  },
  "devDependencies": {
    "copyfiles": "^2.2.0",
-    "prettier": "^2.2.1"
+    "prettier": "^2.2.1",
+    "workspaces-run": "^1.0.1"
  }
 }
@@ -1,12 +0,0 @@
-CONNECTIONS=mysql
-
-LABEL_mysql=MySql
-SERVER_mysql=dbgate.org
-USER_mysql=reader
-PASSWORD_mysql=CovidReader2020
-PORT_mysql=3326
-ENGINE_mysql=mysql@dbgate-plugin-mysql
-SINGLE_CONNECTION=mysql
-SINGLE_DATABASE=covid
-
-PERMISSIONS=files/charts/read
@@ -1,15 +1 @@
-CONNECTIONS=mysql,postgres
-
-LABEL_mysql=MySql localhost
-SERVER_mysql=localhost
-USER_mysql=root
-PASSWORD_mysql=test
-PORT_mysql=3307
-ENGINE_mysql=mysql@dbgate-plugin-mysql
-
-LABEL_postgres=Postgres localhost
-SERVER_postgres=localhost
-USER_postgres=postgres
-PASSWORD_postgres=test
-PORT_postgres=5433
-ENGINE_postgres=postgres@dbgate-plugin-postgres
+DEVMODE=1
@@ -0,0 +1,17 @@
+DEVMODE=1
+
+CONNECTIONS=mysql,postgres
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+LABEL_postgres=Postgres localhost
+SERVER_postgres=localhost
+USER_postgres=postgres
+PASSWORD_postgres=test
+PORT_postgres=5433
+ENGINE_postgres=postgres@dbgate-plugin-postgres
@@ -0,0 +1,15 @@
+DEVMODE=1
+
+CONNECTIONS=mysql
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+SINGLE_CONNECTION=mysql
+SINGLE_DATABASE=Chinook
+
+PERMISSIONS=files/charts/read
@@ -1,13 +1,12 @@
 {
  "name": "dbgate-api",
  "main": "src/index.js",
-  "version": "4.0.0",
+  "version": "4.1.1",
  "homepage": "https://dbgate.org/",
  "repository": {
    "type": "git",
    "url": "https://github.com/dbgate/dbgate.git"
  },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
  "author": "Jan Prochazka",
  "license": "MIT",
  "keywords": [
@@ -19,25 +18,27 @@
  ],
  "dependencies": {
    "async-lock": "^1.2.4",
-    "axios": "^0.19.0",
+    "axios": "^0.21.1",
+    "better-sqlite3-with-prebuilds": "^7.1.8",
    "body-parser": "^1.19.0",
    "bufferutil": "^4.0.1",
    "byline": "^5.0.0",
    "compare-versions": "^3.6.0",
    "cors": "^2.8.5",
    "cross-env": "^6.0.3",
-    "dbgate-sqltree": "^4.0.0",
-    "dbgate-tools": "^4.0.0",
+    "dbgate-sqltree": "^4.1.1",
+    "dbgate-tools": "^4.1.1",
    "eslint": "^6.8.0",
    "express": "^4.17.1",
    "express-basic-auth": "^1.2.0",
    "express-fileupload": "^1.2.0",
-    "find-free-port": "^2.0.0",
-    "fs-extra": "^8.1.0",
+    "fs-extra": "^9.1.0",
+    "fs-reverse": "^0.0.3",
+    "get-port": "^5.1.1",
    "http": "^0.0.0",
    "json-stable-stringify": "^1.0.1",
    "line-reader": "^0.4.0",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.21",
    "ncp": "^2.0.0",
    "nedb-promises": "^4.0.1",
    "node-cron": "^2.0.3",
@@ -49,15 +50,18 @@
    "uuid": "^3.4.0"
  },
  "scripts": {
-    "start": "node src/index.js",
-    "start:portal": "env-cmd nodemon src/index.js",
-    "start:covid": "env-cmd -f .covid-env nodemon src/index.js",
+    "start": "env-cmd node src/index.js",
+    "start:portal": "env-cmd -f .env-portal node src/index.js",
+    "start:singledb": "env-cmd -f .env-singledb node src/index.js",
+    "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db",
+    "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test",
    "ts": "tsc",
    "build": "webpack"
  },
  "devDependencies": {
+    "@types/fs-extra": "^9.0.11",
    "@types/lodash": "^4.14.149",
-    "dbgate-types": "^4.0.0",
+    "dbgate-types": "^4.1.1",
    "env-cmd": "^10.1.0",
    "node-loader": "^1.0.2",
    "nodemon": "^2.0.2",
@@ -68,4 +72,4 @@
  "optionalDependencies": {
    "msnodesqlv8": "^2.0.10"
  }
-}
+}
@@ -1,33 +1,32 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { datadir } = require('../utility/directories');
+const hasPermission = require('../utility/hasPermission');
+const socket = require('../utility/socket');
+const _ = require('lodash');
+
 const currentVersion = require('../currentVersion');
 const platformInfo = require('../utility/platformInfo');
+const connections = require('../controllers/connections');

 module.exports = {
+  settingsValue: {},
+
+  async _init() {
+    try {
+      this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
+    } catch (err) {
+      this.settingsValue = {};
+    }
+  },
+
  get_meta: 'get',
  async get() {
-    // const toolbarButtons = process.env.TOOLBAR;
-    // const toolbar = toolbarButtons
-    //   ? toolbarButtons.split(',').map((name) => ({
-    //       name,
-    //       icon: process.env[`ICON_${name}`],
-    //       title: process.env[`TITLE_${name}`],
-    //       page: process.env[`PAGE_${name}`],
-    //     }))
-    //   : null;
-    // const startupPages = process.env.STARTUP_PAGES ? process.env.STARTUP_PAGES.split(',') : [];
    const permissions = process.env.PERMISSIONS ? process.env.PERMISSIONS.split(',') : null;
-    const singleDatabase =
-      process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE
-        ? {
-            conid: process.env.SINGLE_CONNECTION,
-            database: process.env.SINGLE_DATABASE,
-          }
-        : null;

    return {
-      runAsPortal: !!process.env.CONNECTIONS,
-      // toolbar,
-      // startupPages,
-      singleDatabase,
+      runAsPortal: !!connections.portalConnections,
+      singleDatabase: connections.singleDatabase,
      permissions,
      ...currentVersion,
    };
@@ -37,5 +36,26 @@ module.exports = {
  async platformInfo() {
    return platformInfo;
  },
-  
+
+  getSettings_meta: 'get',
+  async getSettings() {
+    return this.settingsValue;
+  },
+
+  updateSettings_meta: 'post',
+  async updateSettings(values) {
+    if (!hasPermission(`settings/change`)) return false;
+    try {
+      const updated = {
+        ...this.settingsValue,
+        ...values,
+      };
+      await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
+      this.settingsValue = updated;
+      socket.emitChanged(`settings-changed`);
+      return updated;
+    } catch (err) {
+      return false;
+    }
+  },
 };
@@ -8,6 +8,32 @@ const socket = require('../utility/socket');
 const { encryptConnection } = require('../utility/crypting');
 const { handleProcessCommunication } = require('../utility/processComm');

+function getNamedArgs() {
+  const res = {};
+  for (let i = 0; i < process.argv.length; i++) {
+    const name = process.argv[i];
+    if (name.startsWith('--')) {
+      let value = process.argv[i + 1];
+      if (value && value.startsWith('--')) value = null;
+      res[name.substring(2)] = value == null ? true : value;
+      i++;
+    } else {
+      if (name.endsWith('.db') || name.endsWith('.sqlite') || name.endsWith('.sqlite3')) {
+        res.databaseFile = name;
+        res.engine = 'sqlite@dbgate-plugin-sqlite';
+      }
+    }
+  }
+  return res;
+}
+
+function getDatabaseFileLabel(databaseFile) {
+  if (!databaseFile) return databaseFile;
+  const m = databaseFile.match(/[\/]([^\/]+)$/);
+  if (m) return m[1];
+  return databaseFile;
+}
+
 function getPortalCollections() {
  if (process.env.CONNECTIONS) {
    return _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
@@ -17,16 +43,79 @@ function getPortalCollections() {
      user: process.env[`USER_${id}`],
      password: process.env[`PASSWORD_${id}`],
      port: process.env[`PORT_${id}`],
+      databaseUrl: process.env[`URL_${id}`],
+      databaseFile: process.env[`FILE_${id}`],
+      defaultDatabase: process.env[`DATABASE_${id}`],
+      singleDatabase: !!process.env[`DATABASE_${id}`],
      displayName: process.env[`LABEL_${id}`],
    }));
  }
+
+  const args = getNamedArgs();
+  if (args.databaseFile) {
+    return [
+      {
+        _id: 'argv',
+        databaseFile: args.databaseFile,
+        singleDatabase: true,
+        defaultDatabase: getDatabaseFileLabel(args.databaseFile),
+        engine: args.engine,
+      },
+    ];
+  }
+  if (args.databaseUrl) {
+    return [
+      {
+        _id: 'argv',
+        useDatabaseUrl: true,
+        ...args,
+      },
+    ];
+  }
+  if (args.server) {
+    return [
+      {
+        _id: 'argv',
+        ...args,
+      },
+    ];
+  }
+
  return null;
 }
 const portalConnections = getPortalCollections();

+function getSingleDatabase() {
+  if (process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE) {
+    // @ts-ignore
+    const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
+    return {
+      connection,
+      name: process.env.SINGLE_DATABASE,
+    };
+  }
+  // @ts-ignore
+  const arg0 = (portalConnections || []).find(x => x._id == 'argv');
+  if (arg0) {
+    // @ts-ignore
+    if (arg0.singleDatabase) {
+      return {
+        connection: arg0,
+        // @ts-ignore
+        name: arg0.defaultDatabase,
+      };
+    }
+  }
+  return null;
+}
+
+const singleDatabase = getSingleDatabase();
+
 module.exports = {
  datastore: null,
  opened: [],
+  singleDatabase,
+  portalConnections,

  async _init() {
    const dir = datadir();
@@ -46,7 +135,7 @@ module.exports = {
    raw: true,
  },
  test(req, res) {
-    const subprocess = fork(process.argv[1], ['connectProcess', ...process.argv.slice(3)]);
+    const subprocess = fork(process.argv[1], ['--start-process', 'connectProcess', ...process.argv.slice(3)]);
    subprocess.on('message', resp => {
      if (handleProcessCommunication(resp, subprocess)) return;
      // @ts-ignore
@@ -4,6 +4,7 @@ const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const { DatabaseAnalyser } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');
+const config = require('./config');

 module.exports = {
  /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -17,6 +18,19 @@ module.exports = {
    existing.structure = structure;
    socket.emitChanged(`database-structure-changed-${conid}-${database}`);
  },
+  handle_structureTime(conid, database, { analysedTime }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.analysedTime = analysedTime;
+    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+  },
+  handle_version(conid, database, { version }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.serverVersion = version;
+    socket.emitChanged(`database-server-version-changed-${conid}-${database}`);
+  },
+
  handle_error(conid, database, props) {
    const { error } = props;
    console.log(`Error in database connection ${conid}, database ${database}: ${error}`);
@@ -40,13 +54,18 @@ module.exports = {
    const existing = this.opened.find(x => x.conid == conid && x.database == database);
    if (existing) return existing;
    const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['databaseConnectionProcess', ...process.argv.slice(3)]);
+    const subprocess = fork(process.argv[1], [
+      '--start-process',
+      'databaseConnectionProcess',
+      ...process.argv.slice(3),
+    ]);
    const lastClosed = this.closed[`${conid}/${database}`];
    const newOpened = {
      conid,
      database,
      subprocess,
      structure: lastClosed ? lastClosed.structure : DatabaseAnalyser.createEmptyStructure(),
+      serverVersion: lastClosed ? lastClosed.serverVersion : null,
      connection,
      status: { name: 'pending' },
    };
@@ -67,6 +86,7 @@ module.exports = {
      msgtype: 'connect',
      connection: { ...connection, database },
      structure: lastClosed ? lastClosed.structure : null,
+      globalSettings: config.settingsValue,
    });
    return newOpened;
  },
@@ -92,6 +112,14 @@ module.exports = {
    return res;
  },

+  runScript_meta: 'post',
+  async runScript({ conid, database, sql }) {
+    console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'runScript', sql });
+    return res;
+  },
+
  collectionData_meta: 'post',
  async collectionData({ conid, database, options }) {
    const opened = await this.ensureOpened(conid, database);
@@ -99,12 +127,29 @@ module.exports = {
    return res.result;
  },

+  updateCollection_meta: 'post',
+  async updateCollection({ conid, database, changeSet }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet });
+    return res.result;
+  },
+
  status_meta: 'get',
  async status({ conid, database }) {
    const existing = this.opened.find(x => x.conid == conid && x.database == database);
-    if (existing) return existing.status;
+    if (existing) {
+      return {
+        ...existing.status,
+        analysedTime: existing.analysedTime,
+      };
+    }
    const lastClosed = this.closed[`${conid}/${database}`];
-    if (lastClosed) return lastClosed.status;
+    if (lastClosed) {
+      return {
+        ...lastClosed.status,
+        analysedTime: lastClosed.analysedTime,
+      };
+    }
    return {
      name: 'error',
      message: 'Not connected',
@@ -120,7 +165,7 @@ module.exports = {
    } else {
      existing = await this.ensureOpened(conid, database);
    }
-    
+
    return {
      status: 'ok',
      connectionStatus: existing ? existing.status : null,
@@ -128,13 +173,20 @@ module.exports = {
  },

  refresh_meta: 'post',
-  async refresh({ conid, database }) {
-    this.close(conid, database);
+  async refresh({ conid, database, keepOpen }) {
+    if (!keepOpen) this.close(conid, database);

    await this.ensureOpened(conid, database);
    return { status: 'ok' };
  },

+  syncModel_meta: 'post',
+  async syncModel({ conid, database }) {
+    const conn = await this.ensureOpened(conid, database);
+    conn.subprocess.send({ msgtype: 'syncModel' });
+    return { status: 'ok' };
+  },
+
  close(conid, database, kill = true) {
    const existing = this.opened.find(x => x.conid == conid && x.database == database);
    if (existing) {
@@ -152,6 +204,12 @@ module.exports = {
    }
  },

+  disconnect_meta: 'post',
+  async disconnect({ conid, database }) {
+    await this.close(conid, database, true);
+    return { status: 'ok' };
+  },
+
  structure_meta: 'get',
  async structure({ conid, database }) {
    const opened = await this.ensureOpened(conid, database);
@@ -164,6 +222,12 @@ module.exports = {
    // };
  },

+  serverVersion_meta: 'get',
+  async serverVersion({ conid, database }) {
+    const opened = await this.ensureOpened(conid, database);
+    return opened.serverVersion;
+  },
+
  sqlPreview_meta: 'post',
  async sqlPreview({ conid, database, objects, options }) {
    // wait for structure
@@ -2,43 +2,28 @@ const fs = require('fs-extra');
 const axios = require('axios');
 const path = require('path');
 const { extractPackageName } = require('dbgate-tools');
-const { pluginsdir, datadir } = require('../utility/directories');
+const { pluginsdir, packagedPluginsDir } = require('../utility/directories');
 const socket = require('../utility/socket');
 const compareVersions = require('compare-versions');
 const requirePlugin = require('../shell/requirePlugin');
 const downloadPackage = require('../utility/downloadPackage');
 const hasPermission = require('../utility/hasPermission');
-
-// async function loadPackageInfo(dir) {
-//   const readmeFile = path.join(dir, 'README.md');
-//   const packageFile = path.join(dir, 'package.json');
-
-//   if (!(await fs.exists(packageFile))) {
-//     return null;
-//   }
-
-//   let readme = null;
-//   let manifest = null;
-//   if (await fs.exists(readmeFile)) readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
-//   if (await fs.exists(packageFile)) manifest = JSON.parse(await fs.readFile(packageFile, { encoding: 'utf-8' }));
-//   return {
-//     readme,
-//     manifest,
-//   };
-// }
-
-const preinstallPluginMinimalVersions = {
-  'dbgate-plugin-mssql': '1.2.1',
-  'dbgate-plugin-mysql': '1.2.1',
-  'dbgate-plugin-postgres': '1.2.1',
-  'dbgate-plugin-csv': '1.0.8',
-  'dbgate-plugin-excel': '1.0.6',
-};
+const _ = require('lodash');
+const packagedPluginsContent = require('../packagedPluginsContent');

 module.exports = {
  script_meta: 'get',
  async script({ packageName }) {
-    const file = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
+    const packagedContent = packagedPluginsContent();
+
+    if (packagedContent && packagedContent[packageName]) {
+      return packagedContent[packageName].frontend;
+    }
+
+    const file1 = path.join(packagedPluginsDir(), packageName, 'dist', 'frontend.js');
+    const file2 = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
+    // @ts-ignore
+    const file = (await fs.exists(file1)) ? file1 : file2;
    const data = await fs.readFile(file, {
      encoding: 'utf-8',
    });
@@ -62,10 +47,13 @@ module.exports = {
      const { latest } = infoResp.data['dist-tags'];
      const manifest = infoResp.data.versions[latest];
      const { readme } = infoResp.data;
+      // @ts-ignore
+      const isPackaged = await fs.exists(path.join(packagedPluginsDir(), packageName));

      return {
        readme,
        manifest,
+        isPackaged,
      };
    } catch (err) {
      return {
@@ -73,52 +61,63 @@ module.exports = {
        error: err.message,
      };
    }
-
-    // const dir = path.join(pluginstmpdir(), packageName);
-    // if (!(await fs.exists(dir))) {
-    //   await downloadPackage(packageName, dir);
-    // }
-    // return await loadPackageInfo(dir);
-    // return await {
-    //   ...loadPackageInfo(dir),
-    //   installed: loadPackageInfo(path.join(pluginsdir(), packageName)),
-    // };
  },

  installed_meta: 'get',
  async installed() {
-    const files = await fs.readdir(pluginsdir());
+    const packagedContent = packagedPluginsContent();
+
+    const files1 = packagedContent ? _.keys(packagedContent) : await fs.readdir(packagedPluginsDir());
+    const files2 = await fs.readdir(pluginsdir());
+
    const res = [];
-    for (const packageName of files) {
-      const manifest = await fs.readFile(path.join(pluginsdir(), packageName, 'package.json')).then(x => JSON.parse(x));
-      const readmeFile = path.join(pluginsdir(), packageName, 'README.md');
-      if (await fs.exists(readmeFile)) {
-        manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+    for (const packageName of _.union(files1, files2)) {
+      if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
+      try {
+        if (packagedContent && packagedContent[packageName]) {
+          const manifest = {
+            ...packagedContent[packageName].manifest,
+          };
+          manifest.isPackaged = true;
+          manifest.readme = packagedContent[packageName].readme;
+          res.push(manifest);
+        } else {
+          const isPackaged = files1.includes(packageName);
+          const manifest = await fs
+            .readFile(path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'package.json'), {
+              encoding: 'utf-8',
+            })
+            .then(x => JSON.parse(x));
+          const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
+          // @ts-ignore
+          if (await fs.exists(readmeFile)) {
+            manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+          }
+          manifest.isPackaged = isPackaged;
+          res.push(manifest);
+        }
+      } catch (err) {
+        console.log(`Skipped plugin ${packageName}, error:`, err.message);
      }
-      res.push(manifest);
    }
    return res;
-    // const res = await Promise.all(
-    //   files.map((packageName) =>
-    //     fs.readFile(path.join(pluginsdir(), packageName, 'package.json')).then((x) => JSON.parse(x))
-    //   )
-    // );
  },

-  async saveRemovePlugins() {
-    await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
-  },
+  // async saveRemovePlugins() {
+  //   await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
+  // },

  install_meta: 'post',
  async install({ packageName }) {
    if (!hasPermission(`plugins/install`)) return;
    const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
    if (!(await fs.exists(dir))) {
      await downloadPackage(packageName, dir);
    }
    socket.emitChanged(`installed-plugins-changed`);
-    this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
-    await this.saveRemovePlugins();
+    // this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
+    // await this.saveRemovePlugins();
  },

  uninstall_meta: 'post',
@@ -127,7 +126,7 @@ module.exports = {
    const dir = path.join(pluginsdir(), packageName);
    await fs.rmdir(dir, { recursive: true });
    socket.emitChanged(`installed-plugins-changed`);
-    this.removedPlugins.push(packageName);
+    // this.removedPlugins.push(packageName);
    await this.saveRemovePlugins();
  },

@@ -135,6 +134,7 @@ module.exports = {
  async upgrade({ packageName }) {
    if (!hasPermission(`plugins/install`)) return;
    const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
    if (await fs.exists(dir)) {
      await fs.rmdir(dir, { recursive: true });
      await downloadPackage(packageName, dir);
@@ -153,50 +153,51 @@ module.exports = {
  async authTypes({ engine }) {
    const packageName = extractPackageName(engine);
    const content = requirePlugin(packageName);
-    if (!content.driver || content.driver.engine != engine || !content.driver.getAuthTypes) return null;
-    return content.driver.getAuthTypes() || null;
+    const driver = content.drivers.find(x => x.engine == engine);
+    if (!driver || !driver.getAuthTypes) return null;
+    return driver.getAuthTypes() || null;
  },

-  async _init() {
-    const installed = await this.installed();
-    try {
-      this.removedPlugins = (await fs.readFile(path.join(datadir(), 'removed-plugins'), { encoding: 'utf-8' })).split(
-        '\n'
-      );
-    } catch (err) {
-      this.removedPlugins = [];
-    }
+  // async _init() {
+  //   const installed = await this.installed();
+  //   try {
+  //     this.removedPlugins = (await fs.readFile(path.join(datadir(), 'removed-plugins'), { encoding: 'utf-8' })).split(
+  //       '\n'
+  //     );
+  //   } catch (err) {
+  //     this.removedPlugins = [];
+  //   }

-    for (const packageName of Object.keys(preinstallPluginMinimalVersions)) {
-      const installedVersion = installed.find(x => x.name == packageName);
-      if (installedVersion) {
-        // plugin installed, test, whether upgrade
-        const requiredVersion = preinstallPluginMinimalVersions[packageName];
-        if (compareVersions(installedVersion.version, requiredVersion) < 0) {
-          console.log(
-            `Upgrading preinstalled plugin ${packageName}, found ${installedVersion.version}, required version ${requiredVersion}`
-          );
-          await this.upgrade({ packageName });
-        } else {
-          console.log(
-            `Plugin ${packageName} not upgraded, found ${installedVersion.version}, required version ${requiredVersion}`
-          );
-        }
+  //   for (const packageName of Object.keys(preinstallPluginMinimalVersions)) {
+  //     const installedVersion = installed.find(x => x.name == packageName);
+  //     if (installedVersion) {
+  //       // plugin installed, test, whether upgrade
+  //       const requiredVersion = preinstallPluginMinimalVersions[packageName];
+  //       if (compareVersions(installedVersion.version, requiredVersion) < 0) {
+  //         console.log(
+  //           `Upgrading preinstalled plugin ${packageName}, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //         await this.upgrade({ packageName });
+  //       } else {
+  //         console.log(
+  //           `Plugin ${packageName} not upgraded, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //       }

-        continue;
-      }
+  //       continue;
+  //     }

-      if (this.removedPlugins.includes(packageName)) {
-        // plugin was remvoed, don't install again
-        continue;
-      }
+  //     if (this.removedPlugins.includes(packageName)) {
+  //       // plugin was remvoed, don't install again
+  //       continue;
+  //     }

-      try {
-        console.log('Preinstalling plugin', packageName);
-        await this.install({ packageName });
-      } catch (err) {
-        console.error('Error preinstalling plugin', packageName, err);
-      }
-    }
-  },
+  //     try {
+  //       console.log('Preinstalling plugin', packageName);
+  //       await this.install({ packageName });
+  //     } catch (err) {
+  //       console.error('Error preinstalling plugin', packageName, err);
+  //     }
+  //   }
+  // },
 };
@@ -0,0 +1,54 @@
+const fsReverse = require('fs-reverse');
+const fs = require('fs-extra');
+const path = require('path');
+const { datadir } = require('../utility/directories');
+const _ = require('lodash');
+const { filterName } = require('dbgate-tools');
+const socket = require('../utility/socket');
+
+function readCore(reader, skip, limit, filter) {
+  return new Promise((resolve, reject) => {
+    const res = [];
+    let readed = 0;
+    reader.on('data', line => {
+      if (!line && !line.trim()) return;
+      try {
+        const json = JSON.parse(line);
+        if (filterName(filter, json.sql, json.database)) {
+          if (!skip || readed >= skip) {
+            res.push(json);
+          }
+          readed++;
+          if (limit && readed > (skip || 0) + limit) {
+            reader.destroy();
+            resolve(res);
+          }
+        }
+      } catch (err) {
+        reader.destroy();
+        reject(err);
+      }
+    });
+    reader.on('end', () => resolve(res));
+  });
+}
+
+module.exports = {
+  read_meta: 'get',
+  async read({ skip, limit, filter }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    // @ts-ignore
+    if (!(await fs.exists(fileName))) return [];
+    const reader = fsReverse(fileName);
+    const res = await readCore(reader, skip, limit, filter);
+    return res;
+  },
+
+  write_meta: 'post',
+  async write({ data }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    await fs.appendFile(fileName, JSON.stringify(data) + '\n');
+    socket.emitChanged('query-history-changed');
+    return 'OK';
+  },
+};
@@ -5,7 +5,7 @@ const uuidv1 = require('uuid/v1');
 const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
-const { rundir, uploadsdir, pluginsdir } = require('../utility/directories');
+const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
 const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
 const { handleProcessCommunication } = require('../utility/processComm');

@@ -92,7 +92,7 @@ module.exports = {
    const scriptFile = path.join(uploadsdir(), runid + '.js');
    fs.writeFileSync(`${scriptFile}`, scriptText);
    fs.mkdirSync(directory);
-    const pluginNames = fs.readdirSync(pluginsdir());
+    const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
    console.log(`RUNNING SCRIPT ${scriptFile}`);
    // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
    const subprocess = fork(scriptFile, ['--checkParent', ...process.argv.slice(3)], {
@@ -101,7 +101,7 @@ module.exports = {
      env: {
        ...process.env,
        DBGATE_API: global['dbgateApiModulePath'] || process.argv[1],
-        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, path.join(pluginsdir(), name)])),
+        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
      },
    });
    const pipeDispatcher = severity => data =>
@@ -5,6 +5,7 @@ const _ = require('lodash');
 const AsyncLock = require('async-lock');
 const { handleProcessCommunication } = require('../utility/processComm');
 const lock = new AsyncLock();
+const config = require('./config');

 module.exports = {
  opened: [],
@@ -17,6 +18,12 @@ module.exports = {
    existing.databases = databases;
    socket.emitChanged(`database-list-changed-${conid}`);
  },
+  handle_version(conid, { version }) {
+    const existing = this.opened.find(x => x.conid == conid);
+    if (!existing) return;
+    existing.version = version;
+    socket.emitChanged(`server-version-changed-${conid}`);
+  },
  handle_status(conid, { status }) {
    const existing = this.opened.find(x => x.conid == conid);
    if (!existing) return;
@@ -30,7 +37,11 @@ module.exports = {
      const existing = this.opened.find(x => x.conid == conid);
      if (existing) return existing;
      const connection = await connections.get({ conid });
-      const subprocess = fork(process.argv[1], ['serverConnectionProcess', ...process.argv.slice(3)]);
+      const subprocess = fork(process.argv[1], [
+        '--start-process',
+        'serverConnectionProcess',
+        ...process.argv.slice(3),
+      ]);
      const newOpened = {
        conid,
        subprocess,
@@ -55,7 +66,7 @@ module.exports = {
        if (newOpened.disconnected) return;
        this.close(conid, false);
      });
-      subprocess.send({ msgtype: 'connect', ...connection });
+      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: config.settingsValue });
      return newOpened;
    });
    return res;
@@ -75,12 +86,24 @@ module.exports = {
    }
  },

+  disconnect_meta: 'post',
+  async disconnect({ conid }) {
+    await this.close(conid, true);
+    return { status: 'ok' };
+  },
+
  listDatabases_meta: 'get',
  async listDatabases({ conid }) {
    const opened = await this.ensureOpened(conid);
    return opened.databases;
  },

+  version_meta: 'get',
+  async version({ conid }) {
+    const opened = await this.ensureOpened(conid);
+    return opened.version;
+  },
+
  serverStatus_meta: 'get',
  async serverStatus() {
    return {
@@ -106,8 +129,8 @@ module.exports = {
  },

  refresh_meta: 'post',
-  async refresh({ conid }) {
-    this.close(conid);
+  async refresh({ conid, keepOpen }) {
+    if (!keepOpen) this.close(conid);

    await this.ensureOpened(conid);
    return { status: 'ok' };
@@ -65,7 +65,7 @@ module.exports = {
  async create({ conid, database }) {
    const sesid = uuidv1();
    const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['sessionProcess', ...process.argv.slice(3)]);
+    const subprocess = fork(process.argv[1], ['--start-process', 'sessionProcess', ...process.argv.slice(3)]);
    const newOpened = {
      conid,
      database,
@@ -1,5 +1,5 @@

 module.exports = { 
-  version: '4.0.0',
-  buildTime: '2021-04-01T10:48:22.253Z'
+  version: '4.1.1',
+  buildTime: '2021-04-17T07:22:49.702Z'
 };
@@ -1,15 +1,17 @@
 const shell = require('./shell');
+const processArgs = require('./utility/processArgs');
+const dbgateTools = require('dbgate-tools');

-const argument = process.argv[2];
-if (argument && argument.endsWith('Process')) {
+global['DBGATE_TOOLS'] = dbgateTools;
+
+if (processArgs.startProcess) {
  const proc = require('./proc');
-
-  const module = proc[argument];
+  const module = proc[processArgs.startProcess];
  module.start();
-} else if (!module['parent'] && !process.argv.includes('--checkParent')) {
+} else if (!module['parent'] && !processArgs.checkParent) {
  const main = require('./main');

-  main.start(argument);
+  main.start();
 }

 module.exports = {
@@ -6,9 +6,10 @@ const http = require('http');
 const cors = require('cors');
 const io = require('socket.io');
 const fs = require('fs');
-const findFreePort = require('find-free-port');
+const getPort = require('get-port');
 const childProcessChecker = require('./utility/childProcessChecker');
 const path = require('path');
+const crypto = require('crypto');

 const useController = require('./utility/useController');
 const socket = require('./utility/socket');
@@ -26,10 +27,17 @@ const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
 const scheduler = require('./controllers/scheduler');
+const queryHistory = require('./controllers/queryHistory');

 const { rundir } = require('./utility/directories');
+const platformInfo = require('./utility/platformInfo');
+const processArgs = require('./utility/processArgs');
+const timingSafeCheckToken = require('./utility/timingSafeCheckToken');

-function start(argument = null) {
+let authorization = null;
+let checkLocalhostOrigin = null;
+
+function start() {
  // console.log('process.argv', process.argv);

  const app = express();
@@ -49,6 +57,29 @@ function start(argument = null) {
    );
  }

+  app.use(function (req, res, next) {
+    if (authorization && !timingSafeCheckToken(req.headers.authorization, authorization)) {
+      return res.status(403).json({ error: 'Not authorized!' });
+    }
+    if (checkLocalhostOrigin) {
+      if (
+        req.headers.origin &&
+        req.headers.origin != checkLocalhostOrigin &&
+        req.headers.origin != `http://${checkLocalhostOrigin}`
+      ) {
+        console.log('API origin check FAILED');
+        console.log('HEADERS', { ...req.headers, authorization: '***' });
+        return res.status(403).json({ error: 'Not authorized!' });
+      }
+      if (!req.headers.origin && req.headers.host != checkLocalhostOrigin) {
+        console.log('API host check FAILED');
+        console.log('HEADERS', { ...req.headers, authorization: '***' });
+        return res.status(403).json({ error: 'Not authorized!' });
+      }
+    }
+    next();
+  });
+
  app.use(cors());
  app.use(bodyParser.json({ limit: '50mb' }));

@@ -72,6 +103,7 @@ function start(argument = null) {
  useController(app, '/plugins', plugins);
  useController(app, '/files', files);
  useController(app, '/scheduler', scheduler);
+  useController(app, '/query-history', queryHistory);

  // if (process.env.PAGES_DIRECTORY) {
  //   app.use('/pages', express.static(process.env.PAGES_DIRECTORY));
@@ -79,29 +111,32 @@ function start(argument = null) {

  app.use('/runners/data', express.static(rundir()));

-  if (fs.existsSync('/home/dbgate-docker/public')) {
+  if (platformInfo.isDocker) {
    // server static files inside docker container
    app.use(express.static('/home/dbgate-docker/public'));
  } else {
-    if (argument != 'startNodeWeb') {
+    if (!platformInfo.isNpmDist) {
      app.get('/', (req, res) => {
        res.send('DbGate API');
      });
    }
  }

-  if (argument == '--dynport') {
+  if (processArgs.dynport) {
    childProcessChecker();

-    findFreePort(53911, function (err, port) {
+    authorization = crypto.randomBytes(32).toString('hex');
+
+    getPort().then(port => {
+      checkLocalhostOrigin = `localhost:${port}`;
      server.listen(port, () => {
        console.log(`DbGate API listening on port ${port}`);
-        process.send({ msgtype: 'listening', port });
+        process.send({ msgtype: 'listening', port, authorization });
      });
    });
-  } else if (argument == 'startNodeWeb') {
+  } else if (platformInfo.isNpmDist) {
    app.use(express.static(path.join(__dirname, '../../dbgate-web/public')));
-    findFreePort(5000, function (err, port) {
+    getPort({ port: 5000 }).then(port => {
      server.listen(port, () => {
        console.log(`DbGate API listening on port ${port}`);
      });
@@ -2,6 +2,25 @@ const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
+const _ = require('lodash');
+
+function pickSafeConnectionInfo(connection) {
+  return _.mapValues(connection, (v, k) => {
+    if (k == 'engine' || k == 'port' || k == 'authType' || k == 'sshMode' || k == 'passwordMode') return v;
+    if (v === null || v === true || v === false) return v;
+    if (v) return '***';
+    return undefined;
+  });
+}
+
+const formatErrorDetail = (e, connection) => `${e.stack}
+
+Error JSON: ${JSON.stringify(e, undefined, 2)}
+
+Connection: ${JSON.stringify(pickSafeConnectionInfo(connection), undefined, 2)}
+
+Platform: ${process.platform}
+`;

 function start() {
  childProcessChecker();
@@ -14,7 +33,11 @@ function start() {
      process.send({ msgtype: 'connected', ...res });
    } catch (e) {
      console.error(e);
-      process.send({ msgtype: 'error', error: e.message });
+      process.send({
+        msgtype: 'error',
+        error: e.message,
+        detail: formatErrorDetail(e, connection),
+      });
    }
  });
 }
@@ -1,5 +1,7 @@
 const stableStringify = require('json-stable-stringify');
+const { splitQuery } = require('dbgate-query-splitter');
 const childProcessChecker = require('../utility/childProcessChecker');
+const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const connectUtility = require('../utility/connectUtility');
 const { handleProcessCommunication } = require('../utility/processComm');
@@ -11,6 +13,7 @@ let afterConnectCallbacks = [];
 let analysedStructure = null;
 let lastPing = null;
 let lastStatus = null;
+let analysedTime = 0;

 async function checkedAsyncCall(promise) {
  try {
@@ -27,21 +30,42 @@ async function checkedAsyncCall(promise) {
  }
 }

+let loadingModel = false;
+
 async function handleFullRefresh() {
+  loadingModel = true;
  const driver = requireEngineDriver(storedConnection);
+  setStatusName('loadStructure');
  analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection));
+  analysedTime = new Date().getTime();
  process.send({ msgtype: 'structure', structure: analysedStructure });
+  process.send({ msgtype: 'structureTime', analysedTime });
  setStatusName('ok');
+  loadingModel = false;
 }

-async function handleIncrementalRefresh() {
+async function handleIncrementalRefresh(forceSend) {
+  loadingModel = true;
  const driver = requireEngineDriver(storedConnection);
+  setStatusName('checkStructure');
  const newStructure = await checkedAsyncCall(driver.analyseIncremental(systemConnection, analysedStructure));
+  analysedTime = new Date().getTime();
  if (newStructure != null) {
    analysedStructure = newStructure;
+  }
+
+  if (forceSend || newStructure != null) {
    process.send({ msgtype: 'structure', structure: analysedStructure });
  }
+
+  process.send({ msgtype: 'structureTime', analysedTime });
  setStatusName('ok');
+  loadingModel = false;
+}
+
+function handleSyncModel() {
+  if (loadingModel) return;
+  handleIncrementalRefresh();
 }

 function setStatus(status) {
@@ -56,20 +80,34 @@ function setStatusName(name) {
  setStatus({ name });
 }

-async function handleConnect({ connection, structure }) {
+async function readVersion() {
+  const driver = requireEngineDriver(storedConnection);
+  const version = await driver.getVersion(systemConnection);
+  process.send({ msgtype: 'version', version });
+}
+
+async function handleConnect({ connection, structure, globalSettings }) {
  storedConnection = connection;
  lastPing = new Date().getTime();

  if (!structure) setStatusName('pending');
  const driver = requireEngineDriver(storedConnection);
  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection));
+  readVersion();
  if (structure) {
    analysedStructure = structure;
-    handleIncrementalRefresh();
+    handleIncrementalRefresh(true);
  } else {
    handleFullRefresh();
  }
-  setInterval(handleIncrementalRefresh, 30 * 1000);
+
+  if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
+    setInterval(
+      handleIncrementalRefresh,
+      extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 3, 3600) * 1000
+    );
+  }
+
  for (const [resolve] of afterConnectCallbacks) {
    resolve();
  }
@@ -83,6 +121,17 @@ function waitConnected() {
  });
 }

+async function handleRunScript({ msgid, sql }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    await driver.script(systemConnection, sql);
+    process.send({ msgtype: 'response', msgid });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
 async function handleQueryData({ msgid, sql }) {
  await waitConnected();
  const driver = requireEngineDriver(storedConnection);
@@ -105,6 +154,17 @@ async function handleCollectionData({ msgid, options }) {
  }
 }

+async function handleUpdateCollection({ msgid, changeSet }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    const result = await driver.updateCollection(systemConnection, changeSet);
+    process.send({ msgtype: 'response', msgid, result });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
 async function handleSqlPreview({ msgid, objects, options }) {
  await waitConnected();
  const driver = requireEngineDriver(storedConnection);
@@ -140,9 +200,12 @@ function handlePing() {
 const messageHandlers = {
  connect: handleConnect,
  queryData: handleQueryData,
+  runScript: handleRunScript,
+  updateCollection: handleUpdateCollection,
  collectionData: handleCollectionData,
  sqlPreview: handleSqlPreview,
  ping: handlePing,
+  syncModel: handleSyncModel,
  // runCommand: handleRunCommand,
 };

@@ -1,4 +1,5 @@
 const stableStringify = require('json-stable-stringify');
+const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
@@ -31,6 +32,12 @@ async function handleRefresh() {
  }
 }

+async function readVersion() {
+  const driver = requireEngineDriver(storedConnection);
+  const version = await driver.getVersion(systemConnection);
+  process.send({ msgtype: 'version', version });
+}
+
 function setStatus(status) {
  const statusString = stableStringify(status);
  if (lastStatus != statusString) {
@@ -45,14 +52,18 @@ function setStatusName(name) {

 async function handleConnect(connection) {
  storedConnection = connection;
+  const { globalSettings } = storedConnection;
  setStatusName('pending');
  lastPing = new Date().getTime();

  const driver = requireEngineDriver(storedConnection);
  try {
    systemConnection = await connectUtility(driver, storedConnection);
+    readVersion();
    handleRefresh();
-    setInterval(handleRefresh, 30 * 1000);
+    if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
+      setInterval(handleRefresh, extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000);
+    }
  } catch (err) {
    setStatus({
      name: 'error',
@@ -71,7 +82,11 @@ async function handleCreateDatabase({ name }) {
  const driver = requireEngineDriver(storedConnection);
  systemConnection = await connectUtility(driver, storedConnection);
  console.log(`RUNNING SCRIPT: CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
-  await driver.query(systemConnection, `CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
+  if (driver.createDatabase) {
+    await driver.createDatabase(systemConnection, name);
+  } else {
+    await driver.query(systemConnection, `CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
+  }
  await handleRefresh();
 }

@@ -3,7 +3,7 @@ const path = require('path');
 const fs = require('fs');
 const _ = require('lodash');
 const childProcessChecker = require('../utility/childProcessChecker');
-const goSplit = require('../utility/goSplit');
+const { splitQuery } = require('dbgate-query-splitter');

 const { jsldir } = require('../utility/directories');
 const requireEngineDriver = require('../utility/requireEngineDriver');
@@ -17,12 +17,18 @@ let afterConnectCallbacks = [];
 // let currentHandlers = [];

 class TableWriter {
-  constructor(columns, resultIndex) {
+  constructor(structure, resultIndex) {
    this.jslid = uuidv1();
    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
    this.currentRowCount = 0;
    this.currentChangeIndex = 1;
-    fs.writeFileSync(this.currentFile, JSON.stringify({ columns }) + '\n');
+    fs.writeFileSync(
+      this.currentFile,
+      JSON.stringify({
+        ...structure,
+        __isStreamHeader: true,
+      }) + '\n'
+    );
    this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
    this.writeCurrentStats(false, false);
    this.resultIndex = resultIndex;
@@ -92,7 +98,7 @@ class StreamHandler {

  recordset(columns) {
    this.closeCurrentWriter();
-    this.currentWriter = new TableWriter(columns, this.resultIndexHolder.value);
+    this.currentWriter = new TableWriter(Array.isArray(columns) ? { columns } : columns, this.resultIndexHolder.value);
    this.resultIndexHolder.value += 1;

    // this.writeCurrentStats();
@@ -160,7 +166,7 @@ async function handleExecuteQuery({ sql }) {
  const resultIndexHolder = {
    value: 0,
  };
-  for (const sqlItem of goSplit(sql)) {
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('stream'))) {
    await handleStream(driver, resultIndexHolder, sqlItem);
    // const handler = new StreamHandler(resultIndex);
    // const stream = await driver.stream(systemConnection, sqlItem, handler);
@@ -1,9 +1,13 @@
+const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
+
 function copyStream(input, output) {
  return new Promise((resolve, reject) => {
+    const ensureHeader = new EnsureStreamHeaderStream();
    const finisher = output['finisher'] || output;
    finisher.on('finish', resolve);
    finisher.on('error', reject);
-    input.pipe(output);
+    input.pipe(ensureHeader);
+    ensureHeader.pipe(output);
  });
 }

@@ -1,6 +1,5 @@
-const goSplit = require('../utility/goSplit');
+const { splitQuery } = require('dbgate-query-splitter');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');

 async function executeQuery({ connection, sql }) {
@@ -10,9 +9,9 @@ async function executeQuery({ connection, sql }) {
  const pool = await connectUtility(driver, connection);
  console.log(`Connected.`);

-  for (const sqlItem of goSplit(sql)) {
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('script'))) {
    console.log('Executing query', sqlItem);
-    await driver.query(pool, sqlItem);
+    await driver.query(pool, sqlItem, { discardResult: true });
  }

  console.log(`Query finished`);
@@ -6,6 +6,7 @@ const copyStream = require('./copyStream');
 const fakeObjectReader = require('./fakeObjectReader');
 const consoleObjectWriter = require('./consoleObjectWriter');
 const jsonLinesWriter = require('./jsonLinesWriter');
+const jsonArrayWriter = require('./jsonArrayWriter');
 const jsonLinesReader = require('./jsonLinesReader');
 const jslDataReader = require('./jslDataReader');
 const archiveWriter = require('./archiveWriter');
@@ -26,6 +27,7 @@ const dbgateApi = {
  tableReader,
  copyStream,
  jsonLinesWriter,
+  jsonArrayWriter,
  jsonLinesReader,
  fakeObjectReader,
  consoleObjectWriter,
@@ -0,0 +1,52 @@
+const fs = require('fs');
+const stream = require('stream');
+
+class StringifyStream extends stream.Transform {
+  constructor() {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.wasRecord = false;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+
+    if (!this.wasHeader) {
+      skip =
+        chunk.__isStreamHeader ||
+        // TODO remove isArray test
+        Array.isArray(chunk.columns);
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      if (!this.wasRecord) {
+        this.push('[\n');
+      } else {
+        this.push(',\n');
+      }
+      this.wasRecord = true;
+
+      this.push(JSON.stringify(chunk));
+    }
+    done();
+  }
+
+  _flush(done) {
+    if (!this.wasRecord) {
+      this.push('[]\n');
+    } else {
+      this.push('\n]\n');
+    }
+    done();
+  }
+}
+
+async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) {
+  console.log(`Writing file ${fileName}`);
+  const stringify = new StringifyStream();
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = jsonArrayWriter;
@@ -3,9 +3,8 @@ const stream = require('stream');
 const byline = require('byline');

 class ParseStream extends stream.Transform {
-  constructor({ header, limitRows }) {
+  constructor({ limitRows }) {
    super({ objectMode: true });
-    this.header = header;
    this.wasHeader = false;
    this.limitRows = limitRows;
    this.rowsWritten = 0;
@@ -13,7 +12,14 @@ class ParseStream extends stream.Transform {
  _transform(chunk, encoding, done) {
    const obj = JSON.parse(chunk);
    if (!this.wasHeader) {
-      if (!this.header) this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
+      if (
+        !obj.__isStreamHeader &&
+        // TODO remove isArray test
+        !Array.isArray(obj.columns)
+      ) {
+        this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
+      }
+      
      this.wasHeader = true;
    }
    if (!this.limitRows || this.rowsWritten < this.limitRows) {
@@ -24,12 +30,12 @@ class ParseStream extends stream.Transform {
  }
 }

-async function jsonLinesReader({ fileName, encoding = 'utf-8', header = true, limitRows = undefined }) {
+async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
  console.log(`Reading file ${fileName}`);

  const fileStream = fs.createReadStream(fileName, encoding);
  const liner = byline(fileStream);
-  const parser = new ParseStream({ header, limitRows });
+  const parser = new ParseStream({ limitRows });
  liner.pipe(parser);
  return parser;
 }
@@ -8,10 +8,16 @@ class StringifyStream extends stream.Transform {
    this.wasHeader = false;
  }
  _transform(chunk, encoding, done) {
+    let skip = false;
    if (!this.wasHeader) {
-      if (this.header) this.push(JSON.stringify(chunk) + '\n');
+      skip =
+        (chunk.__isStreamHeader ||
+          // TODO remove isArray test
+          Array.isArray(chunk.columns)) &&
+        !this.header;
      this.wasHeader = true;
-    } else {
+    }
+    if (!skip) {
      this.push(JSON.stringify(chunk) + '\n');
    }
    done();
@@ -1,6 +1,8 @@
 const path = require('path');
-const { pluginsdir } = require('../utility/directories');
+const fs = require('fs');
+const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories');
 const nativeModules = require('../nativeModules');
+const platformInfo = require('../utility/platformInfo');

 const loadedPlugins = {};

@@ -8,20 +10,19 @@ const dbgateEnv = {
  dbgateApi: null,
  nativeModules,
 };
-
 function requirePlugin(packageName, requiredPlugin = null) {
  if (!packageName) throw new Error('Missing packageName in plugin');
  if (loadedPlugins[packageName]) return loadedPlugins[packageName];

  if (requiredPlugin == null) {
    let module;
-    const modulePath = path.join(pluginsdir(), packageName, 'dist', 'backend.js');
+    const modulePath = getPluginBackendPath(packageName);
    console.log(`Loading module ${packageName} from ${modulePath}`);
    try {
      // @ts-ignore
      module = __non_webpack_require__(modulePath);
    } catch (err) {
-      console.error('Failed load webpacked module', err);
+      // console.log('Failed load webpacked module', err.message);
      module = require(modulePath);
    }
    requiredPlugin = module.__esModule ? module.default : module;
@@ -1,7 +1,8 @@
 const childProcessChecker = require('../utility/childProcessChecker');
+const processArgs = require('../utility/processArgs');

 async function runScript(func) {
-  if (process.argv.includes('--checkParent')) {
+  if (processArgs.checkParent) {
    childProcessChecker();
  }
  try {
@@ -1,6 +1,5 @@
 const { quoteFullName, fullNameToString } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
 const connectUtility = require('../utility/connectUtility');

 async function tableReader({ connection, pureName, schemaName }) {
@@ -10,6 +9,13 @@ async function tableReader({ connection, pureName, schemaName }) {

  const fullName = { pureName, schemaName };

+  if (driver.dialect.nosql) {
+    // @ts-ignore
+    console.log(`Reading collection ${fullNameToString(fullName)}`);
+    // @ts-ignore
+    return await driver.readQuery(pool, JSON.stringify(fullName));
+  }
+
  const table = await driver.analyseSingleObject(pool, fullName, 'tables');
  const query = `select * from ${quoteFullName(driver.dialect, fullName)}`;
  if (table) {
@@ -29,7 +29,7 @@ class DatastoreProxy {

  async ensureSubprocess() {
    if (!this.subprocess) {
-      this.subprocess = fork(process.argv[1], ['jslDatastoreProcess', ...process.argv.slice(3)]);
+      this.subprocess = fork(process.argv[1], ['--start-process', 'jslDatastoreProcess', ...process.argv.slice(3)]);

      this.subprocess.on('message', message => {
        // @ts-ignore
@@ -0,0 +1,35 @@
+const stream = require('stream');
+
+class EnsureStreamHeaderStream extends stream.Transform {
+  constructor() {
+    super({ objectMode: true });
+    this.wasHeader = false;
+  }
+  _transform(chunk, encoding, done) {
+    if (!this.wasHeader) {
+      if (chunk.__isDynamicStructure) {
+        // ignore dynamic structure header
+        done();
+        return;
+      }
+
+      if (
+        !chunk.__isStreamHeader &&
+        // TODO remove isArray test
+        !Array.isArray(chunk.columns)
+      ) {
+        this.push({
+          __isStreamHeader: true,
+          __isComputedStructure: true,
+          columns: Object.keys(chunk).map(columnName => ({ columnName })),
+        });
+      }
+
+      this.wasHeader = true;
+    }
+    this.push(chunk);
+    done();
+  }
+}
+
+module.exports = EnsureStreamHeaderStream;
@@ -7,6 +7,7 @@ const { getSshTunnelProxy } = require('./sshTunnelProxy');

 async function connectUtility(driver, storedConnection) {
  const connection = {
+    database: storedConnection.defaultDatabase,
    ...decryptConnection(storedConnection),
  };

@@ -38,6 +39,10 @@ async function connectUtility(driver, storedConnection) {
      connection.ssl.key = await fs.readFile(connection.sslKeyFile);
    }

+    if (connection.sslCertFilePassword) {
+      connection.ssl.password = connection.sslCertFilePassword;
+    }
+
    if (!connection.ssl.key && !connection.ssl.ca && !connection.ssl.cert) {
      // TODO: provide this as an option in settings
      // or per-connection as 'reject self-signed certs'
@@ -2,6 +2,7 @@ const os = require('os');
 const path = require('path');
 const fs = require('fs');
 const cleanDirectory = require('./cleanDirectory');
+const platformInfo = require('./platformInfo');

 const createDirectories = {};
 const ensureDirectory = (dir, clean) => {
@@ -39,6 +40,37 @@ const pluginsdir = dirFunc('plugins');
 const archivedir = dirFunc('archive');
 const filesdir = dirFunc('files');

+function packagedPluginsDir() {
+  if (platformInfo.isDevMode) {
+    return path.resolve(__dirname, '../../../../plugins');
+  }
+  if (platformInfo.isDocker) {
+    return '/home/dbgate-docker/plugins';
+  }
+  if (platformInfo.isNpmDist) {
+    // node_modules
+    return global['dbgateApiPackagedPluginsPath'];
+  }
+  if (platformInfo.isElectronBundle) {
+    return path.resolve(__dirname, '../../plugins');
+  }
+  return null;
+}
+
+const packagedPluginList =
+  packagedPluginsDir() != null ? fs.readdirSync(packagedPluginsDir()).filter(x => x.startsWith('dbgate-plugin-')) : [];
+
+function getPluginBackendPath(packageName) {
+  if (packagedPluginList.includes(packageName)) {
+    if (platformInfo.isDevMode) {
+      return path.join(packagedPluginsDir(), packageName, 'src', 'backend', 'index.js');
+    }
+    return path.join(packagedPluginsDir(), packageName, 'dist', 'backend.js');
+  }
+
+  return path.join(pluginsdir(), packageName, 'dist', 'backend.js');
+}
+
 module.exports = {
  datadir,
  jsldir,
@@ -48,4 +80,7 @@ module.exports = {
  ensureDirectory,
  pluginsdir,
  filesdir,
+  packagedPluginsDir,
+  packagedPluginList,
+  getPluginBackendPath,
 };
@@ -1,18 +0,0 @@
-function goSplit(sql) {
-  if (!sql) return [];
-  const lines = sql.split('\n');
-  const res = [];
-  let buffer = '';
-  for (const line of lines) {
-    if (/^\s*go\s*$/i.test(line)) {
-      if (buffer.trim()) res.push(buffer);
-      buffer = '';
-    } else {
-      buffer += line + '\n';
-    }
-  }
-  if (buffer.trim()) res.push(buffer);
-  return res;
-}
-
-module.exports = goSplit;
@@ -1,26 +1,42 @@
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
+const processArgs = require('./processArgs');

-const p = process;
-const platform = p.env.OS_OVERRIDE ? p.env.OS_OVERRIDE : p.platform;
+const platform = process.env.OS_OVERRIDE ? process.env.OS_OVERRIDE : process.platform;
 const isWindows = platform === 'win32';
 const isMac = platform === 'darwin';
 const isLinux = platform === 'linux';
-const isDocker = fs.existsSync('/home/dbgate-docker/build');
+const isDocker = fs.existsSync('/home/dbgate-docker/public');
+const isDevMode = process.env.DEVMODE == '1';
+const isNpmDist = !!global['dbgateApiModulePath'];
+
+// function moduleAvailable(name) {
+//   try {
+//     require.resolve(name);
+//     return true;
+//   } catch (e) {
+//     return false;
+//   }
+// }
+
+const isElectronBundle = processArgs.isElectronBundle;

 const platformInfo = {
  isWindows,
  isMac,
  isLinux,
  isDocker,
-  isSnap: p.env.ELECTRON_SNAP == 'true',
-  isPortable: isWindows && p.env.PORTABLE_EXECUTABLE_DIR,
-  isAppImage: p.env.DESKTOPINTEGRATION === 'AppImageLauncher',
-  sshAuthSock: p.env.SSH_AUTH_SOCK,
+  isElectronBundle,
+  isDevMode,
+  isNpmDist,
+  isSnap: process.env.ELECTRON_SNAP == 'true',
+  isPortable: isWindows && process.env.PORTABLE_EXECUTABLE_DIR,
+  isAppImage: process.env.DESKTOPINTEGRATION === 'AppImageLauncher',
+  sshAuthSock: process.env.SSH_AUTH_SOCK,
  environment: process.env.NODE_ENV,
  platform,
-  runningInWebpack: !!p.env.WEBPACK_DEV_SERVER_URL,
+  runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
  defaultKeyFile: path.join(os.homedir(), '.ssh/id_rsa'),
 };

@@ -0,0 +1,21 @@
+function getNamedArg(name) {
+  const argIndex = process.argv.indexOf(name);
+  if (argIndex > 0) {
+    return process.argv[argIndex + 1];
+  }
+  return null;
+}
+
+const checkParent = process.argv.includes('--checkParent');
+const dynport = process.argv.includes('--dynport');
+const nativeModules = getNamedArg('--native-modules');
+const startProcess = getNamedArg('--start-process');
+const isElectronBundle = process.argv.includes('--is-electron-bundle');
+
+module.exports = {
+  checkParent,
+  nativeModules,
+  startProcess,
+  dynport,
+  isElectronBundle,
+};
@@ -15,7 +15,7 @@ function requireEngineDriver(connection) {
  if (engine.includes('@')) {
    const [shortName, packageName] = engine.split('@');
    const plugin = requirePlugin(packageName);
-    return plugin.driver;
+    return plugin.drivers.find(x => x.engine == engine);
  }
  throw new Error(`Could not found engine driver ${engine}`);
 }
@@ -4,6 +4,8 @@ const portfinder = require('portfinder');
 const stableStringify = require('json-stable-stringify');
 const _ = require('lodash');
 const platformInfo = require('./platformInfo');
+const AsyncLock = require('async-lock');
+const lock = new AsyncLock();

 const sshConnectionCache = {};
 const sshTunnelCache = {};
@@ -34,7 +36,7 @@ async function getSshConnection(connection) {
    password: connection.sshMode == 'userPassword' ? connection.sshPassword : undefined,
    agentSocket: connection.sshMode == 'agent' ? platformInfo.sshAuthSock : undefined,
    privateKey:
-      connection.sshMode == 'keyFile' && connection.sshKeyFile ? await fs.readFile(connection.sshKeyFile) : undefined,
+      connection.sshMode == 'keyFile' && connection.sshKeyfile ? await fs.readFile(connection.sshKeyfile) : undefined,
    skipAutoPrivateKey: true,
    noReadline: true,
  };
@@ -45,36 +47,43 @@ async function getSshConnection(connection) {
 }

 async function getSshTunnel(connection) {
-  const sshConn = await getSshConnection(connection);
  const tunnelCacheKey = stableStringify(_.pick(connection, TUNNEL_FIELDS));
-  if (sshTunnelCache[tunnelCacheKey]) return sshTunnelCache[tunnelCacheKey];

-  const localPort = await portfinder.getPortPromise({ port: 10000, stopPort: 60000 });
-  // workaround for `getPortPromise` not releasing the port quickly enough
-  await new Promise(resolve => setTimeout(resolve, 500));
-  const tunnelConfig = {
-    fromPort: localPort,
-    toPort: connection.port,
-    toHost: connection.server,
-  };
-  try {
-    const tunnel = await sshConn.forward(tunnelConfig);
-    console.log(
-      `Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
-    );
+  return await lock.acquire(tunnelCacheKey, async () => {
+    const sshConn = await getSshConnection(connection);
+    if (sshTunnelCache[tunnelCacheKey]) return sshTunnelCache[tunnelCacheKey];

-    sshTunnelCache[tunnelCacheKey] = {
-      state: 'ok',
-      localPort,
+    const localPort = await portfinder.getPortPromise({ port: 10000, stopPort: 60000 });
+    // workaround for `getPortPromise` not releasing the port quickly enough
+    await new Promise(resolve => setTimeout(resolve, 500));
+    const tunnelConfig = {
+      fromPort: localPort,
+      toPort: connection.port,
+      toHost: connection.server,
    };
-    return sshTunnelCache[tunnelCacheKey];
-  } catch (err) {
-    // error is not cached
-    return {
-      state: 'error',
-      message: err.message,
-    };
-  }
+    try {
+      console.log(
+        `Creating SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
+      );
+
+      const tunnel = await sshConn.forward(tunnelConfig);
+      console.log(
+        `Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
+      );
+
+      sshTunnelCache[tunnelCacheKey] = {
+        state: 'ok',
+        localPort,
+      };
+      return sshTunnelCache[tunnelCacheKey];
+    } catch (err) {
+      // error is not cached
+      return {
+        state: 'error',
+        message: err.message,
+      };
+    }
+  });
 }

 module.exports = {
@@ -0,0 +1,9 @@
+const crypto = require('crypto');
+
+function timingSafeCheckToken(a, b) {
+  if (!a || !b) return false;
+  if (a.length != b.length) return false;
+  return crypto.timingSafeEqual(Buffer.from(a), Buffer.from(b));
+}
+
+module.exports = timingSafeCheckToken;
@@ -1,10 +1,9 @@
 {
-  "version": "4.0.0",
+  "version": "4.1.1",
  "name": "dbgate-datalib",
  "main": "lib/index.js",
  "typings": "lib/index.d.ts",
  "scripts": {
-    "prepare": "yarn build",
    "build": "tsc",
    "start": "tsc --watch"
  },
@@ -12,11 +11,11 @@
    "lib"
  ],
  "dependencies": {
-    "dbgate-sqltree": "^4.0.0",
-    "dbgate-filterparser": "^4.0.0"
+    "dbgate-sqltree": "^4.1.1",
+    "dbgate-filterparser": "^4.1.1"
  },
  "devDependencies": {
-    "dbgate-types": "^4.0.0",
+    "dbgate-types": "^4.1.1",
    "@types/node": "^13.7.0",
    "typescript": "^3.7.5"
  }
@@ -6,6 +6,7 @@ export interface ChangeSetItem {
  pureName: string;
  schemaName?: string;
  insertedRowIndex?: number;
+  document?: any;
  condition?: { [column: string]: string };
  fields?: { [column: string]: string };
 }
@@ -117,6 +118,43 @@ export function setChangeSetValue(
  };
 }

+export function setChangeSetRowData(changeSet: ChangeSet, definition: ChangeSetRowDefinition, document: any): ChangeSet {
+  if (!changeSet || !definition) return changeSet;
+  let [fieldName, existingItem] = findExistingChangeSetItem(changeSet, definition);
+  if (fieldName == 'deletes') {
+    changeSet = revertChangeSetRowChanges(changeSet, definition);
+    [fieldName, existingItem] = findExistingChangeSetItem(changeSet, definition);
+  }
+  if (existingItem) {
+    return {
+      ...changeSet,
+      [fieldName]: changeSet[fieldName].map(item =>
+        item == existingItem
+          ? {
+              ...item,
+              fields: {},
+              document,
+            }
+          : item
+      ),
+    };
+  }
+
+  return {
+    ...changeSet,
+    [fieldName]: [
+      ...changeSet[fieldName],
+      {
+        pureName: definition.pureName,
+        schemaName: definition.schemaName,
+        condition: definition.condition,
+        insertedRowIndex: definition.insertedRowIndex,
+        document,
+      },
+    ],
+  };
+}
+
 // export function batchUpdateChangeSet(
 //   changeSet: ChangeSet,
 //   rowDefinitions: ChangeSetRowDefinition[],
@@ -332,6 +370,7 @@ export function getChangeSetInsertedRows(changeSet: ChangeSet, name?: NamedObjec
 }

 export function changeSetInsertNewRow(changeSet: ChangeSet, name?: NamedObjectInfo): ChangeSet {
+  console.log('INSERT', name);
  const insertedRows = getChangeSetInsertedRows(changeSet, name);
  return {
    ...changeSet,
@@ -25,6 +25,62 @@ function createHeaderText(path) {
  return res;
 }

+function getColumnsForObject(basePath, obj, res: any[], display) {
+  for (const name of getObjectKeys(obj)) {
+    const uniqueName = [...basePath, name].join('.');
+    let column = res.find(x => x.uniqueName == uniqueName);
+    if (!column) {
+      column = getDisplayColumn(basePath, name, display);
+      if (basePath.length > 0) {
+        const lastIndex1 = _.findLastIndex(res, x => x.parentHeaderText.startsWith(column.parentHeaderText));
+        const lastIndex2 = _.findLastIndex(res, x => x.headerText == column.parentHeaderText);
+        // console.log(uniqueName, lastIndex1, lastIndex2);
+        if (lastIndex1 >= 0) res.splice(lastIndex1 + 1, 0, column);
+        else if (lastIndex2 >= 0) res.splice(lastIndex2 + 1, 0, column);
+        else res.push(column);
+      } else {
+        res.push(column);
+      }
+    }
+    if (_.isPlainObject(obj[name]) || _.isArray(obj[name])) {
+      column.isExpandable = true;
+    }
+
+    if (display.isExpandedColumn(column.uniqueName)) {
+      getColumnsForObject([...basePath, name], obj[name], res, display);
+    }
+  }
+}
+
+function getDisplayColumn(basePath, columnName, display) {
+  const uniquePath = [...basePath, columnName];
+  const uniqueName = uniquePath.join('.');
+  return {
+    columnName,
+    headerText: createHeaderText(uniquePath),
+    uniqueName,
+    uniquePath,
+    isStructured: true,
+    parentHeaderText: createHeaderText(basePath),
+    filterType: 'mongo',
+    pureName: display.collection?.pureName,
+    schemaName: display.collection?.schemaName,
+  };
+}
+
+export function analyseCollectionDisplayColumns(rows, display) {
+  const res = [];
+  for (const row of rows || []) {
+    getColumnsForObject([], row, res, display);
+  }
+  return (
+    res.map(col => ({
+      ...col,
+      isChecked: display.isColumnChecked(col),
+    })) || []
+  );
+}
+
 export class CollectionGridDisplay extends GridDisplay {
  constructor(
    public collection: CollectionInfo,
@@ -36,65 +92,13 @@ export class CollectionGridDisplay extends GridDisplay {
    loadedRows
  ) {
    super(config, setConfig, cache, setCache, driver);
-    this.columns = this.getDisplayColumns(loadedRows || []);
+    this.columns = analyseCollectionDisplayColumns(loadedRows, this);
    this.filterable = true;
    this.sortable = true;
-    this.editable = false;
+    this.editable = true;
    this.supportsReload = true;
    this.isDynamicStructure = true;
-  }
-
-  getDisplayColumns(rows) {
-    const res = [];
-    for (const row of rows) {
-      this.getColumnsForObject([], row, res);
-    }
-    return (
-      res.map(col => ({
-        ...col,
-        isChecked: this.isColumnChecked(col),
-      })) || []
-    );
-  }
-
-  getColumnsForObject(basePath, obj, res: any[]) {
-    for (const name of getObjectKeys(obj)) {
-      const uniqueName = [...basePath, name].join('.');
-      let column = res.find(x => x.uniqueName == uniqueName);
-      if (!column) {
-        column = this.getDisplayColumn(basePath, name);
-        if (basePath.length > 0) {
-          const lastIndex1 = _.findLastIndex(res, x => x.parentHeaderText.startsWith(column.parentHeaderText));
-          const lastIndex2 = _.findLastIndex(res, x => x.headerText == column.parentHeaderText);
-          // console.log(uniqueName, lastIndex1, lastIndex2);
-          if (lastIndex1 >= 0) res.splice(lastIndex1 + 1, 0, column);
-          else if (lastIndex2 >= 0) res.splice(lastIndex2 + 1, 0, column);
-          else res.push(column);
-        } else {
-          res.push(column);
-        }
-      }
-      if (_.isPlainObject(obj[name]) || _.isArray(obj[name])) {
-        column.isExpandable = true;
-      }
-
-      if (this.isExpandedColumn(column.uniqueName)) {
-        this.getColumnsForObject([...basePath, name], obj[name], res);
-      }
-    }
-  }
-
-  getDisplayColumn(basePath, columnName) {
-    const uniquePath = [...basePath, columnName];
-    const uniqueName = uniquePath.join('.');
-    return {
-      columnName,
-      headerText: createHeaderText(uniquePath),
-      uniqueName,
-      uniquePath,
-      isStructured: true,
-      parentHeaderText: createHeaderText(basePath),
-      filterType: 'mongo',
-    };
+    this.changeSetKeyFields = ['_id'];
+    this.baseCollection = collection;
  }
 }
@@ -1,17 +1,14 @@
 import _ from 'lodash';
 import { GridConfig, GridCache, GridConfigColumns, createGridCache, GroupFunc } from './GridConfig';
-import { ForeignKeyInfo, TableInfo, ColumnInfo, EngineDriver, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
-import { parseFilter, getFilterType, getFilterValueExpression } from 'dbgate-filterparser';
-import { filterName } from './filterName';
-import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
-import { Expression, Select, treeToSql, dumpSqlSelect, Condition } from 'dbgate-sqltree';
-import { isTypeLogical } from 'dbgate-tools';
+import { TableInfo, EngineDriver, DatabaseInfo, SqlDialect } from 'dbgate-types';
+import { getFilterValueExpression } from 'dbgate-filterparser';
 import { ChangeCacheFunc, ChangeConfigFunc, DisplayColumn } from './GridDisplay';

 export class FormViewDisplay {
  isLoadedCorrectly = true;
  columns: DisplayColumn[];
  public baseTable: TableInfo;
+  dialect: SqlDialect;

  constructor(
    public config: GridConfig,
@@ -19,8 +16,11 @@ export class FormViewDisplay {
    public cache: GridCache,
    protected setCache: ChangeCacheFunc,
    public driver?: EngineDriver,
-    public dbinfo: DatabaseInfo = null
-  ) {}
+    public dbinfo: DatabaseInfo = null,
+    public serverVersion = null
+  ) {
+    this.dialect = (driver?.dialectByVersion && driver?.dialectByVersion(serverVersion)) || driver?.dialect;
+  }

  addFilterColumn(column) {
    if (!column) return;
@@ -33,6 +33,7 @@ export interface GridConfig extends GridConfigColumns {
  formViewKey?: { [uniqueName: string]: string };
  formViewKeyRequested?: { [uniqueName: string]: string };
  formFilterColumns: string[];
+  formColumnFilterText?: string;
 }

 export interface GridCache {
@@ -1,8 +1,17 @@
 import _ from 'lodash';
 import { GridConfig, GridCache, GridConfigColumns, createGridCache, GroupFunc } from './GridConfig';
-import { ForeignKeyInfo, TableInfo, ColumnInfo, EngineDriver, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
+import {
+  ForeignKeyInfo,
+  TableInfo,
+  ColumnInfo,
+  EngineDriver,
+  NamedObjectInfo,
+  DatabaseInfo,
+  CollectionInfo,
+  SqlDialect,
+} from 'dbgate-types';
 import { parseFilter, getFilterType } from 'dbgate-filterparser';
-import { filterName } from './filterName';
+import { filterName } from 'dbgate-tools';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
 import { Expression, Select, treeToSql, dumpSqlSelect, Condition } from 'dbgate-sqltree';
 import { isTypeLogical } from 'dbgate-tools';
@@ -52,10 +61,18 @@ export abstract class GridDisplay {
    public cache: GridCache,
    protected setCache: ChangeCacheFunc,
    public driver?: EngineDriver,
-    public dbinfo: DatabaseInfo = null
-  ) {}
+    public dbinfo: DatabaseInfo = null,
+    public serverVersion = null
+  ) {
+    this.dialect = (driver?.dialectByVersion && driver?.dialectByVersion(serverVersion)) || driver?.dialect;
+  }
+  dialect: SqlDialect;
  columns: DisplayColumn[];
  baseTable?: TableInfo;
+  baseCollection?: CollectionInfo;
+  get baseTableOrCollection(): NamedObjectInfo {
+    return this.baseTable || this.baseCollection;
+  }
  changeSetKeyFields: string[] = null;
  sortable = false;
  filterable = false;
@@ -98,11 +115,7 @@ export abstract class GridDisplay {
  }

  reload() {
-    this.setCache(cache => ({
-      // ...cache,
-      ...createGridCache(),
-      refreshTime: new Date().getTime(),
-    }));
+    this.setCache(reloadDataCacheFunc);
  }

  includeInColumnSet(field: keyof GridConfigColumns, uniqueName: string, isIncluded: boolean) {
@@ -275,8 +288,8 @@ export abstract class GridDisplay {
    return this.config.expandedColumns.includes(uniqueName);
  }

-  toggleExpandedColumn(uniqueName: string) {
-    this.includeInColumnSet('expandedColumns', uniqueName, !this.isExpandedColumn(uniqueName));
+  toggleExpandedColumn(uniqueName: string, value?: boolean) {
+    this.includeInColumnSet('expandedColumns', uniqueName, value == null ? !this.isExpandedColumn(uniqueName) : value);
  }

  getFilter(uniqueName: string) {
@@ -294,6 +307,28 @@ export abstract class GridDisplay {
    this.reload();
  }

+  showFilter(uniqueName) {
+    this.setConfig(cfg => {
+      if (!cfg.filters.uniqueName)
+        return {
+          ...cfg,
+          filters: {
+            ..._.omitBy(cfg.filters, v => !v),
+            [uniqueName]: '',
+          },
+        };
+      return cfg;
+    });
+  }
+
+  removeFilter(uniqueName) {
+    this.setConfig(cfg => ({
+      ...cfg,
+      filters: _.omit(cfg.filters, [uniqueName]),
+    }));
+    this.reload();
+  }
+
  setFilters(dct) {
    this.setConfig(cfg => ({
      ...cfg,
@@ -369,8 +404,12 @@ export abstract class GridDisplay {
  getChangeSetField(row, uniqueName, insertedRowIndex): ChangeSetFieldDefinition {
    const col = this.columns.find(x => x.uniqueName == uniqueName);
    if (!col) return null;
-    if (!this.baseTable) return null;
-    if (this.baseTable.pureName != col.pureName || this.baseTable.schemaName != col.schemaName) return null;
+    const baseTableOrCollection = this.baseTableOrCollection;
+    if (!baseTableOrCollection) return null;
+    if (baseTableOrCollection.pureName != col.pureName || baseTableOrCollection.schemaName != col.schemaName) {
+      return null;
+    }
+
    return {
      ...this.getChangeSetRow(row, insertedRowIndex),
      uniqueName: uniqueName,
@@ -379,10 +418,11 @@ export abstract class GridDisplay {
  }

  getChangeSetRow(row, insertedRowIndex): ChangeSetRowDefinition {
-    if (!this.baseTable) return null;
+    const baseTableOrCollection = this.baseTableOrCollection;
+    if (!baseTableOrCollection) return null;
    return {
-      pureName: this.baseTable.pureName,
-      schemaName: this.baseTable.schemaName,
+      pureName: baseTableOrCollection.pureName,
+      schemaName: baseTableOrCollection.schemaName,
      insertedRowIndex,
      condition: insertedRowIndex == null ? this.getChangeSetCondition(row) : null,
    };
@@ -425,12 +465,75 @@ export abstract class GridDisplay {
    return select;
  }

+  getRowNumberOverSelect(select: Select, offset: number, count: number): Select {
+    const innerSelect: Select = {
+      commandType: 'select',
+      from: select.from,
+      where: select.where,
+      columns: [
+        ...select.columns,
+        {
+          alias: '_rowNumber',
+          exprType: 'rowNumber',
+          orderBy: select.orderBy
+            ? select.orderBy.map(x =>
+                x.exprType != 'column'
+                  ? x
+                  : x.source
+                  ? x
+                  : {
+                      ...x,
+                      source: { alias: 'basetbl' },
+                    }
+              )
+            : [
+                {
+                  ...select.columns[0],
+                  direction: 'ASC',
+                },
+              ],
+        },
+      ],
+    };
+
+    const res: Select = {
+      commandType: 'select',
+      selectAll: true,
+      from: {
+        subQuery: innerSelect,
+        alias: '_RowNumberResult',
+      },
+      where: {
+        conditionType: 'between',
+        expr: {
+          exprType: 'column',
+          columnName: '_RowNumber',
+          source: {
+            alias: '_RowNumberResult',
+          },
+        },
+        left: {
+          exprType: 'value',
+          value: offset + 1,
+        },
+        right: {
+          exprType: 'value',
+          value: offset + count,
+        },
+      },
+    };
+
+    return res;
+  }
+
  getPageQuery(offset: number, count: number) {
    if (!this.driver) return null;
-    const select = this.createSelect();
+    let select = this.createSelect();
    if (!select) return null;
-    if (this.driver.dialect.rangeSelect) select.range = { offset: offset, limit: count };
-    else if (this.driver.dialect.limitSelect) select.topRecords = count;
+    if (this.dialect.rangeSelect) select.range = { offset: offset, limit: count };
+    else if (this.dialect.rowNumberOverPaging && offset > 0)
+      select = this.getRowNumberOverSelect(select, offset, count);
+    else if (this.dialect.limitSelect) select.topRecords = count;
    const sql = treeToSql(this.driver, select, dumpSqlSelect);
    return sql;
  }
@@ -539,4 +642,19 @@ export abstract class GridDisplay {
      formViewKeyRequested: null,
    }));
  }
+
+  switchToJsonView() {
+    this.setConfig(cfg => ({
+      ...cfg,
+      isJsonView: true,
+    }));
+  }
+}
+
+export function reloadDataCacheFunc(cache: GridCache): GridCache {
+  return {
+    // ...cache,
+    ...createGridCache(),
+    refreshTime: new Date().getTime(),
+  };
 }
@@ -1,34 +1,44 @@
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay';
 import { QueryResultColumn } from 'dbgate-types';
 import { GridConfig, GridCache } from './GridConfig';
+import { analyseCollectionDisplayColumns } from './CollectionGridDisplay';

 export class JslGridDisplay extends GridDisplay {
  constructor(
    jslid,
-    columns: QueryResultColumn[],
+    structure,
    config: GridConfig,
    setConfig: ChangeConfigFunc,
    cache: GridCache,
-    setCache: ChangeCacheFunc
+    setCache: ChangeCacheFunc,
+    rows: any
  ) {
    super(config, setConfig, cache, setCache, null);

    this.filterable = true;

-    this.columns = columns
-      .map(col => ({
-        columnName: col.columnName,
-        headerText: col.columnName,
-        uniqueName: col.columnName,
-        uniquePath: [col.columnName],
-        notNull: col.notNull,
-        autoIncrement: col.autoIncrement,
-        pureName: null,
-        schemaName: null,
-      }))
-      ?.map(col => ({
-        ...col,
-        isChecked: this.isColumnChecked(col),
-      }));
+    if (structure.columns) {
+      this.columns = structure.columns
+        .map(col => ({
+          columnName: col.columnName,
+          headerText: col.columnName,
+          uniqueName: col.columnName,
+          uniquePath: [col.columnName],
+          notNull: col.notNull,
+          autoIncrement: col.autoIncrement,
+          pureName: null,
+          schemaName: null,
+        }))
+        ?.map(col => ({
+          ...col,
+          isChecked: this.isColumnChecked(col),
+        }));
+    }
+
+    if (structure.__isDynamicStructure) {
+      this.columns = analyseCollectionDisplayColumns(rows, this);
+    }
+
+    if (!this.columns) this.columns = [];
  }
 }
@@ -12,7 +12,6 @@ import {
  Condition,
  OrderByExpression,
 } from 'dbgate-sqltree';
-import { filterName } from './filterName';
 import { TableGridDisplay } from './TableGridDisplay';
 import stableStringify from 'json-stable-stringify';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
@@ -28,10 +27,22 @@ export class TableFormViewDisplay extends FormViewDisplay {
    setConfig: ChangeConfigFunc,
    cache: GridCache,
    setCache: ChangeCacheFunc,
-    dbinfo: DatabaseInfo
+    dbinfo: DatabaseInfo,
+    displayOptions,
+    serverVersion
  ) {
-    super(config, setConfig, cache, setCache, driver, dbinfo);
-    this.gridDisplay = new TableGridDisplay(tableName, driver, config, setConfig, cache, setCache, dbinfo);
+    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
+    this.gridDisplay = new TableGridDisplay(
+      tableName,
+      driver,
+      config,
+      setConfig,
+      cache,
+      setCache,
+      dbinfo,
+      displayOptions,
+      serverVersion
+    );
    this.gridDisplay.addAllExpandedColumnsToSelected = true;

    this.isLoadedCorrectly = this.gridDisplay.isLoadedCorrectly && !!this.driver;
@@ -253,8 +264,8 @@ export class TableFormViewDisplay extends FormViewDisplay {
    };
  }

-  toggleExpandedColumn(uniqueName: string) {
-    this.gridDisplay.toggleExpandedColumn(uniqueName);
+  toggleExpandedColumn(uniqueName: string, value?: boolean) {
+    this.gridDisplay.toggleExpandedColumn(uniqueName, value);
    this.gridDisplay.reload();
  }

@@ -1,9 +1,9 @@
 import _ from 'lodash';
+import { filterName } from 'dbgate-tools';
 import { GridDisplay, ChangeCacheFunc, DisplayColumn, DisplayedColumnInfo, ChangeConfigFunc } from './GridDisplay';
 import { TableInfo, EngineDriver, ViewInfo, ColumnInfo, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
 import { GridConfig, GridCache, createGridCache } from './GridConfig';
 import { Expression, Select, treeToSql, dumpSqlSelect } from 'dbgate-sqltree';
-import { filterName } from './filterName';

 export class TableGridDisplay extends GridDisplay {
  public table: TableInfo;
@@ -17,9 +17,11 @@ export class TableGridDisplay extends GridDisplay {
    setConfig: ChangeConfigFunc,
    cache: GridCache,
    setCache: ChangeCacheFunc,
-    dbinfo: DatabaseInfo
+    dbinfo: DatabaseInfo,
+    public displayOptions: any,
+    serverVersion
  ) {
-    super(config, setConfig, cache, setCache, driver, dbinfo);
+    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);

    this.table = this.findTable(tableName);
    if (!this.table) {
@@ -168,7 +170,7 @@ export class TableGridDisplay extends GridDisplay {

  processReferences(select: Select, displayedColumnInfo: DisplayedColumnInfo, options) {
    this.addJoinsFromExpandedColumns(select, this.columns, 'basetbl', displayedColumnInfo);
-    if (!options.isExport) {
+    if (!options.isExport && this.displayOptions.showHintColumns) {
      this.addHintsToSelect(select);
    }
  }
@@ -10,9 +10,10 @@ export class ViewGridDisplay extends GridDisplay {
    config: GridConfig,
    setConfig: ChangeConfigFunc,
    cache: GridCache,
-    setCache: ChangeCacheFunc
+    setCache: ChangeCacheFunc,
+    serverVersion
  ) {
-    super(config, setConfig, cache, setCache, driver);
+    super(config, setConfig, cache, setCache, driver, serverVersion);
    this.columns = this.getDisplayColumns(view);
    this.filterable = true;
    this.sortable = true;
@@ -4,7 +4,6 @@ export * from './TableGridDisplay';
 export * from './ViewGridDisplay';
 export * from './JslGridDisplay';
 export * from './ChangeSet';
-export * from './filterName';
 export * from './FreeTableGridDisplay';
 export * from './FreeTableModel';
 export * from './MacroDefinition';
@@ -1,5 +1,4 @@
 [![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
-[![Donate](https://img.shields.io/badge/donate-paypal-blue.svg)](https://paypal.me/JanProchazkaCz/30eur)
 [![NPM version](https://img.shields.io/npm/v/dbgate.svg)](https://www.npmjs.com/package/dbgate)

 # DbGate - database administration tool
@@ -1,7 +1,10 @@
 #!/usr/bin/env node

+const path = require('path');
+
+global.dbgateApiModulePath = path.dirname(path.dirname(require.resolve('dbgate-api')));
+global.dbgateApiPackagedPluginsPath = path.dirname(global.dbgateApiModulePath);
+
 const dbgateApi = require('dbgate-api');

-global.dbgateApiModulePath = require.resolve('dbgate-api');
-
-dbgateApi.getMainModule().start('startNodeWeb');
+dbgateApi.getMainModule().start();
@@ -1,12 +1,11 @@
 {
  "name": "dbgate",
-  "version": "4.0.0",
+  "version": "4.1.1",
  "homepage": "https://dbgate.org/",
  "repository": {
    "type": "git",
    "url": "https://github.com/dbgate/dbgate.git"
  },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
  "description": "Opensource database administration tool - web interface",
  "author": "Jan Prochazka",
  "license": "MIT",
@@ -19,7 +18,13 @@
    "web"
  ],
  "dependencies": {
-    "dbgate-api": "^4.0.0",
-    "dbgate-web": "^4.0.0"
+    "dbgate-api": "^4.1.1",
+    "dbgate-web": "^4.1.1",
+    "dbgate-plugin-csv": "^4.1.1",
+    "dbgate-plugin-excel": "^4.1.1",
+    "dbgate-plugin-mongo": "^4.1.1",
+    "dbgate-plugin-mysql": "^4.1.1",
+    "dbgate-plugin-mssql": "^4.1.1",
+    "dbgate-plugin-postgres": "^4.1.1"
  }
 }
@@ -1,19 +1,19 @@
 {
-  "version": "4.0.0",
+  "version": "4.1.1",
  "name": "dbgate-filterparser",
  "main": "lib/index.js",
  "typings": "lib/index.d.ts",
  "scripts": {
-    "prepare": "yarn build",
    "build": "tsc",
    "start": "tsc --watch",
-    "test": "jest"
+    "test": "jest",
+    "test:ci": "jest --json --outputFile=result.json --testLocationInResults"
  },
  "files": [
    "lib"
  ],
  "devDependencies": {
-    "dbgate-types": "^4.0.0",
+    "dbgate-types": "^4.1.1",
    "@types/jest": "^25.1.4",
    "@types/node": "^13.7.0",
    "jest": "^24.9.0",
@@ -22,8 +22,8 @@
  },
  "dependencies": {
    "@types/parsimmon": "^1.10.1",
-    "dbgate-tools": "^4.0.0",
-    "lodash": "^4.17.15",
+    "dbgate-tools": "^4.1.1",
+    "lodash": "^4.17.21",
    "moment": "^2.24.0",
    "parsimmon": "^1.13.0"
  }
@@ -3,5 +3,9 @@ import { parseFilter } from './parseFilter';
 test('parse string', () => {
  const ast = parseFilter('"123"', 'string');
  console.log(JSON.stringify(ast));
-  expect(ast).toBe(3);
+  expect(ast).toEqual({
+    conditionType: 'like',
+    left: { exprType: 'placeholder' },
+    right: { exprType: 'value', value: '%123%' },
+  });
 });
@@ -0,0 +1 @@
+lib
@@ -0,0 +1,36 @@
+[![NPM version](https://img.shields.io/npm/v/dbgate-query-splitter.svg)](https://www.npmjs.com/package/dbgate-query-splitter)
+
+dbgate-query-splitter
+====================
+
+Splits long SQL query into into particular statements. Designed to have zero dependencies and to be fast.
+
+Supports following SQL dialects:
+* MySQL
+* PostgreSQL
+* SQLite
+* Microsoft SQL Server
+
+## Usage
+
+```js
+import { splitQuery, mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitterOptions } from 'dbgate-query-splitter';
+
+const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`;', mysqlSplitterOptions);
+
+// output is ['SELECT * FROM `table1`', 'SELECT * FROM `table2`']
+
+```
+
+## Contributing
+Please run tests before pushing any changes.
+
+```sh
+yarn test
+```
+
+## Supported syntax
+* Comments
+* Dollar strings (PostgreSQL)
+* GO separators (MS SQL)
+* Custom delimiter, setby DELIMITER keyword (MySQL)
@@ -0,0 +1,5 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  moduleFileExtensions: ['js'],
+};
@@ -0,0 +1,37 @@
+{
+  "version": "4.1.1",
+  "name": "dbgate-query-splitter",
+  "main": "lib/index.js",
+  "typings": "lib/index.d.ts",
+  "description": "SQL Query splitter for verious database engines",
+  "homepage": "https://github.com/dbgate/dbgate/tree/master/packages/query-splitter",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate"
+  },
+  "author": "Jan Prochazka",
+  "license": "MIT",
+  "keywords": [
+    "SQL",
+    "query",
+    "split",
+    "parse"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "start": "tsc --watch",
+    "test": "jest",
+    "test:ci": "jest --json --outputFile=result.json --testLocationInResults"
+  },
+  "files": [
+    "lib"
+  ],
+  "devDependencies": {
+    "dbgate-types": "^4.1.1",
+    "@types/jest": "^25.1.4",
+    "@types/node": "^13.7.0",
+    "jest": "^24.9.0",
+    "ts-jest": "^25.2.1",
+    "typescript": "^3.7.5"
+  }
+}
@@ -0,0 +1,2 @@
+export * from './splitQuery';
+export * from './options';
@@ -0,0 +1,66 @@
+export interface SplitterOptions {
+  stringsBegins: string[];
+  stringsEnds: { [begin: string]: string };
+  stringEscapes: { [begin: string]: string };
+
+  allowSemicolon: boolean;
+  allowCustomDelimiter: boolean;
+  allowGoDelimiter: boolean;
+  allowDollarDollarString: boolean;
+  noSplit: boolean;
+}
+
+export const defaultSplitterOptions: SplitterOptions = {
+  stringsBegins: ["'"],
+  stringsEnds: { "'": "'" },
+  stringEscapes: { "'": "'" },
+
+  allowSemicolon: true,
+  allowCustomDelimiter: false,
+  allowGoDelimiter: false,
+  allowDollarDollarString: false,
+  noSplit: false,
+};
+
+export const mysqlSplitterOptions: SplitterOptions = {
+  ...defaultSplitterOptions,
+
+  allowCustomDelimiter: true,
+  stringsBegins: ["'", '`'],
+  stringsEnds: { "'": "'", '`': '`' },
+  stringEscapes: { "'": '\\', '`': '`' },
+};
+
+export const mssqlSplitterOptions: SplitterOptions = {
+  ...defaultSplitterOptions,
+  allowSemicolon: false,
+  allowGoDelimiter: true,
+
+  stringsBegins: ["'", '['],
+  stringsEnds: { "'": "'", '[': ']' },
+  stringEscapes: { "'": "'" },
+};
+
+export const postgreSplitterOptions: SplitterOptions = {
+  ...defaultSplitterOptions,
+
+  allowDollarDollarString: true,
+
+  stringsBegins: ["'", '"'],
+  stringsEnds: { "'": "'", '"': '"' },
+  stringEscapes: { "'": "'", '"': '"' },
+};
+
+export const sqliteSplitterOptions: SplitterOptions = {
+  ...defaultSplitterOptions,
+
+  stringsBegins: ["'", '"'],
+  stringsEnds: { "'": "'", '"': '"' },
+  stringEscapes: { "'": "'", '"': '"' },
+};
+
+export const noSplitSplitterOptions: SplitterOptions = {
+  ...defaultSplitterOptions,
+
+  noSplit: true,
+};
@@ -0,0 +1,257 @@
+import { SplitterOptions, defaultSplitterOptions } from './options';
+
+const SEMICOLON = ';';
+
+interface SplitExecutionContext {
+  options: SplitterOptions;
+  source: string;
+  position: number;
+  currentDelimiter: string;
+  output: string[];
+  end: number;
+  wasDataOnLine: boolean;
+  currentCommandStart: number;
+
+  //   unread: string;
+  //   currentStatement: string;
+  //   semicolonKeyTokenRegex: RegExp;
+}
+
+function isStringEnd(s: string, pos: number, endch: string, escapech: string) {
+  if (!escapech) {
+    return s[pos] == endch;
+  }
+  if (endch == escapech) {
+    return s[pos] == endch && s[pos + 1] != endch;
+  } else {
+    return s[pos] == endch && s[pos - 1] != escapech;
+  }
+}
+
+interface Token {
+  type: 'string' | 'delimiter' | 'whitespace' | 'eoln' | 'data' | 'set_delimiter' | 'comment' | 'go_delimiter';
+  length: number;
+  value?: string;
+}
+
+const WHITESPACE_TOKEN: Token = {
+  type: 'whitespace',
+  length: 1,
+};
+const EOLN_TOKEN: Token = {
+  type: 'eoln',
+  length: 1,
+};
+const DATA_TOKEN: Token = {
+  type: 'data',
+  length: 1,
+};
+
+function scanDollarQuotedString(context: SplitExecutionContext): Token {
+  if (!context.options.allowDollarDollarString) return null;
+
+  let pos = context.position;
+  const s = context.source;
+
+  const match = /^(\$[a-zA-Z0-9_]*\$)/.exec(s.slice(pos));
+  if (!match) return null;
+  const label = match[1];
+  pos += label.length;
+
+  while (pos < context.end) {
+    if (s.slice(pos).startsWith(label)) {
+      return {
+        type: 'string',
+        length: pos + label.length - context.position,
+      };
+    }
+    pos++;
+  }
+
+  return null;
+}
+
+function scanToken(context: SplitExecutionContext): Token {
+  let pos = context.position;
+  const s = context.source;
+  const ch = s[pos];
+
+  if (context.options.stringsBegins.includes(ch)) {
+    pos++;
+    const endch = context.options.stringsEnds[ch];
+    const escapech = context.options.stringEscapes[ch];
+    while (pos < context.end && !isStringEnd(s, pos, endch, escapech)) {
+      if (endch == escapech && s[pos] == endch && s[pos + 1] == endch) {
+        pos += 2;
+      } else {
+        pos++;
+      }
+    }
+    return {
+      type: 'string',
+      length: pos - context.position + 1,
+    };
+  }
+
+  if (context.currentDelimiter && s.slice(pos).startsWith(context.currentDelimiter)) {
+    return {
+      type: 'delimiter',
+      length: context.currentDelimiter.length,
+    };
+  }
+
+  if (ch == ' ' || ch == '\t' || ch == '\r') {
+    return WHITESPACE_TOKEN;
+  }
+
+  if (ch == '\n') {
+    return EOLN_TOKEN;
+  }
+
+  if (ch == '-' && s[pos + 1] == '-') {
+    while (pos < context.end && s[pos] != '\n') pos++;
+    return {
+      type: 'comment',
+      length: pos - context.position,
+    };
+  }
+
+  if (ch == '/' && s[pos + 1] == '*') {
+    pos += 2;
+    while (pos < context.end) {
+      if (s[pos] == '*' && s[pos + 1] == '/') break;
+      pos++;
+    }
+    return {
+      type: 'comment',
+      length: pos - context.position + 2,
+    };
+  }
+
+  if (context.options.allowCustomDelimiter && !context.wasDataOnLine) {
+    const m = s.slice(pos).match(/^DELIMITER[ \t]+([^\n]+)/i);
+    if (m) {
+      return {
+        type: 'set_delimiter',
+        value: m[1].trim(),
+        length: m[0].length,
+      };
+    }
+  }
+
+  if (context.options.allowGoDelimiter && !context.wasDataOnLine) {
+    const m = s.slice(pos).match(/^GO[\t\r ]*(\n|$)/i);
+    if (m) {
+      return {
+        type: 'go_delimiter',
+        length: m[0].length - 1,
+      };
+    }
+  }
+
+  const dollarString = scanDollarQuotedString(context);
+  if (dollarString) return dollarString;
+
+  return DATA_TOKEN;
+}
+
+function pushQuery(context) {
+  const sql = context.source.slice(context.currentCommandStart, context.position);
+  const trimmed = sql.trim();
+  if (trimmed) context.output.push(trimmed);
+}
+
+export function splitQuery(sql: string, options: SplitterOptions = null): string[] {
+  const usedOptions = {
+    ...defaultSplitterOptions,
+    ...options,
+  };
+
+  if (usedOptions.noSplit) {
+    return [sql];
+  }
+
+  const context: SplitExecutionContext = {
+    source: sql,
+    end: sql.length,
+    currentDelimiter: options?.allowSemicolon === false ? null : SEMICOLON,
+    position: 0,
+    currentCommandStart: 0,
+    output: [],
+    wasDataOnLine: false,
+    options: usedOptions,
+  };
+
+  while (context.position < context.end) {
+    const token = scanToken(context);
+    if (!token) {
+      // nothing special, move forward
+      context.position += 1;
+      continue;
+    }
+    switch (token.type) {
+      case 'string':
+        context.position += token.length;
+        context.wasDataOnLine = true;
+        break;
+      case 'comment':
+        context.position += token.length;
+        context.wasDataOnLine = true;
+        break;
+      case 'eoln':
+        context.position += token.length;
+        context.wasDataOnLine = false;
+        break;
+      case 'data':
+        context.position += token.length;
+        context.wasDataOnLine = true;
+        break;
+      case 'whitespace':
+        context.position += token.length;
+        break;
+      case 'set_delimiter':
+        pushQuery(context);
+        context.currentDelimiter = token.value;
+        context.position += token.length;
+        context.currentCommandStart = context.position;
+        break;
+      case 'go_delimiter':
+        pushQuery(context);
+        context.position += token.length;
+        context.currentCommandStart = context.position;
+        break;
+      case 'delimiter':
+        pushQuery(context);
+        context.position += token.length;
+        context.currentCommandStart = context.position;
+        break;
+    }
+  }
+
+  if (context.end > context.currentCommandStart) {
+    pushQuery(context);
+  }
+
+  //   context.semicolonKeyTokenRegex = buildKeyTokenRegex(SEMICOLON, context);
+  //   let findResult: FindExpResult = {
+  //     expIndex: -1,
+  //     exp: null,
+  //     nextIndex: 0,
+  //   };
+  //   let lastUnreadLength;
+  //   do {
+  //     // console.log('context.unread', context.unread);
+  //     lastUnreadLength = context.unread.length;
+  //     findResult = findKeyToken(context.unread, context.currentDelimiter, context);
+  //     handleKeyTokenFindResult(context, findResult);
+  //     // Prevent infinite loop by returning incorrect result
+  //     if (lastUnreadLength === context.unread.length) {
+  //       read(context, context.unread.length);
+  //     }
+  //   } while (context.unread !== '');
+  //   publishStatement(context);
+
+  // console.log('RESULT', context.output);
+
+  return context.output;
+}
@@ -0,0 +1,79 @@
+import { mysqlSplitterOptions, mssqlSplitterOptions, postgreSplitterOptions, noSplitSplitterOptions } from './options';
+import { splitQuery } from './splitQuery';
+
+test('simple query', () => {
+  const output = splitQuery('select * from A');
+  expect(output).toEqual(['select * from A']);
+});
+
+test('correct split 2 queries', () => {
+  const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`;', mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
+});
+
+test('correct split 2 queries - no end semicolon', () => {
+  const output = splitQuery('SELECT * FROM `table1`;SELECT * FROM `table2`', mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
+});
+
+test('delete empty query', () => {
+  const output = splitQuery(';;;\n;;SELECT * FROM `table1`;;;;;SELECT * FROM `table2`;;; ;;;', mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT * FROM `table1`', 'SELECT * FROM `table2`']);
+});
+
+test('should handle double backtick', () => {
+  const input = ['CREATE TABLE `a``b` (`c"d` INT)', 'CREATE TABLE `a````b` (`c"d` INT)'];
+  const output = splitQuery(input.join(';\n') + ';', mysqlSplitterOptions);
+  expect(output).toEqual(input);
+});
+
+test('semicolon inside string', () => {
+  const input = ['CREATE TABLE a', "INSERT INTO a (x) VALUES ('1;2;3;4')"];
+  const output = splitQuery(input.join(';\n') + ';', mysqlSplitterOptions);
+  expect(output).toEqual(input);
+});
+
+test('semicolon inside identyifier - mssql', () => {
+  const input = ['CREATE TABLE [a;1]', "INSERT INTO [a;1] (x) VALUES ('1')"];
+  const output = splitQuery(input.join(';\n') + ';', {
+    ...mssqlSplitterOptions,
+    allowSemicolon: true,
+  });
+  expect(output).toEqual(input);
+});
+
+test('delimiter test', () => {
+  const input = 'SELECT 1;\n DELIMITER $$\n SELECT 2; SELECT 3; \n DELIMITER ;';
+  const output = splitQuery(input, mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT 1', 'SELECT 2; SELECT 3;']);
+});
+
+test('one line comment test', () => {
+  const input = 'SELECT 1 -- comment1;comment2\n;SELECT 2';
+  const output = splitQuery(input, mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT 1 -- comment1;comment2', 'SELECT 2']);
+});
+
+test('multi line comment test', () => {
+  const input = 'SELECT 1 /* comment1;comment2\ncomment3*/;SELECT 2';
+  const output = splitQuery(input, mysqlSplitterOptions);
+  expect(output).toEqual(['SELECT 1 /* comment1;comment2\ncomment3*/', 'SELECT 2']);
+});
+
+test('dollar string', () => {
+  const input = 'CREATE PROC $$ SELECT 1; SELECT 2; $$ ; SELECT 3';
+  const output = splitQuery(input, postgreSplitterOptions);
+  expect(output).toEqual(['CREATE PROC $$ SELECT 1; SELECT 2; $$', 'SELECT 3']);
+});
+
+test('go delimiter', () => {
+  const input = 'SELECT 1\ngo\nSELECT 2';
+  const output = splitQuery(input, mssqlSplitterOptions);
+  expect(output).toEqual(['SELECT 1', 'SELECT 2']);
+});
+
+test('no split', () => {
+  const input = 'SELECT 1;SELECT 2';
+  const output = splitQuery(input, noSplitSplitterOptions);
+  expect(output).toEqual(['SELECT 1;SELECT 2']);
+});
@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "ES2015",
+    "module": "commonjs",
+    "declaration": true,
+    "skipLibCheck": true,
+    "outDir": "lib",
+    "preserveWatchOutput": true,
+    "esModuleInterop": true
+  },
+  "include": [
+    "src/**/*"
+  ]
+}
@@ -1,5 +1,5 @@
 {
-  "version": "4.0.0",
+  "version": "4.1.1",
  "name": "dbgate-sqltree",
  "main": "lib/index.js",
  "typings": "lib/index.d.ts",
@@ -8,7 +8,6 @@
    "type": "git",
    "url": "https://github.com/dbgate/dbgate.git"
  },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
  "author": "Jan Prochazka",
  "license": "MIT",
  "keywords": [
@@ -20,7 +19,6 @@
    "dbgate"
  ],
  "scripts": {
-    "prepare": "yarn build",
    "build": "tsc",
    "start": "tsc --watch"
  },
@@ -29,10 +27,10 @@
  ],
  "devDependencies": {
    "@types/node": "^13.7.0",
-    "dbgate-types": "^4.0.0",
+    "dbgate-types": "^4.1.1",
    "typescript": "^3.7.5"
  },
  "dependencies": {
-    "lodash": "^4.17.15"
+    "lodash": "^4.17.21"
  }
 }
@@ -62,5 +62,12 @@ export function dumpSqlCondition(dmp: SqlDumper, condition: Condition) {
      dumpSqlSelect(dmp, condition.subQuery);
      dmp.put(')');
      break;
+    case 'between':
+      dumpSqlExpression(dmp, condition.expr);
+      dmp.put(' ^between ');
+      dumpSqlExpression(dmp, condition.left);
+      dmp.put(' ^and ');
+      dumpSqlExpression(dmp, condition.right);
+      break;
  }
 }
@@ -38,5 +38,14 @@ export function dumpSqlExpression(dmp: SqlDumper, expr: Expression) {
    case 'transform':
      dmp.transform(expr.transform, () => dumpSqlExpression(dmp, expr.expr));
      break;
+
+    case 'rowNumber':
+      dmp.put(" ^row_number() ^over (^order ^by ");
+      dmp.putCollection(', ', expr.orderBy, x => {
+        dumpSqlExpression(dmp, x);
+        dmp.put(' %k', x.direction);
+      });
+      dmp.put(")");
+      break;
  }
 }
@@ -92,6 +92,13 @@ export interface NotExistsCondition {
  subQuery: Select;
 }

+export interface BetweenCondition {
+  conditionType: 'between';
+  expr: Expression;
+  left: Expression;
+  right: Expression;
+}
+
 export type Condition =
  | BinaryCondition
  | NotCondition
@@ -99,7 +106,8 @@ export type Condition =
  | CompoudCondition
  | LikeCondition
  | ExistsCondition
-  | NotExistsCondition;
+  | NotExistsCondition
+  | BetweenCondition;

 export interface Source {
  name?: NamedObjectInfo;
@@ -153,13 +161,19 @@ export interface TranformExpression {
  transform: TransformType;
 }

+export interface RowNumberExpression {
+  exprType: 'rowNumber';
+  orderBy: OrderByExpression[];
+}
+
 export type Expression =
  | ColumnRefExpression
  | ValueExpression
  | PlaceholderExpression
  | RawExpression
  | CallExpression
-  | TranformExpression;
+  | TranformExpression
+  | RowNumberExpression;
 export type OrderByExpression = Expression & { direction: 'ASC' | 'DESC' };

 export type ResultField = Expression & { alias?: string };
@@ -1,5 +1,5 @@
 {
-  "version": "4.0.3-rc.1",
+  "version": "4.1.1",
  "name": "dbgate-tools",
  "main": "lib/index.js",
  "typings": "lib/index.d.ts",
@@ -8,7 +8,6 @@
    "type": "git",
    "url": "https://github.com/dbgate/dbgate.git"
  },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
  "author": "Jan Prochazka",
  "license": "MIT",
  "keywords": [
@@ -16,7 +15,6 @@
    "dbgate"
  ],
  "scripts": {
-    "prepare": "yarn build",
    "build": "tsc",
    "start": "tsc --watch",
    "test": "jest",
@@ -27,12 +25,12 @@
  ],
  "devDependencies": {
    "@types/node": "^13.7.0",
-    "dbgate-types": "^4.0.0",
+    "dbgate-types": "^4.1.1",
    "jest": "^24.9.0",
    "ts-jest": "^25.2.1",
    "typescript": "^3.7.5"
  },
  "dependencies": {
-    "lodash": "^4.17.15"
+    "lodash": "^4.17.21"
  }
-}
+}
@@ -2,12 +2,14 @@ import { DatabaseInfo, DatabaseModification, EngineDriver } from 'dbgate-types';
 import _sortBy from 'lodash/sortBy';
 import _groupBy from 'lodash/groupBy';
 import _pick from 'lodash/pick';
+import _compact from 'lodash/compact';

 const fp_pick = arg => array => _pick(array, arg);
 export class DatabaseAnalyser {
  structure: DatabaseInfo;
  modifications: DatabaseModification[];
  singleObjectFilter: any;
+  singleObjectId: string = null;

  constructor(public pool, public driver: EngineDriver) {}

@@ -15,15 +17,30 @@ export class DatabaseAnalyser {
    return DatabaseAnalyser.createEmptyStructure();
  }

-  /** @returns {Promise<import('dbgate-types').DatabaseModification[]>} */
-  async getModifications() {
-    if (this.structure == null) throw new Error('DatabaseAnalyse.getModifications - structure must be filled');
-
+  async _getFastSnapshot(): Promise<DatabaseInfo> {
    return null;
  }

+  async _computeSingleObjectId() {}
+
  async fullAnalysis() {
-    return this._runAnalysis();
+    const res = await this._runAnalysis();
+    // console.log('FULL ANALYSIS', res);
+    return res;
+  }
+
+  async singleObjectAnalysis(name, typeField) {
+    // console.log('Analysing SINGLE OBJECT', name, typeField);
+    this.singleObjectFilter = { ...name, typeField };
+    await this._computeSingleObjectId();
+    const res = await this._runAnalysis();
+    // console.log('SINGLE OBJECT RES', res);
+    const obj =
+      res[typeField]?.length == 1
+        ? res[typeField][0]
+        : res[typeField]?.find(x => x.pureName == name.pureName && x.schemaName == name.schemaName);
+    // console.log('SINGLE OBJECT', obj);
+    return obj;
  }

  async incrementalAnalysis(structure) {
@@ -37,7 +54,7 @@ export class DatabaseAnalyser {
    }
    if (this.modifications.length == 0) return null;
    console.log('DB modifications detected:', this.modifications);
-    return this._runAnalysis();
+    return this.mergeAnalyseResult(await this._runAnalysis());
  }

  mergeAnalyseResult(newlyAnalysed) {
@@ -49,7 +66,7 @@ export class DatabaseAnalyser {
    }

    const res = {};
-    for (const field of ['tables', 'collections', 'views', 'functions', 'procedures', 'triggers']) {
+    for (const field of ['tables', 'collections', 'views', 'matviews', 'functions', 'procedures', 'triggers']) {
      const removedIds = this.modifications
        .filter(x => x.action == 'remove' && x.objectTypeField == field)
        .map(x => x.objectId);
@@ -57,9 +74,19 @@ export class DatabaseAnalyser {
      const addedChangedIds = newArray.map(x => x.objectId);
      const removeAllIds = [...removedIds, ...addedChangedIds];
      res[field] = _sortBy(
-        [...this.structure[field].filter(x => !removeAllIds.includes(x.objectId)), ...newArray],
+        [...(this.structure[field] || []).filter(x => !removeAllIds.includes(x.objectId)), ...newArray],
        x => x.pureName
      );
+
+      // merge missing data from old structure
+      for (const item of res[field]) {
+        const original = (this.structure[field] || []).find(x => x.objectId == item.objectId);
+        if (original) {
+          for (const key in original) {
+            if (!item[key]) item[key] = original[key];
+          }
+        }
+      }
    }

    return res;
@@ -71,15 +98,125 @@ export class DatabaseAnalyser {
    // }
  }

+  getRequestedObjectPureNames(objectTypeField, allPureNames) {
+    if (this.singleObjectFilter) {
+      const { typeField, pureName } = this.singleObjectFilter;
+      if (typeField == objectTypeField) return [pureName];
+    }
+    if (this.modifications) {
+      return this.modifications
+        .filter(x => x.objectTypeField == objectTypeField)
+        .filter(x => x.newName)
+        .map(x => x.newName.pureName);
+    }
+    return allPureNames;
+  }
+
  // findObjectById(id) {
  //   return this.structure.tables.find((x) => x.objectId == id);
  // }

+  createQuery(template, typeFields) {
+    // let res = template;
+    if (this.singleObjectFilter) {
+      const { typeField } = this.singleObjectFilter;
+      if (!this.singleObjectId) return null;
+      if (!typeFields || !typeFields.includes(typeField)) return null;
+      return template.replace(/=OBJECT_ID_CONDITION/g, ` = '${this.singleObjectId}'`);
+    }
+    if (!this.modifications || !typeFields || this.modifications.length == 0) {
+      return template.replace(/=OBJECT_ID_CONDITION/g, ' is not null');
+    }
+    if (this.modifications.some(x => typeFields.includes(x.objectTypeField) && x.action == 'all')) {
+      // do not filter objects
+      return template.replace(/=OBJECT_ID_CONDITION/g, ' is not null');
+    }
+    const filterIds = this.modifications
+      .filter(x => typeFields.includes(x.objectTypeField) && (x.action == 'add' || x.action == 'change'))
+      .map(x => x.objectId);
+    if (filterIds.length == 0) {
+      return template.replace(/=OBJECT_ID_CONDITION/g, " = '0'");
+    }
+    return template.replace(/=OBJECT_ID_CONDITION/g, ` in (${filterIds.map(x => `'${x}'`).join(',')})`);
+  }
+
+  getDeletedObjectsForField(snapshot, objectTypeField) {
+    const items = snapshot[objectTypeField];
+    if (!items) return [];
+    if (!this.structure[objectTypeField]) return [];
+    return this.structure[objectTypeField]
+      .filter(x => !items.find(y => x.objectId == y.objectId))
+      .map(x => ({
+        oldName: _pick(x, ['schemaName', 'pureName']),
+        objectId: x.objectId,
+        action: 'remove',
+        objectTypeField,
+      }));
+  }
+
+  getDeletedObjects(snapshot) {
+    return [
+      ...this.getDeletedObjectsForField(snapshot, 'tables'),
+      ...this.getDeletedObjectsForField(snapshot, 'collections'),
+      ...this.getDeletedObjectsForField(snapshot, 'views'),
+      ...this.getDeletedObjectsForField(snapshot, 'matviews'),
+      ...this.getDeletedObjectsForField(snapshot, 'procedures'),
+      ...this.getDeletedObjectsForField(snapshot, 'functions'),
+      ...this.getDeletedObjectsForField(snapshot, 'triggers'),
+    ];
+  }
+
+  async getModifications() {
+    const snapshot = await this._getFastSnapshot();
+    if (!snapshot) return null;
+
+    // console.log('STRUCTURE', this.structure);
+    // console.log('SNAPSHOT', snapshot);
+
+    const res = [];
+    for (const field in snapshot) {
+      const items = snapshot[field];
+      if (items === null) {
+        res.push({ objectTypeField: field, action: 'all' });
+        continue;
+      }
+      if (items === undefined) {
+        // skip - undefined meens, that field is not supported
+        continue;
+      }
+      for (const item of items) {
+        const { objectId, schemaName, pureName, contentHash } = item;
+        const obj = this.structure[field].find(x => x.objectId == objectId);
+
+        if (obj && contentHash && obj.contentHash == contentHash) continue;
+
+        const action = obj
+          ? {
+              newName: { schemaName, pureName },
+              oldName: _pick(obj, ['schemaName', 'pureName']),
+              action: 'change',
+              objectTypeField: field,
+              objectId,
+            }
+          : {
+              newName: { schemaName, pureName },
+              action: 'add',
+              objectTypeField: field,
+              objectId,
+            };
+        res.push(action);
+      }
+    }
+
+    return [..._compact(res), ...this.getDeletedObjects(snapshot)];
+  }
+
  static createEmptyStructure(): DatabaseInfo {
    return {
      tables: [],
      collections: [],
      views: [],
+      matviews: [],
      functions: [],
      procedures: [],
      triggers: [],
@@ -293,6 +293,20 @@ export class SqlDumper {
  changeViewSchema(obj: ViewInfo, newSchema: string) {}
  renameView(obj: ViewInfo, newSchema: string) {}

+  createMatview(obj: ViewInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropMatview(obj: ViewInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^materialized ^view  %f', obj);
+  }
+  alterMatview(obj: ViewInfo) {
+    this.putRaw(obj.createSql.replace(/create\s+view/i, 'ALTER VIEW'));
+    this.endCommand();
+  }
+  changeMatviewSchema(obj: ViewInfo, newSchema: string) {}
+  renameMatview(obj: ViewInfo, newSchema: string) {}
+
  createProcedure(obj: ProcedureInfo) {
    this.putRaw(obj.createSql);
    this.endCommand();
@@ -7,7 +7,8 @@ import {
  TriggerInfo,
  ViewInfo,
 } from 'dbgate-types';
-import _ from 'lodash';
+import _flatten from 'lodash/flatten';
+import _uniqBy from 'lodash/uniqBy'
 import { SqlDumper } from './SqlDumper';
 import { extendDatabaseInfo } from './structureTools';

@@ -29,6 +30,10 @@ interface SqlGeneratorOptions {
  checkIfViewExists: boolean;
  createViews: boolean;

+  dropMatviews: boolean;
+  checkIfMatviewExists: boolean;
+  createMatviews: boolean;
+
  dropProcedures: boolean;
  checkIfProcedureExists: boolean;
  createProcedures: boolean;
@@ -51,6 +56,7 @@ interface SqlGeneratorObject {
 export class SqlGenerator {
  private tables: TableInfo[];
  private views: ViewInfo[];
+  private matviews: ViewInfo[];
  private procedures: ProcedureInfo[];
  private functions: FunctionInfo[];
  private triggers: TriggerInfo[];
@@ -69,6 +75,7 @@ export class SqlGenerator {
    this.dbinfo = extendDatabaseInfo(dbinfo);
    this.tables = this.extract('tables');
    this.views = this.extract('views');
+    this.matviews = this.extract('matviews');
    this.procedures = this.extract('procedures');
    this.functions = this.extract('functions');
    this.triggers = this.extract('triggers');
@@ -89,6 +96,8 @@ export class SqlGenerator {
      if (this.checkDumper()) return;
      this.dropObjects(this.views, 'View');
      if (this.checkDumper()) return;
+      this.dropObjects(this.matviews, 'Matview');
+      if (this.checkDumper()) return;
      this.dropObjects(this.triggers, 'Trigger');
      if (this.checkDumper()) return;

@@ -113,6 +122,8 @@ export class SqlGenerator {
      if (this.checkDumper()) return;
      this.createObjects(this.views, 'View');
      if (this.checkDumper()) return;
+      this.createObjects(this.matviews, 'Matview');
+      if (this.checkDumper()) return;
      this.createObjects(this.triggers, 'Trigger');
      if (this.checkDumper()) return;
    } finally {
@@ -122,9 +133,9 @@ export class SqlGenerator {

  createForeignKeys() {
    const fks = [];
-    if (this.options.createForeignKeys) fks.push(..._.flatten(this.tables.map(x => x.foreignKeys || [])));
-    if (this.options.createReferences) fks.push(..._.flatten(this.tables.map(x => x.dependencies || [])));
-    for (const fk of _.uniqBy(fks, 'constraintName')) {
+    if (this.options.createForeignKeys) fks.push(..._flatten(this.tables.map(x => x.foreignKeys || [])));
+    if (this.options.createReferences) fks.push(..._flatten(this.tables.map(x => x.dependencies || [])));
+    for (const fk of _uniqBy(fks, 'constraintName')) {
      this.dmp.createForeignKey(fk);
      if (this.checkDumper()) return;
    }
@@ -152,7 +163,7 @@ export class SqlGenerator {
      }
    }
    if (this.options.createIndexes) {
-      for (const index of _.flatten(this.tables.map(x => x.indexes || []))) {
+      for (const index of _flatten(this.tables.map(x => x.indexes || []))) {
        this.dmp.createIndex(index);
      }
    }
@@ -204,7 +215,7 @@ export class SqlGenerator {

  dropTables() {
    if (this.options.dropReferences) {
-      for (const fk of _.flatten(this.tables.map(x => x.dependencies || []))) {
+      for (const fk of _flatten(this.tables.map(x => x.dependencies || []))) {
        this.dmp.dropForeignKey(fk);
      }
    }
--- a/Show More
+++ b/Show More