v4.1.12-beta.2

svelte select fix
v4.1.12-beta.1
2021-05-03 21:08:58 +02:00 · 2021-05-03 21:08:45 +02:00 · 2021-05-03 20:41:19 +02:00 · 2021-05-03 20:39:41 +02:00 · 2021-05-03 18:42:04 +02:00 · 2021-05-03 18:36:38 +02:00
714 changed files with 31107 additions and 24514 deletions
@@ -0,0 +1,33 @@
+---
+name: Bug report
+about: Create a report to help us improve DbGate
+title: 'BUG: Say something here'
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version Information (please complete the following information):**
+ - OS: [e.g. Windows/Mac/Ubuntu]
+ - App Version [see Help -> About]
+ - Install source [e.g. installer/SNAP/Docker/NPM]
+ - Type - Web/Application
+ - Database engine: [e.g. MySQL/PostgreSQL/SQL Server]
+
+**Additional context**
+Anything else you think might be helpful
@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for DbGate
+title: 'FEAT: '
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.
@@ -0,0 +1,18 @@
+---
+name: Question
+about: Ask a question about how to do something
+title: 'QUESTION: Summary of your question'
+labels: ''
+assignees: ''
+
+---
+
+**Details:**
+Details about your question
+
+**Version Information [might be relevant to your issue]**
+Operating System: 
+App Version [help -> about]:
+
+**Screenshot [if appropriate]**:
+A screenshot of the app if that helps
@@ -11,8 +11,8 @@ jobs:

    strategy:
      matrix:
-        os: [ubuntu-18.04, windows-2016]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        # os: [ubuntu-18.04, windows-2016]
+        os: [macOS-10.14, windows-2016, ubuntu-18.04]

    steps:
      - name: Context
@@ -63,11 +63,18 @@ jobs:
        run: |
          mkdir artifacts          

+          cp app/dist/*.deb artifacts/dbgate-beta.deb || true
+          cp app/dist/*.AppImage artifacts/dbgate-beta.AppImage || true
+          cp app/dist/*.exe artifacts/dbgate-beta.exe || true
+          cp app/dist/*windows*.zip artifacts/dbgate-windows-beta.zip || true
+          cp app/dist/*.dmg artifacts/dbgate-beta.dmg || true
+
          mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
          mv app/dist/*.AppImage artifacts/ || true
          mv app/dist/*.deb artifacts/ || true
          mv app/dist/*.snap artifacts/ || true
-#          mv app/dist/*.dmg artifacts/ || true
+          mv app/dist/*.dmg artifacts/ || true

      - name: Upload artifacts
        uses: actions/upload-artifact@v1
@@ -74,9 +74,11 @@ jobs:
          cp app/dist/*.deb artifacts/dbgate-latest.deb || true
          cp app/dist/*.AppImage artifacts/dbgate-latest.AppImage || true
          cp app/dist/*.exe artifacts/dbgate-latest.exe || true
+          cp app/dist/*windows*.zip artifacts/dbgate-windows-latest.zip || true
          cp app/dist/*.dmg artifacts/dbgate-latest.dmg || true

          mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
          mv app/dist/*.AppImage artifacts/ || true
          mv app/dist/*.deb artifacts/ || true
          mv app/dist/*.dmg artifacts/ || true
@@ -0,0 +1,47 @@
+name: Docker image
+
+# on: [push]
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-18.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+      - name: Use Node.js 10.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 10.x
+      - name: yarn install
+        run: |
+          yarn install
+      - name: setCurrentVersion
+        run: |
+          yarn setCurrentVersion
+      - name: Prepare docker image
+        run: |
+          yarn run prepare:docker
+      - name: Build docker image
+        run: |
+          docker build ./docker -t dbgate
+      - name: Push docker image
+        run: |
+          docker tag dbgate dbgate/dbgate:beta
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push dbgate/dbgate:beta
@@ -0,0 +1,121 @@
+name: NPM packages
+
+# on: [push]
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      # - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
+
+# on:
+#   push:
+#     branches:
+#       - production
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-18.04]
+        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+      - name: Use Node.js 10.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 10.x
+
+      - name: Configure NPM token
+        env:
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+        run: |
+          npm config set '//registry.npmjs.org/:_authToken' "${NPM_TOKEN}"
+
+      - name: yarn install
+        run: |
+          yarn install
+          
+      - name: setCurrentVersion
+        run: |
+          yarn setCurrentVersion
+
+      - name: Publish types
+        working-directory: packages/types
+        run: |
+          npm publish
+
+      - name: Publish tools
+        working-directory: packages/tools
+        run: |
+          npm publish
+
+      - name: Publish sqltree
+        working-directory: packages/sqltree
+        run: |
+          npm publish
+                    
+      - name: Publish api
+        working-directory: packages/api
+        run: |
+          npm publish
+
+      - name: Publish datalib
+        working-directory: packages/datalib
+        run: |
+          npm publish
+
+      - name: Publish filterparser
+        working-directory: packages/filterparser
+        run: |
+          npm publish
+
+      - name: Publish web
+        working-directory: packages/web
+        run: |
+          npm publish
+                                        
+      - name: Publish dbgate
+        working-directory: packages/dbgate
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-csv
+        working-directory: plugins/dbgate-plugin-csv
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-excel
+        working-directory: plugins/dbgate-plugin-excel
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mssql
+        working-directory: plugins/dbgate-plugin-mssql
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mysql
+        working-directory: plugins/dbgate-plugin-mysql
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mongo
+        working-directory: plugins/dbgate-plugin-mongo
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-postgres
+        working-directory: plugins/dbgate-plugin-postgres
+        run: |
+          npm publish
@@ -12,6 +12,12 @@ node_modules
 build
 dist

+app/packages/web/public
+app/packages/plugins
+docker/public
+docker/bundle.js
+docker/plugins
+
 # misc
 .DS_Store
 .env.local
@@ -24,3 +30,4 @@ yarn-debug.log*
 yarn-error.log*
 app/src/nativeModulesContent.js
 packages/api/src/nativeModulesContent.js
+.VSCodeCounter
@@ -0,0 +1,62 @@
+# ChangeLog
+
+### 4.1.11
+- FIX: fixed processing postgre query containing $$
+- FIX: fixed postgre analysing procedures & functions
+- FIX: patched svelte crash #105
+- ADDED: ability to disbale background DB model updates
+- ADDED: Duplicate connection
+- ADDED: Duplicate tab
+- FIX: SSH tunnel connection using keyfile auth #106
+- FIX: All tables button fix in export #109
+- CHANGED: Add to favorites moved from toolbar to tab context menu
+- CHANGED: Toolbar design - current tab related commands are delimited
+
+### 4.1.10
+- ADDED: Default database option in connectin settings #96 #92
+- FIX: Bundle size optimalization for Windows #97
+- FIX: Popup menu placement on smaller displays #94
+- ADDED: Browse table data with SQL Server 2008 #93
+- FIX: Prevented malicious origins / DNS rebinding #91
+- ADDED: Handle JSON fields in data editor (eg. jsonb field in Postgres) #90
+- FIX: Fixed crash on Windows with Hyper-V #86
+- ADDED: Show database server version in status bar
+- ADDED: Show detailed info about error, when connect to database fails
+- ADDED: Portable ZIP distribution for Windows #84
+### 4.1.9
+- FIX: Incorrect row count info in query result #83
+
+### 4.1.1
+- CHANGED: Default plugins are now part of installation
+### 4.1.0
+- ADDED: MongoDB support
+- ADDED: Configurable keyboard shortcuts
+- ADDED: JSON row cell data view
+- FIX: Fixed some problems from migration to Svelte
+
+### 4.0.3
+- FIX: fixes for FireFox (mainly incorrent handle of bind:clientHeight, replaces with resizeobserver)
+### 4.0.2
+- FIX: fixed docker and NPM build
+### 4.0.0
+- CHANGED: Excahnged React with Svelte. Changed theme colors. Huge speed and memory optimalization
+- ADDED: SQL Generator (CREATE, INSERT, DROP)
+- ADDED: Command palette (F1). Introduced commands, extended some context menus
+- ADDED: New keyboard shortcuts
+- ADDED: Switch to recent database feature
+- ADDED: Macros from free table editor are available also in table data editor
+- CHANGED: Cell data preview is now in left widgets panel
+- CHANGED: Toolbar refactor
+- FIX: Solved reconnecting expired connection
+
+### 3.9.6
+- ADDED: Connect using SSH Tunnel
+- ADDED: Connect using SSL
+- ADDED: Database connection dialog redesigned
+- ADDED: #63 Ctrl+Enter runs query
+- ADDED: Published dbgate NPM package
+- ADDED: SQL editor context menu
+- FIX: #62 - import, export executed from SNAP installs didn't work
+
+### 3.9.5
+- Start point of changelog
@@ -1,37 +1,53 @@
+[![NPM version](https://img.shields.io/npm/v/dbgate.svg)](https://www.npmjs.com/package/dbgate)
+[![dbgate](https://snapcraft.io/dbgate/badge.svg)](https://snapcraft.io/dbgate)
+[![dbgate](https://snapcraft.io/dbgate/trending.svg?name=0)](https://snapcraft.io/dbgate)
 [![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
-[![Donate](https://img.shields.io/badge/donate-paypal-blue.svg)](https://paypal.me/JanProchazkaCz/30eur)
-[![NPM version](https://img.shields.io/npm/v/dbgate-api.svg)](https://www.npmjs.com/package/dbgate-api)

 # DbGate - database administration tool

-DbGate is fast and efficient database administration tool. It is focused to work with data (filtering, editing, master/detail views etc.)
+DbGate modern, fast and easy to use database manager

-**Try it online** - https://demo.dbgate.org - online demo application
+* Try it online - [demo.dbgate.org](https://demo.dbgate.org) - online demo application
+* Download application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
+* Run web version as [NPM package](https://www.npmjs.com/package/dbgate) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
+
+Supported databases:
+* MySQL
+* PostgreSQL
+* SQL Server
+* MongoDB
+
+![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png)

 ## Features
-* Support for Microsoft SQL Server, Postgre SQL, MySQL
-* Table data browsing - filtering, sorting, related columns using foreign keys
+* Connect to Microsoft SQL Server, Postgre SQL, MySQL, MongoDB
+* Table data editing, with SQL change script preview
 * Master/detail views
 * Query designer
 * Form view for comfortable work with tables with many columns
-* Charts
-* Browsing objects - tables, views, procedures, functions
-* Table data editing, with SQL change script preview
+* JSON view on MongoDB collections
+* Explore tables, views, procedures, functions, MongoDB collections
 * SQL editor, execute SQL script, SQL code formatter, SQL code completion, SQL join wizard
+* Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
 * Import, export from/to CSV, Excel, JSON
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
 * Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
 * Light and dark theme
+* Charts
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture

-![Screenshot](https://raw.githubusercontent.com/dbshell/dbgate/master/screenshot.png)
+## Why is DbGate different
+There are many database managers now, so why DbGate? 
+* Works everywhere - Windows, Linux, Mac, Web browser (+mobile web is planned), without compromises in features
+* Based on standalone NPM packages, scripts can be run without DbGate (example - [CSV export](https://www.npmjs.com/package/dbgate-plugin-csv) )
+* Many data browsing functions based using foreign keys - master/detail, expand columns, expandable form view (on screenshot above)

 ## Design goals
 * Application simplicity - DbGate takes the best and only the best from old [DbGate](http://www.jenasoft.com/dbgate), [DatAdmin](http://www.jenasoft.com/datadmin) and [DbMouse](http://www.jenasoft.com/dbmouse) .
 * Minimal dependencies
-    * Frontend - React, styled-components, socket.io
+    * Frontend - Svelte, socket.io
    * Backend - NodeJs, ExpressJs, socket.io, database connection drivers
    * JavaScript + TypeScript
    * App - electron
@@ -41,35 +57,13 @@ DbGate is fast and efficient database administration tool. It is focused to work
 ## Plugins
 Plugins are standard NPM packages published on [npmjs.com](https://www.npmjs.com).  
 See all [existing DbGate plugins](https://www.npmjs.com/search?q=keywords:dbgateplugin).  
-Visit [dbgate generator homepage](https://github.com/dbshell/generator-dbgate) to see, how to create your own plugin.  
+Visit [dbgate generator homepage](https://github.com/dbgate/generator-dbgate) to see, how to create your own plugin.  

 Currently following extensions can be implemented using plugins:
 - File format parsers/writers
 - Database engine connectors

-## How Can I Contribute?
-You're welcome to contribute to this project! Below are some ideas, how to contribute:
-
-* Create plugins for new import/export formats or database engines
-* Bug fixing
-* Test Mac edition
-* Create unit tests
-* Whatever else
-
-Any help is appreciated!
-
-Feel free to report issues and open merge requests.
-
-## Roadmap
-
-| Feature | Complexity | Schedule |
-|---|---|---|
-| Table designer (structure editor) | big | february 2021 |
-| Support for SQLite | big | 2021 |
-| Filtering, sorting in free table editor | small | ??? |
-| Query designer | medium | december 2020 - done |
-| Using tedious driver instead of mssql | small | january 2021 - done |
-| Filter SQL result sets | small | november 2020 - done |
+Basic set of plugins is part of DbGate git repository and is installed with app. Additional plugins pust be downloaded from NPM (this task is handled by DbGate)

 ## How to run development environment

@@ -78,14 +72,14 @@ yarn
 yarn start
 ```

-If you want to make modifications in TypeScript packages, run TypeScript compiler in watch mode in seconds terminal:
+If you want to make modifications in libraries or plugins, run library compiler in watch mode in the second terminal:
 ```sh
 yarn lib
 ```

 Open http://localhost:5000 in your browser

-You could run electron app, using this server:
+You could run electron app (requires running localhost:5000):
 ```sh
 cd app
 yarn
@@ -93,7 +87,7 @@ yarn start
 ```

 ## How to run built electron app locally
-This mode is very similar to production run of electron app. Electron app forks process with API on dynamically allocated port, works with compiled javascript files.
+This mode is very similar to production run of electron app. Electron app forks process with API on dynamically allocated port, works with compiled javascript files and uses compiled version of plugins (doesn't use localhost:5000)

 ```sh
 cd app
@@ -109,13 +103,12 @@ yarn start:app:local
 ## Packages
 Some dbgate packages can be used also without DbGate. You can find them on [NPM repository](https://www.npmjs.com/search?q=keywords:dbgate)

-* [api](https://github.com/dbshell/dbgate/tree/master/packages/api) - backend, Javascript, ExpressJS [![NPM version](https://img.shields.io/npm/v/dbgate-api.svg)](https://www.npmjs.com/package/dbgate-api)
-* [datalib](https://github.com/dbshell/dbgate/tree/master/packages/datalib) - TypeScript library for utility classes
-* [app](https://github.com/dbshell/dbgate/tree/master/app) - application (JavaScript)
-structure, creating specific queries (JavaScript) [![NPM version](https://img.shields.io/npm/v/dbgate-engines.svg)](https://www.npmjs.com/package/dbgate-engines)
-* [filterparser](https://github.com/dbshell/dbgate/tree/master/packages/filterparser) - TypeScript library for parsing data filter expressions using parsimmon
-* [sqltree](https://github.com/dbshell/dbgate/tree/master/packages/sqltree) - JSON representation of SQL query, functions converting to SQL (TypeScript) [![NPM version](https://img.shields.io/npm/v/dbgate-sqltree.svg)](https://www.npmjs.com/package/dbgate-sqltree)
-* [types](https://github.com/dbshell/dbgate/tree/master/packages/types) - common TypeScript definitions [![NPM version](https://img.shields.io/npm/v/dbgate-types.svg)](https://www.npmjs.com/package/dbgate-types)
-* [web](https://github.com/dbshell/dbgate/tree/master/packages/web) - frontend in React (JavaScript)
-* [tools](https://github.com/dbshell/dbgate/tree/master/packages/tools) - various tools [![NPM version](https://img.shields.io/npm/v/dbgate-tools.svg)](https://www.npmjs.com/package/dbgate-tools)
+* [api](https://github.com/dbgate/dbgate/tree/master/packages/api) - backend, Javascript, ExpressJS [![NPM version](https://img.shields.io/npm/v/dbgate-api.svg)](https://www.npmjs.com/package/dbgate-api)
+* [datalib](https://github.com/dbgate/dbgate/tree/master/packages/datalib) - TypeScript library for utility classes [![NPM version](https://img.shields.io/npm/v/dbgate-datalib.svg)](https://www.npmjs.com/package/dbgate-datalib)
+* [app](https://github.com/dbgate/dbgate/tree/master/app) - application (JavaScript) structure, creating specific queries (JavaScript)
+* [filterparser](https://github.com/dbgate/dbgate/tree/master/packages/filterparser) - TypeScript library for parsing data filter expressions using parsimmon [![NPM version](https://img.shields.io/npm/v/dbgate-filterparser.svg)](https://www.npmjs.com/package/dbgate-filterparser)
+* [sqltree](https://github.com/dbgate/dbgate/tree/master/packages/sqltree) - JSON representation of SQL query, functions converting to SQL (TypeScript) [![NPM version](https://img.shields.io/npm/v/dbgate-sqltree.svg)](https://www.npmjs.com/package/dbgate-sqltree)
+* [types](https://github.com/dbgate/dbgate/tree/master/packages/types) - common TypeScript definitions [![NPM version](https://img.shields.io/npm/v/dbgate-types.svg)](https://www.npmjs.com/package/dbgate-types)
+* [web](https://github.com/dbgate/dbgate/tree/master/packages/web) - frontend in Svelte (JavaScript) [![NPM version](https://img.shields.io/npm/v/dbgate-web.svg)](https://www.npmjs.com/package/dbgate-web)
+* [tools](https://github.com/dbgate/dbgate/tree/master/packages/tools) - various tools [![NPM version](https://img.shields.io/npm/v/dbgate-tools.svg)](https://www.npmjs.com/package/dbgate-tools)

@@ -1,6 +1,6 @@
 {
  "name": "dbgate",
-  "version": "3.9.1-beta.1",
+  "version": "4.1.1",
  "private": true,
  "author": "Jan Prochazka <jenasoft.database@gmail.com>",
  "description": "Opensource database administration tool",
@@ -11,7 +11,7 @@
  },
  "repository": {
    "type": "git",
-    "url": "https://github.com/dbshell/dbgate.git"
+    "url": "https://github.com/dbgate/dbgate.git"
  },
  "build": {
    "appId": "org.dbgate",
@@ -21,7 +21,13 @@
      "artifactName": "dbgate-mac-${version}.${ext}",
      "publish": [
        "github"
-      ]
+      ],
+      "target": {
+        "target": "default",
+        "arch": [
+          "x64"
+        ]
+      }
    },
    "linux": {
      "target": [
@@ -37,9 +43,19 @@
        "github"
      ]
    },
+    "snap": {
+      "publish": [
+        "github",
+        "snapStore"
+      ],
+      "environment": {
+        "ELECTRON_SNAP": "true"
+      }
+    },
    "win": {
      "target": [
-        "nsis"
+        "nsis",
+        "zip"
      ],
      "artifactName": "dbgate-windows-${version}.${ext}",
      "icon": "icon.ico",
@@ -59,16 +75,16 @@
    "start": "cross-env ELECTRON_START_URL=http://localhost:5000 electron .",
    "start:local": "cross-env electron .",
    "dist": "electron-builder",
-    "build": "cd ../packages/api && yarn build && cd ../web && yarn build:app && cd ../../app && yarn dist",
-    "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build:app && cd ../../app && yarn predist",
+    "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist",
+    "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn predist",
    "postinstall": "electron-builder install-app-deps",
-    "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/build/*\" packages && copyfiles \"../packages/web/build/**/*\" packages"
+    "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/public/*\" packages && copyfiles \"../packages/web/public/**/*\" packages && copyfiles --up 3 \"../plugins/dist/**/*\" packages/plugins"
  },
  "main": "src/electron.js",
  "devDependencies": {
    "copyfiles": "^2.2.0",
    "cross-env": "^6.0.3",
-    "electron": "11.1.1",
+    "electron": "11.2.3",
    "electron-builder": "22.9.1"
  },
  "optionalDependencies": {
@@ -1,6 +1,6 @@
 const electron = require('electron');
 const os = require('os');
-const { Menu } = require('electron');
+const { Menu, ipcMain } = require('electron');
 const { fork } = require('child_process');
 const { autoUpdater } = require('electron-updater');
 const Store = require('electron-store');
@@ -20,12 +20,15 @@ const store = new Store();
 // be closed automatically when the JavaScript object is garbage collected.
 let mainWindow;
 let splashWindow;
+let mainMenu;

 log.transports.file.level = 'debug';
 autoUpdater.logger = log;
 // TODO - create settings for this
 // appUpdater.channel = 'beta';

+let commands = {};
+
 function hideSplash() {
  if (splashWindow) {
    splashWindow.destroy();
@@ -34,29 +37,48 @@ function hideSplash() {
  mainWindow.show();
 }

+function commandItem(id) {
+  const command = commands[id];
+  return {
+    id,
+    label: command ? command.menuName || command.toolbarName || command.name : id,
+    accelerator: command ? command.keyText : undefined,
+    enabled: command ? command.enabled : false,
+    click() {
+      mainWindow.webContents.executeJavaScript(`dbgate_runCommand('${id}')`);
+    },
+  };
+}
+
 function buildMenu() {
  const template = [
    {
      label: 'File',
      submenu: [
-        {
-          label: 'Connect to database',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_createNewConnection()`);
-          },
-        },
-        {
-          label: 'New query',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_newQuery()`);
-          },
-        },
+        commandItem('new.connection'),
+        commandItem('file.open'),
+        commandItem('group.save'),
+        commandItem('group.saveAs'),
+        { type: 'separator' },
+        { role: 'close' },
      ],
    },
    {
-      label: 'Edit',
-      submenu: [{ role: 'copy' }, { role: 'paste' }],
+      label: 'Window',
+      submenu: [commandItem('new.query'), { type: 'separator' }, commandItem('tabs.closeAll'), { role: 'minimize' }],
    },
+
+    // {
+    //   label: 'Edit',
+    //   submenu: [
+    //     { role: 'undo' },
+    //     { role: 'redo' },
+    //     { type: 'separator' },
+    //     { role: 'cut' },
+    //     { role: 'copy' },
+    //     { role: 'paste' },
+    //   ],
+    // },
    {
      label: 'View',
      submenu: [
@@ -69,20 +91,7 @@ function buildMenu() {
        { role: 'zoomout' },
        { type: 'separator' },
        { role: 'togglefullscreen' },
-      ],
-    },
-    {
-      role: 'window',
-      submenu: [
-        {
-          label: 'Close all tabs',
-          click() {
-            mainWindow.webContents.executeJavaScript('dbgate_closeAll()');
-          },
-        },
-        { type: 'separator' },
-        { role: 'minimize' },
-        { role: 'close' },
+        commandItem('theme.changeTheme'),
      ],
    },
    {
@@ -97,7 +106,7 @@ function buildMenu() {
        {
          label: 'DbGate on GitHub',
          click() {
-            require('electron').shell.openExternal('https://github.com/dbshell/dbgate');
+            require('electron').shell.openExternal('https://github.com/dbgate/dbgate');
          },
        },
        {
@@ -107,11 +116,12 @@ function buildMenu() {
          },
        },
        {
-          label: 'About',
+          label: 'Report problem or feature request',
          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_showAbout()`);
+            require('electron').shell.openExternal('https://github.com/dbgate/dbgate/issues/new');
          },
        },
+        commandItem('about.show'),
      ],
    },
  ];
@@ -119,6 +129,24 @@ function buildMenu() {
  return Menu.buildFromTemplate(template);
 }

+ipcMain.on('update-commands', async (event, arg) => {
+  commands = JSON.parse(arg);
+  for (const key of Object.keys(commands)) {
+    const menu = mainMenu.getMenuItemById(key);
+    if (!menu) continue;
+    const command = commands[key];
+
+    // rebuild menu
+    if (menu.label != command.text || menu.accelerator != command.keyText) {
+      mainMenu = buildMenu();
+      mainWindow.setMenu(mainMenu);
+      return;
+    }
+
+    menu.enabled = command.enabled;
+  }
+});
+
 function createWindow() {
  const bounds = store.get('winBounds');

@@ -135,13 +163,14 @@ function createWindow() {
    },
  });

-  mainWindow.setMenu(buildMenu());
+  mainMenu = buildMenu();
+  mainWindow.setMenu(mainMenu);

  function loadMainWindow() {
    const startUrl =
      process.env.ELECTRON_START_URL ||
      url.format({
-        pathname: path.join(__dirname, '../packages/web/build/index.html'),
+        pathname: path.join(__dirname, '../packages/web/public/index.html'),
        protocol: 'file:',
        slashes: true,
      });
@@ -176,14 +205,16 @@ function createWindow() {
  } else {
    const apiProcess = fork(path.join(__dirname, '../packages/api/dist/bundle.js'), [
      '--dynport',
+      '--is-electron-bundle',
      '--native-modules',
      path.join(__dirname, 'nativeModules'),
      // '../../../src/nativeModules'
    ]);
    apiProcess.on('message', msg => {
      if (msg.msgtype == 'listening') {
-        const { port } = msg;
+        const { port, authorization } = msg;
        global['port'] = port;
+        global['authorization'] = authorization;
        loadMainWindow();
      }
    });
@@ -717,10 +717,10 @@ electron-updater@^4.3.5:
    lodash.isequal "^4.5.0"
    semver "^7.3.2"

-electron@11.1.1:
-  version "11.1.1"
-  resolved "https://registry.yarnpkg.com/electron/-/electron-11.1.1.tgz#188f036f8282798398dca9513e9bb3b10213e3aa"
-  integrity sha512-tlbex3xosJgfileN6BAQRotevPRXB/wQIq48QeQ08tUJJrXwE72c8smsM/hbHx5eDgnbfJ2G3a60PmRjHU2NhA==
+electron@11.2.3:
+  version "11.2.3"
+  resolved "https://registry.yarnpkg.com/electron/-/electron-11.2.3.tgz#8ad1d9858436cfca0e2e5ea7fea326794ae58ebb"
+  integrity sha512-6yxOc42nDAptHKNlUG/vcOh2GI9x2fqp2nQbZO0/3sz2CrwsJkwR3i3oMN9XhVJaqI7GK1vSCJz0verOkWlXcQ==
  dependencies:
    "@electron/get" "^1.0.1"
    "@types/node" "^12.0.12"
@@ -2,7 +2,6 @@ const fs = require('fs');

 let fillContent = '';

-// if (!process.argv.includes('--electron')) {
 if (process.platform == 'win32') {
  fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');`;
 }
@@ -1,14 +1,16 @@
 {
  "private": true,
+  "version": "4.1.12-beta.2",
  "name": "dbgate-all",
  "workspaces": [
-    "packages/*"
+    "packages/*",
+    "plugins/*"
  ],
  "scripts": {
    "start:api": "yarn workspace dbgate-api start",
    "start:api:portal": "yarn workspace dbgate-api start:portal",
    "start:api:covid": "yarn workspace dbgate-api start:covid",
-    "start:web": "yarn workspace dbgate-web start",
+    "start:web": "yarn workspace dbgate-web dev",
    "start:sqltree": "yarn workspace dbgate-sqltree start",
    "start:tools": "yarn workspace dbgate-tools start",
    "start:datalib": "yarn workspace dbgate-datalib start",
@@ -18,25 +20,27 @@
    "build:filterparser": "yarn workspace dbgate-filterparser build",
    "build:tools": "yarn workspace dbgate-tools build",
    "build:lib": "yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
-    "build:app": "cd app && yarn install && yarn build",
+    "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
    "build:api": "yarn workspace dbgate-api build",
-    "build:web:docker": "yarn workspace dbgate-web build:docker",
-    "build:app:local": "cd app && yarn build:local",
+    "build:web:docker": "yarn workspace dbgate-web build",
+    "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
+    "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch",
+    "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist",
+    "build:app:local": "yarn plugins:copydist && cd app && yarn build:local",
    "start:app:local": "cd app && yarn start:local",
    "setCurrentVersion": "node setCurrentVersion",
    "generatePadFile": "node generatePadFile",
    "fillNativeModules": "node fillNativeModules",
-    "fillNativeModulesElectron": "node fillNativeModules --eletron",
+    "fillNativeModulesElectron": "node fillNativeModules --electron",
    "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
-    "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/build/* docker -u 2 && copyfiles \"packages/web/build/**/*\" docker -u 2",
-    "prepare:docker": "yarn build:web:docker && yarn build:api && yarn copy:docker:build",
-    "prepare": "yarn build:lib",
+    "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
+    "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build",
    "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
-    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\"",
+    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn build:plugins:frontend:watch\"",
    "ts:api": "yarn workspace dbgate-api ts",
    "ts:web": "yarn workspace dbgate-web ts",
    "ts": "yarn ts:api && yarn ts:web",
-    "postinstall": "patch-package && yarn fillNativeModules"
+    "postinstall": "yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend"
  },
  "dependencies": {
    "concurrently": "^5.1.0",
@@ -45,6 +49,7 @@
  },
  "devDependencies": {
    "copyfiles": "^2.2.0",
-    "prettier": "^2.2.1"
+    "prettier": "^2.2.1",
+    "workspaces-run": "^1.0.1"
  }
 }
@@ -1,23 +1 @@
-CONNECTIONS=mysql,postgres
-
-LABEL_mysql=MySql localhost
-SERVER_mysql=localhost
-USER_mysql=root
-PASSWORD_mysql=test
-PORT_mysql=3307
-ENGINE_mysql=mysql
-
-LABEL_postgres=Postgres localhost
-SERVER_postgres=localhost
-USER_postgres=postgres
-PASSWORD_postgres=test
-PORT_postgres=5433
-ENGINE_postgres=postgres
-
-TOOLBAR=home
-ICON_home=mdi mdi-home
-TITLE_home=Home
-PAGE_home=home.html
-STARTUP_PAGES=home
-
-PAGES_DIRECTORY=/home/jena/jenasoft/dbgate-web/pages
+DEVMODE=1
@@ -1,3 +1,5 @@
+DEVMODE=1
+
 CONNECTIONS=mysql

 LABEL_mysql=MySql
@@ -0,0 +1,17 @@
+DEVMODE=1
+
+CONNECTIONS=mysql,postgres
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+LABEL_postgres=Postgres localhost
+SERVER_postgres=localhost
+USER_postgres=postgres
+PASSWORD_postgres=test
+PORT_postgres=5433
+ENGINE_postgres=postgres@dbgate-plugin-postgres
@@ -32,7 +32,7 @@ dbgateApi.runScript(run);

 ```

-Silly example, runs without any dependencies. Copy [fakeObjectReader](https://github.com/dbshell/dbgate/blob/master/packages/api/src/shell/fakeObjectReader.js) to [consoleObjectWriter](https://github.com/dbshell/dbgate/blob/master/packages/api/src/shell/consoleObjectWriter.js) .
+Silly example, runs without any dependencies. Copy [fakeObjectReader](https://github.com/dbgate/dbgate/blob/master/packages/api/src/shell/fakeObjectReader.js) to [consoleObjectWriter](https://github.com/dbgate/dbgate/blob/master/packages/api/src/shell/consoleObjectWriter.js) .

 ```javascript

@@ -1,15 +1,14 @@
 {
  "name": "dbgate-api",
  "main": "src/index.js",
-  "version": "1.0.7",
+  "version": "4.1.1",
  "homepage": "https://dbgate.org/",
  "repository": {
    "type": "git",
-    "url": "https://github.com/dbshell/dbgate.git"
+    "url": "https://github.com/dbgate/dbgate.git"
  },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
  "author": "Jan Prochazka",
-  "license": "GPL",
+  "license": "MIT",
  "keywords": [
    "sql",
    "json",
@@ -26,34 +25,39 @@
    "compare-versions": "^3.6.0",
    "cors": "^2.8.5",
    "cross-env": "^6.0.3",
-    "dbgate-sqltree": "^1.0.0",
-    "dbgate-tools": "^1.0.0",
+    "dbgate-sqltree": "^4.1.1",
+    "dbgate-tools": "^4.1.1",
    "eslint": "^6.8.0",
    "express": "^4.17.1",
    "express-basic-auth": "^1.2.0",
    "express-fileupload": "^1.2.0",
-    "find-free-port": "^2.0.0",
-    "fs-extra": "^8.1.0",
+    "fs-extra": "^9.1.0",
+    "get-port": "^5.1.1",
    "http": "^0.0.0",
+    "json-stable-stringify": "^1.0.1",
    "line-reader": "^0.4.0",
    "lodash": "^4.17.15",
    "ncp": "^2.0.0",
    "nedb-promises": "^4.0.1",
    "node-cron": "^2.0.3",
+    "node-ssh-forward": "^0.7.2",
+    "portfinder": "^1.0.28",
    "simple-encryptor": "^4.0.0",
+    "socket.io": "^2.3.0",
    "tar": "^6.0.5",
    "uuid": "^3.4.0"
  },
  "scripts": {
-    "start": "nodemon src/index.js",
-    "start:portal": "env-cmd nodemon src/index.js",
-    "start:covid": "env-cmd -f .covid-env nodemon src/index.js",
+    "start": "env-cmd node src/index.js",
+    "start:portal": "env-cmd -f .env-portal node src/index.js",
+    "start:covid": "env-cmd -f .env-covid node src/index.js",
    "ts": "tsc",
    "build": "webpack"
  },
  "devDependencies": {
+    "@types/fs-extra": "^9.0.11",
    "@types/lodash": "^4.14.149",
-    "dbgate-types": "^1.0.0",
+    "dbgate-types": "^4.1.1",
    "env-cmd": "^10.1.0",
    "node-loader": "^1.0.2",
    "nodemon": "^2.0.2",
@@ -1,6 +1,24 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { datadir } = require('../utility/directories');
+const hasPermission = require('../utility/hasPermission');
+const socket = require('../utility/socket');
+const _ = require('lodash');
+
 const currentVersion = require('../currentVersion');
+const platformInfo = require('../utility/platformInfo');

 module.exports = {
+  settingsValue: {},
+
+  async _init() {
+    try {
+      this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
+    } catch (err) {
+      this.settingsValue = {};
+    }
+  },
+
  get_meta: 'get',
  async get() {
    // const toolbarButtons = process.env.TOOLBAR;
@@ -31,4 +49,31 @@ module.exports = {
      ...currentVersion,
    };
  },
+
+  platformInfo_meta: 'get',
+  async platformInfo() {
+    return platformInfo;
+  },
+
+  getSettings_meta: 'get',
+  async getSettings() {
+    return this.settingsValue;
+  },
+
+  updateSettings_meta: 'post',
+  async updateSettings(values) {
+    if (!hasPermission(`settings/change`)) return false;
+    try {
+      const updated = {
+        ...this.settingsValue,
+        ...values,
+      };
+      await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
+      this.settingsValue = updated;
+      socket.emitChanged(`settings-changed`);
+      return updated;
+    } catch (err) {
+      return false;
+    }
+  },
 };
@@ -6,6 +6,7 @@ const nedb = require('nedb-promises');
 const { datadir } = require('../utility/directories');
 const socket = require('../utility/socket');
 const { encryptConnection } = require('../utility/crypting');
+const { handleProcessCommunication } = require('../utility/processComm');

 function getPortalCollections() {
  if (process.env.CONNECTIONS) {
@@ -45,8 +46,9 @@ module.exports = {
    raw: true,
  },
  test(req, res) {
-    const subprocess = fork(process.argv[1], ['connectProcess', ...process.argv.slice(3)]);
+    const subprocess = fork(process.argv[1], ['--start-process', 'connectProcess', ...process.argv.slice(3)]);
    subprocess.on('message', resp => {
+      if (handleProcessCommunication(resp, subprocess)) return;
      // @ts-ignore
      const { msgtype } = resp;
      if (msgtype == 'connected' || msgtype == 'error') {
@@ -3,6 +3,8 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const { DatabaseAnalyser } = require('dbgate-tools');
+const { handleProcessCommunication } = require('../utility/processComm');
+const config = require('./config');

 module.exports = {
  /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -16,6 +18,13 @@ module.exports = {
    existing.structure = structure;
    socket.emitChanged(`database-structure-changed-${conid}-${database}`);
  },
+  handle_version(conid, database, { version }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.serverVersion = version;
+    socket.emitChanged(`database-server-version-changed-${conid}-${database}`);
+  },
+
  handle_error(conid, database, props) {
    const { error } = props;
    console.log(`Error in database connection ${conid}, database ${database}: ${error}`);
@@ -39,19 +48,26 @@ module.exports = {
    const existing = this.opened.find(x => x.conid == conid && x.database == database);
    if (existing) return existing;
    const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['databaseConnectionProcess', ...process.argv.slice(3)]);
+    const subprocess = fork(process.argv[1], [
+      '--start-process',
+      'databaseConnectionProcess',
+      ...process.argv.slice(3),
+    ]);
    const lastClosed = this.closed[`${conid}/${database}`];
    const newOpened = {
      conid,
      database,
      subprocess,
      structure: lastClosed ? lastClosed.structure : DatabaseAnalyser.createEmptyStructure(),
+      serverVersion: lastClosed ? lastClosed.serverVersion : null,
      connection,
      status: { name: 'pending' },
    };
    this.opened.push(newOpened);
-    // @ts-ignore
-    subprocess.on('message', ({ msgtype, ...message }) => {
+    subprocess.on('message', message => {
+      // @ts-ignore
+      const { msgtype } = message;
+      if (handleProcessCommunication(message, subprocess)) return;
      if (newOpened.disconnected) return;
      this[`handle_${msgtype}`](conid, database, message);
    });
@@ -64,6 +80,7 @@ module.exports = {
      msgtype: 'connect',
      connection: { ...connection, database },
      structure: lastClosed ? lastClosed.structure : null,
+      globalSettings: config.settingsValue
    });
    return newOpened;
  },
@@ -89,6 +106,20 @@ module.exports = {
    return res;
  },

+  collectionData_meta: 'post',
+  async collectionData({ conid, database, options }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
+    return res.result;
+  },
+
+  updateCollection_meta: 'post',
+  async updateCollection({ conid, database, changeSet }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet });
+    return res.result;
+  },
+
  status_meta: 'get',
  async status({ conid, database }) {
    const existing = this.opened.find(x => x.conid == conid && x.database == database);
@@ -103,10 +134,14 @@ module.exports = {

  ping_meta: 'post',
  async ping({ conid, database }) {
-    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    let existing = this.opened.find(x => x.conid == conid && x.database == database);
+
    if (existing) {
      existing.subprocess.send({ msgtype: 'ping' });
+    } else {
+      existing = await this.ensureOpened(conid, database);
    }
+
    return {
      status: 'ok',
      connectionStatus: existing ? existing.status : null,
@@ -150,6 +185,22 @@ module.exports = {
    // };
  },

+  serverVersion_meta: 'get',
+  async serverVersion({ conid, database }) {
+    const opened = await this.ensureOpened(conid, database);
+    return opened.serverVersion;
+  },
+
+  sqlPreview_meta: 'post',
+  async sqlPreview({ conid, database, objects, options }) {
+    // wait for structure
+    await this.structure({ conid, database });
+
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'sqlPreview', objects, options });
+    return res;
+  },
+
  // runCommand_meta: 'post',
  // async runCommand({ conid, database, sql }) {
  //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);
@@ -78,6 +78,11 @@ module.exports = {
    }
  },

+  saveAs_meta: 'post',
+  async saveAs({ filePath, data, format }) {
+    await fs.writeFile(filePath, serialize(format, data));
+  },
+
  favorites_meta: 'get',
  async favorites() {
    if (!hasPermission(`files/favorites/read`)) return [];
@@ -121,7 +121,13 @@ module.exports = {
  getStats_meta: 'get',
  getStats({ jslid }) {
    const file = `${getJslFileName(jslid)}.stats`;
-    if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, 'utf-8'));
+    if (fs.existsSync(file)) {
+      try {
+        return JSON.parse(fs.readFileSync(file, 'utf-8'));
+      } catch (e) {
+        return {};
+      }
+    }
    return {};
  },

@@ -17,7 +17,7 @@ module.exports = {
  listObjects_meta: 'get',
  async listObjects({ conid, database }) {
    const opened = await databaseConnections.ensureOpened(conid, database);
-    const types = ['tables', 'views', 'procedures', 'functions', 'triggers'];
+    const types = ['tables', 'collections', 'views', 'procedures', 'functions', 'triggers'];
    return types.reduce(
      (res, type) => ({
        ...res,
@@ -2,43 +2,21 @@ const fs = require('fs-extra');
 const axios = require('axios');
 const path = require('path');
 const { extractPackageName } = require('dbgate-tools');
-const { pluginsdir, datadir } = require('../utility/directories');
+const { pluginsdir, packagedPluginsDir } = require('../utility/directories');
 const socket = require('../utility/socket');
 const compareVersions = require('compare-versions');
 const requirePlugin = require('../shell/requirePlugin');
 const downloadPackage = require('../utility/downloadPackage');
 const hasPermission = require('../utility/hasPermission');
-
-// async function loadPackageInfo(dir) {
-//   const readmeFile = path.join(dir, 'README.md');
-//   const packageFile = path.join(dir, 'package.json');
-
-//   if (!(await fs.exists(packageFile))) {
-//     return null;
-//   }
-
-//   let readme = null;
-//   let manifest = null;
-//   if (await fs.exists(readmeFile)) readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
-//   if (await fs.exists(packageFile)) manifest = JSON.parse(await fs.readFile(packageFile, { encoding: 'utf-8' }));
-//   return {
-//     readme,
-//     manifest,
-//   };
-// }
-
-const preinstallPluginMinimalVersions = {
-  'dbgate-plugin-mssql': '1.0.10',
-  'dbgate-plugin-mysql': '1.0.3',
-  'dbgate-plugin-postgres': '1.0.2',
-  'dbgate-plugin-csv': '1.0.8',
-  'dbgate-plugin-excel': '1.0.6',
-};
+const _ = require('lodash');

 module.exports = {
  script_meta: 'get',
  async script({ packageName }) {
-    const file = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
+    const file1 = path.join(packagedPluginsDir(), packageName, 'dist', 'frontend.js');
+    const file2 = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
+    // @ts-ignore
+    const file = (await fs.exists(file1)) ? file1 : file2;
    const data = await fs.readFile(file, {
      encoding: 'utf-8',
    });
@@ -62,10 +40,13 @@ module.exports = {
      const { latest } = infoResp.data['dist-tags'];
      const manifest = infoResp.data.versions[latest];
      const { readme } = infoResp.data;
+      // @ts-ignore
+      const isPackaged = await fs.exists(path.join(packagedPluginsDir(), packageName));

      return {
        readme,
        manifest,
+        isPackaged,
      };
    } catch (err) {
      return {
@@ -73,52 +54,52 @@ module.exports = {
        error: err.message,
      };
    }
-
-    // const dir = path.join(pluginstmpdir(), packageName);
-    // if (!(await fs.exists(dir))) {
-    //   await downloadPackage(packageName, dir);
-    // }
-    // return await loadPackageInfo(dir);
-    // return await {
-    //   ...loadPackageInfo(dir),
-    //   installed: loadPackageInfo(path.join(pluginsdir(), packageName)),
-    // };
  },

  installed_meta: 'get',
  async installed() {
-    const files = await fs.readdir(pluginsdir());
+    const files1 = await fs.readdir(packagedPluginsDir());
+    const files2 = await fs.readdir(pluginsdir());
+
    const res = [];
-    for (const packageName of files) {
-      const manifest = await fs.readFile(path.join(pluginsdir(), packageName, 'package.json')).then(x => JSON.parse(x));
-      const readmeFile = path.join(pluginsdir(), packageName, 'README.md');
-      if (await fs.exists(readmeFile)) {
-        manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+    for (const packageName of _.union(files1, files2)) {
+      if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
+      try {
+        const isPackaged = files1.includes(packageName);
+        const manifest = await fs
+          .readFile(path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'package.json'), {
+            encoding: 'utf-8',
+          })
+          .then(x => JSON.parse(x));
+        const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
+        // @ts-ignore
+        if (await fs.exists(readmeFile)) {
+          manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+        }
+        manifest.isPackaged = isPackaged;
+        res.push(manifest);
+      } catch (err) {
+        console.log(`Skipped plugin ${packageName}, error:`, err.message);
      }
-      res.push(manifest);
    }
    return res;
-    // const res = await Promise.all(
-    //   files.map((packageName) =>
-    //     fs.readFile(path.join(pluginsdir(), packageName, 'package.json')).then((x) => JSON.parse(x))
-    //   )
-    // );
  },

-  async saveRemovePlugins() {
-    await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
-  },
+  // async saveRemovePlugins() {
+  //   await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
+  // },

  install_meta: 'post',
  async install({ packageName }) {
    if (!hasPermission(`plugins/install`)) return;
    const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
    if (!(await fs.exists(dir))) {
      await downloadPackage(packageName, dir);
    }
    socket.emitChanged(`installed-plugins-changed`);
-    this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
-    await this.saveRemovePlugins();
+    // this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
+    // await this.saveRemovePlugins();
  },

  uninstall_meta: 'post',
@@ -127,7 +108,7 @@ module.exports = {
    const dir = path.join(pluginsdir(), packageName);
    await fs.rmdir(dir, { recursive: true });
    socket.emitChanged(`installed-plugins-changed`);
-    this.removedPlugins.push(packageName);
+    // this.removedPlugins.push(packageName);
    await this.saveRemovePlugins();
  },

@@ -135,6 +116,7 @@ module.exports = {
  async upgrade({ packageName }) {
    if (!hasPermission(`plugins/install`)) return;
    const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
    if (await fs.exists(dir)) {
      await fs.rmdir(dir, { recursive: true });
      await downloadPackage(packageName, dir);
@@ -149,7 +131,7 @@ module.exports = {
    return content.commands[command](args);
  },

-  authTypes_meta: 'post',
+  authTypes_meta: 'get',
  async authTypes({ engine }) {
    const packageName = extractPackageName(engine);
    const content = requirePlugin(packageName);
@@ -157,46 +139,46 @@ module.exports = {
    return content.driver.getAuthTypes() || null;
  },

-  async _init() {
-    const installed = await this.installed();
-    try {
-      this.removedPlugins = (await fs.readFile(path.join(datadir(), 'removed-plugins'), { encoding: 'utf-8' })).split(
-        '\n'
-      );
-    } catch (err) {
-      this.removedPlugins = [];
-    }
+  // async _init() {
+  //   const installed = await this.installed();
+  //   try {
+  //     this.removedPlugins = (await fs.readFile(path.join(datadir(), 'removed-plugins'), { encoding: 'utf-8' })).split(
+  //       '\n'
+  //     );
+  //   } catch (err) {
+  //     this.removedPlugins = [];
+  //   }

-    for (const packageName of Object.keys(preinstallPluginMinimalVersions)) {
-      const installedVersion = installed.find(x => x.name == packageName);
-      if (installedVersion) {
-        // plugin installed, test, whether upgrade
-        const requiredVersion = preinstallPluginMinimalVersions[packageName];
-        if (compareVersions(installedVersion.version, requiredVersion) < 0) {
-          console.log(
-            `Upgrading preinstalled plugin ${packageName}, found ${installedVersion.version}, required version ${requiredVersion}`
-          );
-          await this.upgrade({ packageName });
-        } else {
-          console.log(
-            `Plugin ${packageName} not upgraded, found ${installedVersion.version}, required version ${requiredVersion}`
-          );
-        }
+  //   for (const packageName of Object.keys(preinstallPluginMinimalVersions)) {
+  //     const installedVersion = installed.find(x => x.name == packageName);
+  //     if (installedVersion) {
+  //       // plugin installed, test, whether upgrade
+  //       const requiredVersion = preinstallPluginMinimalVersions[packageName];
+  //       if (compareVersions(installedVersion.version, requiredVersion) < 0) {
+  //         console.log(
+  //           `Upgrading preinstalled plugin ${packageName}, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //         await this.upgrade({ packageName });
+  //       } else {
+  //         console.log(
+  //           `Plugin ${packageName} not upgraded, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //       }

-        continue;
-      }
+  //       continue;
+  //     }

-      if (this.removedPlugins.includes(packageName)) {
-        // plugin was remvoed, don't install again
-        continue;
-      }
+  //     if (this.removedPlugins.includes(packageName)) {
+  //       // plugin was remvoed, don't install again
+  //       continue;
+  //     }

-      try {
-        console.log('Preinstalling plugin', packageName);
-        await this.install({ packageName });
-      } catch (err) {
-        console.error('Error preinstalling plugin', packageName, err);
-      }
-    }
-  },
+  //     try {
+  //       console.log('Preinstalling plugin', packageName);
+  //       await this.install({ packageName });
+  //     } catch (err) {
+  //       console.error('Error preinstalling plugin', packageName, err);
+  //     }
+  //   }
+  // },
 };
@@ -5,8 +5,9 @@ const uuidv1 = require('uuid/v1');
 const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
-const { rundir, uploadsdir, pluginsdir } = require('../utility/directories');
+const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
 const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
+const { handleProcessCommunication } = require('../utility/processComm');

 function extractPlugins(script) {
  const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
@@ -23,6 +24,7 @@ const requirePluginsTemplate = plugins =>

 const scriptTemplate = script => `
 const dbgateApi = require(process.env.DBGATE_API);
+dbgateApi.initializeApiEnvironment();
 ${requirePluginsTemplate(extractPlugins(script))}
 require=null;
 async function run() {
@@ -35,6 +37,7 @@ dbgateApi.runScript(run);

 const loaderScriptTemplate = (functionName, props, runid) => `
 const dbgateApi = require(process.env.DBGATE_API);
+dbgateApi.initializeApiEnvironment();
 ${requirePluginsTemplate(extractShellApiPlugins(functionName, props))}
 require=null;
 async function run() {
@@ -89,15 +92,16 @@ module.exports = {
    const scriptFile = path.join(uploadsdir(), runid + '.js');
    fs.writeFileSync(`${scriptFile}`, scriptText);
    fs.mkdirSync(directory);
-    const pluginNames = fs.readdirSync(pluginsdir());
+    const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
    console.log(`RUNNING SCRIPT ${scriptFile}`);
    // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
    const subprocess = fork(scriptFile, ['--checkParent', ...process.argv.slice(3)], {
      cwd: directory,
      stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
      env: {
-        DBGATE_API: process.argv[1],
-        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, path.join(pluginsdir(), name)])),
+        ...process.env,
+        DBGATE_API: global['dbgateApiModulePath'] || process.argv[1],
+        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
      },
    });
    const pipeDispatcher = severity => data =>
@@ -123,8 +127,10 @@ module.exports = {
      subprocess,
    };
    this.opened.push(newOpened);
-    // @ts-ignore
-    subprocess.on('message', ({ msgtype, ...message }) => {
+    subprocess.on('message', message => {
+      // @ts-ignore
+      const { msgtype } = message;
+      if (handleProcessCommunication(message, subprocess)) return;
      this[`handle_${msgtype}`](runid, message);
    });
    return newOpened;
@@ -3,11 +3,14 @@ const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const _ = require('lodash');
 const AsyncLock = require('async-lock');
+const { handleProcessCommunication } = require('../utility/processComm');
 const lock = new AsyncLock();
+const config = require('./config');

 module.exports = {
  opened: [],
  closed: {},
+  lastPinged: {},

  handle_databases(conid, { databases }) {
    const existing = this.opened.find(x => x.conid == conid);
@@ -15,6 +18,12 @@ module.exports = {
    existing.databases = databases;
    socket.emitChanged(`database-list-changed-${conid}`);
  },
+  handle_version(conid, { version }) {
+    const existing = this.opened.find(x => x.conid == conid);
+    if (!existing) return;
+    existing.version = version;
+    socket.emitChanged(`server-version-changed-${conid}`);
+  },
  handle_status(conid, { status }) {
    const existing = this.opened.find(x => x.conid == conid);
    if (!existing) return;
@@ -28,7 +37,11 @@ module.exports = {
      const existing = this.opened.find(x => x.conid == conid);
      if (existing) return existing;
      const connection = await connections.get({ conid });
-      const subprocess = fork(process.argv[1], ['serverConnectionProcess', ...process.argv.slice(3)]);
+      const subprocess = fork(process.argv[1], [
+        '--start-process',
+        'serverConnectionProcess',
+        ...process.argv.slice(3),
+      ]);
      const newOpened = {
        conid,
        subprocess,
@@ -42,8 +55,10 @@ module.exports = {
      this.opened.push(newOpened);
      delete this.closed[conid];
      socket.emitChanged(`server-status-changed`);
-      // @ts-ignore
-      subprocess.on('message', ({ msgtype, ...message }) => {
+      subprocess.on('message', message => {
+        // @ts-ignore
+        const { msgtype } = message;
+        if (handleProcessCommunication(message, subprocess)) return;
        if (newOpened.disconnected) return;
        this[`handle_${msgtype}`](conid, message);
      });
@@ -51,7 +66,7 @@ module.exports = {
        if (newOpened.disconnected) return;
        this.close(conid, false);
      });
-      subprocess.send({ msgtype: 'connect', ...connection });
+      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: config.settingsValue });
      return newOpened;
    });
    return res;
@@ -77,6 +92,12 @@ module.exports = {
    return opened.databases;
  },

+  version_meta: 'get',
+  async version({ conid }) {
+    const opened = await this.ensureOpened(conid);
+    return opened.version;
+  },
+
  serverStatus_meta: 'get',
  async serverStatus() {
    return {
@@ -88,7 +109,12 @@ module.exports = {
  ping_meta: 'post',
  async ping({ connections }) {
    await Promise.all(
-      connections.map(async conid => {
+      _.uniq(connections).map(async conid => {
+        const last = this.lastPinged[conid];
+        if (last && new Date().getTime() - last < 30 * 1000) {
+          return Promise.resolve();
+        }
+        this.lastPinged[conid] = new Date().getTime();
        const opened = await this.ensureOpened(conid);
        opened.subprocess.send({ msgtype: 'ping' });
      })
@@ -4,6 +4,7 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
+const { handleProcessCommunication } = require('../utility/processComm');

 module.exports = {
  /** @type {import('dbgate-types').OpenedSession[]} */
@@ -64,7 +65,7 @@ module.exports = {
  async create({ conid, database }) {
    const sesid = uuidv1();
    const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['sessionProcess', ...process.argv.slice(3)]);
+    const subprocess = fork(process.argv[1], ['--start-process', 'sessionProcess', ...process.argv.slice(3)]);
    const newOpened = {
      conid,
      database,
@@ -73,8 +74,10 @@ module.exports = {
      sesid,
    };
    this.opened.push(newOpened);
-    // @ts-ignore
-    subprocess.on('message', ({ msgtype, ...message }) => {
+    subprocess.on('message', message => {
+      // @ts-ignore
+      const { msgtype } = message;
+      if (handleProcessCommunication(message, subprocess)) return;
      this[`handle_${msgtype}`](sesid, message);
    });
    subprocess.send({ msgtype: 'connect', ...connection, database });
@@ -1,4 +1,5 @@
-module.exports = {
-  version: '3.8.6',
-  buildTime: '2020-12-10T11:14:01.053Z',
+
+module.exports = { 
+  version: '4.1.1',
+  buildTime: '2021-04-17T07:22:49.702Z'
 };
@@ -1,21 +1,17 @@
 const shell = require('./shell');
-// require('socket.io-client');
+const processArgs = require('./utility/processArgs');

-// "socket.io-client": "^2.3.0",
-// "utf-8-validate": "^5.0.2",
-// "uuid": "^3.4.0",
-// "uws": "10.148.1"
-
-const argument = process.argv[2];
-if (argument && argument.endsWith('Process')) {
+if (processArgs.startProcess) {
  const proc = require('./proc');
-
-  const module = proc[argument];
+  const module = proc[processArgs.startProcess];
  module.start();
-} else if (!module['parent'] && !process.argv.includes('--checkParent')) {
+} else if (!module['parent'] && !processArgs.checkParent) {
  const main = require('./main');

-  main.start(argument);
+  main.start();
 }

-module.exports = shell;
+module.exports = {
+  ...shell,
+  getMainModule: () => require('./main'),
+};
@@ -6,8 +6,10 @@ const http = require('http');
 const cors = require('cors');
 const io = require('socket.io');
 const fs = require('fs');
-const findFreePort = require('find-free-port');
+const getPort = require('get-port');
 const childProcessChecker = require('./utility/childProcessChecker');
+const path = require('path');
+const crypto = require('crypto');

 const useController = require('./utility/useController');
 const socket = require('./utility/socket');
@@ -27,8 +29,14 @@ const files = require('./controllers/files');
 const scheduler = require('./controllers/scheduler');

 const { rundir } = require('./utility/directories');
+const platformInfo = require('./utility/platformInfo');
+const processArgs = require('./utility/processArgs');
+const timingSafeCheckToken = require('./utility/timingSafeCheckToken');

-function start(argument = null) {
+let authorization = null;
+let checkLocalhostOrigin = null;
+
+function start() {
  // console.log('process.argv', process.argv);

  const app = express();
@@ -48,6 +56,29 @@ function start(argument = null) {
    );
  }

+  app.use(function (req, res, next) {
+    if (authorization && !timingSafeCheckToken(req.headers.authorization, authorization)) {
+      return res.status(403).json({ error: 'Not authorized!' });
+    }
+    if (checkLocalhostOrigin) {
+      if (
+        req.headers.origin &&
+        req.headers.origin != checkLocalhostOrigin &&
+        req.headers.origin != `http://${checkLocalhostOrigin}`
+      ) {
+        console.log('API origin check FAILED');
+        console.log('HEADERS', { ...req.headers, authorization: '***' });
+        return res.status(403).json({ error: 'Not authorized!' });
+      }
+      if (!req.headers.origin && req.headers.host != checkLocalhostOrigin) {
+        console.log('API host check FAILED');
+        console.log('HEADERS', { ...req.headers, authorization: '***' });
+        return res.status(403).json({ error: 'Not authorized!' });
+      }
+    }
+    next();
+  });
+
  app.use(cors());
  app.use(bodyParser.json({ limit: '50mb' }));

@@ -78,22 +109,34 @@ function start(argument = null) {

  app.use('/runners/data', express.static(rundir()));

-  if (fs.existsSync('/home/dbgate-docker/build')) {
+  if (platformInfo.isDocker) {
    // server static files inside docker container
-    app.use(express.static('/home/dbgate-docker/build'));
+    app.use(express.static('/home/dbgate-docker/public'));
  } else {
-    app.get('/', (req, res) => {
-      res.send('DbGate API');
-    });
+    if (!platformInfo.isNpmDist) {
+      app.get('/', (req, res) => {
+        res.send('DbGate API');
+      });
+    }
  }

-  if (argument == '--dynport') {
+  if (processArgs.dynport) {
    childProcessChecker();

-    findFreePort(53911, function (err, port) {
+    authorization = crypto.randomBytes(32).toString('hex');
+
+    getPort().then(port => {
+      checkLocalhostOrigin = `localhost:${port}`;
+      server.listen(port, () => {
+        console.log(`DbGate API listening on port ${port}`);
+        process.send({ msgtype: 'listening', port, authorization });
+      });
+    });
+  } else if (platformInfo.isNpmDist) {
+    app.use(express.static(path.join(__dirname, '../../dbgate-web/public')));
+    getPort({ port: 5000 }).then(port => {
      server.listen(port, () => {
        console.log(`DbGate API listening on port ${port}`);
-        process.send({ msgtype: 'listening', port });
      });
    });
  } else {
@@ -1,18 +1,43 @@
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');
+const { handleProcessCommunication } = require('../utility/processComm');
+const _ = require('lodash');
+
+function pickSafeConnectionInfo(connection) {
+  return _.mapValues(connection, (v, k) => {
+    if (k == 'engine' || k == 'port' || k == 'authType' || k == 'sshMode' || k == 'passwordMode') return v;
+    if (v === null || v === true || v === false) return v;
+    if (v) return '***';
+    return undefined;
+  });
+}
+
+const formatErrorDetail = (e, connection) => `${e.stack}
+
+Error JSON: ${JSON.stringify(e, undefined, 2)}
+
+Connection: ${JSON.stringify(pickSafeConnectionInfo(connection), undefined, 2)}
+
+Platform: ${process.platform}
+`;

 function start() {
  childProcessChecker();
  process.on('message', async connection => {
+    if (handleProcessCommunication(connection)) return;
    try {
      const driver = requireEngineDriver(connection);
-      const conn = await driver.connect(decryptConnection(connection));
+      const conn = await connectUtility(driver, connection);
      const res = await driver.getVersion(conn);
      process.send({ msgtype: 'connected', ...res });
    } catch (e) {
      console.error(e);
-      process.send({ msgtype: 'error', error: e.message });
+      process.send({
+        msgtype: 'error',
+        error: e.message,
+        detail: formatErrorDetail(e, connection),
+      });
    }
  });
 }
@@ -1,7 +1,10 @@
 const stableStringify = require('json-stable-stringify');
 const childProcessChecker = require('../utility/childProcessChecker');
+const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');
+const { handleProcessCommunication } = require('../utility/processComm');
+const { SqlGenerator } = require('dbgate-tools');

 let systemConnection;
 let storedConnection;
@@ -27,6 +30,7 @@ async function checkedAsyncCall(promise) {

 async function handleFullRefresh() {
  const driver = requireEngineDriver(storedConnection);
+  setStatusName('loadStructure');
  analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection));
  process.send({ msgtype: 'structure', structure: analysedStructure });
  setStatusName('ok');
@@ -34,6 +38,7 @@ async function handleFullRefresh() {

 async function handleIncrementalRefresh() {
  const driver = requireEngineDriver(storedConnection);
+  setStatusName('checkStructure');
  const newStructure = await checkedAsyncCall(driver.analyseIncremental(systemConnection, analysedStructure));
  if (newStructure != null) {
    analysedStructure = newStructure;
@@ -54,20 +59,34 @@ function setStatusName(name) {
  setStatus({ name });
 }

-async function handleConnect({ connection, structure }) {
+async function readVersion() {
+  const driver = requireEngineDriver(storedConnection);
+  const version = await driver.getVersion(systemConnection);
+  process.send({ msgtype: 'version', version });
+}
+
+async function handleConnect({ connection, structure, globalSettings }) {
  storedConnection = connection;
  lastPing = new Date().getTime();

  if (!structure) setStatusName('pending');
  const driver = requireEngineDriver(storedConnection);
-  systemConnection = await checkedAsyncCall(driver.connect(decryptConnection(storedConnection)));
+  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection));
+  readVersion();
  if (structure) {
    analysedStructure = structure;
    handleIncrementalRefresh();
  } else {
    handleFullRefresh();
  }
-  setInterval(handleIncrementalRefresh, 30 * 1000);
+
+  if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', true)) {
+    setInterval(
+      handleIncrementalRefresh,
+      extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 3, 3600) * 1000
+    );
+  }
+
  for (const [resolve] of afterConnectCallbacks) {
    resolve();
  }
@@ -92,6 +111,49 @@ async function handleQueryData({ msgid, sql }) {
  }
 }

+async function handleCollectionData({ msgid, options }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    const result = await driver.readCollection(systemConnection, options);
+    process.send({ msgtype: 'response', msgid, result });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
+async function handleUpdateCollection({ msgid, changeSet }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    const result = await driver.updateCollection(systemConnection, changeSet);
+    process.send({ msgtype: 'response', msgid, result });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
+async function handleSqlPreview({ msgid, objects, options }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+
+  try {
+    const dmp = driver.createDumper();
+    const generator = new SqlGenerator(analysedStructure, options, objects, dmp, driver, systemConnection);
+
+    await generator.dump();
+    process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
+    if (generator.isUnhandledException) {
+      setTimeout(() => {
+        console.log('Exiting because of unhandled exception');
+        process.exit(0);
+      }, 500);
+    }
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, isError: true, errorMessage: err.message });
+  }
+}
+
 // async function handleRunCommand({ msgid, sql }) {
 //   await waitConnected();
 //   const driver = engines(storedConnection);
@@ -106,6 +168,9 @@ function handlePing() {
 const messageHandlers = {
  connect: handleConnect,
  queryData: handleQueryData,
+  updateCollection: handleUpdateCollection,
+  collectionData: handleCollectionData,
+  sqlPreview: handleSqlPreview,
  ping: handlePing,
  // runCommand: handleRunCommand,
 };
@@ -127,6 +192,7 @@ function start() {
  }, 60 * 1000);

  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
    try {
      await handleMessage(message);
    } catch (e) {
@@ -1,5 +1,6 @@
 const childProcessChecker = require('../utility/childProcessChecker');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
+const { handleProcessCommunication } = require('../utility/processComm');

 let lastPing = null;
 let datastore = new JsonLinesDatastore();
@@ -47,6 +48,7 @@ function start() {
  }, 60 * 1000);

  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
    try {
      await handleMessage(message);
    } catch (e) {
@@ -1,7 +1,10 @@
 const stableStringify = require('json-stable-stringify');
+const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');
+const { handleProcessCommunication } = require('../utility/processComm');

 let systemConnection;
 let storedConnection;
@@ -29,6 +32,12 @@ async function handleRefresh() {
  }
 }

+async function readVersion() {
+  const driver = requireEngineDriver(storedConnection);
+  const version = await driver.getVersion(systemConnection);
+  process.send({ msgtype: 'version', version });
+}
+
 function setStatus(status) {
  const statusString = stableStringify(status);
  if (lastStatus != statusString) {
@@ -43,14 +52,18 @@ function setStatusName(name) {

 async function handleConnect(connection) {
  storedConnection = connection;
+  const { globalSettings } = storedConnection;
  setStatusName('pending');
  lastPing = new Date().getTime();

  const driver = requireEngineDriver(storedConnection);
  try {
-    systemConnection = await driver.connect(decryptConnection(storedConnection));
+    systemConnection = await connectUtility(driver, storedConnection);
+    readVersion();
    handleRefresh();
-    setInterval(handleRefresh, 30 * 1000);
+    if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', true)) {
+      setInterval(handleRefresh, extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000);
+    }
  } catch (err) {
    setStatus({
      name: 'error',
@@ -67,9 +80,13 @@ function handlePing() {

 async function handleCreateDatabase({ name }) {
  const driver = requireEngineDriver(storedConnection);
-  systemConnection = await driver.connect(decryptConnection(storedConnection));
+  systemConnection = await connectUtility(driver, storedConnection);
  console.log(`RUNNING SCRIPT: CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
-  await driver.query(systemConnection, `CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
+  if (driver.createDatabase) {
+    await driver.createDatabase(systemConnection, name);
+  } else {
+    await driver.query(systemConnection, `CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
+  }
  await handleRefresh();
 }

@@ -96,6 +113,7 @@ function start() {
  }, 60 * 1000);

  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
    try {
      await handleMessage(message);
    } catch (err) {
@@ -8,6 +8,8 @@ const goSplit = require('../utility/goSplit');
 const { jsldir } = require('../utility/directories');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');
+const { handleProcessCommunication } = require('../utility/processComm');

 let systemConnection;
 let storedConnection;
@@ -15,12 +17,18 @@ let afterConnectCallbacks = [];
 // let currentHandlers = [];

 class TableWriter {
-  constructor(columns, resultIndex) {
+  constructor(structure, resultIndex) {
    this.jslid = uuidv1();
    this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
    this.currentRowCount = 0;
    this.currentChangeIndex = 1;
-    fs.writeFileSync(this.currentFile, JSON.stringify({ columns }) + '\n');
+    fs.writeFileSync(
+      this.currentFile,
+      JSON.stringify({
+        ...structure,
+        __isStreamHeader: true,
+      }) + '\n'
+    );
    this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
    this.writeCurrentStats(false, false);
    this.resultIndex = resultIndex;
@@ -90,7 +98,7 @@ class StreamHandler {

  recordset(columns) {
    this.closeCurrentWriter();
-    this.currentWriter = new TableWriter(columns, this.resultIndexHolder.value);
+    this.currentWriter = new TableWriter(Array.isArray(columns) ? { columns } : columns, this.resultIndexHolder.value);
    this.resultIndexHolder.value += 1;

    // this.writeCurrentStats();
@@ -131,7 +139,7 @@ async function handleConnect(connection) {
  storedConnection = connection;

  const driver = requireEngineDriver(storedConnection);
-  systemConnection = await driver.connect(decryptConnection(storedConnection));
+  systemConnection = await connectUtility(driver, storedConnection);
  for (const [resolve] of afterConnectCallbacks) {
    resolve();
  }
@@ -182,6 +190,7 @@ async function handleMessage({ msgtype, ...other }) {
 function start() {
  childProcessChecker();
  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
    try {
      await handleMessage(message);
    } catch (e) {
@@ -1,9 +1,13 @@
+const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
+
 function copyStream(input, output) {
  return new Promise((resolve, reject) => {
+    const ensureHeader = new EnsureStreamHeaderStream();
    const finisher = output['finisher'] || output;
    finisher.on('finish', resolve);
    finisher.on('error', reject);
-    input.pipe(output);
+    input.pipe(ensureHeader);
+    ensureHeader.pipe(output);
  });
 }

@@ -1,12 +1,13 @@
 const goSplit = require('../utility/goSplit');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');

 async function executeQuery({ connection, sql }) {
  console.log(`Execute query ${sql}`);

  const driver = requireEngineDriver(connection);
-  const pool = await driver.connect(decryptConnection(connection));
+  const pool = await connectUtility(driver, connection);
  console.log(`Connected.`);

  for (const sqlItem of goSplit(sql)) {
@@ -17,6 +17,7 @@ const requirePlugin = require('./requirePlugin');
 const download = require('./download');
 const executeQuery = require('./executeQuery');
 const loadFile = require('./loadFile');
+const initializeApiEnvironment = require('./initializeApiEnvironment');

 const dbgateApi = {
  queryReader,
@@ -37,6 +38,7 @@ const dbgateApi = {
  registerPlugins,
  executeQuery,
  loadFile,
+  initializeApiEnvironment,
 };

 requirePlugin.initializeDbgateApi(dbgateApi);
@@ -0,0 +1,9 @@
+const { handleProcessCommunication } = require('../utility/processComm');
+
+async function initializeApiEnvironment() {
+  process.on('message', async message => {
+    handleProcessCommunication(message);
+  });
+}
+
+module.exports = initializeApiEnvironment;
@@ -3,9 +3,8 @@ const stream = require('stream');
 const byline = require('byline');

 class ParseStream extends stream.Transform {
-  constructor({ header, limitRows }) {
+  constructor({ limitRows }) {
    super({ objectMode: true });
-    this.header = header;
    this.wasHeader = false;
    this.limitRows = limitRows;
    this.rowsWritten = 0;
@@ -13,7 +12,14 @@ class ParseStream extends stream.Transform {
  _transform(chunk, encoding, done) {
    const obj = JSON.parse(chunk);
    if (!this.wasHeader) {
-      if (!this.header) this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
+      if (
+        !obj.__isStreamHeader &&
+        // TODO remove isArray test
+        !Array.isArray(obj.columns)
+      ) {
+        this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
+      }
+      
      this.wasHeader = true;
    }
    if (!this.limitRows || this.rowsWritten < this.limitRows) {
@@ -24,12 +30,12 @@ class ParseStream extends stream.Transform {
  }
 }

-async function jsonLinesReader({ fileName, encoding = 'utf-8', header = true, limitRows = undefined }) {
+async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
  console.log(`Reading file ${fileName}`);

  const fileStream = fs.createReadStream(fileName, encoding);
  const liner = byline(fileStream);
-  const parser = new ParseStream({ header, limitRows });
+  const parser = new ParseStream({ limitRows });
  liner.pipe(parser);
  return parser;
 }
@@ -8,10 +8,16 @@ class StringifyStream extends stream.Transform {
    this.wasHeader = false;
  }
  _transform(chunk, encoding, done) {
+    let skip = false;
    if (!this.wasHeader) {
-      if (this.header) this.push(JSON.stringify(chunk) + '\n');
+      skip =
+        (chunk.__isStreamHeader ||
+          // TODO remove isArray test
+          Array.isArray(chunk.columns)) &&
+        !this.header;
      this.wasHeader = true;
-    } else {
+    }
+    if (!skip) {
      this.push(JSON.stringify(chunk) + '\n');
    }
    done();
@@ -1,11 +1,12 @@
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');

 async function queryReader({ connection, sql }) {
  console.log(`Reading query ${sql}`);

  const driver = requireEngineDriver(connection);
-  const pool = await driver.connect(decryptConnection(connection));
+  const pool = await connectUtility(driver, connection);
  console.log(`Connected.`);
  return await driver.readQuery(pool, sql);
 }
@@ -1,6 +1,8 @@
 const path = require('path');
-const { pluginsdir } = require('../utility/directories');
+const fs = require('fs');
+const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories');
 const nativeModules = require('../nativeModules');
+const platformInfo = require('../utility/platformInfo');

 const loadedPlugins = {};

@@ -8,20 +10,19 @@ const dbgateEnv = {
  dbgateApi: null,
  nativeModules,
 };
-
 function requirePlugin(packageName, requiredPlugin = null) {
  if (!packageName) throw new Error('Missing packageName in plugin');
  if (loadedPlugins[packageName]) return loadedPlugins[packageName];

  if (requiredPlugin == null) {
    let module;
-    const modulePath = path.join(pluginsdir(), packageName, 'dist', 'backend.js');
+    const modulePath = getPluginBackendPath(packageName);
    console.log(`Loading module ${packageName} from ${modulePath}`);
    try {
      // @ts-ignore
      module = __non_webpack_require__(modulePath);
    } catch (err) {
-      console.error('Failed load webpacked module', err);
+      console.log('Failed load webpacked module', err.message);
      module = require(modulePath);
    }
    requiredPlugin = module.__esModule ? module.default : module;
@@ -1,7 +1,8 @@
 const childProcessChecker = require('../utility/childProcessChecker');
+const processArgs = require('../utility/processArgs');

 async function runScript(func) {
-  if (process.argv.includes('--checkParent')) {
+  if (processArgs.checkParent) {
    childProcessChecker();
  }
  try {
@@ -1,14 +1,21 @@
 const { quoteFullName, fullNameToString } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
-const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');

 async function tableReader({ connection, pureName, schemaName }) {
  const driver = requireEngineDriver(connection);
-  const pool = await driver.connect(decryptConnection(connection));
+  const pool = await connectUtility(driver, connection);
  console.log(`Connected.`);

  const fullName = { pureName, schemaName };

+  if (driver.dialect.nosql) {
+    // @ts-ignore
+    console.log(`Reading collection ${fullNameToString(fullName)}`);
+    // @ts-ignore
+    return await driver.readQuery(pool, JSON.stringify(fullName));
+  }
+
  const table = await driver.analyseSingleObject(pool, fullName, 'tables');
  const query = `select * from ${quoteFullName(driver.dialect, fullName)}`;
  if (table) {
@@ -1,12 +1,13 @@
 const { fullNameToString } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
 const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');

 async function tableWriter({ connection, schemaName, pureName, ...options }) {
  console.log(`Writing table ${fullNameToString({ schemaName, pureName })}`);

  const driver = requireEngineDriver(connection);
-  const pool = await driver.connect(decryptConnection(connection));
+  const pool = await connectUtility(driver, connection);
  console.log(`Connected.`);
  return await driver.writeTable(pool, { schemaName, pureName }, options);
 }
@@ -1,5 +1,6 @@
 const { fork } = require('child_process');
 const uuidv1 = require('uuid/v1');
+const { handleProcessCommunication } = require('./processComm');

 class DatastoreProxy {
  constructor(file) {
@@ -28,10 +29,13 @@ class DatastoreProxy {

  async ensureSubprocess() {
    if (!this.subprocess) {
-      this.subprocess = fork(process.argv[1], ['jslDatastoreProcess', ...process.argv.slice(3)]);
+      this.subprocess = fork(process.argv[1], ['--start-process', 'jslDatastoreProcess', ...process.argv.slice(3)]);
+
+      this.subprocess.on('message', message => {
+        // @ts-ignore
+        const { msgtype } = message;
+        if (handleProcessCommunication(message, this.subprocess)) return;

-      // @ts-ignore
-      this.subprocess.on('message', ({ msgtype, ...message }) => {
        // if (this.disconnected) return;
        this[`handle_${msgtype}`](message);
      });
@@ -0,0 +1,35 @@
+const stream = require('stream');
+
+class EnsureStreamHeaderStream extends stream.Transform {
+  constructor() {
+    super({ objectMode: true });
+    this.wasHeader = false;
+  }
+  _transform(chunk, encoding, done) {
+    if (!this.wasHeader) {
+      if (chunk.__isDynamicStructure) {
+        // ignore dynamic structure header
+        done();
+        return;
+      }
+
+      if (
+        !chunk.__isStreamHeader &&
+        // TODO remove isArray test
+        !Array.isArray(chunk.columns)
+      ) {
+        this.push({
+          __isStreamHeader: true,
+          __isComputedStructure: true,
+          columns: Object.keys(chunk).map(columnName => ({ columnName })),
+        });
+      }
+
+      this.wasHeader = true;
+    }
+    this.push(chunk);
+    done();
+  }
+}
+
+module.exports = EnsureStreamHeaderStream;
@@ -0,0 +1,64 @@
+const { SSHConnection } = require('node-ssh-forward');
+const portfinder = require('portfinder');
+const fs = require('fs-extra');
+const { decryptConnection } = require('./crypting');
+const { getSshTunnel } = require('./sshTunnel');
+const { getSshTunnelProxy } = require('./sshTunnelProxy');
+
+async function connectUtility(driver, storedConnection) {
+  const connection = {
+    database: storedConnection.defaultDatabase,
+    ...decryptConnection(storedConnection),
+  };
+
+  if (!connection.port && driver.defaultPort) connection.port = driver.defaultPort.toString();
+
+  if (connection.useSshTunnel) {
+    const tunnel = await getSshTunnelProxy(connection);
+    if (tunnel.state == 'error') {
+      throw new Error(tunnel.message);
+    }
+
+    connection.server = '127.0.0.1';
+    connection.port = tunnel.localPort;
+  }
+
+  // SSL functionality - copied from https://github.com/beekeeper-studio/beekeeper-studio
+  if (connection.useSsl) {
+    connection.ssl = {};
+
+    if (connection.sslCaFile) {
+      connection.ssl.ca = await fs.readFile(connection.sslCaFile);
+    }
+
+    if (connection.sslCertFile) {
+      connection.ssl.cert = await fs.readFile(connection.sslCertFile);
+    }
+
+    if (connection.sslKeyFile) {
+      connection.ssl.key = await fs.readFile(connection.sslKeyFile);
+    }
+
+    if (connection.sslCertFilePassword) {
+      connection.ssl.password = connection.sslCertFilePassword;
+    }
+
+    if (!connection.ssl.key && !connection.ssl.ca && !connection.ssl.cert) {
+      // TODO: provide this as an option in settings
+      // or per-connection as 'reject self-signed certs'
+      // How it works:
+      // if false, cert can be self-signed
+      // if true, has to be from a public CA
+      // Heroku certs are self-signed.
+      // if you provide ca/cert/key files, it overrides this
+      connection.ssl.rejectUnauthorized = false;
+    } else {
+      connection.ssl.rejectUnauthorized = connection.sslRejectUnauthorized;
+    }
+  }
+
+  const conn = await driver.connect(connection);
+  return conn;
+}
+
+module.exports = connectUtility;
@@ -42,28 +42,42 @@ function getEncryptor() {
  return _encryptor;
 }

-function encryptConnection(connection) {
+function encryptPasswordField(connection, field) {
  if (
    connection &&
-    connection.password &&
-    !connection.password.startsWith('crypt:') &&
+    connection[field] &&
+    !connection[field].startsWith('crypt:') &&
    connection.passwordMode != 'saveRaw'
  ) {
    return {
      ...connection,
-      password: 'crypt:' + getEncryptor().encrypt(connection.password),
+      [field]: 'crypt:' + getEncryptor().encrypt(connection[field]),
+    };
+  }
+return connection;
+}
+
+function decryptPasswordField(connection, field) {
+  if (connection && connection[field] && connection[field].startsWith('crypt:')) {
+    return {
+      ...connection,
+      [field]: getEncryptor().decrypt(connection[field].substring('crypt:'.length)),
    };
  }
  return connection;
 }

+function encryptConnection(connection) {
+  connection = encryptPasswordField(connection, 'password');
+  connection = encryptPasswordField(connection, 'sshPassword');
+  connection = encryptPasswordField(connection, 'sshKeyFilePassword');
+  return connection;
+}
+
 function decryptConnection(connection) {
-  if (connection && connection.password && connection.password.startsWith('crypt:')) {
-    return {
-      ...connection,
-      password: getEncryptor().decrypt(connection.password.substring('crypt:'.length)),
-    };
-  }
+  connection = decryptPasswordField(connection, 'password');
+  connection = decryptPasswordField(connection, 'sshPassword');
+  connection = decryptPasswordField(connection, 'sshKeyFilePassword');
  return connection;
 }

@@ -2,6 +2,7 @@ const os = require('os');
 const path = require('path');
 const fs = require('fs');
 const cleanDirectory = require('./cleanDirectory');
+const platformInfo = require('./platformInfo');

 const createDirectories = {};
 const ensureDirectory = (dir, clean) => {
@@ -39,6 +40,37 @@ const pluginsdir = dirFunc('plugins');
 const archivedir = dirFunc('archive');
 const filesdir = dirFunc('files');

+function packagedPluginsDir() {
+  if (platformInfo.isDevMode) {
+    return path.resolve(__dirname, '../../../../plugins');
+  }
+  if (platformInfo.isDocker) {
+    return '/home/dbgate-docker/plugins';
+  }
+  if (platformInfo.isNpmDist) {
+    // node_modules
+    return global['dbgateApiPackagedPluginsPath'];
+  }
+  if (platformInfo.isElectronBundle) {
+    return path.resolve(__dirname, '../../plugins');
+  }
+  return null;
+}
+
+const packagedPluginList =
+  packagedPluginsDir() != null ? fs.readdirSync(packagedPluginsDir()).filter(x => x.startsWith('dbgate-plugin-')) : [];
+
+function getPluginBackendPath(packageName) {
+  if (packagedPluginList.includes(packageName)) {
+    if (platformInfo.isDevMode) {
+      return path.join(packagedPluginsDir(), packageName, 'src', 'backend', 'index.js');
+    }
+    return path.join(packagedPluginsDir(), packageName, 'dist', 'backend.js');
+  }
+
+  return path.join(pluginsdir(), packageName, 'dist', 'backend.js');
+}
+
 module.exports = {
  datadir,
  jsldir,
@@ -48,4 +80,7 @@ module.exports = {
  ensureDirectory,
  pluginsdir,
  filesdir,
+  packagedPluginsDir,
+  packagedPluginList,
+  getPluginBackendPath,
 };
@@ -0,0 +1,43 @@
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const processArgs = require('./processArgs');
+
+const platform = process.env.OS_OVERRIDE ? process.env.OS_OVERRIDE : process.platform;
+const isWindows = platform === 'win32';
+const isMac = platform === 'darwin';
+const isLinux = platform === 'linux';
+const isDocker = fs.existsSync('/home/dbgate-docker/public');
+const isDevMode = process.env.DEVMODE == '1';
+const isNpmDist = !!global['dbgateApiModulePath'];
+
+// function moduleAvailable(name) {
+//   try {
+//     require.resolve(name);
+//     return true;
+//   } catch (e) {
+//     return false;
+//   }
+// }
+
+const isElectronBundle = processArgs.isElectronBundle;
+
+const platformInfo = {
+  isWindows,
+  isMac,
+  isLinux,
+  isDocker,
+  isElectronBundle,
+  isDevMode,
+  isNpmDist,
+  isSnap: process.env.ELECTRON_SNAP == 'true',
+  isPortable: isWindows && process.env.PORTABLE_EXECUTABLE_DIR,
+  isAppImage: process.env.DESKTOPINTEGRATION === 'AppImageLauncher',
+  sshAuthSock: process.env.SSH_AUTH_SOCK,
+  environment: process.env.NODE_ENV,
+  platform,
+  runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
+  defaultKeyFile: path.join(os.homedir(), '.ssh/id_rsa'),
+};
+
+module.exports = platformInfo;
@@ -0,0 +1,21 @@
+function getNamedArg(name) {
+  const argIndex = process.argv.indexOf(name);
+  if (argIndex > 0) {
+    return process.argv[argIndex + 1];
+  }
+  return null;
+}
+
+const checkParent = process.argv.includes('--checkParent');
+const dynport = process.argv.includes('--dynport');
+const nativeModules = getNamedArg('--native-modules');
+const startProcess = getNamedArg('--start-process');
+const isElectronBundle = process.argv.includes('--is-electron-bundle');
+
+module.exports = {
+  checkParent,
+  nativeModules,
+  startProcess,
+  dynport,
+  isElectronBundle,
+};
@@ -0,0 +1,18 @@
+const { handleGetSshTunnelRequest, handleGetSshTunnelResponse } = require('./sshTunnelProxy');
+
+function handleProcessCommunication(message, subprocess) {
+  const { msgtype } = message;
+  if (msgtype == 'getsshtunnel-request') {
+    handleGetSshTunnelRequest(message, subprocess);
+    return true;
+  }
+  if (msgtype == 'getsshtunnel-response') {
+    handleGetSshTunnelResponse(message, subprocess);
+    return true;
+  }
+  return false;
+}
+
+module.exports = {
+  handleProcessCommunication,
+};
@@ -0,0 +1,91 @@
+const { SSHConnection } = require('node-ssh-forward');
+const fs = require('fs-extra');
+const portfinder = require('portfinder');
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const platformInfo = require('./platformInfo');
+const AsyncLock = require('async-lock');
+const lock = new AsyncLock();
+
+const sshConnectionCache = {};
+const sshTunnelCache = {};
+
+const CONNECTION_FIELDS = [
+  'sshHost',
+  'sshPort',
+  'sshLogin',
+  'sshPassword',
+  'sshMode',
+  'sshKeyFile',
+  'sshBastionHost',
+  'sshKeyFilePassword',
+];
+const TUNNEL_FIELDS = [...CONNECTION_FIELDS, 'server', 'port'];
+
+async function getSshConnection(connection) {
+  const connectionCacheKey = stableStringify(_.pick(connection, CONNECTION_FIELDS));
+  if (sshConnectionCache[connectionCacheKey]) return sshConnectionCache[connectionCacheKey];
+
+  const sshConfig = {
+    endHost: connection.sshHost || '',
+    endPort: connection.sshPort || 22,
+    bastionHost: connection.sshBastionHost || '',
+    agentForward: connection.sshMode == 'agent',
+    passphrase: connection.sshMode == 'keyFile' ? connection.sshKeyFilePassword : undefined,
+    username: connection.sshLogin,
+    password: connection.sshMode == 'userPassword' ? connection.sshPassword : undefined,
+    agentSocket: connection.sshMode == 'agent' ? platformInfo.sshAuthSock : undefined,
+    privateKey:
+      connection.sshMode == 'keyFile' && connection.sshKeyfile ? await fs.readFile(connection.sshKeyfile) : undefined,
+    skipAutoPrivateKey: true,
+    noReadline: true,
+  };
+
+  const sshConn = new SSHConnection(sshConfig);
+  sshConnectionCache[connectionCacheKey] = sshConn;
+  return sshConn;
+}
+
+async function getSshTunnel(connection) {
+  const tunnelCacheKey = stableStringify(_.pick(connection, TUNNEL_FIELDS));
+
+  return await lock.acquire(tunnelCacheKey, async () => {
+    const sshConn = await getSshConnection(connection);
+    if (sshTunnelCache[tunnelCacheKey]) return sshTunnelCache[tunnelCacheKey];
+
+    const localPort = await portfinder.getPortPromise({ port: 10000, stopPort: 60000 });
+    // workaround for `getPortPromise` not releasing the port quickly enough
+    await new Promise(resolve => setTimeout(resolve, 500));
+    const tunnelConfig = {
+      fromPort: localPort,
+      toPort: connection.port,
+      toHost: connection.server,
+    };
+    try {
+      console.log(
+        `Creating SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
+      );
+
+      const tunnel = await sshConn.forward(tunnelConfig);
+      console.log(
+        `Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
+      );
+
+      sshTunnelCache[tunnelCacheKey] = {
+        state: 'ok',
+        localPort,
+      };
+      return sshTunnelCache[tunnelCacheKey];
+    } catch (err) {
+      // error is not cached
+      return {
+        state: 'error',
+        message: err.message,
+      };
+    }
+  });
+}
+
+module.exports = {
+  getSshTunnel,
+};
@@ -0,0 +1,30 @@
+const uuidv1 = require('uuid/v1');
+const { getSshTunnel } = require('./sshTunnel');
+
+const dispatchedMessages = {};
+
+async function handleGetSshTunnelRequest({ msgid, connection }, subprocess) {
+  const response = await getSshTunnel(connection);
+  subprocess.send({ msgtype: 'getsshtunnel-response', msgid, response });
+}
+
+function handleGetSshTunnelResponse({ msgid, response }, subprocess) {
+  const { resolve } = dispatchedMessages[msgid];
+  delete dispatchedMessages[msgid];
+  resolve(response);
+}
+
+async function getSshTunnelProxy(connection) {
+  if (!process.send) return getSshTunnel(connection);
+  const msgid = uuidv1();
+  process.send({ msgtype: 'getsshtunnel-request', msgid, connection });
+  return new Promise((resolve, reject) => {
+    dispatchedMessages[msgid] = { resolve, reject };
+  });
+}
+
+module.exports = {
+  handleGetSshTunnelRequest,
+  handleGetSshTunnelResponse,
+  getSshTunnelProxy,
+};
@@ -0,0 +1,9 @@
+const crypto = require('crypto');
+
+function timingSafeCheckToken(a, b) {
+  if (!a || !b) return false;
+  if (a.length != b.length) return false;
+  return crypto.timingSafeEqual(Buffer.from(a), Buffer.from(b));
+}
+
+module.exports = timingSafeCheckToken;
@@ -1,10 +1,9 @@
 {
-  "version": "1.0.0",
+  "version": "4.1.1",
  "name": "dbgate-datalib",
  "main": "lib/index.js",
  "typings": "lib/index.d.ts",
  "scripts": {
-    "prepare": "yarn build",
    "build": "tsc",
    "start": "tsc --watch"
  },
@@ -12,11 +11,11 @@
    "lib"
  ],
  "dependencies": {
-    "dbgate-sqltree": "^1.0.0",
-    "dbgate-filterparser": "^1.0.0"
+    "dbgate-sqltree": "^4.1.1",
+    "dbgate-filterparser": "^4.1.1"
  },
  "devDependencies": {
-    "dbgate-types": "^1.0.0",
+    "dbgate-types": "^4.1.1",
    "@types/node": "^13.7.0",
    "typescript": "^3.7.5"
  }
@@ -6,6 +6,7 @@ export interface ChangeSetItem {
  pureName: string;
  schemaName?: string;
  insertedRowIndex?: number;
+  document?: any;
  condition?: { [column: string]: string };
  fields?: { [column: string]: string };
 }
@@ -117,6 +118,43 @@ export function setChangeSetValue(
  };
 }

+export function setChangeSetRowData(changeSet: ChangeSet, definition: ChangeSetRowDefinition, document: any): ChangeSet {
+  if (!changeSet || !definition) return changeSet;
+  let [fieldName, existingItem] = findExistingChangeSetItem(changeSet, definition);
+  if (fieldName == 'deletes') {
+    changeSet = revertChangeSetRowChanges(changeSet, definition);
+    [fieldName, existingItem] = findExistingChangeSetItem(changeSet, definition);
+  }
+  if (existingItem) {
+    return {
+      ...changeSet,
+      [fieldName]: changeSet[fieldName].map(item =>
+        item == existingItem
+          ? {
+              ...item,
+              fields: {},
+              document,
+            }
+          : item
+      ),
+    };
+  }
+
+  return {
+    ...changeSet,
+    [fieldName]: [
+      ...changeSet[fieldName],
+      {
+        pureName: definition.pureName,
+        schemaName: definition.schemaName,
+        condition: definition.condition,
+        insertedRowIndex: definition.insertedRowIndex,
+        document,
+      },
+    ],
+  };
+}
+
 // export function batchUpdateChangeSet(
 //   changeSet: ChangeSet,
 //   rowDefinitions: ChangeSetRowDefinition[],
@@ -332,6 +370,7 @@ export function getChangeSetInsertedRows(changeSet: ChangeSet, name?: NamedObjec
 }

 export function changeSetInsertNewRow(changeSet: ChangeSet, name?: NamedObjectInfo): ChangeSet {
+  console.log('INSERT', name);
  const insertedRows = getChangeSetInsertedRows(changeSet, name);
  return {
    ...changeSet,
@@ -347,5 +386,6 @@ export function changeSetInsertNewRow(changeSet: ChangeSet, name?: NamedObjectIn
 }

 export function changeSetContainsChanges(changeSet: ChangeSet) {
+  if (!changeSet) return false;
  return changeSet.deletes.length > 0 || changeSet.updates.length > 0 || changeSet.inserts.length > 0;
 }
@@ -0,0 +1,104 @@
+import _ from 'lodash';
+import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc, DisplayColumn } from './GridDisplay';
+import { EngineDriver, ViewInfo, ColumnInfo, CollectionInfo } from 'dbgate-types';
+import { GridConfig, GridCache } from './GridConfig';
+
+function getObjectKeys(obj) {
+  if (_.isArray(obj)) {
+    return Object.keys(obj)
+      .slice(0, 10)
+      .map(x => parseInt(x));
+  }
+  if (_.isPlainObject(obj)) {
+    return Object.keys(obj);
+  }
+  return [];
+}
+
+function createHeaderText(path) {
+  let res = `${path[0]}`;
+  for (let i = 1; i < path.length; i++) {
+    const name = path[i];
+    if (_.isNumber(name)) res += `[${name}]`;
+    else res += `.${name}`;
+  }
+  return res;
+}
+
+function getColumnsForObject(basePath, obj, res: any[], display) {
+  for (const name of getObjectKeys(obj)) {
+    const uniqueName = [...basePath, name].join('.');
+    let column = res.find(x => x.uniqueName == uniqueName);
+    if (!column) {
+      column = getDisplayColumn(basePath, name, display);
+      if (basePath.length > 0) {
+        const lastIndex1 = _.findLastIndex(res, x => x.parentHeaderText.startsWith(column.parentHeaderText));
+        const lastIndex2 = _.findLastIndex(res, x => x.headerText == column.parentHeaderText);
+        // console.log(uniqueName, lastIndex1, lastIndex2);
+        if (lastIndex1 >= 0) res.splice(lastIndex1 + 1, 0, column);
+        else if (lastIndex2 >= 0) res.splice(lastIndex2 + 1, 0, column);
+        else res.push(column);
+      } else {
+        res.push(column);
+      }
+    }
+    if (_.isPlainObject(obj[name]) || _.isArray(obj[name])) {
+      column.isExpandable = true;
+    }
+
+    if (display.isExpandedColumn(column.uniqueName)) {
+      getColumnsForObject([...basePath, name], obj[name], res, display);
+    }
+  }
+}
+
+function getDisplayColumn(basePath, columnName, display) {
+  const uniquePath = [...basePath, columnName];
+  const uniqueName = uniquePath.join('.');
+  return {
+    columnName,
+    headerText: createHeaderText(uniquePath),
+    uniqueName,
+    uniquePath,
+    isStructured: true,
+    parentHeaderText: createHeaderText(basePath),
+    filterType: 'mongo',
+    pureName: display.collection?.pureName,
+    schemaName: display.collection?.schemaName,
+  };
+}
+
+export function analyseCollectionDisplayColumns(rows, display) {
+  const res = [];
+  for (const row of rows || []) {
+    getColumnsForObject([], row, res, display);
+  }
+  return (
+    res.map(col => ({
+      ...col,
+      isChecked: display.isColumnChecked(col),
+    })) || []
+  );
+}
+
+export class CollectionGridDisplay extends GridDisplay {
+  constructor(
+    public collection: CollectionInfo,
+    driver: EngineDriver,
+    config: GridConfig,
+    setConfig: ChangeConfigFunc,
+    cache: GridCache,
+    setCache: ChangeCacheFunc,
+    loadedRows
+  ) {
+    super(config, setConfig, cache, setCache, driver);
+    this.columns = analyseCollectionDisplayColumns(loadedRows, this);
+    this.filterable = true;
+    this.sortable = true;
+    this.editable = true;
+    this.supportsReload = true;
+    this.isDynamicStructure = true;
+    this.changeSetKeyFields = ['_id'];
+    this.baseCollection = collection;
+  }
+}
@@ -1,6 +1,14 @@
 import _ from 'lodash';
 import { GridConfig, GridCache, GridConfigColumns, createGridCache, GroupFunc } from './GridConfig';
-import { ForeignKeyInfo, TableInfo, ColumnInfo, EngineDriver, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
+import {
+  ForeignKeyInfo,
+  TableInfo,
+  ColumnInfo,
+  EngineDriver,
+  NamedObjectInfo,
+  DatabaseInfo,
+  SqlDialect,
+} from 'dbgate-types';
 import { parseFilter, getFilterType, getFilterValueExpression } from 'dbgate-filterparser';
 import { filterName } from './filterName';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
@@ -12,6 +20,7 @@ export class FormViewDisplay {
  isLoadedCorrectly = true;
  columns: DisplayColumn[];
  public baseTable: TableInfo;
+  dialect: SqlDialect;

  constructor(
    public config: GridConfig,
@@ -19,8 +28,11 @@ export class FormViewDisplay {
    public cache: GridCache,
    protected setCache: ChangeCacheFunc,
    public driver?: EngineDriver,
-    public dbinfo: DatabaseInfo = null
-  ) {}
+    public dbinfo: DatabaseInfo = null,
+    public serverVersion = null
+  ) {
+    this.dialect = (driver?.dialectByVersion && driver?.dialectByVersion(serverVersion)) || driver?.dialect;
+  }

  addFilterColumn(column) {
    if (!column) return;
@@ -1,6 +1,15 @@
 import _ from 'lodash';
 import { GridConfig, GridCache, GridConfigColumns, createGridCache, GroupFunc } from './GridConfig';
-import { ForeignKeyInfo, TableInfo, ColumnInfo, EngineDriver, NamedObjectInfo, DatabaseInfo } from 'dbgate-types';
+import {
+  ForeignKeyInfo,
+  TableInfo,
+  ColumnInfo,
+  EngineDriver,
+  NamedObjectInfo,
+  DatabaseInfo,
+  CollectionInfo,
+  SqlDialect,
+} from 'dbgate-types';
 import { parseFilter, getFilterType } from 'dbgate-filterparser';
 import { filterName } from './filterName';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';
@@ -18,9 +27,12 @@ export interface DisplayColumn {
  autoIncrement?: boolean;
  isPrimaryKey?: boolean;
  foreignKey?: ForeignKeyInfo;
+  isExpandable?: boolean;
  isChecked?: boolean;
  hintColumnName?: string;
  dataType?: string;
+  filterType?: boolean;
+  isStructured?: boolean;
 }

 export interface DisplayedColumnEx extends DisplayColumn {
@@ -49,16 +61,25 @@ export abstract class GridDisplay {
    public cache: GridCache,
    protected setCache: ChangeCacheFunc,
    public driver?: EngineDriver,
-    public dbinfo: DatabaseInfo = null
-  ) {}
+    public dbinfo: DatabaseInfo = null,
+    public serverVersion = null
+  ) {
+    this.dialect = (driver?.dialectByVersion && driver?.dialectByVersion(serverVersion)) || driver?.dialect;
+  }
+  dialect: SqlDialect;
  columns: DisplayColumn[];
  baseTable?: TableInfo;
+  baseCollection?: CollectionInfo;
+  get baseTableOrCollection(): NamedObjectInfo {
+    return this.baseTable || this.baseCollection;
+  }
  changeSetKeyFields: string[] = null;
  sortable = false;
  filterable = false;
  editable = false;
  isLoadedCorrectly = true;
  supportsReload = false;
+  isDynamicStructure = false;

  setColumnVisibility(uniquePath: string[], isVisible: boolean) {
    const uniqueName = uniquePath.join('.');
@@ -66,7 +87,7 @@ export abstract class GridDisplay {
      this.includeInColumnSet('hiddenColumns', uniqueName, !isVisible);
    } else {
      this.includeInColumnSet('addedColumns', uniqueName, isVisible);
-      this.reload();
+      if (!this.isDynamicStructure) this.reload();
    }
  }

@@ -94,11 +115,7 @@ export abstract class GridDisplay {
  }

  reload() {
-    this.setCache(cache => ({
-      // ...cache,
-      ...createGridCache(),
-      refreshTime: new Date().getTime(),
-    }));
+    this.setCache(reloadDataCacheFunc);
  }

  includeInColumnSet(field: keyof GridConfigColumns, uniqueName: string, isIncluded: boolean) {
@@ -290,6 +307,28 @@ export abstract class GridDisplay {
    this.reload();
  }

+  showFilter(uniqueName) {
+    this.setConfig(cfg => {
+      if (!cfg.filters.uniqueName)
+        return {
+          ...cfg,
+          filters: {
+            ..._.omitBy(cfg.filters, v => !v),
+            [uniqueName]: '',
+          },
+        };
+      return cfg;
+    });
+  }
+
+  removeFilter(uniqueName) {
+    this.setConfig(cfg => ({
+      ...cfg,
+      filters: _.omit(cfg.filters, [uniqueName]),
+    }));
+    this.reload();
+  }
+
  setFilters(dct) {
    this.setConfig(cfg => ({
      ...cfg,
@@ -365,8 +404,12 @@ export abstract class GridDisplay {
  getChangeSetField(row, uniqueName, insertedRowIndex): ChangeSetFieldDefinition {
    const col = this.columns.find(x => x.uniqueName == uniqueName);
    if (!col) return null;
-    if (!this.baseTable) return null;
-    if (this.baseTable.pureName != col.pureName || this.baseTable.schemaName != col.schemaName) return null;
+    const baseTableOrCollection = this.baseTableOrCollection;
+    if (!baseTableOrCollection) return null;
+    if (baseTableOrCollection.pureName != col.pureName || baseTableOrCollection.schemaName != col.schemaName) {
+      return null;
+    }
+
    return {
      ...this.getChangeSetRow(row, insertedRowIndex),
      uniqueName: uniqueName,
@@ -375,10 +418,11 @@ export abstract class GridDisplay {
  }

  getChangeSetRow(row, insertedRowIndex): ChangeSetRowDefinition {
-    if (!this.baseTable) return null;
+    const baseTableOrCollection = this.baseTableOrCollection;
+    if (!baseTableOrCollection) return null;
    return {
-      pureName: this.baseTable.pureName,
-      schemaName: this.baseTable.schemaName,
+      pureName: baseTableOrCollection.pureName,
+      schemaName: baseTableOrCollection.schemaName,
      insertedRowIndex,
      condition: insertedRowIndex == null ? this.getChangeSetCondition(row) : null,
    };
@@ -421,12 +465,75 @@ export abstract class GridDisplay {
    return select;
  }

+  getRowNumberOverSelect(select: Select, offset: number, count: number): Select {
+    const innerSelect: Select = {
+      commandType: 'select',
+      from: select.from,
+      where: select.where,
+      columns: [
+        ...select.columns,
+        {
+          alias: '_rowNumber',
+          exprType: 'rowNumber',
+          orderBy: select.orderBy
+            ? select.orderBy.map(x =>
+                x.exprType != 'column'
+                  ? x
+                  : x.source
+                  ? x
+                  : {
+                      ...x,
+                      source: { alias: 'basetbl' },
+                    }
+              )
+            : [
+                {
+                  ...select.columns[0],
+                  direction: 'ASC',
+                },
+              ],
+        },
+      ],
+    };
+
+    const res: Select = {
+      commandType: 'select',
+      selectAll: true,
+      from: {
+        subQuery: innerSelect,
+        alias: '_RowNumberResult',
+      },
+      where: {
+        conditionType: 'between',
+        expr: {
+          exprType: 'column',
+          columnName: '_RowNumber',
+          source: {
+            alias: '_RowNumberResult',
+          },
+        },
+        left: {
+          exprType: 'value',
+          value: offset + 1,
+        },
+        right: {
+          exprType: 'value',
+          value: offset + count,
+        },
+      },
+    };
+
+    return res;
+  }
+
  getPageQuery(offset: number, count: number) {
    if (!this.driver) return null;
-    const select = this.createSelect();
+    let select = this.createSelect();
    if (!select) return null;
-    if (this.driver.dialect.rangeSelect) select.range = { offset: offset, limit: count };
-    else if (this.driver.dialect.limitSelect) select.topRecords = count;
+    if (this.dialect.rangeSelect) select.range = { offset: offset, limit: count };
+    else if (this.dialect.rowNumberOverPaging && offset > 0)
+      select = this.getRowNumberOverSelect(select, offset, count);
+    else if (this.dialect.limitSelect) select.topRecords = count;
    const sql = treeToSql(this.driver, select, dumpSqlSelect);
    return sql;
  }
@@ -535,4 +642,19 @@ export abstract class GridDisplay {
      formViewKeyRequested: null,
    }));
  }
+
+  switchToJsonView() {
+    this.setConfig(cfg => ({
+      ...cfg,
+      isJsonView: true,
+    }));
+  }
+}
+
+export function reloadDataCacheFunc(cache: GridCache): GridCache {
+  return {
+    // ...cache,
+    ...createGridCache(),
+    refreshTime: new Date().getTime(),
+  };
 }
@@ -1,34 +1,44 @@
 import { GridDisplay, ChangeCacheFunc, ChangeConfigFunc } from './GridDisplay';
 import { QueryResultColumn } from 'dbgate-types';
 import { GridConfig, GridCache } from './GridConfig';
+import { analyseCollectionDisplayColumns } from './CollectionGridDisplay';

 export class JslGridDisplay extends GridDisplay {
  constructor(
    jslid,
-    columns: QueryResultColumn[],
+    structure,
    config: GridConfig,
    setConfig: ChangeConfigFunc,
    cache: GridCache,
-    setCache: ChangeCacheFunc
+    setCache: ChangeCacheFunc,
+    rows: any
  ) {
    super(config, setConfig, cache, setCache, null);

    this.filterable = true;

-    this.columns = columns
-      .map(col => ({
-        columnName: col.columnName,
-        headerText: col.columnName,
-        uniqueName: col.columnName,
-        uniquePath: [col.columnName],
-        notNull: col.notNull,
-        autoIncrement: col.autoIncrement,
-        pureName: null,
-        schemaName: null,
-      }))
-      ?.map(col => ({
-        ...col,
-        isChecked: this.isColumnChecked(col),
-      }));
+    if (structure.columns) {
+      this.columns = structure.columns
+        .map(col => ({
+          columnName: col.columnName,
+          headerText: col.columnName,
+          uniqueName: col.columnName,
+          uniquePath: [col.columnName],
+          notNull: col.notNull,
+          autoIncrement: col.autoIncrement,
+          pureName: null,
+          schemaName: null,
+        }))
+        ?.map(col => ({
+          ...col,
+          isChecked: this.isColumnChecked(col),
+        }));
+    }
+
+    if (structure.__isDynamicStructure) {
+      this.columns = analyseCollectionDisplayColumns(rows, this);
+    }
+
+    if (!this.columns) this.columns = [];
  }
 }
@@ -19,4 +19,6 @@ export interface MacroDefinition {
 export interface MacroSelectedCell {
  column: string;
  row: number;
+  rowData: any;
+  value: any;
 }
@@ -18,7 +18,7 @@ import stableStringify from 'json-stable-stringify';
 import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet';

 export class TableFormViewDisplay extends FormViewDisplay {
-  // use utility functions from GridDisplay and publish result in FromViewDisplat interface
+  // use utility functions from GridDisplay and publish result in FromViewDisplay interface
  private gridDisplay: TableGridDisplay;

  constructor(
@@ -28,10 +28,22 @@ export class TableFormViewDisplay extends FormViewDisplay {
    setConfig: ChangeConfigFunc,
    cache: GridCache,
    setCache: ChangeCacheFunc,
-    dbinfo: DatabaseInfo
+    dbinfo: DatabaseInfo,
+    displayOptions,
+    serverVersion
  ) {
-    super(config, setConfig, cache, setCache, driver, dbinfo);
-    this.gridDisplay = new TableGridDisplay(tableName, driver, config, setConfig, cache, setCache, dbinfo);
+    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);
+    this.gridDisplay = new TableGridDisplay(
+      tableName,
+      driver,
+      config,
+      setConfig,
+      cache,
+      setCache,
+      dbinfo,
+      displayOptions,
+      serverVersion
+    );
    this.gridDisplay.addAllExpandedColumnsToSelected = true;

    this.isLoadedCorrectly = this.gridDisplay.isLoadedCorrectly && !!this.driver;
@@ -17,9 +17,11 @@ export class TableGridDisplay extends GridDisplay {
    setConfig: ChangeConfigFunc,
    cache: GridCache,
    setCache: ChangeCacheFunc,
-    dbinfo: DatabaseInfo
+    dbinfo: DatabaseInfo,
+    public displayOptions: any,
+    serverVersion
  ) {
-    super(config, setConfig, cache, setCache, driver, dbinfo);
+    super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion);

    this.table = this.findTable(tableName);
    if (!this.table) {
@@ -59,6 +61,7 @@ export class TableGridDisplay extends GridDisplay {
          ...col,
          isChecked: this.isColumnChecked(col),
          hintColumnName: col.foreignKey ? `hint_${col.uniqueName}` : null,
+          isExpandable: !!col.foreignKey,
        })) || []
    );
  }
@@ -167,7 +170,7 @@ export class TableGridDisplay extends GridDisplay {

  processReferences(select: Select, displayedColumnInfo: DisplayedColumnInfo, options) {
    this.addJoinsFromExpandedColumns(select, this.columns, 'basetbl', displayedColumnInfo);
-    if (!options.isExport) {
+    if (!options.isExport && this.displayOptions.showHintColumns) {
      this.addHintsToSelect(select);
    }
  }
@@ -10,9 +10,10 @@ export class ViewGridDisplay extends GridDisplay {
    config: GridConfig,
    setConfig: ChangeConfigFunc,
    cache: GridCache,
-    setCache: ChangeCacheFunc
+    setCache: ChangeCacheFunc,
+    serverVersion
  ) {
-    super(config, setConfig, cache, setCache, driver);
+    super(config, setConfig, cache, setCache, driver, serverVersion);
    this.columns = this.getDisplayColumns(view);
    this.filterable = true;
    this.sortable = true;
@@ -11,3 +11,4 @@ export * from './MacroDefinition';
 export * from './runMacro';
 export * from './FormViewDisplay';
 export * from './TableFormViewDisplay';
+export * from './CollectionGridDisplay';
@@ -4,6 +4,8 @@ import uuidv1 from 'uuid/v1';
 import uuidv4 from 'uuid/v4';
 import moment from 'moment';
 import { MacroDefinition, MacroSelectedCell } from './MacroDefinition';
+import { ChangeSet, setChangeSetValue } from './ChangeSet';
+import { GridDisplay } from './GridDisplay';

 const getMacroFunction = {
  transformValue: code => `
@@ -183,3 +185,55 @@ export function runMacro(
  }
  return data;
 }
+
+export function compileMacroFunction(macro: MacroDefinition, errors = []) {
+  if (!macro) return null;
+  let func;
+  try {
+    func = eval(getMacroFunction[macro.type](macro.code));
+    return func;
+  } catch (err) {
+    errors.push(`Error compiling macro ${macro.name}: ${err.message}`);
+    return null;
+  }
+}
+
+export function runMacroOnValue(compiledFunc, macroArgs, value, rowIndex, row, column, errors = []) {
+  if (!compiledFunc) return value;
+  try {
+    const res = compiledFunc(value, macroArgs, modules, rowIndex, row, column);
+    return res;
+  } catch (err) {
+    errors.push(`Error processing column ${column} on row ${rowIndex}: ${err.message}`);
+    return value;
+  }
+}
+
+export function runMacroOnChangeSet(
+  macro: MacroDefinition,
+  macroArgs: {},
+  selectedCells: MacroSelectedCell[],
+  changeSet: ChangeSet,
+  display: GridDisplay
+): ChangeSet {
+  const errors = [];
+  const compiledMacroFunc = compileMacroFunction(macro, errors);
+  if (!compiledMacroFunc) return null;
+
+  let res = changeSet;
+  for (const cell of selectedCells) {
+    const definition = display.getChangeSetField(cell.rowData, cell.column, undefined);
+    const macroResult = runMacroOnValue(
+      compiledMacroFunc,
+      macroArgs,
+      cell.value,
+      cell.row,
+      cell.rowData,
+      cell.column,
+      errors
+    );
+    res = setChangeSetValue(res, definition, macroResult);
+  }
+
+  return res;
+}
@@ -0,0 +1,27 @@
+[![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
+[![NPM version](https://img.shields.io/npm/v/dbgate.svg)](https://www.npmjs.com/package/dbgate)
+
+# DbGate - database administration tool
+DbGate is fast and easy to use database administration tool for MySQL, PostgreSQL, SQL Server.
+
+## Install using npm
+```sh
+npm install -g dbgate
+```
+
+After installing, you can run dbgate with command:
+```sh
+dbgate
+```
+
+Then open http://localhost:5000 in your browser
+
+## Download electron app
+You can also download binary packages from https://dbgate.org . Or run from source code, as described on [github](https://github.com/dbgate/dbgate)
+
+## Other dbgate packages
+You can use some functionality of dbgate from your JavaScript code. See [dbgate-api](https://npmjs.com/dbgate-api) package.
+
+## Screenshot
+
+![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png)
@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+
+const path = require('path');
+
+global.dbgateApiModulePath = path.dirname(path.dirname(require.resolve('dbgate-api')));
+global.dbgateApiPackagedPluginsPath = path.dirname(global.dbgateApiModulePath);
+
+const dbgateApi = require('dbgate-api');
+
+dbgateApi.getMainModule().start();
@@ -0,0 +1,30 @@
+{
+  "name": "dbgate",
+  "version": "4.1.1",
+  "homepage": "https://dbgate.org/",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate.git"
+  },
+  "description": "Opensource database administration tool - web interface",
+  "author": "Jan Prochazka",
+  "license": "MIT",
+  "bin": {
+    "dbgate": "./bin/dbgate.js"
+  },
+  "keywords": [
+    "sql",
+    "dbgate",
+    "web"
+  ],
+  "dependencies": {
+    "dbgate-api": "^4.1.1",
+    "dbgate-web": "^4.1.1",
+    "dbgate-plugin-csv": "^4.1.1",
+    "dbgate-plugin-excel": "^4.1.1",
+    "dbgate-plugin-mongo": "^4.1.1",
+    "dbgate-plugin-mysql": "^4.1.1",
+    "dbgate-plugin-mssql": "^4.1.1",
+    "dbgate-plugin-postgres": "^4.1.1"
+  }
+}
@@ -1,10 +1,9 @@
 {
-  "version": "1.0.0",
+  "version": "4.1.1",
  "name": "dbgate-filterparser",
  "main": "lib/index.js",
  "typings": "lib/index.d.ts",
  "scripts": {
-    "prepare": "yarn build",
    "build": "tsc",
    "start": "tsc --watch",
    "test": "jest"
@@ -13,7 +12,7 @@
    "lib"
  ],
  "devDependencies": {
-    "dbgate-types": "^1.0.0",
+    "dbgate-types": "^4.1.1",
    "@types/jest": "^25.1.4",
    "@types/node": "^13.7.0",
    "jest": "^24.9.0",
@@ -22,9 +21,9 @@
  },
  "dependencies": {
    "@types/parsimmon": "^1.10.1",
-    "dbgate-tools": "^1.0.0",
+    "dbgate-tools": "^4.1.1",
    "lodash": "^4.17.15",
    "moment": "^2.24.0",
    "parsimmon": "^1.13.0"
  }
-}
+}
@@ -0,0 +1,32 @@
+import P from 'parsimmon';
+
+export const whitespace = P.regexp(/\s*/m);
+
+export function token(parser) {
+  return parser.skip(whitespace);
+}
+
+export function word(str) {
+  return P.string(str).thru(token);
+}
+
+export function interpretEscapes(str) {
+  let escapes = {
+    b: '\b',
+    f: '\f',
+    n: '\n',
+    r: '\r',
+    t: '\t',
+  };
+  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
+    let type = escape.charAt(0);
+    let hex = escape.slice(1);
+    if (type === 'u') {
+      return String.fromCharCode(parseInt(hex, 16));
+    }
+    if (escapes.hasOwnProperty(type)) {
+      return escapes[type];
+    }
+    return type;
+  });
+}
@@ -0,0 +1,121 @@
+import P from 'parsimmon';
+import { interpretEscapes, token, word, whitespace } from './common';
+
+const operatorCondition = operator => value => ({
+  __placeholder__: {
+    [operator]: value,
+  },
+});
+
+const regexCondition = regexString => value => ({
+  __placeholder__: {
+    $regex: regexString.replace('#VALUE#', value),
+    $options: 'i',
+  },
+});
+
+const numberTestCondition = () => value => ({
+  $or: [
+    {
+      __placeholder__: {
+        $regex: `.*${value}.*`,
+        $options: 'i',
+      },
+    },
+    {
+      __placeholder__: value,
+    },
+  ],
+});
+
+const testCondition = (operator, value) => () => ({
+  __placeholder__: {
+    [operator]: value,
+  },
+});
+
+const compoudCondition = conditionType => conditions => {
+  if (conditions.length == 1) return conditions[0];
+  return {
+    [conditionType]: conditions,
+  };
+};
+
+const negateCondition = condition => ({
+  __placeholder__: {
+    $not: condition.__placeholder__,
+  },
+});
+
+const createParser = () => {
+  const langDef = {
+    string1: () =>
+      token(P.regexp(/"((?:\\.|.)*?)"/, 1))
+        .map(interpretEscapes)
+        .desc('string quoted'),
+
+    string2: () =>
+      token(P.regexp(/'((?:\\.|.)*?)'/, 1))
+        .map(interpretEscapes)
+        .desc('string quoted'),
+
+    number: () =>
+      token(P.regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/))
+        .map(Number)
+        .desc('number'),
+
+    noQuotedString: () => P.regexp(/[^\s^,^'^"]+/).desc('string unquoted'),
+
+    value: r => P.alt(r.string1, r.string2, r.number, r.noQuotedString),
+    valueTestNum: r => r.number.map(numberTestCondition()),
+    valueTest: r => r.value.map(regexCondition('.*#VALUE#.*')),
+
+    comma: () => word(','),
+    not: () => word('NOT'),
+    notExists: r => r.not.then(r.exists).map(testCondition('$exists', false)),
+    exists: () => word('EXISTS').map(testCondition('$exists', true)),
+    true: () => word('TRUE').map(testCondition('$eq', true)),
+    false: () => word('FALSE').map(testCondition('$eq', false)),
+
+    eq: r => word('=').then(r.value).map(operatorCondition('$eq')),
+    ne: r => word('!=').then(r.value).map(operatorCondition('$ne')),
+    lt: r => word('<').then(r.value).map(operatorCondition('$lt')),
+    gt: r => word('>').then(r.value).map(operatorCondition('$gt')),
+    le: r => word('<=').then(r.value).map(operatorCondition('$lte')),
+    ge: r => word('>=').then(r.value).map(operatorCondition('$gte')),
+    startsWith: r => word('^').then(r.value).map(regexCondition('#VALUE#.*')),
+    endsWith: r => word('$').then(r.value).map(regexCondition('.*#VALUE#')),
+    contains: r => word('+').then(r.value).map(regexCondition('.*#VALUE#.*')),
+    startsWithNot: r => word('!^').then(r.value).map(regexCondition('#VALUE#.*')).map(negateCondition),
+    endsWithNot: r => word('!$').then(r.value).map(regexCondition('.*#VALUE#')).map(negateCondition),
+    containsNot: r => word('~').then(r.value).map(regexCondition('.*#VALUE#.*')).map(negateCondition),
+
+    element: r =>
+      P.alt(
+        r.exists,
+        r.notExists,
+        r.true,
+        r.false,
+        r.eq,
+        r.ne,
+        r.lt,
+        r.gt,
+        r.le,
+        r.ge,
+        r.startsWith,
+        r.endsWith,
+        r.contains,
+        r.startsWithNot,
+        r.endsWithNot,
+        r.containsNot,
+        r.valueTestNum,
+        r.valueTest
+      ).trim(whitespace),
+    factor: r => r.element.sepBy(whitespace).map(compoudCondition('$and')),
+    list: r => r.factor.sepBy(r.comma).map(compoudCondition('$or')),
+  };
+
+  return P.createLanguage(langDef);
+};
+
+export const mongoParser = createParser();
@@ -3,37 +3,8 @@ import moment from 'moment';
 import { FilterType } from './types';
 import { Condition } from 'dbgate-sqltree';
 import { TransformType } from 'dbgate-types';
-
-const whitespace = P.regexp(/\s*/m);
-
-function token(parser) {
-  return parser.skip(whitespace);
-}
-
-function word(str) {
-  return P.string(str).thru(token);
-}
-
-function interpretEscapes(str) {
-  let escapes = {
-    b: '\b',
-    f: '\f',
-    n: '\n',
-    r: '\r',
-    t: '\t',
-  };
-  return str.replace(/\\(u[0-9a-fA-F]{4}|[^u])/, (_, escape) => {
-    let type = escape.charAt(0);
-    let hex = escape.slice(1);
-    if (type === 'u') {
-      return String.fromCharCode(parseInt(hex, 16));
-    }
-    if (escapes.hasOwnProperty(type)) {
-      return escapes[type];
-    }
-    return type;
-  });
-}
+import { interpretEscapes, token, word, whitespace } from './common';
+import { mongoParser } from './mongoParser';

 const binaryCondition = operator => value => ({
  conditionType: 'binary',
@@ -239,7 +210,8 @@ const createParser = (filterType: FilterType) => {
    yearMonthNum: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthCondition()),
    yearMonthDayNum: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayCondition()),
    yearMonthDayMinute: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?\s+\d\d?:\d\d?/).map(yearMonthDayMinuteCondition()),
-    yearMonthDaySecond: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDaySecondCondition()),
+    yearMonthDaySecond: () =>
+      P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDaySecondCondition()),

    value: r => P.alt(...allowedValues.map(x => r[x])),
    valueTestEq: r => r.value.map(binaryCondition('=')),
@@ -348,9 +320,11 @@ const parsers = {
  string: createParser('string'),
  datetime: createParser('datetime'),
  logical: createParser('logical'),
+  mongo: mongoParser,
 };

 export function parseFilter(value: string, filterType: FilterType): Condition {
+  // console.log('PARSING', value, 'WITH', filterType);
  const ast = parsers[filterType].list.tryParse(value);
  // console.log('AST', ast);
  return ast;
@@ -1,3 +1,3 @@
 // import types from 'dbgate-types';

-export type FilterType = 'number' | 'string' | 'datetime' | 'logical';
+export type FilterType = 'number' | 'string' | 'datetime' | 'logical' | 'mongo';
@@ -37,7 +37,7 @@ console.log("Generated query:", sql);

 ```

-See [TypeScript definitions](https://github.com/dbshell/dbgate/blob/master/packages/sqltree/src/types.ts) for complete list of available SQL command options.
+See [TypeScript definitions](https://github.com/dbgate/dbgate/blob/master/packages/sqltree/src/types.ts) for complete list of available SQL command options.

 ## Installation

@@ -1,16 +1,15 @@
 {
-  "version": "1.0.4",
+  "version": "4.1.1",
  "name": "dbgate-sqltree",
  "main": "lib/index.js",
  "typings": "lib/index.d.ts",
  "homepage": "https://dbgate.org/",
  "repository": {
    "type": "git",
-    "url": "https://github.com/dbshell/dbgate.git"
+    "url": "https://github.com/dbgate/dbgate.git"
  },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
  "author": "Jan Prochazka",
-  "license": "GPL",
+  "license": "MIT",
  "keywords": [
    "sql",
    "mssql",
@@ -20,7 +19,6 @@
    "dbgate"
  ],
  "scripts": {
-    "prepare": "yarn build",
    "build": "tsc",
    "start": "tsc --watch"
  },
@@ -29,10 +27,10 @@
  ],
  "devDependencies": {
    "@types/node": "^13.7.0",
-    "dbgate-types": "^1.0.0",
+    "dbgate-types": "^4.1.1",
    "typescript": "^3.7.5"
  },
  "dependencies": {
    "lodash": "^4.17.15"
  }
-}
+}
@@ -62,5 +62,12 @@ export function dumpSqlCondition(dmp: SqlDumper, condition: Condition) {
      dumpSqlSelect(dmp, condition.subQuery);
      dmp.put(')');
      break;
+    case 'between':
+      dumpSqlExpression(dmp, condition.expr);
+      dmp.put(' ^between ');
+      dumpSqlExpression(dmp, condition.left);
+      dmp.put(' ^and ');
+      dumpSqlExpression(dmp, condition.right);
+      break;
  }
 }
@@ -38,5 +38,14 @@ export function dumpSqlExpression(dmp: SqlDumper, expr: Expression) {
    case 'transform':
      dmp.transform(expr.transform, () => dumpSqlExpression(dmp, expr.expr));
      break;
+
+    case 'rowNumber':
+      dmp.put(" ^row_number() ^over (^order ^by ");
+      dmp.putCollection(', ', expr.orderBy, x => {
+        dumpSqlExpression(dmp, x);
+        dmp.put(' %k', x.direction);
+      });
+      dmp.put(")");
+      break;
  }
 }
@@ -92,6 +92,13 @@ export interface NotExistsCondition {
  subQuery: Select;
 }

+export interface BetweenCondition {
+  conditionType: 'between';
+  expr: Expression;
+  left: Expression;
+  right: Expression;
+}
+
 export type Condition =
  | BinaryCondition
  | NotCondition
@@ -99,7 +106,8 @@ export type Condition =
  | CompoudCondition
  | LikeCondition
  | ExistsCondition
-  | NotExistsCondition;
+  | NotExistsCondition
+  | BetweenCondition;

 export interface Source {
  name?: NamedObjectInfo;
@@ -153,13 +161,19 @@ export interface TranformExpression {
  transform: TransformType;
 }

+export interface RowNumberExpression {
+  exprType: 'rowNumber';
+  orderBy: OrderByExpression[];
+}
+
 export type Expression =
  | ColumnRefExpression
  | ValueExpression
  | PlaceholderExpression
  | RawExpression
  | CallExpression
-  | TranformExpression;
+  | TranformExpression
+  | RowNumberExpression;
 export type OrderByExpression = Expression & { direction: 'ASC' | 'DESC' };

 export type ResultField = Expression & { alias?: string };
@@ -1,22 +1,20 @@
 {
-  "version": "1.0.7",
+  "version": "4.1.1",
  "name": "dbgate-tools",
  "main": "lib/index.js",
  "typings": "lib/index.d.ts",
  "homepage": "https://dbgate.org/",
  "repository": {
    "type": "git",
-    "url": "https://github.com/dbshell/dbgate.git"
+    "url": "https://github.com/dbgate/dbgate.git"
  },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
  "author": "Jan Prochazka",
-  "license": "GPL",
+  "license": "MIT",
  "keywords": [
    "sql",
    "dbgate"
  ],
  "scripts": {
-    "prepare": "yarn build",
    "build": "tsc",
    "start": "tsc --watch",
    "test": "jest",
@@ -27,7 +25,7 @@
  ],
  "devDependencies": {
    "@types/node": "^13.7.0",
-    "dbgate-types": "^1.0.0",
+    "dbgate-types": "^4.1.1",
    "jest": "^24.9.0",
    "ts-jest": "^25.2.1",
    "typescript": "^3.7.5"
@@ -35,4 +33,4 @@
  "dependencies": {
    "lodash": "^4.17.15"
  }
-}
+}
@@ -40,7 +40,7 @@ export class DatabaseAnalyser {
    return this._runAnalysis();
  }

-  mergeAnalyseResult(newlyAnalysed, extractObjectId) {
+  mergeAnalyseResult(newlyAnalysed) {
    if (this.structure == null) {
      return {
        ...DatabaseAnalyser.createEmptyStructure(),
@@ -49,15 +49,15 @@ export class DatabaseAnalyser {
    }

    const res = {};
-    for (const field of ['tables', 'views', 'functions', 'procedures', 'triggers']) {
+    for (const field of ['tables', 'collections', 'views', 'functions', 'procedures', 'triggers']) {
      const removedIds = this.modifications
        .filter(x => x.action == 'remove' && x.objectTypeField == field)
-        .map(x => extractObjectId(x));
+        .map(x => x.objectId);
      const newArray = newlyAnalysed[field] || [];
-      const addedChangedIds = newArray.map(x => extractObjectId(x));
+      const addedChangedIds = newArray.map(x => x.objectId);
      const removeAllIds = [...removedIds, ...addedChangedIds];
      res[field] = _sortBy(
-        [...this.structure[field].filter(x => !removeAllIds.includes(extractObjectId(x))), ...newArray],
+        [...this.structure[field].filter(x => !removeAllIds.includes(x.objectId)), ...newArray],
        x => x.pureName
      );
    }
@@ -78,6 +78,7 @@ export class DatabaseAnalyser {
  static createEmptyStructure(): DatabaseInfo {
    return {
      tables: [],
+      collections: [],
      views: [],
      functions: [],
      procedures: [],
@@ -1,11 +1,20 @@
 import {
  ColumnInfo,
+  ConstraintInfo,
  EngineDriver,
  ForeignKeyInfo,
+  FunctionInfo,
  NamedObjectInfo,
+  PrimaryKeyInfo,
+  ProcedureInfo,
  SqlDialect,
  TableInfo,
  TransformType,
+  TriggerInfo,
+  ViewInfo,
+  IndexInfo,
+  UniqueInfo,
+  CheckInfo,
 } from 'dbgate-types';
 import _isString from 'lodash/isString';
 import _isNumber from 'lodash/isNumber';
@@ -46,11 +55,12 @@ export class SqlDumper {
  }
  putValue(value) {
    if (value === null) this.putRaw('NULL');
-    if (value === true) this.putRaw('1');
-    if (value === false) this.putRaw('0');
+    else if (value === true) this.putRaw('1');
+    else if (value === false) this.putRaw('0');
    else if (_isString(value)) this.putStringValue(value);
    else if (_isNumber(value)) this.putRaw(value.toString());
    else if (_isDate(value)) this.putStringValue(new Date(value).toISOString());
+    else this.putRaw('NULL');
  }
  putCmd(format, ...args) {
    this.put(format, ...args);
@@ -260,4 +270,189 @@ export class SqlDumper {
  }

  allowIdentityInsert(table: NamedObjectInfo, allow: boolean) {}
+  enableConstraints(table: NamedObjectInfo, enabled: boolean) {}
+
+  comment(value: string) {
+    if (!value) return;
+    for (const line of value.split('\n')) {
+      this.put(' -- %s', line.trimRight());
+    }
+  }
+
+  createView(obj: ViewInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropView(obj: ViewInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^view  %f', obj);
+  }
+  alterView(obj: ViewInfo) {
+    this.putRaw(obj.createSql.replace(/create\s+view/i, 'ALTER VIEW'));
+    this.endCommand();
+  }
+  changeViewSchema(obj: ViewInfo, newSchema: string) {}
+  renameView(obj: ViewInfo, newSchema: string) {}
+
+  createProcedure(obj: ProcedureInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropProcedure(obj: ProcedureInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^procedure  %f', obj);
+  }
+  alterProcedure(obj: ProcedureInfo) {
+    this.putRaw(obj.createSql.replace(/create\s+procedure/i, 'ALTER PROCEDURE'));
+    this.endCommand();
+  }
+  changeProcedureSchema(obj: ProcedureInfo, newSchema: string) {}
+  renameProcedure(obj: ProcedureInfo, newSchema: string) {}
+
+  createFunction(obj: FunctionInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropFunction(obj: FunctionInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^function  %f', obj);
+  }
+  alterFunction(obj: FunctionInfo) {
+    this.putRaw(obj.createSql.replace(/create\s+function/i, 'ALTER FUNCTION'));
+    this.endCommand();
+  }
+  changeFunctionSchema(obj: FunctionInfo, newSchema: string) {}
+  renameFunction(obj: FunctionInfo, newSchema: string) {}
+
+  createTrigger(obj: TriggerInfo) {
+    this.putRaw(obj.createSql);
+    this.endCommand();
+  }
+  dropTrigger(obj: TriggerInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^trigger  %f', obj);
+  }
+  alterTrigger(obj: TriggerInfo) {
+    this.putRaw(obj.createSql.replace(/create\s+trigger/i, 'ALTER TRIGGER'));
+    this.endCommand();
+  }
+  changeTriggerSchema(obj: TriggerInfo, newSchema: string) {}
+  renameTrigger(obj: TriggerInfo, newSchema: string) {}
+
+  dropConstraint(cnt: ConstraintInfo) {
+    this.putCmd('^alter ^table %f ^drop ^constraint %i', cnt, cnt.constraintName);
+  }
+  dropForeignKey(fk: ForeignKeyInfo) {
+    if (this.dialect.explicitDropConstraint) {
+      this.putCmd('^alter ^table %f ^drop ^foreign ^key %i', fk, fk.constraintName);
+    } else {
+      this.dropConstraint(fk);
+    }
+  }
+  createForeignKey(fk: ForeignKeyInfo) {
+    this.put('^alter ^table %f ^add ', fk);
+    this.createForeignKeyFore(fk);
+    this.endCommand();
+  }
+  dropPrimaryKey(pk: PrimaryKeyInfo) {
+    if (this.dialect.explicitDropConstraint) {
+      this.putCmd('^alter ^table %f ^drop ^primary ^key', pk);
+    } else {
+      this.dropConstraint(pk);
+    }
+  }
+  createPrimaryKey(pk: PrimaryKeyInfo) {
+    this.putCmd(
+      '^alter ^table %f ^add ^constraint %i ^primary ^key (%,i)',
+      pk,
+      pk.constraintName,
+      pk.columns.map(x => x.columnName)
+    );
+  }
+
+  dropIndex(ix: IndexInfo) {}
+  createIndex(ix: IndexInfo) {}
+
+  dropUnique(uq: UniqueInfo) {
+    this.dropConstraint(uq);
+  }
+  createUniqueCore(uq: UniqueInfo) {
+    this.put(
+      '^constraint %i ^unique (%,i)',
+      uq.constraintName,
+      uq.columns.map(x => x.columnName)
+    );
+  }
+
+  createUnique(uq: UniqueInfo) {
+    this.put('^alter ^table %f ^add ', uq);
+    this.createUniqueCore(uq);
+    this.endCommand();
+  }
+
+  dropCheck(ch: CheckInfo) {
+    this.dropConstraint(ch);
+  }
+
+  createCheckCore(ch: CheckInfo) {
+    this.put('^constraint %i ^check (%s)', ch.constraintName, ch.definition);
+  }
+
+  createCheck(ch: CheckInfo) {
+    this.put('^alter ^table %f ^add ', ch);
+    this.createCheckCore(ch);
+    this.endCommand();
+  }
+
+  renameConstraint(constraint: ConstraintInfo, newName: string) {}
+
+  createColumn(table: TableInfo, column: ColumnInfo, constraints: ConstraintInfo[]) {
+    this.put('^alter ^table %f ^add %i ', table, column.columnName);
+    this.columnDefinition(column);
+    this.inlineConstraints(constraints);
+    this.endCommand();
+  }
+
+  inlineConstraints(constrains: ConstraintInfo[]) {
+    if (constrains == null) return;
+    for (const cnt of constrains) {
+      if (cnt.constraintType == 'primaryKey') {
+        if (cnt.constraintName != null && !this.dialect.anonymousPrimaryKey) {
+          this.put(' ^constraint %i', cnt.constraintName);
+        }
+        this.put(' ^primary ^key ');
+      }
+    }
+  }
+
+  dropColumn(column: ColumnInfo) {
+    this.putCmd('^alter ^table %f ^drop ^column %i', column, column.columnName);
+  }
+
+  renameColumn(column: ColumnInfo, newName: string) {}
+
+  changeColumn(oldcol: ColumnInfo, newcol: ColumnInfo, constraints: ConstraintInfo[]) {}
+
+  dropTable(obj: TableInfo, { testIfExists = false }) {
+    this.putCmd('^drop ^table %f', obj);
+  }
+
+  changeTableSchema(obj: TableInfo, schema: string) {}
+
+  renameTable(obj: TableInfo, newname: string) {}
+
+  beginTransaction() {
+    this.putCmd('^begin ^transaction');
+  }
+
+  commitTransaction() {
+    this.putCmd('^commit');
+  }
+
+  alterProlog() {}
+  alterEpilog() {}
+
+  selectTableIntoNewTable(sourceName: NamedObjectInfo, targetName: NamedObjectInfo) {
+    this.putCmd('^select * ^into %f ^from %f', targetName, sourceName);
+  }
+
+  truncateTable(name: NamedObjectInfo) {
+    this.putCmd('^delete ^from %f', name);
+  }
 }
@@ -0,0 +1,292 @@
+import {
+  DatabaseInfo,
+  EngineDriver,
+  FunctionInfo,
+  ProcedureInfo,
+  TableInfo,
+  TriggerInfo,
+  ViewInfo,
+} from 'dbgate-types';
+import _ from 'lodash';
+import { SqlDumper } from './SqlDumper';
+import { extendDatabaseInfo } from './structureTools';
+
+interface SqlGeneratorOptions {
+  dropTables: boolean;
+  checkIfTableExists: boolean;
+  dropReferences: boolean;
+  createTables: boolean;
+  createReferences: boolean;
+  createForeignKeys: boolean;
+  createIndexes: boolean;
+  insert: boolean;
+  skipAutoincrementColumn: boolean;
+  disableConstraints: boolean;
+  omitNulls: boolean;
+  truncate: boolean;
+
+  dropViews: boolean;
+  checkIfViewExists: boolean;
+  createViews: boolean;
+
+  dropProcedures: boolean;
+  checkIfProcedureExists: boolean;
+  createProcedures: boolean;
+
+  dropFunctions: boolean;
+  checkIfFunctionExists: boolean;
+  createFunctions: boolean;
+
+  dropTriggers: boolean;
+  checkIfTriggerExists: boolean;
+  createTriggers: boolean;
+}
+
+interface SqlGeneratorObject {
+  schemaName: string;
+  pureName: string;
+  objectTypeField: 'tables' | 'views' | 'procedures' | 'functions';
+}
+
+export class SqlGenerator {
+  private tables: TableInfo[];
+  private views: ViewInfo[];
+  private procedures: ProcedureInfo[];
+  private functions: FunctionInfo[];
+  private triggers: TriggerInfo[];
+  public dbinfo: DatabaseInfo;
+  public isTruncated = false;
+  public isUnhandledException = false;
+
+  constructor(
+    dbinfo: DatabaseInfo,
+    public options: SqlGeneratorOptions,
+    public objects: SqlGeneratorObject[],
+    public dmp: SqlDumper,
+    public driver: EngineDriver,
+    public pool
+  ) {
+    this.dbinfo = extendDatabaseInfo(dbinfo);
+    this.tables = this.extract('tables');
+    this.views = this.extract('views');
+    this.procedures = this.extract('procedures');
+    this.functions = this.extract('functions');
+    this.triggers = this.extract('triggers');
+  }
+
+  private handleException = error => {
+    console.log('Unhandled error', error);
+    this.isUnhandledException = true;
+  };
+
+  async dump() {
+    try {
+      process.on('uncaughtException', this.handleException);
+
+      this.dropObjects(this.procedures, 'Procedure');
+      if (this.checkDumper()) return;
+      this.dropObjects(this.functions, 'Function');
+      if (this.checkDumper()) return;
+      this.dropObjects(this.views, 'View');
+      if (this.checkDumper()) return;
+      this.dropObjects(this.triggers, 'Trigger');
+      if (this.checkDumper()) return;
+
+      this.dropTables();
+      if (this.checkDumper()) return;
+
+      this.createTables();
+      if (this.checkDumper()) return;
+
+      this.truncateTables();
+      if (this.checkDumper()) return;
+
+      await this.insertData();
+      if (this.checkDumper()) return;
+
+      this.createForeignKeys();
+      if (this.checkDumper()) return;
+
+      this.createObjects(this.procedures, 'Procedure');
+      if (this.checkDumper()) return;
+      this.createObjects(this.functions, 'Function');
+      if (this.checkDumper()) return;
+      this.createObjects(this.views, 'View');
+      if (this.checkDumper()) return;
+      this.createObjects(this.triggers, 'Trigger');
+      if (this.checkDumper()) return;
+    } finally {
+      process.off('uncaughtException', this.handleException);
+    }
+  }
+
+  createForeignKeys() {
+    const fks = [];
+    if (this.options.createForeignKeys) fks.push(..._.flatten(this.tables.map(x => x.foreignKeys || [])));
+    if (this.options.createReferences) fks.push(..._.flatten(this.tables.map(x => x.dependencies || [])));
+    for (const fk of _.uniqBy(fks, 'constraintName')) {
+      this.dmp.createForeignKey(fk);
+      if (this.checkDumper()) return;
+    }
+  }
+
+  truncateTables() {
+    if (this.options.truncate) {
+      for (const table of this.tables) {
+        this.dmp.truncateTable(table);
+        if (this.checkDumper()) return;
+      }
+    }
+  }
+
+  createTables() {
+    if (this.options.createTables) {
+      for (const table of this.tables) {
+        this.dmp.createTable({
+          ...table,
+          foreignKeys: [],
+          dependencies: [],
+          indexes: [],
+        });
+        if (this.checkDumper()) return;
+      }
+    }
+    if (this.options.createIndexes) {
+      for (const index of _.flatten(this.tables.map(x => x.indexes || []))) {
+        this.dmp.createIndex(index);
+      }
+    }
+  }
+
+  async insertData() {
+    if (!this.options.insert) return;
+
+    this.enableConstraints(false);
+
+    for (const table of this.tables) {
+      await this.insertTableData(table);
+      if (this.checkDumper()) return;
+    }
+
+    this.enableConstraints(true);
+  }
+
+  checkDumper() {
+    if (this.dmp.s.length > 4000000) {
+      if (!this.isTruncated) {
+        this.dmp.putRaw('\n');
+        this.dmp.comment(' *************** SQL is truncated ******************');
+        this.dmp.putRaw('\n');
+      }
+      this.isTruncated = true;
+      return true;
+    }
+    return false;
+  }
+
+  dropObjects(list, name) {
+    if (this.options[`drop${name}s`]) {
+      for (const item of list) {
+        this.dmp[`drop${name}`](item, { testIfExists: this.options[`checkIf${name}Exists`] });
+        if (this.checkDumper()) return;
+      }
+    }
+  }
+
+  createObjects(list, name) {
+    if (this.options[`create${name}s`]) {
+      for (const item of list) {
+        this.dmp[`create${name}`](item);
+        if (this.checkDumper()) return;
+      }
+    }
+  }
+
+  dropTables() {
+    if (this.options.dropReferences) {
+      for (const fk of _.flatten(this.tables.map(x => x.dependencies || []))) {
+        this.dmp.dropForeignKey(fk);
+      }
+    }
+
+    if (this.options.dropTables) {
+      for (const table of this.tables) {
+        this.dmp.dropTable(table, { testIfExists: this.options.checkIfTableExists });
+      }
+    }
+  }
+
+  async insertTableData(table: TableInfo) {
+    const dmpLocal = this.driver.createDumper();
+    dmpLocal.put('^select * ^from %f', table);
+
+    const autoinc = table.columns.find(x => x.autoIncrement);
+    if (autoinc && !this.options.skipAutoincrementColumn) {
+      this.dmp.allowIdentityInsert(table, true);
+    }
+
+    const readable = await this.driver.readQuery(this.pool, dmpLocal.s, table);
+    await this.processReadable(table, readable);
+
+    if (autoinc && !this.options.skipAutoincrementColumn) {
+      this.dmp.allowIdentityInsert(table, false);
+    }
+  }
+
+  processReadable(table: TableInfo, readable) {
+    const columnsFiltered = this.options.skipAutoincrementColumn
+      ? table.columns.filter(x => !x.autoIncrement)
+      : table.columns;
+    const columnNames = columnsFiltered.map(x => x.columnName);
+    let isClosed = false;
+    let isHeaderRead = false;
+
+    return new Promise(resolve => {
+      readable.on('data', chunk => {
+        if (isClosed) return;
+        if (!isHeaderRead) {
+          isHeaderRead = true;
+          return;
+        }
+
+        if (this.checkDumper()) {
+          isClosed = true;
+          resolve(undefined);
+          readable.destroy();
+          return;
+        }
+
+        const columnNamesCopy = this.options.omitNulls ? columnNames.filter(col => chunk[col] != null) : columnNames;
+        this.dmp.put(
+          '^insert ^into %f (%,i) ^values (%,v);&n',
+          table,
+          columnNamesCopy,
+          columnNamesCopy.map(col => chunk[col])
+        );
+      });
+      readable.on('end', () => {
+        resolve(undefined);
+      });
+    });
+  }
+
+  extract(objectTypeField) {
+    return this.dbinfo[objectTypeField].filter(x =>
+      this.objects.find(
+        y => x.pureName == y.pureName && x.schemaName == y.schemaName && y.objectTypeField == objectTypeField
+      )
+    );
+  }
+
+  enableConstraints(enabled) {
+    if (this.options.disableConstraints) {
+      if (this.driver.dialect.enableConstraintsPerTable) {
+        for (const table of this.tables) {
+          this.dmp.enableConstraints(table, enabled);
+        }
+      } else {
+        this.dmp.enableConstraints(null, enabled);
+      }
+    }
+  }
+}
@@ -69,6 +69,7 @@ export function createBulkInsertStreamBase(driver, stream, pool, name, options):
    }
    dmp.putRaw(';');
    // require('fs').writeFileSync('/home/jena/test.sql', dmp.s);
+    // console.log(dmp.s);
    await driver.query(pool, dmp.s);
  };

@@ -7,4 +7,6 @@ export * from './DatabaseAnalyser';
 export * from './driverBase';
 export * from './SqlDumper';
 export * from './testPermission';
-export * from './splitPostgresQuery';
+export * from './SqlGenerator';
+export * from './structureTools';
+export * from './settingsExtractors';
@@ -38,14 +38,26 @@ export function findObjectLike(
  if (!dbinfo) return null;
  if (schemaName) {
    // @ts-ignore
-    return dbinfo[objectTypeField].find(
+    return dbinfo[objectTypeField]?.find(
      x => equalStringLike(x.pureName, pureName) && equalStringLike(x.schemaName, schemaName)
    );
  }
  // @ts-ignore
-  return dbinfo[objectTypeField].find(x => equalStringLike(x.pureName, pureName));
+  return dbinfo[objectTypeField]?.find(x => equalStringLike(x.pureName, pureName));
 }

 export function findForeignKeyForColumn(table: TableInfo, column: ColumnInfo) {
  return (table.foreignKeys || []).find(fk => fk.columns.find(col => col.columnName == column.columnName));
 }
+
+export function makeUniqueColumnNames(res: ColumnInfo[]) {
+  const usedNames = new Set();
+  for (let i = 0; i < res.length; i++) {
+    if (usedNames.has(res[i].columnName)) {
+      let suffix = 2;
+      while (usedNames.has(`${res[i].columnName}${suffix}`)) suffix++;
+      res[i].columnName = `${res[i].columnName}${suffix}`;
+    }
+    usedNames.add(res[i].columnName);
+  }
+}
@@ -0,0 +1,20 @@
+import _ from 'lodash';
+
+export function extractIntSettingsValue(settings, name, defaultValue, min = null, max = null) {
+  const parsed = parseInt(settings[name]);
+  if (_.isNaN(parsed)) {
+    return defaultValue;
+  }
+  if (_.isNumber(parsed)) {
+    if (min != null && parsed < min) return min;
+    if (max != null && parsed > max) return max;
+    return parsed;
+  }
+  return defaultValue;
+}
+
+export function extractBoolSettingsValue(settings, name, defaultValue) {
+  const res = settings[name];
+  if (res == null) return defaultValue;
+  return !!res;
+}
@@ -1,292 +0,0 @@
-const SINGLE_QUOTE = "'";
-const DOUBLE_QUOTE = '"';
-// const BACKTICK = '`';
-const DOUBLE_DASH_COMMENT_START = '--';
-const HASH_COMMENT_START = '#';
-const C_STYLE_COMMENT_START = '/*';
-const SEMICOLON = ';';
-const LINE_FEED = '\n';
-const DELIMITER_KEYWORD = 'DELIMITER';
-
-export interface SplitOptions {
-  multipleStatements?: boolean;
-  retainComments?: boolean;
-}
-
-interface SqlStatement {
-  value: string;
-  supportMulti: boolean;
-}
-
-interface SplitExecutionContext extends Required<SplitOptions> {
-  unread: string;
-  currentDelimiter: string;
-  currentStatement: SqlStatement;
-  output: SqlStatement[];
-}
-
-interface FindExpResult {
-  expIndex: number;
-  exp: string | null;
-  nextIndex: number;
-}
-
-const regexEscapeSetRegex = /[-/\\^$*+?.()|[\]{}]/g;
-const singleQuoteStringEndRegex = /(?<!\\)'/;
-const doubleQuoteStringEndRegex = /(?<!\\)"/;
-// const backtickQuoteEndRegex = /(?<!`)`(?!`)/;
-const doubleDashCommentStartRegex = /--[ \f\n\r\t\v]/;
-const cStyleCommentStartRegex = /\/\*/;
-const cStyleCommentEndRegex = /(?<!\/)\*\//;
-const newLineRegex = /(?:[\r\n]+|$)/;
-const delimiterStartRegex = /(?:^|[\n\r]+)[ \f\t\v]*DELIMITER[ \t]+/i;
-// Best effort only, unable to find a syntax specification on delimiter
-const delimiterTokenRegex = /^(?:'(.+)'|"(.+)"|`(.+)`|([^\s]+))/;
-const semicolonKeyTokenRegex = buildKeyTokenRegex(SEMICOLON);
-const quoteEndRegexDict: Record<string, RegExp> = {
-  [SINGLE_QUOTE]: singleQuoteStringEndRegex,
-  [DOUBLE_QUOTE]: doubleQuoteStringEndRegex,
-  // [BACKTICK]: backtickQuoteEndRegex,
-};
-
-function escapeRegex(value: string): string {
-  return value.replace(regexEscapeSetRegex, '\\$&');
-}
-
-function buildKeyTokenRegex(delimiter: string): RegExp {
-  return new RegExp(
-    '(?:' +
-      [
-        escapeRegex(delimiter),
-        SINGLE_QUOTE,
-        DOUBLE_QUOTE,
-        // BACKTICK,
-        doubleDashCommentStartRegex.source,
-        HASH_COMMENT_START,
-        cStyleCommentStartRegex.source,
-        delimiterStartRegex.source,
-      ].join('|') +
-      ')',
-    'i'
-  );
-}
-
-function findExp(content: string, regex: RegExp): FindExpResult {
-  const match = content.match(regex);
-  let result: FindExpResult;
-  if (match?.index !== undefined) {
-    result = {
-      expIndex: match.index,
-      exp: match[0],
-      nextIndex: match.index + match[0].length,
-    };
-  } else {
-    result = {
-      expIndex: -1,
-      exp: null,
-      nextIndex: content.length,
-    };
-  }
-  return result;
-}
-
-function findKeyToken(content: string, currentDelimiter: string): FindExpResult {
-  let regex;
-  if (currentDelimiter === SEMICOLON) {
-    regex = semicolonKeyTokenRegex;
-  } else {
-    regex = buildKeyTokenRegex(currentDelimiter);
-  }
-  return findExp(content, regex);
-}
-
-function findEndQuote(content: string, quote: string): FindExpResult {
-  if (!(quote in quoteEndRegexDict)) {
-    throw new TypeError(`Incorrect quote ${quote} supplied`);
-  }
-  return findExp(content, quoteEndRegexDict[quote]);
-}
-
-function read(
-  context: SplitExecutionContext,
-  readToIndex: number,
-  nextUnreadIndex?: number,
-  checkSemicolon?: boolean
-): void {
-  if (checkSemicolon === undefined) {
-    checkSemicolon = true;
-  }
-  const readContent = context.unread.slice(0, readToIndex);
-  if (checkSemicolon && readContent.includes(SEMICOLON)) {
-    context.currentStatement.supportMulti = false;
-  }
-  context.currentStatement.value += readContent;
-  if (nextUnreadIndex !== undefined && nextUnreadIndex > 0) {
-    context.unread = context.unread.slice(nextUnreadIndex);
-  } else {
-    context.unread = context.unread.slice(readToIndex);
-  }
-}
-
-function readTillNewLine(context: SplitExecutionContext, checkSemicolon?: boolean): void {
-  const findResult = findExp(context.unread, newLineRegex);
-  read(context, findResult.expIndex, findResult.expIndex, checkSemicolon);
-}
-
-function discard(context: SplitExecutionContext, nextUnreadIndex: number): void {
-  if (nextUnreadIndex > 0) {
-    context.unread = context.unread.slice(nextUnreadIndex);
-  }
-}
-
-function discardTillNewLine(context: SplitExecutionContext): void {
-  const findResult = findExp(context.unread, newLineRegex);
-  discard(context, findResult.expIndex);
-}
-
-function publishStatementInMultiMode(splitOutput: SqlStatement[], currentStatement: SqlStatement): void {
-  if (splitOutput.length === 0) {
-    splitOutput.push({
-      value: '',
-      supportMulti: true,
-    });
-  }
-  const lastSplitResult = splitOutput[splitOutput.length - 1];
-  if (currentStatement.supportMulti) {
-    if (lastSplitResult.supportMulti) {
-      if (lastSplitResult.value !== '' && !lastSplitResult.value.endsWith(LINE_FEED)) {
-        lastSplitResult.value += LINE_FEED;
-      }
-      lastSplitResult.value += currentStatement.value + SEMICOLON;
-    } else {
-      splitOutput.push({
-        value: currentStatement.value + SEMICOLON,
-        supportMulti: true,
-      });
-    }
-  } else {
-    splitOutput.push({
-      value: currentStatement.value,
-      supportMulti: false,
-    });
-  }
-}
-
-function publishStatement(context: SplitExecutionContext): void {
-  const trimmed = context.currentStatement.value.trim();
-  if (trimmed !== '') {
-    if (!context.multipleStatements) {
-      context.output.push({
-        value: trimmed,
-        supportMulti: context.currentStatement.supportMulti,
-      });
-    } else {
-      context.currentStatement.value = trimmed;
-      publishStatementInMultiMode(context.output, context.currentStatement);
-    }
-  }
-  context.currentStatement.value = '';
-  context.currentStatement.supportMulti = true;
-}
-
-function handleKeyTokenFindResult(context: SplitExecutionContext, findResult: FindExpResult): void {
-  switch (findResult.exp?.trim()) {
-    case context.currentDelimiter:
-      read(context, findResult.expIndex, findResult.nextIndex);
-      publishStatement(context);
-      break;
-    // case BACKTICK:
-    case SINGLE_QUOTE:
-    case DOUBLE_QUOTE: {
-      read(context, findResult.nextIndex);
-      const findQuoteResult = findEndQuote(context.unread, findResult.exp);
-      read(context, findQuoteResult.nextIndex, undefined, false);
-      break;
-    }
-    case DOUBLE_DASH_COMMENT_START: {
-      if (context.retainComments) {
-        read(context, findResult.nextIndex);
-        readTillNewLine(context, false);
-      } else {
-        read(context, findResult.expIndex, findResult.expIndex + DOUBLE_DASH_COMMENT_START.length);
-        discardTillNewLine(context);
-      }
-      break;
-    }
-    case HASH_COMMENT_START: {
-      if (context.retainComments) {
-        read(context, findResult.nextIndex);
-        readTillNewLine(context, false);
-      } else {
-        read(context, findResult.expIndex, findResult.nextIndex);
-        discardTillNewLine(context);
-      }
-      break;
-    }
-    case C_STYLE_COMMENT_START: {
-      if (['!', '+'].includes(context.unread[findResult.nextIndex]) || context.retainComments) {
-        // Should not be skipped, see https://dev.mysql.com/doc/refman/5.7/en/comments.html
-        read(context, findResult.nextIndex);
-        const findCommentResult = findExp(context.unread, cStyleCommentEndRegex);
-        read(context, findCommentResult.nextIndex);
-      } else {
-        read(context, findResult.expIndex, findResult.nextIndex);
-        const findCommentResult = findExp(context.unread, cStyleCommentEndRegex);
-        discard(context, findCommentResult.nextIndex);
-      }
-      break;
-    }
-    case DELIMITER_KEYWORD: {
-      read(context, findResult.expIndex, findResult.nextIndex);
-      // MySQL client will return `DELIMITER cannot contain a backslash character` if backslash is used
-      // Shall we reject backslash as well?
-      const matched = context.unread.match(delimiterTokenRegex);
-      if (matched?.index !== undefined) {
-        context.currentDelimiter = matched[0].trim();
-        discard(context, matched[0].length);
-      }
-      discardTillNewLine(context);
-      break;
-    }
-    case undefined:
-    case null:
-      read(context, findResult.nextIndex);
-      publishStatement(context);
-      break;
-    default:
-      // This should never happen
-      throw new Error(`Unknown token '${findResult.exp ?? '(null)'}'`);
-  }
-}
-
-export function splitPostgresQuery(sql: string, options?: SplitOptions): string[] {
-  options = options ?? {};
-  const context: SplitExecutionContext = {
-    multipleStatements: options.multipleStatements ?? false,
-    retainComments: options.retainComments ?? false,
-    unread: sql,
-    currentDelimiter: SEMICOLON,
-    currentStatement: {
-      value: '',
-      supportMulti: true,
-    },
-    output: [],
-  };
-  let findResult: FindExpResult = {
-    expIndex: -1,
-    exp: null,
-    nextIndex: 0,
-  };
-  let lastUnreadLength;
-  do {
-    lastUnreadLength = context.unread.length;
-    findResult = findKeyToken(context.unread, context.currentDelimiter);
-    handleKeyTokenFindResult(context, findResult);
-    // Prevent infinite loop by returning incorrect result
-    if (lastUnreadLength === context.unread.length) {
-      read(context, context.unread.length);
-    }
-  } while (context.unread !== '');
-  publishStatement(context);
-  return context.output.map(v => v.value);
-}
@@ -0,0 +1,84 @@
+import { DatabaseInfo } from 'dbgate-types';
+import _ from 'lodash';
+
+export function addTableDependencies(db: DatabaseInfo): DatabaseInfo {
+  const allForeignKeys = _.flatten(db.tables.map(x => x.foreignKeys || []));
+  return {
+    ...db,
+    tables: db.tables.map(table => ({
+      ...table,
+      dependencies: allForeignKeys.filter(x => x.refSchemaName == table.schemaName && x.refTableName == table.pureName),
+    })),
+  };
+}
+
+function fillTableExtendedInfo(db: DatabaseInfo): DatabaseInfo {
+  return {
+    ...db,
+    tables: (db.tables || []).map(obj => ({
+      ...obj,
+      objectTypeField: 'tables',
+      columns: (obj.columns || []).map(column => ({
+        pureName: obj.pureName,
+        schemaName: obj.schemaName,
+        ...column,
+      })),
+      primaryKey: obj.primaryKey
+        ? {
+            ...obj.primaryKey,
+            pureName: obj.pureName,
+            schemaName: obj.schemaName,
+            constraintType: 'primaryKey',
+          }
+        : undefined,
+      foreignKeys: (obj.foreignKeys || []).map(cnt => ({
+        ...cnt,
+        pureName: obj.pureName,
+        schemaName: obj.schemaName,
+        constraintType: 'foreignKey',
+      })),
+      indexes: (obj.indexes || []).map(cnt => ({
+        ...cnt,
+        pureName: obj.pureName,
+        schemaName: obj.schemaName,
+        constraintType: 'index',
+      })),
+      checks: (obj.checks || []).map(cnt => ({
+        ...cnt,
+        pureName: obj.pureName,
+        schemaName: obj.schemaName,
+        constraintType: 'check',
+      })),
+      uniques: (obj.uniques || []).map(cnt => ({
+        ...cnt,
+        pureName: obj.pureName,
+        schemaName: obj.schemaName,
+        constraintType: 'unique',
+      })),
+    })),
+    collections: (db.collections || []).map(obj => ({
+      ...obj,
+      objectTypeField: 'collections',
+    })),
+    views: (db.views || []).map(obj => ({
+      ...obj,
+      objectTypeField: 'views',
+    })),
+    procedures: (db.procedures || []).map(obj => ({
+      ...obj,
+      objectTypeField: 'procedures',
+    })),
+    functions: (db.functions || []).map(obj => ({
+      ...obj,
+      objectTypeField: 'functions',
+    })),
+    triggers: (db.triggers || []).map(obj => ({
+      ...obj,
+      objectTypeField: 'triggers',
+    })),
+  };
+}
+
+export function extendDatabaseInfo(db: DatabaseInfo): DatabaseInfo {
+  return fillTableExtendedInfo(addTableDependencies(db));
+}
@@ -10,7 +10,7 @@ export interface ColumnReference {

 export interface ConstraintInfo extends NamedObjectInfo {
  constraintName: string;
-  constraintType: string;
+  constraintType: 'primaryKey' | 'foreignKey' | 'index' | 'check' | 'unique';
 }

 export interface ColumnsConstraintInfo extends ConstraintInfo {
@@ -26,7 +26,18 @@ export interface ForeignKeyInfo extends ColumnsConstraintInfo {
  deleteAction: string;
 }

-export interface ColumnInfo {
+export interface IndexInfo extends ColumnsConstraintInfo {
+  isUnique: boolean;
+  indexType: 'normal' | 'clustered' | 'xml' | 'spatial' | 'fulltext';
+}
+
+export interface UniqueInfo extends ColumnsConstraintInfo {}
+
+export interface CheckInfo extends ConstraintInfo {
+  definition: string;
+}
+
+export interface ColumnInfo extends NamedObjectInfo {
  columnName: string;
  notNull: boolean;
  autoIncrement: boolean;
@@ -58,8 +69,13 @@ export interface TableInfo extends DatabaseObjectInfo {
  primaryKey?: PrimaryKeyInfo;
  foreignKeys: ForeignKeyInfo[];
  dependencies?: ForeignKeyInfo[];
+  indexes?: IndexInfo[];
+  uniques?: UniqueInfo[];
+  checks?: CheckInfo[];
 }

+export interface CollectionInfo extends DatabaseObjectInfo {}
+
 export interface ViewInfo extends SqlObjectInfo {
  columns: ColumnInfo[];
 }
@@ -77,6 +93,7 @@ export interface SchemaInfo {

 export interface DatabaseInfoObjects {
  tables: TableInfo[];
+  collections: CollectionInfo[];
  views: ViewInfo[];
  procedures: ProcedureInfo[];
  functions: FunctionInfo[];
@@ -1,8 +1,13 @@
 export interface SqlDialect {
  rangeSelect?: boolean;
  limitSelect?: boolean;
+  rowNumberOverPaging?: boolean;
  stringEscapeChar: string;
  offsetFetchRangeSyntax?: boolean;
  quoteIdentifier(s: string): string;
  fallbackDataType?: string;
+  explicitDropConstraint?: boolean;
+  anonymousPrimaryKey?: boolean;
+  enableConstraintsPerTable?: boolean;
+  nosql?: boolean; // mongo
 }
@@ -24,9 +24,21 @@ export interface EngineAuthType {
  disabledFields: string[];
 }

+export interface ReadCollectionOptions {
+  pureName: string;
+  schemaName?: string;
+
+  countDocuments?: boolean;
+  skip?: number;
+  limit?: number;
+}
+
 export interface EngineDriver {
  engine: string;
  title: string;
+  defaultPort?: number;
+  supportsDatabaseUrl?: boolean;
+  databaseUrlPlaceholder?: string;
  connect({ server, port, user, password, database }): any;
  query(pool: any, sql: string): Promise<QueryResult>;
  stream(pool: any, sql: string, options: StreamOptions);
@@ -49,8 +61,13 @@ export interface EngineDriver {
  analyseFull(pool: any): Promise<DatabaseInfo>;
  analyseIncremental(pool: any, structure: DatabaseInfo): Promise<DatabaseInfo>;
  dialect: SqlDialect;
+  dialectByVersion(version): SqlDialect;
  createDumper(): SqlDumper;
  getAuthTypes(): EngineAuthType[];
+  readCollection(pool: any, options: ReadCollectionOptions): Promise<any>;
+  updateCollection(pool: any, changeSet: any): Promise<any>;
+  getCollectionUpdateScript(changeSet: any): string;
+  createDatabase(pool: any, name: string): Promise;

  analyserClass?: any;
  dumperClass?: any;
@@ -21,6 +21,12 @@ export interface FileFormatDefinition {
  getOutputParams?: (sourceName, values) => any;
 }

+export interface ThemeDefinition {
+  className: string;
+  themeName: string;
+  themeType: 'light' | 'dark';
+}
+
 export interface PluginDefinition {
  packageName: string;
  manifest: any;
@@ -31,4 +37,5 @@ export interface ExtensionsDirectory {
  plugins: PluginDefinition[];
  fileFormats: FileFormatDefinition[];
  drivers: EngineDriver[];
+  themes: ThemeDefinition[];
 }
@@ -4,6 +4,7 @@ export interface OpenedDatabaseConnection {
  conid: string;
  database: string;
  structure: DatabaseInfo;
+  serverVersion?: any;
  subprocess: ChildProcess;
  disconnected?: boolean;
  status?: {
--- a/Show More
+++ b/Show More