v4.3.1-beta.2

This reverts commit da5b96c0a8.

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: dbgate
+patreon: # Replace with a single Patreon username
+open_collective: dbgate
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,33 @@
+---
+name: Bug report
+about: Create a report to help us improve DbGate
+title: 'BUG: Say something here'
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version Information (please complete the following information):**
+ - OS: [e.g. Windows/Mac/Ubuntu]
+ - App Version [see Help -> About]
+ - Install source [e.g. installer/SNAP/Docker/NPM]
+ - Type - Web/Application
+ - Database engine: [e.g. MySQL/PostgreSQL/SQL Server]
+
+**Additional context**
+Anything else you think might be helpful

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for DbGate
+title: 'FEAT: '
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,18 @@
+---
+name: Question
+about: Ask a question about how to do something
+title: 'QUESTION: Summary of your question'
+labels: ''
+assignees: ''
+
+---
+
+**Details:**
+Details about your question
+
+**Version Information [might be relevant to your issue]**
+Operating System: 
+App Version [help -> about]:
+
+**Screenshot [if appropriate]**:
+A screenshot of the app if that helps

.github/workflows/build-app-beta.yaml

@@ -0,0 +1,107 @@
+name: Electron app BETA
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        # os: [ubuntu-18.04, windows-2016]
+        os: [macOS-10.15, windows-2016, ubuntu-18.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Use Node.js 10.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 10.x
+      - name: yarn install
+        run: |
+          yarn install
+      - name: setCurrentVersion
+        run: |
+          yarn setCurrentVersion
+      - name: fillNativeModulesElectron
+        run: |
+          yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
+      - name: Install Snapcraft
+        if: matrix.os == 'ubuntu-18.04'
+        uses: samuelmeuli/action-snapcraft@v1
+      - name: Publish
+        if: matrix.os != 'macOS-10.15'
+        run: |
+          yarn run build:app
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+          WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
+
+      - name: Publish Mac
+        if: matrix.os == 'macOS-10.15'
+        run: |
+          yarn run build:app:mac
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+
+      - name: Save snap login
+        if: matrix.os == 'ubuntu-18.04'
+        run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
+        shell: bash
+        env:
+          SNAPCRAFT_LOGIN: ${{secrets.SNAPCRAFT_LOGIN}}
+
+      - name: publishSnap
+        if: matrix.os == 'ubuntu-18.04'
+        run: |
+          snapcraft login --with snapcraft.login
+          snapcraft upload --release=beta app/dist/*.snap
+
+      - name: Copy artifacts
+        run: |
+          mkdir artifacts          
+
+          cp app/dist/*.deb artifacts/dbgate-beta.deb || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-beta.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-beta-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-beta-armv7l.AppImage || true
+          cp app/dist/*win*.exe artifacts/dbgate-beta.exe || true
+          cp app/dist/*win_x64.zip artifacts/dbgate-windows-beta.zip || true
+          cp app/dist/*win_arm64.zip artifacts/dbgate-windows-beta-arm64.zip || true
+          cp app/dist/*-mac.dmg artifacts/dbgate-beta.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-beta-arm64.dmg || true
+
+          mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
+          mv app/dist/*.AppImage artifacts/ || true
+          mv app/dist/*.deb artifacts/ || true
+          mv app/dist/*.snap artifacts/ || true
+          mv app/dist/*.dmg artifacts/ || true
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ matrix.os }}
+          path: artifacts
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          files: 'artifacts/**'
+          prerelease: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-app.yaml

@@ -3,7 +3,8 @@ name: Electron app
 on:
   push:
     tags:
-      - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      # - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
 
     # branches:
     #   - production
@@ -15,14 +16,14 @@ jobs:
     strategy:
       matrix:
         # os: [ubuntu-18.04, windows-2016]
-        os: [macOS-10.14, windows-2016, ubuntu-18.04]
+        os: [macOS-10.15, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
       - name: Use Node.js 10.x
@@ -35,29 +36,60 @@ jobs:
       - name: setCurrentVersion
         run: |
           yarn setCurrentVersion
+      - name: fillNativeModulesElectron
+        run: |
+          yarn fillNativeModulesElectron
+      - name: fillPackagedPlugins
+        run: |
+          yarn fillPackagedPlugins
+      - name: Install Snapcraft
+        if: matrix.os == 'ubuntu-18.04'
+        uses: samuelmeuli/action-snapcraft@v1
       - name: Publish
         run: |
           yarn run build:app
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }} # token for electron publish
+          WIN_CSC_LINK: ${{ secrets.WINCERT_CERTIFICATE }}
+          WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_PASSWORD }}
 
       - name: generatePadFile
         run: |
           yarn generatePadFile
 
+      - name: Save snap login
+        if: matrix.os == 'ubuntu-18.04'
+        run: 'echo "$SNAPCRAFT_LOGIN" > snapcraft.login'
+        shell: bash
+        env:
+          SNAPCRAFT_LOGIN: ${{secrets.SNAPCRAFT_LOGIN}}
+
+      - name: publishSnap
+        if: matrix.os == 'ubuntu-18.04'
+        run: |
+          snapcraft login --with snapcraft.login
+          snapcraft upload --release=stable app/dist/*.snap
+
       - name: Copy artifacts
         run: |
           mkdir artifacts          
 
           cp app/dist/*.deb artifacts/dbgate-latest.deb || true
-          cp app/dist/*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*x86*.AppImage artifacts/dbgate-latest.AppImage || true
+          cp app/dist/*arm64*.AppImage artifacts/dbgate-latest-arm64.AppImage || true
+          cp app/dist/*armv7l*.AppImage artifacts/dbgate-latest-armv7l.AppImage || true
           cp app/dist/*.exe artifacts/dbgate-latest.exe || true
-          cp app/dist/*.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*win_x64.zip artifacts/dbgate-windows-latest.zip || true
+          cp app/dist/*win_arm64.zip artifacts/dbgate-windows-latest-arm64.zip || true
+          cp app/dist/*-mac.dmg artifacts/dbgate-latest.dmg || true
+          cp app/dist/*-mac_arm64.dmg artifacts/dbgate-latest-arm64.dmg || true
 
           mv app/dist/*.exe artifacts/ || true
+          mv app/dist/*.zip artifacts/ || true
           mv app/dist/*.AppImage artifacts/ || true
           mv app/dist/*.deb artifacts/ || true
           mv app/dist/*.dmg artifacts/ || true
+          mv app/dist/*.snap artifacts/dbgate-latest.snap || true
 
       # - name: Copy artifacts Linux, MacOs
       #   if: matrix.os != 'windows-2016'
@@ -95,7 +127,7 @@ jobs:
           mv app/dist/latest-linux.yml artifacts/latest-linux.yml || true
 
       - name: Copy latest-mac.yml
-        if: matrix.os == 'macOS-10.14'
+        if: matrix.os == 'macOS-10.15'
         run: |
           mv app/dist/latest-mac.yml artifacts/latest-mac.yml || true
 

.github/workflows/build-docker-beta.yaml

@@ -0,0 +1,47 @@
+name: Docker image BETA
+
+# on: [push]
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+'
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-18.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Use Node.js 10.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 10.x
+      - name: yarn install
+        run: |
+          yarn install
+      - name: setCurrentVersion
+        run: |
+          yarn setCurrentVersion
+      - name: Prepare docker image
+        run: |
+          yarn run prepare:docker
+      - name: Build docker image
+        run: |
+          docker build ./docker -t dbgate
+      - name: Push docker image
+        run: |
+          docker tag dbgate dbgate/dbgate:beta
+          docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }}
+          docker push dbgate/dbgate:beta

.github/workflows/build-docker.yaml

@@ -5,7 +5,8 @@ name: Docker image
 on:
   push:
     tags:
-    - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+    # - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
 
 # on:
 #   push:
@@ -20,14 +21,13 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-18.04]
-        # os: [macOS-10.14, windows-2016, ubuntu-18.04]
 
     steps:
       - name: Context
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
         run: echo "$GITHUB_CONTEXT"
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
       - name: Use Node.js 10.x

.github/workflows/build-npm.yaml

@@ -0,0 +1,130 @@
+name: NPM packages
+
+# on: [push]
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      # - 'v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+'
+
+# on:
+#   push:
+#     branches:
+#       - production
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-18.04]
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: Use Node.js 10.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 10.x
+
+      - name: Configure NPM token
+        env:
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+        run: |
+          npm config set '//registry.npmjs.org/:_authToken' "${NPM_TOKEN}"
+
+      - name: yarn install
+        run: |
+          yarn install
+          
+      - name: setCurrentVersion
+        run: |
+          yarn setCurrentVersion
+
+      - name: Publish types
+        working-directory: packages/types
+        run: |
+          npm publish
+
+      - name: Publish tools
+        working-directory: packages/tools
+        run: |
+          npm publish
+
+      - name: Publish sqltree
+        working-directory: packages/sqltree
+        run: |
+          npm publish
+                    
+      - name: Publish api
+        working-directory: packages/api
+        run: |
+          npm publish
+
+      - name: Publish datalib
+        working-directory: packages/datalib
+        run: |
+          npm publish
+
+      - name: Publish filterparser
+        working-directory: packages/filterparser
+        run: |
+          npm publish
+
+      - name: Publish query-splitter
+        working-directory: packages/query-splitter
+        run: |
+          npm publish
+          
+      - name: Publish web
+        working-directory: packages/web
+        run: |
+          npm publish
+                                        
+      - name: Publish dbgate
+        working-directory: packages/dbgate
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-csv
+        working-directory: plugins/dbgate-plugin-csv
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-excel
+        working-directory: plugins/dbgate-plugin-excel
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mssql
+        working-directory: plugins/dbgate-plugin-mssql
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mysql
+        working-directory: plugins/dbgate-plugin-mysql
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-mongo
+        working-directory: plugins/dbgate-plugin-mongo
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-postgres
+        working-directory: plugins/dbgate-plugin-postgres
+        run: |
+          npm publish
+
+      - name: Publish dbgate-plugin-sqlite
+        working-directory: plugins/dbgate-plugin-sqlite
+        run: |
+          npm publish

.github/workflows/run-tests.yaml

@@ -0,0 +1,82 @@
+name: Run tests
+on:
+  push:
+    branches:
+      - master
+      - develop
+
+jobs:
+  test-runner:
+    runs-on: ubuntu-latest
+    container: node:10.18-jessie
+
+    steps:
+      - name: Context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - name: yarn install
+        run: |
+          yarn install
+      - name: Integration tests
+        run: |
+          cd integration-tests
+          yarn test:ci
+#          yarn wait:ci
+      - name: Filter parser tests
+        if: always()
+        run: |
+          cd packages/filterparser
+          yarn test:ci
+      - name: Query spliiter tests
+        if: always()
+        run: |
+          cd packages/query-splitter
+          yarn test:ci
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: integration-tests/result.json
+          action-name: Integration tests
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/filterparser/result.json
+          action-name: Filter parser test results
+      - uses: tanmen/jest-reporter@v1
+        if: always()
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          result-file: packages/query-splitter/result.json
+          action-name: Query splitter test results
+
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: Pwd2020Db
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+      mysql:
+        image: mysql:8.0.18
+        env:
+          MYSQL_ROOT_PASSWORD: Pwd2020Db
+
+      mssql:
+        image: mcr.microsoft.com/mssql/server
+        env:
+          ACCEPT_EULA: Y
+          SA_PASSWORD: Pwd2020Db
+          MSSQL_PID: Express
+          
+      # cockroachdb:
+      #   image: cockroachdb/cockroach

.gitignore

@@ -12,6 +12,12 @@ node_modules
 build
 dist
 
+app/packages/web/public
+app/packages/plugins
+docker/public
+docker/bundle.js
+docker/plugins
+
 # misc
 .DS_Store
 .env.local
@@ -22,3 +28,7 @@ dist
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+app/src/nativeModulesContent.js
+packages/api/src/nativeModulesContent.js
+packages/api/src/packagedPluginsContent.js
+.VSCodeCounter

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "jestrunner.jestCommand": "node_modules/.bin/cross-env DEVMODE=1 LOCALTEST=1 node_modules/.bin/jest"
+}

CHANGELOG.md

@@ -0,0 +1,119 @@
+# ChangeLog
+
+### 4.3.0
+- ADDED: Table structure editor
+- ADDED: Index support
+- ADDED: Unique constraint support
+- ADDED: Context menu for drop/rename table/columns and for drop view/procedure/function
+- ADDED: Added support for Windows arm64 platform
+- FIXED: Search by _id in MongoDB
+
+### 4.2.6
+- FIXED: Fixed MongoDB import
+- ADDED: Configurable thousands separator #136
+- ADDED: Using case insensitive text search in postgres
+
+### 4.2.5
+- FIXED: Fixed crash when using large model on some installations
+- FIXED: Postgre SQL CREATE function 
+- FIXED: Analysing of MySQL when modifyDate is not known
+
+### 4.2.4
+- ADDED: Query history
+- ADDED: One-click exports in desktop app
+- ADDED: JSON array export
+- FIXED: Procedures in PostgreSQL #122
+- ADDED: Support of materialized views for PostgreSQL #123
+- ADDED: Integration tests
+- FIXED: Fixes in DB structure analysis in PostgreSQL, SQLite, MySQL
+- FIXED: Save data in SQLite, PostgreSQL
+- CHANGED: Introduced package dbgate-query-splitter, instead of sql-query-identifier and @verycrazydog/mysql-parse
+
+### 4.2.3
+- ADDED: ARM builds for MacOS and Linux
+- ADDED: Filter by columns in form view
+
+### 4.2.2
+- CHANGED: Further startup optimalization (approx. 2 times quicker start of electron app)
+
+### 4.2.1
+- FIXED: Fixed+optimalized app startup (esp. on Windows)
+
+### 4.2.0
+- ADDED: Support of SQLite database
+- ADDED: Support of Amazon Redshift database
+- ADDED: Support of CockcroachDB
+- CHANGED: DB Model is not auto-refreshed by default, refresh could be invoked from statusbar
+- FIXED: Fixed race conditions on startup
+- FIXED: Fixed broken style in data grid under strange circumstances
+- ADDED: Configure connections with commandline arguments  #108
+- CHANGED: Optimalized algorithm of incremental DB model updates
+- CHANGED: Loading queries from PostgreSQL doesn't need cursors, using streamed query instead
+- ADDED: Disconnect command
+- ADDED: Query executed on server has tab marker (formerly it had only "No DB" marker)
+- ADDED: Horizontal scroll using shift+mouse wheel #113
+- ADDED: Cosmetic improvements of MariaDB support
+
+### 4.1.11
+- FIX: Fixed crash of API process when using SSH tunnel connection (race condition)
+
+### 4.1.11
+- FIX: fixed processing postgre query containing $$
+- FIX: fixed postgre analysing procedures & functions
+- FIX: patched svelte crash #105
+- ADDED: ability to disbale background DB model updates
+- ADDED: Duplicate connection
+- ADDED: Duplicate tab
+- FIX: SSH tunnel connection using keyfile auth #106
+- FIX: All tables button fix in export #109
+- CHANGED: Add to favorites moved from toolbar to tab context menu
+- CHANGED: Toolbar design - current tab related commands are delimited
+
+### 4.1.10
+- ADDED: Default database option in connectin settings #96 #92
+- FIX: Bundle size optimalization for Windows #97
+- FIX: Popup menu placement on smaller displays #94
+- ADDED: Browse table data with SQL Server 2008 #93
+- FIX: Prevented malicious origins / DNS rebinding #91
+- ADDED: Handle JSON fields in data editor (eg. jsonb field in Postgres) #90
+- FIX: Fixed crash on Windows with Hyper-V #86
+- ADDED: Show database server version in status bar
+- ADDED: Show detailed info about error, when connect to database fails
+- ADDED: Portable ZIP distribution for Windows #84
+### 4.1.9
+- FIX: Incorrect row count info in query result #83
+
+### 4.1.1
+- CHANGED: Default plugins are now part of installation
+### 4.1.0
+- ADDED: MongoDB support
+- ADDED: Configurable keyboard shortcuts
+- ADDED: JSON row cell data view
+- FIX: Fixed some problems from migration to Svelte
+
+### 4.0.3
+- FIX: fixes for FireFox (mainly incorrent handle of bind:clientHeight, replaces with resizeobserver)
+### 4.0.2
+- FIX: fixed docker and NPM build
+### 4.0.0
+- CHANGED: Excahnged React with Svelte. Changed theme colors. Huge speed and memory optimalization
+- ADDED: SQL Generator (CREATE, INSERT, DROP)
+- ADDED: Command palette (F1). Introduced commands, extended some context menus
+- ADDED: New keyboard shortcuts
+- ADDED: Switch to recent database feature
+- ADDED: Macros from free table editor are available also in table data editor
+- CHANGED: Cell data preview is now in left widgets panel
+- CHANGED: Toolbar refactor
+- FIX: Solved reconnecting expired connection
+
+### 3.9.6
+- ADDED: Connect using SSH Tunnel
+- ADDED: Connect using SSL
+- ADDED: Database connection dialog redesigned
+- ADDED: #63 Ctrl+Enter runs query
+- ADDED: Published dbgate NPM package
+- ADDED: SQL editor context menu
+- FIX: #62 - import, export executed from SNAP installs didn't work
+
+### 3.9.5
+- Start point of changelog

README.md

@@ -1,72 +1,81 @@
+[![NPM version](https://img.shields.io/npm/v/dbgate.svg)](https://www.npmjs.com/package/dbgate)
+![GitHub All Releases](https://img.shields.io/github/downloads/dbgate/dbgate/total) 
+[![dbgate](https://snapcraft.io/dbgate/badge.svg)](https://snapcraft.io/dbgate)
+[![dbgate](https://snapcraft.io/dbgate/trending.svg?name=0)](https://snapcraft.io/dbgate)
 [![styled with prettier](https://img.shields.io/badge/styled_with-prettier-ff69b4.svg)](https://github.com/prettier/prettier)
-[![Donate](https://img.shields.io/badge/donate-paypal-blue.svg)](https://paypal.me/JanProchazkaCz/30eur)
-[![NPM version](https://img.shields.io/npm/v/dbgate-api.svg)](https://www.npmjs.com/package/dbgate-api)
 
 # DbGate - database administration tool
 
-DbGate is fast and efficient database administration tool. It is focused to work with data (filtering, editing, master/detail views etc.)
+DbGate modern, fast and easy to use database manager
 
-**Try it online** - https://demo.dbgate.org - online demo application
+* Try it online - [demo.dbgate.org](https://demo.dbgate.org) - online demo application
+* Download application for Windows, Linux or Mac from [dbgate.org](https://dbgate.org/download/)
+* Run web version as [NPM package](https://www.npmjs.com/package/dbgate) or as [docker image](https://hub.docker.com/r/dbgate/dbgate)
+
+Supported databases:
+* MySQL
+* PostgreSQL
+* SQL Server
+* MongoDB
+* SQLite
+* Amazon Redshift
+* CockroachDB
+* MariaDB
+
+![Screenshot](https://raw.githubusercontent.com/dbgate/dbgate/master/screenshot.png)
 
 ## Features
-* Support for Microsoft SQL Server, Postgre SQL, MySQL
-* Table data browsing - filtering, sorting, related columns using foreign keys
-* Master/detail views
-* Browsing objects - tables, views, procedures, functions
 * Table data editing, with SQL change script preview
-* SQL editor, execute SQL script, SQL code formatter, SQL code completion, SQL join wizard
+* Light and dark theme
+* Master/detail views
+* Query designer
+* Form view for comfortable work with tables with many columns
+* JSON view on MongoDB collections
+* Explore tables, views, procedures, functions, MongoDB collections
+* SQL editor
+  * execute SQL script
+  * SQL code formatter
+  * SQL code completion
+  * Add SQL LEFT/INNER/RIGHT join utility
+* Mongo JavaScript editor, execute Mongo script (with NodeJs syntax)
 * Runs as application for Windows, Linux and Mac. Or in Docker container on server and in web Browser on client.
 * Import, export from/to CSV, Excel, JSON
 * Free table editor - quick table data editing (cleanup data after import/before export, prototype tables etc.)
 * Archives - backup your data in JSON files on local filesystem (or on DbGate server, when using web application)
-* Light and dark theme
+* Charts
 * For detailed info, how to run DbGate in docker container, visit [docker hub](https://hub.docker.com/r/dbgate/dbgate)
 * Extensible plugin architecture
 
-![Screenshot](https://raw.githubusercontent.com/dbshell/dbgate/master/screenshot.png)
+## How to contribute
+Any contributions are welcome. If you want to contribute without coding, consider following:
+* Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues.
+* Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better
+
+## Why is DbGate different
+There are many database managers now, so why DbGate? 
+* Works everywhere - Windows, Linux, Mac, Web browser (+mobile web is planned), without compromises in features
+* Based on standalone NPM packages, scripts can be run without DbGate (example - [CSV export](https://www.npmjs.com/package/dbgate-plugin-csv) )
+* Many data browsing functions based using foreign keys - master/detail, expand columns, expandable form view (on screenshot above)
 
 ## Design goals
 * Application simplicity - DbGate takes the best and only the best from old [DbGate](http://www.jenasoft.com/dbgate), [DatAdmin](http://www.jenasoft.com/datadmin) and [DbMouse](http://www.jenasoft.com/dbmouse) .
 * Minimal dependencies
-    * Frontend - React, styled-components, socket.io
+    * Frontend - Svelte, socket.io
     * Backend - NodeJs, ExpressJs, socket.io, database connection drivers
     * JavaScript + TypeScript
     * App - electron
-    * There is plan to incorporate SQLite to support work with local datasets
-* Platform independed - will run as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
+* Platform independent - will run as web application in single docker container on server, or as application using Electron platform on Linux, Windows and Mac
 
 ## Plugins
 Plugins are standard NPM packages published on [npmjs.com](https://www.npmjs.com).  
 See all [existing DbGate plugins](https://www.npmjs.com/search?q=keywords:dbgateplugin).  
-Visit [dbgate generator homepage](https://github.com/dbshell/generator-dbgate) to see, how to create your own plugin.  
+Visit [dbgate generator homepage](https://github.com/dbgate/generator-dbgate) to see, how to create your own plugin.  
 
 Currently following extensions can be implemented using plugins:
 - File format parsers/writers
 - Database engine connectors
 
-## How Can I Contribute?
-You're welcome to contribute to this project! Below are some ideas, how to contribute:
-
-* Create plugins for new import/export formats
-* Bug fixing
-* Test Mac edition
-* Improve linux package build, add to APT repository
-* Auto-upgrade of electron application
-
-Any help is appreciated!
-
-Feel free to report issues and open merge requests.
-
-## Roadmap
-
-| Feature | Complexity | Schedule |
-|---|---|---|
-| Query designer | medium | december 2020 |
-| Table designer (structure editor) | big | january 2021 |
-| Filter SQL result sets | small | november 2020 |
-| Filtering, sorting in free table editor | small | november 2020 |
-| Using tedious driver instead of mssql | small | january 2021 |
-| Support for SQLite | big | 2021 |
+Basic set of plugins is part of DbGate git repository and is installed with app. Additional plugins pust be downloaded from NPM (this task is handled by DbGate)
 
 ## How to run development environment
 
@@ -75,14 +84,14 @@ yarn
 yarn start
 ```
 
-If you want to make modifications in TypeScript packages, run TypeScript compiler in watch mode in seconds terminal:
+If you want to make modifications in libraries or plugins, run library compiler in watch mode in the second terminal:
 ```sh
 yarn lib
 ```
 
 Open http://localhost:5000 in your browser
 
-You could run electron app, using this server:
+You could run electron app (requires running localhost:5000):
 ```sh
 cd app
 yarn
@@ -90,7 +99,7 @@ yarn start
 ```
 
 ## How to run built electron app locally
-This mode is very similar to production run of electron app. Electron app forks process with API on dynamically allocated port, works with compiled javascript files.
+This mode is very similar to production run of electron app. Electron app forks process with API on dynamically allocated port, works with compiled javascript files and uses compiled version of plugins (doesn't use localhost:5000)
 
 ```sh
 cd app
@@ -103,16 +112,3 @@ yarn build:app:local
 yarn start:app:local
 ```
 
-## Packages
-Some dbgate packages can be used also without DbGate. You can find them on [NPM repository](https://www.npmjs.com/search?q=keywords:dbgate)
-
-* [api](https://github.com/dbshell/dbgate/tree/master/packages/api) - backend, Javascript, ExpressJS [![NPM version](https://img.shields.io/npm/v/dbgate-api.svg)](https://www.npmjs.com/package/dbgate-api)
-* [datalib](https://github.com/dbshell/dbgate/tree/master/packages/datalib) - TypeScript library for utility classes
-* [app](https://github.com/dbshell/dbgate/tree/master/app) - application (JavaScript)
-structure, creating specific queries (JavaScript) [![NPM version](https://img.shields.io/npm/v/dbgate-engines.svg)](https://www.npmjs.com/package/dbgate-engines)
-* [filterparser](https://github.com/dbshell/dbgate/tree/master/packages/filterparser) - TypeScript library for parsing data filter expressions using parsimmon
-* [sqltree](https://github.com/dbshell/dbgate/tree/master/packages/sqltree) - JSON representation of SQL query, functions converting to SQL (TypeScript) [![NPM version](https://img.shields.io/npm/v/dbgate-sqltree.svg)](https://www.npmjs.com/package/dbgate-sqltree)
-* [types](https://github.com/dbshell/dbgate/tree/master/packages/types) - common TypeScript definitions [![NPM version](https://img.shields.io/npm/v/dbgate-types.svg)](https://www.npmjs.com/package/dbgate-types)
-* [web](https://github.com/dbshell/dbgate/tree/master/packages/web) - frontend in React (JavaScript)
-* [tools](https://github.com/dbshell/dbgate/tree/master/packages/tools) - various tools [![NPM version](https://img.shields.io/npm/v/dbgate-tools.svg)](https://www.npmjs.com/package/dbgate-tools)
-

app/DbGate_pad.xml

@@ -0,0 +1,263 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<XML_DIZ_INFO>
+  <MASTER_PAD_VERSION_INFO>
+    <MASTER_PAD_VERSION>4.0</MASTER_PAD_VERSION>
+    <MASTER_PAD_EDITOR>AppVisor 1.0.43</MASTER_PAD_EDITOR>
+    <MASTER_PAD_EDITOR_URL>http://publisher.appvisor.com</MASTER_PAD_EDITOR_URL>
+    <MASTER_PAD_INFO>Portable Application Description, or PAD(TM) 2004 Association of Software Professionals (ASP)http://www.asp-shareware.org/pad is a data set standard and specification that is commercialsoftware publishers to disseminate certified information about their software application. The PAD format is copyright of the ASP and cannot be used without the formal licensing and permission of both the Association of Software Professionals and its agent AppVisor.com.</MASTER_PAD_INFO>
+    <CERTIFIED>No</CERTIFIED>
+    <CERTIFICATE_ID></CERTIFICATE_ID>
+    <CERTIFICATE_LICENSE></CERTIFICATE_LICENSE>
+  </MASTER_PAD_VERSION_INFO>
+  <RoboSoft>
+    <Company_UIN>CMP-9600EC03382B</Company_UIN>
+    <Product_UIN>APP-A60089A1482B</Product_UIN>
+    <Publish_on_CD>N</Publish_on_CD>
+    <NewsFeed_Search_String></NewsFeed_Search_String>
+    <Search_String>DbGate</Search_String>
+    <Press_Release_Search_String></Press_Release_Search_String>
+    <Backlink></Backlink>
+    <Comments_For_Reviewer></Comments_For_Reviewer>
+    <Search_String_Unique>DbGate</Search_String_Unique>
+    <Search_Engine_Search_String></Search_Engine_Search_String>
+    <Web_Directories_Search_String></Web_Directories_Search_String>
+    <Company_Description></Company_Description>
+  </RoboSoft>
+  <Company_Info>
+    <PublisherID>pid-86000673282b</PublisherID>
+    <BrandID>bid-9600ec03382b</BrandID>
+    <Company_Name>Jan Prochazka</Company_Name>
+    <Country>Czech Republic</Country>
+    <State_Province>Czechia</State_Province>
+    <City_Town>Prague</City_Town>
+    <Zip_Postal_Code>18100</Zip_Postal_Code>
+    <Address_1>Vratislavska 396</Address_1>
+    <Address_2>Praha 8</Address_2>
+    <Company_WebSite_URL>https://dbgate.org/</Company_WebSite_URL>
+    <Contact_Info>
+      <Contact_First_Name>Jan</Contact_First_Name>
+      <Contact_Last_Name>Prochazka</Contact_Last_Name>
+      <Contact_Email>jenasoft.database@gmail.com</Contact_Email>
+      <Author_First_Name>Jan</Author_First_Name>
+      <Author_Last_Name>Prochazka</Author_Last_Name>
+      <Author_Email>jenasoft.database@gmail.com</Author_Email>
+      <Contact_Phone></Contact_Phone>
+    </Contact_Info>
+    <Support_Info>
+      <Sales_Email>jenasoft.database@gmail.com</Sales_Email>
+      <Support_Email>jenasoft.database@gmail.com</Support_Email>
+      <General_Email>jenasoft.database@gmail.com</General_Email>
+      <Sales_Phone></Sales_Phone>
+      <Support_Phone></Support_Phone>
+      <General_Phone></General_Phone>
+      <Fax_Phone></Fax_Phone>
+    </Support_Info>
+    <LinkedinPage></LinkedinPage>
+    <TwitterCompanyPage></TwitterCompanyPage>
+    <FacebookCompanyPage></FacebookCompanyPage>
+  </Company_Info>
+  <Program_Info>
+    <AppID>app-a60089a1482b</AppID>
+    <Program_Name>DbGate</Program_Name>
+    <Program_Version>3.8.40</Program_Version>
+    <Program_Release_Month>01</Program_Release_Month>
+    <Program_Release_Day>21</Program_Release_Day>
+    <Program_Release_Year>2021</Program_Release_Year>
+    <Program_Cost_Dollars>0</Program_Cost_Dollars>
+    <Program_Type>Freeware</Program_Type>
+    <Program_Category_Class>Business::Databases &amp; Tools</Program_Category_Class>
+    <Program_Specific_Category></Program_Specific_Category>
+    <Program_Categories>Business &amp; Finance :: Database Managers,Programming :: SQL,Programming :: Databases &amp; Networks</Program_Categories>
+    <Program_Release_Status>New Release</Program_Release_Status>
+    <Program_Install_Support>Install and Uninstall</Program_Install_Support>
+    <Program_Target_Platform>Windows</Program_Target_Platform>
+    <Program_OS_Support>Win2000,WinXP,Win7 x32,Win7 x64,Windows 8,Windows 10,WinServer,WinOther,Windows2000,Windows2003,Windows Server 2000,Windows Server 2003,Windows Server 2008,Windows Server 2008r2,Windows Server 2012,Windows Tablet PC Edition 2005,Windows Media Center Edition 2005,WinVista,Windows Vista Starter,Windows Vista Home Basic,Windows Vista Home Premium,Windows Vista Business,Windows Vista Enterprise,Windows Vista Ultimate,WinVista x64,Windows Vista Home Basic x64,Windows Vista Home Premium x64,Windows Vista Business x64,Windows Vista Enterprise x64,Windows Vista Ultimate x64</Program_OS_Support>
+    <Program_Language>English</Program_Language>
+    <Program_Change_Info>Form view - for tables with may columns
+Windows authentization for SQL Server
+Support for MariaDB</Program_Change_Info>
+    <Program_System_Requirements>2GB RAM, 100MB disk space</Program_System_Requirements>
+    <Limitations>No limitations</Limitations>
+    <Awards></Awards>
+    <File_Info>
+      <File_Size_Bytes>60973056</File_Size_Bytes>
+      <File_Size_K>59544</File_Size_K>
+      <File_Size_MB>58.15</File_Size_MB>
+    </File_Info>
+    <Expire_Info>
+      <Has_Expire_Info>N</Has_Expire_Info>
+      <Expire_Count></Expire_Count>
+      <Expire_Based_On></Expire_Based_On>
+      <Expire_Year></Expire_Year>
+      <Expire_Other_Info></Expire_Other_Info>
+      <Expire_Month></Expire_Month>
+      <Expire_Day></Expire_Day>
+    </Expire_Info>
+    <Includes_JAVA_VM></Includes_JAVA_VM>
+    <Includes_DirectX></Includes_DirectX>
+    <Includes_VB_Runtime></Includes_VB_Runtime>
+  </Program_Info>
+  <Web_Info>
+    <Application_URLs>
+      <Application_Info_URL>https://dbgate.org/</Application_Info_URL>
+      <Application_Order_URL></Application_Order_URL>
+      <Application_Screenshot_URL>https://raw.githubusercontent.com/dbshell/dbgate/master/screenshot800.png</Application_Screenshot_URL>
+      <Application_Icon_URL>https://raw.githubusercontent.com/dbshell/dbgate/master/app/icon32.png</Application_Icon_URL>
+      <Application_XML_File_URL>http://repository.appvisor.com/app-a60089a1482b/site-01/DbGate_pad.xml</Application_XML_File_URL>
+    </Application_URLs>
+    <Download_URLs>
+      <Primary_Download_URL>https://github.com/dbshell/dbgate/releases/latest/download/dbgate-latest.exe</Primary_Download_URL>
+      <Secondary_Download_URL></Secondary_Download_URL>
+      <Additional_Download_URL_1></Additional_Download_URL_1>
+      <Additional_Download_URL_2></Additional_Download_URL_2>
+    </Download_URLs>
+  </Web_Info>
+  <Program_Descriptions>
+    <English>
+      <Keywords>sql, management, tool, microsoft, sql, mssql, postgres, postgresql, mysql, csv, data, export, import, javascript, database</Keywords>
+      <Char_Desc_45>Modern data manipulation tool</Char_Desc_45>
+      <Char_Desc_80>Modern cross-platform data manipulation tool</Char_Desc_80>
+      <Char_Desc_250>Modern cross-platform data manipulation tool for Microsoft SQL Server, PostgreSQL and MySQL. Query designer. Master/detail views. Advanced data filtering. Export, import. Exports can be saved as jobs</Char_Desc_250>
+      <Char_Desc_450>Modern cross-platform data manipulation tool for Microsoft SQL Server, PostgreSQL and MySQL. Query designer. Charts. Master/detail views. Advanced data filtering. Export, import. Exports can be saved as jobs</Char_Desc_450>
+      <Char_Desc_2000>Modern cross-platform data manipulation tool for Microsoft SQL Server, PostgreSQL and MySQL
+Main features: 
+  Quick database browser
+  Database editor, view and and views, stored procedures, functions, triggers, table relations
+  Export, import - CSV, MS Excel
+  Form view for tables with many columns
+  Query designer
+  Advanced data filtering 
+  JavaScript data scripting
+  Run queries, code completion
+  Display data from referenced tables
+  Create charts, supports for active charts - are refreshed when underlying source is changed</Char_Desc_2000>
+    </English>
+    <Czech>
+      <Keywords>sql, management, tool, microsoft, sql, mssql, postgres, postgresql, mysql, csv, data, export, import, javascript, database</Keywords>
+      <Char_Desc_45>Moderní správce databází</Char_Desc_45>
+      <Char_Desc_80>Správce databází MS SQL Server, MySQL a PostgreSQL</Char_Desc_80>
+      <Char_Desc_250>Nástroj pro správu databází MS SQL Server, MySQL a PostgreSQL</Char_Desc_250>
+      <Char_Desc_450>Nástroj pro správu databází MS SQL Server, MySQL a PostgreSQL</Char_Desc_450>
+      <Char_Desc_2000>Moderní multiplatformní nástroj pro práci s databázemi pro Microsoft SQL Server, PostgreSQL a MySQL. 
+Hlavní funkce: 
+  Prohlížení databáze, tabulek, pohledů, uložených procedur a funkcí
+  Export, import - CSV, MS Excel
+  Zobrazení jednoho řádku tabulky "na výšku" - vhodné pro tabulky s mnoha sloupci
+  Grafický návrh dotazů
+  Pokročilé filtrování dat
+  Advanced data filtering 
+  Skriptování pomocí jazyka JavaScript
+  SQL editor, doplňování kódu
+  Zobrazování referencových dat, na základě cizích klíčů
+  Vytváření grafů, podpora aktivních grafů - obnoví se, pokud se změní zdroj dat</Char_Desc_2000>
+    </Czech>
+  </Program_Descriptions>
+  <Permissions>
+    <Distribution_Permissions>This product can be freely distributed without limitations.</Distribution_Permissions>
+    <EULA>MIT License
+
+Copyright (c) 2021 Jan Prochazka
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.</EULA>
+  </Permissions>
+  <Affiliates>
+    <Affiliates_Regsoft_Order_Page></Affiliates_Regsoft_Order_Page>
+    <Affiliates_Regsoft_Vendor_ID></Affiliates_Regsoft_Vendor_ID>
+    <Affiliates_Regsoft_Product_ID></Affiliates_Regsoft_Product_ID>
+    <Affiliates_FORM></Affiliates_FORM>
+    <Affiliates_FORM_VER></Affiliates_FORM_VER>
+    <Affiliates_VERSION></Affiliates_VERSION>
+    <Affiliates_URL></Affiliates_URL>
+    <Affiliates_Information_Page></Affiliates_Information_Page>
+    <Affiliates_Avangate_Order_Page></Affiliates_Avangate_Order_Page>
+    <Affiliates_Avangate_Vendor_ID></Affiliates_Avangate_Vendor_ID>
+    <Affiliates_Avangate_Product_ID></Affiliates_Avangate_Product_ID>
+    <Affiliates_Avangate_Maximum_Commission_Rate></Affiliates_Avangate_Maximum_Commission_Rate>
+    <Affiliates_BlueSnap_Order_Page></Affiliates_BlueSnap_Order_Page>
+    <Affiliates_BlueSnap_Vendor_ID></Affiliates_BlueSnap_Vendor_ID>
+    <Affiliates_BlueSnap_Product_ID></Affiliates_BlueSnap_Product_ID>
+    <Affiliates_BlueSnap_Maximum_Commission_Rate></Affiliates_BlueSnap_Maximum_Commission_Rate>
+    <Affiliates_BMTMicro_Order_Page></Affiliates_BMTMicro_Order_Page>
+    <Affiliates_BMTMicro_Vendor_ID></Affiliates_BMTMicro_Vendor_ID>
+    <Affiliates_BMTMicro_Product_ID></Affiliates_BMTMicro_Product_ID>
+    <Affiliates_BMTMicro_Maximum_Commission_Rate></Affiliates_BMTMicro_Maximum_Commission_Rate>
+    <Affiliates_eSellerate_Order_Page></Affiliates_eSellerate_Order_Page>
+    <Affiliates_eSellerate_Vendor_ID></Affiliates_eSellerate_Vendor_ID>
+    <Affiliates_eSellerate_Product_ID></Affiliates_eSellerate_Product_ID>
+    <Affiliates_eSellerate_Maximum_Commission_Rate></Affiliates_eSellerate_Maximum_Commission_Rate>
+    <Affiliates_Regnet_Order_Page></Affiliates_Regnet_Order_Page>
+    <Affiliates_Regnet_Vendor_ID></Affiliates_Regnet_Vendor_ID>
+    <Affiliates_Regnet_Product_ID></Affiliates_Regnet_Product_ID>
+    <Affiliates_Regnet_Maximum_Commission_Rate></Affiliates_Regnet_Maximum_Commission_Rate>
+    <Affiliates_Regsoft_Maximum_Commission_Rate></Affiliates_Regsoft_Maximum_Commission_Rate>
+    <Affiliates_ShareIt_Order_Page></Affiliates_ShareIt_Order_Page>
+    <Affiliates_ShareIt_Vendor_ID></Affiliates_ShareIt_Vendor_ID>
+    <Affiliates_ShareIt_Product_ID></Affiliates_ShareIt_Product_ID>
+    <Affiliates_ShareIt_Maximum_Commission_Rate></Affiliates_ShareIt_Maximum_Commission_Rate>
+  </Affiliates>
+  <NewsFeed>
+    <NewsFeed_FORM></NewsFeed_FORM>
+    <NewsFeed_VERSION></NewsFeed_VERSION>
+    <NewsFeed_URL></NewsFeed_URL>
+    <NewsFeed_DESCRIPTION></NewsFeed_DESCRIPTION>
+    <NewsFeed_Feed_URL></NewsFeed_Feed_URL>
+    <NewsFeed_Type></NewsFeed_Type>
+    <NewsFeed_Site_URL></NewsFeed_Site_URL>
+    <NewsFeed_Author_First_Name>Jan</NewsFeed_Author_First_Name>
+    <NewsFeed_Author_Last_Name>Prochazka</NewsFeed_Author_Last_Name>
+    <NewsFeed_Author_Email>jenasoft.database@gmail.com</NewsFeed_Author_Email>
+    <NewsFeed_Language></NewsFeed_Language>
+    <NewsFeed_Title></NewsFeed_Title>
+    <NewsFeed_Description_70></NewsFeed_Description_70>
+    <NewsFeed_Description_250></NewsFeed_Description_250>
+    <NewsFeed_Purpose></NewsFeed_Purpose>
+    <NewsFeed_Site_Name></NewsFeed_Site_Name>
+  </NewsFeed>
+  <Press_Release>
+    <Press_Release></Press_Release>
+    <Headline></Headline>
+    <Summary></Summary>
+    <Press_Release_Plain></Press_Release_Plain>
+  </Press_Release>
+  <Site>
+    <Site_FORM></Site_FORM>
+    <Site_VERSION></Site_VERSION>
+    <Site_DESCRIPTION></Site_DESCRIPTION>
+    <Site_Site_Title>DbGate github</Site_Site_Title>
+    <Site_Site_URL>https://github.com/dbshell/dbgate</Site_Site_URL>
+    <Site_Keywords>database, manager, organizer, explorer, mssql, sqlserver, tsql, sql, mysql, postgre, postgresql,  export, import, filter</Site_Keywords>
+    <Site_Description_100>DbGate developer site</Site_Description_100>
+    <Site_Description_250>Source code for DbGate. Create issues, porticipate, post pull requests. Explore, how to write plugins for DbGate</Site_Description_250>
+    <Site_Description_450>Source code for DbGate. Create issues, porticipate, post pull requests. Explore, how to write plugins for DbGate</Site_Description_450>
+    <Site_Contact_First_Name>Jan</Site_Contact_First_Name>
+    <Site_Contact_Last_Name>Prochazka</Site_Contact_Last_Name>
+    <Site_Contact_Email>jenasoft.database@gmail.com</Site_Contact_Email>
+  </Site>
+  <Dynamic_PAD>
+    <General>
+      <DP_Pad_Enabled>TRUE</DP_Pad_Enabled>
+      <DP_AtFormFill_Enabled>TRUE</DP_AtFormFill_Enabled>
+      <DP_Distributive_Enabled>TRUE</DP_Distributive_Enabled>
+      <DP_Script_Base_URL>http://repository.appvisor.com/</DP_Script_Base_URL>
+      <DP_Pad_Mask>DbGate_pad.xml</DP_Pad_Mask>
+      <DP_Distributive_Primary_URL>https://github.com/dbshell/dbgate/releases/latest/download/dbgate-latest.exe</DP_Distributive_Primary_URL>
+      <DP_Distributive_Mask>dbgate-latest.exe</DP_Distributive_Mask>
+    </General>
+  </Dynamic_PAD>
+</XML_DIZ_INFO>

app/dbgate-pad-template.xml

@@ -6,7 +6,7 @@
     <MASTER_PAD_INFO>Portable Application Description, or PAD for short, is a data set that is used by shareware authors to disseminate information to anyone interested in their software products. To find out more go to http://www.asp-shareware.org/pad</MASTER_PAD_INFO>
   </MASTER_PAD_VERSION_INFO>
   <Company_Info>
-    <Company_Name>JenaSoft</Company_Name>
+    <Company_Name>Jan Prochazka</Company_Name>
     <Address_1>Vratislavska 396</Address_1>
     <Address_2>Praha 8</Address_2>
     <City_Town>Prague</City_Town>
@@ -139,17 +139,34 @@ SOFTWARE.</EULA>
       <Char_Desc_2000>
 Modern cross-platform data manipulation tool for Microsoft SQL Server, PostgreSQL and MySQL
 Main features: 
-  Quick database browser.
-  Database editor, view and and views, stored procedures, functions, triggers, table relations. 
+  Quick database browser
+  Database editor, view and and views, stored procedures, functions, triggers, table relations
   Export, import - CSV, MS Excel
+  Form view for tables with many columns
   Query designer
-  Advanced data filtering. 
-  Export, import. 
-  JavaScript data scripting.
-  Run queries, code completion. 
-  Display data from referenced tables. 
+  Advanced data filtering 
+  JavaScript data scripting
+  Run queries, code completion
+  Display data from referenced tables
   Create charts, supports for active charts - are refreshed when underlying source is changed
 </Char_Desc_2000>
     </English>
+    <Czech>
+      <Char_Desc_250>Správce databází MS SQL Server, MySQL a PostgreSQL</Char_Desc_250>
+      <Char_Desc_2000>
+Moderní multiplatformní nástroj pro práci s databázemi pro Microsoft SQL Server, PostgreSQL a MySQL. 
+Hlavní funkce: 
+  Prohlížení databáze, tabulek, pohledů, uložených procedur a funkcí
+  Export, import - CSV, MS Excel
+  Zobrazení jednoho řádku tabulky "na výšku" - vhodné pro tabulky s mnoha sloupci
+  Grafický návrh dotazů
+  Pokročilé filtrování dat
+  Advanced data filtering 
+  Skriptování pomocí jazyka JavaScript
+  SQL editor, doplňování kódu
+  Zobrazování referencových dat, na základě cizích klíčů
+  Vytváření grafů, podpora aktivních grafů - obnoví se, pokud se změní zdroj dat
+</Char_Desc_2000>
+    </Czech>
   </Program_Descriptions>
 </XML_DIZ_INFO>

app/package.json

@@ -1,49 +1,86 @@
 {
   "name": "dbgate",
-  "version": "3.8.28",
+  "version": "4.1.1",
   "private": true,
   "author": "Jan Prochazka <jenasoft.database@gmail.com>",
+  "description": "Opensource database administration tool",
   "dependencies": {
+    "better-sqlite3-with-prebuilds": "^7.1.8",
     "electron-log": "^4.3.1",
     "electron-store": "^5.1.1",
-    "electron-updater": "^4.3.5"
+    "electron-updater": "^4.3.5",
+    "patch-package": "^6.4.7"
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/dbshell/dbgate.git"
+    "url": "https://github.com/dbgate/dbgate.git"
   },
   "build": {
+    "artifactName": "${productName}-${version}-${os}_${arch}.${ext}",
     "appId": "org.dbgate",
     "mac": {
       "category": "database",
       "icon": "icon512.png",
-      "artifactName": "dbgate-mac-${version}.${ext}",
       "publish": [
         "github"
-      ]
+      ],
+      "target": {
+        "target": "default",
+        "arch": [
+          "arm64",
+          "x64"
+        ]
+      }
     },
     "linux": {
       "target": [
-        "AppImage",
-        "deb"
+        "deb",
+        "snap",
+        {
+          "target": "AppImage",
+          "arch": [
+            "x64",
+            "armv7l",
+            "arm64"
+          ]
+        }
       ],
-      "icon": "icon.png",
-      "artifactName": "dbgate-linux-${version}.${ext}",
+      "icon": "icons/",
       "category": "Development",
-      "synopsis": "Database administration tool for MS SQL, MySQL and PostgreSQL",
+      "synopsis": "Database manager for SQL Server, MySQL, PostgreSQL, MongoDB and SQLite",
       "publish": [
         "github"
       ]
     },
+    "appImage": {
+      "license": "./LICENSE",
+      "category": "Development"
+    },
+    "snap": {
+      "publish": [
+        "github",
+        "snapStore"
+      ],
+      "environment": {
+        "ELECTRON_SNAP": "true"
+      }
+    },
     "win": {
       "target": [
-        "nsis"
+        "nsis",
+        {
+          "target": "zip",
+          "arch": [
+            "x64",
+            "arm64"
+          ]
+        }
       ],
-      "artifactName": "dbgate-windows-${version}.${ext}",
       "icon": "icon.ico",
       "publish": [
         "github"
-      ]
+      ],
+      "rfc3161TimeStampServer": "http://sha256timestamp.ws.symantec.com/sha256/timestamp"
     },
     "files": [
       "packages",
@@ -56,16 +93,20 @@
     "start": "cross-env ELECTRON_START_URL=http://localhost:5000 electron .",
     "start:local": "cross-env electron .",
     "dist": "electron-builder",
-    "build": "cd ../packages/api && yarn build && cd ../web && yarn build:app && cd ../../app && yarn dist",
-    "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build:app && cd ../../app && yarn predist",
-    "postinstall": "electron-builder install-app-deps",
-    "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/build/*\" packages && copyfiles \"../packages/web/build/**/*\" packages"
+    "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist",
+    "build:mac": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && node setMacPlatform x64 && yarn dist && node setMacPlatform arm64 && yarn dist",
+    "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn predist",
+    "postinstall": "electron-builder install-app-deps && patch-package",
+    "predist": "copyfiles ../packages/api/dist/* packages && copyfiles \"../packages/web/public/*\" packages && copyfiles \"../packages/web/public/**/*\" packages && copyfiles --up 3 \"../plugins/dist/**/*\" packages/plugins"
   },
   "main": "src/electron.js",
   "devDependencies": {
     "copyfiles": "^2.2.0",
     "cross-env": "^6.0.3",
-    "electron": "11.1.1",
-    "electron-builder": "22.9.1"
+    "electron": "11.2.3",
+    "electron-builder": "22.10.5"
+  },
+  "optionalDependencies": {
+    "msnodesqlv8": "^2.0.10"
   }
 }

app/setMacPlatform.js

@@ -0,0 +1,8 @@
+const fs = require('fs');
+
+const text = fs.readFileSync('package.json', { encoding: 'utf-8' });
+const json = JSON.parse(text);
+
+json.build.mac.target.arch = process.argv[2];
+
+fs.writeFileSync('package.json', JSON.stringify(json, null, 2), { encoding: 'utf-8' });

app/src/electron.js

@@ -1,6 +1,6 @@
 const electron = require('electron');
 const os = require('os');
-const { Menu } = require('electron');
+const { Menu, ipcMain } = require('electron');
 const { fork } = require('child_process');
 const { autoUpdater } = require('electron-updater');
 const Store = require('electron-store');
@@ -19,17 +19,26 @@ const store = new Store();
 // Keep a global reference of the window object, if you don't, the window will
 // be closed automatically when the JavaScript object is garbage collected.
 let mainWindow;
-let splashWindow;
+let mainMenu;
 
 log.transports.file.level = 'debug';
 autoUpdater.logger = log;
+// TODO - create settings for this
+// appUpdater.channel = 'beta';
 
-function hideSplash() {
-  if (splashWindow) {
-    splashWindow.destroy();
-    splashWindow = null;
-  }
-  mainWindow.show();
+let commands = {};
+
+function commandItem(id) {
+  const command = commands[id];
+  return {
+    id,
+    label: command ? command.menuName || command.toolbarName || command.name : id,
+    accelerator: command ? command.keyText : undefined,
+    enabled: command ? command.enabled : false,
+    click() {
+      mainWindow.webContents.executeJavaScript(`dbgate_runCommand('${id}')`);
+    },
+  };
 }
 
 function buildMenu() {
@@ -37,24 +46,32 @@ function buildMenu() {
     {
       label: 'File',
       submenu: [
-        {
-          label: 'Connect to database',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_createNewConnection()`);
-          },
-        },
-        {
-          label: 'New query',
-          click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_newQuery()`);
-          },
-        },
+        commandItem('new.connection'),
+        commandItem('file.open'),
+        commandItem('group.save'),
+        commandItem('group.saveAs'),
+        commandItem('database.search'),
+        { type: 'separator' },
+        commandItem('tabs.closeTab'),
+        commandItem('file.exit'),
       ],
     },
     {
-      label: 'Edit',
-      submenu: [{ role: 'copy' }, { role: 'paste' }],
+      label: 'Window',
+      submenu: [commandItem('new.query'), { type: 'separator' }, commandItem('tabs.closeAll'), { role: 'minimize' }],
     },
+
+    // {
+    //   label: 'Edit',
+    //   submenu: [
+    //     { role: 'undo' },
+    //     { role: 'redo' },
+    //     { type: 'separator' },
+    //     { role: 'cut' },
+    //     { role: 'copy' },
+    //     { role: 'paste' },
+    //   ],
+    // },
     {
       label: 'View',
       submenu: [
@@ -67,20 +84,7 @@ function buildMenu() {
         { role: 'zoomout' },
         { type: 'separator' },
         { role: 'togglefullscreen' },
-      ],
-    },
-    {
-      role: 'window',
-      submenu: [
-        {
-          label: 'Close all tabs',
-          click() {
-            mainWindow.webContents.executeJavaScript('dbgate_closeAll()');
-          },
-        },
-        { type: 'separator' },
-        { role: 'minimize' },
-        { role: 'close' },
+        commandItem('theme.changeTheme'),
       ],
     },
     {
@@ -95,7 +99,7 @@ function buildMenu() {
         {
           label: 'DbGate on GitHub',
           click() {
-            require('electron').shell.openExternal('https://github.com/dbshell/dbgate');
+            require('electron').shell.openExternal('https://github.com/dbgate/dbgate');
           },
         },
         {
@@ -105,11 +109,12 @@ function buildMenu() {
           },
         },
         {
-          label: 'About',
+          label: 'Report problem or feature request',
           click() {
-            mainWindow.webContents.executeJavaScript(`dbgate_showAbout()`);
+            require('electron').shell.openExternal('https://github.com/dbgate/dbgate/issues/new');
           },
         },
+        commandItem('about.show'),
       ],
     },
   ];
@@ -117,6 +122,24 @@ function buildMenu() {
   return Menu.buildFromTemplate(template);
 }
 
+ipcMain.on('update-commands', async (event, arg) => {
+  commands = JSON.parse(arg);
+  for (const key of Object.keys(commands)) {
+    const menu = mainMenu.getMenuItemById(key);
+    if (!menu) continue;
+    const command = commands[key];
+
+    // rebuild menu
+    if (menu.label != command.text || menu.accelerator != command.keyText) {
+      mainMenu = buildMenu();
+      mainWindow.setMenu(mainMenu);
+      return;
+    }
+
+    menu.enabled = command.enabled;
+  }
+});
+
 function createWindow() {
   const bounds = store.get('winBounds');
 
@@ -126,25 +149,25 @@ function createWindow() {
     title: 'DbGate',
     ...bounds,
     icon: os.platform() == 'win32' ? 'icon.ico' : path.resolve(__dirname, '../icon.png'),
-    show: false,
     webPreferences: {
       nodeIntegration: true,
       enableRemoteModule: true,
     },
   });
 
-  mainWindow.setMenu(buildMenu());
+  mainMenu = buildMenu();
+  mainWindow.setMenu(mainMenu);
 
   function loadMainWindow() {
     const startUrl =
       process.env.ELECTRON_START_URL ||
       url.format({
-        pathname: path.join(__dirname, '../packages/web/build/index.html'),
+        pathname: path.join(__dirname, '../packages/web/public/index.html'),
         protocol: 'file:',
         slashes: true,
       });
     mainWindow.webContents.on('did-finish-load', function () {
-      hideSplash();
+      // hideSplash();
     });
     mainWindow.on('close', () => {
       store.set('winBounds', mainWindow.getBounds());
@@ -155,28 +178,21 @@ function createWindow() {
     }
   }
 
-  splashWindow = new BrowserWindow({
-    width: 300,
-    height: 120,
-    transparent: true,
-    frame: false,
-  });
-  splashWindow.loadURL(
-    url.format({
-      pathname: path.join(__dirname, '../packages/web/build/splash.html'),
-      protocol: 'file:',
-      slashes: true,
-    })
-  );
-
   if (process.env.ELECTRON_START_URL) {
     loadMainWindow();
   } else {
-    const apiProcess = fork(path.join(__dirname, '../packages/api/dist/bundle.js'), ['--dynport']);
-    apiProcess.on('message', (msg) => {
+    const apiProcess = fork(path.join(__dirname, '../packages/api/dist/bundle.js'), [
+      '--dynport',
+      '--is-electron-bundle',
+      '--native-modules',
+      path.join(__dirname, 'nativeModules'),
+      // '../../../src/nativeModules'
+    ]);
+    apiProcess.on('message', msg => {
       if (msg.msgtype == 'listening') {
-        const { port } = msg;
+        const { port, authorization } = msg;
         global['port'] = port;
+        global['authorization'] = authorization;
         loadMainWindow();
       }
     });

app/src/nativeModules.js

@@ -0,0 +1,3 @@
+const content = require('./nativeModulesContent');
+
+module.exports = content;

fillNativeModules.js

@@ -0,0 +1,23 @@
+const fs = require('fs');
+
+let fillContent = '';
+
+if (process.platform == 'win32') {
+  fillContent += `content.msnodesqlv8 = () => require('msnodesqlv8');`;
+}
+fillContent += `content['better-sqlite3-with-prebuilds'] = () => require('better-sqlite3-with-prebuilds');`;
+
+const getContent = (empty) => `
+// this file is generated automatically by script fillNativeModules.js, do not edit it manually
+const content = {};
+
+${empty ? '' : fillContent}
+
+module.exports = content;
+`;
+
+fs.writeFileSync(
+  'packages/api/src/nativeModulesContent.js',
+  getContent(process.argv.includes('--electron') ? true : false)
+);
+fs.writeFileSync('app/src/nativeModulesContent.js', getContent(false));

fillPackagedPlugins.js

@@ -0,0 +1,23 @@
+const fs = require('fs');
+const path = require('path');
+
+function load() {
+  const plugins = {};
+
+  for (const packageName of fs.readdirSync('plugins')) {
+    if (!packageName.startsWith('dbgate-plugin-')) continue;
+    const dir = path.join('plugins', packageName);
+    const frontend = fs.readFileSync(path.join(dir, 'dist', 'frontend.js'), 'utf-8');
+    const readme = fs.readFileSync(path.join(dir, 'README.md'), 'utf-8');
+    const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf-8'));
+    plugins[packageName] = {
+      manifest,
+      frontend,
+      readme,
+    };
+  }
+
+  return plugins;
+}
+
+fs.writeFileSync('packages/api/src/packagedPluginsContent.js', `module.exports = () => (${JSON.stringify(load())});`);

integration-tests/.gitignore

@@ -0,0 +1 @@
+dbtemp

integration-tests/__tests__/alter-database.spec.js

@@ -0,0 +1,68 @@
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const uuidv1 = require('uuid/v1');
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const { getAlterDatabaseScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
+
+function flatSource() {
+  return _.flatten(
+    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+  );
+}
+
+async function testDatabaseDiff(conn, driver, mangle, createObject = null) {
+  await driver.query(conn, `create table t1 (id int not null primary key)`);
+
+  await driver.query(
+    conn,
+    `create table t2 (
+    id int not null primary key, 
+    t1_id int null references t1(id)
+  )`
+  );
+
+  if (createObject) await driver.query(conn, createObject);
+
+  const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
+  let structure2 = _.cloneDeep(structure1);
+  mangle(structure2);
+  structure2 = extendDatabaseInfo(structure2);
+
+  const { sql } = getAlterDatabaseScript(structure1, structure2, {}, structure2, driver);
+  console.log('RUNNING ALTER SQL', driver.engine, ':', sql);
+
+  await driver.script(conn, sql);
+
+  const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));
+
+  expect(structure2Real.tables.length).toEqual(structure2.tables.length);
+  return structure2Real;
+}
+
+describe('Alter database', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Drop referenced table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testDatabaseDiff(conn, driver, db => {
+        _.remove(db.tables, x => x.pureName == 't1');
+      });
+    })
+  );
+
+  test.each(flatSource())(
+    'Drop object - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      const db = await testDatabaseDiff(
+        conn,
+        driver,
+        db => {
+          _.remove(db[type], x => x.pureName == 'obj1');
+        },
+        object.create1
+      );
+      expect(db[type].length).toEqual(0);
+    })
+  );
+});

integration-tests/__tests__/alter-table.spec.js

@@ -0,0 +1,124 @@
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const uuidv1 = require('uuid/v1');
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const { getAlterTableScript, extendDatabaseInfo, generateDbPairingId } = require('dbgate-tools');
+
+function pickImportantTableInfo(table) {
+  return {
+    pureName: table.pureName,
+    columns: table.columns
+      .filter(x => x.columnName != 'rowid')
+      .map(fp.pick(['columnName', 'notNull', 'autoIncrement'])),
+  };
+}
+
+function checkTableStructure(t1, t2) {
+  // expect(t1.pureName).toEqual(t2.pureName)
+  expect(pickImportantTableInfo(t1)).toEqual(pickImportantTableInfo(t2));
+}
+
+async function testTableDiff(conn, driver, mangle) {
+  await driver.query(conn, `create table t0 (id int not null primary key)`);
+
+  await driver.query(
+    conn,
+    `create table t1 (
+    col_pk int not null primary key, 
+    col_std int null, 
+    col_def int null default 12,
+    col_fk int null references t0(id),
+    col_idx int null,
+    col_uq int null unique,
+    col_ref int null unique
+  )`
+  );
+
+  await driver.query(conn, `create index idx1 on t1(col_idx)`);
+
+  await driver.query(conn, `create table t2 (id int not null primary key, fkval int null references t1(col_ref))`);
+
+  const tget = x => x.tables.find(y => y.pureName == 't1');
+  const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn)));
+  let structure2 = _.cloneDeep(structure1);
+  mangle(tget(structure2));
+  structure2 = extendDatabaseInfo(structure2);
+
+  const { sql } = getAlterTableScript(tget(structure1), tget(structure2), {}, structure2, driver);
+  console.log('RUNNING ALTER SQL', driver.engine, ':', sql);
+
+  await driver.script(conn, sql);
+
+  const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn));
+
+  checkTableStructure(tget(structure2Real), tget(structure2));
+  // expect(stableStringify(structure2)).toEqual(stableStringify(structure2Real));
+}
+
+const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'col_idx', 'col_uq'];
+// const TESTED_COLUMNS = ['col_pk'];
+// const TESTED_COLUMNS = ['col_idx'];
+// const TESTED_COLUMNS = ['col_def'];
+// const TESTED_COLUMNS = ['col_std'];
+// const TESTED_COLUMNS = ['col_ref'];
+
+function engines_columns_source() {
+  return _.flatten(engines.map(engine => TESTED_COLUMNS.map(column => [engine.label, column, engine])));
+}
+
+describe('Alter table', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Add column - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableDiff(conn, driver, tbl => {
+        tbl.columns.push({
+          columnName: 'added',
+          dataType: 'int',
+          pairingId: uuidv1(),
+          notNull: false,
+          autoIncrement: false,
+        });
+      });
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Drop column - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column)));
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Change nullability - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(
+        conn,
+        driver,
+        tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, notNull: true } : x)))
+      );
+    })
+  );
+
+  test.each(engines_columns_source())(
+    'Rename column - %s - %s',
+    testWrapper(async (conn, driver, column, engine) => {
+      await testTableDiff(
+        conn,
+        driver,
+        tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x)))
+      );
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Drop index - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableDiff(conn, driver, tbl => {
+        tbl.indexes = [];
+      });
+    })
+  );
+});

integration-tests/__tests__/object-analyse.spec.js

@@ -0,0 +1,89 @@
+const { testWrapper } = require('../tools');
+const engines = require('../engines');
+const _ = require('lodash');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'CREATE TABLE t2 (id int)'];
+
+function flatSource() {
+  return _.flatten(
+    engines.map(engine => (engine.objects || []).map(object => [engine.label, object.type, object, engine]))
+  );
+}
+
+const obj1Match = expect.objectContaining({
+  pureName: 'obj1',
+});
+const view1Match = expect.objectContaining({
+  pureName: 'obj1',
+  columns: expect.arrayContaining([
+    expect.objectContaining({
+      columnName: 'id',
+    }),
+  ]),
+});
+
+describe('Object analyse', () => {
+  test.each(flatSource())(
+    'Full analysis - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure[type].length).toEqual(1);
+      expect(structure[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.create1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(2);
+      expect(structure2[type].find(x => x.pureName == 'obj1')).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Incremental analysis - drop - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      await driver.query(conn, object.create2);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop2);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2[type].length).toEqual(1);
+      expect(structure2[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+
+  test.each(flatSource())(
+    'Create SQL - add - %s - %s',
+    testWrapper(async (conn, driver, type, object, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.query(conn, object.create1);
+      const structure1 = await driver.analyseFull(conn);
+      await driver.query(conn, object.drop1);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+      expect(structure2[type].length).toEqual(0);
+
+      await driver.query(conn, structure1[type][0].createSql);
+
+      const structure3 = await driver.analyseIncremental(conn, structure2);
+
+      expect(structure3[type].length).toEqual(1);
+      expect(structure3[type][0]).toEqual(type.includes('views') ? view1Match : obj1Match);
+    })
+  );
+});

integration-tests/__tests__/query.spec.js

@@ -0,0 +1,163 @@
+const engines = require('../engines');
+const { splitQuery } = require('dbgate-query-splitter');
+const { testWrapper } = require('../tools');
+
+const initSql = ['CREATE TABLE t1 (id int)', 'INSERT INTO t1 (id) VALUES (1)', 'INSERT INTO t1 (id) VALUES (2)'];
+
+expect.extend({
+  dataRow(row, expected) {
+    for (const key in expected) {
+      if (row[key] != expected[key]) {
+        return {
+          pass: false,
+          message: () => `Different key: ${key}`,
+        };
+      }
+    }
+    return {
+      pass: true,
+      message: () => '',
+    };
+  },
+});
+
+class StreamHandler {
+  constructor(resolve) {
+    this.results = [];
+    this.resolve = resolve;
+    this.infoRows = [];
+  }
+  row(row) {
+    this.results[this.results.length - 1].rows.push(row);
+  }
+  recordset(columns) {
+    this.results.push({
+      columns,
+      rows: [],
+    });
+  }
+  done(result) {
+    this.resolve(this.results);
+  }
+  info(msg) {
+    this.infoRows.push(msg);
+  }
+}
+
+function executeStreamItem(driver, conn, sql) {
+  return new Promise(resolve => {
+    const handler = new StreamHandler(resolve);
+    driver.stream(conn, sql, handler);
+  });
+}
+
+async function executeStream(driver, conn, sql) {
+  const results = [];
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('stream'))) {
+    const item = await executeStreamItem(driver, conn, sqlItem);
+    results.push(...item);
+  }
+  return results;
+}
+
+describe('Query', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      const res = await driver.query(conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(res.columns).toEqual([
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ]);
+
+      expect(res.rows).toEqual([
+        expect.dataRow({
+          id: 1,
+        }),
+        expect.dataRow({
+          id: 2,
+        }),
+      ]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple stream query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(driver, conn, 'SELECT id FROM t1 ORDER BY id');
+      expect(results.length).toEqual(1);
+      const res = results[0];
+
+      expect(res.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'More queries - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+      const results = await executeStream(
+        driver,
+        conn,
+        'SELECT id FROM t1 ORDER BY id; SELECT id FROM t1 ORDER BY id DESC'
+      );
+      expect(results.length).toEqual(2);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+
+      const res2 = results[1];
+      expect(res2.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res2.rows).toEqual([expect.dataRow({ id: 2 }), expect.dataRow({ id: 1 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script - return data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      const results = await executeStream(
+        driver,
+        conn,
+        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2); SELECT id FROM t1 ORDER BY id; '
+      );
+      expect(results.length).toEqual(1);
+
+      const res1 = results[0];
+      expect(res1.columns).toEqual([expect.objectContaining({ columnName: 'id' })]);
+      expect(res1.rows).toEqual([expect.dataRow({ id: 1 }), expect.dataRow({ id: 2 })]);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Script - no data - %s',
+    testWrapper(async (conn, driver, engine) => {
+      const results = await executeStream(
+        driver,
+        conn,
+        'CREATE TABLE t1 (id int); INSERT INTO t1 (id) VALUES (1); INSERT INTO t1 (id) VALUES (2) '
+      );
+      expect(results.length).toEqual(0);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Save data query - %s',
+    testWrapper(async (conn, driver, engine) => {
+      for (const sql of initSql) await driver.query(conn, sql);
+
+      await driver.script(
+        conn,
+        'INSERT INTO t1 (id) VALUES (3);INSERT INTO t1 (id) VALUES (4);UPDATE t1 SET id=10 WHERE id=1;DELETE FROM t1 WHERE id=2;'
+      );
+      const res = await driver.query(conn, 'SELECT COUNT(*) AS cnt FROM t1');
+      // console.log(res);
+      expect(res.rows[0].cnt == 3).toBeTruthy();
+    })
+  );
+});

integration-tests/__tests__/table-analyse.spec.js

@@ -0,0 +1,164 @@
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+
+const t1Sql = 'CREATE TABLE t1 (id int not null primary key, val1 varchar(50) null)';
+const ix1Sql = 'CREATE index ix1 ON t1(val1, id)';
+const t2Sql = 'CREATE TABLE t2 (id int not null primary key, val2 varchar(50) null unique)';
+const t3Sql = 'CREATE TABLE t3 (id int not null primary key, valfk int, foreign key (valfk) references t2(id))';
+// const fkSql = 'ALTER TABLE t3 ADD FOREIGN KEY (valfk) REFERENCES t2(id)'
+
+const txMatch = (tname, vcolname, nextcol) =>
+  expect.objectContaining({
+    pureName: tname,
+    columns: [
+      expect.objectContaining({
+        columnName: 'id',
+        notNull: true,
+        dataType: expect.stringContaining('int'),
+      }),
+      expect.objectContaining({
+        columnName: vcolname,
+        notNull: false,
+        dataType: expect.stringMatching(/.*char.*\(50\)/),
+      }),
+      ...(nextcol
+        ? [
+            expect.objectContaining({
+              columnName: 'nextcol',
+              notNull: false,
+              dataType: expect.stringMatching(/.*char.*\(50\)/),
+            }),
+          ]
+        : []),
+    ],
+    primaryKey: expect.objectContaining({
+      columns: [
+        expect.objectContaining({
+          columnName: 'id',
+        }),
+      ],
+    }),
+  });
+
+const t1Match = txMatch('t1', 'val1');
+const t2Match = txMatch('t2', 'val2');
+const t2NextColMatch = txMatch('t2', 'val2', true);
+
+describe('Table analyse', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table structure - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+
+      const structure = await driver.analyseFull(conn);
+
+      expect(structure.tables.length).toEqual(1);
+      expect(structure.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table add - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(1);
+      expect(structure1.tables[0]).toEqual(t2Match);
+
+      await driver.query(conn, t1Sql);
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table remove - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+      expect(structure1.tables.length).toEqual(2);
+      expect(structure1.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure1.tables.find(x => x.pureName == 't2')).toEqual(t2Match);
+
+      await driver.query(conn, 'DROP TABLE t2');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2.tables.length).toEqual(1);
+      expect(structure2.tables[0]).toEqual(t1Match);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table change - incremental analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, t2Sql);
+      const structure1 = await driver.analyseFull(conn);
+
+      if (engine.dbSnapshotBySeconds) await new Promise(resolve => setTimeout(resolve, 1100));
+
+      await driver.query(conn, 'ALTER TABLE t2 ADD nextcol varchar(50)');
+      const structure2 = await driver.analyseIncremental(conn, structure1);
+
+      expect(structure2).toBeTruthy(); // if falsy, no modification is detected
+
+      expect(structure2.tables.length).toEqual(2);
+      expect(structure2.tables.find(x => x.pureName == 't1')).toEqual(t1Match);
+      expect(structure2.tables.find(x => x.pureName == 't2')).toEqual(t2NextColMatch);
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Index - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t1Sql);
+      await driver.query(conn, ix1Sql);
+      const structure = await driver.analyseFull(conn);
+
+      const t1 = structure.tables.find(x => x.pureName == 't1');
+      expect(t1.indexes.length).toEqual(1);
+      expect(t1.indexes[0].columns.length).toEqual(2);
+      expect(t1.indexes[0].columns[0]).toEqual(expect.objectContaining({ columnName: 'val1' }));
+      expect(t1.indexes[0].columns[1]).toEqual(expect.objectContaining({ columnName: 'id' }));
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Unique - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+      const structure = await driver.analyseFull(conn);
+
+      const t2 = structure.tables.find(x => x.pureName == 't2');
+      // const indexesAndUniques = [...t2.uniques, ...t2.indexes];
+      expect(t2.uniques.length).toEqual(1);
+      expect(t2.uniques[0].columns.length).toEqual(1);
+      expect(t2.uniques[0].columns[0]).toEqual(expect.objectContaining({ columnName: 'val2' }));
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Foreign key - full analysis - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await driver.query(conn, t2Sql);
+      await driver.query(conn, t3Sql);
+      // await driver.query(conn, fkSql);
+
+      const structure = await driver.analyseFull(conn);
+
+      const t3 = structure.tables.find(x => x.pureName == 't3');
+      console.log('T3', t3.foreignKeys[0].columns);
+      expect(t3.foreignKeys.length).toEqual(1);
+      expect(t3.foreignKeys[0].columns.length).toEqual(1);
+      expect(t3.foreignKeys[0]).toEqual(expect.objectContaining({ refTableName: 't2' }));
+      expect(t3.foreignKeys[0].columns[0]).toEqual(
+        expect.objectContaining({ columnName: 'valfk', refColumnName: 'id' })
+      );
+    })
+  );
+});

integration-tests/__tests__/table-create.spec.js

@@ -0,0 +1,153 @@
+const _ = require('lodash');
+const fp = require('lodash/fp');
+const engines = require('../engines');
+const { testWrapper } = require('../tools');
+const { extendDatabaseInfo } = require('dbgate-tools');
+
+function createExpector(value) {
+  return _.cloneDeepWith(value, x => {
+    if (_.isPlainObject(x)) {
+      return expect.objectContaining(_.mapValues(x, y => createExpector(y)));
+    }
+  });
+}
+
+function omitTableSpecificInfo(table) {
+  return {
+    ...table,
+    columns: table.columns.map(fp.omit(['dataType'])),
+  };
+}
+
+function checkTableStructure2(t1, t2) {
+  // expect(t1.pureName).toEqual(t2.pureName)
+  expect(t2).toEqual(createExpector(omitTableSpecificInfo(t1)));
+}
+
+async function testTableCreate(conn, driver, table) {
+  await driver.query(conn, `create table t0 (id int not null primary key)`);
+
+  const dmp = driver.createDumper();
+  const table1 = {
+    ...table,
+    pureName: 'tested',
+  };
+  dmp.createTable(table1);
+
+  console.log('RUNNING CREATE SQL', driver.engine, ':', dmp.s);
+  await driver.script(conn, dmp.s);
+
+  const db = extendDatabaseInfo(await driver.analyseFull(conn));
+  const table2 = db.tables.find(x => x.pureName == 'tested');
+
+  checkTableStructure2(table1, table2);
+}
+
+describe('Table create', () => {
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Simple table - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with index - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        indexes: [
+          {
+            constraintName: 'ix1',
+            pureName: 'tested',
+            columns: [{ columnName: 'col2' }],
+          },
+        ],
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with foreign key - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        foreignKeys: [
+          {
+            pureName: 'tested',
+            refTableName: 't0',
+            columns: [{ columnName: 'col2', refColumnName: 'id' }],
+          },
+        ],
+      });
+    })
+  );
+
+  test.each(engines.map(engine => [engine.label, engine]))(
+    'Table with unique - %s',
+    testWrapper(async (conn, driver, engine) => {
+      await testTableCreate(conn, driver, {
+        columns: [
+          {
+            columnName: 'col1',
+            dataType: 'int',
+            notNull: true,
+          },
+          {
+            columnName: 'col2',
+            dataType: 'int',
+            notNull: true,
+          },
+        ],
+        primaryKey: {
+          columns: [{ columnName: 'col1' }],
+        },
+        uniques: [
+          {
+            pureName: 'tested',
+            columns: [{ columnName: 'col2' }],
+          },
+        ],
+      });
+    })
+  );
+});

integration-tests/docker-compose.yaml

@@ -0,0 +1,55 @@
+version: '3'
+services:
+  postgres:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: Pwd2020Db
+    ports: 
+      - 15000:5432
+
+  mysql:
+    image: mysql:8.0.18
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    ports: 
+      - 15001:3306
+    environment:
+      - MYSQL_ROOT_PASSWORD=Pwd2020Db
+
+  mssql:
+    image: mcr.microsoft.com/mssql/server
+    restart: always
+    ports: 
+      - 15002:1433
+    environment:
+      - ACCEPT_EULA=Y
+      - SA_PASSWORD=Pwd2020Db
+      - MSSQL_PID=Express
+      
+  cockroachdb:
+    image: cockroachdb/cockroach
+    ports:
+      - 15003:26257
+    command: start-single-node --insecure
+
+#   mongodb:
+#     image: mongo:4.0.12
+#     restart: always
+#     volumes:
+#       - mongo-data:/data/db
+#       - mongo-config:/data/configdb
+#     ports:
+#       - 27017:27017
+
+
+#   cockroachdb-init:
+#     image: cockroachdb/cockroach
+#     # build: cockroach
+#     # entrypoint: /cockroach/init.sh 
+#     entrypoint: ./cockroach sql --insecure --host="cockroachdb" --execute="CREATE DATABASE IF NOT EXISTS test;"
+
+#     depends_on:
+#       - cockroachdb
+#     restart: on-failure
+

integration-tests/engines.js

@@ -0,0 +1,128 @@
+const views = {
+  type: 'views',
+  create1: 'CREATE VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP VIEW obj1',
+  drop2: 'DROP VIEW obj2',
+};
+const matviews = {
+  type: 'matviews',
+  create1: 'CREATE MATERIALIZED VIEW obj1 AS SELECT id FROM t1',
+  create2: 'CREATE MATERIALIZED VIEW obj2 AS SELECT id FROM t2',
+  drop1: 'DROP MATERIALIZED VIEW obj1',
+  drop2: 'DROP MATERIALIZED VIEW obj2',
+};
+
+const engines = [
+  {
+    label: 'MySQL',
+    connection: {
+      engine: 'mysql@dbgate-plugin-mysql',
+      password: 'Pwd2020Db',
+      user: 'root',
+      server: 'mysql',
+      port: 3306,
+    },
+    local: {
+      server: 'localhost',
+      port: 15001,
+    },
+    // skipOnCI: true,
+    objects: [views],
+    dbSnapshotBySeconds: true,
+  },
+  {
+    label: 'PostgreSQL',
+    connection: {
+      engine: 'postgres@dbgate-plugin-postgres',
+      password: 'Pwd2020Db',
+      user: 'postgres',
+      server: 'postgres',
+      port: 5432,
+    },
+    local: {
+      server: 'localhost',
+      port: 15000,
+    },
+    objects: [
+      views,
+      matviews,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1() LANGUAGE SQL AS $$  select * from t1 $$',
+        create2: 'CREATE PROCEDURE obj2() LANGUAGE SQL AS $$  select * from t2 $$',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+      {
+        type: 'functions',
+        create1:
+          'CREATE FUNCTION obj1() returns int LANGUAGE plpgsql AS $$ declare  res integer; begin select count(*) into res from t1; return res; end; $$',
+        create2:
+          'CREATE FUNCTION obj2() returns int LANGUAGE plpgsql AS $$ declare res integer; begin select count(*) into res from t2; return res; end; $$',
+        drop1: 'DROP FUNCTION obj1',
+        drop2: 'DROP FUNCTION obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQL Server',
+    connection: {
+      engine: 'mssql@dbgate-plugin-mssql',
+      password: 'Pwd2020Db',
+      user: 'sa',
+      server: 'mssql',
+      port: 1433,
+    },
+    local: {
+      server: 'localhost',
+      port: 15002,
+    },
+    objects: [
+      views,
+      {
+        type: 'procedures',
+        create1: 'CREATE PROCEDURE obj1 AS SELECT id FROM t1',
+        create2: 'CREATE PROCEDURE obj2 AS SELECT id FROM t2',
+        drop1: 'DROP PROCEDURE obj1',
+        drop2: 'DROP PROCEDURE obj2',
+      },
+    ],
+  },
+  {
+    label: 'SQLite',
+    generateDbFile: true,
+    connection: {
+      engine: 'sqlite@dbgate-plugin-sqlite',
+    },
+    objects: [views],
+  },
+  {
+    label: 'CockroachDB',
+    connection: {
+      engine: 'cockroach@dbgate-plugin-postgres',
+      user: 'root',
+      server: 'cockroachdb',
+      port: 26257,
+    },
+    local: {
+      server: 'localhost',
+      port: 15003,
+    },
+    skipOnCI: true,
+    objects: [views, matviews],
+  },
+];
+
+const filterLocal = [
+  // filter local testing
+  'MySQL',
+  'PostgreSQL',
+  'SQL Server',
+  'SQLite',
+  'CockroachDB',
+];
+
+module.exports = process.env.CITEST
+  ? engines.filter(x => !x.skipOnCI)
+  : engines.filter(x => filterLocal.find(y => x.label == y));

integration-tests/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "dbgate-integration-tests",
+  "version": "4.1.1",
+  "homepage": "https://dbgate.org/",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbgate/dbgate.git"
+  },
+  "author": "Jan Prochazka",
+  "license": "MIT",
+  "scripts": {
+    "wait:local": "cross-env DEVMODE=1 LOCALTEST=1 node wait.js",
+    "wait:ci": "cross-env DEVMODE=1 CITEST=1 node wait.js",
+
+    "test:local": "cross-env DEVMODE=1 LOCALTEST=1 jest",
+    "test:ci": "cross-env DEVMODE=1 CITEST=1 jest --runInBand --json --outputFile=result.json --testLocationInResults",
+    
+    "run:local": "docker-compose down && docker-compose up -d && yarn wait:local && yarn test:local"
+  },
+  "jest": {
+    "testTimeout": 5000
+  },
+  "devDependencies": {
+    "cross-env": "^7.0.3",
+    "jest": "^27.0.1"
+  },
+  "dependencies": {}
+}

integration-tests/tools.js

@@ -0,0 +1,63 @@
+global.DBGATE_TOOLS = require('dbgate-tools');
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const crypto = require('crypto');
+
+function randomDbName() {
+  const generatedKey = crypto.randomBytes(6);
+  const newKey = generatedKey.toString('hex');
+  return `db${newKey}`;
+}
+
+function extractConnection(engine) {
+  const { connection } = engine;
+
+  if (process.env.LOCALTEST && engine.local) {
+    return {
+      ...connection,
+      ...engine.local,
+    };
+  }
+
+  return connection;
+}
+
+async function connect(engine, database) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+
+  if (engine.generateDbFile) {
+    const conn = await driver.connect({
+      ...connection,
+      databaseFile: `dbtemp/${database}`,
+    });
+    return conn;
+  } else {
+    const conn = await driver.connect(connection);
+    await driver.query(conn, `CREATE DATABASE ${database}`);
+    await driver.close(conn);
+
+    const res = await driver.connect({
+      ...connection,
+      database,
+    });
+    return res;
+  }
+}
+
+const testWrapper = body => async (label, ...other) => {
+  const engine = other[other.length - 1];
+  const driver = requireEngineDriver(engine.connection);
+  const conn = await connect(engine, randomDbName());
+  try {
+    await body(conn, driver, ...other);
+  } finally {
+    await driver.close(conn);
+  }
+};
+
+module.exports = {
+  randomDbName,
+  connect,
+  extractConnection,
+  testWrapper,
+};

integration-tests/wait.js

@@ -0,0 +1,29 @@
+const requireEngineDriver = require('dbgate-api/src/utility/requireEngineDriver');
+const engines = require('./engines');
+const { extractConnection } = require('./tools');
+global.DBGATE_TOOLS = require('dbgate-tools');
+
+async function connectEngine(engine) {
+  const connection = extractConnection(engine);
+  const driver = requireEngineDriver(connection);
+  for (;;) {
+    try {
+      const conn = await driver.connect(connection);
+      await driver.getVersion(conn);
+      console.log(`Connect to ${engine.label} - OK`);
+      await driver.close(conn);
+      return;
+    } catch (err) {
+      console.log(`Waiting for ${engine.label}, error: ${err.message}`);
+      await new Promise(resolve => setTimeout(resolve, 2500));
+      continue;
+    }
+  }
+}
+
+async function run() {
+  await new Promise(resolve => setTimeout(resolve, 10000));
+  await Promise.all(engines.map(engine => connectEngine(engine)));
+}
+
+run();

package.json

@@ -1,41 +1,52 @@
 {
   "private": true,
+  "version": "4.3.1-beta.2",
   "name": "dbgate-all",
   "workspaces": [
-    "packages/*"
+    "packages/*",
+    "plugins/*",
+    "integration-tests"
   ],
   "scripts": {
     "start:api": "yarn workspace dbgate-api start",
     "start:api:portal": "yarn workspace dbgate-api start:portal",
     "start:api:covid": "yarn workspace dbgate-api start:covid",
-    "start:web": "yarn workspace dbgate-web start",
+    "start:web": "yarn workspace dbgate-web dev",
     "start:sqltree": "yarn workspace dbgate-sqltree start",
     "start:tools": "yarn workspace dbgate-tools start",
     "start:datalib": "yarn workspace dbgate-datalib start",
     "start:filterparser": "yarn workspace dbgate-filterparser start",
+    "start:querysplitter": "yarn workspace dbgate-query-splitter start",
     "build:sqltree": "yarn workspace dbgate-sqltree build",
     "build:datalib": "yarn workspace dbgate-datalib build",
     "build:filterparser": "yarn workspace dbgate-filterparser build",
+    "build:querysplitter": "yarn workspace dbgate-query-splitter build",
     "build:tools": "yarn workspace dbgate-tools build",
-    "build:lib": "yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
-    "build:app": "cd app && yarn install && yarn build",
+    "build:lib": "yarn build:querysplitter && yarn build:tools && yarn build:sqltree && yarn build:filterparser && yarn build:datalib",
+    "build:app": "yarn plugins:copydist && cd app && yarn install && yarn build",
+    "build:app:mac": "yarn plugins:copydist && cd app && yarn install && yarn build:mac",
     "build:api": "yarn workspace dbgate-api build",
-    "build:web:docker": "yarn workspace dbgate-web build:docker",
-    "build:app:local": "cd app && yarn build:local",
+    "build:web:docker": "yarn workspace dbgate-web build",
+    "build:plugins:frontend": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn build:frontend",
+    "build:plugins:frontend:watch": "workspaces-run --parallel --only=\"dbgate-plugin-*\" -- yarn build:frontend:watch",
+    "plugins:copydist": "workspaces-run --only=\"dbgate-plugin-*\" -- yarn copydist",
+    "build:app:local": "yarn plugins:copydist && cd app && yarn build:local",
     "start:app:local": "cd app && yarn start:local",
     "setCurrentVersion": "node setCurrentVersion",
     "generatePadFile": "node generatePadFile",
-
-    "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/build/* docker -u 2 && copyfiles \"packages/web/build/**/*\" docker -u 2",
-    "prepare:docker": "yarn build:web:docker && yarn build:api && yarn copy:docker:build",
-
-    "prepare": "yarn build:lib",
+    "fillNativeModules": "node fillNativeModules",
+    "fillNativeModulesElectron": "node fillNativeModules --electron",
+    "fillPackagedPlugins": "node fillPackagedPlugins",
+    "resetPackagedPlugins": "node resetPackagedPlugins",
+    "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src",
+    "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2",
+    "prepare:docker": "yarn plugins:copydist && yarn build:web:docker && yarn build:api && yarn copy:docker:build",
     "start": "concurrently --kill-others-on-fail \"yarn start:api\" \"yarn start:web\"",
-    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\"",
+    "lib": "concurrently --kill-others-on-fail \"yarn start:sqltree\" \"yarn start:filterparser\" \"yarn start:datalib\" \"yarn start:tools\" \"yarn start:querysplitter\" \"yarn build:plugins:frontend:watch\"",
     "ts:api": "yarn workspace dbgate-api ts",
     "ts:web": "yarn workspace dbgate-web ts",
     "ts": "yarn ts:api && yarn ts:web",
-    "postinstall": "patch-package"
+    "postinstall": "yarn resetPackagedPlugins && yarn build:lib && patch-package && yarn fillNativeModules && yarn build:plugins:frontend"
   },
   "dependencies": {
     "concurrently": "^5.1.0",
@@ -43,6 +54,8 @@
     "socket.io": "^2.3.0"
   },
   "devDependencies": {
-    "copyfiles": "^2.2.0"
+    "copyfiles": "^2.2.0",
+    "prettier": "^2.2.1",
+    "workspaces-run": "^1.0.1"
   }
 }

packages/api/.covid-env

@@ -1,12 +0,0 @@
-CONNECTIONS=mysql
-
-LABEL_mysql=MySql
-SERVER_mysql=dbgate.org
-USER_mysql=reader
-PASSWORD_mysql=CovidReader2020
-PORT_mysql=3326
-ENGINE_mysql=mysql@dbgate-plugin-mysql
-SINGLE_CONNECTION=mysql
-SINGLE_DATABASE=covid
-
-PERMISSIONS=files/charts/read

packages/api/.env

@@ -1,23 +1 @@
-CONNECTIONS=mysql,postgres
-
-LABEL_mysql=MySql localhost
-SERVER_mysql=localhost
-USER_mysql=root
-PASSWORD_mysql=test
-PORT_mysql=3307
-ENGINE_mysql=mysql
-
-LABEL_postgres=Postgres localhost
-SERVER_postgres=localhost
-USER_postgres=postgres
-PASSWORD_postgres=test
-PORT_postgres=5433
-ENGINE_postgres=postgres
-
-TOOLBAR=home
-ICON_home=mdi mdi-home
-TITLE_home=Home
-PAGE_home=home.html
-STARTUP_PAGES=home
-
-PAGES_DIRECTORY=/home/jena/jenasoft/dbgate-web/pages
+DEVMODE=1

packages/api/.env-portal

@@ -0,0 +1,17 @@
+DEVMODE=1
+
+CONNECTIONS=mysql,postgres
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+LABEL_postgres=Postgres localhost
+SERVER_postgres=localhost
+USER_postgres=postgres
+PASSWORD_postgres=test
+PORT_postgres=5433
+ENGINE_postgres=postgres@dbgate-plugin-postgres

packages/api/.env-singledb

@@ -0,0 +1,15 @@
+DEVMODE=1
+
+CONNECTIONS=mysql
+
+LABEL_mysql=MySql localhost
+SERVER_mysql=localhost
+USER_mysql=root
+PASSWORD_mysql=test
+PORT_mysql=3307
+ENGINE_mysql=mysql@dbgate-plugin-mysql
+
+SINGLE_CONNECTION=mysql
+SINGLE_DATABASE=Chinook
+
+PERMISSIONS=files/charts/read

packages/api/README.md

@@ -32,7 +32,7 @@ dbgateApi.runScript(run);
 
 ```
 
-Silly example, runs without any dependencies. Copy [fakeObjectReader](https://github.com/dbshell/dbgate/blob/master/packages/api/src/shell/fakeObjectReader.js) to [consoleObjectWriter](https://github.com/dbshell/dbgate/blob/master/packages/api/src/shell/consoleObjectWriter.js) .
+Silly example, runs without any dependencies. Copy [fakeObjectReader](https://github.com/dbgate/dbgate/blob/master/packages/api/src/shell/fakeObjectReader.js) to [consoleObjectWriter](https://github.com/dbgate/dbgate/blob/master/packages/api/src/shell/consoleObjectWriter.js) .
 
 ```javascript
 

packages/api/package.json

@@ -1,15 +1,14 @@
 {
   "name": "dbgate-api",
   "main": "src/index.js",
-  "version": "1.0.7",
+  "version": "4.1.1",
   "homepage": "https://dbgate.org/",
   "repository": {
     "type": "git",
-    "url": "https://github.com/dbshell/dbgate.git"
+    "url": "https://github.com/dbgate/dbgate.git"
   },
-  "funding": "https://www.paypal.com/paypalme/JanProchazkaCz/30eur",
   "author": "Jan Prochazka",
-  "license": "GPL",
+  "license": "MIT",
   "keywords": [
     "sql",
     "json",
@@ -19,43 +18,58 @@
   ],
   "dependencies": {
     "async-lock": "^1.2.4",
-    "axios": "^0.19.0",
+    "axios": "^0.21.1",
+    "better-sqlite3-with-prebuilds": "^7.1.8",
     "body-parser": "^1.19.0",
     "bufferutil": "^4.0.1",
     "byline": "^5.0.0",
+    "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-sqltree": "^1.0.0",
-    "dbgate-tools": "^1.0.0",
+    "dbgate-sqltree": "^4.1.1",
+    "dbgate-tools": "^4.1.1",
     "eslint": "^6.8.0",
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
     "express-fileupload": "^1.2.0",
-    "find-free-port": "^2.0.0",
-    "fs-extra": "^8.1.0",
+    "fs-extra": "^9.1.0",
+    "fs-reverse": "^0.0.3",
+    "get-port": "^5.1.1",
     "http": "^0.0.0",
+    "json-stable-stringify": "^1.0.1",
     "line-reader": "^0.4.0",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.21",
     "ncp": "^2.0.0",
     "nedb-promises": "^4.0.1",
     "node-cron": "^2.0.3",
+    "node-ssh-forward": "^0.7.2",
+    "portfinder": "^1.0.28",
+    "simple-encryptor": "^4.0.0",
+    "socket.io": "^2.3.0",
     "tar": "^6.0.5",
     "uuid": "^3.4.0"
   },
   "scripts": {
-    "start": "nodemon src/index.js",
-    "start:portal": "env-cmd nodemon src/index.js",
-    "start:covid": "env-cmd -f .covid-env nodemon src/index.js",
+    "start": "env-cmd node src/index.js",
+    "start:portal": "env-cmd -f .env-portal node src/index.js",
+    "start:singledb": "env-cmd -f .env-singledb node src/index.js",
+    "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db",
+    "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test",
     "ts": "tsc",
     "build": "webpack"
   },
   "devDependencies": {
+    "@types/fs-extra": "^9.0.11",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "^1.0.0",
+    "dbgate-types": "^4.1.1",
     "env-cmd": "^10.1.0",
+    "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",
-    "typescript": "^3.7.4",
+    "typescript": "^4.4.3",
     "webpack": "^4.42.0",
     "webpack-cli": "^3.3.11"
+  },
+  "optionalDependencies": {
+    "msnodesqlv8": "^2.0.10"
   }
 }

packages/api/src/controllers/archive.js

@@ -18,8 +18,8 @@ module.exports = {
         type: 'jsonl',
       },
       ...folders
-        .filter((x) => x != 'default')
-        .map((name) => ({
+        .filter(x => x != 'default')
+        .map(name => ({
           name,
           type: 'jsonl',
         })),
@@ -39,8 +39,8 @@ module.exports = {
     if (!(await fs.exists(dir))) return [];
     const files = await fs.readdir(dir);
     return files
-      .filter((name) => name.endsWith('.jsonl'))
-      .map((name) => ({
+      .filter(name => name.endsWith('.jsonl'))
+      .map(name => ({
         name: name.slice(0, -'.jsonl'.length),
         type: 'jsonl',
       }));
@@ -84,7 +84,7 @@ module.exports = {
       });
       let structure = null;
       const rows = [];
-      liner.on('line', (line) => {
+      liner.on('line', line => {
         const data = JSON.parse(line);
         if (structure) rows.push(data);
         else structure = data;

packages/api/src/controllers/config.js

@@ -1,34 +1,61 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { datadir } = require('../utility/directories');
+const hasPermission = require('../utility/hasPermission');
+const socket = require('../utility/socket');
+const _ = require('lodash');
+
 const currentVersion = require('../currentVersion');
+const platformInfo = require('../utility/platformInfo');
+const connections = require('../controllers/connections');
 
 module.exports = {
+  settingsValue: {},
+
+  async _init() {
+    try {
+      this.settingsValue = JSON.parse(await fs.readFile(path.join(datadir(), 'settings.json'), { encoding: 'utf-8' }));
+    } catch (err) {
+      this.settingsValue = {};
+    }
+  },
+
   get_meta: 'get',
   async get() {
-    // const toolbarButtons = process.env.TOOLBAR;
-    // const toolbar = toolbarButtons
-    //   ? toolbarButtons.split(',').map((name) => ({
-    //       name,
-    //       icon: process.env[`ICON_${name}`],
-    //       title: process.env[`TITLE_${name}`],
-    //       page: process.env[`PAGE_${name}`],
-    //     }))
-    //   : null;
-    // const startupPages = process.env.STARTUP_PAGES ? process.env.STARTUP_PAGES.split(',') : [];
     const permissions = process.env.PERMISSIONS ? process.env.PERMISSIONS.split(',') : null;
-    const singleDatabase =
-      process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE
-        ? {
-            conid: process.env.SINGLE_CONNECTION,
-            database: process.env.SINGLE_DATABASE,
-          }
-        : null;
 
     return {
-      runAsPortal: !!process.env.CONNECTIONS,
-      // toolbar,
-      // startupPages,
-      singleDatabase,
+      runAsPortal: !!connections.portalConnections,
+      singleDatabase: connections.singleDatabase,
       permissions,
       ...currentVersion,
     };
   },
+
+  platformInfo_meta: 'get',
+  async platformInfo() {
+    return platformInfo;
+  },
+
+  getSettings_meta: 'get',
+  async getSettings() {
+    return this.settingsValue;
+  },
+
+  updateSettings_meta: 'post',
+  async updateSettings(values) {
+    if (!hasPermission(`settings/change`)) return false;
+    try {
+      const updated = {
+        ...this.settingsValue,
+        ...values,
+      };
+      await fs.writeFile(path.join(datadir(), 'settings.json'), JSON.stringify(updated, undefined, 2));
+      this.settingsValue = updated;
+      socket.emitChanged(`settings-changed`);
+      return updated;
+    } catch (err) {
+      return false;
+    }
+  },
 };

packages/api/src/controllers/connections.js

@@ -5,26 +5,117 @@ const nedb = require('nedb-promises');
 
 const { datadir } = require('../utility/directories');
 const socket = require('../utility/socket');
+const { encryptConnection } = require('../utility/crypting');
+const { handleProcessCommunication } = require('../utility/processComm');
+
+function getNamedArgs() {
+  const res = {};
+  for (let i = 0; i < process.argv.length; i++) {
+    const name = process.argv[i];
+    if (name.startsWith('--')) {
+      let value = process.argv[i + 1];
+      if (value && value.startsWith('--')) value = null;
+      res[name.substring(2)] = value == null ? true : value;
+      i++;
+    } else {
+      if (name.endsWith('.db') || name.endsWith('.sqlite') || name.endsWith('.sqlite3')) {
+        res.databaseFile = name;
+        res.engine = 'sqlite@dbgate-plugin-sqlite';
+      }
+    }
+  }
+  return res;
+}
+
+function getDatabaseFileLabel(databaseFile) {
+  if (!databaseFile) return databaseFile;
+  const m = databaseFile.match(/[\/]([^\/]+)$/);
+  if (m) return m[1];
+  return databaseFile;
+}
 
 function getPortalCollections() {
   if (process.env.CONNECTIONS) {
-    return _.compact(process.env.CONNECTIONS.split(',')).map((id) => ({
+    return _.compact(process.env.CONNECTIONS.split(',')).map(id => ({
       _id: id,
       engine: process.env[`ENGINE_${id}`],
       server: process.env[`SERVER_${id}`],
       user: process.env[`USER_${id}`],
       password: process.env[`PASSWORD_${id}`],
       port: process.env[`PORT_${id}`],
+      databaseUrl: process.env[`URL_${id}`],
+      databaseFile: process.env[`FILE_${id}`],
+      defaultDatabase: process.env[`DATABASE_${id}`],
+      singleDatabase: !!process.env[`DATABASE_${id}`],
       displayName: process.env[`LABEL_${id}`],
     }));
   }
+
+  const args = getNamedArgs();
+  if (args.databaseFile) {
+    return [
+      {
+        _id: 'argv',
+        databaseFile: args.databaseFile,
+        singleDatabase: true,
+        defaultDatabase: getDatabaseFileLabel(args.databaseFile),
+        engine: args.engine,
+      },
+    ];
+  }
+  if (args.databaseUrl) {
+    return [
+      {
+        _id: 'argv',
+        useDatabaseUrl: true,
+        ...args,
+      },
+    ];
+  }
+  if (args.server) {
+    return [
+      {
+        _id: 'argv',
+        ...args,
+      },
+    ];
+  }
+
   return null;
 }
 const portalConnections = getPortalCollections();
 
+function getSingleDatabase() {
+  if (process.env.SINGLE_CONNECTION && process.env.SINGLE_DATABASE) {
+    // @ts-ignore
+    const connection = portalConnections.find(x => x._id == process.env.SINGLE_CONNECTION);
+    return {
+      connection,
+      name: process.env.SINGLE_DATABASE,
+    };
+  }
+  // @ts-ignore
+  const arg0 = (portalConnections || []).find(x => x._id == 'argv');
+  if (arg0) {
+    // @ts-ignore
+    if (arg0.singleDatabase) {
+      return {
+        connection: arg0,
+        // @ts-ignore
+        name: arg0.defaultDatabase,
+      };
+    }
+  }
+  return null;
+}
+
+const singleDatabase = getSingleDatabase();
+
 module.exports = {
   datastore: null,
   opened: [],
+  singleDatabase,
+  portalConnections,
 
   async _init() {
     const dir = datadir();
@@ -44,8 +135,9 @@ module.exports = {
     raw: true,
   },
   test(req, res) {
-    const subprocess = fork(process.argv[1], ['connectProcess']);
-    subprocess.on('message', (resp) => {
+    const subprocess = fork(process.argv[1], ['--start-process', 'connectProcess', ...process.argv.slice(3)]);
+    subprocess.on('message', resp => {
+      if (handleProcessCommunication(resp, subprocess)) return;
       // @ts-ignore
       const { msgtype } = resp;
       if (msgtype == 'connected' || msgtype == 'error') {
@@ -59,10 +151,11 @@ module.exports = {
   async save(connection) {
     if (portalConnections) return;
     let res;
+    const encrypted = encryptConnection(connection);
     if (connection._id) {
-      res = await this.datastore.update(_.pick(connection, '_id'), connection);
+      res = await this.datastore.update(_.pick(connection, '_id'), encrypted);
     } else {
-      res = await this.datastore.insert(connection);
+      res = await this.datastore.insert(encrypted);
     }
     socket.emitChanged('connection-list-changed');
     return res;
@@ -78,7 +171,7 @@ module.exports = {
 
   get_meta: 'get',
   async get({ conid }) {
-    if (portalConnections) return portalConnections.find((x) => x._id == conid);
+    if (portalConnections) return portalConnections.find(x => x._id == conid);
     const res = await this.datastore.find({ _id: conid });
     return res[0];
   },

packages/api/src/controllers/databaseConnections.js

@@ -3,6 +3,8 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const { DatabaseAnalyser } = require('dbgate-tools');
+const { handleProcessCommunication } = require('../utility/processComm');
+const config = require('./config');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedDatabaseConnection[]} */
@@ -11,11 +13,24 @@ module.exports = {
   requests: {},
 
   handle_structure(conid, database, { structure }) {
-    const existing = this.opened.find((x) => x.conid == conid && x.database == database);
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
     existing.structure = structure;
     socket.emitChanged(`database-structure-changed-${conid}-${database}`);
   },
+  handle_structureTime(conid, database, { analysedTime }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.analysedTime = analysedTime;
+    socket.emitChanged(`database-status-changed-${conid}-${database}`);
+  },
+  handle_version(conid, database, { version }) {
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (!existing) return;
+    existing.serverVersion = version;
+    socket.emitChanged(`database-server-version-changed-${conid}-${database}`);
+  },
+
   handle_error(conid, database, props) {
     const { error } = props;
     console.log(`Error in database connection ${conid}, database ${database}: ${error}`);
@@ -26,7 +41,7 @@ module.exports = {
     delete this.requests[msgid];
   },
   handle_status(conid, database, { status }) {
-    const existing = this.opened.find((x) => x.conid == conid && x.database == database);
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (!existing) return;
     if (existing.status == status) return;
     existing.status = status;
@@ -36,22 +51,29 @@ module.exports = {
   handle_ping() {},
 
   async ensureOpened(conid, database) {
-    const existing = this.opened.find((x) => x.conid == conid && x.database == database);
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) return existing;
     const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['databaseConnectionProcess']);
+    const subprocess = fork(process.argv[1], [
+      '--start-process',
+      'databaseConnectionProcess',
+      ...process.argv.slice(3),
+    ]);
     const lastClosed = this.closed[`${conid}/${database}`];
     const newOpened = {
       conid,
       database,
       subprocess,
       structure: lastClosed ? lastClosed.structure : DatabaseAnalyser.createEmptyStructure(),
+      serverVersion: lastClosed ? lastClosed.serverVersion : null,
       connection,
       status: { name: 'pending' },
     };
     this.opened.push(newOpened);
-    // @ts-ignore
-    subprocess.on('message', ({ msgtype, ...message }) => {
+    subprocess.on('message', message => {
+      // @ts-ignore
+      const { msgtype } = message;
+      if (handleProcessCommunication(message, subprocess)) return;
       if (newOpened.disconnected) return;
       this[`handle_${msgtype}`](conid, database, message);
     });
@@ -64,6 +86,7 @@ module.exports = {
       msgtype: 'connect',
       connection: { ...connection, database },
       structure: lastClosed ? lastClosed.structure : null,
+      globalSettings: config.settingsValue,
     });
     return newOpened;
   },
@@ -82,16 +105,51 @@ module.exports = {
   async queryData({ conid, database, sql }) {
     console.log(`Processing query, conid=${conid}, database=${database}, sql=${sql}`);
     const opened = await this.ensureOpened(conid, database);
+    // if (opened && opened.status && opened.status.name == 'error') {
+    //   return opened.status;
+    // }
     const res = await this.sendRequest(opened, { msgtype: 'queryData', sql });
     return res;
   },
 
+  runScript_meta: 'post',
+  async runScript({ conid, database, sql }) {
+    console.log(`Processing script, conid=${conid}, database=${database}, sql=${sql}`);
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'runScript', sql });
+    return res;
+  },
+
+  collectionData_meta: 'post',
+  async collectionData({ conid, database, options }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'collectionData', options });
+    return res.result;
+  },
+
+  updateCollection_meta: 'post',
+  async updateCollection({ conid, database, changeSet }) {
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet });
+    return res.result;
+  },
+
   status_meta: 'get',
   async status({ conid, database }) {
-    const existing = this.opened.find((x) => x.conid == conid && x.database == database);
-    if (existing) return existing.status;
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
+    if (existing) {
+      return {
+        ...existing.status,
+        analysedTime: existing.analysedTime,
+      };
+    }
     const lastClosed = this.closed[`${conid}/${database}`];
-    if (lastClosed) return lastClosed.status;
+    if (lastClosed) {
+      return {
+        ...lastClosed.status,
+        analysedTime: lastClosed.analysedTime,
+      };
+    }
     return {
       name: 'error',
       message: 'Not connected',
@@ -100,27 +158,41 @@ module.exports = {
 
   ping_meta: 'post',
   async ping({ conid, database }) {
-    const existing = this.opened.find((x) => x.conid == conid && x.database == database);
+    let existing = this.opened.find(x => x.conid == conid && x.database == database);
+
     if (existing) {
       existing.subprocess.send({ msgtype: 'ping' });
+    } else {
+      existing = await this.ensureOpened(conid, database);
     }
-    return { status: 'ok' };
+
+    return {
+      status: 'ok',
+      connectionStatus: existing ? existing.status : null,
+    };
   },
 
   refresh_meta: 'post',
-  async refresh({ conid, database }) {
-    this.close(conid, database);
+  async refresh({ conid, database, keepOpen }) {
+    if (!keepOpen) this.close(conid, database);
 
     await this.ensureOpened(conid, database);
     return { status: 'ok' };
   },
 
+  syncModel_meta: 'post',
+  async syncModel({ conid, database }) {
+    const conn = await this.ensureOpened(conid, database);
+    conn.subprocess.send({ msgtype: 'syncModel' });
+    return { status: 'ok' };
+  },
+
   close(conid, database, kill = true) {
-    const existing = this.opened.find((x) => x.conid == conid && x.database == database);
+    const existing = this.opened.find(x => x.conid == conid && x.database == database);
     if (existing) {
       existing.disconnected = true;
       if (kill) existing.subprocess.kill();
-      this.opened = this.opened.filter((x) => x.conid != conid || x.database != database);
+      this.opened = this.opened.filter(x => x.conid != conid || x.database != database);
       this.closed[`${conid}/${database}`] = {
         status: {
           ...existing.status,
@@ -132,6 +204,12 @@ module.exports = {
     }
   },
 
+  disconnect_meta: 'post',
+  async disconnect({ conid, database }) {
+    await this.close(conid, database, true);
+    return { status: 'ok' };
+  },
+
   structure_meta: 'get',
   async structure({ conid, database }) {
     const opened = await this.ensureOpened(conid, database);
@@ -144,6 +222,22 @@ module.exports = {
     // };
   },
 
+  serverVersion_meta: 'get',
+  async serverVersion({ conid, database }) {
+    const opened = await this.ensureOpened(conid, database);
+    return opened.serverVersion;
+  },
+
+  sqlPreview_meta: 'post',
+  async sqlPreview({ conid, database, objects, options }) {
+    // wait for structure
+    await this.structure({ conid, database });
+
+    const opened = await this.ensureOpened(conid, database);
+    const res = await this.sendRequest(opened, { msgtype: 'sqlPreview', objects, options });
+    return res;
+  },
+
   // runCommand_meta: 'post',
   // async runCommand({ conid, database, sql }) {
   //   console.log(`Running SQL command , conid=${conid}, database=${database}, sql=${sql}`);

packages/api/src/controllers/files.js

@@ -23,7 +23,7 @@ module.exports = {
     if (!hasPermission(`files/${folder}/read`)) return [];
     const dir = path.join(filesdir(), folder);
     if (!(await fs.exists(dir))) return [];
-    const files = (await fs.readdir(dir)).map((file) => ({ folder, file }));
+    const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
     return files;
   },
 
@@ -34,7 +34,7 @@ module.exports = {
     for (const folder of folders) {
       if (!hasPermission(`files/${folder}/read`)) continue;
       const dir = path.join(filesdir(), folder);
-      const files = (await fs.readdir(dir)).map((file) => ({ folder, file }));
+      const files = (await fs.readdir(dir)).map(file => ({ folder, file }));
       res.push(...files);
     }
     return res;
@@ -78,6 +78,11 @@ module.exports = {
     }
   },
 
+  saveAs_meta: 'post',
+  async saveAs({ filePath, data, format }) {
+    await fs.writeFile(filePath, serialize(format, data));
+  },
+
   favorites_meta: 'get',
   async favorites() {
     if (!hasPermission(`files/favorites/read`)) return [];

packages/api/src/controllers/jsldata.js

@@ -121,7 +121,13 @@ module.exports = {
   getStats_meta: 'get',
   getStats({ jslid }) {
     const file = `${getJslFileName(jslid)}.stats`;
-    if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, 'utf-8'));
+    if (fs.existsSync(file)) {
+      try {
+        return JSON.parse(fs.readFileSync(file, 'utf-8'));
+      } catch (e) {
+        return {};
+      }
+    }
     return {};
   },
 

packages/api/src/controllers/metadata.js

@@ -3,7 +3,7 @@ const fp = require('lodash/fp');
 const databaseConnections = require('./databaseConnections');
 
 function pickObjectNames(array) {
-  return _.sortBy(array, (x) => `${x.schemaName}.${x.pureName}`).map(fp.pick(['pureName', 'schemaName']));
+  return _.sortBy(array, x => `${x.schemaName}.${x.pureName}`).map(fp.pick(['pureName', 'schemaName']));
 }
 
 module.exports = {
@@ -17,7 +17,7 @@ module.exports = {
   listObjects_meta: 'get',
   async listObjects({ conid, database }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
-    const types = ['tables', 'views', 'procedures', 'functions', 'triggers'];
+    const types = ['tables', 'collections', 'views', 'procedures', 'functions', 'triggers'];
     return types.reduce(
       (res, type) => ({
         ...res,
@@ -30,18 +30,18 @@ module.exports = {
   tableInfo_meta: 'get',
   async tableInfo({ conid, database, schemaName, pureName }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
-    const table = opened.structure.tables.find((x) => x.pureName == pureName && x.schemaName == schemaName);
-    const allForeignKeys = _.flatten(opened.structure.tables.map((x) => x.foreignKeys));
+    const table = opened.structure.tables.find(x => x.pureName == pureName && x.schemaName == schemaName);
+    const allForeignKeys = _.flatten(opened.structure.tables.map(x => x.foreignKeys));
     return {
       ...table,
-      dependencies: allForeignKeys.filter((x) => x.refSchemaName == schemaName && x.refTableName == pureName),
+      dependencies: allForeignKeys.filter(x => x.refSchemaName == schemaName && x.refTableName == pureName),
     };
   },
 
   sqlObjectInfo_meta: 'get',
   async sqlObjectInfo({ objectTypeField, conid, database, schemaName, pureName }) {
     const opened = await databaseConnections.ensureOpened(conid, database);
-    const res = opened.structure[objectTypeField].find((x) => x.pureName == pureName && x.schemaName == schemaName);
+    const res = opened.structure[objectTypeField].find(x => x.pureName == pureName && x.schemaName == schemaName);
     return res;
   },
 };

packages/api/src/controllers/plugins.js

@@ -1,42 +1,29 @@
 const fs = require('fs-extra');
 const axios = require('axios');
 const path = require('path');
-const { pluginsdir, datadir } = require('../utility/directories');
+const { extractPackageName } = require('dbgate-tools');
+const { pluginsdir, packagedPluginsDir } = require('../utility/directories');
 const socket = require('../utility/socket');
+const compareVersions = require('compare-versions');
 const requirePlugin = require('../shell/requirePlugin');
 const downloadPackage = require('../utility/downloadPackage');
 const hasPermission = require('../utility/hasPermission');
-
-// async function loadPackageInfo(dir) {
-//   const readmeFile = path.join(dir, 'README.md');
-//   const packageFile = path.join(dir, 'package.json');
-
-//   if (!(await fs.exists(packageFile))) {
-//     return null;
-//   }
-
-//   let readme = null;
-//   let manifest = null;
-//   if (await fs.exists(readmeFile)) readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
-//   if (await fs.exists(packageFile)) manifest = JSON.parse(await fs.readFile(packageFile, { encoding: 'utf-8' }));
-//   return {
-//     readme,
-//     manifest,
-//   };
-// }
-
-const preinstallPlugins = [
-  'dbgate-plugin-mssql',
-  'dbgate-plugin-mysql',
-  'dbgate-plugin-postgres',
-  'dbgate-plugin-csv',
-  'dbgate-plugin-excel',
-];
+const _ = require('lodash');
+const packagedPluginsContent = require('../packagedPluginsContent');
 
 module.exports = {
   script_meta: 'get',
   async script({ packageName }) {
-    const file = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
+    const packagedContent = packagedPluginsContent();
+
+    if (packagedContent && packagedContent[packageName]) {
+      return packagedContent[packageName].frontend;
+    }
+
+    const file1 = path.join(packagedPluginsDir(), packageName, 'dist', 'frontend.js');
+    const file2 = path.join(pluginsdir(), packageName, 'dist', 'frontend.js');
+    // @ts-ignore
+    const file = (await fs.exists(file1)) ? file1 : file2;
     const data = await fs.readFile(file, {
       encoding: 'utf-8',
     });
@@ -50,7 +37,7 @@ module.exports = {
       `http://registry.npmjs.com/-/v1/search?text=${encodeURIComponent(filter)}+keywords:dbgateplugin&size=25&from=0`
     );
     const { objects } = resp.data || {};
-    return (objects || []).map((x) => x.package);
+    return (objects || []).map(x => x.package);
   },
 
   info_meta: 'get',
@@ -60,10 +47,13 @@ module.exports = {
       const { latest } = infoResp.data['dist-tags'];
       const manifest = infoResp.data.versions[latest];
       const { readme } = infoResp.data;
+      // @ts-ignore
+      const isPackaged = await fs.exists(path.join(packagedPluginsDir(), packageName));
 
       return {
         readme,
         manifest,
+        isPackaged,
       };
     } catch (err) {
       return {
@@ -71,48 +61,63 @@ module.exports = {
         error: err.message,
       };
     }
-
-    // const dir = path.join(pluginstmpdir(), packageName);
-    // if (!(await fs.exists(dir))) {
-    //   await downloadPackage(packageName, dir);
-    // }
-    // return await loadPackageInfo(dir);
-    // return await {
-    //   ...loadPackageInfo(dir),
-    //   installed: loadPackageInfo(path.join(pluginsdir(), packageName)),
-    // };
   },
 
   installed_meta: 'get',
   async installed() {
-    const files = await fs.readdir(pluginsdir());
+    const packagedContent = packagedPluginsContent();
+
+    const files1 = packagedContent ? _.keys(packagedContent) : await fs.readdir(packagedPluginsDir());
+    const files2 = await fs.readdir(pluginsdir());
+
     const res = [];
-    for (const packageName of files) {
-      const manifest = await fs
-        .readFile(path.join(pluginsdir(), packageName, 'package.json'))
-        .then((x) => JSON.parse(x));
-      const readmeFile = path.join(pluginsdir(), packageName, 'README.md');
-      if (await fs.exists(readmeFile)) {
-        manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+    for (const packageName of _.union(files1, files2)) {
+      if (!/^dbgate-plugin-.*$/.test(packageName)) continue;
+      try {
+        if (packagedContent && packagedContent[packageName]) {
+          const manifest = {
+            ...packagedContent[packageName].manifest,
+          };
+          manifest.isPackaged = true;
+          manifest.readme = packagedContent[packageName].readme;
+          res.push(manifest);
+        } else {
+          const isPackaged = files1.includes(packageName);
+          const manifest = await fs
+            .readFile(path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'package.json'), {
+              encoding: 'utf-8',
+            })
+            .then(x => JSON.parse(x));
+          const readmeFile = path.join(isPackaged ? packagedPluginsDir() : pluginsdir(), packageName, 'README.md');
+          // @ts-ignore
+          if (await fs.exists(readmeFile)) {
+            manifest.readme = await fs.readFile(readmeFile, { encoding: 'utf-8' });
+          }
+          manifest.isPackaged = isPackaged;
+          res.push(manifest);
+        }
+      } catch (err) {
+        console.log(`Skipped plugin ${packageName}, error:`, err.message);
       }
-      res.push(manifest);
     }
     return res;
-    // const res = await Promise.all(
-    //   files.map((packageName) =>
-    //     fs.readFile(path.join(pluginsdir(), packageName, 'package.json')).then((x) => JSON.parse(x))
-    //   )
-    // );
   },
 
+  // async saveRemovePlugins() {
+  //   await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
+  // },
+
   install_meta: 'post',
   async install({ packageName }) {
     if (!hasPermission(`plugins/install`)) return;
     const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
     if (!(await fs.exists(dir))) {
       await downloadPackage(packageName, dir);
     }
     socket.emitChanged(`installed-plugins-changed`);
+    // this.removedPlugins = this.removedPlugins.filter(x => x != packageName);
+    // await this.saveRemovePlugins();
   },
 
   uninstall_meta: 'post',
@@ -121,8 +126,21 @@ module.exports = {
     const dir = path.join(pluginsdir(), packageName);
     await fs.rmdir(dir, { recursive: true });
     socket.emitChanged(`installed-plugins-changed`);
-    this.removedPlugins.push(packageName);
-    await fs.writeFile(path.join(datadir(), 'removed-plugins'), this.removedPlugins.join('\n'));
+    // this.removedPlugins.push(packageName);
+    await this.saveRemovePlugins();
+  },
+
+  upgrade_meta: 'post',
+  async upgrade({ packageName }) {
+    if (!hasPermission(`plugins/install`)) return;
+    const dir = path.join(pluginsdir(), packageName);
+    // @ts-ignore
+    if (await fs.exists(dir)) {
+      await fs.rmdir(dir, { recursive: true });
+      await downloadPackage(packageName, dir);
+    }
+
+    socket.emitChanged(`installed-plugins-changed`);
   },
 
   command_meta: 'post',
@@ -131,24 +149,55 @@ module.exports = {
     return content.commands[command](args);
   },
 
-  async _init() {
-    const installed = await this.installed();
-    try {
-      this.removedPlugins = (await fs.readFile(path.join(datadir(), 'removed-plugins'), { encoding: 'utf-8' })).split(
-        '\n'
-      );
-    } catch (err) {
-      this.removedPlugins = [];
-    }
-    for (const packageName of preinstallPlugins) {
-      if (this.removedPlugins.includes(packageName)) continue;
-      if (installed.find((x) => x.name == packageName)) continue;
-      try {
-        console.log('Preinstalling plugin', packageName);
-        await this.install({ packageName });
-      } catch (err) {
-        console.error('Error preinstalling plugin', packageName, err);
-      }
-    }
+  authTypes_meta: 'get',
+  async authTypes({ engine }) {
+    const packageName = extractPackageName(engine);
+    const content = requirePlugin(packageName);
+    const driver = content.drivers.find(x => x.engine == engine);
+    if (!driver || !driver.getAuthTypes) return null;
+    return driver.getAuthTypes() || null;
   },
+
+  // async _init() {
+  //   const installed = await this.installed();
+  //   try {
+  //     this.removedPlugins = (await fs.readFile(path.join(datadir(), 'removed-plugins'), { encoding: 'utf-8' })).split(
+  //       '\n'
+  //     );
+  //   } catch (err) {
+  //     this.removedPlugins = [];
+  //   }
+
+  //   for (const packageName of Object.keys(preinstallPluginMinimalVersions)) {
+  //     const installedVersion = installed.find(x => x.name == packageName);
+  //     if (installedVersion) {
+  //       // plugin installed, test, whether upgrade
+  //       const requiredVersion = preinstallPluginMinimalVersions[packageName];
+  //       if (compareVersions(installedVersion.version, requiredVersion) < 0) {
+  //         console.log(
+  //           `Upgrading preinstalled plugin ${packageName}, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //         await this.upgrade({ packageName });
+  //       } else {
+  //         console.log(
+  //           `Plugin ${packageName} not upgraded, found ${installedVersion.version}, required version ${requiredVersion}`
+  //         );
+  //       }
+
+  //       continue;
+  //     }
+
+  //     if (this.removedPlugins.includes(packageName)) {
+  //       // plugin was remvoed, don't install again
+  //       continue;
+  //     }
+
+  //     try {
+  //       console.log('Preinstalling plugin', packageName);
+  //       await this.install({ packageName });
+  //     } catch (err) {
+  //       console.error('Error preinstalling plugin', packageName, err);
+  //     }
+  //   }
+  // },
 };

packages/api/src/controllers/queryHistory.js

@@ -0,0 +1,54 @@
+const fsReverse = require('fs-reverse');
+const fs = require('fs-extra');
+const path = require('path');
+const { datadir } = require('../utility/directories');
+const _ = require('lodash');
+const { filterName } = require('dbgate-tools');
+const socket = require('../utility/socket');
+
+function readCore(reader, skip, limit, filter) {
+  return new Promise((resolve, reject) => {
+    const res = [];
+    let readed = 0;
+    reader.on('data', line => {
+      if (!line && !line.trim()) return;
+      try {
+        const json = JSON.parse(line);
+        if (filterName(filter, json.sql, json.database)) {
+          if (!skip || readed >= skip) {
+            res.push(json);
+          }
+          readed++;
+          if (limit && readed > (skip || 0) + limit) {
+            reader.destroy();
+            resolve(res);
+          }
+        }
+      } catch (err) {
+        reader.destroy();
+        reject(err);
+      }
+    });
+    reader.on('end', () => resolve(res));
+  });
+}
+
+module.exports = {
+  read_meta: 'get',
+  async read({ skip, limit, filter }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    // @ts-ignore
+    if (!(await fs.exists(fileName))) return [];
+    const reader = fsReverse(fileName);
+    const res = await readCore(reader, skip, limit, filter);
+    return res;
+  },
+
+  write_meta: 'post',
+  async write({ data }) {
+    const fileName = path.join(datadir(), 'query-history.jsonl');
+    await fs.appendFile(fileName, JSON.stringify(data) + '\n');
+    socket.emitChanged('query-history-changed');
+    return 'OK';
+  },
+};

packages/api/src/controllers/runners.js

@@ -5,24 +5,26 @@ const uuidv1 = require('uuid/v1');
 const byline = require('byline');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
-const { rundir, uploadsdir, pluginsdir } = require('../utility/directories');
+const { rundir, uploadsdir, pluginsdir, getPluginBackendPath, packagedPluginList } = require('../utility/directories');
 const { extractShellApiPlugins, extractShellApiFunctionName } = require('dbgate-tools');
+const { handleProcessCommunication } = require('../utility/processComm');
 
 function extractPlugins(script) {
   const requireRegex = /\s*\/\/\s*@require\s+([^\s]+)\s*\n/g;
   const matches = [...script.matchAll(requireRegex)];
-  return matches.map((x) => x[1]);
+  return matches.map(x => x[1]);
 }
 
-const requirePluginsTemplate = (plugins) =>
+const requirePluginsTemplate = plugins =>
   plugins
     .map(
-      (packageName) => `const ${_.camelCase(packageName)} = require(process.env.PLUGIN_${_.camelCase(packageName)});\n`
+      packageName => `const ${_.camelCase(packageName)} = require(process.env.PLUGIN_${_.camelCase(packageName)});\n`
     )
-    .join('') + `dbgateApi.registerPlugins(${plugins.map((x) => _.camelCase(x)).join(',')});\n`;
+    .join('') + `dbgateApi.registerPlugins(${plugins.map(x => _.camelCase(x)).join(',')});\n`;
 
-const scriptTemplate = (script) => `
+const scriptTemplate = script => `
 const dbgateApi = require(process.env.DBGATE_API);
+dbgateApi.initializeApiEnvironment();
 ${requirePluginsTemplate(extractPlugins(script))}
 require=null;
 async function run() {
@@ -35,6 +37,7 @@ dbgateApi.runScript(run);
 
 const loaderScriptTemplate = (functionName, props, runid) => `
 const dbgateApi = require(process.env.DBGATE_API);
+dbgateApi.initializeApiEnvironment();
 ${requirePluginsTemplate(extractShellApiPlugins(functionName, props))}
 require=null;
 async function run() {
@@ -89,28 +92,29 @@ module.exports = {
     const scriptFile = path.join(uploadsdir(), runid + '.js');
     fs.writeFileSync(`${scriptFile}`, scriptText);
     fs.mkdirSync(directory);
-    const pluginNames = fs.readdirSync(pluginsdir());
+    const pluginNames = _.union(fs.readdirSync(pluginsdir()), packagedPluginList);
     console.log(`RUNNING SCRIPT ${scriptFile}`);
     // const subprocess = fork(scriptFile, ['--checkParent', '--max-old-space-size=8192'], {
-    const subprocess = fork(scriptFile, ['--checkParent'], {
+    const subprocess = fork(scriptFile, ['--checkParent', ...process.argv.slice(3)], {
       cwd: directory,
       stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
       env: {
-        DBGATE_API: process.argv[1],
-        ..._.fromPairs(pluginNames.map((name) => [`PLUGIN_${_.camelCase(name)}`, path.join(pluginsdir(), name)])),
+        ...process.env,
+        DBGATE_API: global['dbgateApiModulePath'] || process.argv[1],
+        ..._.fromPairs(pluginNames.map(name => [`PLUGIN_${_.camelCase(name)}`, getPluginBackendPath(name)])),
       },
     });
-    const pipeDispatcher = (severity) => (data) =>
+    const pipeDispatcher = severity => data =>
       this.dispatchMessage(runid, { severity, message: data.toString().trim() });
 
     byline(subprocess.stdout).on('data', pipeDispatcher('info'));
     byline(subprocess.stderr).on('data', pipeDispatcher('error'));
-    subprocess.on('exit', (code) => {
+    subprocess.on('exit', code => {
       this.rejectRequest(runid, { message: 'No data retured, maybe input data source is too big' });
       console.log('... EXIT process', code);
       socket.emit(`runner-done-${runid}`, code);
     });
-    subprocess.on('error', (error) => {
+    subprocess.on('error', error => {
       this.rejectRequest(runid, { message: error && (error.message || error.toString()) });
       console.error('... ERROR subprocess', error);
       this.dispatchMessage({
@@ -123,8 +127,10 @@ module.exports = {
       subprocess,
     };
     this.opened.push(newOpened);
-    // @ts-ignore
-    subprocess.on('message', ({ msgtype, ...message }) => {
+    subprocess.on('message', message => {
+      // @ts-ignore
+      const { msgtype } = message;
+      if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](runid, message);
     });
     return newOpened;
@@ -138,7 +144,7 @@ module.exports = {
 
   cancel_meta: 'post',
   async cancel({ runid }) {
-    const runner = this.opened.find((x) => x.runid == runid);
+    const runner = this.opened.find(x => x.runid == runid);
     if (!runner) {
       throw new Error('Invalid runner');
     }

packages/api/src/controllers/scheduler.js

@@ -11,7 +11,7 @@ module.exports = {
   tasks: [],
 
   async unload() {
-    this.tasks.forEach((x) => x.destroy());
+    this.tasks.forEach(x => x.destroy());
     this.tasks = [];
   },
 

packages/api/src/controllers/serverConnections.js

@@ -2,19 +2,30 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const _ = require('lodash');
+const AsyncLock = require('async-lock');
+const { handleProcessCommunication } = require('../utility/processComm');
+const lock = new AsyncLock();
+const config = require('./config');
 
 module.exports = {
   opened: [],
   closed: {},
+  lastPinged: {},
 
   handle_databases(conid, { databases }) {
-    const existing = this.opened.find((x) => x.conid == conid);
+    const existing = this.opened.find(x => x.conid == conid);
     if (!existing) return;
     existing.databases = databases;
     socket.emitChanged(`database-list-changed-${conid}`);
   },
+  handle_version(conid, { version }) {
+    const existing = this.opened.find(x => x.conid == conid);
+    if (!existing) return;
+    existing.version = version;
+    socket.emitChanged(`server-version-changed-${conid}`);
+  },
   handle_status(conid, { status }) {
-    const existing = this.opened.find((x) => x.conid == conid);
+    const existing = this.opened.find(x => x.conid == conid);
     if (!existing) return;
     existing.status = status;
     socket.emitChanged(`server-status-changed`);
@@ -22,42 +33,51 @@ module.exports = {
   handle_ping() {},
 
   async ensureOpened(conid) {
-    const existing = this.opened.find((x) => x.conid == conid);
-    if (existing) return existing;
-    const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['serverConnectionProcess']);
-    const newOpened = {
-      conid,
-      subprocess,
-      databases: [],
-      connection,
-      status: {
-        name: 'pending',
-      },
-      disconnected: false,
-    };
-    this.opened.push(newOpened);
-    delete this.closed[conid];
-    socket.emitChanged(`server-status-changed`);
-    // @ts-ignore
-    subprocess.on('message', ({ msgtype, ...message }) => {
-      if (newOpened.disconnected) return;
-      this[`handle_${msgtype}`](conid, message);
+    const res = await lock.acquire(conid, async () => {
+      const existing = this.opened.find(x => x.conid == conid);
+      if (existing) return existing;
+      const connection = await connections.get({ conid });
+      const subprocess = fork(process.argv[1], [
+        '--start-process',
+        'serverConnectionProcess',
+        ...process.argv.slice(3),
+      ]);
+      const newOpened = {
+        conid,
+        subprocess,
+        databases: [],
+        connection,
+        status: {
+          name: 'pending',
+        },
+        disconnected: false,
+      };
+      this.opened.push(newOpened);
+      delete this.closed[conid];
+      socket.emitChanged(`server-status-changed`);
+      subprocess.on('message', message => {
+        // @ts-ignore
+        const { msgtype } = message;
+        if (handleProcessCommunication(message, subprocess)) return;
+        if (newOpened.disconnected) return;
+        this[`handle_${msgtype}`](conid, message);
+      });
+      subprocess.on('exit', () => {
+        if (newOpened.disconnected) return;
+        this.close(conid, false);
+      });
+      subprocess.send({ msgtype: 'connect', ...connection, globalSettings: config.settingsValue });
+      return newOpened;
     });
-    subprocess.on('exit', () => {
-      if (newOpened.disconnected) return;
-      this.close(conid, false);
-    });
-    subprocess.send({ msgtype: 'connect', ...connection });
-    return newOpened;
+    return res;
   },
 
   close(conid, kill = true) {
-    const existing = this.opened.find((x) => x.conid == conid);
+    const existing = this.opened.find(x => x.conid == conid);
     if (existing) {
       existing.disconnected = true;
       if (kill) existing.subprocess.kill();
-      this.opened = this.opened.filter((x) => x.conid != conid);
+      this.opened = this.opened.filter(x => x.conid != conid);
       this.closed[conid] = {
         ...existing.status,
         name: 'error',
@@ -66,12 +86,24 @@ module.exports = {
     }
   },
 
+  disconnect_meta: 'post',
+  async disconnect({ conid }) {
+    await this.close(conid, true);
+    return { status: 'ok' };
+  },
+
   listDatabases_meta: 'get',
   async listDatabases({ conid }) {
     const opened = await this.ensureOpened(conid);
     return opened.databases;
   },
 
+  version_meta: 'get',
+  async version({ conid }) {
+    const opened = await this.ensureOpened(conid);
+    return opened.version;
+  },
+
   serverStatus_meta: 'get',
   async serverStatus() {
     return {
@@ -82,16 +114,23 @@ module.exports = {
 
   ping_meta: 'post',
   async ping({ connections }) {
-    for (const conid of connections) {
-      const opened = await this.ensureOpened(conid);
-      opened.subprocess.send({ msgtype: 'ping' });
-    }
+    await Promise.all(
+      _.uniq(connections).map(async conid => {
+        const last = this.lastPinged[conid];
+        if (last && new Date().getTime() - last < 30 * 1000) {
+          return Promise.resolve();
+        }
+        this.lastPinged[conid] = new Date().getTime();
+        const opened = await this.ensureOpened(conid);
+        opened.subprocess.send({ msgtype: 'ping' });
+      })
+    );
     return { status: 'ok' };
   },
 
   refresh_meta: 'post',
-  async refresh({ conid }) {
-    this.close(conid);
+  async refresh({ conid, keepOpen }) {
+    if (!keepOpen) this.close(conid);
 
     await this.ensureOpened(conid);
     return { status: 'ok' };

packages/api/src/controllers/sessions.js

@@ -4,6 +4,7 @@ const connections = require('./connections');
 const socket = require('../utility/socket');
 const { fork } = require('child_process');
 const jsldata = require('./jsldata');
+const { handleProcessCommunication } = require('../utility/processComm');
 
 module.exports = {
   /** @type {import('dbgate-types').OpenedSession[]} */
@@ -64,7 +65,7 @@ module.exports = {
   async create({ conid, database }) {
     const sesid = uuidv1();
     const connection = await connections.get({ conid });
-    const subprocess = fork(process.argv[1], ['sessionProcess']);
+    const subprocess = fork(process.argv[1], ['--start-process', 'sessionProcess', ...process.argv.slice(3)]);
     const newOpened = {
       conid,
       database,
@@ -73,8 +74,10 @@ module.exports = {
       sesid,
     };
     this.opened.push(newOpened);
-    // @ts-ignore
-    subprocess.on('message', ({ msgtype, ...message }) => {
+    subprocess.on('message', message => {
+      // @ts-ignore
+      const { msgtype } = message;
+      if (handleProcessCommunication(message, subprocess)) return;
       this[`handle_${msgtype}`](sesid, message);
     });
     subprocess.send({ msgtype: 'connect', ...connection, database });
@@ -83,7 +86,7 @@ module.exports = {
 
   executeQuery_meta: 'post',
   async executeQuery({ sesid, sql }) {
-    const session = this.opened.find((x) => x.sesid == sesid);
+    const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
       throw new Error('Invalid session');
     }
@@ -107,7 +110,7 @@ module.exports = {
 
   kill_meta: 'post',
   async kill({ sesid }) {
-    const session = this.opened.find((x) => x.sesid == sesid);
+    const session = this.opened.find(x => x.sesid == sesid);
     if (!session) {
       throw new Error('Invalid session');
     }

packages/api/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '3.8.6',
-  buildTime: '2020-12-10T11:14:01.053Z'
+  version: '4.1.1',
+  buildTime: '2021-04-17T07:22:49.702Z'
 };

packages/api/src/index.js

@@ -1,21 +1,20 @@
 const shell = require('./shell');
-// require('socket.io-client');
+const processArgs = require('./utility/processArgs');
+const dbgateTools = require('dbgate-tools');
 
-// "socket.io-client": "^2.3.0",
-// "utf-8-validate": "^5.0.2",
-// "uuid": "^3.4.0",
-// "uws": "10.148.1"
+global['DBGATE_TOOLS'] = dbgateTools;
 
-const argument = process.argv[2];
-if (argument && argument.endsWith('Process')) {
+if (processArgs.startProcess) {
   const proc = require('./proc');
-
-  const module = proc[argument];
+  const module = proc[processArgs.startProcess];
   module.start();
-} else if (!module['parent'] && !process.argv.includes('--checkParent')) {
+} else if (!module['parent'] && !processArgs.checkParent) {
   const main = require('./main');
 
-  main.start(argument);
+  main.start();
 }
 
-module.exports = shell;
+module.exports = {
+  ...shell,
+  getMainModule: () => require('./main'),
+};

packages/api/src/main.js

@@ -6,8 +6,10 @@ const http = require('http');
 const cors = require('cors');
 const io = require('socket.io');
 const fs = require('fs');
-const findFreePort = require('find-free-port');
+const getPort = require('get-port');
 const childProcessChecker = require('./utility/childProcessChecker');
+const path = require('path');
+const crypto = require('crypto');
 
 const useController = require('./utility/useController');
 const socket = require('./utility/socket');
@@ -25,10 +27,17 @@ const uploads = require('./controllers/uploads');
 const plugins = require('./controllers/plugins');
 const files = require('./controllers/files');
 const scheduler = require('./controllers/scheduler');
+const queryHistory = require('./controllers/queryHistory');
 
 const { rundir } = require('./utility/directories');
+const platformInfo = require('./utility/platformInfo');
+const processArgs = require('./utility/processArgs');
+const timingSafeCheckToken = require('./utility/timingSafeCheckToken');
 
-function start(argument = null) {
+let authorization = null;
+let checkLocalhostOrigin = null;
+
+function start() {
   // console.log('process.argv', process.argv);
 
   const app = express();
@@ -48,6 +57,29 @@ function start(argument = null) {
     );
   }
 
+  app.use(function (req, res, next) {
+    if (authorization && !timingSafeCheckToken(req.headers.authorization, authorization)) {
+      return res.status(403).json({ error: 'Not authorized!' });
+    }
+    if (checkLocalhostOrigin) {
+      if (
+        req.headers.origin &&
+        req.headers.origin != checkLocalhostOrigin &&
+        req.headers.origin != `http://${checkLocalhostOrigin}`
+      ) {
+        console.log('API origin check FAILED');
+        console.log('HEADERS', { ...req.headers, authorization: '***' });
+        return res.status(403).json({ error: 'Not authorized!' });
+      }
+      if (!req.headers.origin && req.headers.host != checkLocalhostOrigin) {
+        console.log('API host check FAILED');
+        console.log('HEADERS', { ...req.headers, authorization: '***' });
+        return res.status(403).json({ error: 'Not authorized!' });
+      }
+    }
+    next();
+  });
+
   app.use(cors());
   app.use(bodyParser.json({ limit: '50mb' }));
 
@@ -71,6 +103,7 @@ function start(argument = null) {
   useController(app, '/plugins', plugins);
   useController(app, '/files', files);
   useController(app, '/scheduler', scheduler);
+  useController(app, '/query-history', queryHistory);
 
   // if (process.env.PAGES_DIRECTORY) {
   //   app.use('/pages', express.static(process.env.PAGES_DIRECTORY));
@@ -78,22 +111,34 @@ function start(argument = null) {
 
   app.use('/runners/data', express.static(rundir()));
 
-  if (fs.existsSync('/home/dbgate-docker/build')) {
+  if (platformInfo.isDocker) {
     // server static files inside docker container
-    app.use(express.static('/home/dbgate-docker/build'));
+    app.use(express.static('/home/dbgate-docker/public'));
   } else {
-    app.get('/', (req, res) => {
-      res.send('DbGate API');
-    });
+    if (!platformInfo.isNpmDist) {
+      app.get('/', (req, res) => {
+        res.send('DbGate API');
+      });
+    }
   }
 
-  if (argument == '--dynport') {
+  if (processArgs.dynport) {
     childProcessChecker();
 
-    findFreePort(53911, function (err, port) {
+    authorization = crypto.randomBytes(32).toString('hex');
+
+    getPort().then(port => {
+      checkLocalhostOrigin = `localhost:${port}`;
+      server.listen(port, () => {
+        console.log(`DbGate API listening on port ${port}`);
+        process.send({ msgtype: 'listening', port, authorization });
+      });
+    });
+  } else if (platformInfo.isNpmDist) {
+    app.use(express.static(path.join(__dirname, '../../dbgate-web/public')));
+    getPort({ port: 5000 }).then(port => {
       server.listen(port, () => {
         console.log(`DbGate API listening on port ${port}`);
-        process.send({ msgtype: 'listening', port });
       });
     });
   } else {

packages/api/src/nativeModules.js

@@ -0,0 +1,5 @@
+const argIndex = process.argv.indexOf('--native-modules');
+const redirectFile = argIndex > 0 ? process.argv[argIndex + 1] : null;
+
+// @ts-ignore
+module.exports = redirectFile ? __non_webpack_require__(redirectFile) : require('./nativeModulesContent');

packages/api/src/proc/connectProcess.js

@@ -1,17 +1,43 @@
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { handleProcessCommunication } = require('../utility/processComm');
+const _ = require('lodash');
+
+function pickSafeConnectionInfo(connection) {
+  return _.mapValues(connection, (v, k) => {
+    if (k == 'engine' || k == 'port' || k == 'authType' || k == 'sshMode' || k == 'passwordMode') return v;
+    if (v === null || v === true || v === false) return v;
+    if (v) return '***';
+    return undefined;
+  });
+}
+
+const formatErrorDetail = (e, connection) => `${e.stack}
+
+Error JSON: ${JSON.stringify(e, undefined, 2)}
+
+Connection: ${JSON.stringify(pickSafeConnectionInfo(connection), undefined, 2)}
+
+Platform: ${process.platform}
+`;
 
 function start() {
   childProcessChecker();
-  process.on('message', async (connection) => {
+  process.on('message', async connection => {
+    if (handleProcessCommunication(connection)) return;
     try {
       const driver = requireEngineDriver(connection);
-      const conn = await driver.connect(connection);
+      const conn = await connectUtility(driver, connection);
       const res = await driver.getVersion(conn);
       process.send({ msgtype: 'connected', ...res });
     } catch (e) {
       console.error(e);
-      process.send({ msgtype: 'error', error: e.message });
+      process.send({
+        msgtype: 'error',
+        error: e.message,
+        detail: formatErrorDetail(e, connection),
+      });
     }
   });
 }

packages/api/src/proc/databaseConnectionProcess.js

@@ -1,6 +1,11 @@
 const stableStringify = require('json-stable-stringify');
+const { splitQuery } = require('dbgate-query-splitter');
 const childProcessChecker = require('../utility/childProcessChecker');
+const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
+const { handleProcessCommunication } = require('../utility/processComm');
+const { SqlGenerator } = require('dbgate-tools');
 
 let systemConnection;
 let storedConnection;
@@ -8,6 +13,8 @@ let afterConnectCallbacks = [];
 let analysedStructure = null;
 let lastPing = null;
 let lastStatus = null;
+let analysedTime = 0;
+let serverVersion;
 
 async function checkedAsyncCall(promise) {
   try {
@@ -24,21 +31,42 @@ async function checkedAsyncCall(promise) {
   }
 }
 
+let loadingModel = false;
+
 async function handleFullRefresh() {
+  loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
-  analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection));
+  setStatusName('loadStructure');
+  analysedStructure = await checkedAsyncCall(driver.analyseFull(systemConnection, serverVersion));
+  analysedTime = new Date().getTime();
   process.send({ msgtype: 'structure', structure: analysedStructure });
+  process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
+  loadingModel = false;
 }
 
-async function handleIncrementalRefresh() {
+async function handleIncrementalRefresh(forceSend) {
+  loadingModel = true;
   const driver = requireEngineDriver(storedConnection);
-  const newStructure = await checkedAsyncCall(driver.analyseIncremental(systemConnection, analysedStructure));
+  setStatusName('checkStructure');
+  const newStructure = await checkedAsyncCall(driver.analyseIncremental(systemConnection, analysedStructure, serverVersion));
+  analysedTime = new Date().getTime();
   if (newStructure != null) {
     analysedStructure = newStructure;
+  }
+
+  if (forceSend || newStructure != null) {
     process.send({ msgtype: 'structure', structure: analysedStructure });
   }
+
+  process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
+  loadingModel = false;
+}
+
+function handleSyncModel() {
+  if (loadingModel) return;
+  handleIncrementalRefresh();
 }
 
 function setStatus(status) {
@@ -53,20 +81,35 @@ function setStatusName(name) {
   setStatus({ name });
 }
 
-async function handleConnect({ connection, structure }) {
+async function readVersion() {
+  const driver = requireEngineDriver(storedConnection);
+  const version = await driver.getVersion(systemConnection);
+  process.send({ msgtype: 'version', version });
+  serverVersion = version;
+}
+
+async function handleConnect({ connection, structure, globalSettings }) {
   storedConnection = connection;
   lastPing = new Date().getTime();
 
   if (!structure) setStatusName('pending');
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await checkedAsyncCall(driver.connect(storedConnection));
+  systemConnection = await checkedAsyncCall(connectUtility(driver, storedConnection));
+  await checkedAsyncCall(readVersion());
   if (structure) {
     analysedStructure = structure;
-    handleIncrementalRefresh();
+    handleIncrementalRefresh(true);
   } else {
     handleFullRefresh();
   }
-  setInterval(handleIncrementalRefresh, 30 * 1000);
+
+  if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
+    setInterval(
+      handleIncrementalRefresh,
+      extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 3, 3600) * 1000
+    );
+  }
+
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
   }
@@ -80,6 +123,17 @@ function waitConnected() {
   });
 }
 
+async function handleRunScript({ msgid, sql }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    await driver.script(systemConnection, sql);
+    process.send({ msgtype: 'response', msgid });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
 async function handleQueryData({ msgid, sql }) {
   await waitConnected();
   const driver = requireEngineDriver(storedConnection);
@@ -91,6 +145,49 @@ async function handleQueryData({ msgid, sql }) {
   }
 }
 
+async function handleCollectionData({ msgid, options }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    const result = await driver.readCollection(systemConnection, options);
+    process.send({ msgtype: 'response', msgid, result });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
+async function handleUpdateCollection({ msgid, changeSet }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+  try {
+    const result = await driver.updateCollection(systemConnection, changeSet);
+    process.send({ msgtype: 'response', msgid, result });
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, errorMessage: err.message });
+  }
+}
+
+async function handleSqlPreview({ msgid, objects, options }) {
+  await waitConnected();
+  const driver = requireEngineDriver(storedConnection);
+
+  try {
+    const dmp = driver.createDumper();
+    const generator = new SqlGenerator(analysedStructure, options, objects, dmp, driver, systemConnection);
+
+    await generator.dump();
+    process.send({ msgtype: 'response', msgid, sql: dmp.s, isTruncated: generator.isTruncated });
+    if (generator.isUnhandledException) {
+      setTimeout(() => {
+        console.log('Exiting because of unhandled exception');
+        process.exit(0);
+      }, 500);
+    }
+  } catch (err) {
+    process.send({ msgtype: 'response', msgid, isError: true, errorMessage: err.message });
+  }
+}
+
 // async function handleRunCommand({ msgid, sql }) {
 //   await waitConnected();
 //   const driver = engines(storedConnection);
@@ -105,7 +202,12 @@ function handlePing() {
 const messageHandlers = {
   connect: handleConnect,
   queryData: handleQueryData,
+  runScript: handleRunScript,
+  updateCollection: handleUpdateCollection,
+  collectionData: handleCollectionData,
+  sqlPreview: handleSqlPreview,
   ping: handlePing,
+  syncModel: handleSyncModel,
   // runCommand: handleRunCommand,
 };
 
@@ -119,12 +221,14 @@ function start() {
 
   setInterval(() => {
     const time = new Date().getTime();
-    if (time - lastPing > 60 * 1000) {
+    if (time - lastPing > 120 * 1000) {
+      console.log('Database connection not alive, exiting');
       process.exit(0);
     }
   }, 60 * 1000);
 
-  process.on('message', async (message) => {
+  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
     try {
       await handleMessage(message);
     } catch (e) {

packages/api/src/proc/jslDatastoreProcess.js

@@ -1,5 +1,6 @@
 const childProcessChecker = require('../utility/childProcessChecker');
 const JsonLinesDatastore = require('../utility/JsonLinesDatastore');
+const { handleProcessCommunication } = require('../utility/processComm');
 
 let lastPing = null;
 let datastore = new JsonLinesDatastore();
@@ -46,7 +47,8 @@ function start() {
     }
   }, 60 * 1000);
 
-  process.on('message', async (message) => {
+  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
     try {
       await handleMessage(message);
     } catch (e) {

packages/api/src/proc/serverConnectionProcess.js

@@ -1,6 +1,10 @@
 const stableStringify = require('json-stable-stringify');
+const { extractBoolSettingsValue, extractIntSettingsValue } = require('dbgate-tools');
 const childProcessChecker = require('../utility/childProcessChecker');
 const requireEngineDriver = require('../utility/requireEngineDriver');
+const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');
+const { handleProcessCommunication } = require('../utility/processComm');
 
 let systemConnection;
 let storedConnection;
@@ -28,6 +32,12 @@ async function handleRefresh() {
   }
 }
 
+async function readVersion() {
+  const driver = requireEngineDriver(storedConnection);
+  const version = await driver.getVersion(systemConnection);
+  process.send({ msgtype: 'version', version });
+}
+
 function setStatus(status) {
   const statusString = stableStringify(status);
   if (lastStatus != statusString) {
@@ -42,14 +52,18 @@ function setStatusName(name) {
 
 async function handleConnect(connection) {
   storedConnection = connection;
+  const { globalSettings } = storedConnection;
   setStatusName('pending');
   lastPing = new Date().getTime();
 
   const driver = requireEngineDriver(storedConnection);
   try {
-    systemConnection = await driver.connect(storedConnection);
+    systemConnection = await connectUtility(driver, storedConnection);
+    readVersion();
     handleRefresh();
-    setInterval(handleRefresh, 30 * 1000);
+    if (extractBoolSettingsValue(globalSettings, 'connection.autoRefresh', false)) {
+      setInterval(handleRefresh, extractIntSettingsValue(globalSettings, 'connection.autoRefreshInterval', 30, 5, 3600) * 1000);
+    }
   } catch (err) {
     setStatus({
       name: 'error',
@@ -66,9 +80,13 @@ function handlePing() {
 
 async function handleCreateDatabase({ name }) {
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await driver.connect(storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection);
   console.log(`RUNNING SCRIPT: CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
-  await driver.query(systemConnection, `CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
+  if (driver.createDatabase) {
+    await driver.createDatabase(systemConnection, name);
+  } else {
+    await driver.query(systemConnection, `CREATE DATABASE ${driver.dialect.quoteIdentifier(name)}`);
+  }
   await handleRefresh();
 }
 
@@ -88,12 +106,14 @@ function start() {
 
   setInterval(() => {
     const time = new Date().getTime();
-    if (time - lastPing > 60 * 1000) {
+    if (time - lastPing > 120 * 1000) {
+      console.log('Server connection not alive, exiting');
       process.exit(0);
     }
   }, 60 * 1000);
 
-  process.on('message', async (message) => {
+  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
     try {
       await handleMessage(message);
     } catch (err) {

packages/api/src/proc/sessionProcess.js

@@ -3,10 +3,13 @@ const path = require('path');
 const fs = require('fs');
 const _ = require('lodash');
 const childProcessChecker = require('../utility/childProcessChecker');
-const goSplit = require('../utility/goSplit');
+const { splitQuery } = require('dbgate-query-splitter');
 
 const { jsldir } = require('../utility/directories');
 const requireEngineDriver = require('../utility/requireEngineDriver');
+const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');
+const { handleProcessCommunication } = require('../utility/processComm');
 
 let systemConnection;
 let storedConnection;
@@ -14,12 +17,18 @@ let afterConnectCallbacks = [];
 // let currentHandlers = [];
 
 class TableWriter {
-  constructor(columns, resultIndex) {
+  constructor(structure, resultIndex) {
     this.jslid = uuidv1();
     this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     this.currentRowCount = 0;
     this.currentChangeIndex = 1;
-    fs.writeFileSync(this.currentFile, JSON.stringify({ columns }) + '\n');
+    fs.writeFileSync(
+      this.currentFile,
+      JSON.stringify({
+        ...structure,
+        __isStreamHeader: true,
+      }) + '\n'
+    );
     this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
     this.writeCurrentStats(false, false);
     this.resultIndex = resultIndex;
@@ -73,7 +82,7 @@ class StreamHandler {
 
     // use this for cancelling - not implemented
     // this.stream = null;
-    
+
     this.plannedStats = false;
     this.resultIndexHolder = resultIndexHolder;
     this.resolve = resolve;
@@ -89,7 +98,7 @@ class StreamHandler {
 
   recordset(columns) {
     this.closeCurrentWriter();
-    this.currentWriter = new TableWriter(columns, this.resultIndexHolder.value);
+    this.currentWriter = new TableWriter(Array.isArray(columns) ? { columns } : columns, this.resultIndexHolder.value);
     this.resultIndexHolder.value += 1;
 
     // this.writeCurrentStats();
@@ -130,7 +139,7 @@ async function handleConnect(connection) {
   storedConnection = connection;
 
   const driver = requireEngineDriver(storedConnection);
-  systemConnection = await driver.connect(storedConnection);
+  systemConnection = await connectUtility(driver, storedConnection);
   for (const [resolve] of afterConnectCallbacks) {
     resolve();
   }
@@ -157,7 +166,7 @@ async function handleExecuteQuery({ sql }) {
   const resultIndexHolder = {
     value: 0,
   };
-  for (const sqlItem of goSplit(sql)) {
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('stream'))) {
     await handleStream(driver, resultIndexHolder, sqlItem);
     // const handler = new StreamHandler(resultIndex);
     // const stream = await driver.stream(systemConnection, sqlItem, handler);
@@ -180,7 +189,8 @@ async function handleMessage({ msgtype, ...other }) {
 
 function start() {
   childProcessChecker();
-  process.on('message', async (message) => {
+  process.on('message', async message => {
+    if (handleProcessCommunication(message)) return;
     try {
       await handleMessage(message);
     } catch (e) {

packages/api/src/shell/copyStream.js

@@ -1,9 +1,13 @@
+const EnsureStreamHeaderStream = require('../utility/EnsureStreamHeaderStream');
+
 function copyStream(input, output) {
   return new Promise((resolve, reject) => {
+    const ensureHeader = new EnsureStreamHeaderStream();
     const finisher = output['finisher'] || output;
     finisher.on('finish', resolve);
     finisher.on('error', reject);
-    input.pipe(output);
+    input.pipe(ensureHeader);
+    ensureHeader.pipe(output);
   });
 }
 

packages/api/src/shell/executeQuery.js

@@ -1,16 +1,17 @@
-const goSplit = require('../utility/goSplit');
+const { splitQuery } = require('dbgate-query-splitter');
 const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
 
 async function executeQuery({ connection, sql }) {
   console.log(`Execute query ${sql}`);
 
   const driver = requireEngineDriver(connection);
-  const pool = await driver.connect(connection);
+  const pool = await connectUtility(driver, connection);
   console.log(`Connected.`);
 
-  for (const sqlItem of goSplit(sql)) {
+  for (const sqlItem of splitQuery(sql, driver.getQuerySplitterOptions('script'))) {
     console.log('Executing query', sqlItem);
-    await driver.query(pool, sqlItem);
+    await driver.query(pool, sqlItem, { discardResult: true });
   }
 
   console.log(`Query finished`);

packages/api/src/shell/index.js

@@ -6,7 +6,9 @@ const copyStream = require('./copyStream');
 const fakeObjectReader = require('./fakeObjectReader');
 const consoleObjectWriter = require('./consoleObjectWriter');
 const jsonLinesWriter = require('./jsonLinesWriter');
+const jsonArrayWriter = require('./jsonArrayWriter');
 const jsonLinesReader = require('./jsonLinesReader');
+const sqlDataWriter = require('./sqlDataWriter');
 const jslDataReader = require('./jslDataReader');
 const archiveWriter = require('./archiveWriter');
 const archiveReader = require('./archiveReader');
@@ -17,6 +19,7 @@ const requirePlugin = require('./requirePlugin');
 const download = require('./download');
 const executeQuery = require('./executeQuery');
 const loadFile = require('./loadFile');
+const initializeApiEnvironment = require('./initializeApiEnvironment');
 
 const dbgateApi = {
   queryReader,
@@ -25,7 +28,9 @@ const dbgateApi = {
   tableReader,
   copyStream,
   jsonLinesWriter,
+  jsonArrayWriter,
   jsonLinesReader,
+  sqlDataWriter,
   fakeObjectReader,
   consoleObjectWriter,
   jslDataReader,
@@ -37,8 +42,9 @@ const dbgateApi = {
   registerPlugins,
   executeQuery,
   loadFile,
+  initializeApiEnvironment,
 };
 
-requirePlugin.initialize(dbgateApi);
+requirePlugin.initializeDbgateApi(dbgateApi);
 
 module.exports = dbgateApi;

packages/api/src/shell/initializeApiEnvironment.js

@@ -0,0 +1,9 @@
+const { handleProcessCommunication } = require('../utility/processComm');
+
+async function initializeApiEnvironment() {
+  process.on('message', async message => {
+    handleProcessCommunication(message);
+  });
+}
+
+module.exports = initializeApiEnvironment;

packages/api/src/shell/jsonArrayWriter.js

@@ -0,0 +1,52 @@
+const fs = require('fs');
+const stream = require('stream');
+
+class StringifyStream extends stream.Transform {
+  constructor() {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.wasRecord = false;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+
+    if (!this.wasHeader) {
+      skip =
+        chunk.__isStreamHeader ||
+        // TODO remove isArray test
+        Array.isArray(chunk.columns);
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      if (!this.wasRecord) {
+        this.push('[\n');
+      } else {
+        this.push(',\n');
+      }
+      this.wasRecord = true;
+
+      this.push(JSON.stringify(chunk));
+    }
+    done();
+  }
+
+  _flush(done) {
+    if (!this.wasRecord) {
+      this.push('[]\n');
+    } else {
+      this.push('\n]\n');
+    }
+    done();
+  }
+}
+
+async function jsonArrayWriter({ fileName, encoding = 'utf-8' }) {
+  console.log(`Writing file ${fileName}`);
+  const stringify = new StringifyStream();
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = jsonArrayWriter;

packages/api/src/shell/jsonLinesReader.js

@@ -3,9 +3,8 @@ const stream = require('stream');
 const byline = require('byline');
 
 class ParseStream extends stream.Transform {
-  constructor({ header, limitRows }) {
+  constructor({ limitRows }) {
     super({ objectMode: true });
-    this.header = header;
     this.wasHeader = false;
     this.limitRows = limitRows;
     this.rowsWritten = 0;
@@ -13,7 +12,14 @@ class ParseStream extends stream.Transform {
   _transform(chunk, encoding, done) {
     const obj = JSON.parse(chunk);
     if (!this.wasHeader) {
-      if (!this.header) this.push({ columns: Object.keys(obj).map((columnName) => ({ columnName })) });
+      if (
+        !obj.__isStreamHeader &&
+        // TODO remove isArray test
+        !Array.isArray(obj.columns)
+      ) {
+        this.push({ columns: Object.keys(obj).map(columnName => ({ columnName })) });
+      }
+      
       this.wasHeader = true;
     }
     if (!this.limitRows || this.rowsWritten < this.limitRows) {
@@ -24,12 +30,12 @@ class ParseStream extends stream.Transform {
   }
 }
 
-async function jsonLinesReader({ fileName, encoding = 'utf-8', header = true, limitRows = undefined }) {
+async function jsonLinesReader({ fileName, encoding = 'utf-8', limitRows = undefined }) {
   console.log(`Reading file ${fileName}`);
 
   const fileStream = fs.createReadStream(fileName, encoding);
   const liner = byline(fileStream);
-  const parser = new ParseStream({ header, limitRows });
+  const parser = new ParseStream({ limitRows });
   liner.pipe(parser);
   return parser;
 }

packages/api/src/shell/jsonLinesWriter.js

@@ -8,10 +8,16 @@ class StringifyStream extends stream.Transform {
     this.wasHeader = false;
   }
   _transform(chunk, encoding, done) {
+    let skip = false;
     if (!this.wasHeader) {
-      if (this.header) this.push(JSON.stringify(chunk) + '\n');
+      skip =
+        (chunk.__isStreamHeader ||
+          // TODO remove isArray test
+          Array.isArray(chunk.columns)) &&
+        !this.header;
       this.wasHeader = true;
-    } else {
+    }
+    if (!skip) {
       this.push(JSON.stringify(chunk) + '\n');
     }
     done();

packages/api/src/shell/queryReader.js

@@ -1,10 +1,12 @@
-const requireEngineDriver = require("../utility/requireEngineDriver");
+const requireEngineDriver = require('../utility/requireEngineDriver');
+const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');
 
 async function queryReader({ connection, sql }) {
   console.log(`Reading query ${sql}`);
 
   const driver = requireEngineDriver(connection);
-  const pool = await driver.connect(connection);
+  const pool = await connectUtility(driver, connection);
   console.log(`Connected.`);
   return await driver.readQuery(pool, sql);
 }

packages/api/src/shell/requirePlugin.js

@@ -1,25 +1,28 @@
 const path = require('path');
-const { pluginsdir } = require('../utility/directories');
+const fs = require('fs');
+const { pluginsdir, packagedPluginsDir, getPluginBackendPath } = require('../utility/directories');
+const nativeModules = require('../nativeModules');
+const platformInfo = require('../utility/platformInfo');
 
 const loadedPlugins = {};
 
 const dbgateEnv = {
   dbgateApi: null,
+  nativeModules,
 };
-
 function requirePlugin(packageName, requiredPlugin = null) {
   if (!packageName) throw new Error('Missing packageName in plugin');
   if (loadedPlugins[packageName]) return loadedPlugins[packageName];
 
   if (requiredPlugin == null) {
     let module;
-    const modulePath = path.join(pluginsdir(), packageName, 'dist', 'backend.js');
+    const modulePath = getPluginBackendPath(packageName);
     console.log(`Loading module ${packageName} from ${modulePath}`);
     try {
       // @ts-ignore
       module = __non_webpack_require__(modulePath);
     } catch (err) {
-      console.error('Failed load webpacked module', err);
+      // console.log('Failed load webpacked module', err.message);
       module = require(modulePath);
     }
     requiredPlugin = module.__esModule ? module.default : module;
@@ -30,7 +33,7 @@ function requirePlugin(packageName, requiredPlugin = null) {
   return requiredPlugin;
 }
 
-requirePlugin.initialize = (value) => {
+requirePlugin.initializeDbgateApi = value => {
   dbgateEnv.dbgateApi = value;
 };
 

packages/api/src/shell/runScript.js

@@ -1,7 +1,8 @@
 const childProcessChecker = require('../utility/childProcessChecker');
+const processArgs = require('../utility/processArgs');
 
 async function runScript(func) {
-  if (process.argv.includes('--checkParent')) {
+  if (processArgs.checkParent) {
     childProcessChecker();
   }
   try {

packages/api/src/shell/sqlDataWriter.js

@@ -0,0 +1,54 @@
+const fs = require('fs');
+const stream = require('stream');
+const path = require('path');
+const { driverBase } = require('dbgate-tools');
+const requireEngineDriver = require('../utility/requireEngineDriver');
+
+class SqlizeStream extends stream.Transform {
+  constructor({ fileName }) {
+    super({ objectMode: true });
+    this.wasHeader = false;
+    this.tableName = path.parse(fileName).name;
+    this.driver = driverBase;
+  }
+  _transform(chunk, encoding, done) {
+    let skip = false;
+    if (!this.wasHeader) {
+      if (
+        chunk.__isStreamHeader ||
+        // TODO remove isArray test
+        Array.isArray(chunk.columns)
+      ) {
+        skip = true;
+        this.tableName = chunk.pureName;
+        if (chunk.engine) {
+          // @ts-ignore
+          this.driver = requireEngineDriver(chunk.engine) || driverBase;
+        }
+      }
+      this.wasHeader = true;
+    }
+    if (!skip) {
+      const dmp = this.driver.createDumper();
+      dmp.put(
+        '^insert ^into %f (%,i) ^values (%,v);\n',
+        { pureName: this.tableName },
+        Object.keys(chunk),
+        Object.values(chunk)
+      );
+      this.push(dmp.s);
+    }
+    done();
+  }
+}
+
+async function sqlDataWriter({ fileName, driver, encoding = 'utf-8' }) {
+  console.log(`Writing file ${fileName}`);
+  const stringify = new SqlizeStream({ fileName });
+  const fileStream = fs.createWriteStream(fileName, encoding);
+  stringify.pipe(fileStream);
+  stringify['finisher'] = fileStream;
+  return stringify;
+}
+
+module.exports = sqlDataWriter;

packages/api/src/shell/tableReader.js

@@ -1,13 +1,21 @@
 const { quoteFullName, fullNameToString } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
+const connectUtility = require('../utility/connectUtility');
 
 async function tableReader({ connection, pureName, schemaName }) {
   const driver = requireEngineDriver(connection);
-  const pool = await driver.connect(connection);
+  const pool = await connectUtility(driver, connection);
   console.log(`Connected.`);
 
   const fullName = { pureName, schemaName };
 
+  if (driver.dialect.nosql) {
+    // @ts-ignore
+    console.log(`Reading collection ${fullNameToString(fullName)}`);
+    // @ts-ignore
+    return await driver.readQuery(pool, JSON.stringify(fullName));
+  }
+
   const table = await driver.analyseSingleObject(pool, fullName, 'tables');
   const query = `select * from ${quoteFullName(driver.dialect, fullName)}`;
   if (table) {

packages/api/src/shell/tableWriter.js

@@ -1,11 +1,13 @@
 const { fullNameToString } = require('dbgate-tools');
 const requireEngineDriver = require('../utility/requireEngineDriver');
+const { decryptConnection } = require('../utility/crypting');
+const connectUtility = require('../utility/connectUtility');
 
 async function tableWriter({ connection, schemaName, pureName, ...options }) {
   console.log(`Writing table ${fullNameToString({ schemaName, pureName })}`);
 
   const driver = requireEngineDriver(connection);
-  const pool = await driver.connect(connection);
+  const pool = await connectUtility(driver, connection);
   console.log(`Connected.`);
   return await driver.writeTable(pool, { schemaName, pureName }, options);
 }

packages/api/src/utility/DatastoreProxy.js

@@ -1,5 +1,6 @@
 const { fork } = require('child_process');
 const uuidv1 = require('uuid/v1');
+const { handleProcessCommunication } = require('./processComm');
 
 class DatastoreProxy {
   constructor(file) {
@@ -28,10 +29,13 @@ class DatastoreProxy {
 
   async ensureSubprocess() {
     if (!this.subprocess) {
-      this.subprocess = fork(process.argv[1], ['jslDatastoreProcess']);
+      this.subprocess = fork(process.argv[1], ['--start-process', 'jslDatastoreProcess', ...process.argv.slice(3)]);
+
+      this.subprocess.on('message', message => {
+        // @ts-ignore
+        const { msgtype } = message;
+        if (handleProcessCommunication(message, this.subprocess)) return;
 
-      // @ts-ignore
-      this.subprocess.on('message', ({ msgtype, ...message }) => {
         // if (this.disconnected) return;
         this[`handle_${msgtype}`](message);
       });

packages/api/src/utility/EnsureStreamHeaderStream.js

@@ -0,0 +1,35 @@
+const stream = require('stream');
+
+class EnsureStreamHeaderStream extends stream.Transform {
+  constructor() {
+    super({ objectMode: true });
+    this.wasHeader = false;
+  }
+  _transform(chunk, encoding, done) {
+    if (!this.wasHeader) {
+      if (chunk.__isDynamicStructure) {
+        // ignore dynamic structure header
+        done();
+        return;
+      }
+
+      if (
+        !chunk.__isStreamHeader &&
+        // TODO remove isArray test
+        !Array.isArray(chunk.columns)
+      ) {
+        this.push({
+          __isStreamHeader: true,
+          __isComputedStructure: true,
+          columns: Object.keys(chunk).map(columnName => ({ columnName })),
+        });
+      }
+
+      this.wasHeader = true;
+    }
+    this.push(chunk);
+    done();
+  }
+}
+
+module.exports = EnsureStreamHeaderStream;

packages/api/src/utility/childProcessChecker.js

@@ -1,5 +1,5 @@
 let counter = 0;
- 
+
 function childProcessChecker() {
   setInterval(() => {
     try {

packages/api/src/utility/connectUtility.js

@@ -0,0 +1,64 @@
+const { SSHConnection } = require('node-ssh-forward');
+const portfinder = require('portfinder');
+const fs = require('fs-extra');
+const { decryptConnection } = require('./crypting');
+const { getSshTunnel } = require('./sshTunnel');
+const { getSshTunnelProxy } = require('./sshTunnelProxy');
+
+async function connectUtility(driver, storedConnection) {
+  const connection = {
+    database: storedConnection.defaultDatabase,
+    ...decryptConnection(storedConnection),
+  };
+
+  if (!connection.port && driver.defaultPort) connection.port = driver.defaultPort.toString();
+
+  if (connection.useSshTunnel) {
+    const tunnel = await getSshTunnelProxy(connection);
+    if (tunnel.state == 'error') {
+      throw new Error(tunnel.message);
+    }
+
+    connection.server = '127.0.0.1';
+    connection.port = tunnel.localPort;
+  }
+
+  // SSL functionality - copied from https://github.com/beekeeper-studio/beekeeper-studio
+  if (connection.useSsl) {
+    connection.ssl = {};
+
+    if (connection.sslCaFile) {
+      connection.ssl.ca = await fs.readFile(connection.sslCaFile);
+    }
+
+    if (connection.sslCertFile) {
+      connection.ssl.cert = await fs.readFile(connection.sslCertFile);
+    }
+
+    if (connection.sslKeyFile) {
+      connection.ssl.key = await fs.readFile(connection.sslKeyFile);
+    }
+
+    if (connection.sslCertFilePassword) {
+      connection.ssl.password = connection.sslCertFilePassword;
+    }
+
+    if (!connection.ssl.key && !connection.ssl.ca && !connection.ssl.cert) {
+      // TODO: provide this as an option in settings
+      // or per-connection as 'reject self-signed certs'
+      // How it works:
+      // if false, cert can be self-signed
+      // if true, has to be from a public CA
+      // Heroku certs are self-signed.
+      // if you provide ca/cert/key files, it overrides this
+      connection.ssl.rejectUnauthorized = false;
+    } else {
+      connection.ssl.rejectUnauthorized = connection.sslRejectUnauthorized;
+    }
+  }
+
+  const conn = await driver.connect(connection);
+  return conn;
+}
+
+module.exports = connectUtility;

packages/api/src/utility/crypting.js

@@ -0,0 +1,88 @@
+const crypto = require('crypto');
+const simpleEncryptor = require('simple-encryptor');
+const fs = require('fs');
+const path = require('path');
+
+const { datadir } = require('./directories');
+
+const defaultEncryptionKey = 'mQAUaXhavRGJDxDTXSCg7Ej0xMmGCrx6OKA07DIMBiDcYYkvkaXjTAzPUEHEHEf9';
+
+let _encryptionKey = null;
+
+function loadEncryptionKey() {
+  if (_encryptionKey) {
+    return _encryptionKey;
+  }
+  const encryptor = simpleEncryptor.createEncryptor(defaultEncryptionKey);
+
+  const keyFile = path.join(datadir(), '.key');
+
+  if (!fs.existsSync(keyFile)) {
+    const generatedKey = crypto.randomBytes(32);
+    const newKey = generatedKey.toString('hex');
+    const result = {
+      encryptionKey: newKey,
+    };
+    fs.writeFileSync(keyFile, encryptor.encrypt(result), 'utf-8');
+  }
+
+  const encryptedData = fs.readFileSync(keyFile, 'utf-8');
+  const data = encryptor.decrypt(encryptedData);
+  _encryptionKey = data['encryptionKey'];
+  return _encryptionKey;
+}
+
+let _encryptor = null;
+
+function getEncryptor() {
+  if (_encryptor) {
+    return _encryptor;
+  }
+  _encryptor = simpleEncryptor.createEncryptor(loadEncryptionKey());
+  return _encryptor;
+}
+
+function encryptPasswordField(connection, field) {
+  if (
+    connection &&
+    connection[field] &&
+    !connection[field].startsWith('crypt:') &&
+    connection.passwordMode != 'saveRaw'
+  ) {
+    return {
+      ...connection,
+      [field]: 'crypt:' + getEncryptor().encrypt(connection[field]),
+    };
+  }
+return connection;
+}
+
+function decryptPasswordField(connection, field) {
+  if (connection && connection[field] && connection[field].startsWith('crypt:')) {
+    return {
+      ...connection,
+      [field]: getEncryptor().decrypt(connection[field].substring('crypt:'.length)),
+    };
+  }
+  return connection;
+}
+
+function encryptConnection(connection) {
+  connection = encryptPasswordField(connection, 'password');
+  connection = encryptPasswordField(connection, 'sshPassword');
+  connection = encryptPasswordField(connection, 'sshKeyfilePassword');
+  return connection;
+}
+
+function decryptConnection(connection) {
+  connection = decryptPasswordField(connection, 'password');
+  connection = decryptPasswordField(connection, 'sshPassword');
+  connection = decryptPasswordField(connection, 'sshKeyfilePassword');
+  return connection;
+}
+
+module.exports = {
+  loadEncryptionKey,
+  encryptConnection,
+  decryptConnection,
+};

packages/api/src/utility/directories.js

@@ -2,6 +2,7 @@ const os = require('os');
 const path = require('path');
 const fs = require('fs');
 const cleanDirectory = require('./cleanDirectory');
+const platformInfo = require('./platformInfo');
 
 const createDirectories = {};
 const ensureDirectory = (dir, clean) => {
@@ -39,6 +40,37 @@ const pluginsdir = dirFunc('plugins');
 const archivedir = dirFunc('archive');
 const filesdir = dirFunc('files');
 
+function packagedPluginsDir() {
+  if (platformInfo.isDevMode) {
+    return path.resolve(__dirname, '../../../../plugins');
+  }
+  if (platformInfo.isDocker) {
+    return '/home/dbgate-docker/plugins';
+  }
+  if (platformInfo.isNpmDist) {
+    // node_modules
+    return global['dbgateApiPackagedPluginsPath'];
+  }
+  if (platformInfo.isElectronBundle) {
+    return path.resolve(__dirname, '../../plugins');
+  }
+  return null;
+}
+
+const packagedPluginList =
+  packagedPluginsDir() != null ? fs.readdirSync(packagedPluginsDir()).filter(x => x.startsWith('dbgate-plugin-')) : [];
+
+function getPluginBackendPath(packageName) {
+  if (packagedPluginList.includes(packageName)) {
+    if (platformInfo.isDevMode) {
+      return path.join(packagedPluginsDir(), packageName, 'src', 'backend', 'index.js');
+    }
+    return path.join(packagedPluginsDir(), packageName, 'dist', 'backend.js');
+  }
+
+  return path.join(pluginsdir(), packageName, 'dist', 'backend.js');
+}
+
 module.exports = {
   datadir,
   jsldir,
@@ -48,4 +80,7 @@ module.exports = {
   ensureDirectory,
   pluginsdir,
   filesdir,
+  packagedPluginsDir,
+  packagedPluginList,
+  getPluginBackendPath,
 };

packages/api/src/utility/downloadPackage.js

@@ -15,15 +15,14 @@ function extractTarball(tmpFile, destination) {
     fs.createReadStream(tmpFile)
       .pipe(zlib.createGunzip())
       .pipe(tar.extract({ cwd: destination }))
-      .on('error', (err) => reject(err))
+      .on('error', err => reject(err))
       .on('end', () => resolve());
   });
 }
 
-
 function copyDirectory(source, target) {
   return new Promise((resolve, reject) => {
-    ncp(source, target, (err) => {
+    ncp(source, target, err => {
       if (err) reject(err);
       resolve();
     });

packages/api/src/utility/goSplit.js

@@ -1,18 +0,0 @@
-function goSplit(sql) {
-  if (!sql) return [];
-  const lines = sql.split('\n');
-  const res = [];
-  let buffer = '';
-  for (const line of lines) {
-    if (/^\s*go\s*$/i.test(line)) {
-      if (buffer.trim()) res.push(buffer);
-      buffer = '';
-    } else {
-      buffer += line + '\n';
-    }
-  }
-  if (buffer.trim()) res.push(buffer);
-  return res;
-}
-
-module.exports = goSplit;

packages/api/src/utility/platformInfo.js

@@ -0,0 +1,43 @@
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const processArgs = require('./processArgs');
+
+const platform = process.env.OS_OVERRIDE ? process.env.OS_OVERRIDE : process.platform;
+const isWindows = platform === 'win32';
+const isMac = platform === 'darwin';
+const isLinux = platform === 'linux';
+const isDocker = fs.existsSync('/home/dbgate-docker/public');
+const isDevMode = process.env.DEVMODE == '1';
+const isNpmDist = !!global['dbgateApiModulePath'];
+
+// function moduleAvailable(name) {
+//   try {
+//     require.resolve(name);
+//     return true;
+//   } catch (e) {
+//     return false;
+//   }
+// }
+
+const isElectronBundle = processArgs.isElectronBundle;
+
+const platformInfo = {
+  isWindows,
+  isMac,
+  isLinux,
+  isDocker,
+  isElectronBundle,
+  isDevMode,
+  isNpmDist,
+  isSnap: process.env.ELECTRON_SNAP == 'true',
+  isPortable: isWindows && process.env.PORTABLE_EXECUTABLE_DIR,
+  isAppImage: process.env.DESKTOPINTEGRATION === 'AppImageLauncher',
+  sshAuthSock: process.env.SSH_AUTH_SOCK,
+  environment: process.env.NODE_ENV,
+  platform,
+  runningInWebpack: !!process.env.WEBPACK_DEV_SERVER_URL,
+  defaultKeyfile: path.join(os.homedir(), '.ssh/id_rsa'),
+};
+
+module.exports = platformInfo;

packages/api/src/utility/processArgs.js

@@ -0,0 +1,21 @@
+function getNamedArg(name) {
+  const argIndex = process.argv.indexOf(name);
+  if (argIndex > 0) {
+    return process.argv[argIndex + 1];
+  }
+  return null;
+}
+
+const checkParent = process.argv.includes('--checkParent');
+const dynport = process.argv.includes('--dynport');
+const nativeModules = getNamedArg('--native-modules');
+const startProcess = getNamedArg('--start-process');
+const isElectronBundle = process.argv.includes('--is-electron-bundle');
+
+module.exports = {
+  checkParent,
+  nativeModules,
+  startProcess,
+  dynport,
+  isElectronBundle,
+};

packages/api/src/utility/processComm.js

@@ -0,0 +1,18 @@
+const { handleGetSshTunnelRequest, handleGetSshTunnelResponse } = require('./sshTunnelProxy');
+
+function handleProcessCommunication(message, subprocess) {
+  const { msgtype } = message;
+  if (msgtype == 'getsshtunnel-request') {
+    handleGetSshTunnelRequest(message, subprocess);
+    return true;
+  }
+  if (msgtype == 'getsshtunnel-response') {
+    handleGetSshTunnelResponse(message, subprocess);
+    return true;
+  }
+  return false;
+}
+
+module.exports = {
+  handleProcessCommunication,
+};

packages/api/src/utility/requireEngineDriver.js

@@ -1,7 +1,6 @@
 const _ = require('lodash');
 const requirePlugin = require('../shell/requirePlugin');
 
-
 /** @returns {import('dbgate-types').EngineDriver} */
 function requireEngineDriver(connection) {
   let engine = null;
@@ -16,7 +15,7 @@ function requireEngineDriver(connection) {
   if (engine.includes('@')) {
     const [shortName, packageName] = engine.split('@');
     const plugin = requirePlugin(packageName);
-    return plugin.driver;
+    return plugin.drivers.find(x => x.engine == engine);
   }
   throw new Error(`Could not found engine driver ${engine}`);
 }

packages/api/src/utility/sshTunnel.js

@@ -0,0 +1,91 @@
+const { SSHConnection } = require('node-ssh-forward');
+const fs = require('fs-extra');
+const portfinder = require('portfinder');
+const stableStringify = require('json-stable-stringify');
+const _ = require('lodash');
+const platformInfo = require('./platformInfo');
+const AsyncLock = require('async-lock');
+const lock = new AsyncLock();
+
+const sshConnectionCache = {};
+const sshTunnelCache = {};
+
+const CONNECTION_FIELDS = [
+  'sshHost',
+  'sshPort',
+  'sshLogin',
+  'sshPassword',
+  'sshMode',
+  'sshKeyfile',
+  'sshBastionHost',
+  'sshKeyfilePassword',
+];
+const TUNNEL_FIELDS = [...CONNECTION_FIELDS, 'server', 'port'];
+
+async function getSshConnection(connection) {
+  const connectionCacheKey = stableStringify(_.pick(connection, CONNECTION_FIELDS));
+  if (sshConnectionCache[connectionCacheKey]) return sshConnectionCache[connectionCacheKey];
+
+  const sshConfig = {
+    endHost: connection.sshHost || '',
+    endPort: connection.sshPort || 22,
+    bastionHost: connection.sshBastionHost || '',
+    agentForward: connection.sshMode == 'agent',
+    passphrase: connection.sshMode == 'keyFile' ? connection.sshKeyfilePassword : undefined,
+    username: connection.sshLogin,
+    password: connection.sshMode == 'userPassword' ? connection.sshPassword : undefined,
+    agentSocket: connection.sshMode == 'agent' ? platformInfo.sshAuthSock : undefined,
+    privateKey:
+      connection.sshMode == 'keyFile' && connection.sshKeyfile ? await fs.readFile(connection.sshKeyfile) : undefined,
+    skipAutoPrivateKey: true,
+    noReadline: true,
+  };
+
+  const sshConn = new SSHConnection(sshConfig);
+  sshConnectionCache[connectionCacheKey] = sshConn;
+  return sshConn;
+}
+
+async function getSshTunnel(connection) {
+  const tunnelCacheKey = stableStringify(_.pick(connection, TUNNEL_FIELDS));
+
+  return await lock.acquire(tunnelCacheKey, async () => {
+    const sshConn = await getSshConnection(connection);
+    if (sshTunnelCache[tunnelCacheKey]) return sshTunnelCache[tunnelCacheKey];
+
+    const localPort = await portfinder.getPortPromise({ port: 10000, stopPort: 60000 });
+    // workaround for `getPortPromise` not releasing the port quickly enough
+    await new Promise(resolve => setTimeout(resolve, 500));
+    const tunnelConfig = {
+      fromPort: localPort,
+      toPort: connection.port,
+      toHost: connection.server,
+    };
+    try {
+      console.log(
+        `Creating SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
+      );
+
+      const tunnel = await sshConn.forward(tunnelConfig);
+      console.log(
+        `Created SSH tunnel to ${connection.sshHost}-${connection.server}:${connection.port}, using local port ${localPort}`
+      );
+
+      sshTunnelCache[tunnelCacheKey] = {
+        state: 'ok',
+        localPort,
+      };
+      return sshTunnelCache[tunnelCacheKey];
+    } catch (err) {
+      // error is not cached
+      return {
+        state: 'error',
+        message: err.message,
+      };
+    }
+  });
+}
+
+module.exports = {
+  getSshTunnel,
+};

packages/api/src/utility/sshTunnelProxy.js

@@ -0,0 +1,30 @@
+const uuidv1 = require('uuid/v1');
+const { getSshTunnel } = require('./sshTunnel');
+
+const dispatchedMessages = {};
+
+async function handleGetSshTunnelRequest({ msgid, connection }, subprocess) {
+  const response = await getSshTunnel(connection);
+  subprocess.send({ msgtype: 'getsshtunnel-response', msgid, response });
+}
+
+function handleGetSshTunnelResponse({ msgid, response }, subprocess) {
+  const { resolve } = dispatchedMessages[msgid];
+  delete dispatchedMessages[msgid];
+  resolve(response);
+}
+
+async function getSshTunnelProxy(connection) {
+  if (!process.send) return getSshTunnel(connection);
+  const msgid = uuidv1();
+  process.send({ msgtype: 'getsshtunnel-request', msgid, connection });
+  return new Promise((resolve, reject) => {
+    dispatchedMessages[msgid] = { resolve, reject };
+  });
+}
+
+module.exports = {
+  handleGetSshTunnelRequest,
+  handleGetSshTunnelResponse,
+  getSshTunnelProxy,
+};